From 7a8e60520948df5f75cf17c69ddc072cb75d80d4 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 13 Jun 2023 10:34:48 -0500 Subject: [PATCH 001/191] Change serving location --- src/rpc_thallium.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc_thallium.cc b/src/rpc_thallium.cc index 45c2d5e9d..32dec5a38 100644 --- a/src/rpc_thallium.cc +++ b/src/rpc_thallium.cc @@ -36,12 +36,12 @@ void ThalliumRpc::InitServer() { HELOG(kFatal, "RPC init failed for host: {}\n{}", addr, e.what()); } std::string rpc_server_name = server_engine_->self(); + DefineRpcs(); HILOG(kInfo, "Serving {} (i.e., {}) with {} RPC threads as node id {}", rpc_server_name, addr, config_->rpc_.num_threads_, node_id_); - DefineRpcs(); } /** initialize RPC clients */ From 521d6af14de7ffc739ded9af5af2cd831706f400 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 13 Jun 2023 10:45:37 -0500 Subject: [PATCH 002/191] Synchronous full flush --- src/buffer_organizer.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/buffer_organizer.cc b/src/buffer_organizer.cc index 2329c9f27..5f9a130b6 100644 --- a/src/buffer_organizer.cc +++ b/src/buffer_organizer.cc @@ -528,7 +528,11 @@ void BufferOrganizer::GlobalWaitForFullFlush() { for (int i = 0; i < (int)rpc_->hosts_.size(); ++i) { int node_id = i + 1; HILOG(kInfo, "Wait for flush on node {}", node_id) - rpc_->Call(node_id, "RpcWaitForFullFlush"); + if (node_id == rpc_->node_id_) { + LocalWaitForFullFlush(); + } else { + rpc_->Call(node_id, "RpcWaitForFullFlush"); + } } } From 721cdd63eef763e125dbb80888f8171b0f44332e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 13 Jun 2023 10:55:11 -0500 Subject: [PATCH 003/191] Try scoped mutex on mpsc queue --- .../include/hermes_shm/data_structures/ipc/mpsc_queue.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h index 7c6026115..ab5e1b34f 100644 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h @@ -44,7 +44,7 @@ class mpsc_queue_templ : public ShmContainer { ShmArchive>> queue_; std::atomic<_qtok_t> tail_; std::atomic<_qtok_t> head_; - RwLock lock_; + Mutex lock_; public: /**==================================== @@ -57,6 +57,7 @@ class mpsc_queue_templ : public ShmContainer { shm_init_container(alloc); HSHM_MAKE_AR(queue_, GetAllocator(), depth); SetNull(); + lock_.Init(); } /**==================================== @@ -156,6 +157,8 @@ class mpsc_queue_templ : public ShmContainer { _qtok_t tail = tail_.fetch_add(1); size_t size = tail - head + 1; + ScopedMutex lock(lock_, 0); + // Check if there's space in the queue. Resize if necessary. if (size > (*queue_).size()) { if constexpr(EXTENSIBLE) { @@ -213,6 +216,8 @@ class mpsc_queue_templ : public ShmContainer { public: /** Consumer pops the head object */ HSHM_ALWAYS_INLINE qtok_t pop(T &val) { + ScopedMutex lock(lock_, 0); + if constexpr(EXTENSIBLE) { ScopedRwReadLock resize_lock(lock_, 0); return _pop(val); From e0e7e32bc3e4608ef2c657e0acbb021b6a9534c3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 13 Jun 2023 10:58:50 -0500 Subject: [PATCH 004/191] Try updating hermes_shm --- ...dHermesShm.cmake => HermesShmConfig.cmake} | 13 +- hermes_shm/CMakeLists.txt | 27 ++- hermes_shm/benchmark/allocator/allocator.cc | 159 ++++++++----- .../benchmark/data_structure/CMakeLists.txt | 3 +- hermes_shm/benchmark/data_structure/lock.cc | 162 +++++++++++++ hermes_shm/benchmark/data_structure/queue.cc | 175 ++++++++------ .../benchmark/data_structure/test_init.cc | 6 +- .../benchmark/data_structure/test_init.h | 2 + .../data_structure_singleton_macros.h | 6 - .../data_structures/containers/charbuf.h | 55 +++-- .../data_structures/data_structure.h | 18 +- .../hermes_shm/data_structures/ipc/_queue.h | 31 ++- .../ipc/internal/shm_archive.h | 5 +- .../ipc/internal/shm_container.h | 2 +- .../data_structures/ipc/mpsc_queue.h | 117 +++------ .../data_structures/ipc/split_ticket_queue.h | 188 +++++++++++++++ .../data_structures/ipc/spsc_queue.h | 62 ++--- .../data_structures/ipc/ticket_queue.h | 164 +++++++++++++ .../data_structures/ipc/ticket_stack.h | 166 +++++++++++++ .../hermes_shm/data_structures/ipc/vector.h | 14 +- .../serialization/shm_serialize.h | 34 ++- .../smart_ptr/smart_ptr_base.h | 3 +- hermes_shm/include/hermes_shm/hermes_shm.h | 38 +++ .../hermes_shm/introspect/system_info.h | 6 +- .../memory/allocator/allocator_factory.h | 1 - .../hermes_shm/memory/allocator/heap.h | 50 ++++ .../memory/allocator/malloc_allocator.h | 1 - .../hermes_shm/memory/allocator/mp_page.h | 23 +- .../memory/allocator/stack_allocator.h | 10 +- .../hermes_shm/memory/backend/posix_mmap.h | 3 +- hermes_shm/include/hermes_shm/memory/memory.h | 141 ++++++----- .../hermes_shm/memory/memory_manager.h | 7 +- .../hermes_shm/memory/memory_registry.h | 41 ++-- .../hermes_shm/thread/ipc_call_manager.h | 133 ----------- .../include/hermes_shm/thread/lock/mutex.h | 67 +++++- .../include/hermes_shm/thread/lock/rwlock.h | 194 +++++++++++---- .../thread_model/thread_model_factory.h | 26 +- .../hermes_shm/thread/thread_model_manager.h | 36 +-- hermes_shm/include/hermes_shm/types/argpack.h | 59 ++--- .../include/hermes_shm/types/bitfield.h | 33 +-- hermes_shm/include/hermes_shm/util/errors.h | 3 + .../include/hermes_shm/util/singleton.h | 138 +---------- .../util/singleton/_easy_global_singleton.h | 40 ++++ .../util/singleton/_easy_singleton.h | 57 +++++ .../util/singleton/_global_singleton.h | 56 +++++ .../hermes_shm/util/singleton/_singleton.h | 69 ++++++ hermes_shm/readme.md | 2 + hermes_shm/scripts/ci/coverage.sh | 22 ++ hermes_shm/scripts/ci/external/CMakeLists.txt | 18 ++ hermes_shm/scripts/ci/external/test.cc | 16 ++ hermes_shm/scripts/ci/install_deps.sh | 1 + hermes_shm/scripts/ci/install_hshm.sh | 31 ++- .../hermes_shm/packages/hermes_shm/package.py | 4 +- hermes_shm/src/CMakeLists.txt | 9 +- hermes_shm/src/memory/malloc_allocator.cc | 5 +- hermes_shm/src/memory/memory_intercept.cc | 2 +- hermes_shm/src/memory/memory_registry.cc | 1 + .../src/memory/scalable_page_allocator.cc | 5 +- hermes_shm/src/memory/stack_allocator.cc | 8 +- hermes_shm/src/thread/mutex.cc | 113 --------- hermes_shm/src/thread/rwlock.cc | 198 ---------------- hermes_shm/src/thread_factory.cc | 44 ++++ hermes_shm/src/thread_model_manager.cc | 49 ++++ hermes_shm/test/unit/CMakeLists.txt | 2 + .../test/unit/allocators/CMakeLists.txt | 7 + hermes_shm/test/unit/allocators/test_init.h | 2 + .../test/unit/allocators_mpi/CMakeLists.txt | 47 ++++ .../test/unit/allocators_mpi/allocator_mpi.cc | 75 ++++++ .../test/unit/allocators_mpi/test_init.cc | 33 +++ .../test/unit/allocators_mpi/test_init.h | 53 +++++ .../backend/CMakeLists.txt | 7 + .../{data_structures => }/backend/backend.cc | 0 .../backend/memory_manager.cc | 2 + .../backend/memory_slots.cc | 0 .../backend/test_init.cc | 0 .../test/unit/data_structures/CMakeLists.txt | 1 - .../data_structures/containers/CMakeLists.txt | 66 ++++-- .../data_structures/containers/charbuf.cc | 222 ++++++++++++++++++ .../data_structures/containers/mpsc_queue.cc | 156 +----------- .../unit/data_structures/containers/queue.h | 149 ++++++++++++ .../unit/data_structures/containers/slist.cc | 8 +- .../data_structures/containers/spsc_queue.cc | 35 +++ .../data_structures/containers/test_init.h | 2 + .../containers/ticket_queue.cc | 68 ++++++ .../containers_mpi/CMakeLists.txt | 7 + .../containers_mpi/test_init.cc | 2 + .../serialize/shm/CMakeLists.txt | 7 + .../serialize/shm/test_init.cc | 2 + .../data_structures/serialize/shm/test_shm.cc | 13 +- .../serialize/thallium/CMakeLists.txt | 8 + .../serialize/thallium/test_init.h | 2 + hermes_shm/test/unit/thread/CMakeLists.txt | 7 + hermes_shm/test/unit/thread/test_lock.cc | 21 +- hermes_shm/test/unit/types/CMakeLists.txt | 25 +- hermes_shm/test/unit/types/test_util.cc | 8 +- 95 files changed, 2824 insertions(+), 1385 deletions(-) rename hermes_shm/CMake/{FindHermesShm.cmake => HermesShmConfig.cmake} (69%) create mode 100644 hermes_shm/benchmark/data_structure/lock.cc create mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h create mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h create mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h create mode 100644 hermes_shm/include/hermes_shm/hermes_shm.h create mode 100644 hermes_shm/include/hermes_shm/memory/allocator/heap.h delete mode 100644 hermes_shm/include/hermes_shm/thread/ipc_call_manager.h create mode 100644 hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h create mode 100644 hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h create mode 100644 hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h create mode 100644 hermes_shm/include/hermes_shm/util/singleton/_singleton.h create mode 100644 hermes_shm/scripts/ci/coverage.sh create mode 100644 hermes_shm/scripts/ci/external/CMakeLists.txt create mode 100644 hermes_shm/scripts/ci/external/test.cc delete mode 100644 hermes_shm/src/thread/mutex.cc delete mode 100644 hermes_shm/src/thread/rwlock.cc create mode 100644 hermes_shm/src/thread_factory.cc create mode 100644 hermes_shm/src/thread_model_manager.cc create mode 100644 hermes_shm/test/unit/allocators_mpi/CMakeLists.txt create mode 100644 hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc create mode 100644 hermes_shm/test/unit/allocators_mpi/test_init.cc create mode 100644 hermes_shm/test/unit/allocators_mpi/test_init.h rename hermes_shm/test/unit/{data_structures => }/backend/CMakeLists.txt (86%) rename hermes_shm/test/unit/{data_structures => }/backend/backend.cc (100%) rename hermes_shm/test/unit/{data_structures => }/backend/memory_manager.cc (97%) rename hermes_shm/test/unit/{data_structures => }/backend/memory_slots.cc (100%) rename hermes_shm/test/unit/{data_structures => }/backend/test_init.cc (100%) create mode 100644 hermes_shm/test/unit/data_structures/containers/charbuf.cc create mode 100644 hermes_shm/test/unit/data_structures/containers/queue.h create mode 100644 hermes_shm/test/unit/data_structures/containers/spsc_queue.cc create mode 100644 hermes_shm/test/unit/data_structures/containers/ticket_queue.cc diff --git a/hermes_shm/CMake/FindHermesShm.cmake b/hermes_shm/CMake/HermesShmConfig.cmake similarity index 69% rename from hermes_shm/CMake/FindHermesShm.cmake rename to hermes_shm/CMake/HermesShmConfig.cmake index 4f2912b53..579bf9c2d 100644 --- a/hermes_shm/CMake/FindHermesShm.cmake +++ b/hermes_shm/CMake/HermesShmConfig.cmake @@ -15,19 +15,30 @@ find_path( if( HermesShm_INCLUDE_DIR ) get_filename_component(HermesShm_DIR ${HermesShm_INCLUDE_DIR} PATH) + #----------------------------------------------------------------------------- + # Find all packages needed by hermes_shm + #----------------------------------------------------------------------------- find_library( HermesShm_LIBRARY NAMES hermes_shm_data_structures ) + find_library(LIBRT rt) + if(NOT LIBRT) + message(FATAL_ERROR "librt is required for POSIX shared memory") + endif() + #----------------------------------------------------------------------------- + # Mark hermes as found and set all needed packages + #----------------------------------------------------------------------------- if( HermesShm_LIBRARY ) set(HermesShm_LIBRARY_DIR "") get_filename_component(HermesShm_LIBRARY_DIRS ${HermesShm_LIBRARY} PATH) # Set uncached variables as per standard. set(HermesShm_FOUND ON) set(HermesShm_INCLUDE_DIRS ${HermesShm_INCLUDE_DIR}) - set(HermesShm_LIBRARIES ${HermesShm_LIBRARY}) + set(HermesShm_LIBRARIES -lrt -ldl ${HermesShm_LIBRARY}) endif(HermesShm_LIBRARY) + else(HermesShm_INCLUDE_DIR) message(STATUS "FindHermesShm: Could not find hermes_shm.h") endif(HermesShm_INCLUDE_DIR) diff --git a/hermes_shm/CMakeLists.txt b/hermes_shm/CMakeLists.txt index 74fcd0efd..17b3c29b4 100644 --- a/hermes_shm/CMakeLists.txt +++ b/hermes_shm/CMakeLists.txt @@ -88,21 +88,16 @@ if(HERMES_CXX_PROFILE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") message("CXX PROFILER IS ON") endif() -function(make_gprof exec_name exec_dir) - message("gprof -b -A -p -q ${exec_dir}/${exec_name} gmon.out > gprof_out.txt") - add_custom_target( - ${exec_name}_gprof - COMMAND gprof -b -A -p -q ${exec_dir}/${exec_name} gmon.out) -endfunction() +#function(make_gprof exec_name exec_dir) +# add_custom_target( +# ${exec_name}_gprof +# COMMAND gprof -b -A -p -q ${exec_dir}/${exec_name} gmon.out) +#endfunction() #------------------------------------------------------------------------------ # External libraries #------------------------------------------------------------------------------ -# YAML-CPP -find_package(yaml-cpp REQUIRED) -message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") - # Catch2 find_package(Catch2 3.0.1 REQUIRED) message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") @@ -200,3 +195,15 @@ endif() # Install hshm #------------------------------------------------------------------------------ install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) + +configure_file( + ${CMAKE_SOURCE_DIR}/CMake/HermesShmConfig.cmake + ${PROJECT_BINARY_DIR}/CMakeFiles/HermesShmConfig.cmake @ONLY +) + +install( + FILES + ${PROJECT_BINARY_DIR}/CMakeFiles/HermesShmConfig.cmake + DESTINATION + ${CMAKE_INSTALL_PREFIX}/cmake +) diff --git a/hermes_shm/benchmark/allocator/allocator.cc b/hermes_shm/benchmark/allocator/allocator.cc index f2ee35ade..99acbce9c 100644 --- a/hermes_shm/benchmark/allocator/allocator.cc +++ b/hermes_shm/benchmark/allocator/allocator.cc @@ -22,8 +22,7 @@ class AllocatorTestSuite { public: std::string alloc_type_; Allocator *alloc_; - static std::stringstream ss_; - static int test_count_; + Timer timer_; /**==================================== * Test Runner @@ -62,77 +61,113 @@ class AllocatorTestSuite { /** Allocate and Free a single size in a single loop */ void AllocateAndFreeFixedSize(size_t count, size_t size) { - Timer t; - t.Resume(); + StartTimer(); for (size_t i = 0; i < count; ++i) { Pointer p = alloc_->Allocate(size); alloc_->Free(p); } -#pragma omp barrier - t.Pause(); + StopTimer(); - TestOutput("AllocateAndFreeFixedSize", size, t); + TestOutput("AllocateAndFreeFixedSize", size, count, timer_); } /** Allocate a fixed size in a loop, and then free in another loop */ void AllocateThenFreeFixedSize(size_t count, size_t size) { - Timer t; + StartTimer(); std::vector cache(count); - t.Resume(); for (size_t i = 0; i < count; ++i) { cache[i] = alloc_->Allocate(size); } for (size_t i = 0; i < count; ++i) { alloc_->Free(cache[i]); } -#pragma omp barrier - t.Pause(); + StopTimer(); - TestOutput("AllocateThenFreeFixedSize", size, t); + TestOutput("AllocateThenFreeFixedSize", count, size, timer_); + } + + void seq(std::vector &vec, size_t rep, size_t count) { + for (size_t i = 0; i < count; ++i) { + vec.emplace_back(rep); + } } - /** Allocate, Free, Reallocate, Free in a loop */ + /** Allocate a window of pages, free the window. Random page sizes. */ + void AllocateAndFreeRandomWindow(size_t count) { + size_t min_page = 64; + size_t max_page = MEGABYTES(1); + std::mt19937 rng(23522523); + std::vector sizes_; + + seq(sizes_, 64, MEGABYTES(1) / 64); + seq(sizes_, 190, MEGABYTES(1) / 190); + seq(sizes_, KILOBYTES(1), MEGABYTES(1) / KILOBYTES(1)); + seq(sizes_, KILOBYTES(4), MEGABYTES(8) / KILOBYTES(4)); + seq(sizes_, KILOBYTES(32), MEGABYTES(4) / KILOBYTES(4)); + seq(sizes_, MEGABYTES(1), MEGABYTES(64) / MEGABYTES(1)); + std::shuffle(std::begin(sizes_), std::end(sizes_), rng); + std::vector window(sizes_.size()); + size_t num_windows = 500; + + StartTimer(); + for (size_t w = 0; w < num_windows; ++w) { + for (size_t i = 0; i < sizes_.size(); ++i) { + auto &size = sizes_[i]; + window[i] = alloc_->Allocate(size); + } + for (size_t i = 0; i < sizes_.size(); ++i) { + alloc_->Free(window[i]); + } + } + StopTimer(); + + TestOutput("AllocateAndFreeRandomWindow", 0, count, timer_); + } /**==================================== - * Test Output + * Test Helpers * ===================================*/ - /** The CSV header */ - void TestOutputHeader() { - ss_ << "test_name" << "," - << "alloc_type" << "," - << "alloc_size" << "," - << "nthreads" << "," - << "time" << std::endl; + void StartTimer() { + int rank = omp_get_thread_num(); + if (rank == 0) { + timer_.Reset(); + timer_.Resume(); + } +#pragma omp barrier + } + + void StopTimer() { +#pragma omp barrier + int rank = omp_get_thread_num(); + if (rank == 0) { + timer_.Pause(); + } } /** The CSV test case */ - void TestOutput(const std::string &test_name, size_t obj_size, Timer &t) { + void TestOutput(const std::string &test_name, size_t obj_size, + size_t count_per_rank, Timer &t) { int rank = omp_get_thread_num(); if (rank != 0) { return; } - if (test_count_ == 0) { - TestOutputHeader(); - } - ss_ << test_name << "," - << alloc_type_ << "," - << obj_size << "," - << omp_get_num_threads() << "," - << t.GetMsec() << std::endl; - ++test_count_; + int nthreads = omp_get_num_threads(); + double count = (double) count_per_rank * nthreads; + HILOG(kInfo, "{},{},{},{},{},{},{}", + test_name, + alloc_type_, + obj_size, + t.GetMsec(), + nthreads, + count, + count / t.GetMsec()); } /** Print the CSV output */ - static void PrintTestOutput() { - std::cout << ss_.str() << std::endl; + static void PrintTestHeader() { + HILOG(kInfo, "test_name,alloc_type,obj_size,msec,nthreads,count,KOps"); } }; -/** The output text */ -std::stringstream AllocatorTestSuite::ss_; - -/** Number of tests currently conducted */ -int AllocatorTestSuite::test_count_ = 0; - /** The minor number to use for allocators */ static int minor = 1; const std::string shm_url = "test_allocators"; @@ -148,17 +183,19 @@ Allocator* Pretest(MemoryBackendType backend_type, if (rank == 0) { // Create the allocator + backend + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); - alloc = mem_mngr->CreateAllocator( + mem_mngr->CreateAllocator( shm_url, alloc_id, 0, std::forward(args)...); } #pragma omp barrier - if (rank != 0){ - // Retrieve the allocator + backend - alloc = mem_mngr->GetAllocator(alloc_id); - } + alloc = mem_mngr->GetAllocator(alloc_id); + if (alloc == nullptr) { + HELOG(kFatal, "Failed to find the memory allocator?") + } return alloc; } @@ -171,11 +208,9 @@ void Posttest() { HERMES_MEMORY_MANAGER->UnregisterAllocator( alloc_id); HERMES_MEMORY_MANAGER->DestroyBackend(shm_url); - } -# pragma omp barrier - if (rank == 0) { minor += 1; } +# pragma omp barrier } /** A series of allocator benchmarks for a particular thread */ @@ -185,18 +220,27 @@ void AllocatorTest(AllocatorType alloc_type, Args&& ...args) { Allocator *alloc = Pretest( backend_type, std::forward(args)...); - size_t count = 100000; + size_t count = (1 << 20); // Allocate many and then free many - AllocatorTestSuite(alloc_type, alloc).AllocateThenFreeFixedSize( - count, KILOBYTES(1)); + /*AllocatorTestSuite(alloc_type, alloc).AllocateThenFreeFixedSize( + count, KILOBYTES(1));*/ // Allocate and free immediately - AllocatorTestSuite(alloc_type, alloc).AllocateAndFreeFixedSize( - count, KILOBYTES(1)); + /*AllocatorTestSuite(alloc_type, alloc).AllocateAndFreeFixedSize( + count, KILOBYTES(1));*/ + if (alloc_type != AllocatorType::kStackAllocator) { + // Allocate and free randomly + AllocatorTestSuite(alloc_type, alloc).AllocateAndFreeRandomWindow( + count); + } Posttest(); } /** Test different allocators on a particular thread */ void FullAllocatorTestPerThread() { + // Scalable page allocator + AllocatorTest( + AllocatorType::kScalablePageAllocator, + MemoryBackendType::kPosixShmMmap); // Malloc allocator AllocatorTest( AllocatorType::kMallocAllocator, @@ -205,10 +249,6 @@ void FullAllocatorTestPerThread() { AllocatorTest( AllocatorType::kStackAllocator, MemoryBackendType::kPosixShmMmap); - // Scalable page allocator - AllocatorTest( - AllocatorType::kScalablePageAllocator, - MemoryBackendType::kPosixShmMmap); } /** Spawn multiple threads and run allocator tests */ @@ -223,9 +263,10 @@ void FullAllocatorTestThreaded(int nthreads) { } TEST_CASE("AllocatorBenchmark") { - FullAllocatorTestThreaded(1); + AllocatorTestSuite::PrintTestHeader(); + FullAllocatorTestThreaded(8); /*FullAllocatorTestThreaded(2); FullAllocatorTestThreaded(4); - FullAllocatorTestThreaded(8);*/ - AllocatorTestSuite::PrintTestOutput(); + FullAllocatorTestThreaded(8); + FullAllocatorTestThreaded(16);*/ } diff --git a/hermes_shm/benchmark/data_structure/CMakeLists.txt b/hermes_shm/benchmark/data_structure/CMakeLists.txt index 143e958f6..078326c0c 100644 --- a/hermes_shm/benchmark/data_structure/CMakeLists.txt +++ b/hermes_shm/benchmark/data_structure/CMakeLists.txt @@ -15,6 +15,7 @@ add_executable(benchmark_data_structures vector.cc unordered_map.cc queue.cc + lock.cc ) add_dependencies(benchmark_data_structures hermes_shm_data_structures) target_link_libraries(benchmark_data_structures @@ -24,4 +25,4 @@ target_link_libraries(benchmark_data_structures OpenMP::OpenMP_CXX ${Boost_LIBRARIES}) -make_gprof(benchmark_data_structures ${CMAKE_CURRENT_BINARY_DIR}) \ No newline at end of file +# make_gprof(benchmark_data_structures ${CMAKE_CURRENT_BINARY_DIR}) \ No newline at end of file diff --git a/hermes_shm/benchmark/data_structure/lock.cc b/hermes_shm/benchmark/data_structure/lock.cc new file mode 100644 index 000000000..b6646003c --- /dev/null +++ b/hermes_shm/benchmark/data_structure/lock.cc @@ -0,0 +1,162 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "basic_test.h" +#include "test_init.h" + +// Std +#include +#include + +// hermes +#include "hermes_shm/thread/lock.h" + +/** + * A series of performance tests for vectors + * OUTPUT: + * [test_name] [vec_type] [internal_type] [time_ms] + * */ +template +class LockTest { + public: + std::string lock_type_; + std::queue queue_; + LockT lock_; + + /**==================================== + * Test Runner + * ===================================*/ + + /** Test case constructor */ + LockTest() { + if constexpr(std::is_same_v) { + lock_type_ = "std::mutex"; + } else if constexpr(std::is_same_v) { + lock_type_ = "hshm::RwLock"; + } else if constexpr(std::is_same_v) { + lock_type_ = "hshm::Mutex"; + } else { + HELOG(kFatal, "none of the queue tests matched") + } + } + + /** Run the tests */ + void Test(size_t count_per_rank = 100000, int nthreads = 1) { + // AllocateTest(count); + EmplaceTest(count_per_rank, nthreads); + GatherTest(count_per_rank, nthreads); + } + + /**==================================== + * Tests + * ===================================*/ + + /** Emplace after reserving enough space */ + void EmplaceTest(size_t count_per_rank, int nthreads) { + Timer t; + + size_t count = count_per_rank * nthreads; + t.Resume(); + Emplace(count_per_rank, nthreads); + t.Pause(); + + TestOutput("Enqueue", t, count, nthreads); + } + + /** Reader lock scalability */ + void GatherTest(size_t count_per_rank, int nthreads) { + Timer t; + + size_t count = count_per_rank * nthreads; + t.Resume(); + Gather(count_per_rank, nthreads); + t.Pause(); + + TestOutput("Gather", t, count, nthreads); + } + + private: + /**==================================== + * Helpers + * ===================================*/ + + /** Output as CSV */ + void TestOutput(const std::string &test_name, Timer &t, + size_t count, int nthreads) { + HIPRINT("{},{},{},{},{}\n", + test_name, lock_type_, nthreads, t.GetMsec(), + (float)count / t.GetUsec()) + } + + /** Emplace elements into the queue */ + void Emplace(size_t count_per_rank, int nthreads) { + omp_set_dynamic(0); +#pragma omp parallel num_threads(nthreads) + { + for (size_t i = 0; i < count_per_rank; ++i) { + if constexpr(std::is_same_v) { + lock_.lock(); + queue_.emplace(1); + lock_.unlock(); + } else if constexpr(std::is_same_v) { + lock_.Lock(0); + queue_.emplace(1); + lock_.Unlock(); + } else if constexpr(std::is_same_v) { + lock_.WriteLock(0); + queue_.emplace(1); + lock_.WriteUnlock(); + } + } + } + } + + /** Emplace elements into the queue */ + void Gather(size_t count_per_rank, int nthreads) { + std::vector data(count_per_rank * nthreads); + omp_set_dynamic(0); +#pragma omp parallel shared(data) num_threads(nthreads) + { + size_t sum = 0; + for (size_t i = 0; i < count_per_rank; ++i) { + if constexpr(std::is_same_v) { + lock_.lock(); + sum += data[i]; + lock_.unlock(); + } else if constexpr(std::is_same_v) { + lock_.Lock(0); + sum += data[i]; + lock_.Unlock(); + } else if constexpr(std::is_same_v) { + lock_.ReadLock(0); + sum += data[i]; + lock_.ReadUnlock(); + } + } + } + } +}; + +TEST_CASE("LockBenchmark") { + size_t count_per_rank = 1000000; + LockTest().Test(count_per_rank, 1); + LockTest().Test(count_per_rank, 8); + LockTest().Test(count_per_rank, 16); + + LockTest().Test(count_per_rank, 1); + LockTest().Test(count_per_rank, 8); + LockTest().Test(count_per_rank, 16); + + LockTest().Test(count_per_rank, 1); + LockTest().Test(count_per_rank, 8); + LockTest().Test(count_per_rank, 16); +} \ No newline at end of file diff --git a/hermes_shm/benchmark/data_structure/queue.cc b/hermes_shm/benchmark/data_structure/queue.cc index 44f2d70c3..0d69ceb61 100644 --- a/hermes_shm/benchmark/data_structure/queue.cc +++ b/hermes_shm/benchmark/data_structure/queue.cc @@ -19,8 +19,10 @@ // hermes #include "hermes_shm/data_structures/ipc/string.h" +#include "hermes_shm/data_structures/ipc/split_ticket_queue.h" #include #include +#include /** * A series of performance tests for vectors @@ -37,6 +39,8 @@ class QueueTest { ListTPtr queue_ptr_; void *ptr_; hipc::uptr x_; + std::mutex lock_; + int cpu_; /**==================================== * Test Runner @@ -48,10 +52,12 @@ class QueueTest { queue_type_ = "std::queue"; } else if constexpr(std::is_same_v, QueueT>) { queue_type_ = "hipc::mpsc_queue"; - } else if constexpr(std::is_same_v, QueueT>) { - queue_type_ = "hipc::mpsc_queue_ext"; } else if constexpr(std::is_same_v, QueueT>) { queue_type_ = "hipc::spsc_queue"; + } else if constexpr(std::is_same_v, QueueT>) { + queue_type_ = "hipc::ticket_queue"; + } else if constexpr(std::is_same_v, QueueT>) { + queue_type_ = "hipc::split_ticket_queue"; } else { HELOG(kFatal, "none of the queue tests matched") } @@ -60,11 +66,10 @@ class QueueTest { } /** Run the tests */ - void Test() { - size_t count = 10000; + void Test(size_t count_per_rank=100000, int nthreads = 1) { // AllocateTest(count); - EmplaceTest(count); - DequeueTest(count); + EmplaceTest(count_per_rank, nthreads); + DequeueTest(count_per_rank, nthreads); } /**==================================== @@ -76,7 +81,7 @@ class QueueTest { Timer t; t.Resume(); for (size_t i = 0; i < count; ++i) { - Allocate(count); + Allocate(count, count, 1); Destroy(); } t.Pause(); @@ -85,32 +90,33 @@ class QueueTest { } /** Emplace after reserving enough space */ - void EmplaceTest(size_t count) { + void EmplaceTest(size_t count_per_rank, int nthreads) { Timer t; - StringOrInt var(124); - Allocate(count); + size_t count = count_per_rank * nthreads; + Allocate(count, count_per_rank, nthreads); t.Resume(); - Emplace(count); + Emplace(count_per_rank, nthreads); t.Pause(); - TestOutput("Enqueue", t); + TestOutput("Enqueue", t, count, nthreads); Destroy(); } /** Emplace after reserving enough space */ - void DequeueTest(size_t count) { + void DequeueTest(size_t count_per_rank, int nthreads) { Timer t; StringOrInt var(124); - Allocate(count); - Emplace(count); + size_t count = count_per_rank * nthreads; + Allocate(count, count_per_rank, nthreads); + Emplace(count, 1); t.Resume(); - Dequeue(count); + Dequeue(count_per_rank, nthreads); t.Pause(); - TestOutput("Dequeue", t); + TestOutput("Dequeue", t, count, nthreads); Destroy(); } @@ -120,61 +126,84 @@ class QueueTest { * ===================================*/ /** Output as CSV */ - void TestOutput(const std::string &test_name, Timer &t) { - HIPRINT("{},{},{},{}\n", - test_name, queue_type_, internal_type_, t.GetMsec()) + void TestOutput(const std::string &test_name, Timer &t, + size_t count, int nthreads) { + HIPRINT("{},{},{},{},{},{}\n", + test_name, queue_type_, internal_type_, nthreads, t.GetMsec(), + (float)count / t.GetUsec()) } - /** Get element at position i */ - void Dequeue(size_t count) { - for (size_t i = 0; i < count; ++i) { - if constexpr(std::is_same_v>) { - T &x = queue_->front(); - USE(x); - queue_->pop(); - } else if constexpr(std::is_same_v>) { - queue_->pop(*x_); - USE(*x_); - } else if constexpr(std::is_same_v>) { - queue_->pop(*x_); - USE(*x_); - } else if constexpr(std::is_same_v>) { - queue_->pop(*x_); - USE(*x_); + /** Emplace elements into the queue */ + void Emplace(size_t count_per_rank, int nthreads) { + omp_set_dynamic(0); +#pragma omp parallel num_threads(nthreads) + { + StringOrInt var(124); + for (size_t i = 0; i < count_per_rank; ++i) { + if constexpr(std::is_same_v>) { + lock_.lock(); + queue_->emplace(var.Get()); + lock_.unlock(); + } else if constexpr(std::is_same_v>) { + queue_->emplace(var.Get()); + } else if constexpr(std::is_same_v>) { + queue_->emplace(var.Get()); + } else if constexpr(std::is_same_v>) { + queue_->emplace(var.Get()); + } else if constexpr( + std::is_same_v>) { + queue_->emplace(var.Get()); + } } } } - /** Emplace elements into the queue */ - void Emplace(size_t count) { - StringOrInt var(124); - for (size_t i = 0; i < count; ++i) { - if constexpr(std::is_same_v>) { - queue_->emplace(var.Get()); - } else if constexpr(std::is_same_v>) { - queue_->emplace(var.Get()); - } else if constexpr(std::is_same_v>) { - queue_->emplace(var.Get()); - } else if constexpr(std::is_same_v>) { - queue_->emplace(var.Get()); + /** Get element at position i */ + void Dequeue(size_t count_per_rank, int nthreads) { + omp_set_dynamic(0); +#pragma omp parallel num_threads(nthreads) + { + for (size_t i = 0; i < count_per_rank; ++i) { + if constexpr(std::is_same_v>) { + lock_.lock(); + T &x = queue_->front(); + USE(x); + queue_->pop(); + lock_.unlock(); + } else if constexpr(std::is_same_v>) { + queue_->pop(*x_); + USE(*x_); + } else if constexpr(std::is_same_v>) { + queue_->pop(*x_); + USE(*x_); + } else if constexpr(std::is_same_v>) { + while (queue_->pop(*x_).IsNull()); + } else if constexpr( + std::is_same_v>) { + while (queue_->pop(*x_).IsNull()); + } } } } /** Allocate an arbitrary queue for the test cases */ - void Allocate(size_t count) { + void Allocate(size_t count, size_t count_per_rank, int nthreads) { + (void) count_per_rank; (void) nthreads; if constexpr(std::is_same_v>) { queue_ptr_ = new std::queue(); queue_ = queue_ptr_; } else if constexpr(std::is_same_v>) { queue_ptr_ = hipc::make_mptr(count); queue_ = queue_ptr_.get(); - } else if constexpr(std::is_same_v>) { + } else if constexpr(std::is_same_v>) { queue_ptr_ = hipc::make_mptr(count); queue_ = queue_ptr_.get(); - } else if constexpr(std::is_same_v>) { + } else if constexpr(std::is_same_v>) { queue_ptr_ = hipc::make_mptr(count); queue_ = queue_ptr_.get(); + } else if constexpr(std::is_same_v>) { + queue_ptr_ = hipc::make_mptr(count_per_rank, nthreads); + queue_ = queue_ptr_.get(); } } @@ -184,33 +213,47 @@ class QueueTest { delete queue_ptr_; } else if constexpr(std::is_same_v>) { queue_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v>) { - queue_ptr_.shm_destroy(); } else if constexpr(std::is_same_v>) { queue_ptr_.shm_destroy(); + } else if constexpr(std::is_same_v>) { + queue_ptr_.shm_destroy(); + } else if constexpr(std::is_same_v>) { + queue_ptr_.shm_destroy(); } } }; void FullQueueTest() { + const size_t count_per_rank = 1000000; // std::queue tests - QueueTest>().Test(); - // QueueTest>().Test(); + QueueTest>().Test(count_per_rank, 1); + QueueTest>().Test(count_per_rank, 4); + QueueTest>().Test(count_per_rank, 8); + QueueTest>().Test(count_per_rank, 16); + QueueTest>().Test(); - // hipc::mpsc_queue tests - QueueTest>().Test(); - // QueueTest>().Test(); - // QueueTest>().Test(); + // hipc::ticket_queue tests + QueueTest>().Test(count_per_rank, 1); + QueueTest>().Test(count_per_rank, 4); + QueueTest>().Test(count_per_rank, 8); + QueueTest>().Test(count_per_rank, 16); + + // hipc::ticket_queue tests + QueueTest>().Test(count_per_rank, 1); + QueueTest>().Test(count_per_rank, 4); + QueueTest>().Test(count_per_rank, 8); + QueueTest>().Test(count_per_rank, 16); - // hipc::mpsc_ext_queue tests - QueueTest>().Test(); - // QueueTest>().Test(); - // QueueTest>().Test(); + // hipc::mpsc_queue tests + QueueTest>().Test(count_per_rank, 1); + QueueTest>().Test(); + QueueTest>().Test(); // hipc::spsc_queue tests - QueueTest>().Test(); - // QueueTest>().Test(); - // QueueTest>().Test(); + QueueTest>().Test(count_per_rank, 1); + QueueTest>().Test(); + QueueTest>().Test(); + } TEST_CASE("QueueBenchmark") { diff --git a/hermes_shm/benchmark/data_structure/test_init.cc b/hermes_shm/benchmark/data_structure/test_init.cc index 756bc5a38..3582fbec7 100644 --- a/hermes_shm/benchmark/data_structure/test_init.cc +++ b/hermes_shm/benchmark/data_structure/test_init.cc @@ -19,12 +19,16 @@ void MainPretest() { std::string shm_url = "HermesBench"; allocator_id_t alloc_id(0, 1); auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->CreateBackend( + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); + auto backend = mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); + memset(backend->data_, 0, MEGABYTES(16)); // TODO(llogan): back to good allocator mem_mngr->CreateAllocator( shm_url, alloc_id, 0); + // Boost shared memory BOOST_SEGMENT; BOOST_ALLOCATOR(char); diff --git a/hermes_shm/benchmark/data_structure/test_init.h b/hermes_shm/benchmark/data_structure/test_init.h index 61c93054f..dd3cd082e 100644 --- a/hermes_shm/benchmark/data_structure/test_init.h +++ b/hermes_shm/benchmark/data_structure/test_init.h @@ -25,6 +25,8 @@ #include #include +#include + using hshm::ipc::MemoryBackendType; using hshm::ipc::MemoryBackend; using hshm::ipc::allocator_id_t; diff --git a/hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h b/hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h index b0101fa13..38b07173b 100644 --- a/hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h +++ b/hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h @@ -15,9 +15,6 @@ #include -#define HERMES_SYSTEM_INFO hshm::GlobalSingleton::GetInstance() -#define HERMES_SYSTEM_INFO_T hshm::SystemInfo* - #define HERMES_MEMORY_REGISTRY hshm::GlobalSingleton::GetInstance() #define HERMES_MEMORY_REGISTRY_REF hshm::GlobalSingleton::GetRef() #define HERMES_MEMORY_REGISTRY_T hshm::ipc::MemoryRegistry* @@ -26,7 +23,4 @@ #define HERMES_MEMORY_MANAGER_REF hshm::GlobalSingleton::GetRef() #define HERMES_MEMORY_MANAGER_T hshm::ipc::MemoryManager* -#define HERMES_THREAD_MODEL hshm::GlobalSingleton::GetInstance() -#define HERMES_THREAD_MODEL_T hshm::ThreadModelManager* - #endif // HERMES_INCLUDE_HERMES_CONSTANTS_DATA_STRUCTURE_SINGLETON_MACROS_H_H diff --git a/hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h b/hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h index 3a182ca27..ea9c9b004 100644 --- a/hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h +++ b/hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h @@ -28,6 +28,7 @@ struct charbuf { hipc::Allocator *alloc_; /**< The allocator used to allocate data */ char *data_; /**< The pointer to data */ size_t size_; /**< The size of data */ + size_t total_size_; /**< The true size of data buffer */ bool destructable_; /**< Whether or not this container owns data */ /**==================================== @@ -35,26 +36,28 @@ struct charbuf { * ===================================*/ /** Default constructor */ - charbuf() : alloc_(nullptr), data_(nullptr), size_(0), destructable_(false) {} + HSHM_ALWAYS_INLINE charbuf() + : alloc_(nullptr), data_(nullptr), size_(0), + total_size_(0), destructable_(false) {} /**==================================== * Destructor * ===================================*/ /** Destructor */ - ~charbuf() { Free(); } + HSHM_ALWAYS_INLINE ~charbuf() { Free(); } /**==================================== * Emplace Constructors * ===================================*/ /** Size-based constructor */ - explicit charbuf(size_t size) { + HSHM_ALWAYS_INLINE explicit charbuf(size_t size) { Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), size); } /** Allocator + Size-based constructor */ - explicit charbuf(hipc::Allocator *alloc, size_t size) { + HSHM_ALWAYS_INLINE explicit charbuf(hipc::Allocator *alloc, size_t size) { Allocate(alloc, size); } @@ -63,30 +66,31 @@ struct charbuf { * ===================================*/ /** Reference constructor. From char* + size */ - explicit charbuf(char *data, size_t size) - : alloc_(nullptr), data_(data), size_(size), destructable_(false) {} + HSHM_ALWAYS_INLINE explicit charbuf(char *data, size_t size) + : alloc_(nullptr), data_(data), size_(size), + total_size_(size), destructable_(false) {} /** * Reference constructor. From const char* * We assume that the data will not be modified by the user, but * we must cast away the const anyway. * */ - explicit charbuf(const char *data, size_t size) - : alloc_(nullptr), data_(const_cast(data)), - size_(size), destructable_(false) {} + HSHM_ALWAYS_INLINE explicit charbuf(const char *data, size_t size) + : alloc_(nullptr), data_(const_cast(data)), + size_(size), total_size_(size), destructable_(false) {} /**==================================== * Copy Constructors * ===================================*/ /** Copy constructor. From std::string. */ - explicit charbuf(const std::string &data) { + HSHM_ALWAYS_INLINE explicit charbuf(const std::string &data) { Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), data.size()); memcpy(data_, data.data(), data.size()); } /** Copy constructor. From charbuf. */ - charbuf(const charbuf &other) { + HSHM_ALWAYS_INLINE charbuf(const charbuf &other) { if (!Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), other.size())) { return; @@ -95,7 +99,7 @@ struct charbuf { } /** Copy assignment operator */ - charbuf& operator=(const charbuf &other) { + HSHM_ALWAYS_INLINE charbuf& operator=(const charbuf &other) { if (this != &other) { Free(); if (!Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), @@ -116,20 +120,24 @@ struct charbuf { alloc_ = other.alloc_; data_ = other.data_; size_ = other.size_; + total_size_ = other.total_size_; destructable_ = other.destructable_; other.size_ = 0; + other.total_size_ = 0; other.destructable_ = false; } /** Move assignment operator */ - charbuf& operator=(charbuf &&other) { + charbuf& operator=(charbuf &&other) noexcept { if (this != &other) { Free(); alloc_ = other.alloc_; data_ = other.data_; size_ = other.size_; + total_size_ = other.total_size_; destructable_ = other.destructable_; other.size_ = 0; + other.total_size_ = 0; other.destructable_ = false; } return *this; @@ -141,7 +149,7 @@ struct charbuf { /** Destroy and resize */ void resize(size_t new_size) { - if (new_size <= size()) { + if (new_size <= total_size_) { size_ = new_size; return; } @@ -155,30 +163,31 @@ struct charbuf { } destructable_ = true; size_ = new_size; + total_size_ = new_size; } /** Reference data */ - char* data() { + HSHM_ALWAYS_INLINE char* data() { return data_; } /** Reference data */ - char* data() const { + HSHM_ALWAYS_INLINE char* data() const { return data_; } /** Reference size */ - size_t size() const { + HSHM_ALWAYS_INLINE size_t size() const { return size_; } /** Get allocator */ - hipc::Allocator* GetAllocator() { + HSHM_ALWAYS_INLINE hipc::Allocator* GetAllocator() { return alloc_; } /** Convert to std::string */ - std::string str() { + HSHM_ALWAYS_INLINE std::string str() { return std::string(data(), size()); } @@ -186,8 +195,8 @@ struct charbuf { * Comparison Operators * ===================================*/ - int _strncmp(const char *a, size_t len_a, - const char *b, size_t len_b) const { + HSHM_ALWAYS_INLINE int _strncmp(const char *a, size_t len_a, + const char *b, size_t len_b) const { if (len_a != len_b) { return int((int64_t)len_a - (int64_t)len_b); } @@ -231,19 +240,21 @@ struct charbuf { alloc_ = nullptr; data_ = nullptr; size_ = 0; + total_size_ = 0; destructable_ = false; return false; } alloc_ = alloc; data_ = alloc->AllocatePtr(size, p); size_ = size; + total_size_ = size; destructable_ = true; return true; } /** Explicitly free the charbuf */ void Free() { - if (destructable_ && data_ && size_) { + if (destructable_ && data_ && total_size_) { alloc_->FreePtr(data_); } } diff --git a/hermes_shm/include/hermes_shm/data_structures/data_structure.h b/hermes_shm/include/hermes_shm/data_structures/data_structure.h index 8880c8ec0..422238fc8 100644 --- a/hermes_shm/include/hermes_shm/data_structures/data_structure.h +++ b/hermes_shm/include/hermes_shm/data_structures/data_structure.h @@ -14,14 +14,24 @@ #ifndef HERMES_DATA_STRUCTURES_DATA_STRUCTURE_H_ #define HERMES_DATA_STRUCTURES_DATA_STRUCTURE_H_ -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" +#include "ipc/internal/shm_internal.h" #include "hermes_shm/memory/memory_manager.h" -#include "hermes_shm/data_structures/ipc/pair.h" -#include "hermes_shm/data_structures/ipc/string.h" + +#include "containers/charbuf.h" +#include "containers/converters.h" +#include "containers/functional.h" +#include "containers/tuple_base.h" + +#include "ipc/pair.h" +#include "ipc/string.h" #include "ipc/list.h" #include "ipc/vector.h" +#include "ipc/mpsc_queue.h" +#include "ipc/slist.h" +#include "ipc/split_ticket_queue.h" +#include "ipc/spsc_queue.h" +#include "ipc/ticket_queue.h" #include "ipc/unordered_map.h" -#include "hermes_shm/memory/memory_manager.h" namespace hipc = hshm::ipc; diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h index 8386bd5ce..311d7e1ab 100644 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h @@ -23,32 +23,49 @@ struct qtok_t { _qtok_t id_; /** Default constructor */ - qtok_t() = default; + HSHM_ALWAYS_INLINE qtok_t() = default; /** Emplace constructor */ - explicit qtok_t(_qtok_t id) : id_(id) {} + HSHM_ALWAYS_INLINE explicit qtok_t(_qtok_t id) : id_(id) {} /** Copy constructor */ - qtok_t(const qtok_t &other) : id_(other.id_) {} + HSHM_ALWAYS_INLINE qtok_t(const qtok_t &other) : id_(other.id_) {} + + /** Copy assign */ + HSHM_ALWAYS_INLINE qtok_t& operator=(const qtok_t &other) { + if (this != &other) { + id_ = other.id_; + } + return *this; + } /** Move constructor */ - qtok_t(qtok_t &&other) : id_(other.id_) { + HSHM_ALWAYS_INLINE qtok_t(qtok_t &&other) : id_(other.id_) { other.SetNull(); } + /** Move assign */ + HSHM_ALWAYS_INLINE qtok_t& operator=(qtok_t &&other) { + if (this != &other) { + id_ = other.id_; + other.SetNull(); + } + return *this; + } + /** Set to the null qtok */ - void SetNull() { + HSHM_ALWAYS_INLINE void SetNull() { id_ = qtok_t::GetNull().id_; } /** Get the null qtok */ - static qtok_t GetNull() { + HSHM_ALWAYS_INLINE static qtok_t GetNull() { static qtok_t other(std::numeric_limits<_qtok_t>::max()); return other; } /** Check if null */ - bool IsNull() const { + HSHM_ALWAYS_INLINE bool IsNull() const { return id_ == GetNull().id_; } }; diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h index 316b92d86..a4184a106 100644 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h @@ -90,13 +90,14 @@ class ShmArchive { /** Initialize */ template - void shm_init(Args&& ...args) { + HSHM_ALWAYS_INLINE void shm_init(Args&& ...args) { Allocator::ConstructObj(get_ref(), std::forward(args)...); } /** Initialize piecewise */ template - void shm_init_piecewise(ArgPackT_1 &&args1, ArgPackT_2 &&args2) { + HSHM_ALWAYS_INLINE void shm_init_piecewise(ArgPackT_1 &&args1, + ArgPackT_2 &&args2) { return hshm::PassArgPack::Call( MergeArgPacks::Merge( make_argpack(get_ref()), diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h index f561855f3..51dd1bf6a 100644 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h @@ -70,7 +70,7 @@ class ShmContainer {}; /** Typed nullptr */ template -static inline T* typed_nullptr() { +HSHM_ALWAYS_INLINE static T* typed_nullptr() { return reinterpret_cast(NULL); } diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h index ab5e1b34f..b04cb436d 100644 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h @@ -10,8 +10,8 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_templ_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_templ_H_ +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_H_ #include "hermes_shm/data_structures/ipc/internal/shm_internal.h" #include "hermes_shm/thread/lock.h" @@ -21,30 +21,30 @@ namespace hshm::ipc { -/** Forward declaration of mpsc_queue_templ */ -template -class mpsc_queue_templ; +/** Forward declaration of mpsc_queue */ +template +class mpsc_queue; /** - * MACROS used to simplify the mpsc_queue_templ namespace + * MACROS used to simplify the mpsc_queue namespace * Used as inputs to the SHM_CONTAINER_TEMPLATE * */ -#define CLASS_NAME mpsc_queue_templ -#define TYPED_CLASS mpsc_queue_templ -#define TYPED_HEADER ShmHeader> +#define CLASS_NAME mpsc_queue +#define TYPED_CLASS mpsc_queue +#define TYPED_HEADER ShmHeader> /** * A queue optimized for multiple producers (emplace) with a single * consumer (pop). * */ -template -class mpsc_queue_templ : public ShmContainer { +template +class mpsc_queue : public ShmContainer { public: SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) ShmArchive>> queue_; std::atomic<_qtok_t> tail_; std::atomic<_qtok_t> head_; - Mutex lock_; + RwLock lock_; public: /**==================================== @@ -52,12 +52,11 @@ class mpsc_queue_templ : public ShmContainer { * ===================================*/ /** SHM constructor. Default. */ - explicit mpsc_queue_templ(Allocator *alloc, - size_t depth = 1024) { + explicit mpsc_queue(Allocator *alloc, + size_t depth = 1024) { shm_init_container(alloc); HSHM_MAKE_AR(queue_, GetAllocator(), depth); SetNull(); - lock_.Init(); } /**==================================== @@ -65,15 +64,15 @@ class mpsc_queue_templ : public ShmContainer { * ===================================*/ /** SHM copy constructor */ - explicit mpsc_queue_templ(Allocator *alloc, - const mpsc_queue_templ &other) { + explicit mpsc_queue(Allocator *alloc, + const mpsc_queue &other) { shm_init_container(alloc); SetNull(); shm_strong_copy_construct_and_op(other); } /** SHM copy assignment operator */ - mpsc_queue_templ& operator=(const mpsc_queue_templ &other) { + mpsc_queue& operator=(const mpsc_queue &other) { if (this != &other) { shm_destroy(); shm_strong_copy_construct_and_op(other); @@ -82,7 +81,7 @@ class mpsc_queue_templ : public ShmContainer { } /** SHM copy constructor + operator main */ - void shm_strong_copy_construct_and_op(const mpsc_queue_templ &other) { + void shm_strong_copy_construct_and_op(const mpsc_queue &other) { head_ = other.head_.load(); tail_ = other.tail_.load(); (*queue_) = (*other.queue_); @@ -93,8 +92,8 @@ class mpsc_queue_templ : public ShmContainer { * ===================================*/ /** SHM move constructor. */ - mpsc_queue_templ(Allocator *alloc, - mpsc_queue_templ &&other) noexcept { + mpsc_queue(Allocator *alloc, + mpsc_queue &&other) noexcept { shm_init_container(alloc); if (GetAllocator() == other.GetAllocator()) { head_ = other.head_.load(); @@ -108,7 +107,7 @@ class mpsc_queue_templ : public ShmContainer { } /** SHM move assignment operator. */ - mpsc_queue_templ& operator=(mpsc_queue_templ &&other) noexcept { + mpsc_queue& operator=(mpsc_queue &&other) noexcept { if (this != &other) { shm_destroy(); if (GetAllocator() == other.GetAllocator()) { @@ -157,77 +156,35 @@ class mpsc_queue_templ : public ShmContainer { _qtok_t tail = tail_.fetch_add(1); size_t size = tail - head + 1; - ScopedMutex lock(lock_, 0); - - // Check if there's space in the queue. Resize if necessary. + // Check if there's space in the queue. if (size > (*queue_).size()) { - if constexpr(EXTENSIBLE) { - ScopedRwWriteLock resize_lock(lock_, 0); - if (size > (*queue_).size()) { - size_t old_size = (*queue_).size(); - size_t new_size = (RealNumber(5, 4) * (size + 64)).as_int(); - auto new_queue = - hipc::make_uptr>>(new_size); - for (uint64_t i = 0; i < old_size; ++i) { - _qtok_t i_old = (head + i) % old_size; - _qtok_t i_new = (head + i) % new_size; - (*new_queue)[i_new] = std::move((*queue_)[i_old]); - } - (*queue_) = std::move(*new_queue); - } - } else { - while (1) { - head = head_.load(); - size = tail - head + 1; - if (size <= (*queue_).size()) { - break; - } - HERMES_THREAD_MODEL->Yield(); + while (true) { + head = head_.load(); + size = tail - head + 1; + if (size <= (*queue_).size()) { + break; } + HERMES_THREAD_MODEL->Yield(); } } // Emplace into queue at our slot - if constexpr(EXTENSIBLE) { - ScopedRwReadLock resize_lock(lock_, 0); - _emplace(tail, std::forward(args)...); - } else { - _emplace(tail, std::forward(args)...); - } - return qtok_t(tail); - } - - private: - /** Emplace operation */ - template - HSHM_ALWAYS_INLINE void _emplace(_qtok_t tail, Args&& ...args) { uint32_t idx = tail % (*queue_).size(); auto iter = (*queue_).begin() + idx; (*queue_).replace(iter, - hshm::PiecewiseConstruct(), - make_argpack(), - make_argpack(std::forward(args)...)); + hshm::PiecewiseConstruct(), + make_argpack(), + make_argpack(std::forward(args)...)); // Let pop know that the data is fully prepared pair &entry = (*iter); entry.GetFirst().SetBits(1); + return qtok_t(tail); } public: /** Consumer pops the head object */ - HSHM_ALWAYS_INLINE qtok_t pop(T &val) { - ScopedMutex lock(lock_, 0); - - if constexpr(EXTENSIBLE) { - ScopedRwReadLock resize_lock(lock_, 0); - return _pop(val); - } else { - return _pop(val); - } - } - - /** Pop operation */ - HSHM_ALWAYS_INLINE qtok_t _pop(T &val) { + qtok_t pop(T &val) { // Don't pop if there's no entries _qtok_t head = head_.load(); _qtok_t tail = tail_.load(); @@ -249,16 +206,10 @@ class mpsc_queue_templ : public ShmContainer { } }; -template -using mpsc_queue_ext = mpsc_queue_templ; - -template -using mpsc_queue = mpsc_queue_templ; - } // namespace hshm::ipc #undef CLASS_NAME #undef TYPED_CLASS #undef TYPED_HEADER -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_templ_H_ +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h new file mode 100644 index 000000000..c33d9ed5c --- /dev/null +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h @@ -0,0 +1,188 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM__DATA_STRUCTURES_IPC_SPLIT_TICKET_QUEUE_H_ +#define HERMES_SHM__DATA_STRUCTURES_IPC_SPLIT_TICKET_QUEUE_H_ + +#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" +#include "hermes_shm/thread/lock.h" +#include "vector.h" +#include "ticket_queue.h" + +namespace hshm::ipc { + +/** Forward declaration of split_ticket_queue */ +template +class split_ticket_queue; + +/** + * MACROS used to simplify the split_ticket_queue namespace + * Used as inputs to the SHM_CONTAINER_TEMPLATE + * */ +#define CLASS_NAME split_ticket_queue +#define TYPED_CLASS split_ticket_queue +#define TYPED_HEADER ShmHeader> + +/** + * A MPMC queue for allocating tickets. Handles concurrency + * without blocking. + * */ +template +class split_ticket_queue : public ShmContainer { + public: + SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) + ShmArchive>> splits_; + std::atomic rr_tail_, rr_head_; + + public: + /**==================================== + * Default Constructor + * ===================================*/ + + /** SHM constructor. Default. */ + explicit split_ticket_queue(Allocator *alloc, + size_t depth_per_split = 1024, + size_t split = 0) { + shm_init_container(alloc); + if (split == 0) { + split = HERMES_SYSTEM_INFO->ncpu_; + } + HSHM_MAKE_AR(splits_, GetAllocator(), split, depth_per_split); + SetNull(); + } + + /**==================================== + * Copy Constructors + * ===================================*/ + + /** SHM copy constructor */ + explicit split_ticket_queue(Allocator *alloc, + const split_ticket_queue &other) { + shm_init_container(alloc); + SetNull(); + shm_strong_copy_construct_and_op(other); + } + + /** SHM copy assignment operator */ + split_ticket_queue& operator=(const split_ticket_queue &other) { + if (this != &other) { + shm_destroy(); + shm_strong_copy_construct_and_op(other); + } + return *this; + } + + /** SHM copy constructor + operator main */ + void shm_strong_copy_construct_and_op(const split_ticket_queue &other) { + (*splits_) = (*other.splits_); + } + + /**==================================== + * Move Constructors + * ===================================*/ + + /** SHM move constructor. */ + split_ticket_queue(Allocator *alloc, + split_ticket_queue &&other) noexcept { + shm_init_container(alloc); + if (GetAllocator() == other.GetAllocator()) { + (*splits_) = std::move(*other.splits_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + + /** SHM move assignment operator. */ + split_ticket_queue& operator=(split_ticket_queue &&other) noexcept { + if (this != &other) { + shm_destroy(); + if (GetAllocator() == other.GetAllocator()) { + (*splits_) = std::move(*other.splits_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + return *this; + } + + /**==================================== + * Destructor + * ===================================*/ + + /** SHM destructor. */ + void shm_destroy_main() { + (*splits_).shm_destroy(); + } + + /** Check if the list is empty */ + bool IsNull() const { + return (*splits_).IsNull(); + } + + /** Sets this list as empty */ + void SetNull() { + rr_tail_ = 0; + rr_head_ = 0; + } + + /**==================================== + * ticket Queue Methods + * ===================================*/ + + /** Construct an element at \a pos position in the queue */ + template + qtok_t emplace(T &tkt) { + uint16_t rr = rr_tail_.fetch_add(1); + auto &splits = (*splits_); + size_t num_splits = splits.size(); + uint16_t qid_start = rr % num_splits; + for (size_t i = 0; i < num_splits; ++i) { + uint32_t qid = (qid_start + i) % num_splits; + ticket_queue &queue = (*splits_)[qid]; + qtok_t qtok = queue.emplace(tkt); + if (!qtok.IsNull()) { + return qtok; + } + } + return qtok_t::GetNull(); + } + + public: + /** Pop an element from the queue */ + qtok_t pop(T &tkt) { + uint16_t rr = rr_head_.fetch_add(1); + auto &splits = (*splits_); + size_t num_splits = splits.size(); + uint16_t qid_start = rr % num_splits; + for (size_t i = 0; i < num_splits; ++i) { + uint32_t qid = (qid_start + i) % num_splits; + ticket_queue &queue = (*splits_)[qid]; + qtok_t qtok = queue.pop(tkt); + if (!qtok.IsNull()) { + return qtok; + } + } + return qtok_t::GetNull(); + } +}; + +} // namespace hshm::ipc + +#undef TYPED_HEADER +#undef TYPED_CLASS +#undef CLASS_NAME + +#endif // HERMES_SHM__DATA_STRUCTURES_IPC_SPLIT_TICKET_QUEUE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h index a80762d65..bb5a4fbe1 100644 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h @@ -42,7 +42,7 @@ template class spsc_queue_templ : public ShmContainer { public: SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - ShmArchive>> queue_; + ShmArchive> queue_; _qtok_t tail_; _qtok_t head_; @@ -53,7 +53,7 @@ class spsc_queue_templ : public ShmContainer { /** SHM constructor. Default. */ explicit spsc_queue_templ(Allocator *alloc, - size_t depth = 1024) { + size_t depth = 1024) { shm_init_container(alloc); HSHM_MAKE_AR(queue_, GetAllocator(), depth) SetNull(); @@ -150,39 +150,25 @@ class spsc_queue_templ : public ShmContainer { /** Construct an element at \a pos position in the list */ template qtok_t emplace(Args&&... args) { - // Allocate a slot in the queue - // The slot is marked NULL, so pop won't do anything if context switch - _qtok_t tail = tail_ + 1; - - // Emplace into queue at our slot - _emplace(tail, std::forward(args)...); - return qtok_t(tail); - } + // Don't emplace if there is no space + _qtok_t entry_tok = tail_; + size_t size = tail_ - head_; + auto &queue = (*queue_); + if (size >= queue.size()) { + return qtok_t::GetNull(); + } - private: - /** Emplace operation */ - template - HSHM_ALWAYS_INLINE void _emplace(const _qtok_t &tail, Args&& ...args) { - uint32_t idx = tail % (*queue_).size(); - auto iter = (*queue_).begin() + idx; - (*queue_).replace(iter, - hshm::PiecewiseConstruct(), - make_argpack(), - make_argpack(std::forward(args)...)); - - // Let pop know that the data is fully prepared - pair &entry = (*iter); - entry.GetFirst().SetBits(1); + // Do the emplace + _qtok_t idx = entry_tok % queue.size(); + auto iter = queue.begin() + idx; + queue.replace(iter, std::forward(args)...); + tail_ += 1; + return qtok_t(entry_tok); } public: /** Consumer pops the head object */ qtok_t pop(T &val) { - return _pop(val); - } - - /** Pop operation */ - qtok_t _pop(T &val) { // Don't pop if there's no entries _qtok_t head = head_; _qtok_t tail = tail_; @@ -190,17 +176,13 @@ class spsc_queue_templ : public ShmContainer { return qtok_t::GetNull(); } - // Pop the element, but only if it's marked valid - _qtok_t idx = head % (*queue_).size(); - hipc::pair &entry = (*queue_)[idx]; - if (entry.GetFirst().Any(1)) { - (val) = std::move(entry.GetSecond()); - entry.GetFirst().Clear(); - head_ += 1; - return qtok_t(head); - } else { - return qtok_t::GetNull(); - } + // Pop the element + auto &queue = (*queue_); + _qtok_t idx = head % queue.size(); + T &entry = queue[idx]; + (val) = std::move(entry); + head_ += 1; + return qtok_t(head); } }; diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h new file mode 100644 index 000000000..0893bf261 --- /dev/null +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h @@ -0,0 +1,164 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_QUEUE_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_QUEUE_H_ + +#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" +#include "hermes_shm/thread/lock.h" +#include "spsc_queue.h" +#include "_queue.h" + +namespace hshm::ipc { + +/** Forward declaration of ticket_queue */ +template +class ticket_queue; + +/** + * MACROS used to simplify the ticket_queue namespace + * Used as inputs to the SHM_CONTAINER_TEMPLATE + * */ +#define CLASS_NAME ticket_queue +#define TYPED_CLASS ticket_queue +#define TYPED_HEADER ShmHeader> + +/** + * A MPMC queue for allocating tickets. Handles concurrency + * without blocking. + * */ +template +class ticket_queue : public ShmContainer { + public: + SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) + ShmArchive> queue_; + hshm::Mutex lock_; + + public: + /**==================================== + * Default Constructor + * ===================================*/ + + /** SHM constructor. Default. */ + explicit ticket_queue(Allocator *alloc, + size_t depth = 1024) { + shm_init_container(alloc); + HSHM_MAKE_AR(queue_, GetAllocator(), depth); + lock_.Init(); + SetNull(); + } + + /**==================================== + * Copy Constructors + * ===================================*/ + + /** SHM copy constructor */ + explicit ticket_queue(Allocator *alloc, + const ticket_queue &other) { + shm_init_container(alloc); + SetNull(); + shm_strong_copy_construct_and_op(other); + } + + /** SHM copy assignment operator */ + ticket_queue& operator=(const ticket_queue &other) { + if (this != &other) { + shm_destroy(); + shm_strong_copy_construct_and_op(other); + } + return *this; + } + + /** SHM copy constructor + operator main */ + void shm_strong_copy_construct_and_op(const ticket_queue &other) { + (*queue_) = (*other.queue_); + } + + /**==================================== + * Move Constructors + * ===================================*/ + + /** SHM move constructor. */ + ticket_queue(Allocator *alloc, + ticket_queue &&other) noexcept { + shm_init_container(alloc); + if (GetAllocator() == other.GetAllocator()) { + (*queue_) = std::move(*other.queue_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + + /** SHM move assignment operator. */ + ticket_queue& operator=(ticket_queue &&other) noexcept { + if (this != &other) { + shm_destroy(); + if (GetAllocator() == other.GetAllocator()) { + (*queue_) = std::move(*other.queue_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + return *this; + } + + /**==================================== + * Destructor + * ===================================*/ + + /** SHM destructor. */ + void shm_destroy_main() { + (*queue_).shm_destroy(); + } + + /** Check if the list is empty */ + bool IsNull() const { + return (*queue_).IsNull(); + } + + /** Sets this list as empty */ + void SetNull() {} + + /**==================================== + * ticket Queue Methods + * ===================================*/ + + /** Construct an element at \a pos position in the queue */ + template + HSHM_ALWAYS_INLINE qtok_t emplace(T &tkt) { + lock_.Lock(0); + auto qtok = queue_->emplace(tkt); + lock_.Unlock(); + return qtok; + } + + public: + /** Pop an element from the queue */ + HSHM_ALWAYS_INLINE qtok_t pop(T &tkt) { + lock_.Lock(0); + auto qtok = queue_->pop(tkt); + lock_.Unlock(); + return qtok; + } +}; + +} // namespace hshm::ipc + +#undef TYPED_HEADER +#undef TYPED_CLASS +#undef CLASS_NAME + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_QUEUE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h b/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h new file mode 100644 index 000000000..7d419c3a0 --- /dev/null +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h @@ -0,0 +1,166 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_STACK_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_STACK_H_ + +#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" +#include "hermes_shm/thread/lock.h" +#include "vector.h" +#include "_queue.h" +#include "spsc_queue.h" + +namespace hshm::ipc { + +/** Forward declaration of ticket_stack */ +template +class ticket_stack; + +/** + * MACROS used to simplify the ticket_stack namespace + * Used as inputs to the SHM_CONTAINER_TEMPLATE + * */ +#define CLASS_NAME ticket_stack +#define TYPED_CLASS ticket_stack +#define TYPED_HEADER ShmHeader> + +/** + * A MPMC queue for allocating tickets. Handles concurrency + * without blocking. + * */ +template +class ticket_stack : public ShmContainer { + public: + SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) + ShmArchive> queue_; + hshm::Mutex lock_; + + public: + /**==================================== + * Default Constructor + * ===================================*/ + + /** SHM constructor. Default. */ + explicit ticket_stack(Allocator *alloc, + size_t depth = 1024) { + shm_init_container(alloc); + HSHM_MAKE_AR(queue_, GetAllocator(), depth); + lock_.Init(); + SetNull(); + } + + /**==================================== + * Copy Constructors + * ===================================*/ + + /** SHM copy constructor */ + explicit ticket_stack(Allocator *alloc, + const ticket_stack &other) { + shm_init_container(alloc); + SetNull(); + shm_strong_copy_construct_and_op(other); + } + + /** SHM copy assignment operator */ + ticket_stack& operator=(const ticket_stack &other) { + if (this != &other) { + shm_destroy(); + shm_strong_copy_construct_and_op(other); + } + return *this; + } + + /** SHM copy constructor + operator main */ + void shm_strong_copy_construct_and_op(const ticket_stack &other) { + (*queue_) = (*other.queue_); + } + + /**==================================== + * Move Constructors + * ===================================*/ + + /** SHM move constructor. */ + ticket_stack(Allocator *alloc, + ticket_stack &&other) noexcept { + shm_init_container(alloc); + if (GetAllocator() == other.GetAllocator()) { + (*queue_) = std::move(*other.queue_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + + /** SHM move assignment operator. */ + ticket_stack& operator=(ticket_stack &&other) noexcept { + if (this != &other) { + shm_destroy(); + if (GetAllocator() == other.GetAllocator()) { + (*queue_) = std::move(*other.queue_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + return *this; + } + + /**==================================== + * Destructor + * ===================================*/ + + /** SHM destructor. */ + void shm_destroy_main() { + (*queue_).shm_destroy(); + } + + /** Check if the list is empty */ + bool IsNull() const { + return (*queue_).IsNull(); + } + + /** Sets this list as empty */ + void SetNull() { + } + + /**==================================== + * ticket Queue Methods + * ===================================*/ + + /** Construct an element at \a pos position in the queue */ + template + HSHM_ALWAYS_INLINE qtok_t emplace(T &tkt) { + lock_.Lock(0); + auto qtok = queue_->emplace(tkt); + lock_.Unlock(); + return qtok; + } + + public: + /** Pop an element from the queue */ + HSHM_ALWAYS_INLINE qtok_t pop(T &tkt) { + lock_.Lock(0); + auto qtok = queue_->pop(tkt); + lock_.Unlock(); + return qtok; + } +}; + +} // namespace hshm::ipc + +#undef TYPED_HEADER +#undef TYPED_CLASS +#undef CLASS_NAME + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_STACK_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/vector.h b/hermes_shm/include/hermes_shm/data_structures/ipc/vector.h index 62fc8df3c..4a3da9816 100644 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/vector.h +++ b/hermes_shm/include/hermes_shm/data_structures/ipc/vector.h @@ -361,7 +361,7 @@ class vector : public ShmContainer { } /** Destroy all shared memory allocated by the vector */ - void shm_destroy_main() { + HSHM_ALWAYS_INLINE void shm_destroy_main() { erase(begin(), end()); GetAllocator()->Free(vec_ptr_); } @@ -373,7 +373,7 @@ class vector : public ShmContainer { /** * Convert to std::vector * */ - std::vector vec() { + HSHM_ALWAYS_INLINE std::vector vec() { std::vector v; v.reserve(size()); for (T& entry : *this) { @@ -390,7 +390,7 @@ class vector : public ShmContainer { * @param args the arguments to construct * */ template - void reserve(size_t length, Args&& ...args) { + HSHM_ALWAYS_INLINE void reserve(size_t length, Args&& ...args) { if (length == 0) { return; } grow_vector(data_ar(), length, false, std::forward(args)...); } @@ -470,25 +470,25 @@ class vector : public ShmContainer { /** Replace an element at a position */ template - void replace(iterator_t pos, Args&&... args) { + HSHM_ALWAYS_INLINE void replace(iterator_t pos, Args&&... args) { if (pos.is_end()) { return; } ShmArchive *vec = data_ar(); - (*this)[pos.i_].shm_destroy(); + hipc::Allocator::DestructObj((*this)[pos.i_]); HSHM_MAKE_AR(vec[pos.i_], GetAllocator(), std::forward(args)...) } /** Delete the element at \a pos position */ - void erase(iterator_t pos) { + HSHM_ALWAYS_INLINE void erase(iterator_t pos) { if (pos.is_end()) return; shift_left(pos, 1); length_ -= 1; } /** Delete elements between first and last */ - void erase(iterator_t first, iterator_t last) { + HSHM_ALWAYS_INLINE void erase(iterator_t first, iterator_t last) { size_t last_i; if (first.is_end()) return; if (last.is_end()) { diff --git a/hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h b/hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h index fb20fcbbc..49c51e37e 100644 --- a/hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h +++ b/hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h @@ -1,9 +1,17 @@ -// -// Created by llogan on 5/9/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ +#ifndef HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ +#define HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ #define NOREF typename std::remove_reference::type @@ -29,14 +37,16 @@ class ShmSerializer { throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); } }; - ForwardIterateArgpack::Apply(make_argpack(std::forward(args)...), lambda); + ForwardIterateArgpack::Apply(make_argpack( + std::forward(args)...), lambda); return size; } /** Serialize a set of arguments into shared memory */ template HSHM_ALWAYS_INLINE char* serialize(Allocator *alloc, Args&& ...args) { - size_t buf_size = sizeof(allocator_id_t) + shm_buf_size(std::forward(args)...); + size_t buf_size = sizeof(allocator_id_t) + shm_buf_size( + std::forward(args)...); Pointer p; char *buf = alloc->AllocatePtr(buf_size, p); memcpy(buf, &p.allocator_id_, sizeof(allocator_id_t)); @@ -53,7 +63,8 @@ class ShmSerializer { throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); } }; - ForwardIterateArgpack::Apply(make_argpack(std::forward(args)...), lambda); + ForwardIterateArgpack::Apply(make_argpack( + std::forward(args)...), lambda); return buf; } @@ -80,7 +91,8 @@ class ShmSerializer { /** Deserialize an argument from the SHM buffer */ template - HSHM_ALWAYS_INLINE void deserialize(Allocator *alloc, char *buf, hipc::mptr &arg) { + HSHM_ALWAYS_INLINE void deserialize(Allocator *alloc, + char *buf, hipc::mptr &arg) { if constexpr(IS_SHM_ARCHIVEABLE(T)) { OffsetPointer p; memcpy((void*)&p, buf + off_, sizeof(p)); @@ -92,8 +104,8 @@ class ShmSerializer { } }; -} // namespace hshm +} // namespace hshm::ipc #undef NOREF -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ +#endif // HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h b/hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h index c2988fe4f..f861269c8 100644 --- a/hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h +++ b/hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h @@ -199,7 +199,8 @@ class smart_ptr_base { } /** Deserialize from an offset pointer */ - HSHM_ALWAYS_INLINE void shm_deserialize(Allocator *alloc, const OffsetPointer &ar) { + HSHM_ALWAYS_INLINE void shm_deserialize(Allocator *alloc, + const OffsetPointer &ar) { obj_ = alloc->template Convert(ar); if constexpr(unique) { flags_.UnsetBits(POINTER_IS_OWNED); diff --git a/hermes_shm/include/hermes_shm/hermes_shm.h b/hermes_shm/include/hermes_shm/hermes_shm.h new file mode 100644 index 000000000..787de1f85 --- /dev/null +++ b/hermes_shm/include/hermes_shm/hermes_shm.h @@ -0,0 +1,38 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_HERMES_SHM_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_HERMES_SHM_H_ + +#include "thread/thread_model_manager.h" +#include "thread/lock.h" + +#include "util/singleton.h" +#include "util/auto_trace.h" +#include "util/config_parse.h" +#include "util/errors.h" +#include "util/formatter.h" +#include "util/logging.h" +#include "util/partitioner.h" +#include "util/timer.h" +#include "util/type_switch.h" + +#include "types/argpack.h" +#include "types/atomic.h" +#include "types/bitfield.h" +#include "types/real_number.h" + +#include "memory/memory_manager.h" + +#include "data_structures/data_structure.h" + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_HERMES_SHM_H_ diff --git a/hermes_shm/include/hermes_shm/introspect/system_info.h b/hermes_shm/include/hermes_shm/introspect/system_info.h index cc83a26ec..626d35db8 100644 --- a/hermes_shm/include/hermes_shm/introspect/system_info.h +++ b/hermes_shm/include/hermes_shm/introspect/system_info.h @@ -15,7 +15,11 @@ #include #include -#include "hermes_shm/constants/data_structure_singleton_macros.h" +#include "hermes_shm/util/singleton/_global_singleton.h" + +#define HERMES_SYSTEM_INFO \ + hshm::GlobalSingleton::GetInstance() +#define HERMES_SYSTEM_INFO_T hshm::SystemInfo* namespace hshm { diff --git a/hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h b/hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h index 33564a929..9971f96fe 100644 --- a/hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h +++ b/hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h @@ -45,7 +45,6 @@ class AllocatorFactory { auto alloc = std::make_unique(); alloc->shm_init(alloc_id, custom_header_size, - backend->data_, backend->data_size_, std::forward(args)...); return alloc; diff --git a/hermes_shm/include/hermes_shm/memory/allocator/heap.h b/hermes_shm/include/hermes_shm/memory/allocator/heap.h new file mode 100644 index 000000000..2b1d03911 --- /dev/null +++ b/hermes_shm/include/hermes_shm/memory/allocator/heap.h @@ -0,0 +1,50 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_ALLOCATOR_HEAP_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_ALLOCATOR_HEAP_H_ + +#include "allocator.h" +#include "hermes_shm/thread/lock.h" + +namespace hshm::ipc { + +struct HeapAllocator { + std::atomic heap_off_; + size_t heap_size_; + + /** Default constructor */ + HeapAllocator() : heap_off_(0), heap_size_(0) {} + + /** Emplace constructor */ + explicit HeapAllocator(size_t heap_off, size_t heap_size) + : heap_off_(heap_off), heap_size_(heap_size) {} + + /** Explicit initialization */ + void shm_init(size_t heap_off, size_t heap_size) { + heap_off_ = heap_off; + heap_size_ = heap_size; + } + + /** Allocate off heap */ + HSHM_ALWAYS_INLINE OffsetPointer AllocateOffset(size_t size) { + size_t off = heap_off_.fetch_add(size); + if (off + size > heap_size_) { + throw OUT_OF_MEMORY.format(); + } + return OffsetPointer(off); + } +}; + +} // namespace hshm::ipc + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_ALLOCATOR_HEAP_H_ diff --git a/hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h b/hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h index 3ce8b53f6..ec777f7bc 100644 --- a/hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h +++ b/hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h @@ -55,7 +55,6 @@ class MallocAllocator : public Allocator { * */ void shm_init(allocator_id_t id, size_t custom_header_size, - char *buffer, size_t buffer_size); /** diff --git a/hermes_shm/include/hermes_shm/memory/allocator/mp_page.h b/hermes_shm/include/hermes_shm/memory/allocator/mp_page.h index 1209e0f0e..306c65d34 100644 --- a/hermes_shm/include/hermes_shm/memory/allocator/mp_page.h +++ b/hermes_shm/include/hermes_shm/memory/allocator/mp_page.h @@ -18,22 +18,21 @@ namespace hshm::ipc { struct MpPage { - iqueue_entry entry_; /**< Position of page in free list */ - size_t page_size_; /**< The size of the page allocated */ - int flags_; /**< Page flags (e.g., is_allocated?) */ - uint32_t off_; /**< The offset within the page */ - uint32_t cpu_; /**< The CPU the page was alloc'd from */ - - void SetAllocated() { - flags_ = 0x1; + bitfield32_t flags_; /**< Flags of the page (e.g., free/alloc) */ + /** Offset from the start of the page to the beginning of this header */ + uint32_t off_; + size_t page_size_; /**< The total size of the page allocated */ + + HSHM_ALWAYS_INLINE void SetAllocated() { + flags_.SetBits(0x1); } - void UnsetAllocated() { - flags_ = 0; + HSHM_ALWAYS_INLINE void UnsetAllocated() { + flags_.Clear(); } - bool IsAllocated() const { - return flags_ & 0x1; + HSHM_ALWAYS_INLINE bool IsAllocated() const { + return flags_.All(0x1); } }; diff --git a/hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h b/hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h index e4b35ccc2..313a71c6c 100644 --- a/hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h +++ b/hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h @@ -15,13 +15,13 @@ #define HERMES_MEMORY_ALLOCATOR_STACK_ALLOCATOR_H_ #include "allocator.h" +#include "heap.h" #include "hermes_shm/thread/lock.h" namespace hshm::ipc { struct StackAllocatorHeader : public AllocatorHeader { - std::atomic region_off_; - std::atomic region_size_; + HeapAllocator heap_; std::atomic total_alloc_; StackAllocatorHeader() = default; @@ -32,15 +32,15 @@ struct StackAllocatorHeader : public AllocatorHeader { size_t region_size) { AllocatorHeader::Configure(alloc_id, AllocatorType::kStackAllocator, custom_header_size); - region_off_ = region_off; - region_size_ = region_size; + heap_.shm_init(region_off, region_size); total_alloc_ = 0; } }; class StackAllocator : public Allocator { - private: + public: StackAllocatorHeader *header_; + HeapAllocator *heap_; public: /** diff --git a/hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h b/hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h index d56ce4b0a..8b8ba62b5 100644 --- a/hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h +++ b/hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h @@ -82,7 +82,8 @@ class PosixMmap : public MemoryBackend { template T* _Map(size_t size) { T *ptr = reinterpret_cast( - mmap64(nullptr, NextPageSizeMultiple(size), PROT_READ | PROT_WRITE, + mmap64(nullptr, MemoryAlignment::AlignToPageSize(size), + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); if (ptr == MAP_FAILED) { perror("map failed"); diff --git a/hermes_shm/include/hermes_shm/memory/memory.h b/hermes_shm/include/hermes_shm/memory/memory.h index 38e82e558..6e3ad861b 100644 --- a/hermes_shm/include/hermes_shm/memory/memory.h +++ b/hermes_shm/include/hermes_shm/memory/memory.h @@ -90,121 +90,125 @@ struct OffsetPointerBase { atomic_t off_; /**< Offset within the allocator's slot */ /** Default constructor */ - OffsetPointerBase() = default; + HSHM_ALWAYS_INLINE OffsetPointerBase() = default; /** Full constructor */ - explicit OffsetPointerBase(size_t off) : off_(off) {} + HSHM_ALWAYS_INLINE explicit OffsetPointerBase(size_t off) : off_(off) {} /** Full constructor */ - explicit OffsetPointerBase(atomic_t off) : off_(off.load()) {} + HSHM_ALWAYS_INLINE explicit OffsetPointerBase(atomic_t off) + : off_(off.load()) {} /** Pointer constructor */ - explicit OffsetPointerBase(allocator_id_t alloc_id, size_t off) : off_(off) { + HSHM_ALWAYS_INLINE explicit OffsetPointerBase(allocator_id_t alloc_id, + size_t off) + : off_(off) { (void) alloc_id; } /** Copy constructor */ - OffsetPointerBase(const OffsetPointerBase &other) + HSHM_ALWAYS_INLINE OffsetPointerBase(const OffsetPointerBase &other) : off_(other.off_.load()) {} /** Other copy constructor */ - OffsetPointerBase(const OffsetPointerBase &other) + HSHM_ALWAYS_INLINE OffsetPointerBase(const OffsetPointerBase &other) : off_(other.off_.load()) {} /** Move constructor */ - OffsetPointerBase(OffsetPointerBase &&other) noexcept + HSHM_ALWAYS_INLINE OffsetPointerBase(OffsetPointerBase &&other) noexcept : off_(other.off_.load()) { other.SetNull(); } /** Get the offset pointer */ - OffsetPointerBase ToOffsetPointer() { + HSHM_ALWAYS_INLINE OffsetPointerBase ToOffsetPointer() { return OffsetPointerBase(off_.load()); } /** Set to null */ - void SetNull() { + HSHM_ALWAYS_INLINE void SetNull() { off_ = (size_t)-1; } /** Check if null */ - bool IsNull() const { + HSHM_ALWAYS_INLINE bool IsNull() const { return off_.load() == (size_t)-1; } /** Get the null pointer */ - static OffsetPointerBase GetNull() { + HSHM_ALWAYS_INLINE static OffsetPointerBase GetNull() { static const OffsetPointerBase p(-1); return p; } /** Atomic load wrapper */ - inline size_t load( + HSHM_ALWAYS_INLINE size_t load( std::memory_order order = std::memory_order_seq_cst) const { return off_.load(order); } /** Atomic exchange wrapper */ - inline void exchange( + HSHM_ALWAYS_INLINE void exchange( size_t count, std::memory_order order = std::memory_order_seq_cst) { off_.exchange(count, order); } /** Atomic compare exchange weak wrapper */ - inline bool compare_exchange_weak(size_t& expected, size_t desired, - std::memory_order order = - std::memory_order_seq_cst) { + HSHM_ALWAYS_INLINE bool compare_exchange_weak( + size_t& expected, size_t desired, + std::memory_order order = std::memory_order_seq_cst) { return off_.compare_exchange_weak(expected, desired, order); } /** Atomic compare exchange strong wrapper */ - inline bool compare_exchange_strong(size_t& expected, size_t desired, - std::memory_order order = - std::memory_order_seq_cst) { + HSHM_ALWAYS_INLINE bool compare_exchange_strong( + size_t& expected, size_t desired, + std::memory_order order = std::memory_order_seq_cst) { return off_.compare_exchange_weak(expected, desired, order); } /** Atomic add operator */ - inline OffsetPointerBase operator+(size_t count) const { + HSHM_ALWAYS_INLINE OffsetPointerBase operator+(size_t count) const { return OffsetPointerBase(off_ + count); } /** Atomic subtract operator */ - inline OffsetPointerBase operator-(size_t count) const { + HSHM_ALWAYS_INLINE OffsetPointerBase operator-(size_t count) const { return OffsetPointerBase(off_ - count); } /** Atomic add assign operator */ - inline OffsetPointerBase& operator+=(size_t count) { + HSHM_ALWAYS_INLINE OffsetPointerBase& operator+=(size_t count) { off_ += count; return *this; } /** Atomic subtract assign operator */ - inline OffsetPointerBase& operator-=(size_t count) { + HSHM_ALWAYS_INLINE OffsetPointerBase& operator-=(size_t count) { off_ -= count; return *this; } /** Atomic assign operator */ - inline OffsetPointerBase& operator=(size_t count) { + HSHM_ALWAYS_INLINE OffsetPointerBase& operator=(size_t count) { off_ = count; return *this; } /** Atomic copy assign operator */ - inline OffsetPointerBase& operator=(const OffsetPointerBase &count) { + HSHM_ALWAYS_INLINE OffsetPointerBase& operator=( + const OffsetPointerBase &count) { off_ = count.load(); return *this; } /** Equality check */ - bool operator==(const OffsetPointerBase &other) const { + HSHM_ALWAYS_INLINE bool operator==(const OffsetPointerBase &other) const { return off_ == other.off_; } /** Inequality check */ - bool operator!=(const OffsetPointerBase &other) const { + HSHM_ALWAYS_INLINE bool operator!=(const OffsetPointerBase &other) const { return off_ != other.off_; } }; @@ -236,51 +240,51 @@ struct PointerBase { PointerBase() = default; /** Full constructor */ - explicit PointerBase(allocator_id_t id, size_t off) : - allocator_id_(id), off_(off) {} + HSHM_ALWAYS_INLINE explicit PointerBase(allocator_id_t id, size_t off) + : allocator_id_(id), off_(off) {} /** Full constructor using offset pointer */ - explicit PointerBase(allocator_id_t id, OffsetPointer off) : - allocator_id_(id), off_(off) {} + HSHM_ALWAYS_INLINE explicit PointerBase(allocator_id_t id, OffsetPointer off) + : allocator_id_(id), off_(off) {} /** Copy constructor */ - PointerBase(const PointerBase &other) + HSHM_ALWAYS_INLINE PointerBase(const PointerBase &other) : allocator_id_(other.allocator_id_), off_(other.off_) {} /** Other copy constructor */ - PointerBase(const PointerBase &other) + HSHM_ALWAYS_INLINE PointerBase(const PointerBase &other) : allocator_id_(other.allocator_id_), off_(other.off_.load()) {} /** Move constructor */ - PointerBase(PointerBase &&other) noexcept + HSHM_ALWAYS_INLINE PointerBase(PointerBase &&other) noexcept : allocator_id_(other.allocator_id_), off_(other.off_) { other.SetNull(); } /** Get the offset pointer */ - OffsetPointerBase ToOffsetPointer() const { + HSHM_ALWAYS_INLINE OffsetPointerBase ToOffsetPointer() const { return OffsetPointerBase(off_.load()); } /** Set to null */ - void SetNull() { + HSHM_ALWAYS_INLINE void SetNull() { allocator_id_.SetNull(); } /** Check if null */ - bool IsNull() const { + HSHM_ALWAYS_INLINE bool IsNull() const { return allocator_id_.IsNull(); } /** Get the null pointer */ - static PointerBase GetNull() { + HSHM_ALWAYS_INLINE static PointerBase GetNull() { static const PointerBase p(allocator_id_t::GetNull(), OffsetPointer::GetNull()); return p; } /** Copy assignment operator */ - PointerBase& operator=(const PointerBase &other) { + HSHM_ALWAYS_INLINE PointerBase& operator=(const PointerBase &other) { if (this != &other) { allocator_id_ = other.allocator_id_; off_ = other.off_; @@ -289,7 +293,7 @@ struct PointerBase { } /** Move assignment operator */ - PointerBase& operator=(PointerBase &&other) { + HSHM_ALWAYS_INLINE PointerBase& operator=(PointerBase &&other) { if (this != &other) { allocator_id_ = other.allocator_id_; off_.exchange(other.off_.load()); @@ -299,7 +303,7 @@ struct PointerBase { } /** Addition operator */ - PointerBase operator+(size_t size) const { + HSHM_ALWAYS_INLINE PointerBase operator+(size_t size) const { PointerBase p; p.allocator_id_ = allocator_id_; p.off_ = off_ + size; @@ -307,7 +311,7 @@ struct PointerBase { } /** Subtraction operator */ - PointerBase operator-(size_t size) const { + HSHM_ALWAYS_INLINE PointerBase operator-(size_t size) const { PointerBase p; p.allocator_id_ = allocator_id_; p.off_ = off_ - size; @@ -315,24 +319,24 @@ struct PointerBase { } /** Addition assignment operator */ - PointerBase& operator+=(size_t size) { + HSHM_ALWAYS_INLINE PointerBase& operator+=(size_t size) { off_ += size; return *this; } /** Subtraction assignment operator */ - PointerBase& operator-=(size_t size) { + HSHM_ALWAYS_INLINE PointerBase& operator-=(size_t size) { off_ -= size; return *this; } /** Equality check */ - bool operator==(const PointerBase &other) const { + HSHM_ALWAYS_INLINE bool operator==(const PointerBase &other) const { return (other.allocator_id_ == allocator_id_ && other.off_ == off_); } /** Inequality check */ - bool operator!=(const PointerBase &other) const { + HSHM_ALWAYS_INLINE bool operator!=(const PointerBase &other) const { return (other.allocator_id_ != allocator_id_ || other.off_ != off_); } }; @@ -351,23 +355,35 @@ using TypedPointer = Pointer; template using TypedAtomicPointer = AtomicPointer; -/** Round up to the nearest multiple of the alignment */ -static inline size_t NextAlignmentMultiple(size_t alignment, size_t size) { - auto page_size = HERMES_SYSTEM_INFO->page_size_; - size_t new_size = size; - size_t page_off = size % alignment; - if (page_off) { - new_size = size + page_size - page_off; +class MemoryAlignment { + public: + /** + * Round up to the nearest multiple of the alignment + * @param alignment the alignment value (e.g., 4096) + * @param size the size to make a multiple of alignment (e.g., 4097) + * @return the new size (e.g., 8192) + * */ + HSHM_ALWAYS_INLINE static size_t AlignTo(size_t alignment, + size_t size) { + auto page_size = HERMES_SYSTEM_INFO->page_size_; + size_t new_size = size; + size_t page_off = size % alignment; + if (page_off) { + new_size = size + page_size - page_off; + } + return new_size; } - return new_size; -} -/** Round up to the nearest multiple of page size */ -static inline size_t NextPageSizeMultiple(size_t size) { - auto page_size = HERMES_SYSTEM_INFO->page_size_; - size_t new_size = NextAlignmentMultiple(page_size, size); - return new_size; -} + /** + * Round up to the nearest multiple of page size + * @param size the size to align to the PAGE_SIZE + * */ + HSHM_ALWAYS_INLINE static size_t AlignToPageSize(size_t size) { + auto page_size = HERMES_SYSTEM_INFO->page_size_; + size_t new_size = AlignTo(page_size, size); + return new_size; + } +}; } // namespace hshm::ipc @@ -376,7 +392,8 @@ namespace std { /** Allocator ID hash */ template <> struct hash { - std::size_t operator()(const hshm::ipc::allocator_id_t &key) const { + HSHM_ALWAYS_INLINE std::size_t operator()( + const hshm::ipc::allocator_id_t &key) const { return std::hash{}(key.int_); } }; diff --git a/hermes_shm/include/hermes_shm/memory/memory_manager.h b/hermes_shm/include/hermes_shm/memory/memory_manager.h index 229059063..eb412358c 100644 --- a/hermes_shm/include/hermes_shm/memory/memory_manager.h +++ b/hermes_shm/include/hermes_shm/memory/memory_manager.h @@ -17,6 +17,7 @@ #include "hermes_shm/memory/allocator/allocator_factory.h" #include "hermes_shm/memory/memory_registry.h" #include "hermes_shm/constants/macros.h" +#include "hermes_shm/util/logging.h" #include namespace hipc = hshm::ipc; @@ -165,7 +166,7 @@ class MemoryManager { * Convert a process-independent pointer into a process-specific pointer. * */ template - T* Convert(const POINTER_T &p) { + HSHM_ALWAYS_INLINE T* Convert(const POINTER_T &p) { if (p.IsNull()) { return nullptr; } @@ -180,7 +181,7 @@ class MemoryManager { * @param ptr the pointer to convert * */ template - POINTER_T Convert(allocator_id_t allocator_id, T *ptr) { + HSHM_ALWAYS_INLINE POINTER_T Convert(allocator_id_t allocator_id, T *ptr) { return GetAllocator(allocator_id)->template Convert(ptr); } @@ -192,7 +193,7 @@ class MemoryManager { * @param ptr the pointer to convert * */ template - POINTER_T Convert(T *ptr) { + HSHM_ALWAYS_INLINE POINTER_T Convert(T *ptr) { for (auto &alloc : HERMES_MEMORY_REGISTRY_REF.allocators_) { if (alloc && alloc->ContainsPtr(ptr)) { return alloc->template diff --git a/hermes_shm/include/hermes_shm/memory/memory_registry.h b/hermes_shm/include/hermes_shm/memory/memory_registry.h index bbb142735..9ff144cb9 100644 --- a/hermes_shm/include/hermes_shm/memory/memory_registry.h +++ b/hermes_shm/include/hermes_shm/memory/memory_registry.h @@ -17,6 +17,7 @@ #include "backend/memory_backend.h" #include "hermes_shm/memory/allocator/stack_allocator.h" #include "hermes_shm/memory/backend/posix_mmap.h" +#include "hermes_shm/util/errors.h" namespace hipc = hshm::ipc; @@ -42,26 +43,34 @@ class MemoryRegistry { * */ MemoryRegistry(); - /** Register a memory backend */ - MemoryBackend* RegisterBackend(const std::string &url, - std::unique_ptr &backend) { + /** + * Register a unique memory backend. Throws an exception if the backend + * already exists. This is because unregistering a backend can cause + * ramifications across allocators. + * + * @param url the backend's unique identifier + * @param backend the backend to register + * */ + HSHM_ALWAYS_INLINE MemoryBackend* RegisterBackend( + const std::string &url, + std::unique_ptr &backend) { auto ptr = backend.get(); - if (backends_.find(url) != backends_.end()) { - backends_.erase(url); + if (GetBackend(url)) { + throw MEMORY_BACKEND_REPEATED.format(); } backends_.emplace(url, std::move(backend)); return ptr; } /** Unregister memory backend */ - void UnregisterBackend(const std::string &url) { + HSHM_ALWAYS_INLINE void UnregisterBackend(const std::string &url) { backends_.erase(url); } /** * Returns a pointer to a backend that has already been attached. * */ - MemoryBackend* GetBackend(const std::string &url) { + HSHM_ALWAYS_INLINE MemoryBackend* GetBackend(const std::string &url) { auto iter = backends_.find(url); if (iter == backends_.end()) { return nullptr; @@ -70,26 +79,28 @@ class MemoryRegistry { } /** Registers an allocator. */ - Allocator* RegisterAllocator(std::unique_ptr &alloc) { + HSHM_ALWAYS_INLINE Allocator* RegisterAllocator( + std::unique_ptr &alloc) { if (default_allocator_ == nullptr || - default_allocator_ == &root_allocator_ || - default_allocator_->GetId() == alloc->GetId()) { + default_allocator_ == &root_allocator_ || + default_allocator_->GetId() == alloc->GetId()) { default_allocator_ = alloc.get(); } + RegisterAllocator(alloc.get()); auto idx = alloc->GetId().ToIndex(); - allocators_made_[idx] = std::move(alloc); - allocators_[idx] = allocators_made_[idx].get(); - return allocators_[idx]; + auto &alloc_made = allocators_made_[idx]; + alloc_made = std::move(alloc); + return alloc_made.get(); } /** Registers an allocator. */ - void RegisterAllocator(Allocator *alloc) { + HSHM_ALWAYS_INLINE void RegisterAllocator(Allocator *alloc) { auto idx = alloc->GetId().ToIndex(); allocators_[idx] = alloc; } /** Unregisters an allocator */ - void UnregisterAllocator(allocator_id_t alloc_id) { + HSHM_ALWAYS_INLINE void UnregisterAllocator(allocator_id_t alloc_id) { if (alloc_id == default_allocator_->GetId()) { default_allocator_ = &root_allocator_; } diff --git a/hermes_shm/include/hermes_shm/thread/ipc_call_manager.h b/hermes_shm/include/hermes_shm/thread/ipc_call_manager.h deleted file mode 100644 index 5072b9b4a..000000000 --- a/hermes_shm/include/hermes_shm/thread/ipc_call_manager.h +++ /dev/null @@ -1,133 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_THREAD_IPCALL_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_THREAD_IPCALL_H_ - -#include -#include -#include -#include -#include - -namespace hshm::ipc { - -/** Unique id of an IPC function to call */ -typedef uint32_t ipc_call_id_t; - -/** Used for asynchronous polling of IPC call */ -struct IpcFuture { - bitfield32_t *completion_; - - /** Default constructor */ - IpcFuture() = default; - - /** Emplace constructor */ - explicit IpcFuture(char *ptr) { - completion_ = reinterpret_cast(ptr); - } - - bool IsComplete() { - return (*completion_).Any(1); - } -}; - -/** - * Used for single-node, inter-process procedure calls. - * Calls are divided into two groups: readers and writers. - * - * Read calls will occur in the thread that called them since reads can - * progress in parallel. - * - * Write calls will occur in an asynchronous thread. - * - * This implementation requires that all arguments to the IPC function - * can be stored in shared memory. - * */ -class IpcCallManager { - private: - std::vector>> ipcs_; - hipc::Allocator *alloc_; - hipc::Ref> queue_; - - public: - /** Register an IPC to be called server-side */ - template - void RegisterIpc(ipc_call_id_t id, FUNC &&lambda) { - if (id >= ipcs_.size()) { - size_t new_size = (RealNumber(5, 4) * (id + 16)).as_int(); - ipcs_.resize(new_size); - } - ipcs_[id] = lambda; - } - - /** Call a function asynchronously */ - template - IpcFuture AsyncCall(ipc_call_id_t id, Args&& ...args) { - bitfield32_t completion; - - // Get size of serialized message - size_t size = 0; - auto lambda1 = [&size](auto i, auto &&arg) { - if constexpr(std::is_pod()) { - size += sizeof(arg); - } else if constexpr (IS_SHM_ARCHIVEABLE(decltype(arg))) { - size += sizeof(hipc::OffsetPointer); - } else { - throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); - } - }; - hshm::ForwardIterateArgpack::Apply( - make_argpack(completion, id, std::forward(args)...), - lambda1); - - // Allocate SHM message - hipc::OffsetPointer p; - char *ptr = alloc_->AllocatePtr(size, p); - - // Serialize id + args into a buffer - auto lambda2 = [&ptr](auto i, auto &&arg) { - if constexpr(std::is_pod()) { - *reinterpret_cast(ptr) = arg; - ptr += sizeof(arg); - } else if constexpr (IS_SHM_ARCHIVEABLE(decltype(arg))) { - arg >> *reinterpret_cast(ptr); - ptr += sizeof(hipc::OffsetPointer); - } else { - throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); - } - }; - hshm::ForwardIterateArgpack::Apply( - make_argpack(completion, id, std::forward(args)...), - lambda2); - - // Submit request into a queue - queue_->emplace(ptr); - - // Send completion event - IpcFuture future(ptr); - return future; - } - - /** Process this read event if possible */ - template - void ProcessRead() { - } - - /** Process an event in the queue */ - void ProcessWrite() { - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_THREAD_IPCALL_H_ diff --git a/hermes_shm/include/hermes_shm/thread/lock/mutex.h b/hermes_shm/include/hermes_shm/thread/lock/mutex.h index 54834d28b..c2a97fcdf 100644 --- a/hermes_shm/include/hermes_shm/thread/lock/mutex.h +++ b/hermes_shm/include/hermes_shm/thread/lock/mutex.h @@ -15,6 +15,8 @@ #define HERMES_THREAD_MUTEX_H_ #include +#include "hermes_shm/thread/lock.h" +#include "hermes_shm/thread/thread_model_manager.h" namespace hshm { @@ -25,21 +27,46 @@ struct Mutex { #endif /** Default constructor */ - Mutex() : lock_(0) {} + HSHM_ALWAYS_INLINE Mutex() : lock_(0) {} /** Explicit initialization */ - void Init() { + HSHM_ALWAYS_INLINE void Init() { lock_ = 0; } /** Acquire lock */ - void Lock(uint32_t owner); + HSHM_ALWAYS_INLINE void Lock(uint32_t owner) { + do { + for (int i = 0; i < 1; ++i) { + if (TryLock(owner)) { return; } + } + HERMES_THREAD_MODEL->Yield(); + } while (true); + } /** Try to acquire the lock */ - bool TryLock(uint32_t owner); + HSHM_ALWAYS_INLINE bool TryLock(uint32_t owner) { + if (lock_.load() != 0) { + return false; + } + uint32_t tkt = lock_.fetch_add(1); + if (tkt != 0) { + lock_.fetch_sub(1); + return false; + } +#ifdef HERMES_DEBUG_LOCK + owner_ = owner; +#endif + return true; + } /** Unlock */ - void Unlock(); + HSHM_ALWAYS_INLINE void Unlock() { +#ifdef HERMES_DEBUG_LOCK + owner_ = 0; +#endif + lock_.fetch_sub(1); + } }; struct ScopedMutex { @@ -47,19 +74,39 @@ struct ScopedMutex { bool is_locked_; /** Acquire the mutex */ - explicit ScopedMutex(Mutex &lock, uint32_t owner); + HSHM_ALWAYS_INLINE explicit ScopedMutex(Mutex &lock, uint32_t owner) + : lock_(lock), is_locked_(false) { + Lock(owner); + } /** Release the mutex */ - ~ScopedMutex(); + HSHM_ALWAYS_INLINE ~ScopedMutex() { + Unlock(); + } /** Explicitly acquire the mutex */ - void Lock(uint32_t owner); + HSHM_ALWAYS_INLINE void Lock(uint32_t owner) { + if (!is_locked_) { + lock_.Lock(owner); + is_locked_ = true; + } + } /** Explicitly try to lock the mutex */ - bool TryLock(uint32_t owner); + HSHM_ALWAYS_INLINE bool TryLock(uint32_t owner) { + if (!is_locked_) { + is_locked_ = lock_.TryLock(owner); + } + return is_locked_; + } /** Explicitly unlock the mutex */ - void Unlock(); + HSHM_ALWAYS_INLINE void Unlock() { + if (is_locked_) { + lock_.Unlock(); + is_locked_ = false; + } + } }; } // namespace hshm diff --git a/hermes_shm/include/hermes_shm/thread/lock/rwlock.h b/hermes_shm/include/hermes_shm/thread/lock/rwlock.h index 934dbe77a..f01c18b0c 100644 --- a/hermes_shm/include/hermes_shm/thread/lock/rwlock.h +++ b/hermes_shm/include/hermes_shm/thread/lock/rwlock.h @@ -15,54 +15,44 @@ #define HERMES_THREAD_RWLOCK_H_ #include +#include +#include "hermes_shm/thread/lock.h" +#include "hermes_shm/thread/thread_model_manager.h" namespace hshm { -/** The information stored by RwLock */ -union RwLockPayload { - struct { - uint32_t r_; - uint32_t w_; - } bits_; - uint64_t as_int_; - - /** Default constructor */ - RwLockPayload() = default; - - /** Copy constructor */ - RwLockPayload(const RwLockPayload &other) { - as_int_ = other.as_int_; - } - - /** Copy constructor. From uint64_t. */ - explicit RwLockPayload(uint64_t other) { - as_int_ = other; - } - - /** Check if write locked */ - bool IsWriteLocked() const { - return bits_.w_ > 0; - } - - /** Check if read locked */ - bool IsReadLocked() const { - return bits_.r_ > 0; - } +enum class RwLockMode { + kNone, + kWrite, + kRead, }; /** A reader-writer lock implementation */ struct RwLock { - std::atomic payload_; + std::atomic readers_; + std::atomic writers_; + std::atomic ticket_; + std::atomic mode_; + std::atomic cur_writer_; #ifdef HERMES_DEBUG_LOCK uint32_t owner_; #endif /** Default constructor */ - RwLock() : payload_(0) {} + RwLock() + : readers_(0), + writers_(0), + ticket_(0), + mode_(RwLockMode::kNone), + cur_writer_(0) {} /** Explicit constructor */ void Init() { - payload_ = 0; + readers_ = 0; + writers_ = 0; + ticket_ = 0; + mode_ = RwLockMode::kNone; + cur_writer_ = 0; } /** Delete copy constructor */ @@ -70,25 +60,103 @@ struct RwLock { /** Move constructor */ RwLock(RwLock &&other) noexcept - : payload_(other.payload_.load()) {} + : readers_(other.readers_.load()), + writers_(other.writers_.load()), + ticket_(other.ticket_.load()), + mode_(other.mode_.load()), + cur_writer_(other.cur_writer_.load()) {} /** Move assignment operator */ RwLock& operator=(RwLock &&other) noexcept { - payload_ = other.payload_.load(); - return (*this); + if (this != &other) { + readers_ = other.readers_.load(); + writers_ = other.writers_.load(); + ticket_ = other.ticket_.load(); + mode_ = other.mode_.load(); + cur_writer_ = other.cur_writer_.load(); + } + return *this; } /** Acquire read lock */ - void ReadLock(uint32_t owner); + void ReadLock(uint32_t owner) { + RwLockMode mode; + + // Increment # readers. Check if in read mode. + readers_.fetch_add(1); + + // Wait until we are in read mode + do { + UpdateMode(mode); + if (mode == RwLockMode::kRead) { + return; + } + if (mode == RwLockMode::kNone) { + bool ret = mode_.compare_exchange_weak(mode, RwLockMode::kRead); + if (ret) { +#ifdef HERMES_DEBUG_LOCK + owner_ = owner; + HILOG(kDebug, "Acquired read lock for {}", owner); +#endif + return; + } + } + HERMES_THREAD_MODEL->Yield(); + } while (true); + } /** Release read lock */ - void ReadUnlock(); + void ReadUnlock() { + readers_.fetch_sub(1); + } /** Acquire write lock */ - void WriteLock(uint32_t owner); + void WriteLock(uint32_t owner) { + RwLockMode mode; + uint32_t cur_writer; + + // Increment # writers & get ticket + writers_.fetch_add(1); + uint64_t tkt = ticket_.fetch_add(1); + + // Wait until we are in read mode + do { + UpdateMode(mode); + if (mode == RwLockMode::kNone) { + mode_.compare_exchange_weak(mode, RwLockMode::kWrite); + mode = mode_.load(); + } + if (mode == RwLockMode::kWrite) { + cur_writer = cur_writer_.load(); + if (cur_writer == tkt) { +#ifdef HERMES_DEBUG_LOCK + owner_ = owner; + HILOG(kDebug, "Acquired write lock for {}", owner); +#endif + return; + } + } + HERMES_THREAD_MODEL->Yield(); + } while (true); + } /** Release write lock */ - void WriteUnlock(); + void WriteUnlock() { + writers_.fetch_sub(1); + cur_writer_.fetch_add(1); + } + + private: + /** Update the mode of the lock */ + HSHM_ALWAYS_INLINE void UpdateMode(RwLockMode &mode) { + // When # readers is 0, there is a lag to when the mode is updated + // When # writers is 0, there is a lag to when the mode is updated + mode = mode_.load(); + if ((readers_.load() == 0 && mode == RwLockMode::kRead) || + (writers_.load() == 0 && mode == RwLockMode::kWrite)) { + mode_.compare_exchange_weak(mode, RwLockMode::kNone); + } + } }; /** Acquire the read lock in a scope */ @@ -97,16 +165,31 @@ struct ScopedRwReadLock { bool is_locked_; /** Acquire the read lock */ - explicit ScopedRwReadLock(RwLock &lock, uint32_t owner); + explicit ScopedRwReadLock(RwLock &lock, uint32_t owner) + : lock_(lock), is_locked_(false) { + Lock(owner); + } /** Release the read lock */ - ~ScopedRwReadLock(); + ~ScopedRwReadLock() { + Unlock(); + } /** Explicitly acquire read lock */ - void Lock(uint32_t owner); + void Lock(uint32_t owner) { + if (!is_locked_) { + lock_.ReadLock(owner); + is_locked_ = true; + } + } /** Explicitly release read lock */ - void Unlock(); + void Unlock() { + if (is_locked_) { + lock_.ReadUnlock(); + is_locked_ = false; + } + } }; /** Acquire scoped write lock */ @@ -115,16 +198,31 @@ struct ScopedRwWriteLock { bool is_locked_; /** Acquire the write lock */ - explicit ScopedRwWriteLock(RwLock &lock, uint32_t owner); + explicit ScopedRwWriteLock(RwLock &lock, uint32_t owner) + : lock_(lock), is_locked_(false) { + Lock(owner); + } /** Release the write lock */ - ~ScopedRwWriteLock(); + ~ScopedRwWriteLock() { + Unlock(); + } /** Explicity acquire the write lock */ - void Lock(uint32_t owner); + void Lock(uint32_t owner) { + if (!is_locked_) { + lock_.WriteLock(owner); + is_locked_ = true; + } + } /** Explicitly release the write lock */ - void Unlock(); + void Unlock() { + if (is_locked_) { + lock_.WriteUnlock(); + is_locked_ = false; + } + } }; } // namespace hshm diff --git a/hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h b/hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h index 56f8220b0..ff66752a3 100644 --- a/hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h +++ b/hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h @@ -14,37 +14,13 @@ #define HERMES_THREAD_THREAD_FACTORY_H_ #include "thread_model.h" -#include "pthread.h" -#include "argobots.h" -#include "hermes_shm/util/logging.h" namespace hshm::thread_model { class ThreadFactory { public: /** Get a thread instance */ - static std::unique_ptr Get(ThreadType type) { - switch (type) { - case ThreadType::kPthread: { -#ifdef HERMES_PTHREADS_ENABLED - return std::make_unique(); -#else - return nullptr; -#endif - } - case ThreadType::kArgobots: { -#ifdef HERMES_RPC_THALLIUM - return std::make_unique(); -#else - return nullptr; -#endif - } - default: { - HELOG(kWarning, "No such thread type"); - return nullptr; - } - } - } + static std::unique_ptr Get(ThreadType type); }; } // namespace hshm::thread_model diff --git a/hermes_shm/include/hermes_shm/thread/thread_model_manager.h b/hermes_shm/include/hermes_shm/thread/thread_model_manager.h index 141954b3a..abb07a7d4 100644 --- a/hermes_shm/include/hermes_shm/thread/thread_model_manager.h +++ b/hermes_shm/include/hermes_shm/thread/thread_model_manager.h @@ -16,22 +16,22 @@ #include "hermes_shm/thread/thread_model/thread_model.h" #include "hermes_shm/thread/thread_model/thread_model_factory.h" -#include #include #include -#define US_TO_CLOCKS(x) (x * 56) +#include "hermes_shm/util/singleton/_global_singleton.h" +#define HERMES_THREAD_MODEL \ + hshm::GlobalSingleton::GetInstance() +#define HERMES_THREAD_MODEL_T \ + hshm::ThreadModelManager* namespace hshm { -union NodeThreadId; - class ThreadModelManager { public: ThreadType type_; /**< The type of threads used in this program */ std::unique_ptr thread_static_; /**< Functions static to all threads */ - std::mutex lock_; /**< Synchronize */ /** Default constructor */ ThreadModelManager() { @@ -39,34 +39,16 @@ class ThreadModelManager { } /** Set the threading model of this application */ - void SetThreadModel(ThreadType type) { - lock_.lock(); - if (type_ == type) { - lock_.unlock(); - return; - } - type_ = type; - thread_static_ = thread_model::ThreadFactory::Get(type); - if (thread_static_ == nullptr) { - HELOG(kFatal, "Could not load the threading model"); - } - lock_.unlock(); - } + void SetThreadModel(ThreadType type); /** Sleep for a period of time (microseconds) */ - void SleepForUs(size_t us) { - thread_static_->SleepForUs(us); - } + void SleepForUs(size_t us); /** Call Yield */ - void Yield() { - thread_static_->Yield(); - } + void Yield(); /** Call GetTid */ - tid_t GetTid() { - return thread_static_->GetTid(); - } + tid_t GetTid(); }; /** A unique identifier of this thread across processes */ diff --git a/hermes_shm/include/hermes_shm/types/argpack.h b/hermes_shm/include/hermes_shm/types/argpack.h index 7db97462b..bd902f1a3 100644 --- a/hermes_shm/include/hermes_shm/types/argpack.h +++ b/hermes_shm/include/hermes_shm/types/argpack.h @@ -14,6 +14,7 @@ #define HERMES_INCLUDE_HERMES_TYPES_ARGPACK_H_ #include "real_number.h" +#include "hermes_shm/constants/macros.h" #include namespace hshm { @@ -38,15 +39,15 @@ struct ArgPackRecur { recur_; /**< Remaining args */ /** Default constructor */ - ArgPackRecur() = default; + HSHM_ALWAYS_INLINE ArgPackRecur() = default; /** Constructor. Rvalue reference. */ - explicit ArgPackRecur(T arg, Args&& ...args) + HSHM_ALWAYS_INLINE explicit ArgPackRecur(T arg, Args&& ...args) : arg_(std::forward(arg)), recur_(std::forward(args)...) {} /** Forward an rvalue reference (only if argpack) */ template - constexpr decltype(auto) Forward() const { + HSHM_ALWAYS_INLINE constexpr decltype(auto) Forward() const { if constexpr(i == idx) { if constexpr(is_rval) { return std::forward(arg_); @@ -64,11 +65,11 @@ struct ArgPackRecur { template struct ArgPackRecur { /** Default constructor */ - ArgPackRecur() = default; + HSHM_ALWAYS_INLINE ArgPackRecur() = default; /** Forward an rvalue reference (only if argpack) */ template - constexpr void Forward() const { + HSHM_ALWAYS_INLINE constexpr void Forward() const { throw std::logic_error("(Forward) ArgPack index outside of range"); } }; @@ -82,17 +83,17 @@ struct ArgPack { static constexpr const size_t size_ = sizeof...(Args); /** General Constructor. */ - ArgPack(Args&& ...args) // NOLINT + HSHM_ALWAYS_INLINE ArgPack(Args&& ...args) // NOLINT : recur_(std::forward(args)...) {} /** Get forward reference */ template - constexpr decltype(auto) Forward() const { + HSHM_ALWAYS_INLINE constexpr decltype(auto) Forward() const { return recur_.template Forward(); } /** Size */ - constexpr static size_t Size() { + HSHM_ALWAYS_INLINE constexpr static size_t Size() { return size_; } }; @@ -121,7 +122,8 @@ class PassArgPack { public: /** Call function with ArgPack */ template - constexpr static decltype(auto) Call(ArgPackT &&pack, F &&f) { + HSHM_ALWAYS_INLINE constexpr static decltype(auto) Call(ArgPackT &&pack, + F &&f) { return _CallRecur<0, ArgPackT, F>( std::forward(pack), std::forward(f)); } @@ -130,9 +132,10 @@ class PassArgPack { /** Unpacks the ArgPack and passes it to the function */ template - constexpr static decltype(auto) _CallRecur(ArgPackT &&pack, - F &&f, - CurArgs&& ...args) { + HSHM_ALWAYS_INLINE constexpr static decltype(auto) + _CallRecur(ArgPackT &&pack, + F &&f, + CurArgs&& ...args) { if constexpr(i < ArgPackT::Size()) { return _CallRecur( std::forward(pack), @@ -150,16 +153,16 @@ class MergeArgPacks { public: /** Call function with ArgPack */ template - constexpr static decltype(auto) Merge(ArgPacks&& ...packs) { + HSHM_ALWAYS_INLINE constexpr static decltype(auto) + Merge(ArgPacks&& ...packs) { return _MergePacksRecur<0>(make_argpack(std::forward(packs)...)); } private: /** Unpacks the C++ parameter pack of ArgPacks */ - template - constexpr static decltype(auto) _MergePacksRecur(ArgPacksT &&packs, - CurArgs&& ...args) { + template + HSHM_ALWAYS_INLINE constexpr static decltype(auto) _MergePacksRecur( + ArgPacksT &&packs, CurArgs&& ...args) { if constexpr(cur_pack < ArgPacksT::Size()) { return _MergeRecur< cur_pack, ArgPacksT, 0>( @@ -177,9 +180,8 @@ class MergeArgPacks { size_t cur_pack, typename ArgPacksT, size_t i, typename ArgPackT, typename ...CurArgs> - constexpr static decltype(auto) _MergeRecur(ArgPacksT &&packs, - ArgPackT &&pack, - CurArgs&& ...args) { + HSHM_ALWAYS_INLINE constexpr static decltype(auto) _MergeRecur( + ArgPacksT &&packs, ArgPackT &&pack, CurArgs&& ...args) { if constexpr(i < ArgPackT::Size()) { return _MergeRecur( std::forward(packs), @@ -198,8 +200,8 @@ class ProductArgPacks { public: /** The product function */ template - constexpr static decltype(auto) Product(ProductPackT &&prod_pack, - ArgPacks&& ...packs) { + HSHM_ALWAYS_INLINE constexpr static decltype(auto) Product( + ProductPackT &&prod_pack, ArgPacks&& ...packs) { return _ProductPacksRecur<0>( std::forward(prod_pack), make_argpack(std::forward(packs)...)); @@ -212,9 +214,10 @@ class ProductArgPacks { typename ProductPackT, typename OrigPacksT, typename ...NewPacks> - constexpr static decltype(auto) _ProductPacksRecur(ProductPackT &&prod_pack, - OrigPacksT &&orig_packs, - NewPacks&& ...packs) { + HSHM_ALWAYS_INLINE constexpr static decltype(auto) _ProductPacksRecur( + ProductPackT &&prod_pack, + OrigPacksT &&orig_packs, + NewPacks&& ...packs) { if constexpr(cur_pack < OrigPacksT::Size()) { return _ProductPacksRecur( std::forward(prod_pack), @@ -232,7 +235,7 @@ class ProductArgPacks { template struct MakeConstexpr { constexpr static T val_ = Val; - constexpr static T Get() { + HSHM_ALWAYS_INLINE constexpr static T Get() { return val_; } }; @@ -243,14 +246,14 @@ class IterateArgpack { public: /** Apply a function to every element of a tuple */ template - constexpr static void Apply(TupleT &&pack, F &&f) { + HSHM_ALWAYS_INLINE constexpr static void Apply(TupleT &&pack, F &&f) { _Apply<0, TupleT, F>(std::forward(pack), std::forward(f)); } private: /** Apply the function recursively */ template - constexpr static void _Apply(TupleT &&pack, F &&f) { + HSHM_ALWAYS_INLINE constexpr static void _Apply(TupleT &&pack, F &&f) { if constexpr(i < TupleT::Size()) { if constexpr(reverse) { _Apply(std::forward(pack), diff --git a/hermes_shm/include/hermes_shm/types/bitfield.h b/hermes_shm/include/hermes_shm/types/bitfield.h index afea1138f..13a987444 100644 --- a/hermes_shm/include/hermes_shm/types/bitfield.h +++ b/hermes_shm/include/hermes_shm/types/bitfield.h @@ -14,6 +14,7 @@ #define HERMES_INCLUDE_HERMES_TYPES_BITFIELD_H_ #include +#include namespace hshm { @@ -27,35 +28,35 @@ template struct bitfield { T bits_; - bitfield() : bits_(0) {} + HSHM_ALWAYS_INLINE bitfield() : bits_(0) {} - explicit bitfield(T mask) : bits_(mask) {} + HSHM_ALWAYS_INLINE explicit bitfield(T mask) : bits_(mask) {} - inline void SetBits(T mask) { + HSHM_ALWAYS_INLINE void SetBits(T mask) { bits_ |= mask; } - inline void UnsetBits(T mask) { + HSHM_ALWAYS_INLINE void UnsetBits(T mask) { bits_ &= ~mask; } - inline bool Any(T mask) const { + HSHM_ALWAYS_INLINE bool Any(T mask) const { return bits_ & mask; } - inline bool All(T mask) const { + HSHM_ALWAYS_INLINE bool All(T mask) const { return Any(mask) == mask; } - inline void CopyBits(bitfield field, T mask) { + HSHM_ALWAYS_INLINE void CopyBits(bitfield field, T mask) { bits_ &= (field.bits_ & mask); } - inline void Clear() { + HSHM_ALWAYS_INLINE void Clear() { bits_ = 0; } - static T MakeMask(int start, int length) { + HSHM_ALWAYS_INLINE static T MakeMask(int start, int length) { return ((((T)1) << length) - 1) << start; } } __attribute__((packed)); @@ -82,13 +83,13 @@ template -#include "hermes_shm/thread/lock/mutex.h" -#include "hermes_shm/constants/macros.h" - -namespace hshm { - -/** - * Makes a singleton. Constructs the first time GetInstance is called. - * Requires user to define the static storage of obj_ in separate file. - * @tparam T - */ -template -class Singleton { - private: - static T *obj_; - static hshm::Mutex *lock_; - - public: - Singleton() = default; - - /** Get or create an instance of type T */ - inline static T *GetInstance() { - if (!obj_) { - hshm::ScopedMutex lock(*lock_, 0); - if (obj_ == nullptr) { - obj_ = new T(); - } - } - return obj_; - } - - /** Static initialization method for obj */ - static T *_GetObj(); - - /** Static initialization method for lock */ - static hshm::Mutex *_GetLock(); -}; -template -T* Singleton::obj_ = Singleton::_GetObj(); -template -hshm::Mutex* Singleton::lock_ = Singleton::_GetLock(); -#define DEFINE_SINGLETON_CC(T)\ - template<> T* hshm::Singleton::_GetObj() {\ - return nullptr;\ - }\ - template<> hshm::Mutex* hshm::Singleton::_GetLock() {\ - static hshm::Mutex lock;\ - return &lock;\ - } - -/** - * Makes a singleton. Constructs during initialization of program. - * Requires user to define the static storage of obj_ in separate file. - * */ -template -class GlobalSingleton { - public: - static T *obj_; - - public: - GlobalSingleton() = default; - - /** Get instance of type T */ - HSHM_ALWAYS_INLINE static T* GetInstance() { - return obj_; - } - - /** Get ref of type T */ - HSHM_ALWAYS_INLINE static T& GetRef() { - return *obj_; - } - - /** Static initialization method for obj */ - static T& _GetObj(); -}; -template -T* GlobalSingleton::obj_ = &GlobalSingleton::_GetObj(); -#define DEFINE_GLOBAL_SINGLETON_CC(T)\ - template<> T& hshm::GlobalSingleton::_GetObj() {\ - static T obj; \ - return obj;\ - } - -/** - * A class to represent singleton pattern - * Does not require specific initialization of the static variable - * */ -template -class EasySingleton { - protected: - /** static instance. */ - static T* obj_; - static hshm::Mutex lock_; - - public: - /** - * Uses unique pointer to build a static global instance of variable. - * @tparam T - * @return instance of T - */ - template - static T* GetInstance(Args&& ...args) { - if (obj_ == nullptr) { - hshm::ScopedMutex lock(lock_, 0); - if (obj_ == nullptr) { - obj_ = new T(std::forward(args)...); - } - } - return obj_; - } -}; -template -T* EasySingleton::obj_ = nullptr; -template -hshm::Mutex EasySingleton::lock_ = hshm::Mutex(); - -/** - * Makes a singleton. Constructs during initialization of program. - * Does not require specific initialization of the static variable. - * */ -template -class EasyGlobalSingleton { - private: - static T obj_; - public: - EasyGlobalSingleton() = default; - static T* GetInstance() { - return &obj_; - } -}; -template -T EasyGlobalSingleton::obj_; - -} // namespace hshm +#include "singleton/_easy_global_singleton.h" +#include "singleton/_easy_singleton.h" +#include "singleton/_global_singleton.h" +#include "singleton/_singleton.h" #endif // HERMES_SHM_SINGLETON_H diff --git a/hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h new file mode 100644 index 000000000..b396c5067 --- /dev/null +++ b/hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h @@ -0,0 +1,40 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_GLOBAL_SINGLETON_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_GLOBAL_SINGLETON_H_ + +#include +#include "hermes_shm/constants/macros.h" + +namespace hshm { + +/** + * Makes a singleton. Constructs during initialization of program. + * Does not require specific initialization of the static variable. + * */ +template +class EasyGlobalSingleton { + private: + static T obj_; + public: + EasyGlobalSingleton() = default; + static T* GetInstance() { + return &obj_; + } +}; +template +T EasyGlobalSingleton::obj_; + +} // namespace hshm + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_GLOBAL_SINGLETON_H_ diff --git a/hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h new file mode 100644 index 000000000..b896fc9c0 --- /dev/null +++ b/hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h @@ -0,0 +1,57 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_SINGLETON_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_SINGLETON_H_ + +#include +#include "hermes_shm/thread/lock/mutex.h" +#include "hermes_shm/constants/macros.h" + +namespace hshm { + +/** + * A class to represent singleton pattern + * Does not require specific initialization of the static variable + * */ +template +class EasySingleton { + protected: + /** static instance. */ + static T* obj_; + static hshm::Mutex lock_; + + public: + /** + * Uses unique pointer to build a static global instance of variable. + * @tparam T + * @return instance of T + */ + template + static T* GetInstance(Args&& ...args) { + if (obj_ == nullptr) { + hshm::ScopedMutex lock(lock_, 0); + if (obj_ == nullptr) { + obj_ = new T(std::forward(args)...); + } + } + return obj_; + } +}; +template +T* EasySingleton::obj_ = nullptr; +template +hshm::Mutex EasySingleton::lock_ = hshm::Mutex(); + +} // namespace hshm + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_SINGLETON_H_ diff --git a/hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h new file mode 100644 index 000000000..e3be7ec01 --- /dev/null +++ b/hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h @@ -0,0 +1,56 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON__GLOBAL_SINGLETON_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON__GLOBAL_SINGLETON_H_ + +#include +#include "hermes_shm/constants/macros.h" + +namespace hshm { + +/** + * Makes a singleton. Constructs during initialization of program. + * Requires user to define the static storage of obj_ in separate file. + * */ +template +class GlobalSingleton { + public: + static T *obj_; + + public: + GlobalSingleton() = default; + + /** Get instance of type T */ + HSHM_ALWAYS_INLINE static T* GetInstance() { + return obj_; + } + + /** Get ref of type T */ + HSHM_ALWAYS_INLINE static T& GetRef() { + return *obj_; + } + + /** Static initialization method for obj */ + static T& _GetObj(); +}; +template +T* GlobalSingleton::obj_ = &GlobalSingleton::_GetObj(); +#define DEFINE_GLOBAL_SINGLETON_CC(T)\ + template<> T& hshm::GlobalSingleton::_GetObj() {\ + static T obj; \ + return obj;\ + } + +} // namespace hshm + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON__GLOBAL_SINGLETON_H_ diff --git a/hermes_shm/include/hermes_shm/util/singleton/_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_singleton.h new file mode 100644 index 000000000..d7977eea9 --- /dev/null +++ b/hermes_shm/include/hermes_shm/util/singleton/_singleton.h @@ -0,0 +1,69 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_SINGLETON_H_ +#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_SINGLETON_H_ + +#include +#include "hermes_shm/thread/lock/mutex.h" +#include "hermes_shm/constants/macros.h" + +namespace hshm { + +/** + * Makes a singleton. Constructs the first time GetInstance is called. + * Requires user to define the static storage of obj_ in separate file. + * @tparam T + */ +template +class Singleton { + private: + static T *obj_; + static hshm::Mutex *lock_; + + public: + Singleton() = default; + + /** Get or create an instance of type T */ + inline static T *GetInstance() { + if (!obj_) { + hshm::ScopedMutex lock(*lock_, 0); + if (obj_ == nullptr) { + obj_ = new T(); + } + } + return obj_; + } + + /** Static initialization method for obj */ + static T *_GetObj(); + + /** Static initialization method for lock */ + static hshm::Mutex *_GetLock(); +}; + +template +T* Singleton::obj_ = Singleton::_GetObj(); +template +hshm::Mutex* Singleton::lock_ = Singleton::_GetLock(); +#define DEFINE_SINGLETON_CC(T)\ + template<> T* hshm::Singleton::_GetObj() {\ + return nullptr;\ + }\ + template<> hshm::Mutex* hshm::Singleton::_GetLock() {\ + static hshm::Mutex lock;\ + return &lock;\ + } + +} // namespace hshm + +#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_SINGLETON_H_ diff --git a/hermes_shm/readme.md b/hermes_shm/readme.md index a16457d5c..8f65871a8 100644 --- a/hermes_shm/readme.md +++ b/hermes_shm/readme.md @@ -3,3 +3,5 @@ This library contains a variety of data structures and synchronization primitives which are compatible with shared memory. + +[![Coverage Status](https://coveralls.io/repos/github/lukemartinlogan/hermes_shm/badge.svg?branch=master)](https://coveralls.io/github/lukemartinlogan/hermes_shm?branch=master) \ No newline at end of file diff --git a/hermes_shm/scripts/ci/coverage.sh b/hermes_shm/scripts/ci/coverage.sh new file mode 100644 index 000000000..59e8dbd01 --- /dev/null +++ b/hermes_shm/scripts/ci/coverage.sh @@ -0,0 +1,22 @@ +#!/bin/bash +COVERAGE_DIR="${GITHUB_WORKSPACE}/coverage" +mkdir -p "${COVERAGE_DIR}" +cd "${GITHUB_WORKSPACE}/build" +lcov -c -d . -o "${COVERAGE_DIR}/tmp.info" +lcov --remove "${COVERAGE_DIR}/tmp.info" \ + "/usr/*" \ + "*/spack/*" \ + -o "${COVERAGE_DIR}/lcov.info" +genhtml "${COVERAGE_DIR}/lcov.info" --output-directory coverage_report + +echo "Placed coverage info in ${COVERAGE_DIR}/lcov.info" +cd ${COVERAGE_DIR} +`pwd` +ls + +#lcov -c -d . -o "tmp.info" +#lcov --remove "tmp.info" \ +# "/usr/*" \ +# "*/spack/*" \ +# -o "lcov.info" +#genhtml "lcov.info" --output-directory coverage_report diff --git a/hermes_shm/scripts/ci/external/CMakeLists.txt b/hermes_shm/scripts/ci/external/CMakeLists.txt new file mode 100644 index 000000000..32d44c9a3 --- /dev/null +++ b/hermes_shm/scripts/ci/external/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.10) +project(hermes_shm) + +set(CMAKE_CXX_STANDARD 17) + +#------------------------------------------------------------------------------ +# External libraries +#------------------------------------------------------------------------------ + +find_package(HermesShm CONFIG REQUIRED) +message(STATUS "found hermes_shm at ${HermesShm_DIR}") + +#------------------------------------------------------------------------------ +# Build +#------------------------------------------------------------------------------ + +add_executable(test test.cc) +target_link_libraries(test ${HermesShm_LIBRARIES}) \ No newline at end of file diff --git a/hermes_shm/scripts/ci/external/test.cc b/hermes_shm/scripts/ci/external/test.cc new file mode 100644 index 000000000..cbf628b8c --- /dev/null +++ b/hermes_shm/scripts/ci/external/test.cc @@ -0,0 +1,16 @@ +// +// Created by lukemartinlogan on 6/6/23. +// + +#include "hermes_shm/hermes_shm.h" + +int main() { + std::string shm_url = "test_serializers"; + hipc::allocator_id_t alloc_id(0, 1); + auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); + mem_mngr->CreateBackend( + hipc::MemoryManager::GetDefaultBackendSize(), shm_url); + mem_mngr->CreateAllocator(shm_url, alloc_id, 0); +} \ No newline at end of file diff --git a/hermes_shm/scripts/ci/install_deps.sh b/hermes_shm/scripts/ci/install_deps.sh index cd341562e..1b8b819a2 100644 --- a/hermes_shm/scripts/ci/install_deps.sh +++ b/hermes_shm/scripts/ci/install_deps.sh @@ -36,5 +36,6 @@ cd ${GITHUB_WORKSPACE} cp scripts/ci/packages.yaml ${SPACK_DIR}/etc/spack/packages.yaml # Install hermes_shm (needed for dependencies) +# spack repo add scripts/hermes_shm spack install hermes_shm diff --git a/hermes_shm/scripts/ci/install_hshm.sh b/hermes_shm/scripts/ci/install_hshm.sh index 33191e5f6..7dec11efb 100755 --- a/hermes_shm/scripts/ci/install_hshm.sh +++ b/hermes_shm/scripts/ci/install_hshm.sh @@ -14,10 +14,35 @@ SPACK_DIR=${INSTALL_DIR}/spack mkdir -p "${HOME}/install" mkdir build -cd build +pushd build spack load --only dependencies hermes_shm -cmake ../ -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=${HOME}/install -cmake --build . -- -j4 +cmake ../ \ +-DCMAKE_BUILD_TYPE=Debug \ +-DHERMES_ENABLE_COVERAGE=ON \ +-DHERMES_ENABLE_DOXYGEN=ON \ +-DBUILD_HSHM_BENCHMARKS=ON \ +-DBUILD_HSHM_TESTS=ON \ +-DCMAKE_INSTALL_PREFIX=${HOME}/install +make -j8 +make install export CXXFLAGS=-Wall ctest -VV +popd + +# Set proper flags for cmake to find Hermes +INSTALL_PREFIX="${HOME}/install" +export LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LIBRARY_PATH}" +export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}" +export LDFLAGS="-L${INSTALL_PREFIX}/lib:${LDFLAGS}" +export CFLAGS="-I${INSTALL_PREFIX}/include:${CFLAGS}" +export CPATH="${INSTALL_PREFIX}/include:${CPATH}" +export CMAKE_PREFIX_PATH="${INSTALL_PREFIX}:${CMAKE_PREFIX_PATH}" +export CXXFLAGS="-I${INSTALL_PREFIX}/include:${CXXFLAGS}" + +# Run make install unit test +cd scripts/ci/external +mkdir build +cd build +cmake ../ +make -j8 diff --git a/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py b/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py index 36fd17ec6..6beada344 100644 --- a/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py +++ b/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py @@ -7,7 +7,6 @@ class HermesShm(CMakePackage): depends_on('mochi-thallium~cereal@0.8.3') depends_on('catch2@3.0.1') depends_on('mpich@3.3.2:') - depends_on('yaml-cpp') depends_on('boost@1.7:') depends_on('doxygen@1.9.3') @@ -17,8 +16,11 @@ def cmake_args(self): def set_include(self, env, path): env.append_flags('CFLAGS', '-I{}'.format(path)) env.append_flags('CXXFLAGS', '-I{}'.format(path)) + env.prepend_path('INCLUDE', '{}'.format(path)) + env.prepend_path('CPATH', '{}'.format(path)) def set_lib(self, env, path): + env.prepend_path('LIBRARY_PATH', path) env.prepend_path('LD_LIBRARY_PATH', path) env.append_flags('LDFLAGS', '-L{}'.format(path)) diff --git a/hermes_shm/src/CMakeLists.txt b/hermes_shm/src/CMakeLists.txt index 3a1e6e0fe..987ec1d7c 100644 --- a/hermes_shm/src/CMakeLists.txt +++ b/hermes_shm/src/CMakeLists.txt @@ -6,17 +6,17 @@ set(CMAKE_CXX_STANDARD 17) #----------------------------------------------------------------------------- # Build HSHM #----------------------------------------------------------------------------- -add_library(hermes_shm_data_structures SHARED - thread/mutex.cc - thread/rwlock.cc +add_library(hermes_shm_data_structures memory/malloc_allocator.cc memory/stack_allocator.cc memory/scalable_page_allocator.cc memory/memory_registry.cc memory/memory_manager.cc + thread_model_manager.cc + thread_factory.cc data_structure_singleton.cc) target_link_libraries(hermes_shm_data_structures - yaml-cpp pthread -lrt -ldl OpenMP::OpenMP_CXX + pthread -lrt -ldl OpenMP::OpenMP_CXX $<$:thallium>) #----------------------------------------------------------------------------- @@ -51,4 +51,3 @@ endif() if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_shm_data_structures) endif() - diff --git a/hermes_shm/src/memory/malloc_allocator.cc b/hermes_shm/src/memory/malloc_allocator.cc index a5f579011..46c48554d 100644 --- a/hermes_shm/src/memory/malloc_allocator.cc +++ b/hermes_shm/src/memory/malloc_allocator.cc @@ -21,9 +21,8 @@ struct MallocPage { void MallocAllocator::shm_init(allocator_id_t id, size_t custom_header_size, - char *buffer, size_t buffer_size) { - buffer_ = buffer; + buffer_ = nullptr; buffer_size_ = buffer_size; header_ = reinterpret_cast( malloc(sizeof(MallocAllocatorHeader) + custom_header_size)); @@ -45,7 +44,7 @@ OffsetPointer MallocAllocator::AllocateOffset(size_t size) { malloc(sizeof(MallocPage) + size)); page->page_size_ = size; header_->total_alloc_size_ += size; - return OffsetPointer(size_t(page + 1)); + return OffsetPointer((size_t)(page + 1)); } OffsetPointer MallocAllocator::AlignedAllocateOffset(size_t size, diff --git a/hermes_shm/src/memory/memory_intercept.cc b/hermes_shm/src/memory/memory_intercept.cc index 8407f1382..b6a7dead6 100644 --- a/hermes_shm/src/memory/memory_intercept.cc +++ b/hermes_shm/src/memory/memory_intercept.cc @@ -70,7 +70,7 @@ void* valloc(size_t size) { * that is, round up size to nearest pagesize. * */ void* pvalloc(size_t size) { - size_t new_size = hshm::ipc::NextPageSizeMultiple(size); + size_t new_size = hipc::MemoryAlignment::AlignToPageSize(size); return valloc(new_size); } diff --git a/hermes_shm/src/memory/memory_registry.cc b/hermes_shm/src/memory/memory_registry.cc index 89f79cf54..9300217bd 100644 --- a/hermes_shm/src/memory/memory_registry.cc +++ b/hermes_shm/src/memory/memory_registry.cc @@ -23,6 +23,7 @@ MemoryRegistry::MemoryRegistry() { root_backend_.data_, root_backend_.data_size_); default_allocator_ = &root_allocator_; + memset(allocators_, 0, sizeof(allocators_)); RegisterAllocator(&root_allocator_); } diff --git a/hermes_shm/src/memory/scalable_page_allocator.cc b/hermes_shm/src/memory/scalable_page_allocator.cc index e5971781e..3c028a125 100644 --- a/hermes_shm/src/memory/scalable_page_allocator.cc +++ b/hermes_shm/src/memory/scalable_page_allocator.cc @@ -114,7 +114,6 @@ OffsetPointer ScalablePageAllocator::AllocateOffset(size_t size) { header_->total_alloc_.fetch_add(page->page_size_); auto p = Convert(page); page->SetAllocated(); - page->cpu_ = cpu; return p + sizeof(MpPage); } @@ -219,7 +218,7 @@ void ScalablePageAllocator::DividePage(FreeListStats &stats, if (num_divisions > max_divide) { num_divisions = max_divide; } for (size_t i = 0; i < num_divisions; ++i) { rem_page->page_size_ = size_mp; - rem_page->flags_ = 0; + rem_page->flags_.Clear(); rem_page->off_ = 0; stats.AddAlloc(); free_list.enqueue(rem_page); @@ -231,7 +230,7 @@ void ScalablePageAllocator::DividePage(FreeListStats &stats, // Case 3: There is still remaining space after the divisions if (rem_size > 0) { rem_page->page_size_ = rem_size; - rem_page->flags_ = 0; + rem_page->flags_.Clear(); rem_page->off_ = 0; } else { rem_page = nullptr; diff --git a/hermes_shm/src/memory/stack_allocator.cc b/hermes_shm/src/memory/stack_allocator.cc index 8a703e8c7..d6dd7d970 100644 --- a/hermes_shm/src/memory/stack_allocator.cc +++ b/hermes_shm/src/memory/stack_allocator.cc @@ -27,6 +27,7 @@ void StackAllocator::shm_init(allocator_id_t id, size_t region_off = (custom_header_ - buffer_) + custom_header_size; size_t region_size = buffer_size_ - region_off; header_->Configure(id, custom_header_size, region_off, region_size); + heap_ = &header_->heap_; } void StackAllocator::shm_deserialize(char *buffer, @@ -35,6 +36,7 @@ void StackAllocator::shm_deserialize(char *buffer, buffer_size_ = buffer_size; header_ = reinterpret_cast(buffer_); custom_header_ = reinterpret_cast(header_ + 1); + heap_ = &header_->heap_; } size_t StackAllocator::GetCurrentlyAllocatedSize() { @@ -43,15 +45,11 @@ size_t StackAllocator::GetCurrentlyAllocatedSize() { OffsetPointer StackAllocator::AllocateOffset(size_t size) { size += sizeof(MpPage); - if (header_->region_size_ < size) { - return OffsetPointer::GetNull(); - } - OffsetPointer p(header_->region_off_.fetch_add(size)); + OffsetPointer p = heap_->AllocateOffset(size); auto hdr = Convert(p); hdr->SetAllocated(); hdr->page_size_ = size; hdr->off_ = 0; - header_->region_size_.fetch_sub(hdr->page_size_); header_->total_alloc_.fetch_add(hdr->page_size_); return p + sizeof(MpPage); } diff --git a/hermes_shm/src/thread/mutex.cc b/hermes_shm/src/thread/mutex.cc deleted file mode 100644 index e61125679..000000000 --- a/hermes_shm/src/thread/mutex.cc +++ /dev/null @@ -1,113 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "hermes_shm/thread/lock.h" -#include "hermes_shm/thread/thread_model_manager.h" -#include "hermes_shm/util/logging.h" - -namespace hshm { - -/**==================================== - * Mutex - * ===================================*/ - -/** - * Acquire the mutex - * */ -void Mutex::Lock(uint32_t owner) { - do { - for (int i = 0; i < 1; ++i) { - if (TryLock(owner)) { return; } - } - HERMES_THREAD_MODEL->Yield(); - } while (true); -} - -/** - * Attempt to acquire the mutex - * */ -bool Mutex::TryLock(uint32_t owner) { - if (lock_.load() != 0) { - return false; - } - uint32_t tkt = lock_.fetch_add(1); - if (tkt != 0) { - lock_.fetch_sub(1); - return false; - } -#ifdef HERMES_DEBUG_LOCK - owner_ = owner; -#endif - return true; -} - -/** - * Release the mutex - * */ -void Mutex::Unlock() { -#ifdef HERMES_DEBUG_LOCK - owner_ = 0; -#endif - lock_.fetch_sub(1); -} - -/**==================================== - * Scoped Mutex - * ===================================*/ - -/** - * Constructor - * */ -ScopedMutex::ScopedMutex(Mutex &lock, uint32_t owner) -: lock_(lock), is_locked_(false) { - Lock(owner); -} - -/** - * Release the mutex - * */ -ScopedMutex::~ScopedMutex() { - Unlock(); -} - -/** - * Acquire the mutex - * */ -void ScopedMutex::Lock(uint32_t owner) { - if (!is_locked_) { - lock_.Lock(owner); - is_locked_ = true; - } -} - -/** - * Attempt to acquire the mutex - * */ -bool ScopedMutex::TryLock(uint32_t owner) { - if (!is_locked_) { - is_locked_ = lock_.TryLock(owner); - } - return is_locked_; -} - -/** - * Acquire the mutex - * */ -void ScopedMutex::Unlock() { - if (is_locked_) { - lock_.Unlock(); - is_locked_ = false; - } -} - -} // namespace hshm diff --git a/hermes_shm/src/thread/rwlock.cc b/hermes_shm/src/thread/rwlock.cc deleted file mode 100644 index 734a29c2e..000000000 --- a/hermes_shm/src/thread/rwlock.cc +++ /dev/null @@ -1,198 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "hermes_shm/thread/lock/rwlock.h" -#include "hermes_shm/thread/thread_model_manager.h" -#include "hermes_shm/util/logging.h" - -namespace hshm { - -/**==================================== - * Rw Lock - * ===================================*/ - -/** - * Acquire the read lock - * */ -void RwLock::ReadLock(uint32_t owner) { - bool ret = false; - RwLockPayload expected, desired; - do { - for (int i = 0; i < 1; ++i) { - expected.as_int_ = payload_.load(); - if (expected.IsWriteLocked()) { - continue; - } - desired = expected; - desired.bits_.r_ += 1; - ret = payload_.compare_exchange_weak( - expected.as_int_, - desired.as_int_); - if (ret) { -#ifdef HERMES_DEBUG_LOCK - owner_ = owner; - HILOG(kDebug, "Acquired read lock for {}", owner); -#endif - return; - } - } - HERMES_THREAD_MODEL->Yield(); - } while (true); -} - -/** - * Release the read lock - * */ -void RwLock::ReadUnlock() { - bool ret; - RwLockPayload expected, desired; -#ifdef HERMES_DEBUG_LOCK - owner_ = 0; -#endif - do { - expected.as_int_ = payload_.load(); - desired = expected; - desired.bits_.r_ -= 1; - ret = payload_.compare_exchange_weak( - expected.as_int_, - desired.as_int_); - } while (!ret); -} - -/** - * Acquire the write lock - * */ -void RwLock::WriteLock(uint32_t owner) { - bool ret = false; - RwLockPayload expected, desired; - do { - for (int i = 0; i < 1; ++i) { - expected.as_int_ = payload_.load(); - if (expected.IsReadLocked() || expected.IsWriteLocked()) { - continue; - } - desired = expected; - desired.bits_.w_ += 1; - ret = payload_.compare_exchange_weak( - expected.as_int_, - desired.as_int_); - if (ret) { -#ifdef HERMES_DEBUG_LOCK - owner_ = owner; - HILOG(kDebug, "Acquired write lock for {}", owner); -#endif - return; - } - } - HERMES_THREAD_MODEL->Yield(); - } while (true); -} - -/** - * Release the write lock - * */ -void RwLock::WriteUnlock() { - bool ret; - RwLockPayload expected, desired; -#ifdef HERMES_DEBUG_LOCK - owner_ = 0; -#endif - do { - expected.as_int_ = payload_.load(); - desired = expected; - desired.bits_.w_ -= 1; - ret = payload_.compare_exchange_weak( - expected.as_int_, - desired.as_int_); - } while (!ret); -} - -/**==================================== - * ScopedRwReadLock - * ===================================*/ - -/** - * Constructor - * */ -ScopedRwReadLock::ScopedRwReadLock(RwLock &lock, uint32_t owner) -: lock_(lock), is_locked_(false) { - Lock(owner); -} - -/** - * Release the read lock - * */ -ScopedRwReadLock::~ScopedRwReadLock() { - Unlock(); -} - -/** - * Acquire the read lock - * */ -void ScopedRwReadLock::Lock(uint32_t owner) { - if (!is_locked_) { - lock_.ReadLock(owner); - is_locked_ = true; - } -} - -/** - * Release the read lock - * */ -void ScopedRwReadLock::Unlock() { - if (is_locked_) { - lock_.ReadUnlock(); - is_locked_ = false; - } -} - -/**==================================== - * ScopedRwWriteLock - * ===================================*/ - -/** - * Constructor - * */ -ScopedRwWriteLock::ScopedRwWriteLock(RwLock &lock, uint32_t owner) -: lock_(lock), is_locked_(false) { - Lock(owner); -} - -/** - * Release the write lock - * */ -ScopedRwWriteLock::~ScopedRwWriteLock() { - Unlock(); -} - -/** - * Acquire the write lock - * */ -void ScopedRwWriteLock::Lock(uint32_t owner) { - if (!is_locked_) { - lock_.WriteLock(owner); - is_locked_ = true; - } -} - -/** - * Release the write lock - * */ -void ScopedRwWriteLock::Unlock() { - if (is_locked_) { - lock_.WriteUnlock(); - is_locked_ = false; - } -} - -} // namespace hshm diff --git a/hermes_shm/src/thread_factory.cc b/hermes_shm/src/thread_factory.cc new file mode 100644 index 000000000..15b1efd69 --- /dev/null +++ b/hermes_shm/src/thread_factory.cc @@ -0,0 +1,44 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hermes_shm/thread/thread_model/thread_model_factory.h" +#include "hermes_shm/thread/thread_model/thread_model.h" +#include "hermes_shm/thread/thread_model/pthread.h" +#include "hermes_shm/thread/thread_model/argobots.h" +#include "hermes_shm/util/logging.h" + +namespace hshm::thread_model { + +std::unique_ptr ThreadFactory::Get(ThreadType type) { + switch (type) { + case ThreadType::kPthread: { +#ifdef HERMES_PTHREADS_ENABLED + return std::make_unique(); +#else + return nullptr; +#endif + } + case ThreadType::kArgobots: { +#ifdef HERMES_RPC_THALLIUM + return std::make_unique(); +#else + return nullptr; +#endif + } + default: { + HELOG(kWarning, "No such thread type"); + return nullptr; + } + } +} + +} // namespace hshm::thread_model diff --git a/hermes_shm/src/thread_model_manager.cc b/hermes_shm/src/thread_model_manager.cc new file mode 100644 index 000000000..ff4f1c252 --- /dev/null +++ b/hermes_shm/src/thread_model_manager.cc @@ -0,0 +1,49 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hermes_shm/thread/thread_model_manager.h" +#include "hermes_shm/util/logging.h" + +namespace hshm { + +/** Set the threading model of this application */ +void ThreadModelManager::SetThreadModel(ThreadType type) { + static std::mutex lock_; + lock_.lock(); + if (type_ == type) { + lock_.unlock(); + return; + } + type_ = type; + thread_static_ = thread_model::ThreadFactory::Get(type); + if (thread_static_ == nullptr) { + HELOG(kFatal, "Could not load the threading model"); + } + lock_.unlock(); +} + +/** Sleep for a period of time (microseconds) */ +void ThreadModelManager::SleepForUs(size_t us) { + thread_static_->SleepForUs(us); +} + +/** Call Yield */ +void ThreadModelManager::Yield() { + thread_static_->Yield(); +} + +/** Call GetTid */ +tid_t ThreadModelManager::GetTid() { + return thread_static_->GetTid(); +} + +} // namespace hshm diff --git a/hermes_shm/test/unit/CMakeLists.txt b/hermes_shm/test/unit/CMakeLists.txt index d4a927a97..cadd80c1b 100644 --- a/hermes_shm/test/unit/CMakeLists.txt +++ b/hermes_shm/test/unit/CMakeLists.txt @@ -8,5 +8,7 @@ set(TEST_MAIN ${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(data_structures) add_subdirectory(allocators) +add_subdirectory(allocators_mpi) +add_subdirectory(backend) add_subdirectory(types) add_subdirectory(thread) \ No newline at end of file diff --git a/hermes_shm/test/unit/allocators/CMakeLists.txt b/hermes_shm/test/unit/allocators/CMakeLists.txt index 010b1dbb8..28eca3f22 100644 --- a/hermes_shm/test/unit/allocators/CMakeLists.txt +++ b/hermes_shm/test/unit/allocators/CMakeLists.txt @@ -49,3 +49,10 @@ install(TARGETS LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_allocator_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/allocators/test_init.h b/hermes_shm/test/unit/allocators/test_init.h index 764adac1a..b59c6d89e 100644 --- a/hermes_shm/test/unit/allocators/test_init.h +++ b/hermes_shm/test/unit/allocators/test_init.h @@ -37,6 +37,8 @@ Allocator* Pretest() { std::string shm_url = "test_allocators"; allocator_id_t alloc_id(0, 1); auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); mem_mngr->CreateAllocator( diff --git a/hermes_shm/test/unit/allocators_mpi/CMakeLists.txt b/hermes_shm/test/unit/allocators_mpi/CMakeLists.txt new file mode 100644 index 000000000..185a8c8cb --- /dev/null +++ b/hermes_shm/test/unit/allocators_mpi/CMakeLists.txt @@ -0,0 +1,47 @@ +cmake_minimum_required(VERSION 3.10) +project(hermes_shm) + +set(CMAKE_CXX_STANDARD 17) + +#------------------------------------------------------------------------------ +# Build Tests +#------------------------------------------------------------------------------ + +add_executable(test_allocator_mpi_exec + ${TEST_MAIN}/main_mpi.cc + test_init.cc + allocator_mpi.cc) +add_dependencies(test_allocator_mpi_exec hermes_shm_data_structures) +target_link_libraries(test_allocator_mpi_exec + hermes_shm_data_structures Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) + +#------------------------------------------------------------------------------ +# Test Cases +#------------------------------------------------------------------------------ + +# Multi-Thread ALLOCATOR tests +set(MP_ALLOCATORS + StackAllocator + ScalablePageAllocator) +foreach(ALLOCATOR ${MP_ALLOCATORS}) + add_test(NAME test_${ALLOCATOR}_mpi COMMAND + mpirun -n 4 ${CMAKE_BINARY_DIR}/bin/test_allocator_mpi_exec "${ALLOCATOR}Mpi") +endforeach() + +#------------------------------------------------------------------------------ +# Install Targets +#------------------------------------------------------------------------------ +install(TARGETS + test_allocator_mpi_exec + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_allocator_mpi_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc b/hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc new file mode 100644 index 000000000..33d16a892 --- /dev/null +++ b/hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc @@ -0,0 +1,75 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "test_init.h" +#include "basic_test.h" + +struct Record { + char *data; + size_t size; + Pointer ptr; +}; + +void MpiPageAllocationTest(Allocator *alloc, size_t count) { + size_t window_length = 32; + size_t min_page = 64; + size_t max_page = MEGABYTES(1); + std::mt19937 rng(23522523); + std::uniform_int_distribution uni(min_page, max_page); + + MPI_Barrier(MPI_COMM_WORLD); + size_t num_windows = count / window_length; + std::vector window(window_length); + for (size_t w = 0; w < num_windows; ++w) { + for (size_t i = 0; i < window_length; ++i) { + window[i].size = uni(rng); + window[i].data = alloc->AllocatePtr(window[i].size, + window[i].ptr); + memset(window[i].data, (char)i, window[i].size); + } + for (size_t i = 0; i < window_length; ++i) { + VerifyBuffer(window[i].data, window[i].size, (char)i); + alloc->Free(window[i].ptr); + } + } + MPI_Barrier(MPI_COMM_WORLD); +} + +template +Allocator* TestAllocatorMpi() { + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank == 0) { + PretestRank0(); + } + MPI_Barrier(MPI_COMM_WORLD); + if (rank != 0) { + PretestRankN(); + } + HILOG(kInfo, "Allocator: {}", (size_t)alloc_g); + return alloc_g; +} + +TEST_CASE("StackAllocatorMpi") { + auto alloc = TestAllocatorMpi(); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + MpiPageAllocationTest(alloc, 100); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} + +TEST_CASE("ScalablePageAllocatorMpi") { + auto alloc = TestAllocatorMpi(); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + MpiPageAllocationTest(alloc, 1000); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} diff --git a/hermes_shm/test/unit/allocators_mpi/test_init.cc b/hermes_shm/test/unit/allocators_mpi/test_init.cc new file mode 100644 index 000000000..e248a58ea --- /dev/null +++ b/hermes_shm/test/unit/allocators_mpi/test_init.cc @@ -0,0 +1,33 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "basic_test.h" +#include "test_init.h" + +Allocator *alloc_g = nullptr; + +void PretestRankN() { + std::string shm_url = "test_allocators"; + allocator_id_t alloc_id(0, 1); + auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); + mem_mngr->AttachBackend(MemoryBackendType::kPosixShmMmap, shm_url); + alloc_g = mem_mngr->GetAllocator(alloc_id); +} + +void MainPretest() { +} + +void MainPosttest() { +} diff --git a/hermes_shm/test/unit/allocators_mpi/test_init.h b/hermes_shm/test/unit/allocators_mpi/test_init.h new file mode 100644 index 000000000..ef4601edc --- /dev/null +++ b/hermes_shm/test/unit/allocators_mpi/test_init.h @@ -0,0 +1,53 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#ifndef HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ +#define HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ + +#include "hermes_shm/data_structures/data_structure.h" +#include + +using hshm::ipc::PosixShmMmap; +using hshm::ipc::MemoryBackendType; +using hshm::ipc::MemoryBackend; +using hshm::ipc::allocator_id_t; +using hshm::ipc::AllocatorType; +using hshm::ipc::Allocator; +using hshm::ipc::Pointer; + +using hshm::ipc::MemoryBackendType; +using hshm::ipc::MemoryBackend; +using hshm::ipc::allocator_id_t; +using hshm::ipc::AllocatorType; +using hshm::ipc::Allocator; +using hshm::ipc::MemoryManager; +using hshm::ipc::Pointer; + +extern Allocator *alloc_g; + +template +void PretestRank0() { + std::string shm_url = "test_allocators"; + allocator_id_t alloc_id(0, 1); + auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); + mem_mngr->CreateBackend( + MemoryManager::GetDefaultBackendSize(), shm_url); + mem_mngr->CreateAllocator(shm_url, alloc_id, 0); + alloc_g = mem_mngr->GetAllocator(alloc_id); +} + +void PretestRankN(); + +#endif // HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ diff --git a/hermes_shm/test/unit/data_structures/backend/CMakeLists.txt b/hermes_shm/test/unit/backend/CMakeLists.txt similarity index 86% rename from hermes_shm/test/unit/data_structures/backend/CMakeLists.txt rename to hermes_shm/test/unit/backend/CMakeLists.txt index f4c1a2ab6..5ddb37ade 100644 --- a/hermes_shm/test/unit/data_structures/backend/CMakeLists.txt +++ b/hermes_shm/test/unit/backend/CMakeLists.txt @@ -37,3 +37,10 @@ install(TARGETS LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_memory_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/backend/backend.cc b/hermes_shm/test/unit/backend/backend.cc similarity index 100% rename from hermes_shm/test/unit/data_structures/backend/backend.cc rename to hermes_shm/test/unit/backend/backend.cc diff --git a/hermes_shm/test/unit/data_structures/backend/memory_manager.cc b/hermes_shm/test/unit/backend/memory_manager.cc similarity index 97% rename from hermes_shm/test/unit/data_structures/backend/memory_manager.cc rename to hermes_shm/test/unit/backend/memory_manager.cc index 421ca226c..9ec598339 100644 --- a/hermes_shm/test/unit/data_structures/backend/memory_manager.cc +++ b/hermes_shm/test/unit/backend/memory_manager.cc @@ -39,6 +39,8 @@ TEST_CASE("MemoryManager") { if (rank == 0) { std::cout << "Creating SHMEM (rank 0): " << shm_url << std::endl; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); mem_mngr->CreateAllocator( diff --git a/hermes_shm/test/unit/data_structures/backend/memory_slots.cc b/hermes_shm/test/unit/backend/memory_slots.cc similarity index 100% rename from hermes_shm/test/unit/data_structures/backend/memory_slots.cc rename to hermes_shm/test/unit/backend/memory_slots.cc diff --git a/hermes_shm/test/unit/data_structures/backend/test_init.cc b/hermes_shm/test/unit/backend/test_init.cc similarity index 100% rename from hermes_shm/test/unit/data_structures/backend/test_init.cc rename to hermes_shm/test/unit/backend/test_init.cc diff --git a/hermes_shm/test/unit/data_structures/CMakeLists.txt b/hermes_shm/test/unit/data_structures/CMakeLists.txt index 0166a9d6d..f606e5239 100644 --- a/hermes_shm/test/unit/data_structures/CMakeLists.txt +++ b/hermes_shm/test/unit/data_structures/CMakeLists.txt @@ -4,7 +4,6 @@ project(hermes_shm) set(CMAKE_CXX_STANDARD 17) include_directories(${HERMES_SHM_ROOT}/test/unit) -add_subdirectory(backend) add_subdirectory(containers) #add_subdirectory(containers_mpi) add_subdirectory(serialize) diff --git a/hermes_shm/test/unit/data_structures/containers/CMakeLists.txt b/hermes_shm/test/unit/data_structures/containers/CMakeLists.txt index b9269f98a..44a8af076 100644 --- a/hermes_shm/test/unit/data_structures/containers/CMakeLists.txt +++ b/hermes_shm/test/unit/data_structures/containers/CMakeLists.txt @@ -21,6 +21,9 @@ add_executable(test_data_structure_exec unique_ptr.cc unordered_map.cc mpsc_queue.cc + spsc_queue.cc + charbuf.cc + ticket_queue.cc ) add_dependencies(test_data_structure_exec hermes_shm_data_structures) @@ -37,36 +40,48 @@ add_test(NAME test_string COMMAND ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "String") # VECTOR TESTS -add_test(NAME test_vector_of_int COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "VectorOfInt") -add_test(NAME test_vector_of_string COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "VectorOfString") -add_test(NAME test_vector_of_list_of_string COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "VectorOfListOfString") +add_test(NAME test_vector COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "Vector*") # LIST TESTS -add_test(NAME test_list_of_int COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "ListOfInt") -add_test(NAME test_list_of_string COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "ListOfString") +add_test(NAME test_list COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "List*") + +# SLIST TESTS +add_test(NAME test_slist COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "Slist*") # MANUAL PTR TESTS -add_test(NAME test_manual_ptr_of_string COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "ManualPtrOfString") +add_test(NAME test_manual_ptr COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "ManualPtr*") # UNIQUE PTR TESTS -add_test(NAME test_unique_ptr_of_string COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UniquePtrOfString") +add_test(NAME test_unique_ptr COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UniquePtr*") # UNORDERED_MAP TESTS -add_test(NAME test_unordered_map_of_int_int COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UnorderedMapOfIntInt") -add_test(NAME test_unordered_map_of_int_str COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UnorderedMapOfIntString") -add_test(NAME test_unordered_map_of_str_int COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UnorderedMapOfStringInt") -add_test(NAME test_unordered_map_of_str_str COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UnorderedMapOfStringString") +add_test(NAME test_unordered_map COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UnorderedMap*") + +# PAIR TESTS +add_test(NAME test_pair COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "Pair*") + +# IQUEUE TESTS +add_test(NAME test_iqueue COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "IqueueOfMpPage") + +# SPSC TESTS +add_test(NAME test_spsc COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "TestSpsc*") + +# MPSC TESTS +add_test(NAME test_mpsc COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "TestMpsc*") + +# TicketQueue TESTS +add_test(NAME test_tkt_queue COMMAND + ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "TestTicket*") #------------------------------------------------------------------------------ # Install Targets @@ -78,3 +93,10 @@ install(TARGETS LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_data_structure_exec) +endif() diff --git a/hermes_shm/test/unit/data_structures/containers/charbuf.cc b/hermes_shm/test/unit/data_structures/containers/charbuf.cc new file mode 100644 index 000000000..21c35eef6 --- /dev/null +++ b/hermes_shm/test/unit/data_structures/containers/charbuf.cc @@ -0,0 +1,222 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "basic_test.h" +#include "test_init.h" +#include "hermes_shm/data_structures/containers/charbuf.h" + +using hshm::ipc::string; + +void TestCharbuf() { + Allocator *alloc = alloc_g; + + PAGE_DIVIDE("Construct from allocator") { + hshm::charbuf data(256); + memset(data.data(), 0, 256); + REQUIRE(data.size() == 256); + REQUIRE(data.GetAllocator() == alloc); + } + + PAGE_DIVIDE("Construct from malloc") { + char *ptr = (char*)malloc(256); + hshm::charbuf data(ptr, 256); + memset(data.data(), 0, 256); + REQUIRE(data.size() == 256); + REQUIRE(data.GetAllocator() == nullptr); + free(ptr); + } + + PAGE_DIVIDE("Resize null charbuf to higher value") { + hshm::charbuf data; + data.resize(256); + REQUIRE(data.size() == 256); + REQUIRE(data.GetAllocator() == alloc); + } + + PAGE_DIVIDE("Resize null charbuf to 0 value") { + hshm::charbuf data; + data.resize(0); + REQUIRE(data.size() == 0); + REQUIRE(data.GetAllocator() == nullptr); + } + + PAGE_DIVIDE("Resize destructable charbuf to 0 value") { + hshm::charbuf data(8192); + data.resize(0); + REQUIRE(data.size() == 0); + REQUIRE(data.GetAllocator() == alloc); + } + + PAGE_DIVIDE("Resize destructable charbuf to lower value") { + hshm::charbuf data(8192); + data.resize(256); + REQUIRE(data.size() == 256); + REQUIRE(data.GetAllocator() == alloc); + } + + PAGE_DIVIDE("Resize destructable charbuf to higher value") { + hshm::charbuf data(256); + data.resize(8192); + REQUIRE(data.size() == 8192); + REQUIRE(data.GetAllocator() == alloc); + } + + PAGE_DIVIDE("Resize indestructable charbuf to higher value") { + char *ptr = (char*)malloc(256); + hshm::charbuf data(ptr, 256); + data.resize(8192); + REQUIRE(data.size() == 8192); + free(ptr); + } + + PAGE_DIVIDE("Resize indestructable charbuf to lower value") { + char *ptr = (char*)malloc(8192); + hshm::charbuf data(ptr, 8192); + data.resize(256); + REQUIRE(data.size() == 256); + free(ptr); + } + + PAGE_DIVIDE("Move construct from destructable") { + hshm::charbuf data1(8192); + hshm::charbuf data2(std::move(data1)); + REQUIRE(data2.size() == 8192); + } + + PAGE_DIVIDE("Move construct from indestructable") { + char *ptr1 = (char*)malloc(8192); + hshm::charbuf data1(ptr1, 8192); + hshm::charbuf data2(std::move(data1)); + REQUIRE(data2.size() == 8192); + free(ptr1); + } + + PAGE_DIVIDE("Move assign between two destructables") { + hshm::charbuf data1(8192); + hshm::charbuf data2(512); + data1 = std::move(data2); + REQUIRE(data1.size() == 512); + } + + PAGE_DIVIDE("Move assign between two indestructables") { + char *ptr1 = (char*)malloc(8192); + hshm::charbuf data1(ptr1, 8192); + char *ptr2 = (char*)malloc(512); + hshm::charbuf data2(ptr2, 512); + data1 = std::move(data2); + REQUIRE(data1.size() == 512); + free(ptr1); + free(ptr2); + } + + PAGE_DIVIDE("Move assign indestructable -> destructable") { + hshm::charbuf data1(8192); + char *ptr2 = (char*)malloc(512); + hshm::charbuf data2(ptr2, 512); + data1 = std::move(data2); + REQUIRE(data1.size() == 512); + free(ptr2); + } + + PAGE_DIVIDE("Move assign destructable -> indestructable") { + char *ptr1 = (char*)malloc(8192); + hshm::charbuf data1(ptr1, 8192); + hshm::charbuf data2(512); + data1 = std::move(data2); + REQUIRE(data1.size() == 512); + free(ptr1); + } + + PAGE_DIVIDE("Move assign to null") { + hshm::charbuf data1; + hshm::charbuf data2(512); + data1 = std::move(data2); + REQUIRE(data1.size() == 512); + } + + PAGE_DIVIDE("Copy construct from destructable") { + hshm::charbuf data1(8192); + hshm::charbuf data2(data1); + REQUIRE(data1.size() == 8192); + REQUIRE(data2.size() == 8192); + REQUIRE(data1 == data2); + } + + PAGE_DIVIDE("Copy construct from indestructable") { + char *ptr1 = (char*)malloc(8192); + hshm::charbuf data1(ptr1, 8192); + hshm::charbuf data2(data1); + REQUIRE(data1.size() == 8192); + REQUIRE(data2.size() == 8192); + REQUIRE(data1 == data2); + free(ptr1); + } + + PAGE_DIVIDE("Copy assign between two destructables") { + hshm::charbuf data1(8192); + hshm::charbuf data2(512); + data1 = data2; + REQUIRE(data2.size() == 512); + REQUIRE(data1.size() == 512); + REQUIRE(data1 == data2); + } + + PAGE_DIVIDE("Copy assign between two indestructables") { + char *ptr1 = (char*)malloc(8192); + hshm::charbuf data1(ptr1, 8192); + char *ptr2 = (char*)malloc(512); + hshm::charbuf data2(ptr2, 512); + data1 = data2; + REQUIRE(data2.size() == 512); + REQUIRE(data1.size() == 512); + REQUIRE(data1 == data2); + free(ptr1); + free(ptr2); + } + + PAGE_DIVIDE("Copy assign indestructable -> destructable") { + hshm::charbuf data1(8192); + char *ptr2 = (char*)malloc(512); + hshm::charbuf data2(ptr2, 512); + data1 = data2; + REQUIRE(data2.size() == 512); + REQUIRE(data1.size() == 512); + free(ptr2); + } + + PAGE_DIVIDE("Copy assign destructable -> indestructable") { + char *ptr1 = (char*)malloc(8192); + hshm::charbuf data1(ptr1, 8192); + hshm::charbuf data2(512); + data1 = data2; + REQUIRE(data2.size() == 512); + REQUIRE(data1.size() == 512); + free(ptr1); + } + + PAGE_DIVIDE("Copy assign to null") { + char *ptr1 = (char*)malloc(8192); + hshm::charbuf data1; + hshm::charbuf data2(512); + data1 = data2; + REQUIRE(data2.size() == 512); + REQUIRE(data1.size() == 512); + free(ptr1); + } +} + +TEST_CASE("Charbuf") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + TestCharbuf(); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} diff --git a/hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc b/hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc index d089f9e2c..30d9c8d66 100644 --- a/hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc +++ b/hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc @@ -13,151 +13,7 @@ #include "basic_test.h" #include "test_init.h" #include "hermes_shm/data_structures/ipc/mpsc_queue.h" - -template -class QueueTestSuite { - public: - hipc::uptr &queue_; - - public: - /** Constructor */ - explicit QueueTestSuite(hipc::uptr &queue) - : queue_(queue) {} - - /** Producer method */ - void Produce(size_t count_per_rank) { - try { - int rank = omp_get_thread_num(); - for (size_t i = 0; i < count_per_rank; ++i) { - size_t idx = rank * count_per_rank + i; - CREATE_SET_VAR_TO_INT_OR_STRING(T, var, idx); - queue_->emplace(var); - } - } catch (hshm::Error &e) { - HELOG(kFatal, e.what()) - } - } - - /** Consumer method */ - void Consume(size_t nproducers, size_t count_per_rank, - bool inorder) { - std::vector entries; - size_t total_size = nproducers * count_per_rank; - entries.reserve(count_per_rank); - auto entry = hipc::make_uptr(); - auto &entry_ref = *entry; - - // Consume everything - while (entries.size() < total_size) { - auto qtok = queue_->pop(entry_ref); - if (qtok.IsNull()) { - HERMES_THREAD_MODEL->Yield(); - continue; - } - CREATE_GET_INT_FROM_VAR(T, entry_int, entry_ref) - entries.emplace_back(entry_int); - } - - // Ensure there's no data left in the queue - REQUIRE(queue_->pop(entry_ref).IsNull()); - - // Ensure that all elements are here - if (!inorder) { - std::sort(entries.begin(), entries.end()); - } - for (int i = 0; i < (int)entries.size(); ++i) { - REQUIRE(entries[i] == i); - } - } -}; - -template -void ProduceThenConsume(size_t nproducers, size_t count_per_rank) { - size_t depth = 32; - auto queue = hipc::make_uptr(depth); - QueueTestSuite q(queue); - - // Produce all the data - omp_set_dynamic(0); -#pragma omp parallel shared(nproducers, count_per_rank, q) num_threads(nproducers) // NOLINT - { // NOLINT -#pragma omp barrier - q.Produce(count_per_rank); -#pragma omp barrier - } - - // Consume all the data - q.Consume(nproducers, count_per_rank, nproducers == 1); -} - -template -void ProduceAndConsume(size_t nproducers, size_t count_per_rank) { - size_t depth = 32; - auto queue = hipc::make_uptr(depth); - size_t nthreads = nproducers + 1; - QueueTestSuite q(queue); - - // Produce all the data - omp_set_dynamic(0); -#pragma omp parallel shared(nproducers, count_per_rank, q) num_threads(nthreads) // NOLINT - { // NOLINT -#pragma omp barrier - size_t rank = omp_get_thread_num(); - if (rank < nproducers) { - // Producer - q.Produce(count_per_rank); - } else { - // Consumer - q.Consume(nproducers, count_per_rank, false); - } -#pragma omp barrier - } -} - -/** - * TEST MPSC EXTENSIBLE QUEUE - * */ - -TEST_CASE("TestMpscQueueExtInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(1, 32); - ProduceThenConsume, int>(1, 64); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestMpscQueueExtString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, hipc::string>(1, 32); - ProduceThenConsume, hipc::string>(1, 64); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestMpscQueueExtIntMultiThreaded") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(2, 16); - ProduceThenConsume, int>(4, 64); - ProduceThenConsume, int>(8, 256); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestMpscQueueExtStringMultiThreaded") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, hipc::string>(2, 16); - ProduceThenConsume, hipc::string>(4, 64); - ProduceThenConsume, hipc::string>(8, 256); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestMpscQueueExtStringMultiThreaded2") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceAndConsume, hipc::string>(8, 8192); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} +#include "queue.h" /** * TEST MPSC QUEUE @@ -166,27 +22,29 @@ TEST_CASE("TestMpscQueueExtStringMultiThreaded2") { TEST_CASE("TestMpscQueueInt") { Allocator *alloc = alloc_g; REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(1, 32); + ProduceThenConsume, int>(1, 1, 32, 32); REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); } TEST_CASE("TestMpscQueueString") { Allocator *alloc = alloc_g; REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, hipc::string>(1, 32); + ProduceThenConsume, hipc::string>( + 1, 1, 32, 32); REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); } TEST_CASE("TestMpscQueueIntMultiThreaded") { Allocator *alloc = alloc_g; REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceAndConsume, int>(8, 8192); + ProduceAndConsume, int>(8, 1, 8192, 32); REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); } TEST_CASE("TestMpscQueueStringMultiThreaded") { Allocator *alloc = alloc_g; REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceAndConsume, hipc::string>(8, 8192); + ProduceAndConsume, hipc::string>( + 8, 1, 8192, 32); REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); } diff --git a/hermes_shm/test/unit/data_structures/containers/queue.h b/hermes_shm/test/unit/data_structures/containers/queue.h new file mode 100644 index 000000000..6eafd8f15 --- /dev/null +++ b/hermes_shm/test/unit/data_structures/containers/queue.h @@ -0,0 +1,149 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_QUEUE_H_ +#define HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_QUEUE_H_ + +#include "hermes_shm/data_structures/data_structure.h" +#include "omp.h" +#include "hermes_shm/data_structures/ipc/ticket_stack.h" + +template +class QueueTestSuite { + public: + hipc::uptr &queue_; + + public: + /** Constructor */ + explicit QueueTestSuite(hipc::uptr &queue) + : queue_(queue) {} + + /** Producer method */ + void Produce(size_t count_per_rank) { + std::vector idxs; + int rank = omp_get_thread_num(); + try { + for (size_t i = 0; i < count_per_rank; ++i) { + size_t idx = rank * count_per_rank + i; + CREATE_SET_VAR_TO_INT_OR_STRING(T, var, idx); + CREATE_GET_INT_FROM_VAR(T, entry_int, var) + idxs.emplace_back(entry_int); + while (queue_->emplace(var).IsNull()) {} + } + } catch (hshm::Error &e) { + HELOG(kFatal, e.what()) + } + REQUIRE(idxs.size() == count_per_rank); + std::sort(idxs.begin(), idxs.end()); + for (size_t i = 0; i < count_per_rank; ++i) { + size_t idx = rank * count_per_rank + i; + REQUIRE(idxs[i] == idx); + } + } + + /** Consumer method */ + void Consume(std::atomic &count, + size_t total_count, + std::vector &entries) { + auto entry = hipc::make_uptr(); + auto &entry_ref = *entry; + + // Consume everything + while (count < total_count) { + auto qtok = queue_->pop(entry_ref); + if (qtok.IsNull()) { + continue; + } + CREATE_GET_INT_FROM_VAR(T, entry_int, entry_ref) + size_t off = count.fetch_add(1); + if (off >= total_count) { + break; + } + entries[off] = entry_int; + } + + int rank = omp_get_thread_num(); + if (rank == 0) { + // Ensure there's no data left in the queue + REQUIRE(queue_->pop(entry_ref).IsNull()); + + // Ensure the data is all correct + REQUIRE(entries.size() == total_count); + std::sort(entries.begin(), entries.end()); + REQUIRE(entries.size() == total_count); + for (size_t i = 0; i < total_count; ++i) { + REQUIRE(entries[i] == i); + } + } + } +}; + +template +void ProduceThenConsume(size_t nproducers, + size_t nconsumers, + size_t count_per_rank, + size_t depth) { + auto queue = hipc::make_uptr(depth); + QueueTestSuite q(queue); + std::atomic count = 0; + std::vector entries; + entries.resize(count_per_rank * nproducers); + + // Produce all the data + omp_set_dynamic(0); +#pragma omp parallel shared(nproducers, count_per_rank, q, count, entries) num_threads(nproducers) // NOLINT + { // NOLINT +#pragma omp barrier + q.Produce(count_per_rank); +#pragma omp barrier + } + + omp_set_dynamic(0); +#pragma omp parallel shared(nproducers, count_per_rank, q) num_threads(nconsumers) // NOLINT + { // NOLINT +#pragma omp barrier + // Consume all the data + q.Consume(count, count_per_rank * nproducers, entries); +#pragma omp barrier + } +} + +template +void ProduceAndConsume(size_t nproducers, + size_t nconsumers, + size_t count_per_rank, + size_t depth) { + auto queue = hipc::make_uptr(depth); + size_t nthreads = nproducers + nconsumers; + QueueTestSuite q(queue); + std::atomic count = 0; + std::vector entries; + entries.resize(count_per_rank * nproducers); + + // Produce all the data + omp_set_dynamic(0); +#pragma omp parallel shared(nproducers, count_per_rank, q, count) num_threads(nthreads) // NOLINT + { // NOLINT +#pragma omp barrier + size_t rank = omp_get_thread_num(); + if (rank < nproducers) { + // Producer + q.Produce(count_per_rank); + } else { + // Consumer + q.Consume(count, count_per_rank * nproducers, entries); + } +#pragma omp barrier + } +} + +#endif // HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_QUEUE_H_ diff --git a/hermes_shm/test/unit/data_structures/containers/slist.cc b/hermes_shm/test/unit/data_structures/containers/slist.cc index cb9667224..ac6322964 100644 --- a/hermes_shm/test/unit/data_structures/containers/slist.cc +++ b/hermes_shm/test/unit/data_structures/containers/slist.cc @@ -19,7 +19,7 @@ using hshm::ipc::slist; template -void ListTest() { +void SlistTest() { Allocator *alloc = alloc_g; auto lp = hipc::make_uptr>(alloc); ListTestSuite> test(*lp, alloc); @@ -40,20 +40,20 @@ void ListTest() { TEST_CASE("SlistOfInt") { Allocator *alloc = alloc_g; REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ListTest(); + SlistTest(); REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); } TEST_CASE("SlistOfString") { Allocator *alloc = alloc_g; REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ListTest(); + SlistTest(); REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); } TEST_CASE("SlistOfStdString") { Allocator *alloc = alloc_g; REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ListTest(); + SlistTest(); REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); } diff --git a/hermes_shm/test/unit/data_structures/containers/spsc_queue.cc b/hermes_shm/test/unit/data_structures/containers/spsc_queue.cc new file mode 100644 index 000000000..cc665bd0c --- /dev/null +++ b/hermes_shm/test/unit/data_structures/containers/spsc_queue.cc @@ -0,0 +1,35 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "basic_test.h" +#include "test_init.h" +#include "hermes_shm/data_structures/ipc/spsc_queue.h" +#include "queue.h" + +/** + * TEST SPSC QUEUE + * */ + +TEST_CASE("TestSpscQueueInt") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceThenConsume, int>(1, 1, 32, 32); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} + +TEST_CASE("TestSpscQueueString") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceThenConsume, hipc::string>( + 1, 1, 32, 32); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} diff --git a/hermes_shm/test/unit/data_structures/containers/test_init.h b/hermes_shm/test/unit/data_structures/containers/test_init.h index 543d19d24..155cf9375 100644 --- a/hermes_shm/test/unit/data_structures/containers/test_init.h +++ b/hermes_shm/test/unit/data_structures/containers/test_init.h @@ -39,6 +39,8 @@ void Pretest() { std::string shm_url = "test_allocators"; allocator_id_t alloc_id(0, 1); auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); mem_mngr->CreateAllocator(shm_url, alloc_id, 0); diff --git a/hermes_shm/test/unit/data_structures/containers/ticket_queue.cc b/hermes_shm/test/unit/data_structures/containers/ticket_queue.cc new file mode 100644 index 000000000..b5c3e96c6 --- /dev/null +++ b/hermes_shm/test/unit/data_structures/containers/ticket_queue.cc @@ -0,0 +1,68 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "basic_test.h" +#include "test_init.h" +#include "hermes_shm/data_structures/ipc/ticket_queue.h" +#include "hermes_shm/data_structures/ipc/ticket_stack.h" +#include "hermes_shm/data_structures/ipc/split_ticket_queue.h" +#include "queue.h" + +/** + * TEST TICKET QUEUE + * */ + +TEST_CASE("TestTicketStackInt") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceThenConsume, int>(1, 1, 32, 32); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} + +TEST_CASE("TestTicketStackIntMultiThreaded") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceThenConsume, int>(8, 1, 8192, 8192 * 8); + ProduceAndConsume, int>(8, 1, 8192, 64); + ProduceAndConsume, int>(8, 8, 8192, 64); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} + +TEST_CASE("TestTicketQueueInt") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceThenConsume, int>(1, 1, 32, 32); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} + +TEST_CASE("TestTicketQueueIntMultiThreaded") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceAndConsume, int>(8, 1, 8192, 64); + ProduceAndConsume, int>(8, 8, 8192, 64); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} + +TEST_CASE("TestSplitTicketQueueInt") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceThenConsume, int>(1, 1, 32, 32); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} + +TEST_CASE("TestSplitTicketQueueIntMultiThreaded") { + Allocator *alloc = alloc_g; + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); + ProduceAndConsume, int>(8, 1, 8192, 64); + ProduceAndConsume, int>(8, 8, 8192, 64); + REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); +} diff --git a/hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt b/hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt index 422e9b27e..a18256d28 100644 --- a/hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt +++ b/hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt @@ -45,3 +45,10 @@ install(TARGETS LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_data_structure_mpi_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc b/hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc index b5d339ccd..c4517f13a 100644 --- a/hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc +++ b/hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc @@ -21,6 +21,8 @@ void PretestRank0() { std::string shm_url = "test_allocators"; allocator_id_t alloc_id(0, 1); auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); mem_mngr->CreateAllocator(shm_url, alloc_id, sizeof(Pointer)); diff --git a/hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt b/hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt index a36d4b03e..46adc3080 100644 --- a/hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt +++ b/hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt @@ -31,3 +31,10 @@ install(TARGETS LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_shm_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc b/hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc index a723ac90f..a0dd32ebc 100644 --- a/hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc +++ b/hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc @@ -21,6 +21,8 @@ void MainPretest() { std::string shm_url = "test_serializers"; allocator_id_t alloc_id(0, 1); auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); mem_mngr->CreateAllocator(shm_url, alloc_id, 0); diff --git a/hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc b/hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc index d315c542d..aee4add30 100644 --- a/hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc +++ b/hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc @@ -21,7 +21,8 @@ TEST_CASE("SerializePod") { int a = 1; double b = 2; float c = 3; - int size = sizeof(int) + sizeof(double) + sizeof(float) + sizeof(allocator_id_t); + size_t size = sizeof(int) + sizeof(double) + + sizeof(float) + sizeof(allocator_id_t); REQUIRE(istream.shm_buf_size(alloc->GetId(), a, b, c) == size); char *buf = istream.serialize(alloc, a, b, c); @@ -39,18 +40,12 @@ TEST_CASE("SerializePod") { TEST_CASE("SerializeString") { hipc::ShmSerializer istream; Allocator *alloc = HERMES_MEMORY_MANAGER->GetDefaultAllocator(); - hipc::mptr i; - bool string_ar = IS_SHM_ARCHIVEABLE(hipc::string); - bool string_star_ar = IS_SHM_ARCHIVEABLE(std::remove_reference); - bool string_ar2 = IS_SHM_ARCHIVEABLE(std::remove_reference::type); - - hipc::string &y = *i; - auto a = hipc::make_uptr(alloc, "h1"); auto b = hipc::make_uptr(alloc, "h2"); int c; - int size = 2 * sizeof(hipc::OffsetPointer) + sizeof(int) + sizeof(allocator_id_t); + size_t size = 2 * sizeof(hipc::OffsetPointer) + + sizeof(int) + sizeof(allocator_id_t); REQUIRE(istream.shm_buf_size(alloc->GetId(), *a, *b, c) == size); char *buf = istream.serialize(alloc, *a, *b, c); diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt b/hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt index 50cd19fee..e4f893903 100644 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt +++ b/hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt @@ -44,3 +44,11 @@ install(TARGETS LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_thallium_server) + set_coverage_flags(test_thallium_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h b/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h index f111e4580..3bb5f47eb 100644 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h +++ b/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h @@ -64,6 +64,8 @@ void ServerPretest() { std::string shm_url = "test_serializers"; allocator_id_t alloc_id(0, 1); auto mem_mngr = HERMES_MEMORY_MANAGER; + mem_mngr->UnregisterAllocator(alloc_id); + mem_mngr->UnregisterBackend(shm_url); mem_mngr->CreateBackend( MemoryManager::GetDefaultBackendSize(), shm_url); mem_mngr->CreateAllocator(shm_url, alloc_id, 0); diff --git a/hermes_shm/test/unit/thread/CMakeLists.txt b/hermes_shm/test/unit/thread/CMakeLists.txt index 7b4560ae4..69a28e6eb 100644 --- a/hermes_shm/test/unit/thread/CMakeLists.txt +++ b/hermes_shm/test/unit/thread/CMakeLists.txt @@ -35,3 +35,10 @@ install(TARGETS LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_thread_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/thread/test_lock.cc b/hermes_shm/test/unit/thread/test_lock.cc index 7997adb13..bac5d51ee 100644 --- a/hermes_shm/test/unit/thread/test_lock.cc +++ b/hermes_shm/test/unit/thread/test_lock.cc @@ -40,19 +40,8 @@ void MutexTest() { } } -void barrier_for_reads(std::vector &tid_start, size_t left) { - size_t count; - do { - count = 0; - for (size_t i = 0; i < left; ++i) { - count += tid_start[i]; - } - } while (count < left); -} - -void RwLockTest(int producers, int consumers) { +void RwLockTest(int producers, int consumers, size_t loop_count) { size_t nthreads = producers + consumers; - size_t loop_count = 100000; size_t count = 0; RwLock lock; @@ -69,7 +58,7 @@ void RwLockTest(int producers, int consumers) { // The left 2 threads will be readers lock.ReadLock(tid); for (size_t i = 0; i < loop_count; ++i) { - REQUIRE(count < total_size); + REQUIRE(count <= total_size); } lock.ReadUnlock(); } else { @@ -91,7 +80,7 @@ TEST_CASE("Mutex") { } TEST_CASE("RwLock") { - RwLockTest(8, 0); - RwLockTest(7, 1); - RwLockTest(4, 4); + RwLockTest(8, 0, 1000000); + RwLockTest(7, 1, 1000000); + RwLockTest(4, 4, 1000000); } diff --git a/hermes_shm/test/unit/types/CMakeLists.txt b/hermes_shm/test/unit/types/CMakeLists.txt index f4b7cd5d7..f4cdeb082 100644 --- a/hermes_shm/test/unit/types/CMakeLists.txt +++ b/hermes_shm/test/unit/types/CMakeLists.txt @@ -6,22 +6,37 @@ set(CMAKE_CXX_STANDARD 17) #------------------------------------------------------------------------------ # Build Tests #------------------------------------------------------------------------------ -add_executable(test_types +add_executable(test_types_exec ${TEST_MAIN}/main.cc test_init.cc test_argpack.cc test_util.cc) -add_dependencies(test_types hermes_shm_data_structures) -target_link_libraries(test_types +add_dependencies(test_types_exec hermes_shm_data_structures) +target_link_libraries(test_types_exec hermes_shm_data_structures Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) -#------------------------------------------------------------------------------ +add_test(NAME test_types COMMAND + ${CMAKE_BINARY_DIR}/bin/test_types_exec "~[error=FatalError]") +add_test(NAME test_fatal_logger + COMMAND ${CMAKE_BINARY_DIR}/bin/test_types_exec "[error=FatalError]") +set_tests_properties( + test_fatal_logger PROPERTIES + WILL_FAIL TRUE) + +#----------------------------------------------------------------------------- # Install Targets #------------------------------------------------------------------------------ install(TARGETS - test_types + test_types_exec EXPORT ${HERMES_EXPORTED_TARGETS} LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_types_exec) +endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/types/test_util.cc b/hermes_shm/test/unit/types/test_util.cc index 283636699..aec093256 100644 --- a/hermes_shm/test/unit/types/test_util.cc +++ b/hermes_shm/test/unit/types/test_util.cc @@ -102,10 +102,12 @@ TEST_CASE("TestLogger") { HILOG(kInfo, "I'm more likely to be printed (2): {}", 0) HILOG(kDebug, "I won't be printed: {}", 10) -#ifdef TEST_ERRORS - HELOG(kError, "I will NOT cause an EXIT!") + HELOG(kWarning, "I am a WARNING! Will NOT cause an EXIT!") + HELOG(kError, "I am an ERROR! I will NOT cause an EXIT!") +} + +TEST_CASE("TestFatalLogger", "[error=FatalError]") { HELOG(kFatal, "I will cause an EXIT!") -#endif } TEST_CASE("TestFormatter") { From 000b65171e27516e38e959ddc6d61e310ada49ab Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 13 Jun 2023 11:00:26 -0500 Subject: [PATCH 005/191] fix current source dir --- hermes_shm/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermes_shm/CMakeLists.txt b/hermes_shm/CMakeLists.txt index 17b3c29b4..5f76cd2d2 100644 --- a/hermes_shm/CMakeLists.txt +++ b/hermes_shm/CMakeLists.txt @@ -197,7 +197,7 @@ endif() install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) configure_file( - ${CMAKE_SOURCE_DIR}/CMake/HermesShmConfig.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/CMake/HermesShmConfig.cmake ${PROJECT_BINARY_DIR}/CMakeFiles/HermesShmConfig.cmake @ONLY ) From 8ffc121e5aaf12963ffecee84c7ab35dfea1a9ad Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 13 Jun 2023 11:22:01 -0500 Subject: [PATCH 006/191] Add hermes_posix as a dependency of posix_simple_io --- adapter/test/posix/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter/test/posix/CMakeLists.txt b/adapter/test/posix/CMakeLists.txt index 2c86c227e..b18218102 100644 --- a/adapter/test/posix/CMakeLists.txt +++ b/adapter/test/posix/CMakeLists.txt @@ -37,7 +37,7 @@ add_dependencies(hermes_posix_simple_io_omp hermes_posix) target_link_libraries(hermes_posix_simple_io_omp hermes_posix) add_executable(posix_simple_io_omp posix_simple_io_omp.cc) -target_link_libraries(posix_simple_io_omp) +add_dependencies(posix_simple_io_omp hermes_posix) pytest(posix test_hermes_posix_simple_io_omp_default) pytest(posix test_hermes_posix_simple_io_omp_scratch) From b4552304cf0fb84390fee193ff6a8c1e39e0edae Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 13 Jun 2023 11:24:34 -0500 Subject: [PATCH 007/191] Link to hermes_shm_data_structures --- adapter/test/posix/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adapter/test/posix/CMakeLists.txt b/adapter/test/posix/CMakeLists.txt index b18218102..e24b5d9b6 100644 --- a/adapter/test/posix/CMakeLists.txt +++ b/adapter/test/posix/CMakeLists.txt @@ -37,7 +37,8 @@ add_dependencies(hermes_posix_simple_io_omp hermes_posix) target_link_libraries(hermes_posix_simple_io_omp hermes_posix) add_executable(posix_simple_io_omp posix_simple_io_omp.cc) -add_dependencies(posix_simple_io_omp hermes_posix) +add_dependencies(posix_simple_io_omp hermes_shm_data_structures) +target_link_libraries(posix_simple_io_omp hermes_shm_data_structures) pytest(posix test_hermes_posix_simple_io_omp_default) pytest(posix test_hermes_posix_simple_io_omp_scratch) From e7d12122cf0b8060d6878276d1b9e7db67d638a2 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 15 Sep 2023 10:01:48 -0500 Subject: [PATCH 008/191] Initial commit --- .gitignore | 18 +- .travis.yml | 67 - CMake/HermesConfig.cmake | 96 - CMake/LabstorConfig.cmake | 84 + CMake/UseDoxygenDoc.cmake | 33 - CMakeLists.txt | 618 +--- COPYING | 52 - adapter/CMakeLists.txt | 50 - adapter/kvstore/CMakeLists.txt | 62 - adapter/kvstore/README.md | 18 - adapter/kvstore/java/build.gradle | 49 - .../java/gradle/wrapper/gradle-wrapper.jar | Bin 54333 -> 0 bytes .../gradle/wrapper/gradle-wrapper.properties | 5 - adapter/kvstore/java/gradlew | 172 -- adapter/kvstore/java/gradlew.bat | 84 - adapter/kvstore/java/settings.gradle | 18 - .../java/src/kvstore/java/KVStore.java | 14 - .../java/src/kvstore/java/KVTable.java | 160 - .../java/src/test/java/KvstoreJniTest.java | 34 - adapter/kvstore/kvstore.cc | 140 - adapter/kvstore/kvstore.h | 177 -- adapter/kvstore/tests.py | 22 - adapter/mapper/balanced_mapper.cc | 43 - benchmark/CMakeLists.txt | 66 + .../hermes_api_bench.cc | 155 +- .../benchmark/lock => benchmark}/test_init.cc | 4 + .../test_init.cc => benchmark/test_init.h | 7 +- benchmark/test_latency.cc | 346 +++ benchmark/test_zmq.cc | 72 + benchmarks/CMakeLists.txt | 20 - benchmarks/memcpy_bench.cc | 154 - benchmarks/reorganize.cc | 150 - ci/build_hermes.sh | 47 - ci/cluster/Dockerfile | 56 - ci/cluster/cluster_down.sh | 4 - ci/cluster/cluster_test.sh | 4 - ci/cluster/cluster_up.sh | 4 - ci/cluster/cluster_utils.sh | 61 - ci/cluster/docker-compose.yml | 24 - ci/cluster/multi_node_ci_test.sh | 25 - ci/coverage.sh | 10 - ci/external/CMakeLists.txt | 8 - ci/external/test_install.cc | 18 - ci/hermes/packages/hermes/package.py | 57 - ci/hermes/repo.yaml | 2 - ci/install_deps.sh | 43 - ci/install_docs.sh | 41 - ci/install_hermes.sh | 6 - ci/jarvis-util/.coveragerc | 4 - ci/jarvis-util/.gitignore | 139 - ci/jarvis-util/LICENSE | 21 - ci/jarvis-util/README.md | 118 - ci/jarvis-util/bin/jarvis-imports | 8 - ci/jarvis-util/ci/cluster/Dockerfile | 55 - ci/jarvis-util/ci/cluster/docker-compose.yml | 24 - ci/jarvis-util/ci/install_deps.sh | 12 - ci/jarvis-util/ci/install_jarvis.sh | 3 - ci/jarvis-util/ci/install_spack.sh | 12 - ci/jarvis-util/ci/lint.sh | 2 - ci/jarvis-util/ci/run_tests.sh | 5 - ci/jarvis-util/jarvis_util/__init__.py | 25 - .../jarvis_util/introspect/system_info.py | 635 ---- ci/jarvis-util/jarvis_util/jutil_manager.py | 28 - .../jarvis_util/serialize/ini_file.py | 24 - .../jarvis_util/serialize/pickle.py | 24 - .../jarvis_util/serialize/serializer.py | 21 - .../jarvis_util/serialize/text_file.py | 22 - .../jarvis_util/serialize/yaml_file.py | 24 - ci/jarvis-util/jarvis_util/shell/__init__.py | 0 ci/jarvis-util/jarvis_util/shell/exec.py | 59 - ci/jarvis-util/jarvis_util/shell/exec_info.py | 216 -- .../jarvis_util/shell/filesystem.py | 69 - .../jarvis_util/shell/local_exec.py | 103 - ci/jarvis-util/jarvis_util/shell/mpi_exec.py | 55 - ci/jarvis-util/jarvis_util/shell/process.py | 22 - ci/jarvis-util/jarvis_util/shell/pscp.py | 58 - ci/jarvis-util/jarvis_util/shell/pssh_exec.py | 56 - ci/jarvis-util/jarvis_util/shell/scp.py | 115 - ci/jarvis-util/jarvis_util/shell/ssh_exec.py | 56 - ci/jarvis-util/jarvis_util/util/__init__.py | 0 ci/jarvis-util/jarvis_util/util/argparse.py | 454 --- ci/jarvis-util/jarvis_util/util/expand_env.py | 25 - ci/jarvis-util/jarvis_util/util/hostfile.py | 216 -- ci/jarvis-util/jarvis_util/util/import_all.py | 72 - ci/jarvis-util/jarvis_util/util/import_mod.py | 28 - ci/jarvis-util/jarvis_util/util/naming.py | 34 - ci/jarvis-util/jarvis_util/util/size_conv.py | 44 - ci/jarvis-util/pylintrc | 429 --- ci/jarvis-util/requirements.txt | 9 - ci/jarvis-util/setup.py | 23 - ci/jarvis-util/test/unit/argparse_main.py | 4 - ci/jarvis-util/test/unit/print5s.py | 12 - ci/jarvis-util/test/unit/printNone.py | 4 - ci/jarvis-util/test/unit/test_argparse.py | 10 - ci/jarvis-util/test/unit/test_hostfile.py | 57 - ci/jarvis-util/test/unit/test_hostfile.txt | 1 - ci/jarvis-util/test/unit/test_local_exec.py | 65 - ci/jarvis-util/test/unit/test_system_info.py | 137 - ci/lint.sh | 12 - ci/test_hermes.sh | 24 - code_generators/hermes_config.py | 26 - .../codegen/labstor_config}/generator.py | 13 +- .../codegen}/util/conv.py | 0 .../codegen}/util/naming.py | 0 .../codegen}/util/paths.py | 4 +- codegen/hermes_config | 29 + codegen/labstor_config | 28 + codegen/make_macro | 13 + codegen/make_task | 32 + {code_generators => codegen}/preamble.py | 2 +- codegen/refresh_methods | 176 ++ codegen/update_task | 28 + config/labstor_client_default.yaml | 1 + config/labstor_server_default.yaml | 54 + data_stager/CMakeLists.txt | 43 - data_stager/data_stager.h | 74 - data_stager/data_stager_factory.h | 41 - data_stager/stage_in.cc | 46 - data_stager/stage_out.cc | 33 - data_stager/stagers/posix_stager.cc | 106 - data_stager/stagers/posix_stager.h | 40 - data_stager/test/CMakeLists.txt | 4 - data_stager/test/tests.py | 78 - doc/Doxyfile.in | 2685 ----------------- doc/design/hermes-design.org | 816 ----- doc/design/img/Hermes IC.graphml | 245 -- doc/design/img/Hermes IC.png | Bin 42684 -> 0 bytes doc/design/img/adapter.graphml | 92 - doc/design/img/adapter.png | Bin 15781 -> 0 bytes doc/design/img/balanced-mapping.png | Bin 81395 -> 0 bytes doc/design/img/file2object.png | Bin 59544 -> 0 bytes doc/design/img/hermes-bucket-pool.png | Bin 120995 -> 0 bytes doc/design/img/hermes-buckets.png | Bin 179583 -> 0 bytes doc/design/img/hermes-buffering.graphml | 906 ------ doc/design/img/hermes-buffering.png | Bin 72804 -> 0 bytes doc/design/img/hermes-node.png | Bin 124515 -> 0 bytes doc/design/img/hermes-stack.png | Bin 297523 -> 0 bytes doc/design/img/mmap.graphml | 553 ---- doc/design/img/mmap.png | Bin 67976 -> 0 bytes doc/design/img/modules.graphml | 231 -- doc/design/img/modules.png | Bin 26821 -> 0 bytes doc/design/img/perf/dev-bw.png | Bin 7054 -> 0 bytes doc/design/img/perf/tiering-1n.png | Bin 7179 -> 0 bytes doc/design/img/primitives.png | Bin 43788 -> 0 bytes doc/design/img/quality.graphml | 212 -- doc/design/img/quality.png | Bin 38953 -> 0 bytes doc/design/img/sequence.graphml | 1267 -------- doc/design/img/sequence.png | Bin 64902 -> 0 bytes doc/design/img/state-machine.graphml | 216 -- doc/design/img/state-machine.png | Bin 38789 -> 0 bytes doc/design/img/user-core.graphml | 415 --- doc/design/img/user-core.png | Bin 60171 -> 0 bytes doc/design/references.bib | 59 - docker/ctest.Dockerfile | 24 - docker/deps.Dockerfile | 76 - docker/dev.Dockerfile | 23 - docker/user.Dockerfile | 21 - hermes_shm/CMake/HermesShmConfig.cmake | 54 - hermes_shm/CMake/UseDoxygenDoc.cmake | 33 - hermes_shm/CMakeLists.txt | 209 -- hermes_shm/COPYING | 52 - hermes_shm/CPPLINT.cfg | 2 - hermes_shm/benchmark/CMakeLists.txt | 6 - hermes_shm/benchmark/allocator/CMakeLists.txt | 19 - hermes_shm/benchmark/allocator/allocator.cc | 272 -- hermes_shm/benchmark/allocator/test_init.h | 46 - .../benchmark/data_structure/CMakeLists.txt | 28 - hermes_shm/benchmark/data_structure/atomic.cc | 248 -- hermes_shm/benchmark/data_structure/list.cc | 269 -- hermes_shm/benchmark/data_structure/lock.cc | 162 - hermes_shm/benchmark/data_structure/queue.cc | 261 -- hermes_shm/benchmark/data_structure/ref.cc | 41 - hermes_shm/benchmark/data_structure/string.cc | 120 - .../benchmark/data_structure/test_init.cc | 46 - .../benchmark/data_structure/test_init.h | 183 -- .../benchmark/data_structure/unordered_map.cc | 270 -- hermes_shm/benchmark/data_structure/vector.cc | 344 --- hermes_shm/benchmark/lock/CMakeLists.txt | 18 - hermes_shm/benchmark/lock/benchmark_mutex.cc | 66 - hermes_shm/benchmark/lock/test_init.h | 20 - .../code_generators/cpp_macro_generator.py | 17 - hermes_shm/doc/Doxyfile.in | 2685 ----------------- .../data_structure_singleton_macros.h | 26 - .../include/hermes_shm/constants/macros.h | 67 - .../data_structures/containers/charbuf.h | 267 -- .../data_structures/containers/converters.h | 57 - .../data_structures/containers/functional.h | 31 - .../data_structures/containers/tuple_base.h | 238 -- .../data_structures/data_structure.h | 38 - .../hermes_shm/data_structures/ipc/_queue.h | 75 - .../ipc/internal/shm_archive.h | 143 - .../ipc/internal/shm_container.h | 79 - .../ipc/internal/shm_container_macro.h | 66 - .../ipc/internal/shm_internal.h | 24 - .../data_structures/ipc/internal/shm_macros.h | 33 - .../ipc/internal/shm_null_container.h | 127 - .../ipc/internal/shm_smart_ptr.h | 57 - .../template/shm_container_base_example.h | 133 - .../template/shm_container_base_template.h | 61 - .../hermes_shm/data_structures/ipc/iqueue.h | 352 --- .../hermes_shm/data_structures/ipc/list.h | 489 --- .../data_structures/ipc/mpsc_queue.h | 215 -- .../hermes_shm/data_structures/ipc/pair.h | 211 -- .../hermes_shm/data_structures/ipc/slist.h | 506 ---- .../data_structures/ipc/split_ticket_queue.h | 188 -- .../data_structures/ipc/spsc_queue.h | 201 -- .../hermes_shm/data_structures/ipc/string.h | 357 --- .../data_structures/ipc/ticket_queue.h | 164 - .../data_structures/ipc/ticket_stack.h | 166 - .../data_structures/ipc/unordered_map.h | 537 ---- .../hermes_shm/data_structures/ipc/vector.h | 688 ----- .../data_structures/numa_aware/numa_list.h | 169 -- .../data_structures/numa_aware/numa_vector.h | 18 - .../serialization/shm_serialize.h | 111 - .../data_structures/serialization/thallium.h | 222 -- .../smart_ptr/smart_ptr_base.h | 307 -- hermes_shm/include/hermes_shm/hermes_shm.h | 38 - .../hermes_shm/introspect/system_info.h | 48 - .../hermes_shm/memory/allocator/allocator.h | 523 ---- .../memory/allocator/allocator_factory.h | 100 - .../hermes_shm/memory/allocator/heap.h | 50 - .../memory/allocator/malloc_allocator.h | 100 - .../hermes_shm/memory/allocator/mp_page.h | 41 - .../allocator/scalable_page_allocator.h | 218 -- .../memory/allocator/stack_allocator.h | 107 - .../hermes_shm/memory/backend/array_backend.h | 65 - .../memory/backend/memory_backend.h | 90 - .../memory/backend/memory_backend_factory.h | 104 - .../hermes_shm/memory/backend/null_backend.h | 81 - .../hermes_shm/memory/backend/posix_mmap.h | 112 - .../memory/backend/posix_shm_mmap.h | 136 - hermes_shm/include/hermes_shm/memory/memory.h | 404 --- .../hermes_shm/memory/memory_manager.h | 209 -- .../hermes_shm/memory/memory_registry.h | 144 - hermes_shm/include/hermes_shm/thread/lock.h | 21 - .../include/hermes_shm/thread/lock/mutex.h | 114 - .../include/hermes_shm/thread/lock/rwlock.h | 230 -- .../hermes_shm/thread/thread_model/argobots.h | 55 - .../hermes_shm/thread/thread_model/pthread.h | 51 - .../thread/thread_model/thread_model.h | 52 - .../thread_model/thread_model_factory.h | 28 - .../hermes_shm/thread/thread_model_manager.h | 76 - hermes_shm/include/hermes_shm/types/argpack.h | 279 -- hermes_shm/include/hermes_shm/types/atomic.h | 253 -- .../include/hermes_shm/types/bitfield.h | 169 -- .../include/hermes_shm/types/real_number.h | 98 - .../include/hermes_shm/util/auto_trace.h | 94 - .../include/hermes_shm/util/config_parse.h | 241 -- hermes_shm/include/hermes_shm/util/error.h | 64 - hermes_shm/include/hermes_shm/util/errors.h | 67 - .../include/hermes_shm/util/formatter.h | 85 - hermes_shm/include/hermes_shm/util/logging.h | 221 -- .../include/hermes_shm/util/partitioner.h | 156 - .../include/hermes_shm/util/singleton.h | 21 - .../util/singleton/_easy_global_singleton.h | 40 - .../util/singleton/_easy_singleton.h | 57 - .../util/singleton/_global_singleton.h | 56 - .../hermes_shm/util/singleton/_singleton.h | 69 - hermes_shm/include/hermes_shm/util/timer.h | 96 - .../include/hermes_shm/util/type_switch.h | 51 - hermes_shm/readme.md | 7 - hermes_shm/scripts/ci/packages.yaml | 38 - .../packages/hermes_shm/__init__.py | 0 hermes_shm/scripts/hermes_shm/repo.yaml | 2 - hermes_shm/scripts/lint.sh | 14 - hermes_shm/scripts/preamble.py | 93 - hermes_shm/scripts/singleton_generator.py | 98 - hermes_shm/src/CMakeLists.txt | 53 - hermes_shm/src/data_structure_singleton.cc | 25 - hermes_shm/src/memory/malloc_allocator.cc | 82 - hermes_shm/src/memory/memory_intercept.cc | 94 - hermes_shm/src/memory/memory_intercept.h | 16 - hermes_shm/src/memory/memory_manager.cc | 29 - hermes_shm/src/memory/memory_registry.cc | 30 - .../src/memory/scalable_page_allocator.cc | 294 -- hermes_shm/src/memory/stack_allocator.cc | 83 - hermes_shm/src/thread_factory.cc | 44 - hermes_shm/src/thread_model_manager.cc | 49 - hermes_shm/test/CMakeLists.txt | 6 - hermes_shm/test/unit/CMakeLists.txt | 14 - .../test/unit/allocators/CMakeLists.txt | 58 - hermes_shm/test/unit/allocators/allocator.cc | 157 - .../test/unit/allocators/allocator_thread.cc | 48 - hermes_shm/test/unit/allocators/test_init.h | 56 - .../test/unit/allocators_mpi/CMakeLists.txt | 47 - .../test/unit/allocators_mpi/allocator_mpi.cc | 75 - .../test/unit/allocators_mpi/test_init.cc | 33 - .../test/unit/allocators_mpi/test_init.h | 53 - hermes_shm/test/unit/backend/CMakeLists.txt | 46 - hermes_shm/test/unit/backend/backend.cc | 30 - .../test/unit/backend/memory_manager.cc | 80 - hermes_shm/test/unit/backend/memory_slots.cc | 60 - hermes_shm/test/unit/basic_test.h | 72 - .../test/unit/data_structures/CMakeLists.txt | 9 - .../data_structures/containers/CMakeLists.txt | 102 - .../data_structures/containers/charbuf.cc | 222 -- .../unit/data_structures/containers/iqueue.cc | 42 - .../unit/data_structures/containers/iqueue.h | 108 - .../unit/data_structures/containers/list.cc | 63 - .../unit/data_structures/containers/list.h | 192 -- .../data_structures/containers/manual_ptr.cc | 55 - .../data_structures/containers/mpsc_queue.cc | 50 - .../unit/data_structures/containers/pair.cc | 95 - .../unit/data_structures/containers/queue.h | 149 - .../unit/data_structures/containers/slist.cc | 59 - .../data_structures/containers/smart_ptr.h | 86 - .../data_structures/containers/spsc_queue.cc | 35 - .../unit/data_structures/containers/string.cc | 78 - .../data_structures/containers/test_init.cc | 31 - .../data_structures/containers/test_init.h | 52 - .../containers/ticket_queue.cc | 68 - .../unit/data_structures/containers/tuple.cc | 82 - .../data_structures/containers/unique_ptr.cc | 50 - .../containers/unordered_map.cc | 249 -- .../unit/data_structures/containers/vector.cc | 127 - .../unit/data_structures/containers/vector.h | 37 - .../containers_mpi/CMakeLists.txt | 54 - .../containers_mpi/list_vec_mpi.cc | 109 - .../containers_mpi/test_init.cc | 53 - .../containers_mpi/test_init.h | 40 - .../data_structures/serialize/CMakeLists.txt | 7 - .../serialize/shm/CMakeLists.txt | 40 - .../serialize/shm/test_init.cc | 32 - .../data_structures/serialize/shm/test_init.h | 35 - .../data_structures/serialize/shm/test_shm.cc | 64 - .../serialize/thallium/CMakeLists.txt | 54 - .../serialize/thallium/server.cc | 123 - .../serialize/thallium/test_init.cc | 33 - .../serialize/thallium/test_init.h | 85 - .../serialize/thallium/test_thallium.cc | 91 - .../serialize/thallium/test_thallium.sh | 15 - hermes_shm/test/unit/main.cc | 29 - hermes_shm/test/unit/main_mpi.cc | 31 - hermes_shm/test/unit/thread/test_lock.cc | 86 - hermes_shm/test/unit/thread/test_thread.cc | 23 - hermes_shm/test/unit/types/CMakeLists.txt | 42 - hermes_shm/test/unit/types/test_argpack.cc | 180 -- hermes_shm/test/unit/types/test_util.cc | 170 -- include/labstor/api/labstor_client.h | 291 ++ include/labstor/api/labstor_runtime.h | 186 ++ include/labstor/api/manager.h | 85 + .../labstor_task_node_admin_root.template | 29 + .../labstor_task_node_push_root.template | 33 + include/labstor/config/config.h | 73 + .../labstor/config/config_client.h | 40 +- .../labstor/config/config_client_default.h | 5 + include/labstor/config/config_server.h | 96 + .../labstor/config/config_server_default.h | 58 + include/labstor/labstor_constants.h | 29 + include/labstor/labstor_namespace.h | 35 + include/labstor/labstor_types.h | 374 +++ include/labstor/network/local_serialize.h | 76 + include/labstor/network/rpc.h | 226 ++ include/labstor/network/rpc_thallium.h | 315 ++ include/labstor/network/serialize.h | 261 ++ include/labstor/queue_manager/queue.h | 105 + include/labstor/queue_manager/queue_factory.h | 19 + include/labstor/queue_manager/queue_manager.h | 49 + .../queue_manager/queue_manager_client.h | 37 + .../queue_manager/queue_manager_runtime.h | 86 + .../labstor/queue_manager/queues/hshm_queue.h | 288 ++ include/labstor/task_registry/task.h | 456 +++ include/labstor/task_registry/task_lib.h | 143 + include/labstor/task_registry/task_registry.h | 299 ++ include/labstor/work_orchestrator/affinity.h | 27 + include/labstor/work_orchestrator/scheduler.h | 53 + .../work_orchestrator/work_orchestrator.h | 126 + include/labstor/work_orchestrator/worker.h | 367 +++ .../scripts => scripts}/ci/coverage.sh | 0 .../ci/external/CMakeLists.txt | 0 .../scripts => scripts}/ci/external/test.cc | 0 .../scripts => scripts}/ci/install_deps.sh | 0 .../scripts => scripts}/ci/install_docs.sh | 4 +- .../scripts => scripts}/ci/install_hshm.sh | 10 +- {ci => scripts/ci}/packages.yaml | 0 {ci => scripts/ci}/py_hermes_ci/bin/run_test | 8 +- .../py_hermes_ci/py_hermes_ci/test_manager.py | 0 {hermes_shm/scripts => scripts}/docs.sh | 0 .../hermes/packages/hermes}/__init__.py | 0 .../hermes/packages/hermes}/package.py | 24 +- scripts/hermes/repo.yaml | 2 + scripts/lint.sh | 6 + src/CMakeLists.txt | 192 +- src/api/bucket.cc | 260 -- src/api/bucket.h | 243 -- src/api/finalize_hermes.cc | 22 - src/api/hermes.cc | 232 -- src/api/hermes.h | 274 -- src/api/hermes_daemon.cc | 41 - src/api/hermes_singleton.cc | 21 - src/binlog.h | 144 - src/borg_io_clients/borg_io_client.h | 33 - src/borg_io_clients/borg_io_client_factory.h | 46 - src/borg_io_clients/borg_posix_client.h | 85 - src/borg_io_clients/borg_ram_client.h | 63 - src/buffer_organizer.cc | 539 ---- src/buffer_organizer.h | 289 -- src/buffer_pool.cc | 444 --- src/buffer_pool.h | 321 -- src/config_client.cc | 85 +- src/config_server.cc | 203 +- src/config_server.h | 325 -- src/data_placement_engine.cc | 75 - src/data_structures.h | 43 - src/decorator.h | 38 - src/dpe/minimize_io_time.cc | 74 - src/dpe/minimize_io_time.h | 34 - src/dpe/random.cc | 60 - src/dpe/round_robin.cc | 57 - src/hermes_types.cc | 49 - src/hermes_types.h | 567 ---- src/hermes_version.h.in | 21 - src/labstor_client.cc | 9 + src/labstor_runtime.cc | 9 + src/labstor_start_runtime.cc | 13 + src/labstor_stop_runtime.cc | 10 + src/metadata_manager.cc | 953 ------ src/metadata_manager.h | 535 ---- src/metadata_types.cc | 41 - src/metadata_types.h | 395 --- src/prefetcher.cc | 98 - src/prefetcher.h | 57 - src/prefetcher/apriori_prefetcher.cc | 105 - src/prefetcher/apriori_prefetcher.h | 53 - src/prefetcher_factory.h | 50 - src/rpc.cc | 210 -- src/rpc.h | 201 -- src/rpc_factory.h.in | 19 - src/rpc_thallium.cc | 134 - src/rpc_thallium.h | 170 -- src/rpc_thallium_defs.cc | 295 -- src/rpc_thallium_serialization.h | 166 - src/statuses.h | 59 - src/thread_manager.h | 65 - src/thread_pool.h | 104 - src/trait_manager.cc | 87 - src/trait_manager.h | 213 -- src/utils.h | 33 - src/worker.cc | 95 + tasks/CMakeLists.txt | 8 + tasks/bdev/CMakeLists.txt | 9 + tasks/bdev/include/bdev/bdev.h | 165 + tasks/bdev/include/bdev/bdev_lib_exec.h | 306 ++ tasks/bdev/include/bdev/bdev_methods.h | 14 + tasks/bdev/include/bdev/bdev_methods.yaml | 7 + tasks/bdev/include/bdev/bdev_namespace.h | 26 + tasks/bdev/include/bdev/bdev_tasks.h | 319 ++ tasks/hermes/CMakeLists.txt | 10 + .../hermes/config}/hermes_client_default.yaml | 0 .../hermes/config}/hermes_server_default.yaml | 0 .../hermes/include/hermes}/adapter_types.h | 1 - tasks/hermes/include/hermes/bucket.h | 428 +++ {src => tasks/hermes/include/hermes}/config.h | 16 +- .../hermes/include/hermes}/config_client.h | 82 +- .../include/hermes}/config_client_default.h | 8 +- tasks/hermes/include/hermes/config_manager.h | 80 + tasks/hermes/include/hermes/config_server.h | 354 +++ .../include/hermes}/config_server_default.h | 8 +- .../hermes/include/hermes/dpe/dpe.h | 29 +- .../hermes/include/hermes/dpe/dpe_factory.h | 17 +- .../include/hermes/dpe/minimize_io_time.h | 84 + .../hermes/include/hermes}/dpe/random.h | 43 +- tasks/hermes/include/hermes/dpe/round_robin.h | 72 + tasks/hermes/include/hermes/hermes.h | 47 + tasks/hermes/include/hermes/hermes_types.h | 442 +++ tasks/hermes/include/hermes/slab_allocator.h | 150 + {src => tasks/hermes/include/hermes}/status.h | 2 +- tasks/hermes/include/hermes/statuses.h | 21 + tasks/hermes/src/CMakeLists.txt | 54 + tasks/hermes/src/config_manager.cc | 7 + tasks/hermes_adapters/CMakeLists.txt | 23 + .../filesystem/CMakeLists.txt | 22 +- .../hermes_adapters}/filesystem/filesystem.cc | 398 +-- .../hermes_adapters}/filesystem/filesystem.h | 74 +- .../filesystem/filesystem_io_client.h | 27 +- .../filesystem/filesystem_mdm.cc | 2 +- .../filesystem/filesystem_mdm.h | 13 +- .../filesystem/filesystem_mdm_singleton.cc | 0 .../hermes_adapters}/adapter_constants.h | 0 .../include/hermes_adapters/adapter_types.h | 103 + .../include/hermes_adapters/hermes_adapters.h | 72 + .../hermes_adapters_lib_exec.h | 146 + .../hermes_adapters/hermes_adapters_methods.h | 9 + .../hermes_adapters_methods.yaml | 1 + .../hermes_adapters/hermes_adapters_tasks.h | 125 + .../include/hermes_adapters}/interceptor.h | 4 +- .../hermes_adapters}/mapper/abstract_mapper.h | 18 +- .../hermes_adapters}/mapper/balanced_mapper.h | 17 +- .../hermes_adapters}/mapper/mapper_factory.h | 3 +- .../include/hermes_adapters}/real_api.h | 0 .../hermes_adapters}/mpiio/CMakeLists.txt | 12 +- .../hermes_adapters}/mpiio/mpiio_api.cc | 5 +- .../hermes_adapters}/mpiio/mpiio_api.h | 2 +- .../hermes_adapters}/mpiio/mpiio_fs_api.h | 8 +- .../hermes_adapters}/mpiio/mpiio_io_client.cc | 2 - .../hermes_adapters}/mpiio/mpiio_io_client.h | 21 +- .../hermes_adapters}/posix/CMakeLists.txt | 20 +- .../hermes_adapters}/posix/posix_api.cc | 19 +- .../hermes_adapters}/posix/posix_api.h | 2 +- .../hermes_adapters}/posix/posix_fs_api.h | 4 +- .../hermes_adapters}/posix/posix_io_client.cc | 2 - .../hermes_adapters}/posix/posix_io_client.h | 21 +- tasks/hermes_adapters/src/CMakeLists.txt | 54 + tasks/hermes_adapters/src/hermes_adapters.cc | 39 + .../hermes_adapters}/stdio/CMakeLists.txt | 21 +- .../hermes_adapters}/stdio/stdio_api.cc | 15 +- .../hermes_adapters}/stdio/stdio_api.h | 2 +- .../hermes_adapters}/stdio/stdio_fs_api.h | 6 +- .../hermes_adapters}/stdio/stdio_io_client.cc | 2 - .../hermes_adapters}/stdio/stdio_io_client.h | 20 +- .../hermes_adapters}/vfd/CMakeLists.txt | 99 +- .../hermes_adapters}/vfd/H5FDhermes.cc | 2 +- .../hermes_adapters}/vfd/H5FDhermes.h | 0 .../hermes_adapters}/vfd/README.md | 6 +- tasks/hermes_blob_mdm/CMakeLists.txt | 10 + .../include/hermes_blob_mdm/hermes_blob_mdm.h | 428 +++ .../hermes_blob_mdm_lib_exec.h | 594 ++++ .../hermes_blob_mdm/hermes_blob_mdm_methods.h | 23 + .../hermes_blob_mdm_methods.yaml | 17 + .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 1088 +++++++ tasks/hermes_blob_mdm/src/CMakeLists.txt | 54 + tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 652 ++++ tasks/hermes_bucket_mdm/CMakeLists.txt | 10 + .../hermes_bucket_mdm/hermes_bucket_mdm.h | 298 ++ .../hermes_bucket_mdm_lib_exec.h | 530 ++++ .../hermes_bucket_mdm_methods.h | 21 + .../hermes_bucket_mdm_methods.yaml | 16 + .../hermes_bucket_mdm_tasks.h | 849 ++++++ tasks/hermes_bucket_mdm/src/CMakeLists.txt | 54 + .../src/hermes_bucket_mdm.cc | 344 +++ tasks/hermes_mdm/CMakeLists.txt | 10 + .../include/hermes_mdm/hermes_mdm.h | 43 + .../include/hermes_mdm/hermes_mdm_lib_exec.h | 114 + .../include/hermes_mdm/hermes_mdm_methods.h | 8 + .../hermes_mdm/hermes_mdm_methods.yaml | 0 .../include/hermes_mdm/hermes_mdm_tasks.h | 95 + tasks/hermes_mdm/src/CMakeLists.txt | 54 + tasks/hermes_mdm/src/hermes_mdm.cc | 46 + tasks/posix_bdev/CMakeLists.txt | 10 + .../include/posix_bdev/posix_bdev.h | 20 + tasks/posix_bdev/src/CMakeLists.txt | 54 + tasks/posix_bdev/src/posix_bdev.cc | 91 + tasks/ram_bdev/CMakeLists.txt | 10 + tasks/ram_bdev/include/ram_bdev/ram_bdev.h | 21 + tasks/ram_bdev/src/CMakeLists.txt | 54 + tasks/ram_bdev/src/ram_bdev.cc | 67 + tasks_required/CMakeLists.txt | 7 + tasks_required/TASK_NAME/CMakeLists.txt | 10 + .../TASK_NAME/include/TASK_NAME/TASK_NAME.h | 72 + .../include/TASK_NAME/TASK_NAME_lib_exec.h | 146 + .../include/TASK_NAME/TASK_NAME_methods.h | 9 + .../include/TASK_NAME/TASK_NAME_methods.yaml | 1 + .../include/TASK_NAME/TASK_NAME_tasks.h | 125 + tasks_required/TASK_NAME/src/CMakeLists.txt | 54 + tasks_required/TASK_NAME/src/TASK_NAME.cc | 33 + tasks_required/labstor_admin/CMakeLists.txt | 10 + .../include/labstor_admin/labstor_admin.h | 225 ++ .../labstor_admin/labstor_admin_lib_exec.h | 338 +++ .../labstor_admin/labstor_admin_methods.h | 17 + .../labstor_admin/labstor_admin_methods.yaml | 9 + .../labstor_admin/labstor_admin_tasks.h | 410 +++ .../labstor_admin/src/CMakeLists.txt | 54 + .../labstor_admin/src/labstor_admin.cc | 179 ++ tasks_required/proc_queue/CMakeLists.txt | 10 + .../include/proc_queue/proc_queue.h | 87 + .../include/proc_queue/proc_queue_lib_exec.h | 146 + .../include/proc_queue/proc_queue_methods.h | 9 + .../proc_queue/proc_queue_methods.yaml | 1 + .../include/proc_queue/proc_queue_tasks.h | 139 + tasks_required/proc_queue/src/CMakeLists.txt | 54 + tasks_required/proc_queue/src/proc_queue.cc | 52 + tasks_required/remote_queue/CMakeLists.txt | 14 + .../include/remote_queue/remote_queue.h | 95 + .../remote_queue/remote_queue_lib_exec.h | 146 + .../remote_queue/remote_queue_methods.h | 9 + .../remote_queue/remote_queue_methods.yaml | 1 + .../include/remote_queue/remote_queue_tasks.h | 165 + .../remote_queue/src/CMakeLists.txt | 57 + .../remote_queue/src/remote_queue.cc | 378 +++ tasks_required/small_message/CMakeLists.txt | 10 + .../include/small_message/small_message.h | 99 + .../small_message/small_message_lib_exec.h | 210 ++ .../small_message/small_message_methods.h | 11 + .../small_message/small_message_methods.yaml | 3 + .../small_message/small_message_tasks.h | 232 ++ .../small_message/src/CMakeLists.txt | 54 + .../small_message/src/small_message.cc | 51 + .../worch_proc_round_robin/CMakeLists.txt | 10 + .../worch_proc_round_robin.h | 43 + .../worch_proc_round_robin_lib_exec.h | 146 + .../worch_proc_round_robin_methods.h | 9 + .../worch_proc_round_robin_methods.yaml | 1 + .../worch_proc_round_robin_tasks.h | 74 + .../worch_proc_round_robin/src/CMakeLists.txt | 54 + .../src/worch_proc_round_robin.cc | 34 + .../worch_queue_round_robin/CMakeLists.txt | 10 + .../worch_queue_round_robin.h | 43 + .../worch_queue_round_robin_lib_exec.h | 146 + .../worch_queue_round_robin_methods.h | 9 + .../worch_queue_round_robin_methods.yaml | 1 + .../worch_queue_round_robin_tasks.h | 78 + .../src/CMakeLists.txt | 54 + .../src/worch_queue_round_robin.cc | 59 + test/CMakeLists.txt | 65 +- test/data/apriori_schema.yaml | 21 - test/data/asan.supp | 10 - test/data/hermes_client.yaml | 9 - test/data/hermes_client_specific.yaml | 9 - test/data/hermes_server.yaml | 130 - test/data/hermes_server_ares.yaml | 130 - test/data/hermes_server_bpm.yaml | 130 - test/data/hermes_server_prefetch.yaml | 138 - test/hostfile.txt | 4 - test/test_binlog.cc | 89 - test/test_bucket.cc | 212 -- test/test_buffer_pool.cc | 87 - test/test_config.cc | 91 - test/test_multinode_put_get.cc | 75 - test/test_tag.cc | 66 - test/test_trait.cc | 60 - test/test_utils.h | 43 - test/tests.py | 60 - test/unit/CMakeLists.txt | 10 + test/{ => unit}/basic_test.h | 36 +- .../unit/hermes}/CMakeLists.txt | 44 +- .../hermes/config/hermes_server.yaml} | 80 +- test/unit/hermes/config/labstor_server.yaml | 54 + test/unit/hermes/test_bucket.cc | 326 ++ .../unit/hermes}/test_init.cc | 6 +- .../unit/hermes/test_init.h | 7 +- .../unit/hermes_adapters}/CMakeLists.txt | 2 + .../hermes_adapters}/adapter_test_utils.h | 2 +- .../unit/hermes_adapters}/catch_config.h | 0 .../hermes_adapters}/mpiio/CMakeLists.txt | 4 +- .../mpiio/mpiio_adapter_basic_test.cpp | 0 .../mpiio/mpiio_adapter_test.cpp | 22 +- .../unit/hermes_adapters}/mpiio/parallel.cc | 0 .../unit/hermes_adapters}/mpiio/tests.py | 0 .../hermes_adapters}/posix/CMakeLists.txt | 11 +- .../hermes_adapters}/posix/hdf5_write_read.py | 0 .../posix/posix_adapter_basic_test.cpp | 0 .../posix/posix_adapter_mpi_test.cpp | 30 +- .../posix/posix_adapter_rs_test.cpp | 0 .../posix/posix_adapter_shared_test.cpp | 0 .../posix/posix_adapter_test.cpp | 20 +- .../posix/posix_simple_io_mpi.cc | 2 +- .../posix/posix_simple_io_omp.cc | 9 +- .../unit/hermes_adapters}/posix/tests.py | 0 .../hermes_adapters}/stdio/CMakeLists.txt | 8 +- .../stdio/stdio_adapter_basic_test.cpp | 0 .../stdio/stdio_adapter_func_test.cpp | 0 .../stdio_adapter_low_buffer_space_test.cpp | 22 +- .../stdio/stdio_adapter_mapper_test.cpp | 10 +- .../stdio/stdio_adapter_mode_test.cpp | 36 +- .../stdio/stdio_adapter_mpi_test.cpp | 30 +- .../stdio/stdio_adapter_rs_test.cpp | 0 .../stdio/stdio_adapter_shared_test.cpp | 0 .../stdio/stdio_adapter_test.cpp | 18 +- .../unit/hermes_adapters}/stdio/tests.py | 0 .../unit/hermes_adapters}/vfd/CMakeLists.txt | 0 .../vfd/hermes_vfd_basic_test.cc | 2 +- .../hermes_adapters}/vfd/hermes_vfd_test.cc | 18 +- .../unit/hermes_adapters}/vfd/tests.py | 0 test/unit/ipc/CMakeLists.txt | 46 + test/unit/ipc/test_finalize.cc | 11 + .../allocators => test/unit/ipc}/test_init.cc | 10 +- .../test_init.cc => test/unit/ipc/test_init.h | 7 +- test/unit/ipc/test_ipc.cc | 111 + test/unit/ipc/test_serialize.cc | 63 + test/{ => unit}/main.cc | 2 +- test/{ => unit}/main_mpi.cc | 0 traits/CMakeLists.txt | 20 - traits/example/CMakeLists.txt | 41 - traits/example/example_trait.cc | 25 - traits/example/example_trait.h | 50 - traits/prefetcher/CMakeLists.txt | 41 - traits/prefetcher/prefetcher_header.h | 37 - traits/prefetcher/prefetcher_trait.cc | 24 - traits/prefetcher/prefetcher_trait.h | 41 - wrapper/CMakeLists.txt | 26 - wrapper/c/CMakeLists.txt | 41 - wrapper/c/c_wrapper.cc | 78 - wrapper/c/c_wrapper.h | 41 - wrapper/java/CMakeLists.txt | 73 - wrapper/java/build.gradle | 47 - .../java/gradle/wrapper/gradle-wrapper.jar | Bin 55627 -> 0 bytes .../gradle/wrapper/gradle-wrapper.properties | 5 - wrapper/java/gradlew | 172 -- wrapper/java/gradlew.bat | 84 - wrapper/java/settings.gradle | 18 - .../java/src/hermes/cpp/hermes_java_wrapper.h | 177 -- wrapper/java/src/hermes/cpp/src_main_Blob.cc | 22 - wrapper/java/src/hermes/cpp/src_main_Blob.h | 22 - .../java/src/hermes/cpp/src_main_Bucket.cc | 167 - wrapper/java/src/hermes/cpp/src_main_Bucket.h | 93 - .../java/src/hermes/cpp/src_main_Hermes.cc | 50 - wrapper/java/src/hermes/cpp/src_main_Hermes.h | 29 - wrapper/java/src/hermes/java/Blob.java | 58 - wrapper/java/src/hermes/java/Bucket.java | 68 - wrapper/java/src/hermes/java/Hermes.java | 20 - wrapper/java/src/hermes/java/MdLockType.java | 7 - wrapper/java/src/hermes/java/UniqueId.java | 17 - wrapper/java/src/test/java/HermesJniTest.java | 40 - wrapper/java/tests.py | 21 - 704 files changed, 20339 insertions(+), 49940 deletions(-) delete mode 100644 .travis.yml delete mode 100644 CMake/HermesConfig.cmake create mode 100644 CMake/LabstorConfig.cmake delete mode 100644 CMake/UseDoxygenDoc.cmake delete mode 100644 COPYING delete mode 100644 adapter/CMakeLists.txt delete mode 100644 adapter/kvstore/CMakeLists.txt delete mode 100644 adapter/kvstore/README.md delete mode 100644 adapter/kvstore/java/build.gradle delete mode 100644 adapter/kvstore/java/gradle/wrapper/gradle-wrapper.jar delete mode 100644 adapter/kvstore/java/gradle/wrapper/gradle-wrapper.properties delete mode 100755 adapter/kvstore/java/gradlew delete mode 100644 adapter/kvstore/java/gradlew.bat delete mode 100644 adapter/kvstore/java/settings.gradle delete mode 100644 adapter/kvstore/java/src/kvstore/java/KVStore.java delete mode 100644 adapter/kvstore/java/src/kvstore/java/KVTable.java delete mode 100644 adapter/kvstore/java/src/test/java/KvstoreJniTest.java delete mode 100644 adapter/kvstore/kvstore.cc delete mode 100644 adapter/kvstore/kvstore.h delete mode 100644 adapter/kvstore/tests.py delete mode 100644 adapter/mapper/balanced_mapper.cc create mode 100644 benchmark/CMakeLists.txt rename benchmarks/api_bench.cc => benchmark/hermes_api_bench.cc (58%) rename {hermes_shm/benchmark/lock => benchmark}/test_init.cc (91%) rename hermes_shm/test/unit/backend/test_init.cc => benchmark/test_init.h (83%) create mode 100644 benchmark/test_latency.cc create mode 100644 benchmark/test_zmq.cc delete mode 100644 benchmarks/CMakeLists.txt delete mode 100644 benchmarks/memcpy_bench.cc delete mode 100644 benchmarks/reorganize.cc delete mode 100755 ci/build_hermes.sh delete mode 100644 ci/cluster/Dockerfile delete mode 100755 ci/cluster/cluster_down.sh delete mode 100755 ci/cluster/cluster_test.sh delete mode 100755 ci/cluster/cluster_up.sh delete mode 100644 ci/cluster/cluster_utils.sh delete mode 100644 ci/cluster/docker-compose.yml delete mode 100755 ci/cluster/multi_node_ci_test.sh delete mode 100644 ci/coverage.sh delete mode 100644 ci/external/CMakeLists.txt delete mode 100644 ci/external/test_install.cc delete mode 100644 ci/hermes/packages/hermes/package.py delete mode 100644 ci/hermes/repo.yaml delete mode 100755 ci/install_deps.sh delete mode 100755 ci/install_docs.sh delete mode 100755 ci/install_hermes.sh delete mode 100644 ci/jarvis-util/.coveragerc delete mode 100644 ci/jarvis-util/.gitignore delete mode 100644 ci/jarvis-util/LICENSE delete mode 100644 ci/jarvis-util/README.md delete mode 100755 ci/jarvis-util/bin/jarvis-imports delete mode 100644 ci/jarvis-util/ci/cluster/Dockerfile delete mode 100644 ci/jarvis-util/ci/cluster/docker-compose.yml delete mode 100644 ci/jarvis-util/ci/install_deps.sh delete mode 100644 ci/jarvis-util/ci/install_jarvis.sh delete mode 100644 ci/jarvis-util/ci/install_spack.sh delete mode 100644 ci/jarvis-util/ci/lint.sh delete mode 100644 ci/jarvis-util/ci/run_tests.sh delete mode 100644 ci/jarvis-util/jarvis_util/__init__.py delete mode 100644 ci/jarvis-util/jarvis_util/introspect/system_info.py delete mode 100644 ci/jarvis-util/jarvis_util/jutil_manager.py delete mode 100644 ci/jarvis-util/jarvis_util/serialize/ini_file.py delete mode 100644 ci/jarvis-util/jarvis_util/serialize/pickle.py delete mode 100644 ci/jarvis-util/jarvis_util/serialize/serializer.py delete mode 100644 ci/jarvis-util/jarvis_util/serialize/text_file.py delete mode 100644 ci/jarvis-util/jarvis_util/serialize/yaml_file.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/__init__.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/exec.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/exec_info.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/filesystem.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/local_exec.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/mpi_exec.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/process.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/pscp.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/pssh_exec.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/scp.py delete mode 100644 ci/jarvis-util/jarvis_util/shell/ssh_exec.py delete mode 100644 ci/jarvis-util/jarvis_util/util/__init__.py delete mode 100644 ci/jarvis-util/jarvis_util/util/argparse.py delete mode 100644 ci/jarvis-util/jarvis_util/util/expand_env.py delete mode 100644 ci/jarvis-util/jarvis_util/util/hostfile.py delete mode 100644 ci/jarvis-util/jarvis_util/util/import_all.py delete mode 100644 ci/jarvis-util/jarvis_util/util/import_mod.py delete mode 100644 ci/jarvis-util/jarvis_util/util/naming.py delete mode 100644 ci/jarvis-util/jarvis_util/util/size_conv.py delete mode 100644 ci/jarvis-util/pylintrc delete mode 100644 ci/jarvis-util/requirements.txt delete mode 100644 ci/jarvis-util/setup.py delete mode 100644 ci/jarvis-util/test/unit/argparse_main.py delete mode 100644 ci/jarvis-util/test/unit/print5s.py delete mode 100644 ci/jarvis-util/test/unit/printNone.py delete mode 100644 ci/jarvis-util/test/unit/test_argparse.py delete mode 100644 ci/jarvis-util/test/unit/test_hostfile.py delete mode 100644 ci/jarvis-util/test/unit/test_hostfile.txt delete mode 100644 ci/jarvis-util/test/unit/test_local_exec.py delete mode 100644 ci/jarvis-util/test/unit/test_system_info.py delete mode 100644 ci/lint.sh delete mode 100644 ci/test_hermes.sh delete mode 100644 code_generators/hermes_config.py rename {code_generators/code_generators/hermes_config => codegen/codegen/labstor_config}/generator.py (59%) rename {code_generators/code_generators => codegen/codegen}/util/conv.py (100%) rename {code_generators/code_generators => codegen/codegen}/util/naming.py (100%) rename {code_generators/code_generators => codegen/codegen}/util/paths.py (84%) create mode 100755 codegen/hermes_config create mode 100755 codegen/labstor_config create mode 100755 codegen/make_macro create mode 100755 codegen/make_task rename {code_generators => codegen}/preamble.py (98%) create mode 100755 codegen/refresh_methods create mode 100755 codegen/update_task create mode 100644 config/labstor_client_default.yaml create mode 100644 config/labstor_server_default.yaml delete mode 100644 data_stager/CMakeLists.txt delete mode 100644 data_stager/data_stager.h delete mode 100644 data_stager/data_stager_factory.h delete mode 100644 data_stager/stage_in.cc delete mode 100644 data_stager/stage_out.cc delete mode 100644 data_stager/stagers/posix_stager.cc delete mode 100644 data_stager/stagers/posix_stager.h delete mode 100644 data_stager/test/CMakeLists.txt delete mode 100644 data_stager/test/tests.py delete mode 100644 doc/Doxyfile.in delete mode 100644 doc/design/hermes-design.org delete mode 100644 doc/design/img/Hermes IC.graphml delete mode 100644 doc/design/img/Hermes IC.png delete mode 100644 doc/design/img/adapter.graphml delete mode 100644 doc/design/img/adapter.png delete mode 100644 doc/design/img/balanced-mapping.png delete mode 100644 doc/design/img/file2object.png delete mode 100644 doc/design/img/hermes-bucket-pool.png delete mode 100644 doc/design/img/hermes-buckets.png delete mode 100644 doc/design/img/hermes-buffering.graphml delete mode 100644 doc/design/img/hermes-buffering.png delete mode 100644 doc/design/img/hermes-node.png delete mode 100644 doc/design/img/hermes-stack.png delete mode 100644 doc/design/img/mmap.graphml delete mode 100644 doc/design/img/mmap.png delete mode 100644 doc/design/img/modules.graphml delete mode 100644 doc/design/img/modules.png delete mode 100644 doc/design/img/perf/dev-bw.png delete mode 100644 doc/design/img/perf/tiering-1n.png delete mode 100644 doc/design/img/primitives.png delete mode 100644 doc/design/img/quality.graphml delete mode 100644 doc/design/img/quality.png delete mode 100644 doc/design/img/sequence.graphml delete mode 100644 doc/design/img/sequence.png delete mode 100644 doc/design/img/state-machine.graphml delete mode 100644 doc/design/img/state-machine.png delete mode 100644 doc/design/img/user-core.graphml delete mode 100644 doc/design/img/user-core.png delete mode 100644 doc/design/references.bib delete mode 100644 docker/ctest.Dockerfile delete mode 100644 docker/deps.Dockerfile delete mode 100644 docker/dev.Dockerfile delete mode 100644 docker/user.Dockerfile delete mode 100644 hermes_shm/CMake/HermesShmConfig.cmake delete mode 100644 hermes_shm/CMake/UseDoxygenDoc.cmake delete mode 100644 hermes_shm/CMakeLists.txt delete mode 100644 hermes_shm/COPYING delete mode 100644 hermes_shm/CPPLINT.cfg delete mode 100644 hermes_shm/benchmark/CMakeLists.txt delete mode 100644 hermes_shm/benchmark/allocator/CMakeLists.txt delete mode 100644 hermes_shm/benchmark/allocator/allocator.cc delete mode 100644 hermes_shm/benchmark/allocator/test_init.h delete mode 100644 hermes_shm/benchmark/data_structure/CMakeLists.txt delete mode 100644 hermes_shm/benchmark/data_structure/atomic.cc delete mode 100644 hermes_shm/benchmark/data_structure/list.cc delete mode 100644 hermes_shm/benchmark/data_structure/lock.cc delete mode 100644 hermes_shm/benchmark/data_structure/queue.cc delete mode 100644 hermes_shm/benchmark/data_structure/ref.cc delete mode 100644 hermes_shm/benchmark/data_structure/string.cc delete mode 100644 hermes_shm/benchmark/data_structure/test_init.cc delete mode 100644 hermes_shm/benchmark/data_structure/test_init.h delete mode 100644 hermes_shm/benchmark/data_structure/unordered_map.cc delete mode 100644 hermes_shm/benchmark/data_structure/vector.cc delete mode 100644 hermes_shm/benchmark/lock/CMakeLists.txt delete mode 100644 hermes_shm/benchmark/lock/benchmark_mutex.cc delete mode 100644 hermes_shm/benchmark/lock/test_init.h delete mode 100644 hermes_shm/code_generators/cpp_macro_generator.py delete mode 100644 hermes_shm/doc/Doxyfile.in delete mode 100644 hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h delete mode 100644 hermes_shm/include/hermes_shm/constants/macros.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/containers/converters.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/containers/functional.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/containers/tuple_base.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/data_structure.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container_macro.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_internal.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_macros.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_null_container.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_smart_ptr.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_example.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_template.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/iqueue.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/list.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/pair.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/slist.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/string.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/unordered_map.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/ipc/vector.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_list.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_vector.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/serialization/thallium.h delete mode 100644 hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h delete mode 100644 hermes_shm/include/hermes_shm/hermes_shm.h delete mode 100644 hermes_shm/include/hermes_shm/introspect/system_info.h delete mode 100644 hermes_shm/include/hermes_shm/memory/allocator/allocator.h delete mode 100644 hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h delete mode 100644 hermes_shm/include/hermes_shm/memory/allocator/heap.h delete mode 100644 hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h delete mode 100644 hermes_shm/include/hermes_shm/memory/allocator/mp_page.h delete mode 100644 hermes_shm/include/hermes_shm/memory/allocator/scalable_page_allocator.h delete mode 100644 hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h delete mode 100644 hermes_shm/include/hermes_shm/memory/backend/array_backend.h delete mode 100644 hermes_shm/include/hermes_shm/memory/backend/memory_backend.h delete mode 100644 hermes_shm/include/hermes_shm/memory/backend/memory_backend_factory.h delete mode 100644 hermes_shm/include/hermes_shm/memory/backend/null_backend.h delete mode 100644 hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h delete mode 100644 hermes_shm/include/hermes_shm/memory/backend/posix_shm_mmap.h delete mode 100644 hermes_shm/include/hermes_shm/memory/memory.h delete mode 100644 hermes_shm/include/hermes_shm/memory/memory_manager.h delete mode 100644 hermes_shm/include/hermes_shm/memory/memory_registry.h delete mode 100644 hermes_shm/include/hermes_shm/thread/lock.h delete mode 100644 hermes_shm/include/hermes_shm/thread/lock/mutex.h delete mode 100644 hermes_shm/include/hermes_shm/thread/lock/rwlock.h delete mode 100644 hermes_shm/include/hermes_shm/thread/thread_model/argobots.h delete mode 100644 hermes_shm/include/hermes_shm/thread/thread_model/pthread.h delete mode 100644 hermes_shm/include/hermes_shm/thread/thread_model/thread_model.h delete mode 100644 hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h delete mode 100644 hermes_shm/include/hermes_shm/thread/thread_model_manager.h delete mode 100644 hermes_shm/include/hermes_shm/types/argpack.h delete mode 100644 hermes_shm/include/hermes_shm/types/atomic.h delete mode 100644 hermes_shm/include/hermes_shm/types/bitfield.h delete mode 100644 hermes_shm/include/hermes_shm/types/real_number.h delete mode 100644 hermes_shm/include/hermes_shm/util/auto_trace.h delete mode 100644 hermes_shm/include/hermes_shm/util/config_parse.h delete mode 100644 hermes_shm/include/hermes_shm/util/error.h delete mode 100644 hermes_shm/include/hermes_shm/util/errors.h delete mode 100644 hermes_shm/include/hermes_shm/util/formatter.h delete mode 100644 hermes_shm/include/hermes_shm/util/logging.h delete mode 100644 hermes_shm/include/hermes_shm/util/partitioner.h delete mode 100644 hermes_shm/include/hermes_shm/util/singleton.h delete mode 100644 hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h delete mode 100644 hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h delete mode 100644 hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h delete mode 100644 hermes_shm/include/hermes_shm/util/singleton/_singleton.h delete mode 100644 hermes_shm/include/hermes_shm/util/timer.h delete mode 100644 hermes_shm/include/hermes_shm/util/type_switch.h delete mode 100644 hermes_shm/readme.md delete mode 100644 hermes_shm/scripts/ci/packages.yaml delete mode 100644 hermes_shm/scripts/hermes_shm/packages/hermes_shm/__init__.py delete mode 100644 hermes_shm/scripts/hermes_shm/repo.yaml delete mode 100644 hermes_shm/scripts/lint.sh delete mode 100644 hermes_shm/scripts/preamble.py delete mode 100644 hermes_shm/scripts/singleton_generator.py delete mode 100644 hermes_shm/src/CMakeLists.txt delete mode 100644 hermes_shm/src/data_structure_singleton.cc delete mode 100644 hermes_shm/src/memory/malloc_allocator.cc delete mode 100644 hermes_shm/src/memory/memory_intercept.cc delete mode 100644 hermes_shm/src/memory/memory_intercept.h delete mode 100644 hermes_shm/src/memory/memory_manager.cc delete mode 100644 hermes_shm/src/memory/memory_registry.cc delete mode 100644 hermes_shm/src/memory/scalable_page_allocator.cc delete mode 100644 hermes_shm/src/memory/stack_allocator.cc delete mode 100644 hermes_shm/src/thread_factory.cc delete mode 100644 hermes_shm/src/thread_model_manager.cc delete mode 100644 hermes_shm/test/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/allocators/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/allocators/allocator.cc delete mode 100644 hermes_shm/test/unit/allocators/allocator_thread.cc delete mode 100644 hermes_shm/test/unit/allocators/test_init.h delete mode 100644 hermes_shm/test/unit/allocators_mpi/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc delete mode 100644 hermes_shm/test/unit/allocators_mpi/test_init.cc delete mode 100644 hermes_shm/test/unit/allocators_mpi/test_init.h delete mode 100644 hermes_shm/test/unit/backend/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/backend/backend.cc delete mode 100644 hermes_shm/test/unit/backend/memory_manager.cc delete mode 100644 hermes_shm/test/unit/backend/memory_slots.cc delete mode 100644 hermes_shm/test/unit/basic_test.h delete mode 100644 hermes_shm/test/unit/data_structures/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/data_structures/containers/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/data_structures/containers/charbuf.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/iqueue.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/iqueue.h delete mode 100644 hermes_shm/test/unit/data_structures/containers/list.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/list.h delete mode 100644 hermes_shm/test/unit/data_structures/containers/manual_ptr.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/pair.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/queue.h delete mode 100644 hermes_shm/test/unit/data_structures/containers/slist.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/smart_ptr.h delete mode 100644 hermes_shm/test/unit/data_structures/containers/spsc_queue.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/string.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/test_init.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/test_init.h delete mode 100644 hermes_shm/test/unit/data_structures/containers/ticket_queue.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/tuple.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/unique_ptr.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/unordered_map.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/vector.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers/vector.h delete mode 100644 hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/data_structures/containers_mpi/list_vec_mpi.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc delete mode 100644 hermes_shm/test/unit/data_structures/containers_mpi/test_init.h delete mode 100644 hermes_shm/test/unit/data_structures/serialize/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc delete mode 100644 hermes_shm/test/unit/data_structures/serialize/shm/test_init.h delete mode 100644 hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc delete mode 100644 hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/data_structures/serialize/thallium/server.cc delete mode 100644 hermes_shm/test/unit/data_structures/serialize/thallium/test_init.cc delete mode 100644 hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h delete mode 100644 hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.cc delete mode 100644 hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.sh delete mode 100644 hermes_shm/test/unit/main.cc delete mode 100644 hermes_shm/test/unit/main_mpi.cc delete mode 100644 hermes_shm/test/unit/thread/test_lock.cc delete mode 100644 hermes_shm/test/unit/thread/test_thread.cc delete mode 100644 hermes_shm/test/unit/types/CMakeLists.txt delete mode 100644 hermes_shm/test/unit/types/test_argpack.cc delete mode 100644 hermes_shm/test/unit/types/test_util.cc create mode 100644 include/labstor/api/labstor_client.h create mode 100644 include/labstor/api/labstor_runtime.h create mode 100644 include/labstor/api/manager.h create mode 100644 include/labstor/api/template/labstor_task_node_admin_root.template create mode 100644 include/labstor/api/template/labstor_task_node_push_root.template create mode 100644 include/labstor/config/config.h rename src/dpe/round_robin.h => include/labstor/config/config_client.h (59%) create mode 100644 include/labstor/config/config_client_default.h create mode 100644 include/labstor/config/config_server.h create mode 100644 include/labstor/config/config_server_default.h create mode 100644 include/labstor/labstor_constants.h create mode 100644 include/labstor/labstor_namespace.h create mode 100644 include/labstor/labstor_types.h create mode 100644 include/labstor/network/local_serialize.h create mode 100644 include/labstor/network/rpc.h create mode 100644 include/labstor/network/rpc_thallium.h create mode 100644 include/labstor/network/serialize.h create mode 100644 include/labstor/queue_manager/queue.h create mode 100644 include/labstor/queue_manager/queue_factory.h create mode 100644 include/labstor/queue_manager/queue_manager.h create mode 100644 include/labstor/queue_manager/queue_manager_client.h create mode 100644 include/labstor/queue_manager/queue_manager_runtime.h create mode 100644 include/labstor/queue_manager/queues/hshm_queue.h create mode 100644 include/labstor/task_registry/task.h create mode 100644 include/labstor/task_registry/task_lib.h create mode 100644 include/labstor/task_registry/task_registry.h create mode 100644 include/labstor/work_orchestrator/affinity.h create mode 100644 include/labstor/work_orchestrator/scheduler.h create mode 100644 include/labstor/work_orchestrator/work_orchestrator.h create mode 100644 include/labstor/work_orchestrator/worker.h rename {hermes_shm/scripts => scripts}/ci/coverage.sh (100%) rename {hermes_shm/scripts => scripts}/ci/external/CMakeLists.txt (100%) rename {hermes_shm/scripts => scripts}/ci/external/test.cc (100%) rename {hermes_shm/scripts => scripts}/ci/install_deps.sh (100%) rename {hermes_shm/scripts => scripts}/ci/install_docs.sh (86%) rename {hermes_shm/scripts => scripts}/ci/install_hshm.sh (90%) rename {ci => scripts/ci}/packages.yaml (100%) rename {ci => scripts/ci}/py_hermes_ci/bin/run_test (86%) rename {ci => scripts/ci}/py_hermes_ci/py_hermes_ci/test_manager.py (100%) rename {hermes_shm/scripts => scripts}/docs.sh (100%) rename {ci/jarvis-util/jarvis_util/introspect => scripts/hermes/packages/hermes}/__init__.py (100%) rename {hermes_shm/scripts/hermes_shm/packages/hermes_shm => scripts/hermes/packages/hermes}/package.py (58%) create mode 100644 scripts/hermes/repo.yaml create mode 100644 scripts/lint.sh delete mode 100644 src/api/bucket.cc delete mode 100644 src/api/bucket.h delete mode 100644 src/api/finalize_hermes.cc delete mode 100644 src/api/hermes.cc delete mode 100644 src/api/hermes.h delete mode 100644 src/api/hermes_daemon.cc delete mode 100644 src/api/hermes_singleton.cc delete mode 100644 src/binlog.h delete mode 100644 src/borg_io_clients/borg_io_client.h delete mode 100644 src/borg_io_clients/borg_io_client_factory.h delete mode 100644 src/borg_io_clients/borg_posix_client.h delete mode 100644 src/borg_io_clients/borg_ram_client.h delete mode 100644 src/buffer_organizer.cc delete mode 100644 src/buffer_organizer.h delete mode 100644 src/buffer_pool.cc delete mode 100644 src/buffer_pool.h delete mode 100644 src/config_server.h delete mode 100644 src/data_placement_engine.cc delete mode 100644 src/data_structures.h delete mode 100644 src/decorator.h delete mode 100644 src/dpe/minimize_io_time.cc delete mode 100644 src/dpe/minimize_io_time.h delete mode 100644 src/dpe/random.cc delete mode 100644 src/dpe/round_robin.cc delete mode 100644 src/hermes_types.cc delete mode 100644 src/hermes_types.h delete mode 100644 src/hermes_version.h.in create mode 100644 src/labstor_client.cc create mode 100644 src/labstor_runtime.cc create mode 100644 src/labstor_start_runtime.cc create mode 100644 src/labstor_stop_runtime.cc delete mode 100644 src/metadata_manager.cc delete mode 100644 src/metadata_manager.h delete mode 100644 src/metadata_types.cc delete mode 100644 src/metadata_types.h delete mode 100644 src/prefetcher.cc delete mode 100644 src/prefetcher.h delete mode 100644 src/prefetcher/apriori_prefetcher.cc delete mode 100644 src/prefetcher/apriori_prefetcher.h delete mode 100644 src/prefetcher_factory.h delete mode 100644 src/rpc.cc delete mode 100644 src/rpc.h delete mode 100644 src/rpc_factory.h.in delete mode 100644 src/rpc_thallium.cc delete mode 100644 src/rpc_thallium.h delete mode 100644 src/rpc_thallium_defs.cc delete mode 100644 src/rpc_thallium_serialization.h delete mode 100644 src/statuses.h delete mode 100644 src/thread_manager.h delete mode 100644 src/thread_pool.h delete mode 100644 src/trait_manager.cc delete mode 100644 src/trait_manager.h delete mode 100644 src/utils.h create mode 100644 src/worker.cc create mode 100644 tasks/CMakeLists.txt create mode 100644 tasks/bdev/CMakeLists.txt create mode 100644 tasks/bdev/include/bdev/bdev.h create mode 100644 tasks/bdev/include/bdev/bdev_lib_exec.h create mode 100644 tasks/bdev/include/bdev/bdev_methods.h create mode 100644 tasks/bdev/include/bdev/bdev_methods.yaml create mode 100644 tasks/bdev/include/bdev/bdev_namespace.h create mode 100644 tasks/bdev/include/bdev/bdev_tasks.h create mode 100644 tasks/hermes/CMakeLists.txt rename {config => tasks/hermes/config}/hermes_client_default.yaml (100%) rename {config => tasks/hermes/config}/hermes_server_default.yaml (100%) rename {adapter => tasks/hermes/include/hermes}/adapter_types.h (99%) create mode 100644 tasks/hermes/include/hermes/bucket.h rename {src => tasks/hermes/include/hermes}/config.h (90%) rename {src => tasks/hermes/include/hermes}/config_client.h (57%) rename {src => tasks/hermes/include/hermes}/config_client_default.h (58%) create mode 100644 tasks/hermes/include/hermes/config_manager.h create mode 100644 tasks/hermes/include/hermes/config_server.h rename {src => tasks/hermes/include/hermes}/config_server_default.h (97%) rename src/data_placement_engine.h => tasks/hermes/include/hermes/dpe/dpe.h (70%) rename src/data_placement_engine_factory.h => tasks/hermes/include/hermes/dpe/dpe_factory.h (87%) create mode 100644 tasks/hermes/include/hermes/dpe/minimize_io_time.h rename {src => tasks/hermes/include/hermes}/dpe/random.h (55%) create mode 100644 tasks/hermes/include/hermes/dpe/round_robin.h create mode 100644 tasks/hermes/include/hermes/hermes.h create mode 100644 tasks/hermes/include/hermes/hermes_types.h create mode 100644 tasks/hermes/include/hermes/slab_allocator.h rename {src => tasks/hermes/include/hermes}/status.h (98%) create mode 100644 tasks/hermes/include/hermes/statuses.h create mode 100644 tasks/hermes/src/CMakeLists.txt create mode 100644 tasks/hermes/src/config_manager.cc create mode 100644 tasks/hermes_adapters/CMakeLists.txt rename {adapter => tasks/hermes_adapters}/filesystem/CMakeLists.txt (76%) rename {adapter => tasks/hermes_adapters}/filesystem/filesystem.cc (50%) rename {adapter => tasks/hermes_adapters}/filesystem/filesystem.h (68%) rename {adapter => tasks/hermes_adapters}/filesystem/filesystem_io_client.h (94%) rename {adapter => tasks/hermes_adapters}/filesystem/filesystem_mdm.cc (99%) rename {adapter => tasks/hermes_adapters}/filesystem/filesystem_mdm.h (91%) rename {adapter => tasks/hermes_adapters}/filesystem/filesystem_mdm_singleton.cc (100%) rename {adapter => tasks/hermes_adapters/include/hermes_adapters}/adapter_constants.h (100%) create mode 100644 tasks/hermes_adapters/include/hermes_adapters/adapter_types.h create mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h create mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h create mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h create mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml create mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h rename {adapter => tasks/hermes_adapters/include/hermes_adapters}/interceptor.h (93%) rename {adapter => tasks/hermes_adapters/include/hermes_adapters}/mapper/abstract_mapper.h (87%) rename {adapter => tasks/hermes_adapters/include/hermes_adapters}/mapper/balanced_mapper.h (69%) rename {adapter => tasks/hermes_adapters/include/hermes_adapters}/mapper/mapper_factory.h (95%) rename {adapter => tasks/hermes_adapters/include/hermes_adapters}/real_api.h (100%) rename {adapter => tasks/hermes_adapters}/mpiio/CMakeLists.txt (82%) rename {adapter => tasks/hermes_adapters}/mpiio/mpiio_api.cc (99%) rename {adapter => tasks/hermes_adapters}/mpiio/mpiio_api.h (99%) rename {adapter => tasks/hermes_adapters}/mpiio/mpiio_fs_api.h (98%) rename {adapter => tasks/hermes_adapters}/mpiio/mpiio_io_client.cc (99%) rename {adapter => tasks/hermes_adapters}/mpiio/mpiio_io_client.h (84%) rename {adapter => tasks/hermes_adapters}/posix/CMakeLists.txt (80%) rename {adapter => tasks/hermes_adapters}/posix/posix_api.cc (98%) rename {adapter => tasks/hermes_adapters}/posix/posix_api.h (99%) rename {adapter => tasks/hermes_adapters}/posix/posix_fs_api.h (97%) rename {adapter => tasks/hermes_adapters}/posix/posix_io_client.cc (99%) rename {adapter => tasks/hermes_adapters}/posix/posix_io_client.h (83%) create mode 100644 tasks/hermes_adapters/src/CMakeLists.txt create mode 100644 tasks/hermes_adapters/src/hermes_adapters.cc rename {adapter => tasks/hermes_adapters}/stdio/CMakeLists.txt (78%) rename {adapter => tasks/hermes_adapters}/stdio/stdio_api.cc (98%) rename {adapter => tasks/hermes_adapters}/stdio/stdio_api.h (99%) rename {adapter => tasks/hermes_adapters}/stdio/stdio_fs_api.h (95%) rename {adapter => tasks/hermes_adapters}/stdio/stdio_io_client.cc (99%) rename {adapter => tasks/hermes_adapters}/stdio/stdio_io_client.h (83%) rename {adapter => tasks/hermes_adapters}/vfd/CMakeLists.txt (77%) rename {adapter => tasks/hermes_adapters}/vfd/H5FDhermes.cc (99%) rename {adapter => tasks/hermes_adapters}/vfd/H5FDhermes.h (100%) rename {adapter => tasks/hermes_adapters}/vfd/README.md (95%) create mode 100644 tasks/hermes_blob_mdm/CMakeLists.txt create mode 100644 tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h create mode 100644 tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h create mode 100644 tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h create mode 100644 tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml create mode 100644 tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h create mode 100644 tasks/hermes_blob_mdm/src/CMakeLists.txt create mode 100644 tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc create mode 100644 tasks/hermes_bucket_mdm/CMakeLists.txt create mode 100644 tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h create mode 100644 tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h create mode 100644 tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h create mode 100644 tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml create mode 100644 tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h create mode 100644 tasks/hermes_bucket_mdm/src/CMakeLists.txt create mode 100644 tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc create mode 100644 tasks/hermes_mdm/CMakeLists.txt create mode 100644 tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h create mode 100644 tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h create mode 100644 tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h rename ci/jarvis-util/jarvis_util/serialize/__init__.py => tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.yaml (100%) create mode 100644 tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h create mode 100644 tasks/hermes_mdm/src/CMakeLists.txt create mode 100644 tasks/hermes_mdm/src/hermes_mdm.cc create mode 100644 tasks/posix_bdev/CMakeLists.txt create mode 100644 tasks/posix_bdev/include/posix_bdev/posix_bdev.h create mode 100644 tasks/posix_bdev/src/CMakeLists.txt create mode 100644 tasks/posix_bdev/src/posix_bdev.cc create mode 100644 tasks/ram_bdev/CMakeLists.txt create mode 100644 tasks/ram_bdev/include/ram_bdev/ram_bdev.h create mode 100644 tasks/ram_bdev/src/CMakeLists.txt create mode 100644 tasks/ram_bdev/src/ram_bdev.cc create mode 100644 tasks_required/CMakeLists.txt create mode 100644 tasks_required/TASK_NAME/CMakeLists.txt create mode 100644 tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h create mode 100644 tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h create mode 100644 tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h create mode 100644 tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml create mode 100644 tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h create mode 100644 tasks_required/TASK_NAME/src/CMakeLists.txt create mode 100644 tasks_required/TASK_NAME/src/TASK_NAME.cc create mode 100644 tasks_required/labstor_admin/CMakeLists.txt create mode 100644 tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h create mode 100644 tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h create mode 100644 tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h create mode 100644 tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml create mode 100644 tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h create mode 100644 tasks_required/labstor_admin/src/CMakeLists.txt create mode 100644 tasks_required/labstor_admin/src/labstor_admin.cc create mode 100644 tasks_required/proc_queue/CMakeLists.txt create mode 100644 tasks_required/proc_queue/include/proc_queue/proc_queue.h create mode 100644 tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h create mode 100644 tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h create mode 100644 tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml create mode 100644 tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h create mode 100644 tasks_required/proc_queue/src/CMakeLists.txt create mode 100644 tasks_required/proc_queue/src/proc_queue.cc create mode 100644 tasks_required/remote_queue/CMakeLists.txt create mode 100644 tasks_required/remote_queue/include/remote_queue/remote_queue.h create mode 100644 tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h create mode 100644 tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h create mode 100644 tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml create mode 100644 tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h create mode 100644 tasks_required/remote_queue/src/CMakeLists.txt create mode 100644 tasks_required/remote_queue/src/remote_queue.cc create mode 100644 tasks_required/small_message/CMakeLists.txt create mode 100644 tasks_required/small_message/include/small_message/small_message.h create mode 100644 tasks_required/small_message/include/small_message/small_message_lib_exec.h create mode 100644 tasks_required/small_message/include/small_message/small_message_methods.h create mode 100644 tasks_required/small_message/include/small_message/small_message_methods.yaml create mode 100644 tasks_required/small_message/include/small_message/small_message_tasks.h create mode 100644 tasks_required/small_message/src/CMakeLists.txt create mode 100644 tasks_required/small_message/src/small_message.cc create mode 100644 tasks_required/worch_proc_round_robin/CMakeLists.txt create mode 100644 tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h create mode 100644 tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h create mode 100644 tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h create mode 100644 tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml create mode 100644 tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h create mode 100644 tasks_required/worch_proc_round_robin/src/CMakeLists.txt create mode 100644 tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc create mode 100644 tasks_required/worch_queue_round_robin/CMakeLists.txt create mode 100644 tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h create mode 100644 tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h create mode 100644 tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h create mode 100644 tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml create mode 100644 tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h create mode 100644 tasks_required/worch_queue_round_robin/src/CMakeLists.txt create mode 100644 tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc delete mode 100644 test/data/apriori_schema.yaml delete mode 100644 test/data/asan.supp delete mode 100644 test/data/hermes_client.yaml delete mode 100644 test/data/hermes_client_specific.yaml delete mode 100644 test/data/hermes_server.yaml delete mode 100644 test/data/hermes_server_ares.yaml delete mode 100644 test/data/hermes_server_bpm.yaml delete mode 100644 test/data/hermes_server_prefetch.yaml delete mode 100644 test/hostfile.txt delete mode 100644 test/test_binlog.cc delete mode 100644 test/test_bucket.cc delete mode 100644 test/test_buffer_pool.cc delete mode 100644 test/test_config.cc delete mode 100644 test/test_multinode_put_get.cc delete mode 100644 test/test_tag.cc delete mode 100644 test/test_trait.cc delete mode 100644 test/test_utils.h delete mode 100644 test/tests.py create mode 100644 test/unit/CMakeLists.txt rename test/{ => unit}/basic_test.h (57%) rename {hermes_shm/test/unit/thread => test/unit/hermes}/CMakeLists.txt (52%) rename test/{data/hermes_server_ci.yaml => unit/hermes/config/hermes_server.yaml} (71%) create mode 100644 test/unit/hermes/config/labstor_server.yaml create mode 100644 test/unit/hermes/test_bucket.cc rename {hermes_shm/benchmark/allocator => test/unit/hermes}/test_init.cc (89%) rename hermes_shm/test/unit/thread/test_init.cc => test/unit/hermes/test_init.h (83%) rename {adapter/test => test/unit/hermes_adapters}/CMakeLists.txt (94%) rename {adapter/test => test/unit/hermes_adapters}/adapter_test_utils.h (98%) rename {adapter/test => test/unit/hermes_adapters}/catch_config.h (100%) rename {adapter/test => test/unit/hermes_adapters}/mpiio/CMakeLists.txt (92%) rename {adapter/test => test/unit/hermes_adapters}/mpiio/mpiio_adapter_basic_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/mpiio/mpiio_adapter_test.cpp (97%) rename {adapter/test => test/unit/hermes_adapters}/mpiio/parallel.cc (100%) rename {adapter/test => test/unit/hermes_adapters}/mpiio/tests.py (100%) rename {adapter/test => test/unit/hermes_adapters}/posix/CMakeLists.txt (88%) rename {adapter/test => test/unit/hermes_adapters}/posix/hdf5_write_read.py (100%) rename {adapter/test => test/unit/hermes_adapters}/posix/posix_adapter_basic_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/posix/posix_adapter_mpi_test.cpp (93%) rename {adapter/test => test/unit/hermes_adapters}/posix/posix_adapter_rs_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/posix/posix_adapter_shared_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/posix/posix_adapter_test.cpp (94%) rename {adapter/test => test/unit/hermes_adapters}/posix/posix_simple_io_mpi.cc (99%) rename {adapter/test => test/unit/hermes_adapters}/posix/posix_simple_io_omp.cc (95%) rename {adapter/test => test/unit/hermes_adapters}/posix/tests.py (100%) rename {adapter/test => test/unit/hermes_adapters}/stdio/CMakeLists.txt (91%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_basic_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_func_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_low_buffer_space_test.cpp (93%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_mapper_test.cpp (96%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_mode_test.cpp (90%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_mpi_test.cpp (93%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_rs_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_shared_test.cpp (100%) rename {adapter/test => test/unit/hermes_adapters}/stdio/stdio_adapter_test.cpp (93%) rename {adapter/test => test/unit/hermes_adapters}/stdio/tests.py (100%) rename {adapter/test => test/unit/hermes_adapters}/vfd/CMakeLists.txt (100%) rename {adapter/test => test/unit/hermes_adapters}/vfd/hermes_vfd_basic_test.cc (99%) rename {adapter/test => test/unit/hermes_adapters}/vfd/hermes_vfd_test.cc (96%) rename {adapter/test => test/unit/hermes_adapters}/vfd/tests.py (100%) create mode 100644 test/unit/ipc/CMakeLists.txt create mode 100644 test/unit/ipc/test_finalize.cc rename {hermes_shm/test/unit/allocators => test/unit/ipc}/test_init.cc (87%) rename hermes_shm/test/unit/types/test_init.cc => test/unit/ipc/test_init.h (83%) create mode 100644 test/unit/ipc/test_ipc.cc create mode 100644 test/unit/ipc/test_serialize.cc rename test/{ => unit}/main.cc (98%) rename test/{ => unit}/main_mpi.cc (100%) delete mode 100644 traits/CMakeLists.txt delete mode 100644 traits/example/CMakeLists.txt delete mode 100644 traits/example/example_trait.cc delete mode 100644 traits/example/example_trait.h delete mode 100644 traits/prefetcher/CMakeLists.txt delete mode 100644 traits/prefetcher/prefetcher_header.h delete mode 100644 traits/prefetcher/prefetcher_trait.cc delete mode 100644 traits/prefetcher/prefetcher_trait.h delete mode 100644 wrapper/CMakeLists.txt delete mode 100644 wrapper/c/CMakeLists.txt delete mode 100644 wrapper/c/c_wrapper.cc delete mode 100644 wrapper/c/c_wrapper.h delete mode 100644 wrapper/java/CMakeLists.txt delete mode 100644 wrapper/java/build.gradle delete mode 100644 wrapper/java/gradle/wrapper/gradle-wrapper.jar delete mode 100644 wrapper/java/gradle/wrapper/gradle-wrapper.properties delete mode 100755 wrapper/java/gradlew delete mode 100644 wrapper/java/gradlew.bat delete mode 100644 wrapper/java/settings.gradle delete mode 100644 wrapper/java/src/hermes/cpp/hermes_java_wrapper.h delete mode 100644 wrapper/java/src/hermes/cpp/src_main_Blob.cc delete mode 100644 wrapper/java/src/hermes/cpp/src_main_Blob.h delete mode 100644 wrapper/java/src/hermes/cpp/src_main_Bucket.cc delete mode 100644 wrapper/java/src/hermes/cpp/src_main_Bucket.h delete mode 100644 wrapper/java/src/hermes/cpp/src_main_Hermes.cc delete mode 100644 wrapper/java/src/hermes/cpp/src_main_Hermes.h delete mode 100644 wrapper/java/src/hermes/java/Blob.java delete mode 100644 wrapper/java/src/hermes/java/Bucket.java delete mode 100644 wrapper/java/src/hermes/java/Hermes.java delete mode 100644 wrapper/java/src/hermes/java/MdLockType.java delete mode 100644 wrapper/java/src/hermes/java/UniqueId.java delete mode 100644 wrapper/java/src/test/java/HermesJniTest.java delete mode 100644 wrapper/java/tests.py diff --git a/.gitignore b/.gitignore index e3ed55357..c1b647efc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,17 @@ +cmake* +.idea +*.cmd +*.symvers +*.cmd +*.mod +*.mod.c +*.ko +*.order +*.o +*.o.cmd +*.o.d +build + build/ benchmarks/HermesVFD @@ -15,8 +29,8 @@ cmake-build-* .idea .clang-format __pycache__/ -/src/adapter/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt -/adapter/test/posix/Testing/Temporary/LastTest.log +/src/hermes_adapters/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt +/hermes_adapters/test/posix/Testing/Temporary/LastTest.log .cache wrapper/java/out diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3fe28b8d2..000000000 --- a/.travis.yml +++ /dev/null @@ -1,67 +0,0 @@ -language: cpp -os: linux -dist: bionic - -addons: - apt: - update: true - packages: - - build-essential - - cmake - - libboost-dev - - libmpich-dev - - autoconf - - autoconf-archive - - automake - - gfortran - - g++-7 - - libstdc++-7-dev - -before_install: - - test -n $CC && unset CC - - test -n $CXX && unset CXX - - git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git - - (cd $HOME/spack; git describe) || git clone https://github.com/spack/spack $HOME/spack - - | - test -f $HOME/spack/etc/spack/packages.yaml || cat > $HOME/spack/etc/spack/packages.yaml << ' EOF' - packages: - all: - target: [x86_64] - boost: - paths: - boost@1.72.0: /usr - buildable: False - mpich: - paths: - mpich@3.3: /opt/mpich-3.3-intel - buildable: False - cmake: - paths: - cmake@3.10.0: /usr/local/cmake-3.10.0 - buildable: False - autoconf: - paths: - autoconf@2.69: /usr - buildable: False - automake: - paths: - automake@1.15: /usr - buildable: False - libtool: - paths: - libtool@2.4.6: /usr - buildable: False - m4: - paths: - m4@4.17: /usr - buildable: False - EOF -install: - - . $HOME/spack/share/spack/setup-env.sh - - spack install gotcha@develop && spack load gotcha@develop - - spack install glog@0.3.5 && spack load glog@0.3.5 - - spack repo add sds-repo - - spack install mochi-thallium~cereal && spack load -r mochi-thallium~cereal - -script: ./ci/install_hshm.sh - diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake deleted file mode 100644 index f727e2f2c..000000000 --- a/CMake/HermesConfig.cmake +++ /dev/null @@ -1,96 +0,0 @@ -# Find hermes header and library. -# - -# This module defines the following uncached variables: -# Hermes_FOUND, if false, do not try to use hermes. -# Hermes_INCLUDE_DIRS, where to find hermes.h. -# Hermes_LIBRARIES, the libraries to link against to use the hermes library -# Hermes_LIBRARY_DIRS, the directory where the hermes library is found. - -#----------------------------------------------------------------------------- -# Version Strings -#----------------------------------------------------------------------------- -set(HERMES_VERSION_STRING @HERMES_PACKAGE_VERSION@) -set(HERMES_VERSION_MAJOR @HERMES_VERSION_MAJOR@) -set(HERMES_VERSION_MINOR @HERMES_VERSION_MINOR@) -set(HERMES_VERSION_PATCH @HERMES_VERSION_PATCH@) - -#----------------------------------------------------------------------------- -# C++ Version -#----------------------------------------------------------------------------- -set(CMAKE_CXX_STANDARD 17) - -#----------------------------------------------------------------------------- -# Find hermes.h -#----------------------------------------------------------------------------- -find_path(Hermes_INCLUDE_DIR hermes.h) -if (NOT Hermes_INCLUDE_DIR) - set(Hermes_FOUND OFF) - message(SEND_ERROR "FindHermes: Could not find hermes.h") -else() - #----------------------------------------------------------------------------- - # Find hermes install dir and hermes.so - #----------------------------------------------------------------------------- - get_filename_component(Hermes_DIR ${Hermes_INCLUDE_DIR} PATH) - find_library( - Hermes_LIBRARY - NAMES hermes - ) - - #----------------------------------------------------------------------------- - # Mark hermes as found - #----------------------------------------------------------------------------- - if( Hermes_LIBRARY ) - set(Hermes_LIBRARY_DIR "") - get_filename_component(Hermes_LIBRARY_DIRS ${Hermes_LIBRARY} PATH) - # Set uncached variables as per standard. - set(Hermes_FOUND ON) - set(Hermes_INCLUDE_DIRS ${Hermes_INCLUDE_DIR}) - set(Hermes_LIBRARIES ${Hermes_LIBRARY}) - endif(Hermes_LIBRARY) - - #----------------------------------------------------------------------------- - # Find all packages needed by hermes - #----------------------------------------------------------------------------- - # thallium - if(@HERMES_RPC_THALLIUM@) - find_package(thallium CONFIG REQUIRED) - if(thallium_FOUND) - message(STATUS "found thallium at ${thallium_DIR}") - endif() - endif() - - #YAML-CPP - find_package(yaml-cpp REQUIRED) - if(yaml-cpp_FOUND) - message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") - endif() - - #Cereal - find_package(cereal REQUIRED) - if(cereal) - message(STATUS "found cereal") - endif() - - #if(HERMES_COMMUNICATION_MPI) - find_package(MPI REQUIRED COMPONENTS C CXX) - message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") - #endif() - - # librt - if(NOT APPLE) - find_library(LIBRT rt) - if(NOT LIBRT) - message(FATAL_ERROR "librt is required for POSIX shared memory") - endif() - endif() - - #----------------------------------------------------------------------------- - # Set all packages needed by hermes - #----------------------------------------------------------------------------- - - # Set the Hermes library dependencies - set(Hermes_LIBRARIES yaml-cpp thallium - MPI::MPI_CXX stdc++fs dl cereal::cereal - hermes_shm_data_structures hermes) -endif() diff --git a/CMake/LabstorConfig.cmake b/CMake/LabstorConfig.cmake new file mode 100644 index 000000000..221f7ca09 --- /dev/null +++ b/CMake/LabstorConfig.cmake @@ -0,0 +1,84 @@ +# Find labstor header and library. +# + +# This module defines the following uncached variables: +# Labstor_FOUND, if false, do not try to use labstor. +# Labstor_INCLUDE_DIRS, where to find labstor.h. +# Labstor_LIBRARIES, the libraries to link against to use the labstor library +# Labstor_LIBRARY_DIRS, the directory where the labstor library is found. + +find_path( + Labstor_INCLUDE_DIR + labstor/labstor_types.h +) + +if( Labstor_INCLUDE_DIR ) + get_filename_component(Labstor_DIR ${Labstor_INCLUDE_DIR} PATH) + + #----------------------------------------------------------------------------- + # Find all packages needed by labstor + #----------------------------------------------------------------------------- + find_library( + Labstor_LIBRARY + NAMES labstor_client labstor_runtime + ) + + # HermesShm + find_package(HermesShm CONFIG REQUIRED) + message(STATUS "found hermes_shm.h at ${HermesShm_INCLUDE_DIRS}") + + # YAML-CPP + find_package(yaml-cpp REQUIRED) + message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") + + # Catch2 + find_package(Catch2 3.0.1 REQUIRED) + message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") + + # MPICH + if(BUILD_MPI_TESTS) + find_package(MPI REQUIRED COMPONENTS C CXX) + message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") + endif() + + # OpenMP + if(BUILD_OpenMP_TESTS) + find_package(OpenMP REQUIRED COMPONENTS C CXX) + message(STATUS "found omp.h at ${OpenMP_CXX_INCLUDE_DIRS}") + endif() + + # Cereal + find_package(cereal REQUIRED) + if(cereal) + message(STATUS "found cereal") + endif() + + #----------------------------------------------------------------------------- + # Mark hermes as found and set all needed packages + #----------------------------------------------------------------------------- + if( Labstor_LIBRARY ) + set(Labstor_LIBRARY_DIR "") + get_filename_component(Labstor_LIBRARY_DIRS ${Labstor_LIBRARY} PATH) + # Set uncached variables as per standard. + set(Labstor_FOUND ON) + set(Labstor_INCLUDE_DIRS ${Labstor_INCLUDE_DIR}) + set(Labstor_LIBRARIES + ${HermesShm_LIBRARIES} + yaml-cpp + cereal::cereal + -ldl -lrt -lc -pthread ${Labstor_LIBRARY}) + endif(Labstor_LIBRARY) + +else(Labstor_INCLUDE_DIR) + message(STATUS "FindLabstor: Could not find labstor.h") +endif(Labstor_INCLUDE_DIR) + +if(Labstor_FOUND) + if(NOT Labstor_FIND_QUIETLY) + message(STATUS "FindLabstor: Found both labstor.h and liblabstor_client.so") + endif(NOT Labstor_FIND_QUIETLY) +else(Labstor_FOUND) + if(Labstor_FIND_REQUIRED) + message(STATUS "FindLabstor: Could not find labstor.h and/or liblabstor_client.so") + endif(Labstor_FIND_REQUIRED) +endif(Labstor_FOUND) diff --git a/CMake/UseDoxygenDoc.cmake b/CMake/UseDoxygenDoc.cmake deleted file mode 100644 index b9601910b..000000000 --- a/CMake/UseDoxygenDoc.cmake +++ /dev/null @@ -1,33 +0,0 @@ -find_package(Perl REQUIRED) -find_package(Doxygen REQUIRED) - -function(add_doxygen_doc) - set(options) - set(oneValueArgs BUILD_DIR DOXY_FILE TARGET_NAME COMMENT) - set(multiValueArgs) - - cmake_parse_arguments(DOXY_DOC - "${options}" - "${oneValueArgs}" - "${multiValueArgs}" - ${ARGN} - ) - - configure_file( - ${DOXY_DOC_DOXY_FILE} - ${DOXY_DOC_BUILD_DIR}/Doxyfile - @ONLY - ) - - add_custom_target(${DOXY_DOC_TARGET_NAME} - COMMAND - ${DOXYGEN_EXECUTABLE} Doxyfile - WORKING_DIRECTORY - ${DOXY_DOC_BUILD_DIR} - COMMENT - "Building ${DOXY_DOC_COMMENT} with Doxygen" - VERBATIM - ) - - message(STATUS "Added ${DOXY_DOC_TARGET_NAME} [Doxygen] target to build documentation") -endfunction() \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index a8c4391ce..8b42e4d49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,504 +1,200 @@ -# CMakeLists files in this project can -# refer to the root source directory of the project as ${HERMES_SOURCE_DIR} and -# to the root binary directory of the project as ${HERMES_BINARY_DIR}. -cmake_minimum_required (VERSION 3.10) - -if(CMAKE_SIZEOF_VOID_P LESS 8) - message(FATAL_ERROR "Hermes currently requires a 64-bit processor") -endif() - -# Set a consistent MACOSX_RPATH default across all CMake versions. -if(NOT DEFINED CMAKE_MACOSX_RPATH) - set(CMAKE_MACOSX_RPATH 0) -endif() - -project(HERMES) -set(IS_HERMES_MAIN ON) - -#------------------------------------------------------------------------------ -# Compiler optimization -#------------------------------------------------------------------------------ -set(CMAKE_CXX_STANDARD 17) -add_compile_options("-fPIC") -if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -fPIC") - # Ensure that debug logging is enabled - # This will keep logs which have beneath priority 10 - add_compile_definitions(HERMES_LOG_VERBOSITY=10) - message("This is NOT a release build: ${CMAKE_BUILD_TYPE} ${CMAKE_CXX_FLAGS}") -elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fPIC") - # Ensure that debug logging is enabled - # This will keep logs which have beneath priority 10 - add_compile_definitions(HERMES_LOG_VERBOSITY=10) - message("This IS a release + debug build: ${CMAKE_BUILD_TYPE} ${CMAKE_CXX_FLAGS}") -else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC") - # Ensure that debug logging is disabled - # This will keep only VLOGs which have priroity 0 - add_compile_definitions(HERMES_LOG_VERBOSITY=1) - message("This IS a release build: ${CMAKE_BUILD_TYPE} ${CMAKE_CXX_FLAGS}") -endif() - -#------------------------------------------------------------------------------ -# Version information -#------------------------------------------------------------------------------ -set(HERMES_VERSION_MAJOR "1") -set(HERMES_VERSION_MINOR "0") -set(HERMES_VERSION_PATCH "0") -set(HERMES_PACKAGE "hermes") -set(HERMES_PACKAGE_NAME "HERMES") -set(HERMES_PACKAGE_VERSION "${HERMES_VERSION_MAJOR}.${HERMES_VERSION_MINOR}.${HERMES_VERSION_PATCH}") -set(HERMES_PACKAGE_VERSION_MAJOR "${HERMES_VERSION_MAJOR}.${HERMES_VERSION_MINOR}") -set(HERMES_PACKAGE_VERSION_MINOR "${HERMES_VERSION_PATCH}") -set(HERMES_PACKAGE_STRING "${HERMES_PACKAGE_NAME} ${HERMES_PACKAGE_VERSION}") -set(HERMES_PACKAGE_TARNAME "${HERMES_PACKAGE}") - -#set( CMAKE_VERBOSE_MAKEFILE on ) - -#------------------------------------------------------------------------------ -# Setup install and output Directories -#------------------------------------------------------------------------------ -if(NOT HERMES_INSTALL_BIN_DIR) - set(HERMES_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) -endif() -if(NOT HERMES_INSTALL_LIB_DIR) - set(HERMES_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) -endif() -if(NOT HERMES_INSTALL_INCLUDE_DIR) - set(HERMES_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) -endif() -if(NOT HERMES_INSTALL_DATA_DIR) - set(HERMES_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) -endif() - -#------------------------------------------------------------------------------ -# Setup CMake Environment -#------------------------------------------------------------------------------ -# Export compile commands for autocode generation with CLANG -# set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -if(APPLE) - # We are doing a unix-style install i.e. everything will be installed in - # CMAKE_INSTALL_PREFIX/bin and CMAKE_INSTALL_PREFIX/lib etc. as on other unix - # platforms. We still need to setup CMAKE_INSTALL_NAME_DIR correctly so that - # the binaries point to appropriate location for the libraries. - - # 1. Make CMAKE_INSTALL_PREFIX publicly accessible, if it was hidden in - # previous pass - get_property(is_internal CACHE CMAKE_INSTALL_PREFIX PROPERTY TYPE) - if(is_internal STREQUAL "INTERNAL") - set(CMAKE_INSTALL_PREFIX ${CACHED_CMAKE_INSTALL_PREFIX} CACHE PATH "Install prefix" FORCE) - else() - set(CMAKE_INSTALL_PREFIX ${CACHED_CMAKE_INSTALL_PREFIX} CACHE PATH "Install prefix") - endif() - unset(MACOSX_APP_INSTALL_PREFIX CACHE) - - set(CMAKE_INSTALL_NAME_DIR "@rpath") - mark_as_advanced( - CMAKE_OSX_ARCHITECTURES - CMAKE_OSX_DEPLOYMENT_TARGET - CMAKE_OSX_SYSROOT - ) -endif() - -if(NOT CMAKE_INSTALL_RPATH) - set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") - set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -endif() - -#------------------------------------------------------------------------------ -if(NOT HERMES_EXTERNALLY_CONFIGURED) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." - ) - set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" - ) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." - ) -endif() - -set(HERMES_CMAKE_DIR "${HERMES_SOURCE_DIR}/CMake") -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${HERMES_CMAKE_DIR}) -set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${ORTOOLS_DIR} ${ORTOOLS_ROOT}) - -#------------------------------------------------------------------------------ -# Disallow in-source build -#------------------------------------------------------------------------------ -if("${HERMES_SOURCE_DIR}" STREQUAL "${HERMES_BINARY_DIR}") - message(FATAL_ERROR - "HERMES requires an out of source Build. " - "Please create a separate binary directory and run CMake there.") -endif() - -#------------------------------------------------------------------------------ -# Set a default build type if none was specified -#------------------------------------------------------------------------------ -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") - set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" - "MinSizeRel" "RelWithDebInfo") -endif() - -if(NOT "${CMAKE_CXX_STANDARD}") - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_EXTENSIONS OFF) -endif() +cmake_minimum_required(VERSION 3.10) +project(labstor) #----------------------------------------------------------------------------- -# Targets built within this project are exported at Install time for use -# by other projects. +# Define Options #----------------------------------------------------------------------------- -if(NOT HERMES_EXPORTED_TARGETS) - set(HERMES_EXPORTED_TARGETS "hermes-targets") -endif() +option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" ON) +option(BUILD_MPI_TESTS "Build tests which depend on MPI" ON) +option(BUILD_OpenMP_TESTS "Build tests which depend on OpenMP" ON) +option(LABSTOR_ENABLE_COVERAGE "Check how well tests cover code" OFF) +option(LABSTOR_ENABLE_DOXYGEN "Check how well the code is documented" OFF) -#------------------------------------------------------------------------------ -# Build options -#------------------------------------------------------------------------------ -option(BUILD_SHARED_LIBS "Build with shared libraries." ON) -option(HERMES_COMMUNICATION_MPI "Use MPI as the communication layer." ON) -option(HERMES_ENABLE_DOXYGEN "Enable hermes documentation." OFF) -option(HERMES_BUILD_BUFFER_POOL_VISUALIZER "Build the BufferPool visualizer" OFF) -option(HERMES_USE_ADDRESS_SANITIZER "Enable -fsanitize=address in Debug builds" OFF) -option(HERMES_USE_THREAD_SANITIZER "Enable -fsanitize=thread in Debug builds" OFF) -option(HERMES_RPC_THALLIUM "Use Thallium as the RPC library." ON) -option(HERMES_BUILD_BENCHMARKS "Build the Hermes benchmark suite." ON) -option(HERMES_ENABLE_COVERAGE "Enable code coverage." OFF) option(HERMES_ENABLE_POSIX_ADAPTER "Build the Hermes POSIX adapter." ON) option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." ON) -option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON) -option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." ON) +option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." OFF) +option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF) option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF) option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) -option(HERMES_BUILD_ADAPTER_TESTS "Enable installation of tests." ON) -option(HERMES_INSTALL_TESTS "Enable installation of tests." OFF) -option(HERMES_ONLY_RPC "Avoids the shared-memory case in RPC for testing" OFF) -option(HERMES_ENABLE_PROFILING "Enables profiling of certain functions" OFF) -option(HERMES_ENABLE_C_BINDINGS "Used for compiling java API" OFF) -option(HERMES_ENABLE_JAVA_BINDINGS "Used for compiling java API" OFF) -option(HERMES_ENABLE_PYTHON_BINDINGS "Used for compiling python API" OFF) -option(BUILD_HSHM_TESTS "Build data structure tests" OFF) -option(HERMES_PTHREADS_ENABLED "Support spawning pthreads" ON) -option(HERMES_DEBUG_LOCK "Used for debugging locks" OFF) - -# Compile definitions -if (HERMES_PTHREADS_ENABLED) - add_compile_definitions(HERMES_PTHREADS_ENABLED) -endif() -if (HERMES_RPC_THALLIUM) - add_compile_definitions(HERMES_RPC_THALLIUM) -endif() -if (HERMES_DEBUG_LOCK) - message("Lock debugging enabled") - add_compile_definitions(HERMES_DEBUG_LOCK) -endif() - -# Calculate code coverage with debug mode -if(HERMES_ENABLE_COVERAGE) - if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - message(STATUS "Use code coverage with debug mode") - set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "" FORCE) - endif() -endif() - -if(BUILD_SHARED_LIBS) - set(HERMES_BUILD_SHARED_LIBS 1) - set(HERMES_LIBTYPE SHARED) +#----------------------------------------------------------------------------- +# Compiler Optimization +#----------------------------------------------------------------------------- +set(CMAKE_CXX_STANDARD 17) +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + message("IN DEBUG MODE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0") + add_compile_definitions(HERMES_LOG_VERBOSITY=10) else() - message(FATAL_ERROR "Must build hermes with shared libs") + message("IN RELEASE MODE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O3") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3") + add_compile_definitions(HERMES_LOG_VERBOSITY=1) endif() +add_compile_options(-march=native -fomit-frame-pointer) -# TODO(chogan): Expose this once we have more than one communication layer -mark_as_advanced(HERMES_COMMUNICATION_MPI) -# TODO(chogan): Expose this once we support more than one RPC layer -mark_as_advanced(HERMES_RPC_THALLIUM) - -if (CMAKE_BUILD_TYPE STREQUAL "Debug") - if (NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-g3.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g3" - CACHE STRING "" FORCE) - endif() - - if (HERMES_USE_ADDRESS_SANITIZER) - if(NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=address.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address" - CACHE STRING "" FORCE) - endif() - else() - string(REPLACE - "-fsanitize=address" "" FLAGS_NO_SANITIZE "${CMAKE_CXX_FLAGS_DEBUG}") - set(CMAKE_CXX_FLAGS_DEBUG "${FLAGS_NO_SANITIZE}" CACHE STRING "" FORCE) - endif() - - if (HERMES_USE_THREAD_SANITIZER AND HERMES_USE_ADDRESS_SANITIZER) - message(FATAL_ERROR "Cannont use -fsanitize=address and -fsanitize=thread " - "at the same time") - else() - if (HERMES_USE_THREAD_SANITIZER) - if(NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=thread.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=thread" - CACHE STRING "" FORCE) - endif() - else() - string(REPLACE - "-fsanitize=thread" "" FLAGS_NO_SANITIZE "${CMAKE_CXX_FLAGS_DEBUG}") - set(CMAKE_CXX_FLAGS_DEBUG "${FLAGS_NO_SANITIZE}" CACHE STRING "" FORCE) - endif() - endif() +#----------------------------------------------------------------------------- +# Targets built within this project are exported at Install time for use +# by other projects. +#----------------------------------------------------------------------------- +if(NOT LABSTOR_EXPORTED_TARGETS) + set(LABSTOR_EXPORTED_TARGETS "labstor-targets") endif() -#------------------------------------------------------------------------------- -function(hermes_set_lib_options libtarget libname libtype) - if(${libtype} MATCHES "SHARED") - if(WIN32 AND NOT MINGW) - set(LIB_RELEASE_NAME "${libname}") - set(LIB_DEBUG_NAME "${libname}_D") - set(LIB_VERSION ${HERMES_PACKAGE_VERSION_MAJOR}) - else() - set(LIB_RELEASE_NAME "${libname}") - set(LIB_DEBUG_NAME "${libname}_debug") - set(LIB_VERSION ${HERMES_PACKAGE_VERSION}) - endif() - else() - if(WIN32 AND NOT MINGW) - set(LIB_RELEASE_NAME "lib${libname}") - set(LIB_DEBUG_NAME "lib${libname}_D") - else() - # if the generator supports configuration types or if the CMAKE_BUILD_TYPE has a value - if(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - set(LIB_RELEASE_NAME "${libname}") - set(LIB_DEBUG_NAME "${libname}_debug") - else() - set(LIB_RELEASE_NAME "lib${libname}") - set(LIB_DEBUG_NAME "lib${libname}_debug") - endif() - endif() - endif() +#----------------------------------------------------------------------------- +# Find Packages +#----------------------------------------------------------------------------- - set_target_properties(${libtarget} - PROPERTIES - DEBUG_OUTPUT_NAME ${LIB_DEBUG_NAME} - RELEASE_OUTPUT_NAME ${LIB_RELEASE_NAME} - MINSIZEREL_OUTPUT_NAME ${LIB_RELEASE_NAME} - RELWITHDEBINFO_OUTPUT_NAME ${LIB_RELEASE_NAME} - VERSION ${LIB_VERSION} - SOVERSION ${LIB_VERSION} - ) +# HermesShm +find_package(HermesShm CONFIG REQUIRED) +message(STATUS "found hermes_shm.h at ${HermesShm_INCLUDE_DIRS}") - #----- Use MSVC Naming conventions for Shared Libraries - if(MINGW AND ${libtype} MATCHES "SHARED") - set_target_properties(${libtarget} - PROPERTIES - IMPORT_SUFFIX ".lib" - IMPORT_PREFIX "" - PREFIX "" - ) - endif() -endfunction() +# YAML-CPP +find_package(yaml-cpp REQUIRED) +message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") -#----------------------------------------------------------------------------- -# Dependencies common to all subdirectories -#----------------------------------------------------------------------------- -# thallium -if(HERMES_RPC_THALLIUM) - find_package(thallium CONFIG REQUIRED) - if(thallium_FOUND) - message(STATUS "found thallium at ${thallium_DIR}") - endif() +# Catch2 +find_package(Catch2 3.0.1 REQUIRED) +message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") + +# MPICH +if(BUILD_MPI_TESTS) + find_package(MPI REQUIRED COMPONENTS C CXX) + message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") endif() -#YAML-CPP -find_package(yaml-cpp REQUIRED) -if(yaml-cpp_FOUND) - message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") +# OpenMP +if(BUILD_OpenMP_TESTS) + find_package(OpenMP REQUIRED COMPONENTS C CXX) + message(STATUS "found omp.h at ${OpenMP_CXX_INCLUDE_DIRS}") endif() -#Cereal +# Cereal find_package(cereal REQUIRED) if(cereal) - message(STATUS "found cereal") -endif() - -#if(HERMES_COMMUNICATION_MPI) -find_package(MPI REQUIRED COMPONENTS C CXX) -message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") -#endif() - -# librt -if(NOT APPLE) - find_library(LIBRT rt) - if(NOT LIBRT) - message(FATAL_ERROR "librt is required for POSIX shared memory") - endif() + message(STATUS "found cereal") endif() -# HDF5 -if(HERMES_ENABLE_VFD) - set(HERMES_REQUIRED_HDF5_VERSION 1.14.0) - set(HERMES_REQUIRED_HDF5_COMPONENTS C) - find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} CONFIG NAMES hdf5 - COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS} shared) - if(HDF5_FOUND) - message(STATUS "found HDF5 ${HDF5_VERSION} at ${HDF5_INCLUDE_DIR}") - set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - ${HDF5_INCLUDE_DIR}) - set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} - ${HDF5_C_SHARED_LIBRARY}) - else() - # Allow for HDF5 autotools builds - find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} MODULE REQUIRED - COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS}) - if(HDF5_FOUND) - set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - ${HDF5_INCLUDE_DIRS}) - set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} - ${HDF5_LIBRARIES}) - else() - message(FATAL_ERROR "Could not find HDF5, please set HDF5_DIR (1.13.0) or HDF5_ROOT (1.13.1).") - endif() - endif() +# Pkg-Config +find_package(PkgConfig REQUIRED) +if(PkgConfig) + message(STATUS "found pkg config") endif() -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage" CACHE STRING - "Flags to the coverage program to perform coverage inspection" - ) - mark_as_advanced(COVERAGE_FLAGS) +# Zeromq +#pkg_check_modules(ZMQ REQUIRED libzmq) +#include_directories(${ZMQ_INCLUDE_DIRS}) +#message("Found libzmq at: ${ZMQ_INCLUDE_DIRS}") - macro(set_coverage_flags target) - set_target_properties(${target} - PROPERTIES - COMPILE_FLAGS ${COVERAGE_FLAGS} - LINK_FLAGS ${COVERAGE_FLAGS} - ) - endmacro() +# Thallium +find_package(thallium CONFIG REQUIRED) +if(thallium_FOUND) + message(STATUS "found thallium at ${thallium_DIR}") endif() -#----------------------------------------------------------------------------- -# Enable Testing +#------------------------------------------------------------------------------ +# Setup CMake Environment +#------------------------------------------------------------------------------ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables.") +set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries.") + +#------------------------------------------------------------------------------ +# Setup install and output Directories +#------------------------------------------------------------------------------ +if(NOT LABSTOR_INSTALL_BIN_DIR) + set(LABSTOR_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) +endif() +if(NOT LABSTOR_INSTALL_LIB_DIR) + set(LABSTOR_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) +endif() +if(NOT LABSTOR_INSTALL_INCLUDE_DIR) + set(LABSTOR_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) +endif() +if(NOT LABSTOR_INSTALL_DATA_DIR) + set(LABSTOR_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) +endif() + +#----------------------------------------------------------------------------- +# Build Labstor Main Packages +#----------------------------------------------------------------------------- +# Main includes +include_directories(${HermesShm_INCLUDE_DIRS}) +include_directories(${CMAKE_SOURCE_DIR}) +include_directories(${CMAKE_SOURCE_DIR}/include) +# Task includes +include_directories(${CMAKE_SOURCE_DIR}/tasks_required) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/labstor_admin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/small_message/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/remote_queue/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/worch_proc_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/worch_queue_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/proc_queue/include) + +include_directories(${CMAKE_SOURCE_DIR}/tasks) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/bdev/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/ram_bdev/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/posix_bdev/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_mdm/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_blob_mdm/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_bucket_mdm/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_adapters/include) + +# Test includes +include_directories(${CMAKE_SOURCE_DIR}/test/unit) + +set(Labstor_CLIENT_LIBRARIES + ${HermesShm_LIBRARIES} + yaml-cpp + cereal::cereal + -ldl -lrt -lc -pthread labstor_client) +set(Labstor_CLIENT_DEPS + labstor_client) +set(Labstor_RUNTIME_LIBRARIES + ${Labstor_CLIENT_LIBRARIES} + labstor_runtime) +set(Labstor_RUNTIME_DEPS + labstor_client labstor_runtime) + +set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) +add_subdirectory(src) +add_subdirectory(tasks_required) +add_subdirectory(tasks) +add_subdirectory(benchmark) +add_custom_target(lint COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/lint.sh ${CMAKE_SOURCE_DIR}) + +#----------------------------------------------------------------------------- +# Build + Enable Testing #----------------------------------------------------------------------------- # General function used to hook ctest to python test tool lib function(pytest test_type test_name) - set(script ${CMAKE_SOURCE_DIR}/ci/py_hermes_ci/bin/run_test) - add_test(NAME ${test_name} - COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER}) + set(script ${CMAKE_SOURCE_DIR}/scripts/ci/py_hermes_ci/bin/run_test) + add_test(NAME ${test_name} + COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER}) endfunction() -include(CTest) - -# Add testing directory -if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) - find_package(Catch2 3.0.1 REQUIRED) - find_program(IOR_EXE ior) - if(IOR_EXE) - message(STATUS "Found ior at ${IOR_EXE}") - set(HERMES_HAVE_IOR "YES") - else() - message(WARNING "Couldn't find the 'ior' executable. Some tests will be skipped.") - set(HERMES_HAVE_IOR "NO") - endif() - enable_testing() -endif() - -#----------------------------------------------------------------------------- -# Source -#----------------------------------------------------------------------------- - -set(HERMES_EXPORTED_LIBS "") - -include_directories(${CMAKE_SOURCE_DIR}/hermes_shm/include) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src) -add_subdirectory(${CMAKE_SOURCE_DIR}/hermes_shm) -add_subdirectory(${CMAKE_SOURCE_DIR}/wrapper) - -add_custom_target(rpc COMMAND bash - ${CMAKE_SOURCE_DIR}/code_generators/code_generators/rpc/rpcgen.sh - "${CMAKE_SOURCE_DIR}" - "${CMAKE_BINARY_DIR}") -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/data_stager) - -##### ADAPTERS -if(HERMES_ENABLE_STDIO_ADAPTER OR HERMES_ENABLE_POSIX_ADAPTER OR - HERMES_ENABLE_MPIIO_ADAPTER OR HERMES_ENABLE_PUBSUB_ADAPTER OR - HERMES_ENABLE_VFD) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/adapter) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/traits) -endif() - -#----------------------------------------------------------------------------- -# Testing Sources -#----------------------------------------------------------------------------- -if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) - add_subdirectory(test) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/data_stager/test) -endif() - -#----------------------------------------------------------------------------- -# Documentation -#----------------------------------------------------------------------------- - -if(HERMES_ENABLE_DOXYGEN) - include(UseDoxygenDoc) - - add_doxygen_doc( - BUILD_DIR - ${CMAKE_CURRENT_BINARY_DIR}/_build - DOXY_FILE - ${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in - TARGET_NAME - dox - COMMENT - "HTML documentation" - ) -endif() +enable_testing() +add_subdirectory(test) #----------------------------------------------------------------------------- -# Benchmarks +# Install LabStor Headers #----------------------------------------------------------------------------- -if(HERMES_BUILD_BENCHMARKS) - message("Building benchmarks") - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/benchmarks) -endif() +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) -install( - FILES - test/data/hermes_server.yaml - TYPE - DATA - RENAME - hermes_sample.yaml -) - -add_custom_target(lint - COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/ci/lint.sh ${CMAKE_CURRENT_SOURCE_DIR} -) - -#----------------------------------------------------------------------------- -# Install HermesConfig.cmake -#----------------------------------------------------------------------------- configure_file( - ${HERMES_SOURCE_DIR}/CMake/HermesConfig.cmake - ${HERMES_BINARY_DIR}/CMakeFiles/HermesConfig.cmake @ONLY + ${CMAKE_CURRENT_SOURCE_DIR}/CMake/LabstorConfig.cmake + ${PROJECT_BINARY_DIR}/CMakeFiles/LabstorConfig.cmake @ONLY ) install( - FILES - ${HERMES_BINARY_DIR}/CMakeFiles/HermesConfig.cmake - DESTINATION - ${CMAKE_INSTALL_PREFIX}/cmake -) + FILES + ${PROJECT_BINARY_DIR}/CMakeFiles/LabstorConfig.cmake + DESTINATION + ${CMAKE_INSTALL_PREFIX}/cmake +) \ No newline at end of file diff --git a/COPYING b/COPYING deleted file mode 100644 index bb9689a91..000000000 --- a/COPYING +++ /dev/null @@ -1,52 +0,0 @@ -Copyright Notice and License Terms for -Hermes I/O Buffering Software Library and Utilities - ------------------------------------------------------------------------------- - -Hermes I/O Buffering Software Library and Utilities -Copyright 2018-2021, The HDF Group and Illinois Institute of Technology - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of The HDF Group, Illinois Institute of Technology, nor - the names of Contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -DISCLAIMER: - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -You are under no obligation whatsoever to provide any bug fixes, patches, or -upgrades to the features, functionality or performance of the source code -("Enhancements") to anyone; however, if you choose to make your Enhancements -available either publicly, or directly to the copyright holders, without -imposing a separate written license agreement for such Enhancements, then you -hereby grant the following license: a non-exclusive, royalty-free perpetual -license to install, use, modify, prepare derivative works, incorporate into -other computer software, distribute, and sublicense such enhancements or -derivative works thereof, in binary and source code form. - ------------------------------------------------------------------------------- - -Hermes was developed with support from the National Science Foundation (NSF) -under award NSF OCI-1835764. diff --git a/adapter/CMakeLists.txt b/adapter/CMakeLists.txt deleted file mode 100644 index 1f4b884f0..000000000 --- a/adapter/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ -# Set hermes to preload if adapter is linked. -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_PRELOAD -DHERMES_RPC_THALLIUM") - -set(HERMES_SRC_DIR ${CMAKE_SOURCE_DIR}/src) -set(HERMES_ADAPTER_DIR ${CMAKE_SOURCE_DIR}/adapter) -set(HERMES_IO_CLIENT_DIR ${CMAKE_SOURCE_DIR}/io_client) - -include_directories( - ${CMAKE_SOURCE_DIR} - ${HERMES_SRC_DIR} - ${HERMES_ADAPTER_DIR} - ${HERMES_IO_CLIENT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}) - -add_subdirectory(filesystem) -if(HERMES_ENABLE_POSIX_ADAPTER) - add_subdirectory(posix) -endif() -if(HERMES_ENABLE_STDIO_ADAPTER) - add_subdirectory(stdio) -endif() -if(HERMES_ENABLE_MPIIO_ADAPTER) - add_subdirectory(mpiio) -endif() -if(HERMES_ENABLE_VFD) - add_subdirectory(vfd) -endif() -if(HERMES_ENABLE_KVSTORE) - add_subdirectory(kvstore) -endif() -if(HERMES_BUILD_ADAPTER_TESTS) - add_subdirectory(test) -endif() - -#----------------------------------------------------------------------------- -# Specify project header files to be installed -#----------------------------------------------------------------------------- -file(GLOB_RECURSE HERMES_HEADERS "*.h") - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install( - FILES - ${HERMES_HEADERS} - DESTINATION - ${HERMES_INSTALL_INCLUDE_DIR}/adapter - COMPONENT - headers -) \ No newline at end of file diff --git a/adapter/kvstore/CMakeLists.txt b/adapter/kvstore/CMakeLists.txt deleted file mode 100644 index dcafdcf73..000000000 --- a/adapter/kvstore/CMakeLists.txt +++ /dev/null @@ -1,62 +0,0 @@ -#----------------------------------------------------------------------------- -# Build KVStore adapter -#----------------------------------------------------------------------------- -include_directories( - ${CMAKE_SOURCE_DIR} - ${HERMES_SRC_DIR} - ${HERMES_ADAPTER_DIR} - ${HERMES_IO_CLIENT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}) - -# Create KVStore Adapter -set(INTERCEPTOR_DEPS hermes) -add_library(hermes_kvstore SHARED - ${CMAKE_CURRENT_SOURCE_DIR}/kvstore.cc) -add_dependencies(hermes_kvstore ${INTERCEPTOR_DEPS}) -target_link_libraries(hermes_kvstore ${INTERCEPTOR_DEPS}) - -#----------------------------------------------------------------------------- -# Build Java API -#----------------------------------------------------------------------------- -add_custom_command(TARGET hermes_kvstore POST_BUILD - COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR}/java && - ${CMAKE_CURRENT_SOURCE_DIR}/java/gradlew build -x test) -add_custom_target(build_hermes_kvstore_java - COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR}/java && - ${CMAKE_CURRENT_SOURCE_DIR}/java/gradlew build -x test) -pytest(kvstore test_hermes_kvstore_java) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - hermes_kvstore - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_kvstore - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) -EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) -endif() - -#----------------------------------------------------------------------------- -# Add Target(s) to Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_kvstore) -endif() diff --git a/adapter/kvstore/README.md b/adapter/kvstore/README.md deleted file mode 100644 index 96b62be0e..000000000 --- a/adapter/kvstore/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# KVStore - -The Hermes KVStore adapter provides a simplistic in-memory key-value store -interface similar to that of Redis. Currently, we only provide the minimum -functionality to benchmark KVStore over the YCSB benchmark. - -## Current Operation Support - -Below we describe the operations our KVStore currently supports: -1. Create a table: a table contains a set of records. -2. Create a record: a record is a )>. The ARRAY -is stored as a single blob in Hermes. Updating a field in a record will -require reading the entire blob. -3. Update a record -4. Get a set of fields from a record -5. Scan a subset of records -6. Delete a record -7. Delete a table \ No newline at end of file diff --git a/adapter/kvstore/java/build.gradle b/adapter/kvstore/java/build.gradle deleted file mode 100644 index 1c0d21482..000000000 --- a/adapter/kvstore/java/build.gradle +++ /dev/null @@ -1,49 +0,0 @@ -/* - * This build file was generated by the Gradle 'init' task. - * - * This generated file contains a commented-out sample Java project to get you started. - * For more details take a look at the Java Quickstart chapter in the Gradle - * user guide available at https://docs.gradle.org/4.4.1/userguide/tutorial_java_projects.html - */ - -// Apply the java plugin to add support for Java -apply plugin: 'java' - -// In this section you declare where to find the dependencies of your project -repositories { - // Use 'jcenter' for resolving your dependencies. - // You can declare any Maven/Ivy/file repository here. - jcenter() -} - -// In this section you declare the dependencies for your production and test code -dependencies { - // The production code uses the SLF4J logging API at compile time - compile 'org.slf4j:slf4j-api:1.7.25' - - // Declare the dependency for your favourite test framework you want to use in your tests. - // TestNG is also supported by the Gradle Test task. Just change the - // testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add - // 'test.useTestNG()' to your build script. - testCompile 'junit:junit:4.12' - - compile files('../../../wrapper/java/build/libs/hermes-1.0.0.jar') -} - -sourceSets { - main { - java { - srcDir 'src/kvstore/java' - } - } - - test { - java { - srcDir 'src/test/java' - } - } -} - -test { - useJUnit() -} \ No newline at end of file diff --git a/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.jar b/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 99340b4ad18d3c7e764794d300ffd35017036793..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 54333 zcmagFV|ZrKvM!pAZQHhO+qP}9lTN{$T84PPtG)}GB2|29NtYb{F0ec zn4*)R{|0*oFi$%u(WPm}q9{8kG5s~rnst(Sf^+A3>-z2AhWtCq|1&r6zu!)-R{z(u z{=YhK|D|K)YGQ70@jt4f{i~{%tBJF-h3o%ia_y^|e*Jgo-+zA%!+$mrb#iidbN_3v zU}EE_=5FHZZsE#kW^dx=7OSb_wV;Xog+7r+CXQ+K4N4lGWN6w{P1Z)n8nztKKV>uA<7mtWOO80;QOQ9F=EuxE6B*)#W-04 zR6ns|YhjU-E#qKL6m1~&7_pW!JroRk6OZ!0u|`5#(QFp0YunX)9JN~=|Lgv%#-N_5c!me}oq7E~Us>av-(aVRVow%C)3 z`bILp8{1tIi#(-4eb4>*$gi^Q(=&QZj=To)%2bjHRA}z966Go_$h!LJp(dSMXq2XY4R;@@OSRS%n;X}L)wBJ;9w_>b z7>nq-M*r7p$2x2sgK?UHnma@Vf{hsM-7taoo2-CTzrjXw0Uh-~f-&vN5ELwZFaEc0 zEf^^-M*U#+D9pZ+E^W3hJpk^&{*HTXbo+g=sJ!w#V-5?YeyJC2;Tgu)@i&92&`>3G z|EQUYI78f|P94*3Xjs9~3M^L!p+%TZ;Pl|TXn*>PY`>Vl!uQSgvYz&`eY7@ey0lOy zbpPn<6O0RALmE7wra50KI&(hzgfF9MhNpP>vvG4oPd_OmFv5z>^>jm4oQ>aAv6$8R z+`^HBRZtT*p6-QE=w>z$o%u`1FD}c9tD!UE1LzYFW+<%0uDJ~r-5g7yUn5hdORh%F zGpi1Mjq-q%k-NpXTC=j^k{4!)P;QVJpwXHkGDvcS05!7!*YXltLHE*%5M8+#@7HV* zjdyqTiQs5Ji_|-GVV>Q=j2;l#rIdWOw#@=3;Xj)5N<(2jv{(Y{DX92P%8QHfiXNJmxeTmhc9JH_I4rWsu zkV~_*AWxQCWUD5YFdXbj(<>ubX6d`wM!Z=IZ5_x6N@*@3E2VT4oe&jITa_r+tBljQlB+2clPX1_!PC4Wa*^zrxrNN~?^WJU|M@gqF zn1}Tex29;0nSFnnP-4l*gukpd4T9J$t+aCMRonaSePR_Oj}SrS4M~d)OHi@1Lk2$Z zWkeJ;!1U+q9zC43(1Zy|iCIJZNlc|XsDBL?)*AJ8Q}}O$VE_N|QQ6hW+egI1-P*#@-Nwws z-RZxAM5>yB6RHTh09eYfg^i0w!;-9nU04)nO9Y&|f&fq)_&@^_wNwDLPbU8$+FVOXJZ>vylN5bn}Wb%y#g%wJ@Zf`~sb&aT_z8wWFX|p94L8txi(d_iOpDw1NE>irPnOBFm{+XdO$3D6ygE!F z1OwFi z2ZR)*5&>z$nn?guY!QDlvhv~*p&9HP>Z)X2P*EnZp6uXPaTuQLj(oL_Vj?--?C>*~ zq9QsZeA(q%?tjh04GcW7kQ)7pBv!EV$Nq?jZs8>Z)z>G%6N6lJaZN3PoCuV9$i?I9 z55e=QMI-Os8(_JZQ9Be=NSTNmN68pyb?GQKCo^NP2c}I%FfW;QW9So=)gr!fIcL_$ z8KOeyhs0K!Cn|nG{_7@6xJ8*pLIeS!#0LQ(`JWfqKekY=4xBOivWI}u+9pn{B$1|c zgYk{&Ps}%5ydMnklBl5INyxh7O~&@|Eo3$(9Aq=opsspcabkPy3L7mhf{Lb8in`B6 z7XSPg^oB8{#z!qHx^m5?iQNUHebFbe_&vozPao_9d z4Co3CBewKTXRt}P`rww8`Mu6+H+{C&!5P!ZOHQ!S5R=V|1sJ3wwl_wf>5kT|&+123 z$*afe2OxO;HVd%5c%uiIb+ZmR@InCA7rmW4(Z`f6G;e)WZ24u2u=k=H6T^e$htB$KvRc};FX&^j*(g#x!jLJ_6A=y(4<#`q z#Mzqx)umpXkVz@4XKPEbu*3FCp%|DM(g>syk~&yz4s7X0CQf#jJ-%yC<`vG2Y{|t{ zbRLJ0li#M8txNs)k=qCkE48pfwKG*wc*=!eCgn z%nq4#J{yIyMV$>AIt~*b8J=k+$d6TWg&rG@)XJi+D$DIDm+cc4$1K;}(*yaD<_VR8 zNyAOr$Om!5n4eR%9Rc|3*MvlPlh;}9T}Tvaz%`X-RG3OwmX@3pWtkNlq0@usI-_a1 za3uY#rsG4>OguPND1z-Ld^!mWR>La$Ws;`fR`bARKC@#J87U>pvT_R1s_6$6A}Tg6 zOS#Q+Z%y4%qNZG&ptN2(}oqj}t?z$oPJ;Rk9FOn=wqY zs{GTpF+%ybHsq;E7gNK(Matb( zds>T&+%&7`H_hcSM%^3OcMI4qS$X7jZTz0=zFEoczAr7owpXIp#h`z3AC@L1Rds|`L1%;R=AjNG z+>!B}%W|t<;i<5xD?a(E4K<^+oE&J+Ql|(_F%e)hoEj-XgnjkG6G>$=E_Bv6jMeJf7G#yr7+5%bXhb@ zp?PKR5zd+Ik$H~{!FeYF63)D6C+tk?X|6Ed|POzXjSd62@_8Ms7 zq=)m8u=E{NkG?UnU9ZC^;BySKu#nHH!*A|ORKSSCVn!IY%+H~YFjiu%9;}8du#m>t znJ%JJ4P{W8mT}XJD6o(KdRl%q8dDC8X3i7k-$_QQ`2<$*UOGL}w)L=`+1b3li1RN%XPPSBfcM`fhx5WPYfUm0(=~8v{1OFK;qj|H zpVzOlV?8DBMAYp)WSU%;?|1*+X|pux1}0RlOH?oJ6#VLH)68NhJ;u{zKGO`*F7rd{ zMuFFt<#eWQ@7QjB!G1A;QPKVl_Oo(^X_E70DGxvfD9DDeed@ z_aWCRss)}qwWE{q;65Xh=w#t7vLi*}&W0(rwck3QW#UhqSQozpa&{sG}b42R&z?tkM-ticENafjhg? z*7)A}T`!7tT-J%q;T`JIhAw%pr?K&XP*<}2iCONRY+uRN#HVB|77s3cS~V6Ui!Wc= z*G6MCm$Ek?y;KC|P*tb3%)`O~%cfTUfc`Y;*eYcXTQNu`wy{uts0p>I^bQt9#w=zP z$x54>arGGpD3WYv^b(d!E+&#WxeVG$GOx9=W}Meo{j3{^ zI^w+;Z8%R=#gXOjtcEO?G-?K^D#vmt=i|^IObL* zr=sgC<@4Fag+)lw+5mrpVLr^-(Xj`5$w-im)%?jvt%pn>ph5kKq`t$`98&I{ju9XE zxa}DGq`-@{aUVJ)fN|kh7BWUBAXGx~j_LnuCnHyiHtuD)ezs!L-G!xbQ(@(0_s3Q+ zr9$5~I<(BdY5Hy=PPtzdBQBwQ0F#^<{u>^N{ZF18D?40^^o2|4^5j^)Im`zz==KjP4rZ>D3iN|!t@2K8Fvgl6wGo{v6 zFZ!ZF|1Q4v&D@J?)pZFs`r%^s9hYv6Nb8>#-LW}NRzg2BS)TH#p{sGJFb|ZxJ!%eo zLoSzRGlsMBW<5gpm*>#ScUq&Z2_r+BtWRNDTBSgvwWZ|hujfdEoym(V8c9%pnVKm% zE6^Fv`?>RU+O(-Omjf-849KzPnpe=b4b&d#kLpa^Fu8(|r4ttT1vaJ}Xg7g>chKU*HO&ftcR7B0MWy)(fu zATlqRuZNog=H*P#d2Uqr(|HWIlW;tE^T}CvFmRf_Gkfno{f)O2%WrNp=$$3vTMQ}*3;qar9P2~nfDw)GMQX&NGVMd%GKwCOK)!V0>~W+#Q6J`6 znUDN>7xPB4rMX3&c_LiAC9mI;x-rBlEvf^ddb)EqWr8ZFRf#UY5mbkZJiwPyKJgY3 z6rP|DR!C?1N!fHI0o!cQ9cCI5g2kwZe zLr94lu+4n|pCmQ}=3i++nL@8^%4_I-^8l6jed%%6(A$0l?|acJM7>PW zU=EYIJs*~HQsU|5DB-^C>?`~Vhb9CHc%^vHEE4pkEuX_H>qap>pWbPJO48@emkL=j zh}t&SG^H(lT$FceilJ00pRX@^L`1Jps3vZ7LH$o0{4rDK5;A}FxOfQeI4v#{>XFqv z;d0`BXG}7IeraOVgEQyuP62=ZXcg3$?hrJsU%1INR1?}{-&7*IshkP+Z*M|l?$wyo zWO9Gu@At7Bb&smwd5aUQ8oz7S^&*`6R5+HU5EePbyhNk897lAFMzoDavfC8BpT0hp zyl^KPDxB56s62t@=)@BZ-n*~yWN~mka}#{-zrZ}1#x&%cE0qDMqK^VpGn24b55Lfh z0oo~j;RkGYeB4P_D>#8@Y!kdBuAPLqB)!9fL$RM88$Jn=>(g46USbrJ?} zN^@k_#kAK}Nnap8-cM1DuIXu;)d~}z+175|&fZvYM72`ts}Gn|d|G>d2~Z?W1Fp+O z#de!zJD^HjccE6Fzz2}Vl$h+&g5U$L=T6|i!83so%qWJCARvc-c>%Wn z?5~8knZ1Xbji-gAjlG44nb}{L6-*pWtp4e7s-~VB+A&U`pw9f-IR_^qH_9l-tT^jR z$b&F#7rA7=I=65+I8M(C*2VdDWfa^l`5$QsvB_#^-OsIY< zTaROTUqO1use+ZS%A0>zQzBA3+RO`IZ}@S*_OYK<(C4)j5g2#qI&a=U2O@tY42Qfe zcRg7g4T8}^;t)@(k4SCV`fX8`kt!9|;*AG~zWN{q0#nI4pf0QD)m#uD+mi_WMf zMK4A(?~!qJHXZNLZcA~|MaSh0ugWl&`%A?sw6c=$rw36$9h#Pl!k`BN9BwbotQvo= zkdIA`GQX$I-p7jeE=2Iw9>#f9st3InCm+fSkdKdkHuZT!*SMoU<#4f7Sjt$dLliR+N(<9N&iRV|JbZII!Z zYZ?8v=KGNee@3$nh2F!*>P3xHDnVw?k>2mBhvw0`^c>%*UfNN6skk)RCQZCHeLKxJ z@T`^ru;4j&^zciI)D7pBYA9ebgO6k7)%!d`lh1@p%IL1V++w*^-7nRGn6Yz>btB^l zD(JCW*?UkvQjv5|Ik&WP*>X%?WAuAkl_m@Vv}V+>sL86i$*Vs(=(pelsn2(EU;10mV>Uf{KYxR!%Q?bScq<9nEl)iQ~NEeeMEJbsQ4lJoqyZpP4xV z3{=w28nQRc368(_EsUUcT~X%^viHX|?hBzIi4q`ZzB^d!cRzBk(+*U-%?=002Qrt= zML6g5Z-Ynt^khb7=Ih_NO{c!WI(MZQ(#^J&OlavgJ5q_o%Lq9? zWrC>)B`AV3f~J1ry)**yv0rGm*p-Lc0r@yDJb@&$JUFe83tjeA+JR$M_Nd}EifL1J zX#~t$v+s9uEX7=QkcQBdR<L3GH-ynkKnaKl#2~uF0 zV?S3*RusvPGE&1_7&gqXjpXT?z)@TYMI8)7M;WUnyG)tGZ3Z+A-^)kcSW1Y+L8R^z zY2)ioCbKY-A_dJOo#luxCV;_29mIy`T5B3$PT^-U8^=r~)Ljh5B#gZQ687b3$|RFV z^jGN#=Lr@c7rJpBXiy=x=Zryo%hE+Y14U*InO#Po_*7!kI98%y;q0X}7U%fZG(t91 zZ9_9pbu@6}woL#RfAFT1tTszO*wwTU^A~LOk#1di#mrZ8Su}G*!&E^&(Ruj`jiVT> zjX5#%PXl;fUyYfaPOVmiX_qYol8IYluZ0vFwfvi4K4h?UA#r=-(Qs=&A$39Ys$rfd zw~Zf&7sHengA4pDWE}p4MiGHvp^oD1s&S_JK`2H==$k|1(gkqHGxVf_J$=DLq)7{2 z1um;w!cz6qg~8K&%!&wioLEZ3cqg4m0Ya;cfs`i<%HLL>G^x9m!QS{28{NAs2ZuLo1t#=~zSySzaF<=E> z{Ahi!)>UB;+2=g(8G(K8J7^ZB4Zj}|-_#j#S5~{C&p+vm@!P{UL3sp*!PwZP)e99s z{ZEKX|IjRV^vWN9=FE{=Z&-c(iByu{a$8~Gtj4iGLCbXRGn$%gsam$H!H^CUCS+In@Fi>^5s~%(9k?qSUPsVA93)<>61bX`pLXC2#vd_J47u&I9Sdd7gYE? zs5#&d6xA(=1j?XI;V!%iK2`vA#3eV0XJg0pp#i`P%7yaa7X)irl;6V*JiD*MtWuoS zK|iWfJ8qd2N?mUV*})Xyntz&=gLE_@DUQhn=9dV+)L0Sg7l1$ScxKK_ohyxFw8E!b z=6Ytgnj{GB<5;R(=COQ`ui%sMt<6~JFn$7G32=NYHYrKIFjQWxq^$Ob{#g4({VS#} z*l&gEhH-EB*$2Pe7;97V=q&--NE4sw_Jh=k@n*Gq!?YK#+*wy*Mw|xO>d(J1Cg7F6 zoBDs*1eL#R!hgdxOZ^{WTf@!5RocnHLf*;A&cpe?xMnpyC0r5oFUdrMrAO5o>QhyYP)Eyl_mc z0B(yxTI@QS4^QSDf=mQ}__#~8?r#oS%w@qY$)0Hghw?*0_V%vxMelJJ#Vilofc7;M zE9_XR4~DEbpo4A>^PTRhtyk$mPMO}3X)wuJG#gElhB&i`zMxN&OvC|-2zj?^V+;o5 zqb0((AcdQ&#tQEr7#PpUmKi5`?~)5Gv>O)C1`E~R(O!%7X%BGK-tkEA_g?r=9yfi` zZ#Nm~J#orcHM_UCgT|?TILjos*D)lReE}4xJ&t4EOvJe^2`B9VvH<3QC~^&=?=<}61Ox1Zt;tHd<`3+H0qG}9SDzuiVZwj`IOjOlry}iiNYjx^x;ne>E*=45ormsLC&%#T(>EH~64% z`lkWM3kp%!G7Zlsrhm;M&ueUx%V7T!Uf3WYwEsD)`o}WW`bSU2|89%;FLFW~${%eR z=Z{cAz}>6;4|`@(co7gL`>@2&mC(gT1Z;CWdoxpMQN-5<3G&=T#TZKuasrubpgZLz zn+%nV-fC=e!gR~2;`0{WreFM@F20E-_wPZU5JH>pTW!nlhIt<|gYn;1h&d})09_jP zLvjaqsvX_({WcgBsQMx4blzb^^$+_RuFtj@_5A{Y8|R~T7~P5zsY6u*=ZCv^>fI<# zsyCJx9EdLrw7~t^q~rC2JI{`8vEhLBzjQwA*6C;$`N<}2+P(nnR<$X{R=Fw8G_|+# zFz(o)Ab-iuXQ02Z1c6`$X*A3$2TeNC*rAC~*@w4y7*YKAry1;Yl{Y?-<}Hf9`tazD z3r7A=G+4pP9W`3ujtoZrh}FkSP04ExQNGp<#Otes&r1sbN0RwN3XxG-Oz+;P7KXpV znroYW@6{;5-7{XGzuIuyjRVB??S5C%-?O?mp6J(MIBj4kQS}QW{bGc9K-~Vqbr0rQw4uh_{+XZCzu7A^BKHfYXZTH~Tm6xZ0 zFTG5$Kg<^_eVvfDeG`L$Qf!h@>uBTmGuN%sa&TILm<*gP!e~7HT70EU{uWb7T~Y9q z6dT=Kmc6vN^U9X~_}{O-lKYL$;hEAWV<jZ{>$ z@gk$_I6l-z?HFbs{Nih_l6NN&y~GAT(->VZCbS#9y&)Md-Muf1oA{={FX>~mi;GoP zqq~cc8%3uin)kY__{AIQ>o2JkOaWkTHc;w6XBWo+XAFW!SHE5jeRPsHJOko zyRL6rW`Ulv=K|GV300NEI&Vpd!Lhb5#PSgOvna=LbL=+e@ViVE!Sv7`*{9@UM64bU zdv-uCCEhtx4yDAA5eHS#{b4AtarW-w#9mQ|Ln0`Le)Kte%$_Df+m0oIlnYzeTo|m( zAkz)IfyRlVBp?V5fBp!bC3`dj9XBx2h~}jgu)WhZ#6zj7_VC$mWIZYoH{HrXttZw?>L+`9|wQVQ~8v3BhKgau+?XvVXt;_N#K&=%wwaD$=vZ zL@$1P^A0~E-xS(ddu4nB6gL$WJ3m&N9MhK-aJ`J?>10XUkJo{GMMVwux=;TEbkT>{ zXeuua*q(Tc)8M|YO)|t^rW@!xs~_HEQN|z(yvNvG5Rat(9Xb#JE2M#D(VpvYpbOw@ zl`P=T7q2MNpSQ8cayz|^s+biotxz2TRiH=zCZTRuBuTG2WU8N@&L_&o(I`)kwX#%5m0X5_%W@J2jz)qWdw%-BEr7>sS|(c8 zNwS4sNJhhk5_OlvS5qFVyt(FXFj2#P$>+Zqp_`>mqA43mqEM5x7h=+tl&z*nGjlg* z>%tOzLghsq`? z+IYb8(neimh=$M}cez^J!|Y0632<)~V82AuGK^*is-P%jX)|T%#D&-oX0?O2(V70J ziPt*~b>l>n=}|F6x^4F{HM$!ipD1v8o7Yy`W=@Z8TXcWy*>L0MMvCq|74*#mE*`To z5iKlSR?f)}Nb6#$GK^|XgG-qfXQIQ|IjF2Na7r5bP&_TNXxZ_PhVZj?o!-*oa@2h2 zxr{x;YVcE2U*VHEKjJe+@iSh;I3Pgy@rYa0c&${{ zAP(1Y3jy|_UdLSlOoFpf?bA6)+evP@jp>PMm?q;s zJ~F+{kfW+cFs}p6J;^gA>oDuW?voAMgd>eHgynshhsk_e)_Q~ANE;5&c=x@-q?i$X zXT2d@6QODi#5_TD%oN+-q5*f1Or37*_J`F1FD$5Y z?KDi`BlpKA7sBXUjuHkr{kxZ(0Si8zu`H`+m2Y9*$5#U$Zi5H+F^Fz|j>DFzyUdpO zI8C#-EJtv{-@p*Ww8LUoAb*Av0~+!NZ#Xul#BiPmef(12!F=9fuaJ%|>10R_zyRlu zp`j#Db_FFapnD>ChsevSc(6tm-_LDQPY_f(v2pM~~ zXj0&n3R-?+Jy;Z8Aic>Fh6Q;C7T6oFU@1d2QO}5Gf76n7E(O2N7sa5#YDoi^BflId zsAq{5wmT)S(n~E6wLV3+jcB5d9@bfe^l2*jP4vK}L`uyir#G>rv{$6bQNgKbvg+Jm z4?EETsVQ%I2rjCo&4K@*Wc&!um^Xmys+&FBiPPmp^3#K1vpX(rY0dzlLV_%<7$&=s zkZ&%e$9;`sMR?mjMC3$9AF$p4;Z=aaG@zL?lFA+Jx$dG! zW5h~~bo-45@S1WD;wYY@?Po$7Odb#j_J?h(GzA}V4_j54x;l6882cjym>J&;-;rBp zIO-B|%WX4(a1O*56QX{W<#tjF%_n&nv+Pk zZ;REvXc$mX2Yp&rS*OGEy`4(QYz z8@UxwAW!cl!PrxqFDH-IagN5-yQgn{y5}zXX>o8`Hqt~->F8$-TXbEENtnax0kSEb zhjqrS5n3!79}VdffBsDEm1=bG4&47qUYGTgZMt*3MQP`Z4Oj#5Fm+f*Nk7@A zE3#-zq-G_Qe1fZa{Y`HWS)lWKP}{D~#&2ftnGv^NHauAd8AiSF2DSN?%yKV$;`+X{55t7bDuS88dl(~`6S$1&kc`8JpKzDfr7q=m&xCxDr*k91 zI+%V}BC*N9jf<%wBi|rYFgBi@gGvb)9C{0q$%6sduJ7bta9$l7ZS5O$<;>D$7%9xb z0bw{EYJ};s>Cwh>GBsF#kp4B>J-*O(tNNSeuKyL%|D5g0{IiekA6c%%UyyWw86NOw z^7IHDr5o1p(e$>v8Wv1M`N6P_l~b1Q%v~@Wis)xJDqkuN^jP}G>%IRb5eVjq--$0Y zbYKx31&0!J@+sx9)^rN>stHz@(Kx%Fm1<}+8Hx6$sY*(wrWKOFgkJzlW5si*NXv)L z)VbJzc6LfBfJ(Jt{H6#Qz=1n(W5(1WyZV$8-A*3!ReYotF6^uf)e(xRKj9$fak=tf zfBiEE>|Z_2t;T*?{EL=yBK`NCI=Z==*xUcJUoThd-49(0Cs65$b;9yHvaYk9^_=7i zJpx>zGE6WmjDuLZFrFy5nMV_!JtQUD`2~G?)gzz{Xj`pWYTKh8?NymXf$nf#yY?3S z=X>9q)=R}TS9A8JeH!$U&r=T1wtxDw|2E$?kI(ydeJ`Xj-WL#F@1`A|C}mfS6`NIl zEJ2AhIs}p}5?s-inle#b1asleoWfc5Xo0lRA|tUsv0uRywUk^Co+-FV+CNbpm79#x zDw#X{p46uZji_HrV%Mz6zX^?Kl#2IO#$OhoMCB|plrOW_{GyMVpKhG|A_}TRanr&t zNH&Ny{3RzN`qc!m;@~gerGquFg6|sjyaxmf&DfZL8 z52lqjboSoWR7N>+|HS2*rQ(ELF=se%4_{zovPg{*hNk~*zfliyN`%^l=|tNXFpiU-v5qAFZo5F89(4q zNZRaVCFP&oJnCEu(}=sCj+v_nTwU^NU8g-l*mrr2&1uosyVCFbHk!>RTpu5x-feYvNb8Ou)^H9+I zZGlm5c-fslzWyBhr(^DCsnMt>&eb3y0iR1>+_Q0doQVIA-Yl`x7fM2P?RDAWPmBr1 zjJ&vLZ;{czHI$}Hw4d&%m$^U$mWMyq_QGudP9Skp`i&TZ(EKecBh3&n1%77v*39P< z?IT6y{ITR7Vei%oH?VFW>!mc#XtxQV<^F-&H|GM;%v;r7WuiFB(OtFM8Mhskza6}2 zC}6)<#-u1sOS$woy|8KLZ7_((8yGKS{Zbq!v^z7o(|#YSK^w^!+0gG*`9k?sv|sNn zH@d!e_uW2lQJc3pCf^SgwRAvptq zvCrH?JT@pDdBVtJ1&bMQzHRPosBN&mmT9xBoF4V}i^yQ;HGnkCZ~oF(*Us&%Wqj~) z9>V$6XSCKe!i~b^Dx0>;bC@e0`->gT?BL!q)4-eV-T5l*-7edK-POYi&nmXJSY`(2 znVxgtVc=cm8-zdJKu9TQ*$+}>T0=09iMf4LV5>8i#*aR24t#E(4 z;!e$_UBy+8H|q)ABIN|C%jV{IBv%i$Li5e@&$d>vrPlMdD{d(cpvPm8pN`aiCxByO zeOhyil6mIX1OKv{SLfkUQKtW)C=Aaq3-qJ0*brvem6_A4<@bKH9aqD zKy!q->YR=^TjdJ9X{NeA=Ix}0b-i*k7?N|$0%6P_!#LlYZj^Us_>Ld9(nXj(Ui@Jh z-dapXQhVMJG2L+zN(!Evk2^sh?EuA<}BN zSiQLBwkt{0n*@MYTAy6jMe^40YzQECAHMP<%MI7<=0;(~3EVv}hH)ieN3?-=go~)U zTv3Rt!KFma6k6@g2N~A;#A)1K(N+(5f7~9HVRG4Ig^0P z-fkrN^Aq_i%%qhNYLU`VmepGuVYi@a2gT!C_~Jdb2Gd@HFHAgSP392mn-2&Qi(*Pz z_APV%-V2oud&m!SL?5+%bzOl-sTWNXRRZ!Xu>&oU%Tek~LX(3w=M4Ec))89Y>+(x8 z4nx582=*>Dg^KJ**HdEQs3-u^D6xSnt+MF)4)r1Kcfn-WE;q_aAJXWLSMV*O?(89` z4A`3>jo}&gm-m0ex^m$GOVf--;TZO2ekEW-l8X!d5 zYsgT?2tC=Uag&G}Nr+8Yi1*07l43R}rgP;`&ozlObuWuN+AzE-N;T9`*=d0u%WJx| zz@;TCyMW8;fO@A>zj`N{>mW-?Qp9kqqmc55Hf82WS1AHExlmz6&;a-c8A6`lu#ad$5L-1jmP!13i+t}i}R?ur1VeS-tBqcI325yl(ekNI?zHUsV%bL;Tz|Sc< zCXA#-(u8@@Y@ii6W@hgg>daC_8Dj3u+#lZgbaWe#2zKc2(q%@-IK+%1R5MmlB}*pJ z=x;Z4da5mq5In{*;Bm3nt+h0|7~ID_w&E=_W3U?(UC&n+`*t^X4g)8>k73JPJR-g4 zm-e451+qdIwOAmEBBvDxO|#}w8hESu7z9u`s{H7Cy4D-2vU5ag{x)d~+**sqGfzEb zt!U>HVli%ZZfd}xm^r8v@-N%E)4Nido%!Jr8PU&PI2MC9RjHkA;;_g^W~UT*A}l5U)Ri-L&j;!FV#m!*{1PgX9y*ZQ<6O+Cg&Rb9P) z23P&X)}1T%%2L-=pKb=P88!!FSKmn&rx4(wNpPe$<@gF<2u3T`OQXQ}0^JDde^N(3 znnLw<9FwLkHJsZ&XZ6io88RZ{U)@cZi6kMn>{v*a^c!cr>^*D!N%~#e$rD#bPt!AE z0gjCycYl)x&mBPkpaQ@Fb*PL(4ECJ{YD55mg5E}L9%cN@9u&q|oRU=<)3mc0t2QgG zx4$e0S^3_Y99l^dK>(OB^G-FsW(8^~m+QV{tB#qQbrT;b3n9E{n;6*fnp~N2d1_Lk zz4d<3Piqw&R&fVF)uk&}3neKdgY>yxHkw`?T~^JtecK;D)lKWLjKz936;D-07ktLV z$8EXr`H%5>HYK|p#B&mj7sJri7rJ*7YO9ReYJ8xR9EvZG;&ENz4>GLpbSJ0V9n0y> zYTzUn9+<~kN=&#OK-GcTEGIrUdrD{B@POgcOkrTPgI=OhMF&H(6F7U4&+HDk18ghn1N3(TZA}CK!Q(4 zjc{x0Rytw}`N{rL0JlSBnZ-E{T%<5%5s0H!Ya}(Po$~%I@5r*hl3vhSU*b-OE>E8A zp_7TlQUo|qniy2gs32FH4*ceGi(!8qVw(2NT50PD=LLl%B|FNBQg zPsa>YA(R7X8JSXK>I)52yXo*xFr-nRht4NfyjN zLHXu}RZ8+~4I6~hT&-p_U8tsZ%LmisxF;Jpw)?j#Wi$ADqh(K+PL05b=g9u2>JNvl ziI&d{(Gnl3e$v8W!EA6^p~-C(8>Y?TyKi50{nEK^tW4(tHV?dFZVdJjDq z`j&Tg?M@}{4(V!nvRYaJ#8wU(T@E&weeEJLe6&Q0EJu+RcTO-#e9zNE-SE^1n@&r; z%951{hJq?_7KX0*MP;?$*280aFUq=$CHN^MZG2|8Pwoy{``obRExP5xcx(lCWFaF` z8iK##X+GM=l_K_$T!esUyDd6)I=+8zlB7bN5TqjAC({a1IQ_W6`IJwaLDKlJHrtZ;J8QgtqGdJTuz@A@H-+k41lrlyK6ON`Z@}VgatND(Vn1rKGcW1% zg){R?@u)-O2>KE9*E~vdB}$~Q40-il#ijZeN?+kkyv7$yU*(rWL2U%2`%_ z+ZT)>bb;*bl6L?u{R*J_+p6Z@Y)lytJ&O%&r$G2DcWf+L!Ax}j_oJRoN z9P*KcBc^B2QIl24Z$)jMYhU_;@)_3FJSa4%)fKpPwO=nTU7d&nGNEzp`_5ci)ZF`O zjI7SafVa57ThEie*?nU}-<<4(F~U>0Q|R~zW7qYnc}eS(v$zvriqMI(32sD*#7%n2 zIKo&-V&+%}#bg_r9e_y=Rt{I-7W1b|>B5!kp0Iah?x8zRaf#!_$Kb)N&bo=`N)E~5Q!1bEGDK4Y@}4W ze@sOkurijoCLq%_XSALsFRlh8xn*}vz7Vqj5qfKoFwg(9ES9y~d|@8{7k1R^Epn|l zf5?gpH;CV#kO}Z{Wg`3TStlSns^5N)J@!@@f&z^@!ppt@El3X+*uqANH&eWxmS+6ufb%R4Ue(;LoGc70o#Rv-* z$M026Y!^5W`fmB#A_@#=b`r^{OA$-16ZErURnqZNW&=enuz zaWVF|u&w@b}i782gv(o!`hT7!}@N15$tapk=Q3pjEo&a3xNJ zZ`IE&&ymgJmm$-5*U?{>!`RKq+cWxwjO<&e#tL0rmxpBO`^eY96a?vWGfS)KP7DM) zJ4*N*f?zx?EFM>!%&EAhiH@==*L|-^W&cq{{0b^H&B53flS6kZ*$IPsIFIvy(21ii z+7jv*91x(bC)qYnFl!I_(4BlWP}9Ml+#A^j5&&2IsqMqHw9|cdHW+&XXL7I>ViQi& z?U_*4mWDJNW4X=xrzG;hy@9fAHorODQWN0zZga#MRQTtrIH(T#_zkx3pe8^{sOqGA zm1EXb@f^g4Bg#pqff*Z--;anPLtJVwcwlFb4rzgg!)LcUFzdW8{BCf32;$TIEUkgm z;2wB^c-LF(6q*`HOUOB4n-%&58nG#hW=&Vnh``vLa5U?grG(Y5%y;5$`-6wnoe`%c znD@l=j6AvXM70xqmJOsO!)ySN+M7jl8;TOb-!8KxVyV+aV=2X-QC^YonjY? zLveR^hvF2s;_mM5_TAZ;-JSXG?3Ya-A%Q?HzjNPn-se2BKke0+cv0|5OCxPD?#;jl zLQZO2dm`iwqb!@N*BJZIUkRC5=wK2`%$|S8><4YG2U`t&h1KIi>6g+4{nZTr3Duh& zF|~w+OwoohSkEPcE~ju7hd-TD@>U04`fW|J>bX5k@< zcfH$5NnoS=R1a8RQ{IhOIb8)#?g?H%QLcY)RwK@L&DJ8r_?>Y5jhPdPp`GM~wbA4S z!6l~hxjlYy83+ZL_b{tv$MH$GOlL?OeX-~8A=>a^=KO0{m;j{THP8aV4`7!G} zAt6R2D0q)6g9LFo@Q-Mz4~H|h1ctouU3VyS%1ZL0D&{ggr7P!=W}FS5U12ZSE^-Gv z=h!v)e~n-2B*`wmJiS)$0zCnHhokLUQ28QhM0b^LgpOxI_IQ7QMz z0%N_?nHlxCmMwDh9B8saymRU0-78n}#^1V^_h~|MLcbRg9IQI0tzYZ>eNumHBG^^l zW(&96?jEx;=oIc`i|eVT%j`Se+FLW;Gh+3(M|mEgE)}_gf2AJMOAM_K za^lspe9WCr|8{*5J_GdEjP1dy1Y1gPP~?>}N;@`L{v&zp@U7DxDz+`Q66%VN;k%Df z-D}-3?{{3}X=718#Df~)@fOlPACyB4@~#>Q*-NqCyx}xU$UMmX8?ig@PMR+$#wI9^ zykI0<$XoA7wP>5};F)~kgV@KEn(x-O0!*`XTdiT{(-q&1QP$|aqxHj5Y%HnY!#BRC z5D%^q&no=!820j+4SX2FoeGc@aN`w7+vvO0ivc*ytRgkK?6J~yI@u4T^!R)C#`L>h zn-o5(=%{!J`3Q(P6XoiDf^G)N9$|9v#()S+O{?bG;VF{i$-2=NdR4GzZ@Be+k+2_| z^(6{P44T;Am*-X8Hmba@%2W%K+e&1G*2iqX0yO~NOF`U|AH64BI{&HJI~Uq7+~jT#rk* z6kirdC_%0-7En#VU5`PuXh8b>kEH?N;HL0vDzBEA@NcS`-1V176Me2Ju25Mmj#tA% zmW_n?srwXet6qRO#cHrdapHJ`UYho@bo==GYnG7$!N^&V`7`6h|2g19;y=10dgd!4}wpVWy zclXD85<2maZCRO|u~`8)%il zopbvo;N#;3aSKJ^r{tD7Sc)TE;A$zAQZT)!UidfRwIoLpq4+9Ie#@5rw^5zzLppfUj%YZ8dCB`zO_DDR?cxc$E4N$jPwD_KFh zH^<|%1DH-_^WX`v{$5i4utgG;#ls~iju+MzW{dV|W!e_PLspV32$vA%#oB@0hs|Q` zAN+iCc(Kqe0Bf0~`(a%jQOET3$S~Zvaa_z!GaK1gzOi z4deF)7sq7zW&oW1!jvj3#%J zuSZdWDIOM@I-we{`s9-9!DIeQ1lh=2!mC9aWZBIU+H*3V3>_&m2s73^p)(vbg}5^^ zp*)zKYgaoDe=SLC66?4|P@vHjI0y*Wzb{EeBUeiy7Z)=-V_P$43RwpeBU@3RQpVZJ z*v-|-;U9FzBxM~Ld>|!v(tLg1wZf^Xc}1tuE8OOEWk*Lw2n~X?F%*l~mqs>$RK9%~ z@uytmr*yC4Rh~c(u^KNMV5StscPY)~mCdB5{I94@>dGjbDhfH0oqXzBSf32KO?&`*bqx1}g6EE%JHzC8ykEj5v=}Q^H0MzH} z4MP&22~q1oFJjg5o7j-fle=rF=@Hx8xUGfyW(shmbGCwr@xq`O^7RmygFmI-*9m!g1jL>VYDYFPPv^h0u;0+y!T$AdGJdk6I|8o6@&0DP@qeg+%1`L$;5#){jc?qYz9FZKhD1Uf9X zV*!U(bu|1${nS?@IQ*g5VlRymQ^Rk|IP$wLtiJjKc9;g?JCQ6DSIQU$X!}fKSC}k! zYVloI1B+%K;uxDzJ31`B_?xJQOcK}YG+orx2I|*-m;#E9w(X78*wk`EO%qKKHp%pm z+6(mvKKZ5JVzz0x+`Rm4oxQEytc1BI8cF`)hQq z{@!`Hmd&O87t3HNXPMRPblsl+;?B8cujAuwUAnE=DQ$pxlH&MQpXR+ zZCk$@Oh|@_M%Pl3_Tq1YQH-MEx9q2pgzja(nQa4#Mu~MHXVe-h4LP3=&N_p+bWHWD zmImnB8-#ukyv&F&bTsn%dd@l$bQ;R0L-spi-aSjl)alpBd3^i1`ug?0-)6O1)nc{D z2@=kPw{GZUZlPTX>FEboHayh=C#THU=rV@IT7uYvxc$0m_@anT`(Y|NKLf<*VOYch zPt|^CC#3;8JSyOxd-19fP{Le68fm4!*O|=m2IIlb3!SCWjyvEvK!1R!(;q8#Nc}{M zJ7W2iRw;h%g4m66oYGpb_4cjCvF@49)dC zLl7hK64iv+;Dqpkb97MdOC(LyPEC-dMDI0Ln);vFrK}pw>Ncyh=LU~_RoVk|^$|JS zT4wZOwdz*bZg%nC4;4|VeVbPvh#o+Tm~v5HjWsvUuY!miYbH0rlY@~OMr`;a8O;#^ zmY#ZO%n^DE*`^=HK}AW-S)v2p8(-GRQh)0xN#;ZwnlIYpoOYlhcTbFs^pEr&So3bo z%@20sok}-Xo|x}**CdedjE>Ub`0IG^(9j+$+FOyf23us{*~5#BePkKM-mpNy6ZdP5 zsdsc&wcOOU^xF88DLKA;-r;srn|40hW@%0m5s= z@`e0d9E3EbAO;DiX!w}?0c1tiD_6aEnpimXGp;A3Oh#1Flf?37mo=pGyW$v1XDeSs$zhC=#HLg_Z>90X)qZQyP6QIr zCKsmlKuI#aAAHv%%Lj}%OIYlcDD-pM5LjJiQAao>Q+hnl3o5zgemcz2fqukTYFu0#nN%mssKTF1yNw&e(hBP~$k|2& zwoY7uYBk2}`efHkTH78{Tc4EK=;Bnf*QS`kH6-cnk?Dxrb6I^nO; z&%cOmO})oA6M)JkKA=-3^S|$vDqcV_u^mua40PIbwK8*&G_nWUbpDrFrjCk^8pa2w z6c;JMXeYa*PGM7sjxv)ZW#bnN3QRCcm?HfX(oCu_fXmK^6sW`fi?#uQ#YwOO(j?d9 zTnh7Wl;5L7*`_?%-omJ9{lkilpU-(ychjQ$$IEN10BA?JgWu6WVGu3x3G;Ekn=D=G zbuW3e!*i~ox;&Zkue>cgNEf&RKg+&CY$a|+$vT<=Ibv0MDbGsOG0zfII?be=80^TBCZ3o#ipx4;+Je)dcou(`~yH#zTs@1$D`@!iJEW1ebp+9U5=~68d7%S-Q zD#w^y#59FAnt8b7MPblzvy~==!nfM#ryYP1lcSa7@7)OaEVEiATV%4#b(7=)rbgb> z)g^@etj8|vvmUsT4d}}Z>#n2R+M1hlyGTlIgJOX(3Gd`F_BJt;KLwKbN44zbM7&nO zqQ!nQHYCfc6ZNMsa^D}LJ+j~%9ntsPfXJl(&XsLuki7rp>cTT@;a28-wd^Dn-HuR+p%1i+5Cz1s1}I z4BFQyIR{?s{!XbcHnBaXoU9&Iku#5;1K7jsrZ(G+U98KWm^+Nla2}oVRqI^p9XP8E zP+dsHE7w7)P??rV?vg_`M<5VDYGFR+RV)o6F^M09=jI0KVG{46W9ICzVdm@xMlf9! z1=XnfMms=2DQqZ%F(jpE3NjQ$+u38((5=34o+^5Qnx2liz-4l~Cp{#IdhXEDkq3{s z${I_6%|9CFj;c~{=ms)1uQpui4}D4X{025JNCstNs%Wewzp~JLmf`zxfA*1jF}$0y z;hDbIuW24vBUg949BFoyFqB)h@+Fs|Nmg~!2&N^CcvMBe+f4flx`n{CS##ES`8LHf zV4f_csY2n{To_Dq6y}xZr`{CCq6ZWihn6$t-Veri%VIzr@K z5T>|@G%b2p)_g#qjg!w2)*SEGD1bYew+oU5HXRS1iRBq`s@e`QxbE+PWU=zb?JCfB zHNDqH;62FakKB~=hkNgFPtW3*uZJ63{qchv#~T6r`A8z}2ecEnzzYk&iG79~nON-G zzLuvG(!^d}62i-f0kPKUp+$Hj1b2)QJ~e|8T0;ctQX@Y}0W$7A^oP+iWX+aF)Pa<) zaU;Jg8s;9O(H)9M+KPxP`m%(#T}MR96)Ww$V=$e>F=bO@jMLqYg>J76eJJs!%ObN3 zvl6|0uSe2KlP{=2$%DAQ&#l3~-UY@H^~K|XK*clgKJdS&GyfrY6LGLNx3X|^Hu~?Y zVDwKr1#}^lq0iOBg<$@m@A>&;=cJjrTWqk%YAk99n3BFC)sD&J`!Q>J9M>vVWGvUA z#68GnnVmYhm8-wMvNf>X=DeTlzP?UP8GsyI(S*sQju>)A5=0?fI0+0*@P2)4^Xi!p z#*4J#HU0BoCvbJ9{;=TkSbmJxBSO=k~w8VtRu0V>TgyFH1 zEf^e-x1oC5lWCx7>mG|UV{o=>i58o`F~_!A7fcBN64EE)G~FJ{vK^LU?wft)>N_6I zE6_L2FVeE?W*eThDA0QY&)Meisuwv|vfDQ6qNcLFd~Ll3<@Yr74hz;Vys0cm*L+vu z78pA^6stxYre2*@>b&d18j_2Yrf0vD>@rrDomHfVm47~zi%a}Uun=8FEo-sZCimY| zj!`9+7m%2|o210!xxai@4Z8UTtYw!w9uV~|wMW3VHB`ey~68T2CzL^=HW2gbnD zc^(st0Y(>vFZUg|VP?)5bCx^J)Nt&nK+m1>8H7`3B9{gCfH{RBtBOnK(k-r@hXwpTvFK!<>K;E6!08_mD( z-T$aeWWcwc9n_Ms z@%d4SS%JC5ZYAtpc&xY5NZP63J=AD8-3rT>CwKYE{~70><@Kvl-8k>=_V96|BkeI` zcVb+p^Ckz|0IHVo6KT5+g$Q|fg7xG|6h>hbW+xK?R^fV5N7&N_#{qu(oHcv!n<@Ji zE^ed~V~EG7W`qkO#WN$(#zeo*)0U5yC$Y;W@gTG(|I|0)z4@>{&*Veo+pBoy?A>Ru zhMdEBiT)>?DdNLzbR*w{Lxt=W@o>#mwrcfypYv&VGO1$ztz?gd`biCby^@3Kq~C=*FiUjeh;op!;EcONfBq&8+Csg zCkWK3mlh;*;+0&6R8=fimQA|}J26!1`$1*rxwHcmr%o5Z(Kr`1_le6HhA1RmE0=#l z@L8~F_g-@u%AH~yJ^XU9GV7g#D=#Blh)k43r-Z^~af2N**0F{3nm8fAJ=uq+!J}kT zX-kKba7Uy8T#@bPmz8OOT1BOD^Te^u*^d5%GU8MCQBlTLO}yvo(4|;^b`DNTub5lH zK4ycXCH75CgPs)iHQ-G7qvsXSLyDHXX*suKxz>2R#+kRDeXLCQz>n03_ctj$*2GC z^l}R2a*nF4A^K}Lwv5j4yGCgT`F8e#Eox42sa^#wSua(l!ecUao&KrR*V``u#5Z{J zw**`{e!qmECA&mtEmRtN`Dr$O7LI6K)Y&S6G6Py13j?5&NO!LT=39(LH#FtujIG-1 zcwKJ^#`wMYc;|WrQRTI7d#Vd(7+7JP{CwOWxEECcQn z5D{>3RSMs`D0?{x;%W2qF($0WDE=~=<55JyO?(>d&2iggLy{I`Rh6R42F_ZaA zEE$e@UY1NXRp3)plM@rz)`pqv)-gL@pXaEz&qlBC)Jc{t8N3Usl4taW%`gf6MvxJ{ z+17-lkTHn+A&;tLNs{VnuDyHF6|xEQLN=`99_%6j@x{P+qS|w1vIT)K*9JiwYM=?t z^6@r;ShSk;hV|feEwbXu2*sSrqPC=}C4gq+76Ddgha6}wvF?w&wv%SSbmyZvQk_rf zYybEjRq8~1dUP7V@;+)M2v5`#EZP}S%419Z5*MZ~=9&_wFXGw~))jWtPvp%v%}3-- zPPUe7MYv7GV1BZ~&P2NIcSJn`n%}OZK$mL;Tgbc7#F4f-EM9A{8sVacn6Yp;-;p%X~8BN}Hn(QvPqA1EmO|3Nr339Ay z9O@{Dl|S3lNv>-k0x`{u$0Q?=mKKcq;^f;5lpSUowY=FKL6W>qkaDoBJ7AO42^w`| z{%Krh>{BL1n&Rr$?6LD^jpGIL8#*SJO+PxD9ya(MHaamdO|f1JhcXz_-n?s{M+Pq#*QEDH*NwwOS=E4q3l%-Oe( z^8Id$t@dHkkM&FK_0Q}F<@p35CHM`w7K?qfqC(~jny+CDF(gWH=$JVvSjpv|py^~h zH%-j9%zjg4cC1S{agLEIN-W0U6|538v(1`*FfH^=hnC@BnmeU*O zlo#MVm6g6M#D-7Yd}|S4H^F!2#C*m@Bh+8$(lyIXym#9aR>B(=e6dX{`;BEJep;w& zys+xM=R;5;KeYN~t0>)5K)7z)h?|mx{Gwi=4Hhdf=BKbs2!#t7BpF3UNds$|ouVR` zJqXwJG5oO8R_;&#ZBJo1#iaQc#j8TKrFk3xEe{bnCSf6YNaL`$^b4t0c4Rx|4-KrU z6A1H>US0)NMGR<48nwQM_rT;Hx49OGgM>v?*~zNT?*PxB8#np#BOL|J9<3X{Wa$^B zhyjzm-d#;Ott9~UkIzKvnDc6ezkS3obnry8k;PPnJmrQjRN{AHGzleJqpu%)T^&5W?^u z$sUft{STw#e*qbEG^10}a*VXia!k^)BU1BnlOuayzD&@KjLS7a{skui{?`A>Gyy8@ z{v*@m?|*{H#MbJ61dzlH-zD<`uN^AD{pepj+y3_BLdJiK9x-thaj>%kqA9Xg_GZBA zhdU5o`A=LWN%bH8^6-&#&Ea-cn}3k^3tU(E(}X2qrY(ibQD=S+nc*(l8KxkJ6a@5b z<}$y9v#L(Rq}Zgj-1)ddeNxOk-urtnXLnWzgRB^-tZ9W5qE~Hf2~)^_ig+ zXZ|R(EAf}A-P#CioOWT^QXv1D7%rpRU5&UwHecR&^5^;A~ocjJUW86E=+@Z_W`0SNH z(!eGn=J7$L`h}or?#@Q*g^of{_-GTARAwTNL+TfvmN=(+E6A&mJcfs5G_)ZeT_e>H zsHRxN*}9BmI(@-pl~Q*8tGGJ(uV?u9SnTj@3~6%9P-Ul(r45+t%UL)&Qz0+d%2|K% zQdOKo`7g8%SsS%IzZ4Wg@>M*)hb2}_#=~-PQXk?B38W7S6hBe#*@W;8GzfYFvg=X? z)D2tyx)+xN$mYEqW@4R%v+%)?+cMM+miBj=s)1IaB%N)nxF zCXrGr)8!%C@A+ZF;xbL(^Wz*DZ+ceMR~;9J)WXMl#VB^qM~42=o)%7hP7B#^g|{wEW@P zTQ65+>5wKa$6+Veq#kXpHZ8MuR-Q+Uro!LTvf}FtUw*h1B}ZlFmpO9;NA$3&7M7e$ zH!GJ_*hHt7X(z+eF41N1+hwz^8+{RhRI&|*Cg%*U^A_C|8zPxsQ9HmsWN(jL$384| zclN6Gh%D}42Cfn=9IBL5GhoDci%$+3>#LXS-2yz%eIJfFz2$2kU!nG_?Xi&N^0T76 zwGfIvPwMqMlX$T8zXJJk_E_^icr{9u-SYZ-&88hcqyTlZ?m#13T-=6K)a&Rdk9h3s zswirbXzq`~@A=_q0r?S2Ka~c%VB`itvm;7}ONCR6a>YH~nxifSdvQad|Fp>DCCg5?OJyeZcw6Jn?2PVJjbG4 zU2ve97E%k_7K;9`@CWNNPNYp}+Dn1>QeE2+%1KfLLe8_B zayRa^^&__&ppvJ1&NMbGVdT#D5J1-X)JN|ps0;Rq-s`zI`+}teyHfTQRr3k*jGNFm zHQ%kr48V-I(NYbDBr8!|@)qZbQ`uhQr=!Yfh8$O$_e zpJbQdjfW;{HNm9NGb{jH&T2vt^Cwgr*VdjZur8gxLKFfg=5_)YNdG?1%;7qdsMoq% zw&L)cFfgQ`pTaaGyiQl*r_%!6gE{fg-H|VFMHq?E%x1>%^;e|(q;gO0u*)WW1LT^d zqLJa#O!7y4ntdGkkWsT?hG3U%*ZZs2`^PRf;9pO@>YGKc^&A!hU6Oye1W$rF+3%2l z4QLsh*`t%9&JL!mAGVW*uX}=*<$~e+R?w_Gb}N(7l{}AO|Lb+M`(L4qBXYzW2_Shx z2G}$IO~3r7BT2~F!p#mC#saheHFNxLs4gjfLK>U}BUIsqt)Qt{+m7y}@1ju+dNC;| zJczlqw&>LruQ)zUl$GtortM4X6{s%_iK+S#2$kVBhfvFbch49gFNSi(uqrwj`z|h0 z-a2w;5bAAExK9deOc@UTa41|u>W{kbC3Em$9WX&_tyW6XDR@+gJ_MGyjS1 zG0{~;_hl>*eaI?@lzN(Im@fz5|{;c_;`xn!#7FjN|=e3NudH?jhMFyUt-()mPotZ zjZ?kMwJ!wgL#W_o3|0e&$`d;l5`dRh))vebgfEqsI>JG{HV=y|t!dh*N`?dFMBw>= zmhEFy9LmI=QMjg{eB8~#clc9@GlPW#wrHiT;gqnDv#UC5FbqR%RRh!bhqC2Sj}P@7 zlvB4gZ)KKp;y%x|0&CL?Hf7tiH>~bvwPwiJ`D8{-$VC8r--P>4FxGPvUBj|!hM=XP zTRMXRIYK9_YX>Oof=nj=O?pIqQrKR|so2r!A;0MAKr;Bxyq@7GlL(96T$V zD%%{UU5=>2U)Gu+ zO8UcJfgMq!lbdET~4$zykuh4*#NIZdLO zy*j!m)PlDr$tC$BEN z+ud1fTYWy(?N4u|45uJ2UlXq?Q!DQIg&Q<76K|!KSSiV3-)(B!3FQ<7J6A(}Vb8U3 z8#1+!wCYNFtA)%Z^pJ^6>+yx<2Gq0VI-w+5DR~scsZJAETKe8>*tEaMt zWP}y`9E&$7|C|vqd`Z!I)QS(&6l$6_=Lihr-Y)-Q)nQ}Sf>YMc%&1H(sCv9CAjK*CUy>P*Bl zZI!M%e5E$${LPgyb2yDeVMLF;z83f~$uLy6@=Bd@l7p>%dQ2QfSE6T7zD0-(r$rjGx|R57~O^j!+hu1f}+K8}jzPF;0*M zTQb|PNkKK&(Lgcd(fX9MD&cink&X7G`)}rM@zv$juWstRydKwYR>?F^)so>S7x2Tp zi%A=gtW-f2s$6b-RqMucj?>R`9{NcMmgPK*z^-8OijCGF) zRhlSh=2%n|whGaoDJCK8ODw)9Z+L`X5rb7jP$2QU@aPT@5vQ(mo{&D2!%0XRAs50L zP!>gm^*1_l0#t)c8rI&ZLT);^o{vJGTmv&j0JhNdusaV^Lbklpq-V)|Fm zNp!q+%AV3FmA8yKQ%%~U%n@-x*UO^~?C?`tVwwiy=fbfLHfcDHLwW@p2nowuh*A-G zK1OhlfxF6oQ{%_fo5+pUqC2hCx%UoR- zM3ZNSTwX#jUD{qm*rOQKi~Cj??D9a%qj@(eWnOcNOoMw3%zuNoGNMGw$T5sGai{jS zG$$h0f^SLEf`Ugklj6}z(ARfka^VrJZZbvuwOxR*Pn$^<80T58OSZ_3o`1ILk1W;q z=pUZwZPAFbF zE&u`-OuiZ-kstkfC=}UgdD^b5*j~x0psM?;za1#M{j|c&c8#8kvkWT(c8S zntQ}LTjO5`Mm(i)ot*YupHR^P#~5y$Q+!eCp`CKMMLj@v;7F@ELsPHqcOCwS?qc4^ z;i>jL7f-H7PmalDiZOll!P^@w>9u@j*LCFaUx1v}4W)pDAs(#qeF;g^AK*-HqOJ`# z$Q8BHFVUvYu}&arDSWM6%!aQ^c*u_A)^^}oc>Bxn!)jm_TMTTkN5Id&>~a3m1{95) zfhMy5Y2*_cCG@x8J+V*ACT)24{q}GhktszC2(e!D-|nRhR$x;1j*PtD!_ZW8+a39D z1&GZzPTxJjhEc*`L_|Z5Z0(!3?5a|0hvinK^)75CsI3TB-N-K!fV@;;T#eI{p7r@e zQuC0ax}?%dddx`S`JF?rBviq05ABADB-a>1(&KnxDZL-NMceu0g`bjAC*0t^7Dteq zU{StMI6gTq(Qki8 zAe-Rmh+lqO=lqD>5y3e;!=lG(4~=^|9M4SS-b_o&O5+3KB=1lpfo8B(aT!)WWGsW} z<86n-jMmUu$aeS_qohz+7n?7)Zqqjy=MVhaqz|U`l|cB zN z)Z2=gU2g_s=*lXDup*Iphx7xD^}ux2BeG)L6~c|Sh(s)-&NBrE8V8KF(j!Q2i?noo zNp%^hHNKiYDq`LUt*svweFaPwOAmXhVwTJGl?yE%wh3g}%-@Y}h50*tJYj!vg|s+z zQ9QB!Mkzle2u@^kAMfBPxQ>8TUq-mOdzucE@rU@sxKl>VmWs)8IK-RHa@T4WY_v8X zkNr|?e|AEhDEMggMWA|{$HO+>dhtFeRAno;gx1(?AyI{GPA*C3LLI=Q1&DWPQsZOc z{#xvgslrDzYZZ~#@Ccjq%Lj9w9&9N5M0VtGUYV91tKpUdXnQ|q7*00TJCgN3X@i$7Vgr^X$Yoi)hF_hsVnjBT%jHhc5`Ea zKTaS{9Kz|v%&QW&8=!@1_8uUc))dlY0cVQK^&WfD(9oulmKM0Kb@alv%kI=?x~_Zj za4pj^_Kp`wPGFFP5-=Xyu#Slz^^sa|(K#6Bsmq(G_|Hb1Dxi!+fsr^QT;?uD->fIa zP?{R>8d#v5LXR-aa+CCR*D2bh?vgcxiIpJIk3_)=wiK!$5(*zt{a`*MWei8@3FQRNZEt^7X3Y2w>RCuP0%&BvEvM@5G~?_8JmqI&g#R$9#ut zE&+#`K2vr9A1@E!f~YchsS!YVuKA+d_YIWR6IZATN&XssS**q^EHyWE{AAV$YN#(8E zhySYfW82}^Xatd;MH+7mUkMql585x6^PAb52I98#<){`pNB$f}zz7d;00N8GqNIQF z&)3ceu92$yqT1QEecGo63}V_;KdHza)F32(o5vz%!jl?sgJP~E+D+XZ8OX&O>bQwq~Bnt%S@fn`8Qld!VpE=o-B1kz|yAF8V7=gLE=>Tckw)6B!ZRZXiIeg@JM1 zP``?Q>6TCbV6u9CpIlKquq)MS_<=Z)PfQsZW7)#=nEV{o3N9Kqi%@Vnno08PXL)YU z#=GR7(ZeI7j6X|Q&ZhYG?c}j4f+SM(1&8ba;Ulo<85t>G+dAdAlq<22yha>2tx1R| z)J$aDSSv^|iHmOu8GsL5*fgW567n8t5PEh!ogQ77#2Ls0j0C0~=Wu-_;6XY66p`pJ zIlqUnDwMt)uiAvAWs`7Qe>ENFk>hLnmS|7#{hWBjkvzeud_Zbjy4Fq^uirCndfsp! zA`2sTmb;9ZSy11?iZ>-uKoC5|lKxM)Z~8|dlzHy^UqU)niaC!@fSt$}7&F5A|GnJ) zqZ0|6nL9WG6G#4SE3#8T*TC?%uXl86u#`khYPKvCB4G*v`68tVKi7ix!!D7t)b6X1 zoW@T*i-s~_yl}_|{I^*o?ZM_vo>``M;&;@n>|gjI%z=n8Z12tnlYST6Q+^L?4UaXz zri3Vn2P75!o?)?OW$n}486JtNr6-S+0o5JEj8XV)8fv1}!=FUD`(>z!RQIrzbeD?% zCrgxDOh*(7Cl?$+3og|8<(;@|j1ifG_Ppx6a*K_2x$>-vjd^D^-m2I7ANvY*kp|}w z{RPDJu5m}d75opXJ{AX$EoZh2gqifVDwa$XK(u*hrIyE>b2NQkhnA8}MtM=B*BA|K zJHocwd7hQkqHBc_cE*0P0@Qx7k>!H6oEOr&ARf5cJn2T9JR)U5`k4`T&p6ndhyoq z443$_k-<575l1Eg_2XPB(}aqVvhN z2*kU(W2c)qeIWNMjD^a)z3$AlsIRzP*f_Es5Rco)2xdhr#|njdw{^s*E1|j(v+OA3 z9XGz%{Y8JAjxpsE-$r)eAv~CGUKxcc>&UsOI>oylJLDeOxuz#j=IIBukkD;$VclW5 z-9tAbE0EsBQZzHV0JO5`b}Z?6@yMXIqyPv)zsx-;V=QG93axgxR zF9kqKcWJ;02N2Zf;`{jA))wn(`X~u83DpQW0In)Ltd=Yk();KD(^Wz^KvT8r@7s}9 zrVf*h7TSc%c5wR~M}=}90$HehqUk&vmR9c~p6<-n{mx&d>Ui%VqXvQt@};McO{8k(te9=Qn!rj7$Wpt9g$ z%(%`eWeAlfBGAz{*PmVEn@i+E2>6j0EN2WkSpD7|ctiF>&bkVvF9;d%3!)4ICkPX@ zzoN2>^JXQ)wC>U<^UAoVCvr2Sk~u-jM!#7)j7-NBu~h|~u~jEHh=Np-A~(^o_i=fN z9#Jg&Hn|6i+Poqvud3`*m#`Ig2-h-6Hq;E)IxsKA#xijr0)4!Pa1pY--1 zu8ER8hFIGVa1fIT36SWN`HPl|f5I1*1X%InCk+RxIt)?74qMk*H*30)f{nk(Y;(FF zd4c^R{XlvU_Q784%jUpyN-MYB#?snT_y3C&&Ow*EdlL!-q!txO@&CURsEj=@-Tm*A z-QU}uimQ>U+23cpbk z#gnotL3Cd(zJ({wLyGAN*~vgI2DcJ?0s)pMiPFbO*%`{fM&G-!jK_;#tFP-mn|>pg zpT4(*-^HV#6d}!^5Ldxq$iI+ceMkPHLbOLZDe@H=4|z51lN1(Kz?0xFUpR-{2FObxxQ1d`e_>*z=l;uI#J3Ml@e!E!Jm4$MYRTwJk4Zl z*tr&b^A3LdFLfc>#B!GRRpJV<`9muldh4aB$=1oyDF&2^N7^W@2W0ux2u|2yL*ZJY z1n6~6K@`e{K(teONo1xPK1&v*bnN_;%iq08L5kuzao~J4iNA>7%I3$5`$WB!gLnzn zBIY-%gq?b+)rxgD;pyG`jkXM}to14_Gem5Gwo_1-ng!Phdn_)4v<$y=rQv8_``i*} z^LcYng9=pgMn;ol$$Euldr(4aU|T58*d0TW^`k|-8Y)rX$pc*< ziPj7%;=r;yLZsFLxu9wesIYD?+tHYL2}%RfI#?betR&_@Gi2e}RpnO)D_gB=R4Q4f zZs{s6+>t~QI$@KPXdRW+NfhYNQ@=4Xt=i73+6J>#4A=fra|zY}IM<-B%(0R$vyxq! zg@<&4p*+2Q#$~s);b&n?;#+Rz#fMQ5CEwIWh1on4r3T@_X_myZ7o0HQl{|MKWb+`U zN13nDm~hx{E$e`}3<@tBr;j3aS$NvyM4i?ui*uZqvF9p@qbHAQ0x(%urb!k3jB=FI z8r@0Cy|)cc>=vmk1@`pk^CFxp$S~Mc=?d}Pht-LcMIwl#&3p;hV@+_ojaJ<`q_aDt z-_JI3qrkanYTLz-%lJAUVj49|4eEqDi>FHl{$FKZ0i4CMbsHcd4smyPcXxMpcXt=! z?(UutcNd7e6L)umctUs|_uTvEoE+}CysnyJs7kNZU(cTD?%loD{-Uza+YKd1tXAirrHuwjs>2DP*+*Vm7gK%`2kFNqo_Hz~TMknF)xV_w#HwEjo$*7+I20av^*q5r zJgONK?;$c}i2r1k73vph!cnzhN0-SSn{m=AM+hB*>$&3BD)|C=C58!%ROZ?0^^QXg zQ=Dv&v2pJcCQSVWy0pHJ=cu63A!sZ+CKfriumon`voO%8m?NLd*M4g9RaD+IcX%c#MJ>8~;IaeDBt2629L^1RE{ zOAn$lD^Qh9>pYrPSy$7XKMmF7eP5!haUg+pjwznuAxrjJ*)ZxeNd`P7V_sA0XyU{+ znh$mri<~BVc+J+2r=1c~>;jtb3(d^U(%>ro9yQm;HEvThlY!dIW@lIX~h(*_@jZU0%x#S!QSN>LVlR;ad@B7=w4S zrvr2HhMZ1;vA>@)Y73RJ@Fm&3L!#y*I*4U?yK$JhD5q`O3D#3AD!aI)qPh3AAWp1L zQz5e}#nBZ6Z_BEZ$H3wWLZFXR%IEx8Yk4pJ+xN%-P?^1K84*E-4{E?Rl%7csw?ATE zElXfHPa*eujTD_poomy*ap^|BeW4)FD!y^^(ProZ?vKc!4pNhI)BwpbDZ(Nx((Nlem_lG@zZ-Oc2v zEhN9)W0yqyO%)v!uZ>~qW(UdUQ`>O^noJ1Yq0mvyIR{jvlE^aTqu4o%+9pf&-q2|i z8`)B|E~v-CNM0q9p%OxhGc11ArS~|EoiL>=j6=pEs1+wn4H9>u)(Z}Fa*hqn?VSY$ zm*+IQ?e!!plY5CrYPncAXTxqPGjeAXEpBq82XB8g8O9Q|uVV@hDy>~`YSlG=M4sam zwIXkp3?xZ%OjxTRSxC>b2y00cB@kgY zzRR@PGzt{R>v%*$qlKBk{?%g$2rQhh{wOjrm}Xs_VObrBAcmAVwChz~*>HnIP@ zu<^TxDNX@ce+j_)e{0_OZ?kvSwKM$Nys=zy!D?CthLhQLtH)N9#8PEQZYrMWG?wXO zJQYPj?ngO=XkWa9#JAZ_PBa3BJNTpyzNMkFOg5^$12lfsR6M_A+7 z?N6||KY4;y1-F6|&=&IwOL0k>UfN4`GkisZpivkxk)bWp)8c;(Nrt2tj}q(HII}60 zXTeIu-*6lXNWkIJ%@pRiM8>t*MUS55O=Av4`y*v%2kC9LyG;fk?&J|bm%kAeYrA`6 z<7)t!U(Es2$Ax&P<2rRld5&GiG(3F6#<3GJZ6ai$`99NRnhT!9sb>_UOx(+fbB`bQ#>{xpncN3p4wUq59Dk!sT z0-yk0{r`I9`*KN88Z>;t7n~3{wHkRG}a8KpRPeF?OH9L3{y(JD|j9<*lSg$ zWhSYsWrwjlhL@5gUEX;&*YtCyop9C1*&BA-4gt60I=VRv(U2))h!pKi$rC^XT$}h4 zxd*n<;pQ4l_lHXAk74 z>_*3*(Z7c4_@T#Y-3+@WHPt}JKDhP{Yei1E5n*4*rd~d-m$l|CDe=LBol?kzDw2(0 zjCRUvBk&j;b%NC+gP6!4FV;$5`oz3D0XAO_uzBX+Su6cv^DkHL35x&ngnOONPRcT& zR1SIY+73a4)A7KBf{mFZQs~g}Z}!urx{A)lHsC)&d4p*`z7~S(HvSPI`w4Aa6FuER zPNHGee%y7;ehlCbzdou2#10}{@bRM2L)`b~y|1O)MG#3=9%#VPm+g!7&7UW(rqCDs zoHd}5P))>dysx^)bi<6Y>FSuoOUC*@)XfvHwChw`ZQNoV&KbtQk62`+(i%A2%cUUp z#C}R^;PlS(be^!t)}$%$tO=^P#xl)nkd9-VphvIQ_DF4o@tRV;R8<4tqPyxwFnKQU zD8=fO4s?fTPK_PnlFS=~cfD8`{|lWh8)fqV4eE)_EA8xBrlxbW5)2gKKqFKq&C+|S z#v?Z!`~iiI(WY8womr1|`MgBKGWzU*C<<{|WXq8DEaL%9(9sF?9ml2fv|OtMZ^AE^ z%>~~{uiYFdnS#}d*(4i{I<|A~^f5Q~fE^)Fp$g9t(CIR1%QR{)cqx~JP0Ct%&-V;0 z3abM47~GSVI;&7oYYbeB7Oo|hwz#4kaxj-hD9PCeB-R!CkyXpQXK;G=Wmeu3gAilktgqAi8s$s zjJRofy&7_CM*^avlU&EeiqYIAJa_$A?uf>0l98WUAd6E; zF$fZxKp^RZ7E>TC^(rdnJM>>if((3zx+1_Cu)@E~D*bwI=#RJJZ-YRBoaRqiCC*ya zS|>@#y3|eQa8e;sD+Fx0HwT8Gj0FsM4{?kx<84YCY8E>@UIr!^>Xw|v_OdjrhhL7LEn zddbeW_DxD4rPpYAM>fse^lU5NykpQ`cjJ0+KxM@?18jwEd7^T;UXG6$FO$HRq<>T; zi1(u_H5@NGaTUPw( z%V-szf+Qofl2C@?gmp#*0Wm_ZsA{ z4cjqk*XV^(cI^lOChzQAt?1)F+$O><#2awY&HZvyG=9? zlk$paMdz>)z#xvBY=EcRH?Prcg$k(T9Di3r^`1Y?sT`5pL?z zU|1su*a?m|n)GWrXm+;)f663p)g8%0dJPgM_|(Y9hRRLYHz=zbBF5HSh^&x1-_mIi zb$3y}B6re4ovV$$>-I3zja;AUBAjtXEGd6zUOhpfcUHiVbVr%weta+trW;RG}6PtBW`7cB^I!S{eIJ>=QAejMEDC?M=AC+*JIZ z6mvqRSO$kfNvBg|n{nOjyAdqq;|OvPR|FXO|HoYS*S`Aisn0K$`z(^5b|K&T8NC7zNA zX$=AlpIC5yh-jRU1Py^3`cOWUs?*?U;%(*U!8B0!i`OAUeYM{47o!cia&>*(V-K`z z?q_Mt*4IakQCY9ztIhObYMCtBRY+8lYjyRssG}$;_AO&00|X%r1GfXeD%1_?AcVF; z8O^LELE$^@4DOZZaiX>%xKp_b9eMFLwc^Pvp0bVZ^8`RGoNS5Qtx6$~RH|M@MGE5J zaqEc8a7e#ZRQAsySaw@^pDINU^HAY{R$&L`PSED5c^m-7hq_%kuAp0mRxK!K40*+G zi-)1?B3a~DcIg(%7<#c~VK~|WYSY{1ci=2u3nG92|SY?q)Essvm zZO0zDvP05cnqc7Lr)<=>&I;4;vf|jDThjh&F?JbQ-xv2ePl*a@oFA1YqQ~PqS}mlT zK``U_APx``tMxX(yQe!aO7Y?uspGXi;yL2E2d#Ty`&Y0ebg|SxrIfaz&OX^2d5O=O zB6ZI@0%pIb-tY{_6_=@#ypM7wO@;TpIx(54lJ>L$@27$1A_&csDS`+hpy2G0*Re%5 zP;wZHGnRP*FBL#PpriOI9ZaGt8&Ta9=^|l%y?PU7 z+;SJ@6ibYHAMhz$JI2>0;nb7`yv=5+3&urSk-|MOZJp6+=SZJxZKRRsg*D4p9qWNr;$sak(oK>P zyE1Q+-a|-{cd})tpC8GA8S0kNiuI(g%qrMJ7%Gl#v@^*O1RUtan23iE2bomax8 z<8d2&DJ-p3iDKpL4Gq%8x`g)n5FT)Cm$&k}`)Zwi*mF3m>YbYH2%Of3H+{@XQ1g$Y zro^lyRaz2nR4QH*(eskNBCD0)T}(GL**9YShPgY3PGb%}B7_#(MX-oj=7VB`c>}lp zFz?@i@xEf75k74#ZKR}Sm z9x#}TfMAkA7_f~TykNY70!0M1fEt}S0{N|dgTPdYc=w2%v3fE1A7Pz`TUv=;Lmj?Znq4srOB!}d(FH7jBGNa$<`(QhB&Mi4i4j^M zD@&+XZVDZ)e}lq3MS~hlFn$CXS6ybZGtV?%e2du&8Ke@pkvGC6T^*T)69Y2M)Gqjz z(=)xAmDj!3cowy5SSlv;2#K^bVXE+yYd0;CN^2XTseVjT{@71v-rX!#UQ>T6Pg)qK%w#%oz*WRr>OQN9Bi0tFh7UbQz|N(xLj; zQ>&)gR!iZuT3h&bL#s6ow0g`apU)4u8LC{JR;_F5I7ub6l1X(7QH9iAO%KnehpJDU zrR07u)@Oy^j39HHBpzuCwIWcs#3|ZQ3EpYEiYL0AI%GL4gH zDQs}R!Gs*D%dlcbAD9aLil3Mo>CDQVN_$T!;QD(+U;V3@YI0fu?5=nBDuK}m&^d~7 zbptnF2*1LL|{+)(4HQtLF5tJI3l-yv5qaWQ&f*OY@bWe}biZ01ogJ zVt(-Dx<)M8mnenP0Wt>iCZsZW64M~PBbSv3&Na5BYGM=DG7#OEt9Qr4>*}~`GrM9~ zH)M(G1WJL7u4#xGNW))0bJS^m+rcxW&}qM25ciqe!=@xv#yuuH>?tf6ed3Y-G(me~6EwONlN{1rURG=E@`8Wq*$xMlu+(1jLaxW)wn=<8@V0EX*UjeYlUziQ{V4D-%U(M#zmLUmS)2Y>-M6z$SJDG075#vLc@1MAuivIm-D;ClnAkB~q$o zSmF+>ic<2m1*uMXG8G~$F>|_vWj>XtJF5ry5GR+|h53=}T7-#Wrz}2i)F^=*z zSa_jGPiQ%0+C>v0^2qW$AOsV?JVb?Ls-3E(axOe2#6%+Q8_2iW6x&i#xJJAS89av& z#zYaBbsOWamWU%<+%?Q_E7-P7%qi7Ph~|>X;No$X92d)5$C^0FyEV&1Bxdh=w>TCT zEQ1}F)awdz$aQBjm7SKB}h3o+MMW_84K z0vLMA@yLPY^CD}^ALQu@jbxb9+Vcl3p&f-s1hm7En2gRU9JIQP@>^$NtMvF8W;K^E z>DjxLEQMJV(&{YvN%UBUDpVv>rbld8T0XHld_I(D5zl^PT+td2MRqPs`gk0?V12}I zTgz!N8S<2nBSKQGwoXcJ9N&JFn~^w`MO=S*V&15yj*ydW!(Z18WzDFMdzGniN#w>C z&+$Y5i(|T^p=Y}p5lg#8zl-~DzgX1*8~wHnpH}}q`KT={zls^hgl>V{9$#lo4O9tH zpExO3^A?o)M!qpfk)(;iU=v3X!Y9N711owONeN#Qvgvq!%`ZvOJodpzam=Y4OZsHs ztT}MAd_t%5*Hv7~`AM~-27#B6j3b0hq{~T*kkdoaM?>rBeo~Io)Uw&fAZx zzF`!&1lpP4e$ayrr_6PhX%dXuP($J8t;vAVz;+zmT2Ke6au@tWHjpCHQ0@6XfMmIr zdFwzXoYOBSSp6Wedu?@Yra+I*eVNz%1k+I?>3+i4J46_yqD&^R+_q7~Qflg>w8rFt z+?UbFcjQ>qsx2ll*>x6rFeH0LYKZbucMyr z#1hRp8?lWwi^q7=xQX|%0HWGrgf6JCZ zl^TO7b3Q+tYp}{8FHCvb`O)U$KIy5Iq#KkM!;MMJTy)hjJprC<*Wf%1+7vnR^vcXk zV2{74D(6N{iseVPu;L;`<+CQ6nF=xf`)xgbs+jMA7|~w*R0+eu=!WK-JEEbAjEI36 zRLUf|(x`R?8iE#eH!}^3y?eHh?y1{qR~4xcM&57D3f$`ZR4CBsx6KMJ%Px9hA)HHE zU>6-DG|UGa?b>m(lp>pWspz-q?ecJ$2NeT7kQ!0#YCWXl)S;RcMbXhe+lpEa?Cn4v z*JNF? zxJ>A5oH7Lymhv4Uh4Bs#KCfi{GxtUoH0wtU7m-uiZE?Fsq8qf^YLA8JhrX(!&c)~l z#XeW@t%8`c9{cQx=t_|<(cNDavEgBeg|^*30Dfv zp0x~IkiS0JVy0a=tLe3spw?1B(^kO=S8(*Q>7v?{4{#=At^ zC{jTwvZdzTuNm!0jQ*rIp_m7%SErhYJAoWj(i6q-yKwI3^h5-y=UnX`e6ankYSKh& zE7YsLgjtU*bf_d;V+EcE+v^U?m|itk9M?qDAMXKqLkHJAuPcEP`iq=N+kI)9_#Cd% zYy+rD5a($-c*(HW&ZGukBSTcNYm237DtWbGk-4P5ld9jWU9Rj{USJPF(%)pC=}GG9 zy*3Tls@DmF6I;~;_fp+8TsDR5uY_ZDA8`qD#L@ks)A@D?y;|ESKB3hy2jpSo$;mMy zf>3U*B$qDf$*Q&GqxQ0K;Fdm^+y)-+G)v|86Rycy#FKs5&e4dQ`u;dpWrf-8#W*Qh z(-hqn@?KBn6CrL9IZr#F3mn_Q zl{rF=5_7Us-zkPU%&(SC2eDTym=c^l$Z%1UK76nBm4sivP|v_de@tgP z*m~5UaZ?qK7Ym1 zckN}-^6+bpqoQ+$ir!=Mn#|LU5l}h>DLtkUMSC47RZDN+wW0@KGQ}3BUX$z)9 zgY=Y)?+f!`vZK(|K&NLq&b=fiTOXsN;;KRSUY&0M&#c-d=2`W^c+6(5jc{v4pWKz3 zQ6!gJ*?MI0p{=IgwPjeFZ)}7wA@ZLqg@2x70{$^Q;JUoq<~Qyk2PD4HOD8dUu`bnt z`tJQ{*9u1EF5v@CaYCCShCR!!k^+_wQ}AbfQ(mvM;z=Z$XE<>3ph zr;FXOJkl@r$fF~9C}&`HOG4R=iW~vyy4&D^I#9Y|-@v{uk(C&L$*YqE$$mcaW!}f^ zW9Z~{6&!^CS?j%Y({`L-4n_stnar-NGJ&CgpQ{60yNGafocO9aX}FU&-3Ep@nr6m!8qLFL}#K=}E@H!FZJ7w@DZX=_@^ zR@kYY=u+ecjnLb_Hng# zZT_I5MY5nl%sj)Z-9U4XCeg;{j$0*tK&}x(ip0e2A zW=7FJ9Nv~)aCrl7(~LusdI_&be3Y@`M4^e+pKvsRaz^;5Et5? zY-YBY6{XLVb>1l>>h8g9h9(h0tpt;%v+R%{@e8ScFF27_ zDw>l*2pX*Zij*^^9+-V*EqcJ_`z`c`?K2I`hatM6-fg)?qCvNqw9sNBYN6FJ(TJa@ z2n{#`7;0)5g~hCBm`M#wQwed^lyT&a+Ofa+y>)u9dl;o(bz>(L9PJl zdf^F;J$*Nl#Ka4VcGJX&A;|7|gW9E_EEUD3g$If;^KTy`lz5MD8YZ%L6iMC3ibw^4 z+sgx%F!DT9w=Gxm1%fW-MHJ%H*WJG()6(D&Ye zTPsohB9tQvdr;g(Q480iVEu%ujJOA`%m(q5a)AZaxEI)3o<~H{TIAUlYvEoRN$0ox z;hkdpbKlYGkQS)(@hCSIn4PVKKxnVjk^13HT|Rwg-ABR459Vp4lUnkk8Dx-ST77Hu zJN7%QZNAtwV7^}wyab>*RPb)r6}zgF5;?`&6;q0HStIiD_PFz}9)*2N>}RE$b_ZBe z6EEZw3o5)KB(J$YZHjQ(jDanpjGTZ4*EAWU9H3Z5OctZe4<)7DM__c?C9t zTUXmHE4Ui7c!tsX1DIkbrGnQ1TWsnwyr6kZ{Bwsbu%l(k6$anO=ApM)5|5ec8B~Uq zFTM<$vlPyJ5Z@wO){E3!-|&O0qvgdkn#H%SyO5p8cV% z!ik-q#6Xw-1k3nfMf5tRqeJ??E9DwPDg(1xYbB(8l*n1 z9gquWw7*HG!E}FHon&*2`C;xG0TM!Bl+IdQ6;|WYhsm=?xk6V=;~)B2m!J0Qb*;#9 zETYO)-t>pL!iL(0+iP^vc2-Ai5yn?%ZN0L)wr~4Vx+9YV_g^Lm{>T*~#c7sAL)Jio376uqEh0&bB~n{a8#M@cb3Ey2&Qdi!pTTwsU2e4t9Dp@!2L?YQWAZ>ka#!hjH^1 z_tQf^_nTXqRq1T_Kqz`2m~fS8G=V($`;KheVtC$nr1Ye@i0^xdb97K*Sr6Vu`kxpr%u43j8Mw|iIOGjv z6o(zgvZ>TxO|93dVZ}_)l_EpWRH&u;$3+M(BBN zJ5yp30B&z9%LpOgl9ORa5g884s@PB$YBYbUR4{p4W+cDU*_Q$2_b$4CGF4oy6I9Gp z*+Lw%F?2?7(>|~Mov~OY*nT^D15;;Y#9^RSl<5A*ENOZbqd2Gbl+gW{NU%gQh@{K- zxlu{j9^(=Cg(RQbP`a|Xv4S|Vxpi&tPCAi4Uz{X3u4kqHHQO16euL>nnw-`1s5U_c0KC-1fUfnkHaCOi0f;dKmrGbsVa>MSz8t!@5|HEkh*YSLw#{$ z!M@D~Rr1?w3vC&=cyUF5c1}cBelcXOUmUgrozg|Lq4Gzt(^98qGx2>hHkU?DN;;4i3XDK2Hr49~0C#PX?IGFDKJxL1s#ur>%-&aInys6U~WR zYaBt?brZ!F+qxTB+S?6~is}fUuz~4$wmym(4kOoArAV7XLSJ zr>Z~LtW*zca8x>l&t*}S5!?Kn4Ly`QZ_YhIxIe!qG&tR5rf*Jd6M zd{pQhKeq=5ZMS7fKPcX&k3f!TopVtSX@C1!evi5)8gXPj-WT!FS9zRB-p^T(D{npN z7BNVu(w7vFO#fV^mN^g>=elrkHgf z&4~nW>*syQ2f;q7fu_QBHYLQipdy4vtkThKCR$?LbVSD(t4!JS?iT0zY6Y~^!qXvf z<}*7LyZe!7bo9CxVu1H%UzgVVOyRuEEW^qgjOur(fky?DLjx7&`iQq=)^n{Q*ko68VL?9Ek4 zb}6RkN5`HkTVypyC>_>83%|9(YY%iMfD&%a&D0VSiPU?rp}66~)2yx-Ky&i9b=n#G zq;%TmoH6E4+qL(K#KVbx8}qSzjkh?Cd(@e3+nqM#fhJ6|sv4bo)2q!NN{dJ|kHa?o z)q==|Za8jW(PWtLIqg$_GT|mmB#d?JExb#-6ING$MkOxqtH@21b&)7o!7lJ2h_%zP zeCbV5v2+ZNSUwM5C4VXJ7W`{shp_IK0jlKZj9O0GJH;JBc&{Zb`)47M)q z9k^evA-&hi_1)>^hJ4x0eG4Lc*0t$(C_*hKtw+RM|A0snezsO-fjzaNZ{FQw0I#O_ zSs2UAGBw0ClG~41?aIYEcwHaKLfH-;Vq^?rdMcd^ekyL#hBA|On4NFQH_2!Y$!~IV zMVT4|iYo|nv9i$xeWW9VR!Fd}YCeL&2*ns)X{<$mrW1dV(|nD%#tS0Y&r1We!q3nK zd}_6BC={=`nquO~KwEWcJp`^Mkxo`L6`MtgckATDJ}qLG58l6rpoS^dvk-P$X8Zs> zel6o|>2!Y;S6+(ULRS^25xhhy+o*9F)C2r|&u!m|ma`l8j$*|%vhnk`^atwO)00gS z&w0{9VE0c-68s&*{E>Nh{Dh$P7cT-m<1*S1+W>o>IDj)E6o22f`P;bu$04{sM)wGL z^?w}+X=tQ8!7I=};3YxMp&SZDZj&hmro}bw$$#r-&CjkHXyKf;LJ8{)F;3KRn;jw+ zE#P?#*hoL7DnJnRNXcM*$T;&@W2OJ}!{f&*9awTw0~r}v30z;k_xChexjTWOX`&SQ z(H}DVC@F|+)xFL>Hj#h2*{;do><9&gfO8+QuvO+QtQo#eUj6D{tn7b8NxokU(XFR7 zya8d<;+5oEJ4nG_=@wZPzC%|)Al3z^rOL~oJ+AgN@ z&uC;tCp=(}67I_G8^2B7puZp3eMmHCUO1sm*S~Y6*gC1Bq$y{~z^SXQKxJxUEsrKc zlT4kSB!DFaBaqJN^W>}sko;xZwGOi4I@JU?SE~q(2&ukJM zq_||o&g`){8WW}U^P#W76pgC<)S_mSk(uy=9=492YzU z*`{&9;13MSjm>Gb7SxU1*DjH&53E1vCPT>t2E4K&nuPn!D}lr$C@cp6N;3ev7#ln_WRT)oPAkCWgaY^tN_ z2k~RDqqIxYoT1%QVk&KL!Y|inkZQ{LgA^oKU? zVh73-Z|uWP!9Mz47i-!kGx(YnE97O}_80SE!DG%OEzb0lVqKITImG_hjJ=Ad*Z?hJ zkKNL>n~28fC89-Ca}k0yRgNS0CcGc9?&wUu8%@ND-ywsI{&jGL!N>#E!I-~Kj)WZj zay{1f&ByBZ29NqleRuC778*KPu@heI{D<)}T6h4?fK(c*iY#*>Lh)0+btm_C#9CAgDLsKg5)Ys2*QEk!&H9 z(9R2NQJfy6v#wn_6+hU$hV|d zEyLk{!g{M&VWm39Jf3mWANEC&EON@* zY*N1u0akV9@`3pJc8(G%SGUF{9@BGtf&LL@S@hTQ7MqE<21mxMD4!2?_-b^dMJ9gJ zG=mATR$mCMqPTjFrqqrZp0{jDgGK8-U=H2Y>HRLkhQ%+#8C0!$jYWaR_>>zBRpwH+ za4apwGCq9lgwro`gdYM#z82md6S5Gg=v7`Aya)I&twz+wrP`tGTS`yds~tFL!1i#>P*k!XVXCmR|MYvuqaRQXL2;yNTL6dQe@Q$#(cRoFPx3hBh#;p;Is8g%(BA7tERjUn}L=0#LT6< zezFR>eA>R->ceayM(xQpn&F z>>sFkG8K)%#mIK`tKS(SlH27Pq3?fmD)x{~v#zp|4!lRO6lr@Z#^po_!0R1meFicZbB}70)K%yGJ z|6jdE00Ok{Pv8LC`&R*og1sn`f(pEpqEbTCFRKl|>hby6G9|$E^T$4ypGu|wS51;v zN>oTtUV%zd=q19>8vh{%oRIrdNeA3p+Zq3j^XH%Zrv^YZ=1=7_;QnWge@nvu6V=Z? zcrHhP+dozOw_N?-n9tnQlFk5)oB{OE&ooxg1q<-*{sMr{+S=i##=MQBuBjCu6!BSE z{$<-td<~590C+Qi3;!7xaHj$Yg!u)u06-zY!dlnhKQeSL0f(&F?B4^ri3iYLhQ9!E z06ZiA1PB;8TwHk_9ZUfFzNY%mf*3CuqAy^_-v8801&ICQ{)Hg}&_Dmc@W+JgCEOP& z>XjS-%n;zB{R`X?@;|}>+G}iQXm2k8NFr@$^&jc}m+)S}G_YELuG|5b%+Evg&t(bl zGX0{p0BK_jM|*%gh>)p;A+NqZfI(8%O85V=slOxuE_Ped0Cc!4pu_)a9HD^M{}%+p zcDjHJj*0*&K@n?9LkVkZb4QzBcg<0mSzIANIGzE}QnKfU1MXmX{UbL-mh`@ zj#cbS04Pg<3jp+|f(`fuzi6`Pf2{7;cp0TAw_gB#%mm2I@K-!=s$b(-**oZ3So}8| z&VX#@AppFo*MN@+&YxNb1cb@>Pk6s>U4bi2ixS`g0RdJIKR?LN#arZ;XcE@O|Mh9V z%ya$>1}FYYu;0ph{?evi=Fom-!I1plEWhrCm$^otF+~)9iTQH!@t4e_FI)UF)88{X ziRv%W|81nd$ouya`emZEXJ}xJe}evN+O?OcFEf@rqrw{g-=O_Z`Ac2`{sj1GWk1W$ z93{`i1_1b{m+EKi&dbU#GnPD8&iZ@h|0Iq0@3X;|@GrAaJj0Wj{H<;Of&ZPA#7m-= zO7hP{;}(A-`cwP=i>~}j_?MFC&+vv;{{;VHviuVEr4slvY_Ih{!T#CL@DlT-tn4%9 zh~qzD{t*T6-@c451zev|o1A_F^}oxxzHI(WQOai`Nw?o6`lEaPEm8TB;H3cIGXbCH zZxH;09NamwGP0v&WfK}UzG44;F_kVxX z{%2pk3~qVGchCF{{C^I0dD(9-V*#GYngQ#P|Mg>jJ;{2x1O1GdRQor~e|5*-_oH97 z^QHIu8PmG;e`EgB81>Sl_>An>@pt6k_b>kK7rgLtJ@bfu{u|HBY3HBJ=zk33FFh~M zG#Fj~H>LY;ewhEX`9EHlKRZo+dR?B2B>?Qd`Cw$kK>+KZp81p@ki D&=%~j diff --git a/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.properties b/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 2c2bbe5f9..000000000 --- a/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-bin.zip diff --git a/adapter/kvstore/java/gradlew b/adapter/kvstore/java/gradlew deleted file mode 100755 index cccdd3d51..000000000 --- a/adapter/kvstore/java/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/adapter/kvstore/java/gradlew.bat b/adapter/kvstore/java/gradlew.bat deleted file mode 100644 index e95643d6a..000000000 --- a/adapter/kvstore/java/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/adapter/kvstore/java/settings.gradle b/adapter/kvstore/java/settings.gradle deleted file mode 100644 index 371bf447f..000000000 --- a/adapter/kvstore/java/settings.gradle +++ /dev/null @@ -1,18 +0,0 @@ -/* - * This settings file was generated by the Gradle 'init' task. - * - * The settings file is used to specify which projects to include in your build. - * In a single project build this file can be empty or even removed. - * - * Detailed information about configuring a multi-project build in Gradle can be found - * in the user guide at https://docs.gradle.org/4.4.1/userguide/multi_project_builds.html - */ - -/* -// To declare projects as part of a multi-project build use the 'include' method -include 'shared' -include 'api' -include 'services:webservice' -*/ - -rootProject.name = 'hermes_kvstore-1.0.0' diff --git a/adapter/kvstore/java/src/kvstore/java/KVStore.java b/adapter/kvstore/java/src/kvstore/java/KVStore.java deleted file mode 100644 index 8bd924830..000000000 --- a/adapter/kvstore/java/src/kvstore/java/KVStore.java +++ /dev/null @@ -1,14 +0,0 @@ -package hermes_kvstore.java; -import hermes_kvstore.java.KVTable; -import hermes.java.Hermes; - -public class KVStore { - public static void connect() { - Hermes hermes = Hermes.getInstance(); - hermes.create(); - } - - public static KVTable getTable(String table_name) { - return new KVTable(table_name); - } -} diff --git a/adapter/kvstore/java/src/kvstore/java/KVTable.java b/adapter/kvstore/java/src/kvstore/java/KVTable.java deleted file mode 100644 index 596226ada..000000000 --- a/adapter/kvstore/java/src/kvstore/java/KVTable.java +++ /dev/null @@ -1,160 +0,0 @@ -package hermes_kvstore.java; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.HashMap; -import java.util.Set; -import java.util.HashSet; -import java.io.IOException; -import java.io.ByteArrayOutputStream; -import java.io.ByteArrayInputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - -import hermes.java.Hermes; -import hermes.java.Bucket; -import hermes.java.Blob; -import hermes.java.MdLockType; -import hermes.java.UniqueId; - -public class KVTable { - Bucket bkt_; - - /** Emplace Constructor */ - KVTable(String table_name) { - Hermes hermes = Hermes.getInstance(); - bkt_ = hermes.getBucket(table_name); - } - - /** Serialize a Map into a blob */ - private Blob blobSerialize(T obj) throws IOException { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(baos); - oos.writeObject(obj); - oos.close(); - byte[] bytes = baos.toByteArray(); - return new Blob(ByteBuffer.wrap(bytes)); - } catch (IOException e) { - e.printStackTrace(); - throw e; - } - } - - /** Deserialize a Map from a blob */ - private T blobDeserialize(Blob blob) throws IOException, ClassNotFoundException { - try { - byte[] bytes = blob.array(); - ByteArrayInputStream istream = new ByteArrayInputStream(bytes); - ObjectInputStream ois = new ObjectInputStream(istream); - T obj = (T) ois.readObject(); - ois.close(); - blob.close(); - return obj; - } catch (IOException e) { - e.printStackTrace(); - throw e; - } catch (ClassNotFoundException e) { - e.printStackTrace(); - throw e; - } - } - - public String getFieldKey(String key, String field_name) { - return String.format("%s-%s", key, field_name); - } - - public String getKvMetadataKey() { - return "hermes-kvmetadata"; - } - - /** - * Insert a new record into the table - * */ - public void insert(String key, Map val) throws IOException { - /*UniqueId md_id = bkt_.getBlobId(getKvMetadataKey()); - if (md_id.isNull()) { - Set keys = new HashSet(val.keySet()); - Blob md_blob = blobSerialize(keys); - bkt_.put(getKvMetadataKey(), md_blob); - md_blob.close(); - } - for (Map.Entry entry : val.entrySet()) { - String field_key = getFieldKey(key, entry.getKey()); - Blob blob = blobSerialize(entry.getValue()); - bkt_.put(field_key, blob); - blob.close(); - }*/ - Blob blob = blobSerialize(val); - bkt_.put(key, blob); - blob.close(); - } - - /** - * Create or insert a record into the table - * - * @param key the record key - * @param val the values to update in the record - * @return None - * */ - public void update(String key, Map val) throws IOException, ClassNotFoundException { - /*insert(key, val);*/ - UniqueId blob_id = bkt_.getBlobId(key); - if (blob_id.isNull()) { - insert(key, val); - } else { - bkt_.lockBlob(blob_id, MdLockType.kExternalWrite); - Blob orig_blob = bkt_.get(blob_id); - Map old_val = blobDeserialize(orig_blob); - old_val.putAll(val); - Blob new_blob = blobSerialize(old_val); - bkt_.put(key, new_blob); - bkt_.unlockBlob(blob_id, MdLockType.kExternalWrite); - new_blob.close(); - } - } - - /** - * Get a subset of fields from a record - * - * @param key the record key - * @param field_set the field in the record to update - * @return The blob containing only the field's data - * */ - public Map read(String key, Set field_set) throws IOException, ClassNotFoundException { - /*HashMap map = new HashMap(); - if (field_set.isEmpty()) { - UniqueId md_id = bkt_.getBlobId(getKvMetadataKey()); - Blob md_blob = bkt_.get(md_id); - field_set = (HashSet)blobDeserialize(md_blob); - } - for (String field_name : field_set) { - UniqueId blob_id = bkt_.getBlobId(getFieldKey(key, field_name)); - Blob blob = bkt_.get(blob_id); - map.put(field_name, blobDeserialize(blob)); - } - return map;*/ - - UniqueId blob_id = bkt_.getBlobId(key); - bkt_.lockBlob(blob_id, MdLockType.kExternalRead); - Blob orig_blob = bkt_.get(blob_id); - Map old_val = blobDeserialize(orig_blob); - bkt_.unlockBlob(blob_id, MdLockType.kExternalRead); - return old_val; - } - - public Map read(String key) throws IOException, ClassNotFoundException { - return read(key, new HashSet()); - } - - /** Delete a record */ - public void erase(String key) { - UniqueId blob_id = bkt_.getBlobId(key); - bkt_.destroyBlob(blob_id); - } - - /** Destroy this table */ - public void destroy() { - bkt_.destroy(); - } -}; diff --git a/adapter/kvstore/java/src/test/java/KvstoreJniTest.java b/adapter/kvstore/java/src/test/java/KvstoreJniTest.java deleted file mode 100644 index 12168bf67..000000000 --- a/adapter/kvstore/java/src/test/java/KvstoreJniTest.java +++ /dev/null @@ -1,34 +0,0 @@ -import org.junit.Test; -import static org.junit.Assert.*; -import java.nio.*; -import java.util.*; -import hermes_kvstore.java.KVStore; -import hermes_kvstore.java.KVTable; - -import java.lang.management.ManagementFactory; - -public class KvstoreJniTest { - @Test - public void testKvPutGet() throws Exception { - String pid = ManagementFactory.getRuntimeMXBean().getName(); - System.out.println(pid); - KVStore.connect(); - KVTable table = KVStore.getTable("hello");/**/ - - HashMap record = new HashMap(); - try { - record.put("f0", "abcde"); - record.put("f1", "12345"); - record.put("f2", "ABCDE"); - table.update("0", record); - table.update("0", record); - Map read_record = table.read("0"); - for (Map.Entry entry : read_record.entrySet()) { - assertTrue(entry.getValue().equals(record.get(entry.getKey()))); - } - } catch (Exception e) { - e.printStackTrace(); - throw e; - } - } -} diff --git a/adapter/kvstore/kvstore.cc b/adapter/kvstore/kvstore.cc deleted file mode 100644 index a40648c15..000000000 --- a/adapter/kvstore/kvstore.cc +++ /dev/null @@ -1,140 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "kvstore.h" - -namespace hermes::adapter { - -/**==================================== - * KVRecord - * ===================================*/ - -/** Convert the record from a single array of bytes */ -KVRecord::KVRecord(hapi::Blob &blob) { - BlobSerializer serial(blob); - while (serial.off_ < blob.size()) { - std::string field_name; - hapi::Blob field_val; - serial >> field_name; - serial >> field_val; - record_.emplace(std::move(field_name), std::move(field_val)); - } -} - -/** Convert the record into a single array of bytes */ -hapi::Blob KVRecord::value() { - BlobSerializeCounter counter; - - // Get the total size of the serialized blob - for (auto &[field_name, field_val] : record_) { - counter << field_name; - counter << field_val; - } - - // Produce the serialized blob - hapi::Blob blob(counter.size_); - BlobSerializer serial(blob); - for (auto &[field_name, field_val] : record_) { - serial << field_name; - serial << field_val; - } - - return blob; -} - -/**==================================== - * KVTable - * ===================================*/ - -/** - * Create or insert a record into the table - * - * @param key the record key - * @param val the values to update in the record - * @return None - * */ -void KVTable::Update(const std::string &key, KVRecord &val) { - hermes::BlobId blob_id; - bkt_.GetBlobId(key, blob_id); - if (!blob_id.IsNull()) { - hapi::Context ctx; - hapi::Blob record_serial; - bkt_.LockBlob(blob_id, MdLockType::kExternalWrite); - bkt_.Get(blob_id, record_serial, ctx); - KVRecord orig(record_serial); - for (auto &[field, field_val] : val.record_) { - orig.record_[field] = field_val; - } - bkt_.Put(key, orig.value(), blob_id, ctx); - bkt_.UnlockBlob(blob_id, MdLockType::kExternalWrite); - } else { - hapi::Context ctx; - hapi::Blob record_serial; - bkt_.LockBlob(blob_id, MdLockType::kExternalWrite); - bkt_.Put(key, val.value(), blob_id, ctx); - bkt_.UnlockBlob(blob_id, MdLockType::kExternalWrite); - } -} - -/** - * Get a subset of fields from a record - * - * @param key the record key - * @param field the field in the record to update - * @return The blob containing only the field's data - * */ -KVRecord KVTable::Read(const std::string &key, const KVFieldSet &field) { - hapi::Blob blob; - hermes::BlobId blob_id; - hapi::Context ctx; - bkt_.GetBlobId(key, blob_id); - bkt_.Get(blob_id, blob, ctx); - KVRecord orig_record(blob); - if (field.size() == 0) { - return orig_record; - } - - KVRecord new_record; - for (const std::string &field_name : field) { - new_record.record_[field_name] = orig_record.record_[field_name]; - } - return new_record; -} - -/** Delete a record */ -void KVTable::Erase(const std::string &key) { - hapi::Context ctx; - hermes::BlobId blob_id; - bkt_.GetBlobId(key, blob_id); - bkt_.DestroyBlob(blob_id, ctx); -} - -/** Destroy this table */ -void KVTable::Destroy() { - bkt_.Destroy(); -} - -/**==================================== - * KVStore - * ===================================*/ - -/** Connect to Hermes */ -void KVStore::Connect() { - HERMES->Create(hermes::HermesType::kClient); -} - -/** Get or create a table */ -KVTable KVStore::GetTable(const std::string table_name) { - return KVTable(HERMES->GetBucket(table_name)); -} - -} // namespace hermes::adapter diff --git a/adapter/kvstore/kvstore.h b/adapter/kvstore/kvstore.h deleted file mode 100644 index b914e9bb0..000000000 --- a/adapter/kvstore/kvstore.h +++ /dev/null @@ -1,177 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ADAPTER_KVSTORE_KVSTORE_H_ -#define HERMES_ADAPTER_KVSTORE_KVSTORE_H_ - -#include -#include -#include "hermes.h" - -namespace hermes::adapter { - -/** Denotes the set of fields for a record */ -typedef std::unordered_set KVFieldSet; - -/** Measure the serialized size */ -struct BlobSerializeCounter { - size_t size_; - - /** Constructor */ - BlobSerializeCounter() : size_(0) {} - - /** Serialize std::string into blob */ - BlobSerializeCounter& operator<<(const std::string &data) { - (*this) << data.size(); - size_ += data.size(); - return *this; - } - - /** Serialize blob into blob */ - BlobSerializeCounter& operator<<(const hapi::Blob &data) { - (*this) << data.size(); - size_ += data.size(); - return *this; - } - - /** Serialize size_t into blob */ - BlobSerializeCounter& operator<<(const size_t &data) { - size_ += sizeof(size_t); - return *this; - } -}; - -/** Serialize data into and out of blob */ -struct BlobSerializer { - hapi::Blob &blob_; - size_t off_; - - /** Constructor */ - explicit BlobSerializer(hapi::Blob &blob) : blob_(blob), off_(0) {} - - /** Serialize a string type */ - template - void SerializeString(const StringT &data) { - (*this) << data.size(); - memcpy(blob_.data() + off_, data.data(), data.size()); - off_ += data.size(); - } - - /** Deserialize a string type */ - template - void DeserializeString(StringT &data) { - size_t size; - (*this) >> size; - data.resize(size); - memcpy(data.data(), blob_.data() + off_, data.size()); - off_ += data.size(); - } - - /** Serialize std::string into blob */ - BlobSerializer& operator<<(const std::string &data) { - SerializeString(data); - return *this; - } - - /** Deserialize std::string from blob */ - BlobSerializer& operator>>(std::string &data) { - DeserializeString(data); - return *this; - } - - /** Serialize blob into blob */ - BlobSerializer& operator<<(const hapi::Blob &data) { - SerializeString(data); - return *this; - } - - /** Deserialize blob from blob */ - BlobSerializer& operator>>(hapi::Blob &data) { - DeserializeString(data); - return *this; - } - - /** Serialize size_t into blob */ - BlobSerializer& operator<<(const size_t &data) { - memcpy(blob_.data() + off_, &data, sizeof(size_t)); - off_ += sizeof(size_t); - return *this; - } - - /** Deserialize size_t from blob */ - BlobSerializer& operator>>(size_t &data) { - memcpy(&data, blob_.data() + off_, sizeof(size_t)); - off_ += sizeof(size_t); - return *this; - } -}; - -/** Denotes a record to place in a table */ -struct KVRecord { - std::unordered_map record_; - - /** Default constructor */ - KVRecord() = default; - - /** Convert the record from a single array of bytes */ - explicit KVRecord(hapi::Blob &blob); - - /** Convert the record into a single array of bytes */ - hapi::Blob value(); -}; - -/** Denotes a set of records */ -class KVTable { - public: - hapi::Bucket bkt_; - - /** Emplace Constructor */ - explicit KVTable(const hapi::Bucket &bkt) : bkt_(bkt) {} - - /** - * Create or insert a record into the table - * - * @param key the record key - * @param val the values to update in the record - * @return None - * */ - void Update(const std::string &key, KVRecord &val); - - /** - * Get a subset of fields from a record - * - * @param key the record key - * @param field the field in the record to update - * @return The blob containing only the field's data - * */ - KVRecord Read(const std::string &key, const KVFieldSet &field); - - /** Delete a record */ - void Erase(const std::string &key); - - /** Destroy this table */ - void Destroy(); -}; - -/** The key-value store instance */ -class KVStore { - public: - /** Connect to Hermes */ - void Connect(); - - /** Get or create a table */ - KVTable GetTable(const std::string table_name); -}; - -} // namespace hermes::adapter - -#endif // HERMES_ADAPTER_KVSTORE_KVSTORE_H_ diff --git a/adapter/kvstore/tests.py b/adapter/kvstore/tests.py deleted file mode 100644 index e5718b51b..000000000 --- a/adapter/kvstore/tests.py +++ /dev/null @@ -1,22 +0,0 @@ -from py_hermes_ci.test_manager import TestManager -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - - -class KvstoreTestManager(TestManager): - def spawn_all_nodes(self): - return self.spawn_info() - - def set_paths(self): - self.KVSTORE_CMD = f"./gradlew test" - self.disable_testing = False - - def test_hermes_kvstore_java(self): - return - spawn_info = self.spawn_info(nprocs=1, - hermes_conf='hermes_server', - cwd=f"{self.MY_DIR}/java") - self.start_daemon(spawn_info) - node = Exec(self.KVSTORE_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code diff --git a/adapter/mapper/balanced_mapper.cc b/adapter/mapper/balanced_mapper.cc deleted file mode 100644 index 3afece24e..000000000 --- a/adapter/mapper/balanced_mapper.cc +++ /dev/null @@ -1,43 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "balanced_mapper.h" -#include "api/hermes.h" - -namespace hermes::adapter { - -/** - * Convert a range defined by \a off and \a size into specific - * blob placements. - */ -void BalancedMapper::map(size_t off, size_t size, - size_t page_size, - BlobPlacements &ps) { - HILOG(kDebug, "Mapping File with offset {} and size {}", off, size); - size_t kPageSize = page_size; - size_t size_mapped = 0; - while (size > size_mapped) { - BlobPlacement p; - p.bucket_off_ = off + size_mapped; - p.page_ = p.bucket_off_ / kPageSize; - p.page_size_ = page_size; - p.blob_off_ = p.bucket_off_ % kPageSize; - auto left_size_page = kPageSize - p.blob_off_; - p.blob_size_ = left_size_page < size - size_mapped ? left_size_page - : size - size_mapped; - ps.emplace_back(p); - size_mapped += p.blob_size_; - } -} - - -} // namespace hermes::adapter diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 000000000..4eb76ca54 --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,66 @@ +cmake_minimum_required(VERSION 3.10) +project(labstor) + +set(CMAKE_CXX_STANDARD 17) + +#------------------------------------------------------------------------------ +# Build Tests +#------------------------------------------------------------------------------ + +add_executable(test_performance_exec + ${TEST_MAIN}/main.cc + test_init.cc + test_latency.cc) +add_dependencies(test_performance_exec + ${Labstor_RUNTIME_DEPS} hermes) +target_link_libraries(test_performance_exec + ${Labstor_RUNTIME_LIBRARIES} hermes Catch2::Catch2 + MPI::MPI_CXX) + +#add_executable(test_performance_exec +# ${TEST_MAIN}/main.cc +# test_init.cc +# test_latency.cc +# test_zmq.cc +#) +#add_dependencies(test_performance_exec +# ${Labstor_CLIENT_DEPS} hermes) +#target_link_libraries(test_performance_exec +# ${Labstor_CLIENT_LIBRARIES} hermes Catch2::Catch2 +# MPI::MPI_CXX ${ZMQ_LIBRARIES}) + +add_executable(test_hermes_api + hermes_api_bench.cc) +add_dependencies(test_hermes_api + ${Labstor_CLIENT_DEPS} hermes) +target_link_libraries(test_hermes_api + ${Labstor_CLIENT_LIBRARIES} hermes + Catch2::Catch2 MPI::MPI_CXX) + +#------------------------------------------------------------------------------ +# Test Cases +#------------------------------------------------------------------------------ + +# STRING TESTS +add_test(NAME test_performance COMMAND + ${CMAKE_BINARY_DIR}/bin/test_performance_exec "TestIpc") + +#------------------------------------------------------------------------------ +# Install Targets +#------------------------------------------------------------------------------ +install(TARGETS + test_performance_exec + test_hermes_api + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(test_performance_exec) + set_coverage_flags(test_hermes_api) +endif() diff --git a/benchmarks/api_bench.cc b/benchmark/hermes_api_bench.cc similarity index 58% rename from benchmarks/api_bench.cc rename to benchmark/hermes_api_bench.cc index 3cdd40c91..928d53053 100644 --- a/benchmarks/api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -13,10 +13,11 @@ #include #include #include -#include "hermes.h" -#include "bucket.h" +#include "hermes/hermes.h" +#include "hermes/bucket.h" +#include "labstor/work_orchestrator/affinity.h" -namespace hapi = hermes::api; +namespace hapi = hermes; using Timer = hshm::HighResMonotonicTimer; /** Gather times per-process */ @@ -36,20 +37,18 @@ void GatherTimes(std::string test_name, size_t io_size, Timer &t) { } /** Each process PUTS into the same bucket, but with different blob names */ -void PutTest(hapi::Hermes *hermes, - int nprocs, int rank, +void PutTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size) { Timer t; - auto bkt = hermes->GetBucket("hello"); - hermes::api::Context ctx; - hermes::BlobId blob_id; + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); hermes::Blob blob(blob_size); t.Resume(); for (int j = 0; j < repeat; ++j) { for (size_t i = 0; i < blobs_per_rank; ++i) { size_t blob_name_int = rank * blobs_per_rank + i; std::string name = std::to_string(blob_name_int); - bkt.Put(name, blob, blob_id, ctx); + bkt.Put(name, blob, ctx); } } t.Pause(); @@ -60,20 +59,18 @@ void PutTest(hapi::Hermes *hermes, * Each process GETS from the same bucket, but with different blob names * MUST run PutTest first. * */ -void GetTest(hapi::Hermes *hermes, - int nprocs, int rank, +void GetTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size) { Timer t; - auto bkt = hermes->GetBucket("hello"); - hermes::api::Context ctx; - hermes::BlobId blob_id; + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); t.Resume(); for (int j = 0; j < repeat; ++j) { for (size_t i = 0; i < blobs_per_rank; ++i) { size_t blob_name_int = rank * blobs_per_rank + i; std::string name = std::to_string(blob_name_int); hermes::Blob ret; - bkt.GetBlobId(name, blob_id); + hermes::BlobId blob_id = bkt.GetBlobId(name); bkt.Get(blob_id, ret, ctx); } } @@ -82,34 +79,37 @@ void GetTest(hapi::Hermes *hermes, } /** Each process PUTs then GETs */ -void PutGetTest(hapi::Hermes *hermes, - int nprocs, int rank, int repeat, +void PutGetTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size) { - PutTest(hermes, nprocs, rank, repeat, blobs_per_rank, blob_size); + PutTest(nprocs, rank, repeat, blobs_per_rank, blob_size); MPI_Barrier(MPI_COMM_WORLD); - GetTest(hermes, nprocs, rank, repeat, blobs_per_rank, blob_size); + GetTest(nprocs, rank, repeat, blobs_per_rank, blob_size); } /** Each process creates a set of buckets */ -void CreateBucketTest(hapi::Hermes *hermes, int nprocs, int rank, +void CreateBucketTest(int nprocs, int rank, size_t bkts_per_rank) { Timer t; t.Resume(); + hapi::Context ctx; + std::unordered_map mdm_; for (size_t i = 0; i < bkts_per_rank; ++i) { - int bkt_name = rank * bkts_per_rank + i; - hapi::Bucket bkt = hermes->GetBucket(std::to_string(bkt_name)); + int bkt_name_int = rank * bkts_per_rank + i; + std::string bkt_name = std::to_string(bkt_name_int); + hermes::Bucket bkt(bkt_name, ctx); } t.Pause(); GatherTimes("CreateBucket", bkts_per_rank * nprocs, t); } /** Each process gets existing buckets */ -void GetBucketTest(hapi::Hermes *hermes, int nprocs, int rank, +void GetBucketTest(int nprocs, int rank, size_t bkts_per_rank) { // Initially create the buckets + hapi::Context ctx; for (size_t i = 0; i < bkts_per_rank; ++i) { int bkt_name = rank * bkts_per_rank + i; - hapi::Bucket bkt = hermes->GetBucket(std::to_string(bkt_name)); + hermes::Bucket bkt(std::to_string(bkt_name), ctx); } // Get existing buckets @@ -117,104 +117,42 @@ void GetBucketTest(hapi::Hermes *hermes, int nprocs, int rank, t.Resume(); for (size_t i = 0; i < bkts_per_rank; ++i) { int bkt_name = rank * bkts_per_rank + i; - hapi::Bucket bkt = hermes->GetBucket(std::to_string(bkt_name)); + hapi::Bucket bkt(std::to_string(bkt_name), ctx); } t.Pause(); GatherTimes("CreateBucket", bkts_per_rank * nprocs, t); } -/** Each process creates (no data) a blob in a single bucket */ -void CreateBlobOneBucketTest(hapi::Hermes *hermes, - int nprocs, int rank, - size_t blobs_per_rank) { - Timer t; - hapi::Bucket bkt = hermes->GetBucket("CreateBlobOneBucket"); - hapi::Context ctx; - hermes::BlobId blob_id; - - t.Resume(); - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.TryCreateBlob(name, blob_id, ctx); - } - t.Pause(); - GatherTimes("CreateBlobOneBucket", blobs_per_rank * nprocs, t); -} - -/** Each process creates (no data) a blob in a process-specific bucket */ -void CreateBlobPerBucketTest(hapi::Hermes *hermes, - int nprocs, int rank, - size_t blobs_per_rank) { - Timer t; - hapi::Bucket bkt = hermes->GetBucket( - hshm::Formatter::format("CreateBlobPerBucket{}", rank)); - hapi::Context ctx; - hermes::BlobId blob_id; - - t.Resume(); - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.TryCreateBlob(name, blob_id, ctx); - } - t.Pause(); - GatherTimes("CreateBlobPerBucket", nprocs * blobs_per_rank, t); -} - /** Each process deletes a number of buckets */ -void DeleteBucketTest(hapi::Hermes *hermes, - int nprocs, int rank, +void DeleteBucketTest(int nprocs, int rank, size_t bkt_per_rank, size_t blobs_per_bucket) { Timer t; hapi::Context ctx; - hermes::BlobId blob_id; // Create the buckets for (size_t i = 0; i < bkt_per_rank; ++i) { - hapi::Bucket bkt = hermes->GetBucket( - hshm::Formatter::format("DeleteBucket{}", rank)); + hapi::Bucket bkt(hshm::Formatter::format("DeleteBucket{}", rank), ctx); + hapi::Blob blob; for (size_t j = 0; j < blobs_per_bucket; ++j) { std::string name = std::to_string(j); - bkt.TryCreateBlob(name, blob_id, ctx); + bkt.Put(name, blob, ctx); } } // Delete the buckets t.Resume(); for (size_t i = 0; i < bkt_per_rank; ++i) { - hapi::Bucket bkt = hermes->GetBucket( - hshm::Formatter::format("DeleteBucket{}", rank)); - size_t blob_name_int = rank * bkt_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.TryCreateBlob(name, blob_id, ctx); + hapi::Bucket bkt(hshm::Formatter::format("DeleteBucket{}", rank), ctx); + bkt.Destroy(); } t.Pause(); GatherTimes("DeleteBucket", nprocs * bkt_per_rank * blobs_per_bucket, t); } /** Each process deletes blobs from a single bucket */ -void DeleteBlobOneBucket(hapi::Hermes *hermes, - int nprocs, int rank, +void DeleteBlobOneBucket(int nprocs, int rank, size_t blobs_per_rank) { - Timer t; - CreateBlobOneBucketTest(hermes, nprocs, rank, blobs_per_rank); - MPI_Barrier(MPI_COMM_WORLD); - hapi::Bucket bkt = hermes->GetBucket("CreateBlobOneBucket"); - hermes::BlobId blob_id; - hapi::Context ctx; - - // Delete blobs from a bucket - t.Resume(); - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.GetBlobId(name, blob_id); - bkt.DestroyBlob(blob_id, ctx); - } - t.Pause(); - GatherTimes("CreateBlobOneBucket", nprocs * blobs_per_rank, t); } @@ -248,12 +186,17 @@ int main(int argc, char **argv) { MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - auto hermes = hapi::Hermes::Create(hermes::HermesType::kClient); + TRANSPARENT_LABSTOR(); + HERMES->ClientInit(); // Get mode REQUIRE_ARGC_GE(2) std::string mode = argv[1]; + // Set CPU affinity + // TODO(logan): remove + ProcessAffiner::SetCpuAffinity(getpid(), 8); + MPI_Barrier(MPI_COMM_WORLD); HIPRINT("Beginning {}\n", mode) @@ -263,39 +206,29 @@ int main(int argc, char **argv) { REQUIRE_ARGC(4) size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); size_t blobs_per_rank = atoi(argv[3]); - PutTest(hermes, nprocs, rank, 1, blobs_per_rank, blob_size); + PutTest(nprocs, rank, 1, blobs_per_rank, blob_size); } else if (mode == "putget") { REQUIRE_ARGC(4) size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); size_t blobs_per_rank = atoi(argv[3]); - PutGetTest(hermes, nprocs, rank, 1, blobs_per_rank, blob_size); + PutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size); } else if (mode == "create_bkt") { REQUIRE_ARGC(3) size_t bkts_per_rank = atoi(argv[2]); - CreateBucketTest(hermes, nprocs, rank, bkts_per_rank); + CreateBucketTest(nprocs, rank, bkts_per_rank); } else if (mode == "get_bkt") { REQUIRE_ARGC(3) size_t bkts_per_rank = atoi(argv[2]); - GetBucketTest(hermes, nprocs, rank, bkts_per_rank); - } else if (mode == "create_blob_1bkt") { - REQUIRE_ARGC(3) - size_t blobs_per_rank = atoi(argv[2]); - CreateBlobOneBucketTest(hermes, nprocs, rank, blobs_per_rank); - } else if (mode == "create_blob_Nbkt") { - REQUIRE_ARGC(3) - size_t blobs_per_rank = atoi(argv[2]); - CreateBlobPerBucketTest(hermes, nprocs, rank, blobs_per_rank); + GetBucketTest(nprocs, rank, bkts_per_rank); } else if (mode == "del_bkt") { REQUIRE_ARGC(4) size_t bkt_per_rank = atoi(argv[2]); size_t blobs_per_bkt = atoi(argv[3]); - DeleteBucketTest(hermes, nprocs, rank, bkt_per_rank, blobs_per_bkt); + DeleteBucketTest(nprocs, rank, bkt_per_rank, blobs_per_bkt); } else if (mode == "del_blobs") { REQUIRE_ARGC(4) size_t blobs_per_rank = atoi(argv[2]); - DeleteBlobOneBucket(hermes, nprocs, rank, blobs_per_rank); + DeleteBlobOneBucket(nprocs, rank, blobs_per_rank); } - - hermes->Finalize(); MPI_Finalize(); } diff --git a/hermes_shm/benchmark/lock/test_init.cc b/benchmark/test_init.cc similarity index 91% rename from hermes_shm/benchmark/lock/test_init.cc rename to benchmark/test_init.cc index 0fe134776..63132b16a 100644 --- a/hermes_shm/benchmark/lock/test_init.cc +++ b/benchmark/test_init.cc @@ -10,9 +10,13 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "labstor/api/labstor_client.h" +#include "basic_test.h" #include "test_init.h" void MainPretest() { + TRANSPARENT_LABSTOR(); } void MainPosttest() { diff --git a/hermes_shm/test/unit/backend/test_init.cc b/benchmark/test_init.h similarity index 83% rename from hermes_shm/test/unit/backend/test_init.cc rename to benchmark/test_init.h index 82bac123b..a6c71f3ec 100644 --- a/hermes_shm/test/unit/backend/test_init.cc +++ b/benchmark/test_init.h @@ -11,8 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "basic_test.h" +#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ -void MainPretest() {} +#include "labstor/labstor_types.h" -void MainPosttest() {} +#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/benchmark/test_latency.cc b/benchmark/test_latency.cc new file mode 100644 index 000000000..ea8f14c34 --- /dev/null +++ b/benchmark/test_latency.cc @@ -0,0 +1,346 @@ +// +// Created by llogan on 7/1/23. +// + +#include +#include "basic_test.h" +#include "labstor/api/labstor_client.h" +#include "labstor_admin/labstor_admin.h" +#include "small_message/small_message.h" +#include "hermes_shm/util/timer.h" +#include "labstor/work_orchestrator/affinity.h" +#include "hermes/hermes.h" +#include "labstor/work_orchestrator/worker.h" +#include "labstor/api/labstor_runtime.h" + +/** The performance of getting a queue */ +TEST_CASE("TestGetQueue") { + /*labstor::QueueId qid(0, 3); + LABSTOR_ADMIN->CreateQueueRoot(labstor::DomainId::GetLocal(), qid, + 16, 16, 256, + hshm::bitfield32_t(0)); + LABSTOR_CLIENT->GetQueue(qid); + + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + labstor::MultiQueue *queue = LABSTOR_CLIENT->GetQueue(qid); + REQUIRE(queue->id_ == qid); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec());*/ +} + +/** Single-thread performance of allocating + freeing tasks */ +TEST_CASE("TestHshmAllocateFree") { + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + size_t count = (1 << 8); + size_t reps = ops / count; + for (size_t i = 0; i < reps; ++i) { + std::vector tasks(count); + for (size_t j = 0; j < count; ++j) { + tasks[j] = LABSTOR_CLIENT->NewTaskRoot().ptr_; + } + for (size_t j = 0; j < count; ++j) { + LABSTOR_CLIENT->DelTask(tasks[j]); + } + } + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of emplacing, and popping a mpsc_ptr_queue */ +TEST_CASE("TestPointerQueueEmplacePop") { + size_t ops = (1 << 20); + auto queue_ptr = hipc::make_uptr>(ops); + auto queue = queue_ptr.get(); + hipc::Pointer p; + + hshm::Timer t; + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + queue->emplace(p); + queue->pop(p); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of empacling + popping vec */ +TEST_CASE("TestPointerQueueVecEmplacePop") { + auto queues_ptr = hipc::make_uptr>>(16); + auto queues = queues_ptr.get(); + hipc::Pointer p; + + hshm::Timer t; + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + t.Resume(); + auto &queue = (*queues)[0]; + queue.emplace(p); + queue.pop(p); + t.Pause(); + } + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of getting, emplacing, and popping a queue */ +TEST_CASE("TestHshmQueueEmplacePop") { + labstor::QueueId qid(0, 3); + u32 ops = (1 << 20); + std::vector queue_info = { + {16, 16, ops, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + labstor::LaneData entry; + auto task = LABSTOR_CLIENT->NewTaskRoot(); + entry.p_ = task.shm_; + + hshm::Timer t; + t.Resume(); + labstor::Lane &lane = queue->GetLane(0, 0); + for (size_t i = 0; i < ops; ++i) { + queue->Emplace(0, 0, entry); + lane.pop(); + } + t.Pause(); + + LABSTOR_CLIENT->DelTask(task); + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of getting a lane from a queue */ +TEST_CASE("TestHshmQueueGetLane") { + labstor::QueueId qid(0, 3); + std::vector queue_info = { + {16, 16, 256, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + labstor::LaneGroup group = queue->GetGroup(0); + + hshm::Timer t; + size_t ops = (1 << 20); + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + queue->GetLane(0, i % group.num_lanes_); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of getting, emplacing, and popping a queue */ +TEST_CASE("TestHshmQueueAllocateEmplacePop") { + labstor::QueueId qid(0, 3); + std::vector queue_info = { + {16, 16, 256, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + labstor::Lane &lane = queue->GetLane(0, 0); + + hshm::Timer t; + size_t ops = (1 << 20); + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + labstor::LaneData entry; + auto task = LABSTOR_CLIENT->NewTaskRoot(); + entry.p_ = task.shm_; + queue->Emplace(0, 0, entry); + lane.pop(); + LABSTOR_CLIENT->DelTask(task); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Time to spawn and join a thread */ +TEST_CASE("TestSpawnJoinThread") { + hshm::Timer t; + t.Resume(); + size_t count = 16 * (1 << 10); + for (int i = 0; i < count; ++i) { + std::thread thread([]() { }); + thread.join(); + } + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", count / t.GetUsec()); +} + +/** Time to spawn and join a thread */ +TEST_CASE("TestSpawnJoinArgoThread") { + hshm::Timer t; + t.Resume(); + ABT_xstream xstream; + ABT_thread tl_thread_; + size_t count = 16 * (1 << 10); + ABT_init(0, nullptr); + int ret = ABT_xstream_create(ABT_SCHED_NULL, &xstream); + if (ret != ABT_SUCCESS) { + HELOG(kFatal, "Could not create argobots xstream"); + } + for (int i = 0; i < count; ++i) { + int ret = ABT_thread_create_on_xstream(xstream, + [](void *args) { }, nullptr, + ABT_THREAD_ATTR_NULL, &tl_thread_); + if (ret != ABT_SUCCESS) { + HELOG(kFatal, "Couldn't spawn worker"); + } + ABT_thread_join(tl_thread_); + } + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", count / t.GetUsec()); +} + +void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) { + LABSTOR_RUNTIME->Create(); + + labstor::Worker worker(0); + std::vector> queues; + for (u32 i = 0; i < num_queues; ++i) { + labstor::QueueId qid(0, i + 1); + std::vector queue_info = { + {num_lanes, num_lanes, 256, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + queues.emplace_back(std::move(queue)); + for (u32 j = 0; j < num_lanes; ++j) { + worker.PollQueues({{0, j, queue.get()}}); + } + } + + labstor::small_message::Client client; + LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetLocal(), "small_message");\ + client.CreateRoot(labstor::DomainId::GetLocal(), "ipc_test"); + + hshm::Timer t; + t.Resume(); + // size_t ops = (1 << 20); + size_t ops = 256; + for (size_t i = 0; i < ops; ++i) { + hipc::LPointer task; + labstor::TaskNode task_node(labstor::TaskId((u32)0, (u64)i)); + task = client.AsyncMdPushEmplace(queues[num_queues - 1].get(), + task_node, + labstor::DomainId::GetLocal()); + worker.Run(); + LABSTOR_CLIENT->DelTask(task); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Time for worker to process a request */ +TEST_CASE("TestWorkerLatency") { + TestWorkerIterationLatency(1, 16); + TestWorkerIterationLatency(5, 16); + TestWorkerIterationLatency(10, 16); + TestWorkerIterationLatency(15, 16); +} + +/** Time to process a request */ +TEST_CASE("TestRoundTripLatency") { + HERMES->ClientInit(); + labstor::small_message::Client client; + LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetLocal(), "small_message"); +// int count = 25; +// for (int i = 0; i < count; ++i) { +// labstor::small_message::Client client2; +// client2.CreateRoot(labstor::DomainId::GetLocal(), "ipc_test" + std::to_string(i)); +// } + client.CreateRoot(labstor::DomainId::GetLocal(), "ipc_test"); + hshm::Timer t; + + int pid = getpid(); + ProcessAffiner::SetCpuAffinity(pid, 8); + + t.Resume(); + size_t ops = (1 << 20); + // size_t ops = 1024; + for (size_t i = 0; i < ops; ++i) { + client.MdPushRoot(labstor::DomainId::GetLocal()); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Time to process a request */ +//TEST_CASE("TestHermesGetBlobIdLatency") { +// HERMES->ClientInit(); +// hshm::Timer t; +// +// int pid = getpid(); +// ProcessAffiner::SetCpuAffinity(pid, 8); +// hermes::Bucket bkt = HERMES_FILESYSTEM_API->Open("/home/lukemartinlogan/hi.txt"); +// +// t.Resume(); +// size_t ops = (1 << 20); +// hermes::Context ctx; +// std::string data(ctx.page_size_, 0); +// for (size_t i = 0; i < ops; ++i) { +// bkt.GetBlobId(std::to_string(i)); +// } +// t.Pause(); +// +// HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +//} +// +///** Time to process a request */ +//TEST_CASE("TestHermesFsWriteLatency") { +// HERMES->ClientInit(); +// hshm::Timer t; +// +// int pid = getpid(); +// ProcessAffiner::SetCpuAffinity(pid, 8); +// hermes::Bucket bkt = HERMES_FILESYSTEM_API->Open("/home/lukemartinlogan/hi.txt"); +// +// t.Resume(); +// size_t ops = 150; +// hermes::Context ctx; +// ctx.page_size_ = 4096; +// std::string data(ctx.page_size_, 0); +// for (size_t i = 0; i < ops; ++i) { +// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, ctx); +// } +// t.Pause(); +// +// HILOG(kInfo, "Latency: {} MBps", ops * 4096 / t.GetUsec()); +//} +// +///** Time to process a request */ +//TEST_CASE("TestHermesFsReadLatency") { +// HERMES->ClientInit(); +// hshm::Timer t; +// +// int pid = getpid(); +// ProcessAffiner::SetCpuAffinity(pid, 8); +// hermes::Bucket bkt = HERMES_FILESYSTEM_API->Open("/home/lukemartinlogan/hi.txt"); +// +// size_t ops = 1024; +// hermes::Context ctx; +// ctx.page_size_ = 4096; +// std::string data(ctx.page_size_, 0); +// for (size_t i = 0; i < ops; ++i) { +// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, ctx); +// } +// +// t.Resume(); +// for (size_t i = 0; i < ops; ++i) { +// HERMES_FILESYSTEM_API->Read(bkt, data.data(), i * ctx.page_size_, data.size(), false, ctx); +// } +// t.Pause(); +// +// HILOG(kInfo, "Latency: {} MBps", ops * 4096 / t.GetUsec()); +//} \ No newline at end of file diff --git a/benchmark/test_zmq.cc b/benchmark/test_zmq.cc new file mode 100644 index 000000000..2f95ea8e6 --- /dev/null +++ b/benchmark/test_zmq.cc @@ -0,0 +1,72 @@ +// +// Created by lukemartinlogan on 8/25/23. +// + +#include +#include "basic_test.h" +#include "labstor/api/labstor_client.h" +#include "labstor_admin/labstor_admin.h" +#include "small_message/small_message.h" +#include "hermes_shm/util/timer.h" +#include "labstor/work_orchestrator/affinity.h" + +/** ZeroMQ allocate + free request */ +TEST_CASE("TestZeromqAllocateFree") { + zmq::context_t context(1); + + // Create a PUSH socket to push requests + zmq::socket_t pushSocket(context, ZMQ_PUSH); + + // Bind the PUSH socket to a specific address + pushSocket.bind("ipc:///tmp/shared_memory"); + + // Create a PULL socket to receive requests + zmq::socket_t pullSocket(context, ZMQ_PULL); + + // Connect the PULL socket to the same address + pullSocket.connect("ipc:///tmp/shared_memory"); + + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + zmq::message_t message(sizeof(labstor::Task)); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of zeromq */ +TEST_CASE("TestZeromqAllocateEmplacePop") { + zmq::context_t context(1); + + // Create a PUSH socket to push requests + zmq::socket_t pushSocket(context, ZMQ_PUSH); + + // Bind the PUSH socket to a specific address + pushSocket.bind("ipc:///tmp/shared_memory"); + + // Create a PULL socket to receive requests + zmq::socket_t pullSocket(context, ZMQ_PULL); + + // Connect the PULL socket to the same address + pullSocket.connect("ipc:///tmp/shared_memory"); + + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + // Send a request + zmq::message_t message(sizeof(labstor::Task)); + pushSocket.send(message, zmq::send_flags::none); + + // Receive the request + zmq::message_t receivedMessage; + zmq::recv_result_t result = pullSocket.recv(receivedMessage); + REQUIRE(receivedMessage.size() == sizeof(labstor::Task)); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} \ No newline at end of file diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt deleted file mode 100644 index faae961b1..000000000 --- a/benchmarks/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -include_directories( - ${CMAKE_SOURCE_DIR} - ${PROJECT_SOURCE_DIR}/src/api - ${PROJECT_SOURCE_DIR}/test - ${PROJECT_SOURCE_DIR}/io_client -) - -set(BENCHMARKS - api_bench - memcpy_bench - reorganize) - -foreach(benchmark ${BENCHMARKS}) - message("Building ${benchmark}") - add_executable(${benchmark} ${benchmark}.cc) - add_dependencies(${benchmark} hermes hermes_prefetcher_trait) - target_link_libraries(${benchmark} hermes hermes_prefetcher_trait - MPI::MPI_CXX - $<$:thallium>) -endforeach() diff --git a/benchmarks/memcpy_bench.cc b/benchmarks/memcpy_bench.cc deleted file mode 100644 index 267439cb6..000000000 --- a/benchmarks/memcpy_bench.cc +++ /dev/null @@ -1,154 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// #define HERMES_ENABLE_PROFILING 1 - -#include "mpi.h" -#include -#include "hermes.h" -#include "hermes_shm/memory/backend/posix_shm_mmap.h" -#include - -using hshm::ipc::PosixShmMmap; -using Timer = hshm::HighResMonotonicTimer; -hipc::MemoryBackend *backend; - -void GatherTimes(const std::string &backend_type, - const std::string &test_name, - size_t size_per_rank, Timer &t) { - MPI_Barrier(MPI_COMM_WORLD); - int nprocs, rank; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - double time = t.GetSec(), max; - MPI_Reduce(&time, &max, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - if (rank == 0) { - HIPRINT("{} {} {}: MBps: {}\n", - nprocs, backend_type, test_name, - size_per_rank * nprocs / t.GetUsec()) - } -} - -template -size_t GetBlobOff(int rank, size_t i, - size_t blobs_per_rank, size_t blob_size) { - if constexpr(SHM) { - return (rank * blobs_per_rank + i) * blob_size; - } else { - return i * blob_size; - } -} - -template -void PutTest(const std::string &backend_type, int rank, int repeat, - size_t blobs_per_rank, size_t blob_size) { - Timer t; - std::vector data(blob_size); - t.Resume(); - for (int j = 0; j < repeat; ++j) { - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_off = GetBlobOff(rank, i, blobs_per_rank, blob_size); - memcpy(backend->data_ + blob_off, data.data(), data.size()); - } - } - t.Pause(); - GatherTimes(backend_type, "Put", blobs_per_rank * blob_size * repeat, t); -} - -template -void GetTest(const std::string &backend_type, int rank, int repeat, - size_t blobs_per_rank, size_t blob_size) { - Timer t; - std::vector data(blob_size); - t.Resume(); - for (int j = 0; j < repeat; ++j) { - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_off = GetBlobOff(rank, i, blobs_per_rank, blob_size); - memcpy(data.data(), backend->data_ + blob_off, data.size()); - } - } - t.Pause(); - GatherTimes(backend_type, "Get", blobs_per_rank * blob_size * repeat, t); -} - -template -void MemcpyBench(int nprocs, int rank, - size_t blob_size, - size_t blobs_per_rank) { - std::string shm_url = "test_mem_backend"; - std::string backend_type; - size_t backend_size; - hipc::MemoryBackendType type = hipc::MemoryBackendType::kPosixShmMmap; - - if constexpr(std::is_same_v) { - backend_type = "kPosixShmMmap"; - type = hipc::MemoryBackendType::kPosixShmMmap; - backend_size = nprocs * blob_size * blobs_per_rank; - } else if constexpr(std::is_same_v) { - backend_type = "kPosixMmap"; - type = hipc::MemoryBackendType::kPosixMmap; - backend_size = blob_size * blobs_per_rank; - } else { - HELOG(kFatal, "Invalid backend type"); - } - - // Create the backend - try { - if constexpr(std::is_same_v) { - MPI_Barrier(MPI_COMM_WORLD); - if (rank == 0) { - backend = HERMES_MEMORY_MANAGER-> - CreateBackend(backend_size, shm_url); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank != 0) { - backend = HERMES_MEMORY_MANAGER->AttachBackend(type, shm_url); - } - MPI_Barrier(MPI_COMM_WORLD); - } else if constexpr(std::is_same_v) { - MPI_Barrier(MPI_COMM_WORLD); - backend = HERMES_MEMORY_MANAGER-> - CreateBackend(backend_size, shm_url); - MPI_Barrier(MPI_COMM_WORLD); - } - } catch (std::exception &e) { - HELOG(kFatal, "{}\n", e.what()); - } - - if (backend == nullptr) { - HELOG(kFatal, "Backend was null: {}", backend_type) - } - - PutTest(backend_type, rank, 1, blobs_per_rank, blob_size); - MPI_Barrier(MPI_COMM_WORLD); - GetTest(backend_type, rank, 1, blobs_per_rank, blob_size); -} - -int main(int argc, char **argv) { - int rank, nprocs; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc != 3) { - printf("USAGE: ./memcpy_bench [blob_size (K/M/G)] [blobs_per_rank]\n"); - exit(1); - } - size_t blob_size = hshm::ConfigParse::ParseSize(argv[1]); - size_t blobs_per_rank = atoi(argv[2]); - - MemcpyBench(nprocs, rank, blob_size, blobs_per_rank); - MemcpyBench(nprocs, rank, - blob_size, blobs_per_rank); - - MPI_Finalize(); -} diff --git a/benchmarks/reorganize.cc b/benchmarks/reorganize.cc deleted file mode 100644 index 785f0fbe3..000000000 --- a/benchmarks/reorganize.cc +++ /dev/null @@ -1,150 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// #define HERMES_ENABLE_PROFILING 1 - -#include "mpi.h" -#include -#include "hermes.h" -#include "hermes_shm/memory/backend/posix_shm_mmap.h" -#include -#include "traits/prefetcher/prefetcher_trait.h" - -using hshm::ipc::PosixShmMmap; -using Timer = hshm::HighResMonotonicTimer; -hipc::MemoryBackend *backend; -const char *kBucketName = "/tmp/test_hermes/hi.txt"; - -void GatherTimes(const std::string &test_name, - size_t total_size, - Timer &t, Timer &io_t) { - MPI_Barrier(MPI_COMM_WORLD); - int nprocs, rank; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - double time = t.GetUsec(), max_runtime; - MPI_Reduce(&time, &max_runtime, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - double io_time = t.GetUsec(), max_io_time; - MPI_Reduce(&io_time, &max_io_time, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - if (rank == 0) { - HIPRINT("{} {}: MBps: {}, Time: {}\n", - nprocs, test_name, - total_size / max_io_time, - max_runtime) - } -} - -/** Each process PUTS into the same bucket, but with different blob names */ -void PutTest(int nprocs, int rank, - size_t blobs_per_checkpt, - size_t num_checkpts, - size_t blob_size, - int compute_sec) { - Timer t, io_t; - auto bkt = HERMES->GetBucket(kBucketName); - hermes::api::Context ctx; - hermes::BlobId blob_id; - hermes::Blob blob(blob_size); - t.Resume(); - - size_t blobs_per_rank = blobs_per_checkpt * num_checkpts; - size_t cur_blob = 0; - for (size_t i = 0; i < num_checkpts; ++i) { - io_t.Resume(); - for (size_t j = 0; j < blobs_per_checkpt; ++j) { - size_t blob_name_int = rank * blobs_per_rank + cur_blob; - std::string name = std::to_string(blob_name_int); - bkt.Put(name, blob, blob_id, ctx); - ++cur_blob; - } - io_t.Pause(); - sleep(compute_sec); - } - t.Pause(); - GatherTimes("Put", nprocs * blobs_per_rank * blob_size, t, io_t); -} - -/** - * Each process GETS from the same bucket, but with different blob names - * MUST run PutTest first. - * */ -void GetTest(int nprocs, int rank, - size_t blobs_per_checkpt, - size_t num_checkpts, - size_t blob_size, - int compute_sec) { - Timer t, io_t; - auto bkt = HERMES->GetBucket(kBucketName); - hermes::api::Context ctx; - hermes::BlobId blob_id; - t.Resume(); - size_t blobs_per_rank = blobs_per_checkpt * num_checkpts; - size_t cur_blob = 0; - for (size_t i = 0; i < num_checkpts; ++i) { - io_t.Resume(); - for (size_t j = 0; j < blobs_per_checkpt; ++j) { - size_t blob_name_int = rank * blobs_per_rank + cur_blob; - std::string name = std::to_string(blob_name_int); - hermes::Blob ret; - bkt.GetBlobId(name, blob_id); - bkt.Get(blob_id, ret, ctx); - ++cur_blob; - } - io_t.Pause(); - sleep(compute_sec); - } - t.Pause(); - GatherTimes("Get", nprocs * blobs_per_rank * blob_size, t, io_t); -} - -int main(int argc, char **argv) { - int rank, nprocs; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc != 7) { - printf("USAGE: ./reorganize [prefetch] [blob_size (K/M/G)] " - "[blobs_per_checkpt] [num_checkpts] " - "[compute_put (sec)] [compute_get (sec)]\n"); - exit(1); - } - int with_prefetch = atoi(argv[1]); - size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); - size_t blobs_per_checkpt = atoi(argv[3]); - size_t num_checkpts = atoi(argv[4]); - int compute_put = atoi(argv[5]); - int compute_get = atoi(argv[6]); - - // Start Hermes - HERMES->Create(hermes::HermesType::kClient); - - // Register the Apriori trait - hermes::TraitId apriori_trait = - HERMES->RegisterTrait( - "apriori", hermes::PrefetcherType::kApriori); - if (with_prefetch) { - auto bkt = HERMES->GetBucket(kBucketName); - bkt.AttachTrait(apriori_trait); - } - - PutTest(nprocs, rank, blobs_per_checkpt, num_checkpts, - blob_size, compute_put); - MPI_Barrier(MPI_COMM_WORLD); - GetTest(nprocs, rank, blobs_per_checkpt, num_checkpts, - blob_size, compute_get); - - // Finalize - HERMES->Finalize(); - MPI_Finalize(); -} diff --git a/ci/build_hermes.sh b/ci/build_hermes.sh deleted file mode 100755 index ae1a9b563..000000000 --- a/ci/build_hermes.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -set -x -set -e -set -o pipefail - -INSTALL_DIR="${HOME}" -INSTALL_PREFIX="${HOME}/install" -mkdir -p "${INSTALL_PREFIX}" - -# This will build our small python library for running unit tests -pushd ci/jarvis-util -python3 -m pip install -r requirements.txt -python3 -m pip install -e . -popd - -mkdir build -pushd build - -# Load hermes dependencies via spack -# Copy from install_deps.sh -SPACK_DIR=${INSTALL_DIR}/spack -set +x -. ${SPACK_DIR}/share/spack/setup-env.sh -set -x -spack load --only dependencies hermes - -# Build hermes -# export CXXFLAGS="${CXXFLAGS} -std=c++17 -Werror -Wall -Wextra" -cmake \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DBUILD_SHARED_LIBS=ON \ - -DHERMES_ENABLE_COVERAGE=ON \ - -DHERMES_BUILD_BENCHMARKS=OFF \ - -DHERMES_BUILD_BUFFER_POOL_VISUALIZER=OFF \ - -DHERMES_USE_ADDRESS_SANITIZER=OFF \ - -DHERMES_USE_THREAD_SANITIZER=OFF \ - -DHERMES_RPC_THALLIUM=ON \ - -DBUILD_TESTING=ON \ - -DHERMES_ENABLE_VFD=ON \ - -DHERMES_DEBUG_LOCK=OFF \ - .. -make -j8 -make install - -popd diff --git a/ci/cluster/Dockerfile b/ci/cluster/Dockerfile deleted file mode 100644 index d3600723a..000000000 --- a/ci/cluster/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM ubuntu:20.04 - -ENV USER mpirun - -ENV DEBIAN_FRONTEND=noninteractive \ - HOME=/home/${USER} - -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends \ - sudo \ - apt-utils \ - && apt-get install -y --no-install-recommends \ - openssh-server \ - gcc \ - g++ \ - libfabric-dev \ - mpich \ - binutils \ - libjson-c-dev \ - libunwind-dev \ - && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -RUN mkdir /var/run/sshd -RUN echo 'root:${USER}' | chpasswd -RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config - -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -ENV NOTVISIBLE "in users profile" -RUN echo "export VISIBLE=now" >> /etc/profile - -# ------------------------------------------------------------ -# Add an 'mpirun' user -# ------------------------------------------------------------ - -ARG USER_ID -ARG GROUP_ID - -RUN addgroup --gid ${GROUP_ID} ${USER} -RUN adduser --disabled-password --gecos "" --uid ${USER_ID} --gid ${GROUP_ID} ${USER} && \ - echo "${USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - -# ------------------------------------------------------------ -# Set-Up SSH with our Github deploy key -# ------------------------------------------------------------ - -ENV SSHDIR ${HOME}/.ssh/ - -RUN mkdir -p ${SSHDIR} -RUN echo "StrictHostKeyChecking no" > ${SSHDIR}/config - -RUN chmod -R 600 ${SSHDIR}* && \ - chown -R ${USER}:${USER} ${SSHDIR} - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/ci/cluster/cluster_down.sh b/ci/cluster/cluster_down.sh deleted file mode 100755 index 71377a9bf..000000000 --- a/ci/cluster/cluster_down.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -. cluster_utils.sh -hermes_cluster_down diff --git a/ci/cluster/cluster_test.sh b/ci/cluster/cluster_test.sh deleted file mode 100755 index 6ec34caaf..000000000 --- a/ci/cluster/cluster_test.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -. cluster_utils.sh -hermes_cluster_test ${1:-} diff --git a/ci/cluster/cluster_up.sh b/ci/cluster/cluster_up.sh deleted file mode 100755 index ce7ef43bf..000000000 --- a/ci/cluster/cluster_up.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -. cluster_utils.sh -hermes_cluster_up diff --git a/ci/cluster/cluster_utils.sh b/ci/cluster/cluster_utils.sh deleted file mode 100644 index b1a388034..000000000 --- a/ci/cluster/cluster_utils.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -x -e - -node_names=($(awk '/hostname:/ {print $2}' docker-compose.yml)) -docker_user=mpirun -docker_home=/home/${docker_user} -script_dir="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)" -hermes_dir=${script_dir}/../.. -cluster_conf=${hermes_dir}/test/data/hermes_server_ci.yaml -hermes_build_dir=${hermes_dir}/build -project_name="$(basename ${script_dir})" -host1=${project_name}_${node_names[0]}_1 -host2=${project_name}_${node_names[1]}_1 - - -# Build images and start a cluster -function hermes_cluster_up() { - local num_workers=${1:-1} - - # Build the images, passing our user id and group id so the container user can - # modify the .gcda coverage files - for n in "${node_names[@]}"; do - docker-compose build --build-arg GROUP_ID=$(id -g) --build-arg USER_ID=$(id -u) ${n} - done - - docker-compose up -d --scale ${node_names[0]}=1 --scale ${node_names[1]}=${num_workers} --no-recreate - - for h in ${host1} ${host2}; do - # Create the hosts file - docker exec --user ${docker_user} -w ${hermes_build_dir} ${h} \ - bash -c "echo -e \"${host1}\n${host2}\n\" > hermes_hosts" - # Copy ssh keys to ${docker_home}/.ssh - docker exec ${h} bash -c "cp ${HOME}/.ssh/id_rsa ${docker_home}/.ssh/id_rsa" - docker exec ${h} bash -c "cp ${HOME}/.ssh/id_rsa.pub ${docker_home}/.ssh/id_rsa.pub" - docker exec ${h} bash -c "cp ${HOME}/.ssh/id_rsa.pub ${docker_home}/.ssh/authorized_keys" - docker exec ${h} chown -R ${docker_user}:${docker_user} ${docker_home}/.ssh - # Create the data directory - docker exec ${h} bash -c "mkdir /tmp/test_hermes" - done -} - -function hermes_cluster_test() { - local allocate_tty=${1:-} - local hosts=${host1},${host2} - - docker-compose exec ${allocate_tty} \ - -e GLOG_vmodule=rpc_thallium=1 \ - -e LSAN_OPTIONS=suppressions=../test/data/asan.supp \ - --user ${docker_user} \ - -w ${hermes_build_dir} \ - ${node_names[0]} \ - echo `mpirun -n 2 -ppn 1 -hosts ${hosts} bin/hermes_daemon ${cluster_conf} &` \ - && sleep 3 \ - && mpirun -n 4 -ppn 2 -hosts ${hosts} -genv ${cluster_conf} bin/test_multinode_put_get -} - -# Stop the cluster -function hermes_cluster_down() { - docker-compose down -} diff --git a/ci/cluster/docker-compose.yml b/ci/cluster/docker-compose.yml deleted file mode 100644 index 389c73c45..000000000 --- a/ci/cluster/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3" - -services: - node1: - build: . - links: - - node2 - networks: - - net - volumes: - - $HOME:$HOME - hostname: node1 - - node2: - build: . - networks: - - net - volumes: - - $HOME:$HOME - hostname: node2 - -networks: - net: - driver: bridge diff --git a/ci/cluster/multi_node_ci_test.sh b/ci/cluster/multi_node_ci_test.sh deleted file mode 100755 index 5fcfe0238..000000000 --- a/ci/cluster/multi_node_ci_test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -x -e - -if [[ "${CI}" != "true" ]]; then - echo "This script is only meant to be run within Github actions" - exit 1 -fi - -echo "TODO(llogan): Add back. disabling for now" -exit 0 - -. cluster_utils.sh - -# Create ssh keys for the cluster to use -echo -e 'y\n' | ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa - -# Start a two node cluster -hermes_cluster_up - -# Run the Hermes tests on the cluster (without allocating a tty) -hermes_cluster_test "-T" - -# Shutdown the cluster -hermes_cluster_down diff --git a/ci/coverage.sh b/ci/coverage.sh deleted file mode 100644 index c6b27ad2c..000000000 --- a/ci/coverage.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -COVERAGE_DIR="${GITHUB_WORKSPACE}/coverage" -mkdir -p "${COVERAGE_DIR}" -cd "${GITHUB_WORKSPACE}/build" -lcov -c -d . -o "${COVERAGE_DIR}/tmp.info" -lcov --remove "${COVERAGE_DIR}/tmp.info" \ - "/usr/*" \ - "*/spack/*" \ - -o "${COVERAGE_DIR}/lcov.info" -genhtml "${COVERAGE_DIR}/lcov.info" --output-directory coverage_report diff --git a/ci/external/CMakeLists.txt b/ci/external/CMakeLists.txt deleted file mode 100644 index 826f316b6..000000000 --- a/ci/external/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -project(UNIT_TESTS) -cmake_minimum_required (VERSION 3.10) - -set(CMAKE_MESSAGE_LOG_LEVEL "VERBOSE") -find_package(Hermes CONFIG REQUIRED) -message("Found Hermes at ${Hermes_INCLUDE_DIR} ${Hermes_LIBRARIES}") -add_executable(test_hermes_install test_install.cc) -target_link_libraries(test_hermes_install ${Hermes_LIBRARIES}) \ No newline at end of file diff --git a/ci/external/test_install.cc b/ci/external/test_install.cc deleted file mode 100644 index f7cbe04ac..000000000 --- a/ci/external/test_install.cc +++ /dev/null @@ -1,18 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes.h" - -int main() { - HERMES->Create(hermes::HermesType::kClient); - hapi::Bucket bkt = HERMES->GetBucket("hi"); -} diff --git a/ci/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py deleted file mode 100644 index 0a3462f95..000000000 --- a/ci/hermes/packages/hermes/package.py +++ /dev/null @@ -1,57 +0,0 @@ -from spack import * - -class Hermes(CMakePackage): - homepage = "http://www.cs.iit.edu/~scs/assets/projects/Hermes/Hermes.html" - url = "https://github.com/HDFGroup/hermes/tarball/master" - git = "https://github.com/HDFGroup/hermes.git" - version('master', branch='master') - version('pnnl', branch='pnnl') - version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', branch='dev') - variant('vfd', default=False, description='Enable HDF5 VFD') - variant('ares', default=False, description='Enable full libfabric install') - variant('debug', default=False, description='Enable debug mode') - depends_on('mochi-thallium~cereal@0.8.3') - depends_on('cereal') - depends_on('catch2@3.0.1') - depends_on('mpich@3.3.2:') - depends_on('yaml-cpp') - depends_on('boost@1.7:') - # TODO(): we need to figure out how to add a python library as a dependency - # depends_on('py-jarvis-util') - depends_on('libfabric@1.14.1 fabrics=mlx,rxd,rxm,shm,sockets,tcp,udp,verbs,xpmem', - when='+ares') - depends_on('hdf5@1.14.0:', when='+vfd') - - def cmake_args(self): - args = ['-DCMAKE_INSTALL_PREFIX={}'.format(self.prefix), - '-DHERMES_RPC_THALLIUM=ON', - '-DHERMES_INSTALL_TESTS=ON', - '-DBUILD_TESTING=ON'] - if '+debug' in self.spec: - args.append('-DCMAKE_BUILD_TYPE=Debug') - if '+vfd' in self.spec: - args.append(self.define('HERMES_ENABLE_VFD', 'ON')) - return args - - def set_include(self, env, path): - env.append_flags('CFLAGS', '-I{}'.format(path)) - env.append_flags('CXXFLAGS', '-I{}'.format(path)) - env.prepend_path('CPATH', '{}'.format(path)) - env.prepend_path('CMAKE_PREFIX_PATH', '{}'.format(path)) - - def set_lib(self, env, path): - env.prepend_path('LD_LIBRARY_PATH', path) - env.prepend_path('LIBRARY_PATH', '{}'.format(path)) - env.append_flags('LDFLAGS', '-L{}'.format(path)) - - def set_flags(self, env): - self.set_include(env, '{}/include'.format(self.prefix)) - self.set_include(env, '{}/include'.format(self.prefix)) - self.set_lib(env, '{}/lib'.format(self.prefix)) - self.set_lib(env, '{}/lib64'.format(self.prefix)) - - def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - self.set_flags(spack_env) - - def setup_run_environment(self, env): - self.set_flags(env) diff --git a/ci/hermes/repo.yaml b/ci/hermes/repo.yaml deleted file mode 100644 index fdd6f5fb9..000000000 --- a/ci/hermes/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: hermes diff --git a/ci/install_deps.sh b/ci/install_deps.sh deleted file mode 100755 index 45e6be4c4..000000000 --- a/ci/install_deps.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -x -set -e -set -o pipefail - -sudo apt-get install -y pkg-config - -# Where to install data -INSTALL_DIR="${HOME}" -INSTALL_PREFIX="${HOME}/install" -SPACK_DIR=${INSTALL_DIR}/spack -SPACK_VERSION=0.18.1 - -# Install spack -echo "Installing dependencies at ${INSTALL_DIR}" -mkdir -p ${INSTALL_DIR} -git clone https://github.com/spack/spack ${SPACK_DIR} -pushd ${SPACK_DIR} -git checkout v${SPACK_VERSION} -popd - -set +x -. ${SPACK_DIR}/share/spack/setup-env.sh -set -x - -# This will allow Spack to skip building some packages that are directly -# available from the system. For example, autoconf, cmake, m4, etc. -# Modify ci/pckages.yaml to skip building compilers or build tools via Spack. -cp ci/packages.yaml ${SPACK_DIR}/etc/spack/packages.yaml - -# This will override Spack's default package repository to allow building -# a custom package when the same package is available from Spack. -spack repo add ./ci/hermes - -# This will build our small python library for running unit tests -cd ci/jarvis-util -python3 -m pip install -r requirements.txt -python3 -m pip install -e . - -# NOTE(llogan): Modify version string per release. -HERMES_VERSION=1.0.0 -spack install hermes +vfd diff --git a/ci/install_docs.sh b/ci/install_docs.sh deleted file mode 100755 index 38d213025..000000000 --- a/ci/install_docs.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# set -x -# set -e -# set -o pipefail - -mkdir build -pushd build - -#LOCAL=local -INSTALL_PREFIX="${HOME}/${LOCAL}" - -export CXXFLAGS="${CXXFLAGS} -std=c++17 -Werror -Wall -Wextra" -cmake \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} \ - -DCMAKE_BUILD_RPATH=${INSTALL_PREFIX}/lib \ - -DCMAKE_INSTALL_RPATH=${INSTALL_PREFIX}/lib \ - -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DCMAKE_CXX_COMPILER=`which mpicxx` \ - -DCMAKE_C_COMPILER=`which mpicc` \ - -DBUILD_SHARED_LIBS=ON \ - -DHERMES_ENABLE_DOXYGEN=ON \ - -DHERMES_ENABLE_COVERAGE=ON \ - -DHERMES_BUILD_BENCHMARKS=ON \ - -DHERMES_COMMUNICATION_MPI=ON \ - -DHERMES_BUILD_BUFFER_POOL_VISUALIZER=ON \ - -DORTOOLS_DIR=${INSTALL_PREFIX} \ - -DHERMES_USE_ADDRESS_SANITIZER=ON \ - -DHERMES_USE_THREAD_SANITIZER=OFF \ - -DHERMES_RPC_THALLIUM=ON \ - -DHERMES_ENABLE_VFD=ON \ - -DBUILD_TESTING=ON \ - .. -make dox >& out.txt -# cat out.txt | grep warning | grep -v "ignoring unsupported tag" -popd -rec="$( grep warning build/out.txt | grep -v "ignoring unsupported tag" | wc -l )" -echo "$rec" -exit "$rec" - diff --git a/ci/install_hermes.sh b/ci/install_hermes.sh deleted file mode 100755 index b34a3aa40..000000000 --- a/ci/install_hermes.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. ci/build_hermes.sh -pushd build -ctest -VV -popd diff --git a/ci/jarvis-util/.coveragerc b/ci/jarvis-util/.coveragerc deleted file mode 100644 index f1db9952b..000000000 --- a/ci/jarvis-util/.coveragerc +++ /dev/null @@ -1,4 +0,0 @@ -# .coveragerc -[run] -source = . -omit = *test* \ No newline at end of file diff --git a/ci/jarvis-util/.gitignore b/ci/jarvis-util/.gitignore deleted file mode 100644 index 80ced04e4..000000000 --- a/ci/jarvis-util/.gitignore +++ /dev/null @@ -1,139 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -.idea -hostfile.txt -lcov.info - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ -/.idea/jarvis-cd.iml -/.idea/misc.xml -/.idea/modules.xml -/.idea/inspectionProfiles/profiles_settings.xml -/.idea/inspectionProfiles/Project_Default.xml -/.idea/vcs.xml -/.idea/deployment.xml diff --git a/ci/jarvis-util/LICENSE b/ci/jarvis-util/LICENSE deleted file mode 100644 index 5db7c2cc8..000000000 --- a/ci/jarvis-util/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022-present Luke Logan and other contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/ci/jarvis-util/README.md b/ci/jarvis-util/README.md deleted file mode 100644 index 711a1466e..000000000 --- a/ci/jarvis-util/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Jarvis UTIL - -Jarvis-util is a library which contains various utilities to aid with -creating shell scripts within Python. This library contains wrappers -for executing shell commands locally, SSH, SCP, MPI, argument parsing, -and various other random utilities. - -![Build](https://github.com/lukemartinlogan/jarvis-util/workflows/GitHub%20Actions/badge.svg) - -[![Coverage Status](https://coveralls.io/repos/github/lukemartinlogan/jarvis-util/badge.svg?branch=master)](https://coveralls.io/github/lukemartinlogan/jarvis-util?branch=master) - -## Installation - -For now, we only consider manual installation -```bash -cd jarvis-util -python3 -m pip install -r requirements.txt -python3 -m pip install -e . -``` - -## Executing a program - -The following code will execute a command on the local machine. -The output will NOT be collected into an in-memory buffer. -The output will be printed to the terminal as it occurs. - -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - -node = Exec('echo hello', LocalExecInfo(collect_output=False)) -``` - -Programs can also be executed asynchronously: -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - -node = Exec('echo hello', LocalExecInfo(collect_output=False, - exec_async=True)) -node.wait() -``` - -Various examples of outputs: -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - -# Will ONLY print to the terminal -node = Exec('echo hello', LocalExecInfo(collect_output=False)) -# Will collect AND print to the terminal -node = Exec('echo hello', LocalExecInfo(collect_output=True)) -# Will collect BUT NOT print to the terminal -node = Exec('echo hello', LocalExecInfo(collect_output=True, - hide_output=True)) -# Will collect, pipe to file, and print to terminal -node = Exec('echo hello', LocalExecInfo(collect_output=True, - pipe_stdout='/tmp/stdout.txt', - pipe_stderr='/tmp/stderr.txt')) -``` - -This is useful if you have a program which runs using a daemon mode. - -## Executing an MPI program - -The following code will execute the "hostname" command on the local -machine 24 times using MPI. - -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.mpi_exec import MpiExecInfo - -node = Exec('hostname', MpiExecInfo(hostfile=None, - nprocs=24, - ppn=None, - collect_output=False)) -``` - -## Executing an SSH program - -The following code will execute the "hostname" command on all machines -in the hostfile - -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.pssh_exec import PsshExecInfo - -node = Exec('hostname', PsshExecInfo(hostfile="/tmp/hostfile.txt", - collect_output=False)) -``` - -## The contents of a hostfile - -A hostfile can have the following syntax: -``` -ares-comp-01 -ares-comp-[02-04] -ares-comp-[05-09,11,12-14]-40g -``` - -These will be expanded internally by PSSH and MPI. - -# Unit tests - -We run our unit tests in a docker container, which is located underneath -the CI directory. This is because we need to test multi-node functionality, -without having multiple nodes. To setup unit testing, perform the following: - -1. Install Docker -2. Setup our "ci" container -3. Run the unit tests - -``` -``` - -# Contributing - -We use the Google Python Style guide (pylintrc). \ No newline at end of file diff --git a/ci/jarvis-util/bin/jarvis-imports b/ci/jarvis-util/bin/jarvis-imports deleted file mode 100755 index 25fc19a9a..000000000 --- a/ci/jarvis-util/bin/jarvis-imports +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 - -from jarvis_util.util.import_all import * -import pathlib -import os - - -build_global_import_from_bin('jarvis_util') diff --git a/ci/jarvis-util/ci/cluster/Dockerfile b/ci/jarvis-util/ci/cluster/Dockerfile deleted file mode 100644 index 548937e02..000000000 --- a/ci/jarvis-util/ci/cluster/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -# Install ubuntu 20.04 -FROM ubuntu:20.04 -LABEL maintainer="llogan@hawk.iit.edu" -LABEL version="0.0" -LABEL description="An example docker image" - -# Disable Prompt During Packages Installation -ARG DEBIAN_FRONTEND=noninteractive - -# Update ubuntu -RUN apt update && apt install - -# Install some basic packages -RUN apt install -y \ - openssh-server \ - sudo git nano vim \ - docker \ - mpich \ - gcc \ - g++ \ - gfortran \ - libtool \ - libtool-bin \ - automake \ - autoconf - -# Create a new user -RUN useradd -m sshuser -RUN usermod -aG sudo sshuser -RUN passwd -d sshuser - -# Copy the host's SSH keys -# Docker requires COPY be relative to the current working -# directory. We cannot pass ~/.ssh/id_rsa unfortunately... -ENV SSHDIR=/home/sshuser/.ssh -RUN sudo -u sshuser mkdir ${SSHDIR} -COPY id_rsa ${SSHDIR}/id_rsa -COPY id_rsa.pub ${SSHDIR}/id_rsa.pub - -# Authorize host SSH keys -RUN sudo -u sshuser touch ${SSHDIR}/authorized_keys -RUN cat ${SSHDIR}/id_rsa.pub >> ${SSHDIR}/authorized_keys - -# Set SSH permissions -RUN chmod 700 ${SSHDIR} -RUN chmod 644 ${SSHDIR}/id_rsa.pub -RUN chmod 600 ${SSHDIR}/id_rsa -RUN chmod 600 ${SSHDIR}/authorized_keys - -# Enable passwordless SSH -RUN sed -i 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/' /etc/ssh/sshd_config - -# Start SSHD and wait forever -RUN mkdir /run/sshd -CMD ["/usr/sbin/sshd", "-D"] \ No newline at end of file diff --git a/ci/jarvis-util/ci/cluster/docker-compose.yml b/ci/jarvis-util/ci/cluster/docker-compose.yml deleted file mode 100644 index aa7477089..000000000 --- a/ci/jarvis-util/ci/cluster/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3" - -services: - node1: - build: . - links: - - node2 - networks: - - net - hostname: node1 - stdin_open: true - tty: true - - node2: - build: . - networks: - - net - hostname: node2 - stdin_open: true - tty: true - -networks: - net: - driver: bridge \ No newline at end of file diff --git a/ci/jarvis-util/ci/install_deps.sh b/ci/jarvis-util/ci/install_deps.sh deleted file mode 100644 index 0d1f0447d..000000000 --- a/ci/jarvis-util/ci/install_deps.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -sudo apt update -sudo apt install -y \ -docker \ -mpich \ -gcc \ -g++ \ -gfortran \ -libtool \ -libtool-bin \ -automake \ -autoconf \ No newline at end of file diff --git a/ci/jarvis-util/ci/install_jarvis.sh b/ci/jarvis-util/ci/install_jarvis.sh deleted file mode 100644 index 58aa283c0..000000000 --- a/ci/jarvis-util/ci/install_jarvis.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -python3 -m pip install -r requirements.txt -python3 -m pip install -e . diff --git a/ci/jarvis-util/ci/install_spack.sh b/ci/jarvis-util/ci/install_spack.sh deleted file mode 100644 index 0d1f0447d..000000000 --- a/ci/jarvis-util/ci/install_spack.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -sudo apt update -sudo apt install -y \ -docker \ -mpich \ -gcc \ -g++ \ -gfortran \ -libtool \ -libtool-bin \ -automake \ -autoconf \ No newline at end of file diff --git a/ci/jarvis-util/ci/lint.sh b/ci/jarvis-util/ci/lint.sh deleted file mode 100644 index a1af3ff48..000000000 --- a/ci/jarvis-util/ci/lint.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -pylint "${PWD}"/jarvis_util \ No newline at end of file diff --git a/ci/jarvis-util/ci/run_tests.sh b/ci/jarvis-util/ci/run_tests.sh deleted file mode 100644 index abd4e290a..000000000 --- a/ci/jarvis-util/ci/run_tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -coverage run -m pytest -rm -rf "*.pyc" -coverage report -coverage-lcov \ No newline at end of file diff --git a/ci/jarvis-util/jarvis_util/__init__.py b/ci/jarvis-util/jarvis_util/__init__.py deleted file mode 100644 index 1019dd213..000000000 --- a/ci/jarvis-util/jarvis_util/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Import all modules""" -from jarvis_util.util.expand_env import * -from jarvis_util.util.naming import * -from jarvis_util.util.hostfile import * -from jarvis_util.util.size_conv import * -from jarvis_util.util.import_all import * -from jarvis_util.util.import_mod import * -from jarvis_util.util.argparse import * -from jarvis_util.serialize.ini_file import * -from jarvis_util.serialize.yaml_file import * -from jarvis_util.serialize.text_file import * -from jarvis_util.serialize.serializer import * -from jarvis_util.serialize.pickle import * -from jarvis_util.shell.filesystem import * -from jarvis_util.shell.exec import * -from jarvis_util.shell.exec_info import * -from jarvis_util.shell.ssh_exec import * -from jarvis_util.shell.pssh_exec import * -from jarvis_util.shell.process import * -from jarvis_util.shell.pscp import * -from jarvis_util.shell.scp import * -from jarvis_util.shell.mpi_exec import * -from jarvis_util.shell.local_exec import * -from jarvis_util.introspect.system_info import * -from jarvis_util.jutil_manager import * diff --git a/ci/jarvis-util/jarvis_util/introspect/system_info.py b/ci/jarvis-util/jarvis_util/introspect/system_info.py deleted file mode 100644 index 5a9d94482..000000000 --- a/ci/jarvis-util/jarvis_util/introspect/system_info.py +++ /dev/null @@ -1,635 +0,0 @@ -""" -This module provides methods for querying the information of the host -system. This can be used to make scripts more portable. -""" - -import re -import platform -from jarvis_util.shell.exec import Exec -from jarvis_util.util.size_conv import SizeConv -from jarvis_util.serialize.yaml_file import YamlFile -import json -import pandas as pd -import numpy as np -from enum import Enum -import shlex - -# pylint: disable=C0121 - -class SystemInfo: - """ - This class queries information about the host machine, such as OS, - CPU, and kernel - """ - - instance_ = None - - @staticmethod - def get_instance(): - if SystemInfo.instance_ is None: - SystemInfo.instance_ = SystemInfo() - return SystemInfo.instance_ - - def __init__(self): - with open('/etc/os-release', 'r', encoding='utf-8') as fp: - lines = fp.read().splitlines() - self.os = self._detect_os_type(lines) - self.os_like = self._detect_os_like_type(lines) - self.os_version = self._detect_os_version(lines) - self.ksemantic = platform.platform() - self.krelease = platform.release() - self.ktype = platform.system() - self.cpu = platform.processor() - self.cpu_family = platform.machine() - - def _detect_os_type(self, lines): - for line in lines: - if 'ID=' in line: - if 'ubuntu' in line: - return 'ubuntu' - elif 'centos' in line: - return 'centos' - elif 'debian' in line: - return 'debian' - - def _detect_os_like_type(self, lines): - for line in lines: - if 'ID_LIKE=' in line: - if 'ubuntu' in line: - return 'ubuntu' - elif 'centos' in line: - return 'centos' - elif 'debian' in line: - return 'debian' - - def _detect_os_version(self, lines): - for line in lines: - grp = re.match('VERSION_ID=\"(.*)\"', line) - if grp: - return grp.group(1) - - def __hash__(self): - return hash(str([self.os, self.os_like, - self.os_version, self.ksemantic, - self.krelease, self.ktype, - self.cpu, self.cpu_family])) - - def __eq__(self, other): - return ( - (self.os == other.os) and - (self.os_like == other.os_like) and - (self.os_version == other.os_version) and - (self.ksemantic == other.ksemantic) and - (self.krelease == other.krelease) and - (self.cpu == other.cpu) and - (self.cpu_family == other.cpu_family) - ) - - -class Lsblk(Exec): - """ - List all block devices in the system per-node. Lsblk will return - a JSON output - - A table is stored per-host: - parent: the parent device of the partition (e.g., /dev/sda or NaN) - device: the name of the partition (e.g., /dev/sda1) - size: total size of the partition (bytes) - mount: where the partition is mounted (if anywhere) - model: the exact model of the device - tran: the transport of the device (e.g., /dev/nvme) - rota: whether or not the device is rotational - host: the host this record corresponds to - """ - - def __init__(self, exec_info): - cmd = 'lsblk -o NAME,SIZE,MODEL,TRAN,MOUNTPOINT,ROTA -J -s' - super().__init__(cmd, exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - lsblk_data = json.loads(stdout)['blockdevices'] - partitions = [] - devs = {} - for partition in lsblk_data: - dev = partition['children'][0] - partitions.append({ - 'parent': f'/dev/{dev["name"]}', - 'device': f'/dev/{partition["name"]}', - 'size': SizeConv.to_int(partition['size']), - 'mount': partition['mountpoint'], - 'host': host - }) - devs[dev['name']] = { - 'parent': f'/dev/{dev["name"]}', - 'size': SizeConv.to_int(dev['size']), - 'model': dev['model'], - 'tran': dev['tran'].lower(), - 'mount': dev['mountpoint'], - 'rota': dev['rota'], - 'host': host - } - devs = list(devs.values()) - part_df = pd.DataFrame(partitions) - dev_df = pd.DataFrame(devs) - total_df = pd.merge(part_df, - dev_df[['parent', 'model', 'tran', 'host']], - on=['parent', 'host']) - dev_df = dev_df.rename(columns={'parent': 'device'}) - total_df = pd.concat([total_df, dev_df]) - self.df = total_df - - -class Blkid(Exec): - """ - List all filesystems (even those unmounted) and their properties - - Stores a per-host table with the following: - device: the device (or partition) which stores the data (e.g., /dev/sda) - fs_type: the type of filesystem (e.g., ext4) - uuid: filesystem-levle uuid from the FS metadata - partuuid: the partition-lable UUID for the partition - host: the host this entry corresponds to - """ - def __init__(self, exec_info): - cmd = 'blkid' - super().__init__(cmd, exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - devices = stdout.splitlines() - dev_list = [] - for dev in devices: - dev_dict = {} - toks = shlex.split(dev) - dev_name = toks[0].split(':')[0] - dev_dict['device'] = dev_name - dev_dict['host'] = host - for tok in toks[1:]: - keyval = tok.split('=') - key = keyval[0].lower() - val = ' '.join(keyval[1:]) - dev_dict[key] = val - dev_list.append(dev_dict) - df = pd.DataFrame(dev_list) - df = df.rename(columns={'type': 'fs_type'}) - self.df = df - - -class ListFses(Exec): - """ - List all mounted filesystems - - Will store a per-host dictionary containing: - device: the device which contains the filesystem - fs_size: total size of the filesystem - used: total nubmer of bytes used - avail: total number of bytes remaining - use%: the percent of capacity used - fs_mount: where the filesystem is mounted - host: the host this entry corresponds to - """ - - def __init__(self, exec_info): - cmd = 'df -h' - super().__init__(cmd, exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - lines = stdout.strip().splitlines() - columns = ['device', 'fs_size', 'used', - 'avail', 'use%', 'fs_mount', 'host'] - rows = [line.split() + [host] for line in lines[1:]] - df = pd.DataFrame(rows, columns=columns) - # pylint: disable=W0108 - df.loc[:, 'fs_size'] = df['fs_size'].apply( - lambda x : SizeConv.to_int(x)) - df.loc[:, 'used'] = df['used'].apply( - lambda x: SizeConv.to_int(x)) - df.loc[:, 'avail'] = df['avail'].apply( - lambda x : SizeConv.to_int(x)) - # pylint: enable=W0108 - self.df = df - - -class FiInfo(Exec): - """ - List all networks and their information - provider: network protocol (e.g., sockets, tcp, ib) - fabric: IP address - domain: network domain - version: network version - type: packet type (e.g., DGRAM) - protocol: protocol constant - host: the host this network corresponds to - """ - def __init__(self, exec_info): - super().__init__('fi_info', exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - lines = stdout.strip().splitlines() - providers = [] - for line in lines: - if 'provider' in line: - providers.append({ - 'provider': line.split(':')[1], - 'host': host - }) - else: - splits = line.split(':') - key = splits[0].strip() - val = splits[1].strip() - if 'fabric' in key: - val = val.split('/')[0] - providers[-1][key] = val - self.df = pd.DataFrame(providers) - - -class StorageDeviceType(Enum): - PMEM='pmem' - NVME='nvme' - SSD='ssd' - HDD='hdd' - - -class ResourceGraph: - """ - Stores helpful information about storage and networking info for machines. - - Two tables are stored to make decisions on application deployment. - fs: - parent: the parent device of the partition (e.g., /dev/sda or NaN) - device: the name of the device (e.g., /dev/sda1 or /dev/sda) - size: total size of the device (bytes) - mount: where the device is mounted (if anywhere) - model: the exact model of the device - rota: whether the device is rotational or not - tran: the transport of the device (e.g., /dev/nvme) - fs_type: the type of filesystem (e.g., ext4) - uuid: filesystem-levle uuid from the FS metadata - fs_size: total size of the filesystem - avail: total number of bytes remaining - shared: whether the this is a shared service or not - host: the host this record corresponds to - net: - provider: network protocol (e.g., sockets, tcp, ib) - fabric: IP address - domain: network domain - host: the host this network corresponds to - - TODO: Need to verify on more than ubuntu20.04 - TODO: Can we make this work for windows? - TODO: Can we make this work even when hosts have different OSes? - """ - - def __init__(self): - self.lsblk = None - self.blkid = None - self.list_fs = None - self.fi_info = None - self.fs_columns = [ - 'parent', 'device', 'size', 'mount', 'model', 'rota', - 'tran', 'fs_type', 'uuid', 'fs_size', - 'avail', 'shared', 'host' - ] - self.net_columns = [ - 'provider', 'fabric', 'domain', 'host' - ] - self.all_fs = pd.DataFrame(columns=self.fs_columns) - self.all_net = pd.DataFrame(columns=self.net_columns) - self.fs_settings = { - 'register': [], - 'filter_mounts': {} - } - self.net_settings = { - 'register': [], - 'track_ips': {} - } - self.hosts = None - - def build(self, exec_info, introspect=True): - """ - Build a resource graph. - - :param exec_info: Where to collect resource information - :param introspect: Whether to introspect system info, or rely solely - on admin-defined settings - :return: self - """ - if introspect: - self._introspect(exec_info) - self.apply() - return self - - def _introspect(self, exec_info): - """ - Introspect the cluster for resources. - - :param exec_info: Where to collect resource information - :return: None - """ - self.lsblk = Lsblk(exec_info) - self.blkid = Blkid(exec_info) - self.list_fs = ListFses(exec_info) - self.fi_info = FiInfo(exec_info) - self.hosts = exec_info.hostfile.hosts - self.all_fs = pd.merge(self.lsblk.df, - self.blkid.df, - on=['device', 'host'], - how='outer') - self.all_fs['shared'] = False - self.all_fs = pd.merge(self.all_fs, - self.list_fs.df, - on=['device', 'host'], - how='outer') - self.all_fs['shared'].fillna(True, inplace=True) - self.all_fs.drop(['used', 'use%', 'fs_mount', 'partuuid'], - axis=1, inplace=True) - self.all_fs['mount'].fillna(value='', inplace=True) - net_df = self.fi_info.df - net_df['speed'] = np.nan - net_df.drop(['version', 'type', 'protocol'], - axis=1, inplace=True) - self.all_net = net_df - - def save(self, path): - """ - Save the resource graph YAML file - - :param path: the path to save the file - :return: None - """ - graph = { - 'hosts': self.hosts, - 'fs': self.all_fs.to_dict('records'), - 'net': self.all_net.to_dict('records'), - 'fs_settings': self.fs_settings, - 'net_settings': self.net_settings - } - YamlFile(path).save(graph) - - def load(self, path): - """ - Load resource graph from storage. - - :param path: The path to the resource graph YAML file - :return: self - """ - graph = YamlFile(path).load() - self.hosts = graph['hosts'] - self.all_fs = pd.DataFrame(graph['fs']) - self.all_net = pd.DataFrame(graph['net']) - self.fs = None - self.net = None - self.fs_settings = graph['fs_settings'] - self.net_settings = graph['net_settings'] - self.apply() - return self - - def set_hosts(self, hosts): - """ - Set the set of hosts this resource graph covers - - :param hosts: Hostfile() - :return: None - """ - self.hosts = hosts.hosts_ip - - def add_storage(self, hosts, **kwargs): - """ - Register a storage device record - - :param hosts: Hostfile() indicating set of hosts to make record for - :param kwargs: storage record - :return: None - """ - for host in hosts.hosts: - record = kwargs.copy() - record['host'] = host - self.fs_settings['register'].append(record) - - def add_net(self, hosts, **kwargs): - """ - Register a network record - - :param hosts: Hostfile() indicating set of hosts to make record for - :param kwargs: net record - :return: None - """ - for host, ip in zip(hosts.hosts, hosts.hosts_ip): - record = kwargs.copy() - record['fabric'] = ip - record['host'] = host - self.net_settings['register'].append(record) - - def filter_fs(self, mount_re, - mount_suffix=None, tran=None): - """ - Track all filesystems + devices matching the mount regex. - - :param mount_re: The regex to match a set of mountpoints - :param mount_suffix: After the mount_re is matched, append this path - to the mountpoint to indicate where users can access data. A typical - value for this is /${USER}, indicating the mountpoint has a subdirectory - per-user where they can access data. - :param shared: Whether this mount point is shared - :param tran: The transport of this device - :return: self - """ - self.fs_settings['filter_mounts']['mount_re'] = { - 'mount_re': mount_re, - 'mount_suffix': mount_suffix, - 'tran': tran - } - return self - - def filter_ip(self, ip_re, speed=None): - """ - Track all IPs matching the regex. The IPs with this regex all have - a certain speed. - - :param ip_re: The regex to match - :param speed: The speed of the fabric - :return: self - """ - self.net_settings['track_ips'][ip_re] = { - 'ip_re': ip_re, - 'speed': SizeConv.to_int(speed) if speed is not None else speed - } - return self - - def filter_hosts(self, hosts, speed=None): - """ - Track all ips matching the hostnames. - - :param hosts: Hostfile() of the hosts to filter for - :param speed: Speed of the interconnect (e.g., 1gbps) - :return: self - """ - for host in hosts.hosts_ip: - self.filter_ip(host, speed) - return self - - def apply(self): - """ - Apply fs and net settings to the resource graph - - :return: self - """ - self._apply_fs_settings() - self._apply_net_settings() - # self.fs.size = self.fs.size.fillna(0) - # self.fs.avail = self.fs.avail.fillna(0) - # self.fs.fs_size = self.fs.fs_size.fillna(0) - return self - - def _apply_fs_settings(self): - if len(self.fs_settings) == 0: - self.fs = self.all_fs - return - df = self.all_fs - self.fs = pd.DataFrame(columns=self.all_net.columns) - for fs_set in self.fs_settings['filter_mounts'].values(): - mount_re = fs_set['mount_re'] - mount_suffix = fs_set['mount_suffix'] - tran = fs_set['tran'] - with_mount = df[df.mount.str.contains(mount_re)] - if mount_suffix is not None: - with_mount['mount'] += mount_suffix - if tran is not None: - with_mount['tran'] = tran - self.fs = pd.concat([self.fs, with_mount]) - admin_df = pd.DataFrame(self.fs_settings['register'], - columns=self.fs_columns) - self.fs = pd.concat([self.fs, admin_df]) - - def _apply_net_settings(self): - if len(self.net_settings) == 0: - self.net = self.all_net - return - self.net = pd.DataFrame(columns=self.all_net.columns) - df = self.all_net - for net_set in self.net_settings['track_ips'].values(): - ip_re = net_set['ip_re'] - speed = net_set['speed'] - with_ip = df[df['fabric'].str.contains(ip_re)] - with_ip['speed'] = speed - self.net = pd.concat([self.net, with_ip]) - admin_df = pd.DataFrame(self.net_settings['register'], - columns=self.net_columns) - self.net = pd.concat([self.net, admin_df]) - - def find_shared_storage(self): - """ - Find the set of shared storage services - - :return: Dataframe - """ - df = self.fs - return df[df.shared == True] - - def find_storage(self, - dev_types=None, - is_mounted=True, - common=False, - count_per_node=None, - count_per_dev=None, - min_cap=None, - min_avail=None): - """ - Find a set of storage devices. - - :param dev_types: Search for devices of type in order. Either a list - or a string. - :param is_mounted: Search only for mounted devices - :param common: Remove mount points that are not common across all hosts - :param count_per_node: Choose only a subset of devices matching query - :param count_per_dev: Choose only a subset of devices matching query - :param min_cap: Remove devices with too little overall capacity - :param min_avail: Remove devices with too little available space - :return: Dataframe - """ - df = self.fs - # Remove pfs - df = df[df.shared == False] - # Filter devices by whether or not a mount is needed - if is_mounted: - df = df[df.mount.notna()] - # Find devices of a particular type - if dev_types is not None: - matching_devs = pd.DataFrame(columns=df.columns) - if isinstance(dev_types, str): - dev_types = [dev_types] - for dev_type in dev_types: - if dev_type == StorageDeviceType.HDD: - devs = df[(df.tran == 'sata') & (df.rota == True)] - elif dev_type == StorageDeviceType.SSD: - devs = df[(df.tran == 'sata') & (df.rota == False)] - elif dev_type == StorageDeviceType.NVME: - devs = df[(df.tran == 'nvme')] - matching_devs = pd.concat([matching_devs, devs]) - df = matching_devs - # Get the set of mounts common between all hosts - if common: - df = df.groupby(['mount']).filter( - lambda x: len(x) == len(self.hosts)).reset_index(drop=True) - # Remove storage with too little capacity - if min_cap is not None: - df = df[df.size >= min_cap] - # Remove storage with too little available space - if min_avail is not None: - df = df[df.avail >= min_avail] - # Take a certain number of each device per-host - if count_per_dev is not None: - df = df.groupby(['tran', 'rota', 'host']).\ - head(count_per_dev).reset_index(drop=True) - # Take a certain number of matched devices per-host - if count_per_node is not None: - df = df.groupby('host').head(count_per_node).reset_index(drop=True) - return df - - def find_net_info(self, hosts, - providers=None): - """ - Find the set of networks common between each host. - - :param hosts: A Hostfile() data structure containing the set of - all hosts to find network information for - :param providers: The network protocols to search for. - :return: Dataframe - """ - df = self.net - # Get the set of fabrics corresponding to these hosts - df = df[df.fabric.isin(hosts.hosts_ip)] - # Filter out protocols which are not common between these hosts - df = df.groupby('provider').filter( - lambda x: len(x) == len(hosts)).reset_index(drop=True) - # Choose only a subset of providers - if providers is not None: - if isinstance(providers, str): - providers = [providers] - df = df[df.provider.isin(providers)] - return df - -# pylint: enable=C0121 diff --git a/ci/jarvis-util/jarvis_util/jutil_manager.py b/ci/jarvis-util/jarvis_util/jutil_manager.py deleted file mode 100644 index c229b2445..000000000 --- a/ci/jarvis-util/jarvis_util/jutil_manager.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -This file contains properties which are globally accessible to all -jarvis-util modules. This can be used to configure various aspects -of jarvis, such as output. -""" - - -class JutilManager: - """ - A singleton which stores various properties that can be queried by - internally by jutil modules. This includes properties such output - management. - """ - - instance_ = None - - @staticmethod - def get_instance(): - if JutilManager.instance_ is None: - JutilManager.instance_ = JutilManager() - return JutilManager.instance_ - - def __init__(self): - self.collect_output = False - self.hide_output = False - self.debug_mpi_exec = False - self.debug_local_exec = False - diff --git a/ci/jarvis-util/jarvis_util/serialize/ini_file.py b/ci/jarvis-util/jarvis_util/serialize/ini_file.py deleted file mode 100644 index 8da174ff1..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/ini_file.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -This module contains methods to serialize and deserialize data from -a human-readable ini file. -""" -import configparser -from jarvis_util.serialize.serializer import Serializer - - -class IniFile(Serializer): - """ - This class contains methods to serialize and deserialize data from - a human-readable ini file. - """ - def __init__(self, path): - self.path = path - - def load(self): - config = configparser.ConfigParser() - config.read(self.path) - return config - - def save(self, data): - with open(self.path, 'w', encoding='utf-8') as fp: - data.write(fp) diff --git a/ci/jarvis-util/jarvis_util/serialize/pickle.py b/ci/jarvis-util/jarvis_util/serialize/pickle.py deleted file mode 100644 index a90ae12de..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/pickle.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -This module contains methods to serialize and deserialize data from -a pickle file. -""" - -import pickle as pkl -from jarvis_util.serialize.serializer import Serializer - - -class PickleFile(Serializer): - """ - This class serializes and deserializes data from a pickle file - """ - - def __init__(self, path): - self.path = path - - def load(self): - with open(self.path, 'rb') as fp: - return pkl.load(fp) - - def save(self, data): - with open(self.path, 'wb') as fp: - pkl.dump(data, fp) diff --git a/ci/jarvis-util/jarvis_util/serialize/serializer.py b/ci/jarvis-util/jarvis_util/serialize/serializer.py deleted file mode 100644 index 8647945a7..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/serializer.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -This module contains an abstract class used to define classes which -serialize data to a file. -""" - -from abc import ABC, abstractmethod - - -class Serializer(ABC): - """ - An abstract class which loads serialized data from a file and - saves serialized data to a file. - """ - - @abstractmethod - def load(self): - pass - - @abstractmethod - def save(self, data): - pass diff --git a/ci/jarvis-util/jarvis_util/serialize/text_file.py b/ci/jarvis-util/jarvis_util/serialize/text_file.py deleted file mode 100644 index 214c6bb9f..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/text_file.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -This module stores data into a file in a human-readable way -""" -from jarvis_util.serialize.serializer import Serializer - - -class TextFile(Serializer): - """ - This class stores data directly into a file using str() as the - serialization method. The data is intended to be human-readable. - """ - def __init__(self, path): - self.path = path - - def load(self): - with open(self.path, 'r', encoding='utf-8') as fp: - data = fp.read() - return data - - def save(self, data): - with open(self.path, 'w', encoding='utf-8') as fp: - fp.write(data) diff --git a/ci/jarvis-util/jarvis_util/serialize/yaml_file.py b/ci/jarvis-util/jarvis_util/serialize/yaml_file.py deleted file mode 100644 index 5892f6bca..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/yaml_file.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -This module contains methods to serialize and deserialize data from -a human-readable YAML file. -""" -from jarvis_util.serialize.serializer import Serializer -import yaml - - -class YamlFile(Serializer): - """ - This class contains methods to serialize and deserialize data from - a human-readable YAML file. - """ - def __init__(self, path): - self.path = path - - def load(self): - with open(self.path, 'r', encoding='utf-8') as fp: - return yaml.load(fp, Loader=yaml.FullLoader) - return None - - def save(self, data): - with open(self.path, 'w', encoding='utf-8') as fp: - yaml.dump(data, fp) diff --git a/ci/jarvis-util/jarvis_util/shell/__init__.py b/ci/jarvis-util/jarvis_util/shell/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ci/jarvis-util/jarvis_util/shell/exec.py b/ci/jarvis-util/jarvis_util/shell/exec.py deleted file mode 100644 index f9fef066d..000000000 --- a/ci/jarvis-util/jarvis_util/shell/exec.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -This module provides mechanisms to execute binaries either locally or -remotely. -""" - -from .local_exec import LocalExec -from .pssh_exec import PsshExec -from .pssh_exec import SshExec -from .mpi_exec import MpiExec -from .exec_info import ExecInfo, ExecType, Executable - - -class Exec(Executable): - """ - This class is a factory which wraps around various shell command - execution stragies, such as MPI and SSH. - """ - - def __init__(self, cmd, exec_info=None): - """ - Execute a command or list of commands - - :param cmd: list of commands or a single command string - :param exec_info: Info needed to execute processes locally - """ - super().__init__() - if exec_info is None: - exec_info = ExecInfo() - if exec_info.exec_type == ExecType.LOCAL: - self.exec_ = LocalExec(cmd, exec_info) - elif exec_info.exec_type == ExecType.SSH: - self.exec_ = SshExec(cmd, exec_info) - elif exec_info.exec_type == ExecType.PSSH: - self.exec_ = PsshExec(cmd, exec_info) - elif exec_info.exec_type == ExecType.MPI: - self.exec_ = MpiExec(cmd, exec_info) - self.set_exit_code() - self.set_output() - - def wait(self): - self.exec_.wait() - self.set_output() - self.set_exit_code() - return self.exit_code - - def set_output(self): - self.stdout = self.exec_.stdout - self.stderr = self.exec_.stderr - if isinstance(self.stdout, str): - if hasattr(self.exec_, 'addr'): - host = self.exec_.addr - else: - host = 'localhost' - self.stdout = {host: self.stdout} - self.stderr = {host: self.stderr} - - def set_exit_code(self): - self.exec_.set_exit_code() - self.exit_code = self.exec_.exit_code diff --git a/ci/jarvis-util/jarvis_util/shell/exec_info.py b/ci/jarvis-util/jarvis_util/shell/exec_info.py deleted file mode 100644 index 2b28246ae..000000000 --- a/ci/jarvis-util/jarvis_util/shell/exec_info.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -This module contains data structures for determining how to execute -a subcommand. This includes information such as storing SSH keys, -passwords, working directory, etc. -""" - -from enum import Enum -from jarvis_util.util.hostfile import Hostfile -import os -from abc import ABC, abstractmethod - - -class ExecType(Enum): - """ - Different program execution methods. - """ - - LOCAL = 'LOCAL' - SSH = 'SSH' - PSSH = 'PSSH' - MPI = 'MPI' - - -class ExecInfo: - """ - Contains all information needed to execute a program. This includes - parameters such as the path to key-pairs, the hosts to run the program - on, number of processes, etc. - """ - def __init__(self, exec_type=ExecType.LOCAL, nprocs=None, ppn=None, - user=None, pkey=None, port=None, - hostfile=None, hosts=None, env=None, - sleep_ms=0, sudo=False, cwd=None, - collect_output=None, pipe_stdout=None, pipe_stderr=None, - hide_output=None, exec_async=False, stdin=None): - """ - - :param exec_type: How to execute a program. SSH, MPI, Local, etc. - :param nprocs: Number of processes to spawn. E.g., MPI uses this - :param ppn: Number of processes per node. E.g., MPI uses this - :param user: The user to execute command under. E.g., SSH, PSSH - :param pkey: The path to the private key. E.g., SSH, PSSH - :param port: The port to use for connection. E.g., SSH, PSSH - :param hostfile: The hosts to launch command on. E.g., PSSH, MPI - :param hosts: A list (or single string) of host names to run command on. - :param env: The environment variables to use for command. - :param sleep_ms: Sleep for a period of time AFTER executing - :param sudo: Execute command with root privilege. E.g., SSH, PSSH - :param cwd: Set current working directory. E.g., SSH, PSSH - :param collect_output: Collect program output in python buffer - :param pipe_stdout: Pipe STDOUT into a file. (path string) - :param pipe_stderr: Pipe STDERR into a file. (path string) - :param hide_output: Whether to print output to console - :param exec_async: Whether to execute program asynchronously - :param stdin: Any input needed by the program. Only local - """ - - self.exec_type = exec_type - self.nprocs = nprocs - self.user = user - self.pkey = pkey - self.port = port - self.ppn = ppn - self.hostfile = hostfile - self._set_hostfile(hostfile=hostfile, hosts=hosts) - self.env = env - self.basic_env = {} - self._set_env(env) - self.cwd = cwd - self.sudo = sudo - self.sleep_ms = sleep_ms - self.collect_output = collect_output - self.pipe_stdout = pipe_stdout - self.pipe_stderr = pipe_stderr - self.hide_output = hide_output - self.exec_async = exec_async - self.stdin = stdin - self.keys = ['exec_type', 'nprocs', 'ppn', 'user', 'pkey', 'port', - 'hostfile', 'env', 'sleep_ms', 'sudo', - 'cwd', 'hosts', 'collect_output', - 'pipe_stdout', 'pipe_stderr', 'hide_output', - 'exec_async', 'stdin'] - - def _set_env(self, env): - if env is None: - self.env = {} - else: - self.env = env - basic_env = [ - 'PATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'CMAKE_PREFIX_PATH', - 'PYTHON_PATH', 'CPATH', 'INCLUDE', 'JAVA_HOME' - ] - self.basic_env = {} - for key in basic_env: - if key not in os.environ: - continue - self.basic_env[key] = os.getenv(key) - for key, val in self.basic_env.items(): - if key not in self.env: - self.env[key] = val - self.basic_env.update(self.env) - if 'LD_PRELOAD' in self.basic_env: - del self.basic_env['LD_PRELOAD'] - - def _set_hostfile(self, hostfile=None, hosts=None): - if hostfile is not None: - if isinstance(hostfile, str): - self.hostfile = Hostfile(hostfile=hostfile) - elif isinstance(hostfile, Hostfile): - self.hostfile = hostfile - else: - raise Exception('Hostfile is neither string nor Hostfile') - if hosts is not None: - if isinstance(hosts, list): - self.hostfile = Hostfile(all_hosts=hosts) - elif isinstance(hosts, str): - self.hostfile = Hostfile(all_hosts=[hosts]) - elif isinstance(hosts, Hostfile): - self.hostfile = hosts - else: - raise Exception('Host set is neither str, list or Hostfile') - - if hosts is not None and hostfile is not None: - raise Exception('Must choose either hosts or hostfile, not both') - - if self.hostfile is None: - self.hostfile = Hostfile() - - def mod(self, **kwargs): - self._mod_kwargs(kwargs) - return ExecInfo(**kwargs) - - def _mod_kwargs(self, kwargs): - for key in self.keys: - if key not in kwargs and hasattr(self, key): - kwargs[key] = getattr(self, key) - - def copy(self): - return self.mod() - - -class Executable(ABC): - """ - An abstract class representing a class which is intended to run - shell commands. This includes SSH, MPI, etc. - """ - - def __init__(self): - self.exit_code = None - self.stdout = '' - self.stderr = '' - - def failed(self): - return self.exit_code != 0 - - @abstractmethod - def set_exit_code(self): - pass - - @abstractmethod - def wait(self): - pass - - def smash_cmd(self, cmds): - """ - Convert a list of commands into a single command for the shell - to execute. - - :param cmds: A list of commands or a single command string - :return: - """ - if isinstance(cmds, list): - return ' && '.join(cmds) - elif isinstance(cmds, str): - return cmds - else: - raise Exception('Command must be either list or string') - - def wait_list(self, nodes): - for node in nodes: - node.wait() - - def smash_list_outputs(self, nodes): - """ - Combine the outputs of a set of nodes into a single output. - For example, used if executing multiple commands in sequence. - - :param nodes: - :return: - """ - self.stdout = '\n'.join([node.stdout for node in nodes]) - self.stderr = '\n'.join([node.stderr for node in nodes]) - - def per_host_outputs(self, nodes): - """ - Convert the outputs of a set of nodes to a per-host dictionary. - Used if sending commands to multiple hosts - - :param nodes: - :return: - """ - self.stdout = {} - self.stderr = {} - self.stdout = {node.addr: node.stdout for node in nodes} - self.stderr = {node.addr: node.stderr for node in nodes} - - def set_exit_code_list(self, nodes): - """ - Set the exit code from a set of nodes. - - :param nodes: The set of execution nodes that have been executed - :return: - """ - for node in nodes: - if node.exit_code: - self.exit_code = node.exit_code diff --git a/ci/jarvis-util/jarvis_util/shell/filesystem.py b/ci/jarvis-util/jarvis_util/shell/filesystem.py deleted file mode 100644 index a5cb2b55e..000000000 --- a/ci/jarvis-util/jarvis_util/shell/filesystem.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -This module contains various wrappers over typical filesystem commands seen -in shell scripts. This includes operations such as creating directories, -changing file permissions, etc. -""" -from .exec import Exec - - -class Mkdir(Exec): - """ - Create directories + subdirectories. - """ - - def __init__(self, paths, exec_info=None): - """ - Create directories + subdirectories. Does not fail if the dirs - already exist. - - :param paths: A list of paths or a single path string. - :param exec_info: Info needed to execute the mkdir command - """ - - if isinstance(paths, str): - paths = [paths] - path = ' '.join(paths) - super().__init__(f'mkdir -p {path}', exec_info) - - -class Rm(Exec): - """ - Remove a file and its subdirectories - """ - - def __init__(self, paths, exec_info=None): - """ - Execute file or directory remove. - - :param paths: Either a list of paths or a single path string - :param exec_info: Information needed to execute rm - """ - - if isinstance(paths, str): - paths = [paths] - path = ' '.join(paths) - super().__init__(f'rm -rf {path}', exec_info) - - -class Chmod(Exec): - """ - Change the mode of a file - """ - - def __init__(self, path=None, mode=None, modes=None, exec_info=None): - """ - Change the mode of a file - - :param path: path to file to mode change - :param mode: the mode to change to - :param modes: A list of tuples [(Path, Mode)] - :param exec_info: How to execute commands - """ - cmds = [] - if path is not None and mode is not None: - cmds.append(f'chmod {mode} {path}') - if modes is not None: - cmds += [f'chmod {mode[1]} {mode[0]}' for mode in modes] - if len(cmds) == 0: - raise Exception('Must set either path+mode or modes') - super().__init__(cmds, exec_info) diff --git a/ci/jarvis-util/jarvis_util/shell/local_exec.py b/ci/jarvis-util/jarvis_util/shell/local_exec.py deleted file mode 100644 index c5487fe1e..000000000 --- a/ci/jarvis-util/jarvis_util/shell/local_exec.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Provides methods for executing a program or workflow locally. This class -is intended to be called from Exec, not by general users. -""" - -import time -import subprocess -import os -import sys -import io -import threading -from jarvis_util.jutil_manager import JutilManager -from .exec_info import ExecInfo, ExecType, Executable - - -class LocalExec(Executable): - """ - Provides methods for executing a program or workflow locally. - """ - - def __init__(self, cmd, exec_info): - """ - Execute a program or workflow - - :param cmd: list of commands or a single command string - :param exec_info: Info needed to execute processes locally - """ - - super().__init__() - jutil = JutilManager.get_instance() - cmd = self.smash_cmd(cmd) - - # Managing console output and collection - self.collect_output = exec_info.collect_output - self.pipe_stdout = exec_info.pipe_stdout - self.pipe_stderr = exec_info.pipe_stderr - self.pipe_stdout_fp = None - self.pipe_stderr_fp = None - self.hide_output = exec_info.hide_output - # pylint: disable=R1732 - if self.collect_output is None: - self.collect_output = jutil.collect_output - if self.hide_output is None: - self.hide_output = jutil.hide_output - # pylint: enable=R1732 - self.stdout = io.StringIO() - self.stderr = io.StringIO() - self.last_stdout_size = 0 - self.last_stderr_size = 0 - self.executing_ = True - self.exit_code = 0 - - # Copy ENV - self.env = exec_info.env.copy() - for key, val in os.environ.items(): - if key not in self.env: - self.env[key] = val - - # Managing command execution - self.cmd = cmd - self.sudo = exec_info.sudo - self.stdin = exec_info.stdin - self.exec_async = exec_info.exec_async - self.sleep_ms = exec_info.sleep_ms - if exec_info.cwd is None: - self.cwd = os.getcwd() - else: - self.cwd = exec_info.cwd - if jutil.debug_local_exec: - print(cmd) - self._start_bash_processes() - - def _start_bash_processes(self): - if self.sudo: - self.cmd = f'sudo {self.cmd}' - time.sleep(self.sleep_ms) - # pylint: disable=R1732 - self.proc = subprocess.Popen(self.cmd, - cwd=self.cwd, - env=self.env, - shell=True) - # pylint: enable=R1732 - if not self.exec_async: - self.wait() - - def wait(self): - self.proc.wait() - self.set_exit_code() - return self.exit_code - - def set_exit_code(self): - self.exit_code = self.proc.returncode - - def get_pid(self): - if self.proc is not None: - return self.proc.pid - else: - return None - - -class LocalExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.LOCAL, **kwargs) diff --git a/ci/jarvis-util/jarvis_util/shell/mpi_exec.py b/ci/jarvis-util/jarvis_util/shell/mpi_exec.py deleted file mode 100644 index e104bf50f..000000000 --- a/ci/jarvis-util/jarvis_util/shell/mpi_exec.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -This module provides methods to execute a process in parallel using the -Message Passing Interface (MPI). This module assumes MPI is installed -on the system. This class is intended to be called from Exec, -not by general users. -""" - -from jarvis_util.jutil_manager import JutilManager -from jarvis_util.shell.local_exec import LocalExec -from .exec_info import ExecInfo, ExecType - - -class MpiExec(LocalExec): - """ - This class contains methods for executing a command in parallel - using MPI. - """ - - def __init__(self, cmd, exec_info): - """ - Execute a command using MPI - - :param cmd: A command (string) to execute - :param exec_info: Information needed by MPI - """ - - self.cmd = cmd - self.nprocs = exec_info.nprocs - self.ppn = exec_info.ppn - self.hostfile = exec_info.hostfile - self.mpi_env = exec_info.env - super().__init__(self.mpicmd(), - exec_info.mod(env=exec_info.basic_env)) - - def mpicmd(self): - params = [f"mpirun -n {self.nprocs}"] - if self.ppn is not None: - params.append(f"-ppn {self.ppn}") - if len(self.hostfile): - if self.hostfile.is_subset() or self.hostfile.path is None: - params.append(f"--host {','.join(self.hostfile.hosts)}") - else: - params.append(f"--hostfile {self.hostfile.path}") - params += [f"-genv {key}={val}" for key, val in self.mpi_env.items()] - params.append(self.cmd) - cmd = " ".join(params) - jutil = JutilManager.get_instance() - if jutil.debug_mpi_exec: - print(cmd) - return cmd - - -class MpiExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.MPI, **kwargs) diff --git a/ci/jarvis-util/jarvis_util/shell/process.py b/ci/jarvis-util/jarvis_util/shell/process.py deleted file mode 100644 index 7aba2f0fc..000000000 --- a/ci/jarvis-util/jarvis_util/shell/process.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -This module provides various wrappers for methods which manage processes -in the cluster. Examples include killing processes, determining whether -or not a process exists, etc. -""" - -from .exec import Exec - - -class Kill(Exec): - """ - Kill all processes which match the name regex. - """ - - def __init__(self, cmd, exec_info): - """ - Kill all processes which match the name regex. - - :param cmd: A regex for the command to kill - :param exec_info: Info needed to execute the command - """ - super().__init__(f"pkill {cmd}", exec_info) diff --git a/ci/jarvis-util/jarvis_util/shell/pscp.py b/ci/jarvis-util/jarvis_util/shell/pscp.py deleted file mode 100644 index f74da34d0..000000000 --- a/ci/jarvis-util/jarvis_util/shell/pscp.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -This module provides methods to distribute a command among multiple -nodes using SSH. This class is intended to be called from Exec, -not by general users. -""" - -from .scp import Scp -from .exec_info import Executable - - -class Pscp(Executable): - """ - Execute commands on multiple hosts using SSH. - """ - - def __init__(self, paths, exec_info): - """ - Copy files to a set of remote hosts via rsync. - - Case 1: Paths is a single file: - paths = '/tmp/hi.txt' - '/tmp/hi.txt' will be copied to user@host:/tmp/hi.txt - - Case 2: Paths is a list of files: - paths = ['/tmp/hi1.txt', '/tmp/hi2.txt'] - Repeat Case 1 twice. - - Case 3: Paths is a list of tuples of files: - paths = [('/tmp/hi.txt', '/tmp/remote_hi.txt')] - '/tmp/hi.txt' will be copied to user@host:'/tmp/remote_hi.txt' - - :param paths: Either a path to a file, a list of files, or a list of - tuples of files. - :param exec_info: Connection information for SSH - """ - super().__init__() - self.exec_async = exec_info.exec_async - self.hosts = exec_info.hostfile.hosts - self.scp_nodes = [] - self.stdout = {} - self.stderr = {} - self.hosts = exec_info.hostfile.hosts - for host in self.hosts: - ssh_exec_info = exec_info.mod(hostfile=None, - hosts=host, - exec_async=True) - self.scp_nodes.append(Scp(paths, ssh_exec_info)) - if self.exec_async: - self.wait() - - def wait(self): - self.wait_list(self.scp_nodes) - self.per_host_outputs(self.scp_nodes) - self.set_exit_code() - - def set_exit_code(self): - self.set_exit_code_list(self.scp_nodes) - diff --git a/ci/jarvis-util/jarvis_util/shell/pssh_exec.py b/ci/jarvis-util/jarvis_util/shell/pssh_exec.py deleted file mode 100644 index 720869391..000000000 --- a/ci/jarvis-util/jarvis_util/shell/pssh_exec.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -This module provides methods to distribute a command among multiple -nodes using SSH. This class is intended to be called from Exec, -not by general users. -""" - -from .ssh_exec import SshExec -from .local_exec import LocalExec -from .exec_info import ExecInfo, ExecType, Executable - - -class PsshExec(Executable): - """ - Execute commands on multiple hosts using SSH. - """ - - def __init__(self, cmd, exec_info): - """ - Execute commands on multiple hosts. - - :param cmd: A list of commands or a single command string - :param exec_info: Info needed to execute command with SSH - """ - super().__init__() - self.cmd = self.smash_cmd(cmd) - self.exec_async = exec_info.exec_async - self.hosts = exec_info.hostfile.hosts - self.execs_ = [] - self.stdout = {} - self.stderr = {} - if len(self.hosts): - for host in self.hosts: - ssh_exec_info = exec_info.mod(hostfile=None, - hosts=host, - exec_async=True) - self.execs_.append(SshExec(cmd, ssh_exec_info)) - else: - self.execs_.append( - LocalExec(cmd, exec_info)) - return - if not self.exec_async: - self.wait() - - def wait(self): - self.wait_list(self.execs_) - self.per_host_outputs(self.execs_) - self.set_exit_code() - - def set_exit_code(self): - self.set_exit_code_list(self.execs_) - - -class PsshExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.PSSH, **kwargs) - diff --git a/ci/jarvis-util/jarvis_util/shell/scp.py b/ci/jarvis-util/jarvis_util/shell/scp.py deleted file mode 100644 index 8b39301e6..000000000 --- a/ci/jarvis-util/jarvis_util/shell/scp.py +++ /dev/null @@ -1,115 +0,0 @@ -""" -This module provides methods to execute a single command remotely using SSH. -This class is intended to be called from Exec, not by general users. -""" -from .local_exec import LocalExec -from .exec_info import Executable - - -class _Scp(LocalExec): - """ - This class provides methods to copy data over SSH using the "rsync" - command utility in Linux - """ - - def __init__(self, src_path, dst_path, exec_info): - """ - Copy a file or directory from source to destination via rsync - - :param src_path: The path to the file on the host - :param dst_path: The desired file path on the remote host - :param exec_info: Info needed to execute command with SSH - """ - - self.addr = exec_info.hostfile.hosts[0] - self.src_path = src_path - self.dst_path = dst_path - self.user = exec_info.user - self.pkey = exec_info.pkey - self.port = exec_info.port - self.sudo = exec_info.sudo - super().__init__(self.rsync_cmd(src_path, dst_path), - exec_info.mod(env=exec_info.basic_env)) - - def rsync_cmd(self, src_path, dst_path): - lines = ['rsync -ha'] - if self.pkey is not None or self.port is not None: - ssh_lines = ['ssh'] - if self.pkey is not None: - ssh_lines.append(f'-i {self.pkey}') - if self.port is not None: - ssh_lines.append(f'-p {self.port}') - ssh_cmd = ' '.join(ssh_lines) - lines.append(f'-e \'{ssh_cmd}\'') - lines.append(src_path) - if self.user is not None: - lines.append(f'{self.user}@{self.addr}:{dst_path}') - else: - lines.append(f'{self.addr}:{dst_path}') - rsync_cmd = ' '.join(lines) - return rsync_cmd - - -class Scp(Executable): - """ - Secure copy data between two hosts. - """ - - def __init__(self, paths, exec_info): - """ - Copy files via rsync. - - Case 1: Paths is a single file: - paths = '/tmp/hi.txt' - '/tmp/hi.txt' will be copied to user@host:/tmp/hi.txt - - Case 2: Paths is a list of files: - paths = ['/tmp/hi1.txt', '/tmp/hi2.txt'] - Repeat Case 1 twice. - - Case 3: Paths is a list of tuples of files: - paths = [('/tmp/hi.txt', '/tmp/remote_hi.txt')] - '/tmp/hi.txt' will be copied to user@host:'/tmp/remote_hi.txt' - - :param paths: Either a path to a file, a list of files, or a list of - tuples of files. - :param exec_info: Connection information for SSH - """ - - super().__init__() - self.paths = paths - self.exec_info = exec_info - self.scp_nodes = [] - if isinstance(paths, str): - self._exec_single_path(paths) - if isinstance(paths, list): - if len(paths) == 0: - raise Exception('Must have at least one path to scp') - elif isinstance(paths[0], str): - self._exec_many_paths(paths) - elif isinstance(paths[0], tuple): - self._exec_many_paths_tuple(paths) - elif isinstance(paths[0], list): - self._exec_many_paths_tuple(paths) - if not self.exec_info.exec_async: - self.wait() - - def _exec_single_path(self, path): - self.scp_nodes.append(_Scp(path, path, self.exec_info)) - - def _exec_many_paths(self, paths): - for path in paths: - self.scp_nodes.append(_Scp(path, path, self.exec_info)) - - def _exec_many_paths_tuple(self, path_tlist): - for src, dst in path_tlist: - self.scp_nodes.append(_Scp(src, dst, self.exec_info)) - - def wait(self): - self.wait_list(self.scp_nodes) - self.smash_list_outputs(self.scp_nodes) - self.set_exit_code() - return self.exit_code - - def set_exit_code(self): - self.set_exit_code_list(self.scp_nodes) diff --git a/ci/jarvis-util/jarvis_util/shell/ssh_exec.py b/ci/jarvis-util/jarvis_util/shell/ssh_exec.py deleted file mode 100644 index bd9b85135..000000000 --- a/ci/jarvis-util/jarvis_util/shell/ssh_exec.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -This module provides methods to execute a single command remotely using SSH. -This class is intended to be called from Exec, not by general users. -""" -from .local_exec import LocalExec -from .exec_info import ExecInfo, ExecType - - -class SshExec(LocalExec): - """ - This class provides methods to execute a command via SSH. - """ - - def __init__(self, cmd, exec_info): - """ - Execute a command remotely via SSH - - :param cmd: A list of commands or a single command string - :param exec_info: Info needed to execute command with SSH - """ - - cmd = self.smash_cmd(cmd) - self.addr = exec_info.hostfile.hosts[0] - self.user = exec_info.user - self.pkey = exec_info.pkey - self.port = exec_info.port - self.sudo = exec_info.sudo - self.ssh_env = exec_info.env - super().__init__(self.ssh_cmd(cmd), - exec_info.mod(env=exec_info.basic_env)) - - def ssh_cmd(self, cmd): - lines = ['ssh'] - if self.pkey is not None: - lines.append(f'-i {self.pkey}') - if self.port is not None: - lines.append(f'-p {self.port}') - if self.user is not None: - lines.append(f'{self.user}@{self.addr}') - else: - lines.append(f'{self.addr}') - ssh_cmd = ' '.join(lines) - - cmd_lines = [] - if self.ssh_env is not None: - for key, val in self.ssh_env.items(): - cmd_lines.append(f'{key}={val}') - cmd_lines.append(cmd) - env_cmd = ' '.join(cmd_lines) - real_cmd = f'{ssh_cmd} \"{env_cmd}\"' - return real_cmd - - -class SshExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.SSH, **kwargs) diff --git a/ci/jarvis-util/jarvis_util/util/__init__.py b/ci/jarvis-util/jarvis_util/util/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ci/jarvis-util/jarvis_util/util/argparse.py b/ci/jarvis-util/jarvis_util/util/argparse.py deleted file mode 100644 index e58486be5..000000000 --- a/ci/jarvis-util/jarvis_util/util/argparse.py +++ /dev/null @@ -1,454 +0,0 @@ -""" -This module contains an argument parser which defines -""" - -import sys -import os -from abc import ABC, abstractmethod -import shlex -from tabulate import tabulate - - -class ArgParse(ABC): - """ - A class for parsing command line arguments. - Parsed menu name stored in self.menu_name - Parsed menu arguments stored in self.kwargs - Parsed remaining arguments stored in self.remainder - """ - - def __init__(self, args=None, exit_on_fail=True): - if args is None: - args = sys.argv[1:] - elif isinstance(args, str): - args = shlex.split(args) - args = ' '.join(args) - self.binary_name = os.path.basename(sys.argv[0]) - self.orig_args = shlex.split(args) - self.args = self.orig_args - self.error = None - self.exit_on_fail = exit_on_fail - self.menus = [] - self.vars = {} - self.remainder = None - self.pos_required = False - self.use_remainder = False - - self.menu = None - self.menu_name = None - self.kwargs = {} - self.define_options() - self._parse() - - @abstractmethod - def define_options(self): - """ - User-defined options menu - - :return: - """ - pass - - def process_args(self): - """ - After args have been parsed, can call this function to process - the arguments. Assumes that derived ArgParse class has a function - for each menu option. - - :return: - """ - - func_name = self.menu_name.replace(' ', '_') - func = getattr(self, func_name) - func(self) - - def add_menu(self, name=None, msg=None, - use_remainder=False): - """ - A menu is a container of arguments. - - :param name: The name that appears in the CLI to trigger the menu. - Spaces indicate menu nesting. E.g., 'repo add' will trigger the - menu argparser only if 'repo' and 'add' appear next to each other - in the argument list. - :param msg: The message to print if the user selects an improper menu - in the CLI. - :param use_remainder: Whether or not the menu should store all remaining - arguments for further use later. - :return: - """ - - toks = [] - if name is not None: - toks = name.split() - self.menus.append({ - 'name_str': ' '.join(toks), - 'name': toks, - 'msg': msg, - 'num_required': 0, - 'pos_opts': [], - 'kw_opts': {}, - 'use_remainder': use_remainder - }) - self.pos_required = False - self.menu = self.menus[-1] - - def start_required(self): - """ - Define a set of required positional arguments. - - :return: None - """ - self.pos_required = True - - def end_required(self): - """ - Finish the set of required positional arguments. - - :return: None - """ - self.pos_required = False - - def add_arg(self, - name, - argtype=str, - choices=None, - default=None, - msg=None, - action=None, - aliases=None): - """ - Append an argument to a menu. - Arguments can either be positional or key-value. - Positional arguments do NOT start with a dash - Key-value arguments are separated by dashes - - :param name: The name of the argument. If name starts with a dash, - it will be interpreted as a positional arg - :param argtype: The type of the argument being stored. - :param choices: The set of acceptable inputs as a list - :param default: The default value to store - :param msg: The help message to print if there is a problem - :param action: An action to execute if the argument exists - :param aliases: Other names for the same thing (list) - :return: - """ - - # Add all aliases - if aliases is not None: - for alias in aliases: - if '-' in alias: - self.add_arg(alias, argtype, choices, default, msg, action) - else: - raise f"Can't have a non-keyword alias: {alias}" - # Handle the specific boolean argument case - is_kwarg = '-' in name - if is_kwarg and argtype == bool: - self._add_bool_kw_arg(name, default, msg) - return - # Add general argument - menu = self.menu - arg = { - 'name': name, - 'dict_name': self._get_opt_name(name), - 'type': argtype, - 'choices': choices, - 'default': default, - 'action': action, - 'msg': msg, - 'required': self.pos_required, - 'has_input': True - } - if is_kwarg: - self.pos_required = False - menu['kw_opts'][name] = arg - else: - if self.pos_required: - menu['num_required'] += 1 - menu['pos_opts'].append(arg) - self.kwargs[arg['dict_name']] = default - - def _add_bool_kw_arg(self, - name, - default, - msg=None, - is_other=False, - dict_name=None): - """ - Boolean arguments can be indicated using a +-. - + indicates true, - indicates false. - - :param name: The name of the boolean arg - :param default: Default value of the boolean arg - :param msg: Help message - :param is_other: Indicates this is an alias of the +- syntax. - :param dict_name: Name to make the argument in final kwargs - :return: None - """ - menu = self.menu - if dict_name is None: - dict_name = self._get_opt_name(name, True) - arg = { - 'name': name, - 'dict_name': dict_name, - 'type': bool, - 'choices': None, - 'default': default, - 'action': None, - 'msg': msg, - 'required': False, - 'has_input': not is_other - } - if not is_other: - self._add_bool_kw_arg('--with-' + name.strip('-'), - True, msg, True, dict_name) - self._add_bool_kw_arg('--no-' + name.strip('-'), - False, msg, True, dict_name) - self.pos_required = False - menu['kw_opts'][name] = arg - - def _parse(self): - """ - Parse the CLI arguments. - Will modify self.menu to indicate which menu is used - Will modify self.args to create a key-value store of arguments - - :return: None. - """ - self.menus.sort(key=lambda x: len(x['name']), reverse=True) - self._parse_menu() - - def _parse_menu(self): - """ - Determine which menu is used in the CLI. - - :return: Modify self.menu. No return value. - """ - - self.menu = None - for menu in self.menus: - menu_name = menu['name'] - if len(menu_name) > len(self.args): - continue - if menu_name == self.args[0:len(menu_name)]: - self.menu = menu - break - if self.menu is None: - self._invalid_menu() - self.menu_name = self.menu['name_str'] - self.add_arg('-h', - default=None, - msg='print help message', - action=self._print_help, - aliases=['--help']) - menu_name = self.menu['name'] - self.use_remainder = self.menu['use_remainder'] - self.args = self.args[len(menu_name):] - self._parse_args() - - def _parse_args(self): - self._set_defaults() - i = self._parse_pos_args() - self._parse_kw_args(i) - - def _set_defaults(self): - all_opts = self.menu['pos_opts'] + list(self.menu['kw_opts'].values()) - for opt_info in all_opts: - if opt_info['default'] is None: - continue - self.__dict__[opt_info['dict_name']] = opt_info['default'] - - def _parse_pos_args(self): - """ - Parse positional arguments - Modify the self.kwargs dictionary - - :return: - """ - - i = 0 - args = self.args - menu = self.menu - while i < len(menu['pos_opts']): - # Get the positional arg info - opt_name = menu['pos_opts'][i]['name'] - opt_dict_name = menu['pos_opts'][i]['dict_name'] - opt_type = menu['pos_opts'][i]['type'] - opt_choices = menu['pos_opts'][i]['choices'] - if i >= len(args): - if i >= menu['num_required']: - break - else: - self._missing_positional(opt_name) - - # Get the arg value - arg = args[i] - if arg in menu['kw_opts']: - break - arg = self._convert_opt(opt_name, opt_type, opt_choices, arg) - - # Set the argument - self.kwargs[opt_dict_name] = arg - i += 1 - return i - - def _parse_kw_args(self, i): - """ - Parse key-word arguments. - Modify the self.kwargs dictionary - - :param i: The starting index in the self.args list where kv pairs start - :return: - """ - - menu = self.menu - args = self.args - while i < len(args): - # Get argument name - opt_name = args[i] - if opt_name not in menu['kw_opts']: - if self.use_remainder: - self.remainder = ' '.join(args[i:]) - return - else: - self._invalid_kwarg(opt_name) - - # Get argument type - opt = menu['kw_opts'][opt_name] - opt_has_input = opt['has_input'] - opt_dict_name = opt['dict_name'] - opt_type = opt['type'] - opt_default = opt['default'] - opt_action = opt['action'] - opt_choices = opt['choices'] - if not opt_has_input: - arg = opt_default - i += 1 - elif opt_action is not None: - opt_action() - arg = None - i += 1 - elif self._next_is_kw_value(i): - arg = args[i + 1] - i += 2 - elif opt_default is not None: - arg = opt_default - i += 1 - else: - arg = None - self._invalid_kwarg_default(opt_name) - - # Convert argument to type - arg = self._convert_opt(opt_name, opt_type, opt_choices, arg) - - # Set the argument - self.kwargs[opt_dict_name] = arg - - def _convert_opt(self, opt_name, opt_type, opt_choices, arg): - if opt_type is not None: - # pylint: disable=W0702 - try: - arg = opt_type(arg) - if opt_choices is not None: - if arg not in opt_choices: - self._invalid_choice(opt_name, arg) - except: - self._invalid_type(opt_name, opt_type) - # pylint: enable=W0702 - return arg - - def _next_is_kw_value(self, i): - if i + 1 >= len(self.args): - return False - return self.args[i + 1] not in self.menu['kw_opts'] - - def _get_opt_name(self, opt_name, is_bool_arg=False): - """ - Normalize option names - '-' are converted into '_' - '--with-' and '--no-' are removed - '+' and '-' for boolean args are removed - - :param opt_name: The menu option name - :param is_bool_arg: Whether the arg is a boolean arg - :return: - """ - - if not is_bool_arg: - return opt_name.strip('-').replace('-', '_') - else: - return opt_name.replace('--with-', '', 1)\ - .replace('--no-', '', 1).\ - strip('-').replace('-', '_') - - def _invalid_menu(self): - self._print_error('Could not find a menu') - - def _invalid_choice(self, opt_name, arg): - self._print_menu_error(f'{opt_name}={arg} is not a valid choice') - - def _missing_positional(self, opt_name): - self._print_menu_error(f'{opt_name} was required, but not defined') - - def _invalid_kwarg(self, opt_name): - self._print_menu_error(f'{opt_name} is not a valid key-word argument') - - def _invalid_kwarg_default(self, opt_name): - self._print_menu_error( - f'{opt_name} was not given a value, but requires one') - - def _invalid_type(self, opt_name, opt_type): - self._print_menu_error(f'{opt_name} was not of type {opt_type}') - - def _print_menu_error(self, msg): - self._print_error(f'{self.menu["name_str"]} {msg}') - - def _print_error(self, msg): - print(f'{msg}') - self._print_help() - if self.exit_on_fail: - sys.exit(1) - else: - raise Exception(msg) - - def _print_help(self): - if self.menu is not None: - self._print_menu_help() - else: - self._print_menus() - - def _print_menus(self): - for menu in self.menus: - self.menu = menu - self._print_menu_help(True) - - def _print_menu_help(self, only_usage=False): - pos_args = [] - for arg in self.menu['pos_opts']: - if arg['required']: - pos_args.append(f'[{arg["name"]}]') - else: - pos_args.append(f'[{arg["name"]} (opt)]') - pos_args = ' '.join(pos_args) - menu_str = self.menu['name_str'] - print(f'USAGE: {self.binary_name} {menu_str} {pos_args} ...') - if self.menu['msg'] is not None: - print(self.menu['msg']) - print() - if only_usage: - return - - headers = ['Name', 'Default', 'Type', 'Description'] - table = [] - all_opts = self.menu['pos_opts'] + list(self.menu['kw_opts'].values()) - for arg in all_opts: - default = arg['default'] - if self._is_bool_arg(arg): - default = None - table.append( - [arg['name'], default, arg['type'], arg['msg']]) - print(tabulate(table, headers=headers)) - - def _is_bool_arg(self, arg): - return arg['type'] == bool and (arg['name'].startswith('--with-') or - arg['name'].startswith('--no-')) diff --git a/ci/jarvis-util/jarvis_util/util/expand_env.py b/ci/jarvis-util/jarvis_util/util/expand_env.py deleted file mode 100644 index a9d8aefa7..000000000 --- a/ci/jarvis-util/jarvis_util/util/expand_env.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -This module contains functions for expanding environment variables for -dictionaries -""" - -import os - - -def expand_env(data): - """ - Expand environment variables for dictionaries - - :param data: A dict where strings may contain environment variables to - expand - :return: - """ - if isinstance(data, str): - return os.path.expandvars(data) - if isinstance(data, dict): - for key, val in data.items(): - data[key] = expand_env(val) - if isinstance(data, (list, tuple)): - for i, val in enumerate(data): - data[i] = expand_env(val) - return data diff --git a/ci/jarvis-util/jarvis_util/util/hostfile.py b/ci/jarvis-util/jarvis_util/util/hostfile.py deleted file mode 100644 index 4d22ae233..000000000 --- a/ci/jarvis-util/jarvis_util/util/hostfile.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -This module contains methods for parsing hostfiles and storing hosts -""" - -import os -import socket -import re -import itertools - - -class Hostfile: - """ - Parse a hostfile or store a set of hosts passed in manually. - """ - - def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, - text=None, find_ips=True): - """ - Constructor. Parse hostfile or store existing host list. - - :param hostfile: The path to the hostfile - :param all_hosts: a list of strings representing all hostnames - :param all_hosts_ip: a list of strings representing all host IPs - :param text: Text of a hostfile - :param find_ips: Whether to construct host_ip and all_host_ip fields - """ - self.hosts_ip = [] - self.hosts = [] - self.all_hosts = [] - self.all_hosts_ip = [] - self.path = hostfile - self.find_ips = find_ips - - # Set the host ips directly - if all_hosts_ip is not None: - self.all_hosts_ip = all_hosts_ip - self.hosts_ip = all_hosts_ip - self.find_ips = False - - # Direct constructor - if all_hosts is not None: - self._set_hosts(all_hosts) - - # From hostfile path - elif hostfile is not None: - self._load_hostfile(self.path) - - # From hostfile text - elif text is not None: - self.parse(text) - - # Both hostfile and hosts are None - else: - self._set_hosts(['localhost']) - - def _load_hostfile(self, path): - """ - Expand a hostfile - - :param path: the path to the hostfile - :return: - """ - if not os.path.exists(path): - raise Exception('hostfile not found') - self.path = path - with open(path, 'r', encoding='utf-8') as fp: - text = fp.read() - self.parse(text) - return self - - def parse(self, text): - """ - Parse a hostfile text. - - :param text: Hostfile text - :param set_hosts: Whether or not to set hosts - :return: None - """ - - lines = text.strip().splitlines() - hosts = [] - for line in lines: - self._expand_line(hosts, line) - self._set_hosts(hosts) - - def _expand_line(self, hosts, line): - """ - Will expand brackets in a host declaration. - E.g., host-[0-5,...]-name - - :param hosts: the current set of hosts - :param line: the line to parse - :return: None - """ - toks = re.split(r'[\[\]]', line) - brkts = [tok for i, tok in enumerate(toks) if i % 2 == 1] - num_set = [] - - # Get the expanded set of numbers for each bracket - for i, brkt in enumerate(brkts): - num_set.append([]) - self._expand_set(num_set[-1], brkt) - - # Expand the host string - host_nums = self._product(num_set) - for host_num in host_nums: - host = [] - for i, tok in enumerate(toks): - if i % 2 == 1: - host.append(host_num[int(i/2)]) - else: - host.append(tok) - hosts.append(''.join(host)) - - def _expand_set(self, num_set, brkt): - """ - Expand a bracket set. - The bracket initially has notation: [0-5,0-9,...] - """ - - rngs = brkt.split(',') - for rng in rngs: - self._expand_range(num_set, rng) - - def _expand_range(self, num_set, brkt): - """ - Expand a range. - The range has notation: A-B or A - - :param num_set: the numbers in the range - :param brkt: - :return: - """ - if len(brkt) == 0: - return - if '-' in brkt: - min_max = brkt.split('-') - if len(min_max[0]) == len(min_max[1]): - nums = range(int(min_max[0]), int(min_max[1]) + 1) - num_set += [str(num).zfill(len(min_max[0])) for num in nums] - else: - nums = range(int(min_max[0]), int(min_max[1]) + 1) - num_set += [str(num) for num in nums] - else: - num_set.append(brkt) - - def _product(self, num_set): - """ - Return the cartesian product of the number set - - :param num_set: The numbers to product - :return: - """ - return list(itertools.product(*num_set)) - - def _set_hosts(self, all_hosts): - self.all_hosts = all_hosts - if self.find_ips: - self.all_hosts_ip = [socket.gethostbyname(host) - for host in all_hosts] - self.hosts = self.all_hosts - if self.find_ips: - self.hosts_ip = self.all_hosts_ip - return self - - def subset(self, count): - sub = Hostfile() - sub.path = self.path - sub.all_hosts = self.all_hosts - sub.all_hosts_ip = self.all_hosts_ip - sub.hosts = self.hosts[:count] - sub.hosts_ip = self.hosts_ip[:count] - return sub - - def is_subset(self): - return len(self.hosts) != len(self.all_hosts) - - def save(self, path): - self.all_hosts = self.hosts - self.all_hosts_ip = self.hosts_ip - self.path = path - with open(path, 'w', encoding='utf-8') as fp: - fp.write('\n'.join(self.all_hosts)) - return self - - def ip_list(self): - return self.hosts_ip - - def hostname_list(self): - return self.hosts - - def enumerate(self): - return enumerate(self.hosts) - - def host_str(self, sep=','): - return sep.join(self.hosts) - - def ip_str(self, sep=','): - return sep.join(self.hosts_ip) - - def __len__(self): - return len(self.hosts) - - def __getitem__(self, idx): - return self.hosts[idx] - - def __str__(self): - return str(self.hosts) - - def __repr__(self): - return str(self) - - def __eq__(self, other): - return (self.hosts == other.hosts and - self.all_hosts == other.all_hosts) - diff --git a/ci/jarvis-util/jarvis_util/util/import_all.py b/ci/jarvis-util/jarvis_util/util/import_all.py deleted file mode 100644 index 8a10e6125..000000000 --- a/ci/jarvis-util/jarvis_util/util/import_all.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -This file contains methods to automate large import __init__.py files -""" - -import pathlib -import os - - -def _import_recurse(root_path, root, stmts): - """ - Identify the set of files in the current "root" directory - - :param root_path: The path to the root of the python package - :param root: The current subdirectory of the python package - :param stmts: The current set of import statements - :return: - """ - for file in os.listdir(root): - file = os.path.join(root, file) - if os.path.isfile(file): - file = os.path.relpath(file, root_path) - ext = file.split('.') - if ext[-1] == 'py': - toks = ext[0].split('/') - if toks[-1] == '__init__': - continue - import_stmt = '.'.join(toks) - stmts.append(f'from {import_stmt} import *') - elif os.path.isdir(file): - _import_recurse(root_path, file, stmts) - return stmts - - -def import_all(root_path, root): - """ - Create all import statement to do: from root import *. - - :param root_path: The root of the python repo - :param root: The current directory we are in within the repo - :return: - """ - stmts = [] - _import_recurse(root_path, root, stmts) - return '\"\"\"Import all modules\"\"\"\n' + '\n'.join(stmts) + '\n' - - -def build_global_import_file(root_path, pkg_name): - """ - Build a file to be able to do: from pkg_name import * - - :param root_path: The path to the python package's root directory - :param pkg_name: The name of the python package - :return: - """ - path = os.path.join(root_path, pkg_name) - imports = import_all(root_path, path) - with open(os.path.join(path, '__init__.py'), 'w', - encoding='utf-8') as fp: - fp.write(imports) - - -def build_global_import_from_bin(pkg_name): - """ - Build a file to be able to do: from pkg_name import * - This function is assumed to be called in the "bin" directory - of the main python repo - - :param pkg_name: The name of the python package being built - :return: - """ - root_path = str(pathlib.Path(__file__).parent.parent.parent.resolve()) - build_global_import_file(root_path, pkg_name) diff --git a/ci/jarvis-util/jarvis_util/util/import_mod.py b/ci/jarvis-util/jarvis_util/util/import_mod.py deleted file mode 100644 index 9c55c508d..000000000 --- a/ci/jarvis-util/jarvis_util/util/import_mod.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -This file contains helper methods to load a class dynamically from a file -""" - -import sys - -# NOTE(llogan): To get the path of the directory this file is in, use -# str(pathlib.Path(__file__).parent.resolve()) - - -def load_class(import_str, path, class_name): - """ - Loads a class from a python file. - - :param import_str: A python import string. E.g., for "myrepo.dir1.pkg" - :param path: The absolute path to the directory which contains the - beginning of the import statement. Let's say you have a git repo located - at "/home/hello/myrepo". The git repo has a subdirectory called "myrepo", - so "/home/hello/myrepo/myrepo". In this case, path would be - "/home/hello/myrepo". The import string "myrepo.dir1.pkg" will find - the "myrepo" part of the import string at "/home/hello/myrepo/myrepo". - :param class_name: The name of the class in the file - :return: - """ - sys.path.insert(0, path) - module = __import__(import_str, fromlist=[class_name]) - sys.path.pop(0) - return getattr(module, class_name) diff --git a/ci/jarvis-util/jarvis_util/util/naming.py b/ci/jarvis-util/jarvis_util/util/naming.py deleted file mode 100644 index af6505633..000000000 --- a/ci/jarvis-util/jarvis_util/util/naming.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -This module contains methods to create strings which follow a particular -naming convention. -""" - -import re - - -def to_camel_case(string): - """ - Convert a string in snake case to camel case - - :param string: - :return: - """ - if string is None: - return - words = re.sub(r'(_|-)+', ' ', string).split() - words = [word.capitalize() for word in words] - return ''.join(words) - - -def to_snake_case(string): - """ - Convert a string in CamelCase to snake case - :param string: - :return: - """ - if string is None: - return - words = re.split('([A-Z][a-z0-9_]*)', string) - words = [word for word in words if len(word)] - string = '_'.join(words) - return string.lower() diff --git a/ci/jarvis-util/jarvis_util/util/size_conv.py b/ci/jarvis-util/jarvis_util/util/size_conv.py deleted file mode 100644 index 266959bf0..000000000 --- a/ci/jarvis-util/jarvis_util/util/size_conv.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -This module provides methods to convert a semantic size string to an integer. -""" - - -class SizeConv: - """ - A class which provides methods to convert a semantic size string to an int. - """ - - @staticmethod - def to_int(text): - text = text.lower() - if 'k' in text: - return SizeConv.kb(text) - if 'm' in text: - return SizeConv.mb(text) - if 'g' in text: - return SizeConv.gb(text) - if 't' in text: - return SizeConv.tb(text) - if 'p' in text: - return SizeConv.pb(text) - return int(text) - - @staticmethod - def kb(num): - return int(float(num.split('k')[0]) * (1 << 10)) - - @staticmethod - def mb(num): - return int(float(num.split('m')[0]) * (1 << 20)) - - @staticmethod - def gb(num): - return int(float(num.split('g')[0]) * (1 << 30)) - - @staticmethod - def tb(num): - return int(float(num.split('t')[0]) * (1 << 40)) - - @staticmethod - def pb(num): - return int(float(num.split('p')[0]) * (1 << 50)) diff --git a/ci/jarvis-util/pylintrc b/ci/jarvis-util/pylintrc deleted file mode 100644 index 5371019ba..000000000 --- a/ci/jarvis-util/pylintrc +++ /dev/null @@ -1,429 +0,0 @@ -# This Pylint rcfile contains a best-effort configuration to uphold the -# best-practices and style described in the Google Python style guide: -# https://google.github.io/styleguide/pyguide.html -# -# Its canonical open-source location is: -# https://google.github.io/styleguide/pylintrc - -[MASTER] - -# Files or directories to be skipped. They should be base names, not paths. -ignore=third_party - -# Files or directories matching the regex patterns are skipped. The regex -# matches against base names, not paths. -ignore-patterns= - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=4 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=abstract-method, - apply-builtin, - arguments-differ, - attribute-defined-outside-init, - backtick, - bad-option-value, - basestring-builtin, - buffer-builtin, - c-extension-no-member, - consider-using-enumerate, - cmp-builtin, - cmp-method, - coerce-builtin, - coerce-method, - delslice-method, - div-method, - duplicate-code, - eq-without-hash, - execfile-builtin, - file-builtin, - filter-builtin-not-iterating, - fixme, - getslice-method, - global-statement, - hex-method, - idiv-method, - implicit-str-concat, - import-error, - import-self, - import-star-module-level, - inconsistent-return-statements, - input-builtin, - intern-builtin, - invalid-str-codec, - locally-disabled, - long-builtin, - long-suffix, - map-builtin-not-iterating, - misplaced-comparison-constant, - missing-function-docstring, - metaclass-assignment, - next-method-called, - next-method-defined, - no-absolute-import, - no-else-break, - no-else-continue, - no-else-raise, - no-else-return, - no-init, # added - no-member, - no-name-in-module, - no-self-use, - nonzero-method, - oct-method, - old-division, - old-ne-operator, - old-octal-literal, - old-raise-syntax, - parameter-unpacking, - print-statement, - raising-string, - range-builtin-not-iterating, - raw_input-builtin, - rdiv-method, - reduce-builtin, - relative-import, - reload-builtin, - round-builtin, - setslice-method, - signature-differs, - standarderror-builtin, - suppressed-message, - sys-max-int, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-boolean-expressions, - too-many-branches, - too-many-instance-attributes, - too-many-locals, - too-many-nested-blocks, - too-many-public-methods, - too-many-return-statements, - too-many-statements, - trailing-newlines, - unichr-builtin, - unicode-builtin, - unnecessary-pass, - unpacking-in-except, - useless-else-on-loop, - useless-object-inheritance, - useless-suppression, - using-cmp-argument, - wrong-import-order, - xrange-builtin, - zip-builtin-not-iterating, - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=main,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl - -# Regular expression matching correct function names -function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct constant names -const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct attribute names -attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ - -# Regular expression matching correct argument names -argument-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=^_?[A-Z][a-zA-Z0-9]*$ - -# Regular expression matching correct module names -module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ - -# Regular expression matching correct method names -method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=10 - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt -# lines made too long by directives to pytype. - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=(?x)( - ^\s*(\#\ )??$| - ^\s*(from\s+\S+\s+)?import\s+.+$) - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=yes - -# Maximum number of lines in a module -max-module-lines=99999 - -# String used as indentation unit. The internal Google style guide mandates 2 -# spaces. Google's externaly-published style guide says 4, consistent with -# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google -# projects (like TensorFlow). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=TODO - - -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=yes - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging,absl.logging,tensorflow.io.logging - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec, - sets - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant, absl - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls, - class_ - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=StandardError, - Exception, - BaseException diff --git a/ci/jarvis-util/requirements.txt b/ci/jarvis-util/requirements.txt deleted file mode 100644 index 70849fea6..000000000 --- a/ci/jarvis-util/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -pyyaml -pylint==2.15.0 -coverage==5.5 -# coverage -coverage-lcov==0.2.4 -# coverage-lcov -pytest==6.2.5 -pandas -tabulate \ No newline at end of file diff --git a/ci/jarvis-util/setup.py b/ci/jarvis-util/setup.py deleted file mode 100644 index a1bdd4e91..000000000 --- a/ci/jarvis-util/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -import setuptools - -setuptools.setup( - name="jarvis_util", - packages=setuptools.find_packages(), - version="0.0.1", - author="Luke Logan", - author_email="llogan@hawk.iit.edu", - description="Various wrappers around shell utilities", - url="https://github.com/scs-lab/jarvis-util", - classifiers = [ - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Development Status :: 0 - Pre-Alpha", - "Environment :: Other Environment", - "Intended Audience :: Developers", - "License :: None", - "Operating System :: OS Independent", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Application Configuration", - ], - long_description="" -) diff --git a/ci/jarvis-util/test/unit/argparse_main.py b/ci/jarvis-util/test/unit/argparse_main.py deleted file mode 100644 index 7b636ca60..000000000 --- a/ci/jarvis-util/test/unit/argparse_main.py +++ /dev/null @@ -1,4 +0,0 @@ -from jarvis_util.util.argparse import ArgParse - -if __name__ == 'main': - args = ArgParse() \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/print5s.py b/ci/jarvis-util/test/unit/print5s.py deleted file mode 100644 index a6581b31c..000000000 --- a/ci/jarvis-util/test/unit/print5s.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -NOTE: this is a helper utility for test_local_exec -""" - -import time -import sys - - -for i in range(5): - sys.stdout.write(f"COUT: {i}\n") - sys.stderr.write(f"CERR: {i}\n") - time.sleep(1) \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/printNone.py b/ci/jarvis-util/test/unit/printNone.py deleted file mode 100644 index cf728ec39..000000000 --- a/ci/jarvis-util/test/unit/printNone.py +++ /dev/null @@ -1,4 +0,0 @@ -from jarvis_util.shell.local_exec import LocalExec, LocalExecInfo - -spawn_info = LocalExecInfo(hide_output=True) -LocalExec("echo hello", spawn_info) diff --git a/ci/jarvis-util/test/unit/test_argparse.py b/ci/jarvis-util/test/unit/test_argparse.py deleted file mode 100644 index 3c3f3d32f..000000000 --- a/ci/jarvis-util/test/unit/test_argparse.py +++ /dev/null @@ -1,10 +0,0 @@ -from jarvis_util.util.argparse import ArgParse -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo -import pathlib -from unittest import TestCase - - -class TestArgparse(TestCase): - def test_argparse_main(self): - pass \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/test_hostfile.py b/ci/jarvis-util/test/unit/test_hostfile.py deleted file mode 100644 index 47bd7c853..000000000 --- a/ci/jarvis-util/test/unit/test_hostfile.py +++ /dev/null @@ -1,57 +0,0 @@ -from jarvis_util.util.hostfile import Hostfile -import pathlib -from unittest import TestCase - - -class TestHostfile(TestCase): - def test_no_expand_int(self): - host = Hostfile(text='0', find_ips=False) - self.assertTrue(len(host.hosts) == 1) - self.assertTrue(host.hosts[0] == '0') - - def test_no_expand(self): - host = Hostfile(text='ares-comp-01', find_ips=False) - self.assertTrue(len(host.hosts) == 1) - self.assertTrue(host.hosts[0] == 'ares-comp-01') - - def test_expand_set(self): - host = Hostfile(text='ares-comp-[01-04]-40g', find_ips=False) - self.assertTrue(len(host.hosts) == 4) - self.assertTrue(host.hosts[0] == 'ares-comp-01-40g') - self.assertTrue(host.hosts[1] == 'ares-comp-02-40g') - self.assertTrue(host.hosts[2] == 'ares-comp-03-40g') - self.assertTrue(host.hosts[3] == 'ares-comp-04-40g') - - def test_expand_two_sets(self): - host = Hostfile(text='ares-comp-[01-02]-40g-[01-02]', find_ips=False) - self.assertTrue(len(host.hosts) == 4) - self.assertTrue(host.hosts[0] == 'ares-comp-01-40g-01') - self.assertTrue(host.hosts[1] == 'ares-comp-01-40g-02') - self.assertTrue(host.hosts[2] == 'ares-comp-02-40g-01') - self.assertTrue(host.hosts[3] == 'ares-comp-02-40g-02') - - def test_subset(self): - host = Hostfile(text='ares-comp-[01-02]-40g-[01-02]', find_ips=False) - host = host.subset(3) - self.assertTrue(len(host.hosts) == 3) - self.assertTrue(host.is_subset()) - self.assertTrue(host.hosts[0] == 'ares-comp-01-40g-01') - self.assertTrue(host.hosts[1] == 'ares-comp-01-40g-02') - self.assertTrue(host.hosts[2] == 'ares-comp-02-40g-01') - - def test_read_hostfile(self): - HERE = str(pathlib.Path(__file__).parent.resolve()) - hf = Hostfile(hostfile=f'{HERE}/test_hostfile.txt', find_ips=False) - print(hf.hosts) - self.assertEqual(len(hf), 15) - - def test_save_hostfile(self): - HERE = str(pathlib.Path(__file__).parent.resolve()) - hf = Hostfile(hostfile=f'{HERE}/test_hostfile.txt', find_ips=False) - hf_sub = hf.subset(4) - self.assertEqual(len(hf_sub), 4) - hf_sub.save('/tmp/test_hostfile.txt') - hf_sub_reload = Hostfile(hostfile=f'/tmp/test_hostfile.txt', - find_ips=False) - self.assertEqual(len(hf_sub_reload), 4) - self.assertEqual(hf_sub, hf_sub_reload) diff --git a/ci/jarvis-util/test/unit/test_hostfile.txt b/ci/jarvis-util/test/unit/test_hostfile.txt deleted file mode 100644 index 7ecc164e9..000000000 --- a/ci/jarvis-util/test/unit/test_hostfile.txt +++ /dev/null @@ -1 +0,0 @@ -ares-comp-[01-10,11,12-15]-40g \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/test_local_exec.py b/ci/jarvis-util/test/unit/test_local_exec.py deleted file mode 100644 index c0ced8c3b..000000000 --- a/ci/jarvis-util/test/unit/test_local_exec.py +++ /dev/null @@ -1,65 +0,0 @@ -import pathlib -import os -from jarvis_util.shell.local_exec import LocalExec, LocalExecInfo -from jarvis_util.shell.exec import Exec -from unittest import TestCase - - -class TestLocalExec(TestCase): - def _setup_files(self): - self.stdout = '/tmp/test_out.log' - self.stderr = '/tmp/test_err.log' - try: - os.remove(self.stdout) - except OSError: - pass - try: - os.remove(self.stderr) - except: - pass - - def test_default(self): - ret = Exec("echo hello") - self.assertEqual(ret.exit_code, 0) - self.assertEqual(len(ret.stdout['localhost']), 0) - - def test_pipe_stdout(self): - self._setup_files() - spawn_info = LocalExecInfo(pipe_stdout=self.stdout, - pipe_stderr=self.stderr, - collect_output=True) - ret = Exec("echo hello", spawn_info) - self.assertEqual(ret.stdout['localhost'].strip(), "hello") - self.assertEqual(ret.stderr['localhost'].strip(), "") - self.assertFile(self.stdout, "hello") - self.assertFile(self.stderr, "") - - def test_hide_stdout(self): - HERE = str(pathlib.Path(__file__).parent.resolve()) - PRINTNONE = os.path.join(HERE, 'printNone.py') - spawn_info = LocalExecInfo(collect_output=True) - ret = Exec(f"python3 {PRINTNONE}", spawn_info) - self.assertEqual(ret.stdout['localhost'].strip(), "") - self.assertEqual(ret.stderr['localhost'].strip(), "") - - def test_periodic_print(self): - self._setup_files() - HERE = str(pathlib.Path(__file__).parent.resolve()) - PRINT5s = os.path.join(HERE, 'print5s.py') - ret = Exec(f"python3 {PRINT5s}", - LocalExecInfo(pipe_stdout=self.stdout, - pipe_stderr=self.stderr)) - stdout_data = "\n".join([f"COUT: {i}" for i in range(5)]) - stderr_data = "\n".join([f"CERR: {i}" for i in range(5)]) - self.assertFile(self.stdout, stdout_data) - self.assertFile(self.stderr, stderr_data) - - def assertFile(self, path, data, strip=True): - self.assertTrue(os.path.exists(path)) - with open(path, 'r') as fp: - if strip: - data = data.strip() - file_data = fp.read().strip() - else: - file_data = fp.read() - self.assertEqual(data, file_data) diff --git a/ci/jarvis-util/test/unit/test_system_info.py b/ci/jarvis-util/test/unit/test_system_info.py deleted file mode 100644 index a9b6c9d91..000000000 --- a/ci/jarvis-util/test/unit/test_system_info.py +++ /dev/null @@ -1,137 +0,0 @@ -from jarvis_util.util.argparse import ArgParse -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo -from jarvis_util.util.hostfile import Hostfile -from jarvis_util.introspect.system_info import Lsblk, \ - ListFses, FiInfo, Blkid, ResourceGraph, StorageDeviceType -from jarvis_util.util.size_conv import SizeConv -import pathlib -import itertools -from unittest import TestCase - - -class TestSystemInfo(TestCase): - def test_lsblk(self): - Lsblk(LocalExecInfo(hide_output=True)) - - def test_list_fses(self): - ListFses(LocalExecInfo(hide_output=True)) - - def test_fi_info(self): - FiInfo(LocalExecInfo(hide_output=True)) - - def test_blkid(self): - Blkid(LocalExecInfo(hide_output=True)) - - def test_resource_graph(self): - rg = ResourceGraph() - rg.build(LocalExecInfo(hide_output=True)) - rg.save('/tmp/resource_graph.yaml') - rg.load('/tmp/resource_graph.yaml') - rg.filter_fs(r'/$', '/${USER}', 'NVME') - rg.filter_hosts(Hostfile(), '1gbps') - rg.save('/tmp/resource_graph.yaml') - - def test_custom_resource_graph(self): - rg = ResourceGraph() - all_hosts = ['host1', 'host2', 'host3'] - all_hosts_ip = ['192.168.1.0', '192.168.1.1', '192.168.1.2'] - providers = ['tcp', 'ib', 'roce'] - hosts = Hostfile(all_hosts=all_hosts, all_hosts_ip=all_hosts_ip) - - # Add networks for each node - rg.set_hosts(hosts) - for provider in providers: - rg.add_net(hosts, - provider=provider) - rg.add_net(hosts.subset(1), - provider='uncommon') - - # Add common storage for each node - rg.add_storage(hosts, - device='/dev/sda1', - mount='/', - tran='sata', - rota=True, - size=SizeConv.to_int('10g'), - shared=False) - rg.add_storage(hosts, - device='/dev/sda2', - mount='/mnt/hdd/$USER', - tran='sata', - rota=True, - size=SizeConv.to_int('200g'), - shared=False) - rg.add_storage(hosts, - device='/dev/sdb1', - mount='/mnt/ssd/$USER', - tran='sata', - rota=False, - size=SizeConv.to_int('50g'), - shared=False) - rg.add_storage(hosts, - device='/dev/nvme0n1', - mount='/mnt/nvme/$USER', - tran='nvme', - rota=False, - size=SizeConv.to_int('100g'), - shared=False) - rg.add_storage(hosts.subset(1), - device='/dev/nvme0n2', - mount='/mnt/nvme2/$USER', - tran='nvme', - rota=False, - size=SizeConv.to_int('10g'), - shared=False) - rg.add_storage(hosts, - device='/dev/nvme0n3', - tran='nvme', - rota=False, - size=SizeConv.to_int('100g'), - shared=False) - - # Filter only mounts in '/mnt' - rg.filter_fs('/mnt/*') - - # Apply changes - rg.apply() - - # Find all mounted NVMes - df = rg.find_storage([StorageDeviceType.NVME]) - self.assertTrue(len(df[df.tran == 'nvme']) == 4) - self.assertTrue(len(df[df.tran == 'sata']) == 0) - self.assertTrue(len(df) == 4) - - # Find all mounted & common NVMes and SSDs - df = rg.find_storage([StorageDeviceType.NVME, - StorageDeviceType.SSD], - common=True) - self.assertTrue(len(df[df.tran == 'nvme']) == 3) - self.assertTrue(len(df[df.tran == 'sata']) == 3) - self.assertTrue(len(df) == 6) - - # Select a single nvme per-node - df = rg.find_storage([StorageDeviceType.NVME, - StorageDeviceType.SSD], - common=True, - count_per_node=1) - self.assertTrue(len(df[df.tran == 'nvme']) == 3) - self.assertTrue(len(df[df.tran == 'sata']) == 0) - self.assertTrue(len(df) == 3) - - # Select a single nvme and ssd per-node - df = rg.find_storage([StorageDeviceType.NVME, - StorageDeviceType.SSD], - common=True, - count_per_dev=1) - self.assertTrue(len(df[df.tran == 'nvme']) == 3) - self.assertTrue(len(df[df.tran == 'sata']) == 3) - self.assertTrue(len(df) == 6) - - # Find common networks between hosts - df = rg.find_net_info(hosts) - self.assertTrue(len(df) == 9) - - # Find common tcp networks - df = rg.find_net_info(hosts, providers='tcp') - self.assertTrue(len(df) == 3) diff --git a/ci/lint.sh b/ci/lint.sh deleted file mode 100644 index 63ecafd32..000000000 --- a/ci/lint.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -HERMES_ROOT=$1 -ADAPTER=${HERMES_ROOT}/adapter - -cpplint --recursive \ ---exclude="${HERMES_ROOT}/src/config_server_default.h" \ ---exclude="${HERMES_ROOT}/src/config_client_default.h" \ ---exclude="${ADAPTER}/posix/posix_api.h" \ ---exclude="${ADAPTER}/stdio/stdio_api.h" \ ---exclude="${ADAPTER}/mpiio/mpiio_api.h" \ -"${HERMES_ROOT}/adapter" "${HERMES_ROOT}/benchmarks" "${HERMES_ROOT}/data_stager" \ -"${HERMES_ROOT}/src" "${HERMES_ROOT}/test" diff --git a/ci/test_hermes.sh b/ci/test_hermes.sh deleted file mode 100644 index e31c7714e..000000000 --- a/ci/test_hermes.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Run main unit tests -pushd build -ctest -VV -R test_hermes_posix_basic_small -popd build - -# Set proper flags for cmake to find Hermes -INSTALL_PREFIX="${HOME}/install" -export LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LIBRARY_PATH}" -export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}" -export LDFLAGS="-L${INSTALL_PREFIX}/lib:${LDFLAGS}" -export CFLAGS="-I${INSTALL_PREFIX}/include:${CFLAGS}" -export CPATH="${INSTALL_PREFIX}/include:${CPATH}" -export CMAKE_PREFIX_PATH="${INSTALL_PREFIX}:${CMAKE_PREFIX_PATH}" -export CXXFLAGS="-I${INSTALL_PREFIX}/include:${CXXFLAGS}" - -# Run make install unit test -cd ci/external -mkdir build -cd build -cmake ../ -make -j8 - diff --git a/code_generators/hermes_config.py b/code_generators/hermes_config.py deleted file mode 100644 index f6e770ef7..000000000 --- a/code_generators/hermes_config.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -USAGE: - cd code_generators/bin - python3 hermes_config.py - -OUTPUT: - ${HERMES}/src/config_client_default.h (if client) - ${HERMES}/src/config_server_default.h (if server) -""" - -from code_generators.hermes_config.generator import create_config -from code_generators.util.paths import HERMES_ROOT - -create_config( - path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", - var_name="kClientDefaultConfigStr", - config_path=f"{HERMES_ROOT}/src/config_client_default.h", - macro_name="CLIENT" -) - -create_config( - path=f"{HERMES_ROOT}/config/hermes_server_default.yaml", - var_name="kServerDefaultConfigStr", - config_path=f"{HERMES_ROOT}/src/config_server_default.h", - macro_name="SERVER" -) \ No newline at end of file diff --git a/code_generators/code_generators/hermes_config/generator.py b/codegen/codegen/labstor_config/generator.py similarity index 59% rename from code_generators/code_generators/hermes_config/generator.py rename to codegen/codegen/labstor_config/generator.py index a59951050..29b5f1e56 100644 --- a/code_generators/code_generators/hermes_config/generator.py +++ b/codegen/codegen/labstor_config/generator.py @@ -1,5 +1,12 @@ import sys, os +def print_macro(path, macro_name): + with open(path) as fp: + lines = fp.read().splitlines() + macro_def = f'#define {macro_name}\\\n' + macro_body = '\\\n'.join(lines) + print(f'{macro_def}{macro_body}') + def create_config(path, var_name, config_path, macro_name): with open(path) as fp: yaml_config_lines = fp.read().splitlines() @@ -15,10 +22,10 @@ def create_config(path, var_name, config_path, macro_name): # Create the configuration config_lines = [] - config_lines.append(f"#ifndef HERMES_SRC_CONFIG_{macro_name}_DEFAULT_H_") - config_lines.append(f"#define HERMES_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#ifndef LABSTOR_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#define LABSTOR_SRC_CONFIG_{macro_name}_DEFAULT_H_") config_lines += string_lines - config_lines.append(f"#endif // HERMES_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#endif // LABSTOR_SRC_CONFIG_{macro_name}_DEFAULT_H_") # Persist config = "\n".join(config_lines) diff --git a/code_generators/code_generators/util/conv.py b/codegen/codegen/util/conv.py similarity index 100% rename from code_generators/code_generators/util/conv.py rename to codegen/codegen/util/conv.py diff --git a/code_generators/code_generators/util/naming.py b/codegen/codegen/util/naming.py similarity index 100% rename from code_generators/code_generators/util/naming.py rename to codegen/codegen/util/naming.py diff --git a/code_generators/code_generators/util/paths.py b/codegen/codegen/util/paths.py similarity index 84% rename from code_generators/code_generators/util/paths.py rename to codegen/codegen/util/paths.py index 2052c3aab..ce4ce8fe9 100644 --- a/code_generators/code_generators/util/paths.py +++ b/codegen/codegen/util/paths.py @@ -1,11 +1,11 @@ import os,sys import pathlib -def GetHermesRoot(): +def GetLabstorRoot(): util_path = pathlib.Path(__file__).parent.resolve() code_generators_path = os.path.dirname(util_path) code_generators_path = os.path.dirname(code_generators_path) hermes_path = os.path.dirname(code_generators_path) return hermes_path -HERMES_ROOT=GetHermesRoot() \ No newline at end of file +LABSTOR_ROOT=GetLabstorRoot() \ No newline at end of file diff --git a/codegen/hermes_config b/codegen/hermes_config new file mode 100755 index 000000000..28a523f97 --- /dev/null +++ b/codegen/hermes_config @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +""" +USAGE: + cd code_generators/bin + python3 hermes_config.py + +OUTPUT: + ${HERMES}/src/config_client_default.h (if client) + ${HERMES}/src/config_server_default.h (if server) +""" + +from codegen.labstor_config.generator import create_config +from codegen.util.paths import LABSTOR_ROOT + +HERMES_ROOT = f'{LABSTOR_ROOT}/tasks/hermes' +create_config( + path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", + var_name="kHermesClientDefaultConfigStr", + config_path=f"{HERMES_ROOT}/include/hermes/config_client_default.h", + macro_name="CLIENT" +) + +create_config( + path=f"{HERMES_ROOT}/config/hermes_server_default.yaml", + var_name="kHermesServerDefaultConfigStr", + config_path=f"{HERMES_ROOT}/include/hermes/config_server_default.h", + macro_name="SERVER" +) diff --git a/codegen/labstor_config b/codegen/labstor_config new file mode 100755 index 000000000..7060e4c95 --- /dev/null +++ b/codegen/labstor_config @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +""" +USAGE: + cd codegen/bin + python3 labstor_config.py + +OUTPUT: + ${LABSTOR}/src/config_client_default.h (if client) + ${LABSTOR}/src/config_server_default.h (if server) +""" + +from codegen.labstor_config.generator import create_config +from codegen.util.paths import LABSTOR_ROOT + +create_config( + path=f"{LABSTOR_ROOT}/config/labstor_client_default.yaml", + var_name="kLabstorClientDefaultConfigStr", + config_path=f"{LABSTOR_ROOT}/include/labstor/config/config_client_default.h", + macro_name="CLIENT" +) + +create_config( + path=f"{LABSTOR_ROOT}/config/labstor_server_default.yaml", + var_name="kLabstorServerDefaultConfigStr", + config_path=f"{LABSTOR_ROOT}/include/labstor/config/config_server_default.h", + macro_name="SERVER" +) \ No newline at end of file diff --git a/codegen/make_macro b/codegen/make_macro new file mode 100755 index 000000000..d9d0c7a95 --- /dev/null +++ b/codegen/make_macro @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./make_macro [PATH] +""" + +import os +import sys +from codegen.labstor_config.generator import print_macro + +PATH = sys.argv[1] +MACRO_NAME = os.path.basename(PATH).upper().split('.')[0] +print_macro(PATH, MACRO_NAME) diff --git a/codegen/make_task b/codegen/make_task new file mode 100755 index 000000000..0330e2784 --- /dev/null +++ b/codegen/make_task @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./make_task [TASK_ROOT] +""" + +import os +import sys +from codegen.util.paths import LABSTOR_ROOT + +def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): + with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: + text = fp.read() + text = text.replace('TASK_NAME', TASK_NAME) + rel_path = rel_path.replace('TASK_NAME', TASK_NAME) + with open(f'{TASK_ROOT}/{rel_path}', 'w') as fp: + fp.write(text) + +TASK_ROOT = sys.argv[1] +TASK_NAME = os.path.basename(TASK_ROOT) +TASK_TEMPL_ROOT = f'{LABSTOR_ROOT}/tasks_required/TASK_NAME' + +os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) +os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/TASK_NAME.cc', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_lib_exec.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_tasks.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_methods.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_methods.yaml', TASK_NAME) diff --git a/code_generators/preamble.py b/codegen/preamble.py similarity index 98% rename from code_generators/preamble.py rename to codegen/preamble.py index b55ea20d1..46036cffa 100644 --- a/code_generators/preamble.py +++ b/codegen/preamble.py @@ -2,7 +2,7 @@ Prepends the preabmle to all files in the repo USAGE: - python3 scripts/preamble.py ${HERMES_ROOT} + python3 scripts/preamble.py ${LABSTOR_ROOT} """ import sys,os,re diff --git a/codegen/refresh_methods b/codegen/refresh_methods new file mode 100755 index 000000000..600e355d8 --- /dev/null +++ b/codegen/refresh_methods @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./referesh_methods [TASK_ROOT] +""" + +import yaml +import os +import sys +from codegen.util.paths import LABSTOR_ROOT + +TASK_ROOT = sys.argv[1] +TASK_NAME = os.path.basename(TASK_ROOT) +METHODS_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.h' +METHODS_YAML = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.yaml' +LIB_EXEC_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_lib_exec.h' +METHOD_MACRO = f'LABSTOR_{TASK_NAME.upper()}_METHODS_H_' +LIB_EXEC_MACRO = f'LABSTOR_{TASK_NAME.upper()}_LIB_EXEC_H_' + +with open(METHODS_YAML) as fp: + methods = yaml.load(fp, Loader=yaml.FullLoader) +if methods is None: + methods = {} +if 'kLast' in methods: + del methods['kLast'] +methods = sorted(methods.items(), key=lambda x: x[1]) +if TASK_NAME != 'labstor_admin': + methods.insert(0, ('kConstruct', -2)) + methods.insert(1, ('kDestruct', -1)) + +# Produce the TASK_NAME_methods.h file +lines = [] +lines += [f'#ifndef {METHOD_MACRO}', + f'#define {METHOD_MACRO}', + '', + '/** The set of methods in the admin task */', + 'struct Method : public TaskMethod {'] +for method_enum_name, method_off in methods: + if method_enum_name == 'kConstruct': + continue + if method_enum_name == 'kDestruct': + continue + lines += f' TASK_METHOD_T {method_enum_name} = kLast + {method_off};', +lines += ['};', '', f'#endif // {METHOD_MACRO}'] +with open(METHODS_H, 'w') as fp: + fp.write('\n'.join(lines)) + + +# Produce the TASK_NAME_lib_exec.h file +lines = [] +lines += [f'#ifndef {LIB_EXEC_MACRO}', + f'#define {LIB_EXEC_MACRO}', + ''] +## Create the Run method +lines += ['/** Execute a task */', + 'void Run(u32 method, Task *task) override {', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' {method_name}(reinterpret_cast<{task_name} *>(task));', + f' break;', + f' }}'] +lines += [' }'] +lines += ['}'] + +## Create the ReplicateStart method +lines += ['/** Ensure there is space to store replicated outputs */', + 'void ReplicateStart(u32 method, u32 count, Task *task) override {', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' labstor::CALL_REPLICA_START(count, reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] +lines += [' }'] +lines += ['}'] + +## Create the ReplicateEnd method +lines += ['/** Determine success and handle failures */', + 'void ReplicateEnd(u32 method, Task *task) override {', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' labstor::CALL_REPLICA_END(reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] +lines += [' }'] +lines += ['}'] + +## Create the SaveStart Method +lines += ['/** Serialize a task when initially pushing into remote */', + 'std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override {', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar << *reinterpret_cast<{task_name}*>(task);', + f' break;', + f' }}'] +lines += [' }'] +lines += [' return ar.Get();'] +lines += ['}'] + +## Create the LoadStart Method +lines += ['/** Deserialize a task when popping from remote queue */', + 'TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override {', + ' TaskPointer task_ptr;', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask<{task_name}>(task_ptr.p_);', + f' ar >> *reinterpret_cast<{task_name}*>(task_ptr.task_);', + f' break;', + f' }}'] +lines += [' }'] +lines += [' return task_ptr;'] +lines += ['}'] + +## Create the SaveEnd Method +lines += ['/** Serialize a task when returning from remote queue */', + 'std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override {', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar << *reinterpret_cast<{task_name}*>(task);', + f' break;', + f' }}'] +lines += [' }'] +lines += [' return ar.Get();'] +lines += ['}'] + +## Create the LoadEnd Method +lines += ['/** Deserialize a task when returning from remote queue */', + 'void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override {', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar.Deserialize(replica, *reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] +lines += [' }'] +lines += ['}'] + +## Create the CheckIfConcurrent Method +lines += ['/** Get the grouping of the task */', + 'u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override {', + ' switch (method) {'] +for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' return reinterpret_cast<{task_name}*>(task)->GetGroup(group);', + f' }}'] +lines += [' }'] +lines += [' return -1;'] +lines += ['}'] + +## Finish the file +lines += ['', f'#endif // {METHOD_MACRO}'] + +## Write TASK_NAME_lib_exec.h +with open(LIB_EXEC_H, 'w') as fp: + fp.write('\n'.join(lines)) diff --git a/codegen/update_task b/codegen/update_task new file mode 100755 index 000000000..ffcdd5015 --- /dev/null +++ b/codegen/update_task @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./make_task [TASK_ROOT] +""" + +import os +import sys +from codegen.util.paths import LABSTOR_ROOT + +def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): + with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: + text = fp.read() + text = text.replace('TASK_NAME', TASK_NAME) + rel_path = rel_path.replace('TASK_NAME', TASK_NAME) + with open(f'{TASK_ROOT}/{rel_path}', 'w') as fp: + fp.write(text) + +TASK_ROOT = sys.argv[1] +TASK_NAME = os.path.basename(TASK_ROOT) +TASK_TEMPL_ROOT = f'{LABSTOR_ROOT}/tasks_required/TASK_NAME + +os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) +os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/TASK_NAME.cc', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME.h', TASK_NAME) diff --git a/config/labstor_client_default.yaml b/config/labstor_client_default.yaml new file mode 100644 index 000000000..0cb21544a --- /dev/null +++ b/config/labstor_client_default.yaml @@ -0,0 +1 @@ +thread_model: kStd \ No newline at end of file diff --git a/config/labstor_server_default.yaml b/config/labstor_server_default.yaml new file mode 100644 index 000000000..d0fc09001 --- /dev/null +++ b/config/labstor_server_default.yaml @@ -0,0 +1,54 @@ +### Runtime orchestration settings +work_orchestrator: + # The number of worker threads to spawn + max_workers: 4 + +### Queue Manager settings +queue_manager: + # The default depth of allocated queues + queue_depth: 256 + # The maximum number of lanes per queue + max_lanes: 16 + # The maximum number of queues + max_queues: 1024 + # The shared memory allocator to use + shm_allocator: kScalablePageAllocator + # The name of the shared memory region to create + shm_name: "labstor_shm" + # The size of the shared memory region to allocate + shm_size: 0g + +### Define properties of RPCs +rpc: + # A path to a file containing a list of server names, 1 per line. If your + # servers are named according to a pattern (e.g., server-1, server-2, etc.), + # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this + # option is not empty, it will override anything in `rpc_server_base_name`. + host_file: "" + + # Host names can be defined using the following syntax: + # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... + # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... + host_names: ["localhost"] + + # The RPC protocol. This must come from the documentation of the specific RPC + # library in use. + protocol: "ofi+sockets" + + # RPC domain name for verbs transport. Blank for tcp. + domain: "" + + # Desired RPC port number. + port: 8080 + + # The number of handler threads for each RPC server. + num_threads: 4 + +### Task Registry +task_registry: [ + 'hermes_mdm', + 'hermes_blob_mdm', + 'hermes_bucket_mdm', + 'posix_bdev', + 'ram_bdev' +] \ No newline at end of file diff --git a/data_stager/CMakeLists.txt b/data_stager/CMakeLists.txt deleted file mode 100644 index 3fc1b9008..000000000 --- a/data_stager/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -#----------------------------------------------------------------------------- -# Build Stage In -#----------------------------------------------------------------------------- -include_directories( - ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/adapter - ${CMAKE_CURRENT_SOURCE_DIR}) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_RPC_THALLIUM") - -add_library(hermes_data_stager SHARED - ${CMAKE_CURRENT_SOURCE_DIR}/stagers/posix_stager.cc) -add_dependencies(hermes_data_stager hermes_posix_io_client) -target_link_libraries(hermes_data_stager hermes_posix_io_client) - -add_executable(stage_in ${CMAKE_CURRENT_SOURCE_DIR}/stage_in.cc) -add_dependencies(stage_in hermes_data_stager) -target_link_libraries(stage_in hermes_data_stager) - -add_executable(stage_out ${CMAKE_CURRENT_SOURCE_DIR}/stage_out.cc) -add_dependencies(stage_out hermes_data_stager) -target_link_libraries(stage_out hermes_data_stager) - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DESTINATION ${HERMES_INSTALL_INCLUDE_DIR} - FILES_MATCHING PATTERN "*.h") - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - hermes_data_stager - stage_in - stage_out - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) \ No newline at end of file diff --git a/data_stager/data_stager.h b/data_stager/data_stager.h deleted file mode 100644 index 3c9f09622..000000000 --- a/data_stager/data_stager.h +++ /dev/null @@ -1,74 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_DATA_STAGER_STAGE_IN_H_ -#define HERMES_DATA_STAGER_STAGE_IN_H_ - -#include -#include - -#include "hermes_types.h" -#include "posix/posix_io_client.h" - -namespace hermes { - -enum class DataStagerType { - kPosix, - kHdf5 -}; - -class DataStagerTypeConv { - public: - static DataStagerType from_url(const std::string &url) { - if (url.rfind("h5::", 0) != std::string::npos) { - return DataStagerType::kHdf5; - } else { - return DataStagerType::kPosix; - } - } -}; - -class DataStager { - public: - virtual void StageIn(std::string url, hapi::PlacementPolicy dpe) = 0; - virtual void StageIn(std::string url, off_t off, size_t size, - hapi::PlacementPolicy dpe) = 0; - virtual void StageOut(std::string url) = 0; - - protected: - void DivideRange(off_t off, size_t size, off_t &new_off, size_t &new_size) { - int nprocs = 1; - int rank = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - int ranks_for_io = nprocs; - - // Ensure that all ranks perform at least 32MB of I/O - size_t min_io_per_rank = MEGABYTES(32); - if (size < nprocs * min_io_per_rank) { - ranks_for_io = size / min_io_per_rank; - if (size % min_io_per_rank) { - ranks_for_io += 1; - } - } - - new_size = size / ranks_for_io; - new_off = off + new_size * rank; - if (rank == ranks_for_io - 1) { - new_size += size % ranks_for_io; - } - } -}; - -} // namespace hermes - -#endif // HERMES_DATA_STAGER_STAGE_IN_H_ diff --git a/data_stager/data_stager_factory.h b/data_stager/data_stager_factory.h deleted file mode 100644 index 371942869..000000000 --- a/data_stager/data_stager_factory.h +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_DATA_STAGER_STAGE_FACTORY_H_ -#define HERMES_DATA_STAGER_STAGE_FACTORY_H_ - -#include "data_stager.h" -#include "stagers/posix_stager.h" - -namespace hermes { - -class DataStagerFactory { - public: - static std::unique_ptr Get(const std::string& url) { - return Get(DataStagerTypeConv::from_url(url)); - } - - static std::unique_ptr Get(DataStagerType stager) { - switch (stager) { - case DataStagerType::kPosix: { - return std::make_unique(); - } - default: { - return nullptr; - } - } - } -}; - -} // namespace hermes - -#endif // HERMES_DATA_STAGER_STAGE_FACTORY_H_ diff --git a/data_stager/stage_in.cc b/data_stager/stage_in.cc deleted file mode 100644 index 75a321d3d..000000000 --- a/data_stager/stage_in.cc +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include "hermes_types.h" -#include -#include "data_stager_factory.h" -#include - -using hermes::api::PlacementPolicyConv; -using hermes::api::PlacementPolicy; -using hermes::DataStager; -using hermes::DataStagerFactory; - -int main(int argc, char **argv) { - if (argc != 5) { - std::cout << "Usage: mpirun -n [nprocs]" << - " ./stage_in [url] [offset] [size] [dpe]" << std::endl; - exit(1); - } - MPI_Init(&argc, &argv); - HERMES->Create(hermes::HermesType::kClient); - std::string url = argv[1]; - off_t off = (off_t) hshm::ConfigParse::ParseSize(argv[2]); - size_t size = hshm::ConfigParse::ParseSize(argv[3]); - PlacementPolicy dpe = PlacementPolicyConv::to_enum(argv[4]); - auto stager = DataStagerFactory::Get(url); - if (size == 0) { - HILOG(kInfo, "Full stage-in") - stager->StageIn(url, dpe); - } else { - HILOG(kInfo, "Partial stage-in") - stager->StageIn(url, off, size, dpe); - } - HERMES->Finalize(); - MPI_Finalize(); -} diff --git a/data_stager/stage_out.cc b/data_stager/stage_out.cc deleted file mode 100644 index b99db66e3..000000000 --- a/data_stager/stage_out.cc +++ /dev/null @@ -1,33 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include "posix/posix_io_client.h" -#include "hermes_types.h" -#include -#include "data_stager_factory.h" - -using hermes::api::PlacementPolicyConv; -using hermes::api::PlacementPolicy; -using hermes::DataStager; -using hermes::DataStagerFactory; - -int main(int argc, char **argv) { - if (argc != 1) { - std::cout << "Usage: ./stage_out" << std::endl; - exit(1); - } - HERMES->Create(hermes::HermesType::kClient); - HERMES->Flush(); - HERMES->Finalize(); - MPI_Finalize(); -} diff --git a/data_stager/stagers/posix_stager.cc b/data_stager/stagers/posix_stager.cc deleted file mode 100644 index d1d1254f5..000000000 --- a/data_stager/stagers/posix_stager.cc +++ /dev/null @@ -1,106 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "posix_stager.h" - - -using hermes::api::PlacementPolicyConv; -using hermes::api::PlacementPolicy; -using hermes::adapter::fs::AdapterStat; -using hermes::adapter::fs::IoStatus; -using hermes::adapter::fs::File; -using hermes::adapter::fs::FsIoOptions; -namespace stdfs = std::filesystem; - -namespace hermes { - -void PosixStager::StageIn(std::string path, PlacementPolicy dpe) { - if (stdfs::is_directory(path)) { - DirectoryStageIn(path, dpe); - } else if (stdfs::is_regular_file(path)) { - FileStageIn(path, dpe); - } else { - HELOG(kError, "{} is neither directory or file", path); - } -} - -void PosixStager::FileStageIn(std::string path, PlacementPolicy dpe) { - off_t per_proc_off; - size_t per_proc_size; - DivideRange(0, stdfs::file_size(path), per_proc_off, per_proc_size); - FileStageIn(path, per_proc_off, per_proc_size, dpe); -} - -void PosixStager::DirectoryStageIn(std::string path, PlacementPolicy dpe) { - HILOG(kInfo, "Staging in the directory {}", path) - for (auto &file_path : stdfs::directory_iterator(path)) { - FileStageIn(file_path.path(), dpe); - } -} - -void PosixStager::StageIn(std::string path, off_t off, - size_t size, PlacementPolicy dpe) { - if (stdfs::is_directory(path)) { - HELOG(kError, "Posix stage-in with offset is " - "not supported for directories") - } else if (stdfs::is_regular_file(path)) { - FileStageIn(path, off, size, dpe); - } else { - HELOG(kError, "{} is neither directory or file", path); - } -} - -void PosixStager::FileStageIn(std::string path, - off_t off, size_t size, PlacementPolicy dpe) { - auto fs_api = HERMES_POSIX_FS; - std::vector buf(size); - AdapterStat stat; - bool stat_exists; - IoStatus io_status; - HILOG(kInfo, "Staging in {}", path) - File f = fs_api->Open(stat, path); - fs_api->Read(f, stat, buf.data(), off, size, - io_status, FsIoOptions::WithDpe(dpe)); - fs_api->Close(f, stat_exists); -} - -void PosixStager::StageOut(std::string path) { - if (stdfs::is_regular_file(path)) { - FileStageOut(path); - } else if (stdfs::is_directory(path)) { - DirectoryStageOut(path); - } else { - HILOG(kDebug, "Posix stage-out is neither a file or directory") - } -} - -void PosixStager::FileStageOut(std::string path) { - HILOG(kInfo, "Staging out the file {}", path) - auto fs_api = HERMES_POSIX_FS; - AdapterStat stat; - bool stat_exists; - File f = fs_api->Open(stat, path); - if (!f.status_) { - HELOG(kError, "Couldn't open file: {}", path) - return; - } - fs_api->Close(f, stat_exists); -} - -void PosixStager::DirectoryStageOut(std::string path) { - HILOG(kInfo, "Staging out the directory {}", path) - for (auto &file_path : stdfs::directory_iterator(path)) { - FileStageOut(file_path.path()); - } -} - -} // namespace hermes diff --git a/data_stager/stagers/posix_stager.h b/data_stager/stagers/posix_stager.h deleted file mode 100644 index 234ac8be1..000000000 --- a/data_stager/stagers/posix_stager.h +++ /dev/null @@ -1,40 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_DATA_STAGER_STAGERS_POSIX_STAGE_H -#define HERMES_DATA_STAGER_STAGERS_POSIX_STAGE_H - -#include "../data_stager.h" -#include "posix/posix_fs_api.h" -#include "posix/posix_api.h" - -namespace hermes { - -class PosixStager : public DataStager { - public: - void StageIn(std::string url, hapi::PlacementPolicy dpe) override; - void FileStageIn(std::string path, hapi::PlacementPolicy dpe); - void DirectoryStageIn(std::string path, hapi::PlacementPolicy dpe); - - void StageIn(std::string url, off_t off, size_t size, - hapi::PlacementPolicy dpe) override; - void FileStageIn(std::string path, off_t off, size_t size, - hapi::PlacementPolicy dpe); - - void StageOut(std::string url) override; - void FileStageOut(std::string path); - void DirectoryStageOut(std::string path); -}; - -} // namespace hermes - -#endif // HERMES_DATA_STAGER_STAGERS_POSIX_STAGE_H diff --git a/data_stager/test/CMakeLists.txt b/data_stager/test/CMakeLists.txt deleted file mode 100644 index 20b66297c..000000000 --- a/data_stager/test/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -pytest(data_stager test_data_stager_posix_large_aligned) -pytest(data_stager test_data_stager_posix_large_unaligned) -pytest(data_stager test_data_stager_posix_small_misaligned) -pytest(data_stager test_data_stager_directory) \ No newline at end of file diff --git a/data_stager/test/tests.py b/data_stager/test/tests.py deleted file mode 100644 index 834484f26..000000000 --- a/data_stager/test/tests.py +++ /dev/null @@ -1,78 +0,0 @@ -from py_hermes_ci.test_manager import TestManager -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo -from jarvis_util.util.size_conv import SizeConv -import os - - -class DataStagerTestManager(TestManager): - def spawn_all_nodes(self): - return self.spawn_info() - - def set_paths(self): - self.TEST_STAGE_IN_CMD = f"{self.CMAKE_BINARY_DIR}/bin/stage_in" - - def test_data_stager_posix_large_aligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_large_aligned.bin' - self._make_file(path, '128m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} 0 128m kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_posix_large_unaligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_large_unaligned.bin' - self._make_file(path, '128m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} .5m 127.5m " - f"kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_posix_small_aligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_small_aligned.bin' - self._make_file(path, '64m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} 0 64m kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_posix_small_misaligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_small_misaligned.bin' - self._make_file(path, '64m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} .5m 63.5m kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_directory(self): - os.makedirs('/tmp/test_hermes/a', exist_ok=True) - path = '/tmp/test_hermes/a/test_data_stager_posix_small_misaligned.bin' - self._make_file(path + "0", '64m') - self._make_file(path + "1", '128m') - self._make_file(path + "2", '4k') - self._make_file(path + "3", '16k') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} 0 0 kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def _make_file(self, path, size): - size = SizeConv.to_int(size) - with open(path, 'w') as fp: - fp.write("\0" * size) \ No newline at end of file diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in deleted file mode 100644 index 0f405154e..000000000 --- a/doc/Doxyfile.in +++ /dev/null @@ -1,2685 +0,0 @@ -# Doxyfile 1.9.5 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). -# -# Note: -# -# Use doxygen to compare the used configuration file with the template -# configuration file: -# doxygen -x [configFile] -# Use doxygen to compare the used configuration file with the template -# configuration file without replacing the environment variables or CMake type -# replacement variables: -# doxygen -x_noenv [configFile] - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the configuration -# file that follow. The default is UTF-8 which is also the encoding used for all -# text before the first occurrence of this tag. Doxygen uses libiconv (or the -# iconv built into libc) for the transcoding. See -# https://www.gnu.org/software/libiconv/ for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = Hermes - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = 1.0.0-beta - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = "Hierarchical Distributed I/O Buffering System" - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 -# sub-directories (in 2 levels) under the output directory of each output format -# and will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to -# control the number of sub-directories. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# Controls the number of sub-directories that will be created when -# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every -# level increment doubles the number of directories, resulting in 4096 -# directories at level 8 which is the default and also the maximum value. The -# sub-directories are organized in 2 levels, the first level always has a fixed -# numer of 16 directories. -# Minimum value: 0, maximum value: 8, default value: 8. -# This tag requires that the tag CREATE_SUBDIRS is set to YES. - -CREATE_SUBDIRS_LEVEL = 8 - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, -# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English -# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, -# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with -# English messages), Korean, Korean-en (Korean with English messages), Latvian, -# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, -# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, -# Swedish, Turkish, Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = NO - -# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line -# such as -# /*************** -# as being the beginning of a Javadoc-style comment "banner". If set to NO, the -# Javadoc-style will behave just like regular comments and it will not be -# interpreted by doxygen. -# The default value is: NO. - -JAVADOC_BANNER = NO - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = NO - -# By default Python docstrings are displayed as preformatted text and doxygen's -# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the -# doxygen's special commands can be used and the contents of the docstring -# documentation blocks is shown as doxygen documentation. -# The default value is: YES. - -PYTHON_DOCSTRING = YES - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:^^" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". Note that you cannot put \n's in the value part of an alias -# to insert newlines (in the resulting output). You can put ^^ in the value part -# of an alias to insert a newline as if a physical newline was in the original -# file. When you need a literal { or } or , in the value part of an alias you -# have to escape them by means of a backslash (\), this can lead to conflicts -# with the commands \{ and \} for these it is advised to use the version @{ and -# @} or use a double escape (\\{ and \\}) - -ALIASES = "status=A Status object." \ - "ctx{1}=ctx The Context for this \1." \ - "bool{1}=true if \1, otherwise false." - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice -# sources only. Doxygen will then generate output that is more tailored for that -# language. For instance, namespaces will be presented as modules, types will be -# separated into more groups, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_SLICE = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, -# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: -# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser -# tries to guess whether the code is fixed or free formatted code, this is the -# default for Fortran type files). For instance to make doxygen treat .inc files -# as Fortran files (default is PHP), and .f files as C (default is Fortran), -# use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. When specifying no_extension you should add -# * to the FILE_PATTERNS. -# -# Note see also the list of default file extension mappings. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See https://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 5. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 5 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = NO - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use -# during processing. When set to 0 doxygen will based this on the number of -# cores available in the system. You can set it explicitly to a value larger -# than 0 to get more control over the balance between CPU load and processing -# speed. At this moment only the input processing can be done using multiple -# threads. Since this is still an experimental feature the default is set to 1, -# which effectively disables parallel processing. Please report any issues you -# encounter. Generating dot graphs in parallel is controlled by the -# DOT_NUM_THREADS setting. -# Minimum value: 0, maximum value: 32, default value: 1. - -NUM_PROC_THREADS = 1 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual -# methods of a class will be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIV_VIRTUAL = NO - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = NO - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = NO - -# If this flag is set to YES, the name of an unnamed parameter in a declaration -# will be determined by the corresponding definition. By default unnamed -# parameters remain unnamed in the output. -# The default value is: YES. - -RESOLVE_UNNAMED_PARAMS = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# declarations. If set to NO, these declarations will be included in the -# documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = NO - -# With the correct setting of option CASE_SENSE_NAMES doxygen will better be -# able to match the capabilities of the underlying filesystem. In case the -# filesystem is case sensitive (i.e. it supports files in the same directory -# whose names only differ in casing), the option must be set to YES to properly -# deal with such files in case they appear in the input. For filesystems that -# are not case sensitive the option should be set to NO to properly deal with -# output files written for symbols that only differ in casing, such as for two -# classes, one named CLASS and the other named Class, and to also support -# references to files without having to specify the exact matching casing. On -# Windows (including Cygwin) and MacOS, users should typically set this option -# to NO, whereas on Linux or other Unix flavors it should typically be set to -# YES. -# Possible values are: SYSTEM, NO and YES. -# The default value is: SYSTEM. - -CASE_SENSE_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class -# will show which file needs to be included to use the class. -# The default value is: YES. - -SHOW_HEADERFILE = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = NO - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = NO - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. See also section "Changing the -# layout of pages" for information. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = YES - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as documenting some parameters in -# a documented function twice, or documenting parameters that don't exist or -# using markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete -# function parameter documentation. If set to NO, doxygen will accept that some -# parameters have no documentation without warning. -# The default value is: YES. - -WARN_IF_INCOMPLETE_DOC = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong parameter -# documentation, but not about the absence of documentation. If EXTRACT_ALL is -# set to YES then this flag will automatically be disabled. See also -# WARN_IF_INCOMPLETE_DOC -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS -# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but -# at the end of the doxygen process doxygen will return with a non-zero status. -# Possible values are: NO, YES and FAIL_ON_WARNINGS. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# See also: WARN_LINE_FORMAT -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# In the $text part of the WARN_FORMAT command it is possible that a reference -# to a more specific place is given. To make it easier to jump to this place -# (outside of doxygen) the user can define a custom "cut" / "paste" string. -# Example: -# WARN_LINE_FORMAT = "'vi $file +$line'" -# See also: WARN_FORMAT -# The default value is: at line $line of file $file. - -WARN_LINE_FORMAT = "at line $line of file $file" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). In case the file specified cannot be opened for writing the -# warning and error messages are written to standard error. When as file - is -# specified the warning and error messages are written to standard output -# (stdout). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = @PROJECT_SOURCE_DIR@/adapter \ - @PROJECT_SOURCE_DIR@/gotcha_intercept \ - @PROJECT_SOURCE_DIR@/src - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: -# https://www.gnu.org/software/libiconv/) for the list of possible encodings. -# See also: INPUT_FILE_ENCODING -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify -# character encoding on a per file pattern basis. Doxygen will compare the file -# name with each pattern and apply the encoding instead of the default -# INPUT_ENCODING) if there is a match. The character encodings are a list of the -# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding -# "INPUT_ENCODING" for further information on supported encodings. - -INPUT_FILE_ENCODING = - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# Note the list of default checked file patterns might differ from the list of -# default file extension mappings. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, -# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C -# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, -# *.vhdl, *.ucf, *.qsf and *.ice. - -FILE_PATTERNS = *.h \ - *.cc - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = @PROJECT_SOURCE_DIR@/src/stb_ds.h - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# ANamespace::AClass, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that doxygen will use the data processed and written to standard output -# for further processing, therefore nothing else, like debug statements or used -# commands (so in case of a Windows batch file always use @echo OFF), should be -# written to standard output. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = front_page.md - -# The Fortran standard specifies that for fixed formatted Fortran code all -# characters from position 72 are to be considered as comment. A common -# extension is to allow longer lines before the automatic comment starts. The -# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can -# be processed before the automatic comment starts. -# Minimum value: 7, maximum value: 10000, default value: 72. - -FORTRAN_COMMENT_AFTER = 72 - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# entity all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see https://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot of -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output -# should be rendered with a dark or light theme. Default setting AUTO_LIGHT -# enables light output unless the user preference is dark output. Other options -# are DARK to always use dark mode, LIGHT to always use light mode, AUTO_DARK to -# default to dark mode unless the user prefers light mode, and TOGGLE to let the -# user toggle between dark and light mode via a button. -# Possible values are: LIGHT Always generate light output., DARK Always generate -# dark output., AUTO_LIGHT Automatically set the mode according to the user -# preference, use light mode if no preference is set (the default)., AUTO_DARK -# Automatically set the mode according to the user preference, use dark mode if -# no preference is set. and TOGGLE Allow to user to switch between light and -# dark mode via a button.. -# The default value is: AUTO_LIGHT. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE = AUTO_LIGHT - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a color-wheel, see -# https://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use gray-scales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - -# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML -# documentation will contain a main index with vertical navigation menus that -# are dynamically created via JavaScript. If disabled, the navigation index will -# consists of multiple levels of tabs that are statically embedded in every HTML -# page. Disable this option to support browsers that do not have JavaScript, -# like the Qt help browser. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_MENUS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = NO - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 100 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: -# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To -# create a documentation set, doxygen will generate a Makefile in the HTML -# output directory. Running make will produce the docset in that directory and -# running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy -# genXcode/_index.html for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag determines the URL of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDURL = - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# on Windows. In the beginning of 2021 Microsoft took the original page, with -# a.o. the download links, offline the HTML help workshop was already many years -# in maintenance mode). You can download the HTML help workshop from the web -# archives at Installation executable (see: -# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo -# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the main .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location (absolute path -# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to -# run qhelpgenerator on the generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine tune the look of the index (see "Fine-tuning the output"). As an -# example, the default style sheet generated by doxygen has an example that -# shows how to put an image at the root of the tree instead of the PROJECT_NAME. -# Since the tree basically has the same information as the tab index, you could -# consider setting DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = YES - -# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the -# FULL_SIDEBAR option determines if the side bar is limited to only the treeview -# area (value NO) or if it should extend to the full height of the window (value -# YES). Setting this to YES gives a layout similar to -# https://docs.readthedocs.io with more room for contents, but less room for the -# project logo, title, and description. If either GENERATE_TREEVIEW or -# DISABLE_INDEX is set to NO, this option has no effect. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FULL_SIDEBAR = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email -# addresses. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -OBFUSCATE_EMAILS = YES - -# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg -# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see -# https://inkscape.org) to generate formulas as SVG images instead of PNGs for -# the HTML output. These images will generally look nicer at scaled resolutions. -# Possible values are: png (the default) and svg (looks nicer but requires the -# pdf2svg or inkscape tool). -# The default value is: png. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FORMULA_FORMAT = png - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands -# to create new LaTeX commands to be used in formulas as building blocks. See -# the section "Including formulas" for details. - -FORMULA_MACROFILE = - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# https://www.mathjax.org) which uses client side JavaScript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. -# Note that the different versions of MathJax have different requirements with -# regards to the different settings, so it is possible that also other MathJax -# settings have to be changed when switching between the different MathJax -# versions. -# Possible values are: MathJax_2 and MathJax_3. -# The default value is: MathJax_2. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_VERSION = MathJax_2 - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. For more details about the output format see MathJax -# version 2 (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 -# (see: -# http://docs.mathjax.org/en/latest/web/components/output.html). -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility. This is the name for Mathjax version 2, for MathJax version 3 -# this will be translated into chtml), NativeMML (i.e. MathML. Only supported -# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This -# is the name for Mathjax version 3, for MathJax version 2 this will be -# translated into HTML-CSS) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. The default value is: -# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 -# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/ - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# for MathJax version 2 (see -# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# For example for MathJax version 3 (see -# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): -# MATHJAX_EXTENSIONS = ams -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /Node, -# Edge and Graph Attributes specification You need to make sure dot is able -# to find the font, which can be done by putting it in a standard location or by -# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the -# directory containing the font. Default graphviz fontsize is 14. -# The default value is: fontname=Helvetica,fontsize=10. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" - -# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can -# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about -# arrows shapes. -# The default value is: labelfontname=Helvetica,labelfontsize=10. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" - -# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes -# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification -# The default value is: shape=box,height=0.2,width=0.4. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" - -# You can set the path where dot can find font specified with fontname in -# DOT_COMMON_ATTR and others dot attributes. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_FONTPATH = - -# If the CLASS_GRAPH tag is set to YES (or GRAPH) then doxygen will generate a -# graph for each documented class showing the direct and indirect inheritance -# relations. In case HAVE_DOT is set as well dot will be used to draw the graph, -# otherwise the built-in generator will be used. If the CLASS_GRAPH tag is set -# to TEXT the direct and indirect inheritance relations will be shown as texts / -# links. -# Possible values are: NO, YES, TEXT and GRAPH. -# The default value is: YES. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a -# graph for each documented class showing the direct and indirect implementation -# dependencies (inheritance, containment, and class references variables) of the -# class with other documented classes. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. See also the chapter Grouping -# in the manual. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -UML_LOOK = NO - -# If the UML_LOOK tag is enabled, the fields and methods are shown inside the -# class node. If there are many fields or methods and many nodes the graph may -# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the -# number of items for each type to make the size more manageable. Set this to 0 -# for no limit. Note that the threshold may be exceeded by 50% before the limit -# is enforced. So when you set the threshold to 10, up to 15 fields may appear, -# but if the number exceeds 15, the total amount of fields shown is limited to -# 10. -# Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag UML_LOOK is set to YES. - -UML_LIMIT_NUM_FIELDS = 10 - -# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and -# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS -# tag is set to YES, doxygen will add type and arguments for attributes and -# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen -# will not generate fields with class member information in the UML graphs. The -# class diagrams will look similar to the default class diagrams but using UML -# notation for the relationships. -# Possible values are: NO, YES and NONE. -# The default value is: NO. -# This tag requires that the tag UML_LOOK is set to YES. - -DOT_UML_DETAILS = NO - -# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters -# to display on a single line. If the actual line length exceeds this threshold -# significantly it will wrapped across multiple lines. Some heuristics are apply -# to avoid ugly line breaks. -# Minimum value: 0, maximum value: 1000, default value: 17. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_WRAP_THRESHOLD = 17 - -# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and -# collaboration graphs will show the relations between templates and their -# instances. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -TEMPLATE_RELATIONS = NO - -# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to -# YES then doxygen will generate a graph for each documented file showing the -# direct and indirect include dependencies of the file with other documented -# files. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -INCLUDE_GRAPH = YES - -# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are -# set to YES then doxygen will generate a graph for each documented file showing -# the direct and indirect include dependencies of the file with other documented -# files. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH tag is set to YES then doxygen will generate a call -# dependency graph for every global function or class method. -# -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. Disabling a call graph can be -# accomplished by means of the command \hidecallgraph. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -CALL_GRAPH = YES - -# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller -# dependency graph for every global function or class method. -# -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable caller graphs for selected -# functions only using the \callergraph command. Disabling a caller graph can be -# accomplished by means of the command \hidecallergraph. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -CALLER_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical -# hierarchy of all classes instead of a textual one. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the -# dependencies a directory has on other directories in a graphical way. The -# dependency relations are determined by the #include relations between the -# files in the directories. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -DIRECTORY_GRAPH = YES - -# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels -# of child directories generated in directory dependency graphs by dot. -# Minimum value: 1, maximum value: 25, default value: 1. -# This tag requires that the tag DIRECTORY_GRAPH is set to YES. - -DIR_GRAPH_MAX_DEPTH = 1 - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. For an explanation of the image formats see the section -# output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). -# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order -# to make the SVG files visible in IE 9+ (other browsers do not have this -# requirement). -# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, -# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and -# png:gdiplus:gdiplus. -# The default value is: png. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_IMAGE_FORMAT = png - -# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to -# enable generation of interactive SVG images that allow zooming and panning. -# -# Note that this requires a modern browser other than Internet Explorer. Tested -# and working are Firefox, Chrome, Safari, and Opera. -# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make -# the SVG files visible. Older versions of IE do not have SVG support. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -INTERACTIVE_SVG = NO - -# The DOT_PATH tag can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_PATH = @DOXYGEN_DOT_PATH@ - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the \dotfile -# command). -# This tag requires that the tag HAVE_DOT is set to YES. - -DOTFILE_DIRS = - -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). - -MSCFILE_DIRS = - -# The DIAFILE_DIRS tag can be used to specify one or more directories that -# contain dia files that are included in the documentation (see the \diafile -# command). - -DIAFILE_DIRS = - -# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file or to the filename of jar file -# to be used. If left blank, it is assumed PlantUML is not used or called during -# a preprocessing step. Doxygen will generate a warning when it encounters a -# \startuml command in this case and will not generate output for the diagram. - -PLANTUML_JAR_PATH = - -# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a -# configuration file for plantuml. - -PLANTUML_CFG_FILE = - -# When using plantuml, the specified paths are searched for files specified by -# the !include statement in a plantuml block. - -PLANTUML_INCLUDE_PATH = - -# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes -# that will be shown in the graph. If the number of nodes in a graph becomes -# larger than this value, doxygen will truncate the graph, which is visualized -# by representing a node as a red box. Note that doxygen if the number of direct -# children of the root node in a graph is already larger than -# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that -# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. -# Minimum value: 0, maximum value: 10000, default value: 50. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_GRAPH_MAX_NODES = 100 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs -# generated by dot. A depth value of 3 means that only nodes reachable from the -# root by following a path via at most 3 edges will be shown. Nodes that lay -# further from the root node will be omitted. Note that setting this option to 1 -# or 2 may greatly reduce the computation time needed for large code bases. Also -# note that the size of a graph can be further restricted by -# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. -# Minimum value: 0, maximum value: 1000, default value: 0. -# This tag requires that the tag HAVE_DOT is set to YES. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) support -# this, this feature is disabled by default. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_MULTI_TARGETS = NO - -# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page -# explaining the meaning of the various boxes and arrows in the dot generated -# graphs. -# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal -# graphical representation for inheritance and collaboration diagrams is used. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate -# files that are used to generate the various graphs. -# -# Note: This setting is not only used for dot files but also for msc temporary -# files. -# The default value is: YES. - -DOT_CLEANUP = YES diff --git a/doc/design/hermes-design.org b/doc/design/hermes-design.org deleted file mode 100644 index 4db22eea0..000000000 --- a/doc/design/hermes-design.org +++ /dev/null @@ -1,816 +0,0 @@ -#+TITLE: Hermes Design -#+AUTHOR: The Hermes Project -#+DATE: [2019-08-28 Wed] -#+OPTIONS: num:4 toc:t - -For a Hermes overview see cite:KougkasEtAl2018. - -* Models - - See cite:Maier2009. - -** Purpose/objective - /(What the client wants)/ - - - Distributed deep memory and storage hierarchies - - In HPC and cloud architectures - - Require expert knowledge (currently) - - Independent layers - - Vertical /and/ horizontal I/O - - Accelerate I/O through - - Delay write - - Pre-read/cache - - Smart placement and transfer - - Baseline: PFS, ... - - Optimization problem - - Policies - - Enhance and extend /current/ software - - Transparent, no code changes, adapters (POSIX, MPI-IO), =LD_PRELOAD= - - Hermes API for new development - - Audience - - NSF -> science and research community - -*** Use cases - -**** HPC - -**** Cloud - -*** Differences between Hermes (buffer pool) and the UNIX buffer cache - - Hermes buffers have no direct storage counterparts such as disk blocks. - - Hermes buffers are typically used in buffer allocations (sequences of - buffers) and state is associated with such allocations rather than - individual buffers. - - Hermes maintains buffer integrity but has no integrity concept for - entities in storage clients. Features such as `file system integrity' - need to be implemented by the corresponding adapters. -*** TODO Why? - Ask why the project is being done in the first place. Think about what - completing the project would be like. What would the finished project look - like? - - -** Behavioral or functional - /(What the system does)/ - -*** Core functions - - These are /user-level/ functions. The only user-visible primitives are - buckets, vBuckets, and traits. They are finite resources, i.e., a creation - function might return an error, if system limits are exceeded. - - Buckets are collections of /named/ (and timestamped = immutable) blobs that - are internally represented by a sequence of buffers in the storage - hierarchy. - - #+CAPTION: Hermes Primitives label:fig:HERMES-primitves - #+ATTR_HTML: :width 50% - [[./img/primitives.png]] - - vBuckets are used to group buckets by capabilities or features expressed in - traits. Traits are represented as sets of key/value pairs. - - #+CAPTION: Hermes Core User Functions label:fig:HERMES-user-core - #+ATTR_HTML: :width 50% - [[./img/user-core.png]] - - Nomenclature used in examples: - - #+BEGIN_EXAMPLE - ctx - context - vbkt - vBucket handle - bkt - bucket handle - name - byte vector / unicode string - data - user data - trs - traits - key - a trait property name (unicode string) - val - a trait property value (byte vector) - status - error code - #+END_EXAMPLE - - 1. =vbkt <- acquire(name, trs, ctx)= - - purpose: :: acquire a vBucket with a set of traits - - input: :: name, set of vBucket traits, context - - output: :: vBucket handle - A trait could be represented as a set of key/value pairs. - 2. =status <- release(vbkt, ctx)= - - purpose: :: release a vBucket - - input: :: vBucket handle, context - - output: :: error code - The context controls the side-effects on buckets (if any). - 3. =status <- edit(vbkt, key, val, ctx)= - - purpose: :: update a vBucket's trait property - - input: :: vBucket handle, trait property name, trait property value, - context - - output: :: error code - Here we assume either that the trait property name is unambiguous or - that it's clear from the context which trait is referenced. - 4. =bkt <- acquire(vbkt, name, ctx)= - - purpose: :: acquire a bucket and link it to the vBucket referenced - - input: :: vBucket handle, name, context - - output: :: bucket handle - 5. =status <- release(bkt, ctx)= - - purpose: :: release a bucket and flush/release the underlying buffers - - input: :: bucket handle, context - - output: :: error code - The context controls the side-effects on buffers (if any). - 6. =status <- put(bkt, name, data, ctx)= - - purpose: :: write user data to a bucket (buffer) - - input: :: bucket handle, name, user data, context - - output: :: error code - 7. =data <- get(bkt, name, ctx)= - - purpose: :: read user data from a bucket (buffer) - - input: :: bucket handle, name, context - - output: :: user data - A =get= may trigger a "cache miss" in which case I/O clients, etc., will - get involved. - 8. =status <- link(vbkt, bkt, ctx)= - - purpose: :: associate a bucket with a vBucket - - input: :: vBucket handle, bucket handle, context - - output: :: error code - 9. =status <- unlink(vbkt, bkt, ctx)= - - *purpose:* :: dissociate a bucket from a vBucket - - *input:* :: vBucket handle, bucket handle, context - - *output:* :: error code - -*** Basic examples - - For files in a POSIX file system, the following parallels might be drawn: - - trait "=" directory-level metadata, file invariants - - vBucket "=" directory or file system - - bucket "=" (kernel) file table entry - - Let's look at the simplest POSIX use cases! - -**** UNIX =write= - - #+begin_src C -n - // FILE* fp = fopen("test.dat", "a+"); - hvbkt = hermes::acquire($HOME, trs, ctx); - hbkt = hermes::acquire(hvbkt, "test.dat", ctx); - int* data = ... ; - // fwrite(data, sizeof(int), count, fp); - hermes::put(hbkt, key(0, count), data, ctx); - // long offset = ... ; - // fseek(fp, offset, SEEK_SET); - // fwrite(data, sizeof(int), count, fp); - hermes::put(hbkt, key(offset, count), data, ctx); - // fclose(fp); - hermes::release(hbkt, ctx); - hermes::release(hvbkt, ctx); - #+end_src - - - line 1 :: What are the traits in =trs=? - - line 2 :: This could be a side effect of 3 by acquiring a bucket on the - "default vBucket." - - line 3 :: =ctx= contains the access mode, etc. - - line 6 :: The =key= function calculates the "name" of the stream to be - written. - - line 13 :: This might be unnecessary when the default vBucket is used or - the vBucket represents a file system. - -**** UNIX =read= - - #+begin_src C -n - // FILE* fp = fopen("test.dat", "r"); - hvbkt = hermes::acquire($HOME, trs, ctx); - hbkt = hermes::acquire(hvbkt, "test.dat", ctx); - // int* data = ... ; - // fread(data, sizeof(int), count, fp); - data = hermes::get(hbkt, key(0, count), ctx); - // long offset = ... ; - // fseek(fp, offset, SEEK_SET); - // fread(data, sizeof(int), count, fp); - data = hermes::get(hbkt, key(offset, count), ctx); - // fclose(fp); - hermes::release(hbkt, ctx); - hermes::release(hvbkt, ctx); - #+end_src - - - line 1 :: What are the traits in =trs=? - - line 2 :: This could be a side effect of 3 by acquiring a bucket on the - "default vBucket." - - line 3 :: =ctx= contains the access mode, etc. - - line 6 :: The =key= function calculates the "name" of the stream to be - written. - - line 13 :: This might be unnecessary when the default vBucket is used or - the vBucket represents a file system. - - -** Form - /(What the system is)/ - - - I/O middleware library - - =LD_PRELOAD=-able - - API adapters - - UNIX - - MPI-IO - - HDF5 - - HDF5 Virtual File Driver (VFD) - - HDF5 Virtual Object Layer (VOL) plugin - - Services - - Application coordinator - - Tools - - System profiler - - Schema parser - -*** Core components - - #+CAPTION: Hermes Components - #+ATTR_HTML: :width 50% - #+NAME: fig:HERMES - [[./img/hermes-stack.png]] - - - API Adapters :: Intercept standard API calls and "translate" into Hermes - calls. - - Hermes API :: Getting Hermes things done - - Data placement engine (DPE) :: Oracle - - Prefetcher (PRE) :: Trigger read ahead - - Metadata manager (MDM) :: Hold user- and Hermes-internal metadata - - Messaging service (MSG) :: =BCAST "Hello, World!"= - - Data organizer (ORG) :: Execute data placement decisions. Respond to - events. (e.g., no space left on device) - - Buffer pool manager (BPM) :: Buffer management - - I/O clients (IOC) :: Storage interfaces that do the actual I/O. - -**** Buffer pool manager - - - Manages a (finite) buffer pool $BP = \{b_i\}_{i\in I}$ - - The $b_i$ are pre-allocated (across the hierarchy) fixed-size buffers - - A /buffer pool allocation/ is a sequence of buffers - $(b_{i_1},\ldots,b_{i_n}), b_{i_k}\in BP, 1\leq k\leq n$ - - Certain operations are supported on buffer pool allocations - - /Buffer fission/ - the splitting of one or more buffers in an allocation - into smaller buffers - - /Buffer fusion/ - the joining of two or more adjacent buffers into a - larger buffer - - What else? - - What are the possible /states/ of a buffer allocation? - - Chemistry analogy, molecules, polymer chains, etc. - - What's the initial size/layer distribution? Ratio... - -*** Allocation of core functions to components - - The messaging service is present in all of these. - However, it doesn't really do anything specific to the - behavior of these functions. - - #+CAPTION: Core function allocation - #+TBLNAME: table-1 - | | *DPE* | *MDM* | *ORG* | *PRE* | *BUM* | *IOC* | - |-------------------+-----+-----+-----+-----+-----+-----| - | =create= (vBucket) | | [x] | | | | | - |-------------------+-----+-----+-----+-----+-----+-----| - | =destroy= (vBucket) | | [x] | [*] | | [*] | [*] | - |-------------------+-----+-----+-----+-----+-----+-----| - | =create= (bucket) | | [x] | | | | | - |-------------------+-----+-----+-----+-----+-----+-----| - | =destroy= (bucket) | | [x] | [*] | | [*] | [*] | - |-------------------+-----+-----+-----+-----+-----+-----| - | =put= | [x] | | [*] | | [x] | [x] | - |-------------------+-----+-----+-----+-----+-----+-----| - | =get= | [x] | | [*] | [x] | [*] | [x] | - |-------------------+-----+-----+-----+-----+-----+-----| - | =edit= | | [x] | [*] | [*] | | | - |-------------------+-----+-----+-----+-----+-----+-----| - | =link= | | [x] | | | | | - |-------------------+-----+-----+-----+-----+-----+-----| - | =unlink= | | [x] | | | | | - #+TBLFM: $1=create= vBucket - - #+BEGIN_EXAMPLE - [x] - All function use cases involve this component - [*] - Some function use cases involve this component - #+END_EXAMPLE - -**** TODO Review the allocation table. -**** TODO Break down the MDM. -**** TODO What is the synchronization cycle? - We must prove that the buffering is self-regulating and the application - never sees inconsistent data. How much fault tolerance do we need / can we - afford? - -*** Distribution - - #+CAPTION: Hermes Node - #+ATTR_HTML: :width 50% - #+NAME: fig:HERMES-node - [[./img/hermes-node.png]] - -*** Adapters - - #+CAPTION: Hermes Adapter - #+ATTR_HTML: :width 25% - #+NAME: fig:HERMES-adapter - [[./img/adapter.png]] - - 1. The Unix Universal I/O Adapter - - =open= - - =creat= - - =close= - - =read= - - =write= - - =lseek= - - =writev= - - =readv= - - =pread= - - =pwrite= - - 2. The MPI-IO Adapter - -**** TODO Look at the HDF5 source and compile a list of MPI-IO calls what we need - - #+CAPTION: Adaptor modules label:fig:adaptors - #+ATTR_HTML: :width 50% - [[./img/modules.png]] -*** Modules - -**** Hermes core API - - - Configuration - - Layers (tiers), buffer pools, buffers - - vBuckets (mappings) - - Instrumentation - - Messaging - - Data placement engine - - Data organizer - - Transfer content between buffers in /different/ buffer pools - - Sync variants - - Gather blocks to buffers - - Scatter buffers to blocks - - Asynchronous and ahead/delayed variants - -*** Configuration - - - XML - -** Performance objectives or requirements - /(How effectively the system does it)/ - - - Metrics - * (Difference to caching metrics) - * Layer occupancy - * Network traffic - * Latency - * Throughput - * Overflow handling - * Scalability - * Capacity - * Concurrency - * Resource contention - - Baseline (host FS or PFS) - - Fault tolerance - - Data consistency - - Thread safety - -*** Policies - - 1. Maximum application bandwidth - 2. Maximum data locality - 3. Hot-data - 4. User-defined -** Data - /(The information retained in the system and its interrelationships)/ - - - Layer configuration - - Buffers, buckets, vBuckets - - Profile and instrumentation data - - Policies - - Node status - - File properties - -*** Metadata Management - - - vBucket list - #+BEGIN_EXAMPLE - | vBucket ID | Name | List of Bucket Links | Trait(s) | RC | - |------------+------+----------------------+----------+----+ - #+END_EXAMPLE - - vBucket free list - - vBucket name to vBucket ID map (bi-directional?) Maybe just one-way: - =name -> ID= - - /Can a vBucket have more than one trait?/ (No, for now.) - - /Can traits be shared?/ (The vBucket is the trait.) - - - Bucket list - #+BEGIN_EXAMPLE - | Bucket ID | Name | Map of Buffer IDs | vBucket IDs | RC | - |-----------+------+-------------------+-------------+----+ - #+END_EXAMPLE - - Bucket free list - - Bucket name to bucket ID map (bi-directional) - - - Buffer list - #+BEGIN_EXAMPLE - | Buffer ID | Tier | Size | Status | Place | - |-----------+------+------+--------+-------| - #+END_EXAMPLE - - Buffer free list (by tier) - -** Managerial - /(The process by which the system is constructed and managed)/ - -*** Milestones - -**** TODO Create a Hermes library skeleton - - CMake, C++11 - - Draft of the Hermes low-level API - DEADLINE: <2019-04-30 Tue> -**** TODO Create a Hermes POSIX adapter skeleton - - CMake, C++11, C99 - - Draft of the Hermes high-level API - DEADLINE: <2019-05-31 Fri> - -*** Testing - - -* Detailed Design - -** POSIX Adapter - - The POSIX adapter assumes a /paged/ view of a logical file, a byte vector. - - *What's the relationship between the page size and the pre-defined buffer - sizes in buffer pools?* - - #+CAPTION: Hermes File to Object Mapping:fig:HERMES-file-2-object - #+ATTR_HTML: :width 50% - [[./img/file2object.png]] - - #+CAPTION: Hermes Balanced File Mapping:fig:HERMES-balanced-map - #+ATTR_HTML: :width 50% - [[./img/balanced-mapping.png]] - - - This adapter supports a subset of [[https://pubs.opengroup.org/onlinepubs/9699919799/][POSIX.1]] calls (see below) related to file - I/O. It also supports /poor-man's parallel I/O/ to multiple independent files - (MIF). - -*** Metadata structures - - We call them tables, but they are /maps/... - -**** File table (FT) - - The =File ID= is obtained from =fstat=. The first two fields of the returned - =struct stat= uniquely identify a file. - - #+begin_example - File ID := (st_dev, st_ino) - #+end_example - - #+begin_example - File ID -> (Bucket ID, Page Size, Reference Count) - #+end_example - - - Do we need to track EOA/EOF? - -**** File descriptor table (FDT) - - #+begin_example - File Desc. -> (File ID, Cursor, Append Mode) - #+end_example - -**** Working set (WS) - - The set of page keys currently present in file buckets. - - #+begin_example - File Page Key := (File ID, Page Key) - #+end_example - -**** Working set statistics (WSS) - - #+begin_example - File Page Key -> (read count, write count) - #+end_example - - *WARNING:* Generally, the key sets of WS and WSS will be different. The - former is a subset of the latter. - -*** Supported APIs - -**** =open= - - #+begin_src c - int open(const char *pathname, int flags); - int open(const char *pathname, int flags, mode_t mode); - #+end_src - - #+begin_example - - Open/create the file, obtain new file descriptor - - Compute File ID - - IF File ID in FT: - - Increment Reference Count - - ELSE: - - Call flock and place an exclusive lock on the file - - Acquire a Hermes bucket - - Create a new entry in FT - - Release the lock - - Add entry to FDT - - RETURN file descriptor - #+end_example - -**** =write= - - #+begin_src c - ssize_t write(int fd, const void *buf, size_t count); - #+end_src - - #+begin_example - - Compute File Page Key Range - - IF File Desc. IN Append Mode: - - Move Cursor to end of file - - FORALL File Page Key IN File Page Key Range: - - IF writing full page: - - Call hermes::put on file bucket w/ File Page Key and user buffer - - ELSE: - - IF File Page Key in WS: - - Call hermes::get to retrieve buffer associated with File Page Key - - ELSE: - - Call client_read to retrieve the data - - Merge w/ user buffer - - Call hermes::put - - UNLESS File Page Key in WSS: - - Add (File Page Key, (0,0)) to WSS - - Increment write count for File Page Key - - Advance Cursor for this File Desc. - - RETURN the number of bytes written - #+end_example - -**** =read= - - #+begin_src c - ssize_t read(int fd, void *buf, size_t count); - #+end_src - - #+begin_example - - Compute File Page Key Range - - FORALL File Page Key in Page Key Range: - - IF File Page Key in WS: - - hermes::get the data - - ELSE: - - Call client_read to retrieve the data - - hermes::put the data (optional?) - - IF reading full page: - - Copy data to user buffer - - ELSE: - - Merge data into user buffer - - UNLESS File Page Key in WSS: - - Add (File Page Key, (0,0)) to WSS - - Increment read count for File Page Key - - Advance Cursor for this File Desc. - - RETURN the number of bytes read - #+end_example - -**** =lseek= - - #+begin_src c - off_t lseek(int fd, off_t offset, int whence); - #+end_src - - #+begin_example - - Call lseek - - Update the Cursor for File Desc. fd - - RETURN the new offset - #+end_example - -**** =posix_fallocate= - - #+begin_src c - int posix_fallocate(int fd, off_t offset, off_t len); - #+end_src - -**** =close= - - #+begin_src c - int close(int fd); - #+end_src - - #+begin_example - - Call client_close - - Remove the corresponding entry from FDT - - Decrement the Reference Count - - IF Reference Count == 0: - - Call client_flock and place an exclusive lock on the file - - Release the Hermes bucket - - Remove the entry from FT - - Release the lock - #+end_example - -*** System calls in =H5FDsec2.c= - - - =open= - - =close= - - =read=, =pread= - - =write=, =pwrite= - - =lseek= - - =stat= - - =ftruncate= - - =flock= - - =fallocate= (?) - - -* Glossary - - We follow closely standard terminology for Unix-like systems with minor - adaptations. (cite:Bach1986, cite:Kerrisk2010.) - - - Block :: A configurable but fixed unit or granule of I/O. All I/O - operations occur in multiples of blocks and all offsets are - block-aligned. Let /B/ denote the configurable but fixed - /block size/ in bytes. (A typical value for /B/ is 4096 bytes.) - - Buffer :: A storage object which consists of metadata and data. - 1. A header that identifies the buffer, its size, status, etc. - 2. A byte stream, the buffer content. - - Buffer Pool :: During Hermes initialization, a set of buffers according to - a predefined size distribution is created across storage - layers. (See figure ref:fig:HERMES-buffer-pool.) - As I/O activity commences, the buffer pool divides - into "regions" such as the following: - - Unused or available buffers - - Buffers used a RAM cache - - Buffers mapped to (regions of) files in NVMe devices - - Buffers mapped to (regions of) files in burst buffers - - Etc. - - A PFS "swap file" - - Bucket :: A data structure that represents a collection of /named/ streams - (buffers). - - Trait :: A set of properties represented as - (unicode string) key/(byte array) value pairs. - - Virtual Bucket (vBucket) :: A data structure that represents a set of - traits common among a set of /named/ buckets. - - Visibility :: A vBucket can be /shared/ or /private/. The visibility - depends on control flow granularity, i.e., threads, MPI - processes, etc. - - #+CAPTION: Hermes Buffer Pool label:fig:HERMES-buffer-pool - #+ATTR_HTML: :width 50% - [[./img/hermes-bucket-pool.png]] - -** Tasks - -*** TODO Clarify how we identify buffers, buckets, and vBuckets - - Should we be able to infer the MPI rank, bucket size, and score from a - bucket's ID? -*** TODO Clarify the buffer pool initialization - - What's our thinking on a bucket size distribution? -*** TODO Clarify the vBucket management - - -* Miscellaneous - - -** Magic numbers - - | Size (Bytes) | Comment | - |--------------+---------------------------------------------| - | 256 | ext4 inode size | - | 512 | Default sector size of a Linux block device | - | 4096 | Default memory page size | - | 4096 | Default ext4 inode block size | - - -** Paged allocation and buffering in HDF5 - - For the technical details of HDF5 file space management, paged allocation, - aggregation, and buffering see cite:Chaarawi2015, cite:ChoiEtAl2017. - These features are designed to be used together and don't make much sense - on their own. - - - If an I/O request is equal or larger than a file's page size, the - page buffer calls directly into the VFD layer. - - SWMR writers are not supported because of the flush dependence. - SWMR readers are fine as long as on refresh they evict all pages - which have evicted metadata cache entries on them. - -*** Parallel issues - - - Metadata :: Writes are collective and this works if rank 0 does all - writing (=H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY=). - Distributed write strategies are currently not supported. - _Main reason_: a new distributed algorithm for distributing - entries according to pages would be necessary. The overhead - could be substantial and void potential savings or make - matters worse. Likewise, PB can be effective if metadata - reads are collective and process 0 does all the reading - and broadcasting. - - Raw data :: Currently unsupported. _Rationale_: Unlike metadata access, - raw data access is not "atomic" and can be done collectively - or independently. Independent access might benefit, but, - since it can be used simultaneously w/ collective I/O, this - might lead to data inconsistency. - -*** API - - Paged buffering is supported in the HDF5 library since version 1.10.1. The - page buffer sits right above the VFD layer and acts as a cache for small - (meta-)data I/O. With this feature enabled, I/O is always done in multiples - of pages. - - 1. Set the file space allocation strategy in the file creation property - list to =H5F_FSPACE_STRATEGY_PAGE=: - #+begin_src C - herr_t H5Pset_file_space_strategy - ( - hid_t fcpl, - H5F_fspace_strategy_t strategy, - hbool_t persist, - hsize_t threshold - ); - #+end_src - - =persist= controls whether free space should be persisted. - - =threshold= is the smallest free-space section size that the free space - manager will track. - 2. Specify the page size of file space allocation: - #+begin_src C - herr_t H5Pset_file_space_page_size (hid_t fcpl, hsize_t fsp_size); - #+end_src - The default page size is 4096 bytes. - 3. To enable page buffering set the maximum page buffer size on the file - access property list: - #+begin_src C - herr_t H5Pset_page_buffer_size - ( - hid_t fapl, - size_t buf_size, - unsigned min_meta_perc, - unsigned min_raw_perc - ); - #+end_src - - =buf_size= should be a multiple of the paged allocation size. - - =min_meta_perc= - the minimum metadata percentage to keep in the page - buffer before allowing pages containing metadata to be evicted. - - =min_raw_perc= - Minimum raw data percentage to keep in the page buffer - before allowing pages containing raw data to be evicted. - -**** TODO Compile a list of HDF5 features that we might want to disable - - -** Templates - -*** Diagrams - -**** Sequence - - #+CAPTION: A sequence diagram label:fig:sequence-diagram - #+ATTR_HTML: :width 50% - [[./img/sequence.png]] - -**** State - - #+CAPTION: A state machine diagram label:fig:state-machine-diagram - #+ATTR_HTML: :width 50% - [[./img/state-machine.png]] - -**** Quality - - #+CAPTION: The quality triangle label:fig:quality-triangle - #+ATTR_HTML: :width 40% - [[./img/quality.png]] - -*** Code outline template - - #+BEGIN_EXAMPLE - terminate:= false - initialize application/application protocols - ask for current state (image request) - Loop - Get_event - Case Event_Type is - -- "normal" (non-fault-tolerant-related) requests to - -- perform actions; only happens if this unit is the - -- current primary address space - when X => Process X - Send state data updates to other address spaces - when Y => Process Y - Send state data updates to other address spaces - ... - when Terminate_Directive => clean up resources; terminate:= true - when State_Data_Update => apply to state data - -- will only happen if this unit is a secondary address - -- space, receiving the update from the primary after it - -- has completed a "normal" action sending, receiving - -- state data - when Image_Request => send current state data to new address space - when State_Data_Image => Initialize state data - when Switch_Directive => notify service packages of change in rank - -- these are requests that come in after a PAS/SAS - -- switchover; they report services that they had - -- requested from the old (failed) PAS which this unit - -- (noe the PAS) must complete. A, B, etc. are the names - -- of the clients. - when Recon_from_A => reconstitute A - when Recom_from_B => reconstitute B - ... - when others => log error - end case - exit when terminate - end loop - #+END_EXAMPLE - - - bibliographystyle:unsrtnat - bibliography:references.bib diff --git a/doc/design/img/Hermes IC.graphml b/doc/design/img/Hermes IC.graphml deleted file mode 100644 index 88b35dc11..000000000 --- a/doc/design/img/Hermes IC.graphml +++ /dev/null @@ -1,245 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - Hermes IC - - - - - - - - - - - PUT - - - - - - - - - - - GET - - - - - - - - - - - ACQ - - - - - - - - - - - REL - - - - - - - - - - - MOD - - - - - - - - - - - SND - - - - - - - - - - - RCV - - - - - - - - - - - I/O Clients - - - - - - - - - - - Hermes API - - - - - - - - - - - Bucket - - - - - - - - - - - vBucket - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ACQuire -RELease -MODe -SeND (write) -ReCeiVe (read) - - - - - - - - - - - B(L)OB - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - (v)Bucket MODe - -0 - Bucket -1 - vBucket - - - - - - - - - diff --git a/doc/design/img/Hermes IC.png b/doc/design/img/Hermes IC.png deleted file mode 100644 index 20311c8c46e9c36b1ae854cafc891832f79d0afd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 42684 zcmb@uby$_()-}8V#Ud2xmhJ|T76j=|QIHPlZY){^HYq5b0#Z^+O2{TeknWNa5GiTC zx%K?cdCv2`?|^2_kFK5*PLUHG1d-KSG|jSnfx*WfxuOezoUsjU_OO^p%*dX zl}DFw7!ZiZ2!%VhwY`ki-gx>E>I{Eoj%Ro;plD5}DINQEhYnLS=b~nISRBb!LiKF< z>x6gi5MC{PEOxnHRP&A$q4(9RSEIS=e}BciG5TA~wOiQN)5F7qe;hgeJIc?aSKRf> zBx>WN=aox|IR;$tabolxEclBMJ8w=3e>TaZP2f+{BdK8Y+a(xe*zl(g`~QbGHzga+ zPLDKBv3CuQ;Zie>Qo&CwwikPzJb8j_yzQTx8bJfsI=L?J&x*(XFt+#Mtv9EHL!yl4#eWa zIsH7-{7hkFxnyJ}c3;mC10i*xgJ+-SRLA&GZygVT5RnRQKHJPHEDjjZK_Hwi!Z6Po z@lRcv2aOLeArNwpah?TkCbE4qe(Qli@L-}pYJA^yH$oK?@z|NR zWyaXA@#{_^k_v&q!6d^zn>$K;;NKVm4@n5)a5oq~It-|XORBe!m72k6<3&!AEuXUD{T<;zbaM8dw1R*@|If|0**CBAS@C zMXL2?gsL&uHw;9R#h?FiHy>J-AP}^XFifc9(3=K>EG&c+`4b$hYczG2e>{5t@ul?` z2MgD)cxR^v6K=I1&z#hH<`Idhf3YM2@sK#$B>hQ=PFjQR8@L+-7zSIyopUHp1ma^6 zo@#7g6=U^L&MgEY3FgXrUNq6nKaUx{C<0%M{5g=`pp|$;NG0llgNsWg?36z8cfq76 zXlm(x&`)j(?yZlLkdn&ZWi~M}A>lS6Ou;~$VwsbUw2^eGPqTLVhX>dwY8B8?gqI`+YdtD2_$W^MAA%5nr|Nb~p-EEYj`srQ%&F_`b= z*-4*)kCiosqt%Ya+S-KvA282f#v$ct3Z7&Z}(fbh$6<_+UHMe|L4z z_9IhVXupQ0LPK@LhZ{bySFfN4YAdWTQeMZXa-8Nem{*Vb{E|w^A)ZZ#fPy0-O7b)* z*>>_1-|8`$2h*4wOO@iE%RKpaA#n~C}547IG(MTDqb^A)B-G~Ixmc~LDP zFO|EYK9spjtCwUNfe3+5-VpOH8;Y&1d&!*B7mLJ4@W56e{S-^|mYLnrxXFw;Ts+|- z;vr=R56|<@?}mLiVt&rNq8p60xQS>IhA|0y2@6s=KKh^Qy_jv)N{yJ|`19iZ@?B-( z>DjT$M<++Wd*fJRKdO;zldVKYN56aUF;&=E+727xb;%R8nIRJ*_e6j{@YK(_*9X6M zpdhuN*8hp`?~O<*(S%`@%Q!e|BW3fbSY}z~ZQ2z^3?{N*Qmh%R8l~XrfFt`$*{{;m z(~Sd9bS7$zK4IzEi${MVfgn%5sNcTy^?1f8~1>c5OiE$0u+-(Hhy{ zC@N7o$)C&1%b%lYBJ7(;8*1?<{bz?Jf7M@MA3=EKK(?w*zFoRBnzOHE{mXKHF{VCyk4GKQ%dVVdGR_H>TG87|NY z3Fwz__l=pJO*#5nJnS$@J`v%z($C1l6MdR8T46;a>akYu`}+<)jnvioRB_*MY6Rl< z!v>t`I`7yET2MtW%=Wh=7M@lbRy)k)tO>~QPTsTiy%IpmDkQYGIoo2I67_37N-{Vk zM9ZC=gEwwgLC|MDi~^rxuj2H|-3v!_~@6`#|KA^a}$|7Qf+AHyYifj(uXZLQc98HA9E!f+9Y zpC-vTM_a9Gr8PxX@dbHIw5HLl*JxjxkX_j`I+!!xyyBpWPicPFx_1 zYQyG#jKjWcTwt-wsh*AqY=Vs+PCc}qAbZ%<83oFEw(z}8r0H=6l=tdmD) z_PELI*IXO7aYNCFTB;aKVDHoOSAv6s;c*0P2dpPQxGS>XHrUr*_ZR`2-FF7TR^E-QX&pHI#1!1_F=w_TTk6mKaE!g;J=c_0eOUFv6M4 zsLb^I3I-WHJ$*r{uC@uJ{WaPSCrwSwZ@DUr-s_DKB;0Jy^3JzZ@7%c)beZTf30Jo> zHa>OHzH+Kqtz0CzU5Re-Jt*U1uFq0fFRJ~reOp48seM`L!+GAoAV2|D&V-$F&CK!t zeGhT7A^0?@68=8N`@fo-o8b{9-Ik;K&DDc@--sr18VV^ibf^e&QCNL>O-9@DJ@H;} zY;5d!mA#Uqf<-St0X6lA?@&-ELslQ;=uf=2xjwBd;W#DM_AO9Nar?DycF-Xb_4VC1 z)=-<961xLh@7Do*4tn_dbl*i#CskiT%tXKM{Ig6(qalZ$iX?2>)<37}Z3Cb2iCKT8 z^*Q=v;oT?org(u-Y2O+PI34M8e%ax(lL6;+dtwpS*f?Q=+ zCrj&U?4jb8?74okGL?4SD6;A=rnm?m>3bHW*ClhPO4elj$(T|DPrWMz$m3=S7!|*c zpT9y3SIO$&NjrQQ5s^i3e{)uAv?}jHKHuWUn)ujW82kbrWoTGf3{HN@Q`Ot|ON_)( zBdy%F;<$)^@`@;9F(zDLeE7ruV$FB)abZ%iHj)MhIr6J4# ziiuv2XjZ#;nmxIzZN3Fhm*Z4BPEU<>@tJpAWz|8id;v%73!Ro)m+vAj3l|5Q?@I-q z><)*wL{i+WafCq9wa-%tMw&#Z+%6K!yCYauNe|dhJ{HHj)hRk88h^!KJ z7;-vbr|<4+_2ma2++wtua}LTfgRYR=gSFPmRn8dmoN0POoN^(dCiP`__y@NY%+4dtxXRooye4Rq#QD)_&hh;J7_$THzdd*R6Sgm_}&^+Ywz4OxctF`g! zC=t0yb%7M2C46bY_hbB2S%j^7spv~yd)HOE1hW7=OG`9{=Jw@6u*;jGgui=47M~X%0<|xMJV=i9l1youeaPq;&P{KMi zJp8ReMJ|nswfJjkJSNI$f()hZ@Dho4ZnMHn9Hd`mMi@# zElX$mmy-YTRJ~{~t4gM-shCA)+|lo!g;hs1umkgr>O~|mb`O@)t`x5jaD}dHiyXZs zKr{_IX$05v{n8S_I=Y%2A!5&>%1J)N;s9DbFS3BfuSX$s32zk-}5Ccx!$=4Htf`b>oy!jY#atI|vRYBoe zH7|b8t*-lv(78aT*698t0Xje@q7EL!hOAcbVBP*)#D?K`%N??S{mmP_>0#c}2%c+y z-gV_I{C6#VLL-tS;{Mk53O}z#F#lNp7laq~zlI<(3OsSQeXCu79C=6pcjE^z8^QJI z2g^Sk2eG&K7J1TV>PaAN0lLeKK!#KJe00?5=Cp1#>TfQ)_NQ#$jFdENOqY1zI zr!1Z_sU+Ktmdn~y#+$IUdfSvhTWH+iXHb?V=3Q>pn_8EuqpYl)B^NnVY?$|6^YSU- zWioRW1;hj{upuP3Jl2vJrtKKDL$X~*%Pq2U0lFJfqR?2cpy0trkMK5Dbp4Zsa@j#m zMy9|hh~imF=CimgdXevST-=?{xaO?)-Y-K36fc}zL&;1>mnj`|8S=%VFzgx25CHMR z`5Mn62jA)x!q#3iIp0J`+k&W3{oWa`kYL%HS}er%;6vOC2&UQUWWHkntKag}qG%)~ zgoTBD_tsX!?!V~Tg!HKY6OH}*1JHGaUuv;6>^kuyGF4Sohw&=Tdd7J1z|%T{r9bsf zaSA$#GG{2xV(SMG-Xl+A`~S3khMW`*D$nd>&OxwNSkM2L($$C@83-^|Oe5-O^?0!X$ za_4Uh!oixstD|p8#)^$$W?gY=P<2_aDLMegc8ih8cWWezo&na{o zP*%nvVks*t>lOb!v4DzooNnO!kehKv`^I^FOz^Pr8%eNH>33-AuU)$atde4F!GXQT z_VIRap?hBqP9lrOQ;sXd9Q(s4Y}w$0HBV@HbY`F8r*)=E1~xp*>r8T7SlsS~z?yi1 zk18eiRnrVO{`Gm_gZH*&z$5kGutXytRuPfI{v0LKXB_JcO#(%#lBE~#SP z!h9=bhjK*i-%p?I+y(r(B6-?IV!SyHOVB^=v%AWQmpt9y+IpwohD{%keL=y)=BL@y z6Xy4W(Jipc=1gOc=-5B5?`-~%xQ}r+PM2{sfONA6-F-(L15Ji*Icp zHV)7W3V=c^lL}#kC8f%;Q|j|7$VX4#bPfS%(s-`?im(3nl<#nfv0gGXQFrH5RJ2FC zlXzt+V?2yPPA;q<8I=;`U1Yl)53uvkj+Z3EpDk9)Ldm_dQ=B^l?O0h`I!>&etN!7T z!GS>wpYqoj3y8%P**7BY19(8fnO&gT-{OI-;5j41SM>0wf+G3jj??M0Q@yg~&)3qO zBgH(|$8v-M_Gd$obsk$UsYC}nf@0|JDwfNiLADFmkSwV`eddO03Bk|G%v>J%vGu(x zTTNR+?#`WwTGzYIU(c%aHRC@*BpoMMjO;}_cSPSnDxxbk7A7Wx*`U+A`?W7AZ%MoJ zkbH%nwsOvCz8!aCYPKarXWRy8oF=OMOq_8E1)CTe+w2(C+M~F>DEdAg%fxr~U1m#!=7g0@xF+wyRrb^2D?vr4)j{b$8UBwF zUj%s2O!f*1*YU6dn$yre!lM-aYP#J-kT~Erk=`H``|tX04FuXhH>7|=Etd2`nBUVx{u3lx~i0yfgz z&r>e#G;->d7<~^3YJW-9l>o@`+cA---cTlJ~H6La8zWcMyzXVSMb zPqO_D`7Jib{l`$aq?(ZCyZg6hgM%SrG^X>Ji$>pb23}=LA9O%Geve+xZ3f0>vye^e z5KBs`4<74^U(y0kZ#90enE3RKsY?Ab9UWbVE94WqzV5>5z~}~VXhW8j2&k!3MLivE zcMCLfva+&zZcG?J0t8^sa8MrVVzr9~5XQF@Tw$}ay&TO?n1vR2`{p);sgj~1-Ibih z+WO3_EQJ|PAwFeD%t10x-;tcqYc0eM?qDg%%d5YT4JV8#vPezzxEc_QzEumdq1HW! zybOi|%;Fdf)e8g(eKWd(f9Gro1vUGh^3#*PWM$3S1wnTzkmgk}$A6 zu5HBLJI{H)COW|Qf1wMV>FSOPxEdN7f{%t;%@YtwPq{lw&0!`Z1(9NO`KHJCRu)Mo z1xB;gIZBDS)h)re6?^@U9eLNstGOH9tIlXAWrA4Gen6S#6tD>}#Kgk-7yS5^*ts{w zA~JCdP^2r4RpzOpyFDZrKqbF{n@|)2RNXZM$@4oZHe^8z5(C_>*7W-v))sA+&#kS| z^$;uJF(3NseLYjPlJC4}^@3pi{c~E+-s+aqj1(l6gI~7*T`|+wu54WqI6CMq}D?5v4*W)Pg%w<{1U%#Dmsn@4Oxi39!xathP_L!-GrG+?! zne`wHm1rC=I@aY(`9vv^t4y+lv*kLqtgP&c;}AS+!vrdq6AmxN&+M!+PW#>V|@IC^Kk{i4K@QqY!6RZ%fKEbP+d z%l-E^qHU+42*-09OThOQWL6FHRNnxOnEv3dYHO*t zH|qEUf?M9|V*n+*JHvS_7aRr^U8#UQ`eV}-?mDUS?72Q5NX&@t$Kq(+ro477qo`zq z5RfY^?#9=QX8`gLUpBvp{(M0rY4o0aF)uB^cA}=tsk$Jlsd{E@y3yDUvC*1lK7te} z{EjbqOe-XuY0?r5Ab9~*0T5Nt3!@NrpM>AOx*fmV(cU_v_t*jO7h(2O?NoeX51StE z@9#q#h2hc?Nyg5`#l?L9Xvtxqsc)-5lX^YyVC%anC={f@OOy1)iV^wI0o!(a;yyc9 z$29?>>8}>2ozb(furM(NS0xMCCmXhnQ}K1Pd;Tg)!zH%II+rG)cR$nL8cS+zBehLQzn|P zvLC0Tr#~AtW<#ggAj+eB3w;$kyEO>Pcy%;`jX=DdwOu>;JxxM!-*lxvhmaE*jN~7s zWjI>jKRd=EkzQnq3E|MG-TRml%-!XL=1>S(qz=a#{C(R-8UxRgl9K3T!__ZK$oc>& zYUF-YyOf$mC7ZKdpp`e}^MiBBo!_#12G=0C@np}?=LDcTV93Djo||>=#Hj38yDt0z z-w5G{ur2~pOFb#W9-kUgZaM}B1t|!{UNKl;ob)nEr@#C1qRO2}a>0BX*g2-pB5~N& zFLX3m_t6D(eY=9hGw?m($8DI)l{)o5+__QAXp`+y2Mx`q91%k@iGX8aDt7s;tkhMd zCvH1DE(;y|`(6@1#`86@p!BmA`%$T17?RARjpWeCY&|=fKC?g}gGkP&pMK5rT6*IN z?SM3|_sa_emOjvR+ssM|K5RmiFLfnG(Rk~0m41Tc>iz28tfkQjeu@X+XT%JGJdyDz z#|bDZhT=!{{8nEWd?5p#7i~H3-2C)>8@4DKOUoEJy8~jib5;z7?bvpu^s;(E@%}|V zWp@UmbR>t;>Lm&;69M$x0$Ask1GO!4;WK!Hq2!RmLsa~CdrD|^Yfi^OpEg@zG%g2iOS>jqk^k#F?!wd(vbAdVOkLc*=2&~q=eL-{Ju=xUwzMF+GV2FX8?OsX4*Jq_@ zB^rScmpI;(K`m|=7@}!L7ntTZvZ)euJUpddzvePy1)&c?;&hZ^7Gho!YZPQ@fe}nb z)u$tKE_LyQk)@kBkQP5G>tmgGpPMk>0Y8VkZ4>5=1Ji*>FKoP!mqvV)C$!-8s=)qiQsN{ zOU7$@Zd4)d+(&VJXbj7`)B{;My0{xTggQ1`~23DS0hQ)k@Hd9UTe zAM3_Wjpt8lwXYesu7r?F*eHJ>A|)Nkl)X|mn!K>Ekd#C!yEkx=&kv{zkij)AhHeMy=ZlfejAphx!W%p}*!oe=}sW z8l+P^1qOr37m6|v?H*81p_2DaL#O#wBgnqFry?$nO~)3R89Z*)aWiNq2^T7djjmIe z%SBPE6hj@!9^P5m56dDk_H&+Y@DB?M1EOKZwd?ln+i1GVF*CvB@T~tXHNPd!bcZgq zFyGheEB>`{~1?`^~}0ez9Z=Lfmre28gV z_%AH6Pdili6~;69{V()EBq>PHs#C0($=$sh@x(l?K$dW*$e`sze*2zXsRY37tYV+v z8=Mdlv!9xtU>$HyYcflbqXj9pl#vuIsNiFtZsFuf6#cQzG+}_28oglQ=SGcbnHq3t zqmasRZ(GPrQ0(_N;9An*+ADy_LLc0wZQ+}u-N#j8EuIkQNrkbMo z&#xI+I_+;cPvcEiHG!sC8OZ^K>(i9F(`w5H1RC;Cw%G1$eEb=X%hjB-d0kB`LoyA! z<09HD)Wi#r@Zyy#S()3vrvpjGUoW&rQi$pCOHQ8S1L+%VGiZ&D@u2@FP+o09Eewo| zzgCt(A`7*rm4WUUfyml?e~QO!HIxAj0AU6&5P00w&v3wOgFw7MB9Z+cJ%6%l<%Z%0 z8`S%_M-X!oT)*Ci9j$?`mlBPQNk?xQc` ztatt1NtCMgQeIyEYKjFYW-WM_Foj(AYn&jHI!x4L?CDubr<2%NE0r zga1)DZo#I&cUTEx3O>ZDMce7o^jUZhpP2hcJmBCoxLw;TR^()53rqoHmmtXY1OUDx zH|{&s*|O@mo67iDK|n(szv8oatCfeh{-mVw^xjT1>Wyf%c?C-kw88~Qv}onz#8U~W zWJ&S5q={x@fPXytMlMI2M7hkhUV1n3l3JXt3YDFmy=gECeSoT+($CVv{$HOB9qSnj zm~8c24|i;)o&+BM3K?Pg_U?hGt^MHJcvDEH;pS76&!LClf}yqmQYvj-()4wGdIkby zvdEk0=*?fBY4q~dZx~kDK?N%|s8|GoXaFx^prR63!4vq#qk;Tu(5spFKgwnj~ZC`gOLE~M>aAwaN+(@f8KmGz=zq{S#yJ0mmE-C z5c}ug*88D-SNMHy<>s_si*2?LwS-@ip2W48T|pucg=7e7Oj@uBv^{3{tD(V&TM$$P z4zXbB^_|4G+stWxn)cZjR}Ej`rXQHw*MHSGcDEX9kN(5(F6Jl_ss z&GzW==k|8wDW`tv43N!WnaE?))qJ2Ym~;rFn4gL#5LJXBZ)ez3^KAZA^#;j4(jZ2KT=oba7LC=;EAl3Is3Lm-fiOn!t=)qC6DOiWQ>nU* z^PanY2kAua(HeRUY3s4CQ>go^>Qm>)@&0v~|0@>$KNl9vjrJ=5+%UWgcTEwp?~u>z zaKz1>YVa4W&*9Tv-(E@!RNOMt+f!|1kA82}l@R&*rsGr*DRe??qZ1q|lV6aGS4LM_ z*yL8n0bf)(OpcYA-9617u+ITK$2h)-n+Pzj8V1=OCGgZHZ=6D1Zp1RM0d4KN*sbk+ z@50eicd-kGJGAe}`;${2J=NCb!c{#q1uG}mqWT?xvH$*vRqq`o8Kh2IXC^_}#g;3Y z&bv=%vAqG+SdRm+HLx$zE+8PY>PgNK(sbIW!XsR2es-ag$&We!I+VA7QDgTFO@)?W|PTj|)|v0mfyiF@Iq7&zx{@a^8D?l|pk?^!fHuZ;5`5 z<^U`!-WnMR$z8|uRCMy03_mJr5(Y%ufK<*A+N!nKTB>f>;?Gk{qvm-i1*WoUz{NdP zFRF4ZdyWg=lHf!D%Zx&B`Q}-2J^|D?h4v026qmJQc5S zBs8rNJwu?oZPL_NK683RG<{w?;Ml8D|R&vs7}`(zt}6LiEDCU%k$xOskOYqh}`{6fTBZlwJfJB_kt)I_+dH zc`fYxRMA#53#L3ai0vMm$Y?Pn3#8cXkn$VbU?Gs-IySJSpI**xXzrxz&1s9`c~I*T z#EGr7Z2^^=UoAx_vMMLp9mHDLgG}=k5Y~>EkI?A{pw=4^9Xq9KlIEKeZfxxpcn&F3&7tnfpc2}N-wI~fdJ=&oEs4Anjzcvd{N z5x99`0Mn>P;Y~!(7B?u@kP@b7SlK7P3M|2vnJT8X{SUFaR=ws3Le@;NPP;B8?s|rT zjS~=`ip{k~f0X(ixUgouBe{gwq^E76&XjtBiR}3xW$VUej$9M89RS{j33OZ_?a-Fe ztvlD9PmjG7+(Rlbr_+cC2r>g+Ca@Sch>Zr-W_eSyvv~~4MPv;qo0`r6Wy!9cFQwz+ zUYZ(P}>xQtQy@(Mch~p^a?a((Z*q8%>NA|;Kj$o z;Pr&R|0l+P7hlg52Kra{% zptf>MVT*vi({9P=PPB<2gyHr29xOgPDC@v+f;}z=#QtntKqR9;FM*N>$)>;9@TP*% z)VK$+Cw>dxYjIzX05E{M!CnKctb*X@MD4e+Ndejxw96NPu!o4d`{^nfnKuM=*8q3TRRhPGUCqnOJ7jcL)SX+y8XcG>;V=J4fdo9Yt3$;fVNokV zk4O6hfUrW=(T)SWAv%JM5Bvcc0Ei6W`6&c#vyDiD_W^etq&HK8NLf54`QYy@)RYr0 zBrN1v_hlmayBt1TQu!kqm(YE%>ZPs{W^94*>;zN7y^CbKrq)k!(nQ?D6W^@>8|mZi zBI3bnP@a92nU+?n{+zJ+$e$5?H|bKfud@?t28D)R)D3NFY64JGFd-gE+oJ8WF;bQV zog;F$(qx{`Z-3L|)D7k@qkI=K%D-q7ow8MOJ*F0|d^Wb33#DP>CRYgeV)2_f3LZ2L_t2ayZ5lX*p49U*gU?E98V`UOJE}^?v z*tG+caRemUfE>755r_bsFvhUkPEMtZSAUZG`Rsv?2juTt{W7}xfGrd=hzcqy z&?V$nhu~2WRR5w)=HozpM{D(3nwld$p!B`eG*L-_y45$P@!TS*^E>EC@>^^xPqDAl z`0rd3bYJP0Ek*0ESP|#D%vpYq3dHdj!HEO{``j|k7qa1NNd8jmzBQ?%hV-4mgg7QRm zJyrW1u|0Dy(8A2Q^-F0R#obp4>w6M7Qz{d4pgx0OS#WU?i+rql=p|ie4+691cUWDen|Atg=ttXR-rJZQ2gLdKF@n4tbm7Cjb>EGNTYV@7Fk7&r ztkegW9;^uVuKb$v{aN5w|5%)C1AKm?KL!d~907kYq4J5KhjF&kp3dA|tLRnS67LO{ zy}tXVA9^a>XoDK3jIpH{q62T05|T(Jn+7JvYfK07S&8Gguz(DH{f z_ZHu7JNYl#0ACcXlbr62J7%W?Y@@r)C|muziGUXnf`SBdhOYV+H#l}iyArt@p*92V zfdNBLxeUsmd!$=7Kgg29-Qw1HOQ@#LB$Q{2rpSz~AZhyDh=qQaRM?3Y^=x^8TAsiz zuOj`~W1#*N+;)Zk&z8T^Ve(%2tA=OnHimAwKWU+7XG2{$XFCrTtb-vTL+JSjiVP}( z)m-wMci@1;j7mP!qp#^0Q5^>5<~Q80rONCgv`sz08^guLB_I&j*$MTX`j*QbNa0U$ zzBS*Dp_2t)S2-xl&HT4+y^2d>JDQo90ZF#cxf_V+5L^nT(GH$6u-m^4M)#ov1kzu? zpK9>Yqj;=YH@Kj`_`ydZGT<5t8+?yJ1joY0VbRDuMJFSWFvS297Vr%B|D42-Iy{da6%W$W!Y)B5?0D5A9(0MxypC9*Lf_0c z4#TP{ot9AWk&E}L-BeXBCntTcJe-8hI=qyLx}-##VY>*$wycfMS`V* zUi9kN*RRC_Pme*qD?HzZ(~MNo6U zU>>(#S5r=tIDr|+OmnP&><l4l3_+_E!Te0~rs;J{ps zTEC|TNbd?BUM`c-(@#_7t=`*<@!CqFuf@DIkxQy&d*z#Bx>f8_|L;ouI`O{W_jtVl zc4h)77F4%f%>DetWFua_bQs9*gS{K4JGNA}y*@k4F0~(x<2Y64ot?+?$iimFAnnD3 zRePZC1_wTk#A^xvL!gD4TTH=Z47HOTr}7jpP=F@JBok8KKpM5U&&569TafjT9FKLi zf>+Ms8I_0?Di_nW`08v2*~WKi7$ zzwZD5PzQ>{XS_g*dHg4~`?F*=?fgw(iXqBE<{E#sQ0fZJ{!_ zAED5~9+}a;+g-o`#yw^$0<*GaGW4MyoBnKgFeN^4PClrzd$k<5F@ygDEm)lld;SBb(Uz0#ZC z$AJ@8vg5&~qEQADJ}3@IWiXwAV!$erqqR>j8b{*~V7SeB08*ds4u3m{o|u@J@}lt( z<#n11V2UE5nay^Smi`0>Lm;PqE>Kt>arg^Cs8j>n|G^*YlPq5g^|LB1-Lm$zc1jtnLU9qiZzIp~w`{5}$raV{BJ5KchT%?31^oSWkC=7ry^WK|u%U9_G<)-_l-bA0&lA#9*QJW$9C zGMY!4YZL{8r2k{_aE%y^J~b^F^rlIGW)R=xAm`X{@#nH(Iz)d{a6&PEL!$O zRo#_)FA^0%Y~{>G9xoxV3>nZLS|hiCW3&f>nKpZV4r<@oW(z)$zRd+3c<85@h-s7W zhWw#1tKu=- z@j-31C~J27MM1WAS0UE?PT}^ll{4|PzU9&nz+4|k>d8pv4IBAJz6?cK0oiL_pjVPw zQ#Q+i{jUc#Q6@jyZ;k?wWxoVTiiPTH;btrH3UibT$mM0DQp7>%_InAqP9~v|S%8&< zLLlp$9Sz{V$rPJx5p5nePjchs<3pa_)lTidrYSEUN~^JFUISW|c?qz*jrAVu`-+DOhDKjBh-rh&SJGhEb3@(MLL#g#{@O~F>2{m#3ect z;zOBBfkgj$rW5Pa5i3hDe=4rfqjc_p;c(w80hKKH15S;3m+AZ5aB$>^a8`aHq$&5l zGbu^7*`yn2Bcu$t#dn5prRz4d3xOS?tVfC%kfbQXfdC<}&G+moW+C#yg14079J-Y1 z>i`8MQw#f#0t~ux4Nm&wAGPk2dt)ZnVE@@tsCi zFoPrsn1_TyhVxD^uiyE^u@gdYS`wkV)Ym)5aoBfU*4qk*3(gau^1w;~g|PO+9SFit z0NSARGagVDFr4A(q?|@To;NWScoP^NWWBz;BydvvEQFe5kM77jc~~ycRn@plTjq__x-#!F1LoNA1ebtxVZCTh z7D01AwU{m$x7k2k{3R=`&0r_%z$|S={*~2-m7(I4dU}<^9QTchSCUk{304uXWqEm% zzeu*Va$LqU+gg^14;gugHGMml-%7QtS&BH~_^LLDgeZGxJWLJQw6XeXx?|a|!M;qt zch{}+;eHU#^}2BW_=>8V3|IIlsi}>!R zI5c?O)@m(sMtHq~Mfe4ZnVNFS%{C?~n&H4e?a!0?IjbT=Ypq&03rn{ z_SH8CgoBi?*w*DknXoU*D}*qTtR(qw?`;0i8|HGWmd?Z{*4%ZQPt8xhW^lR@a-gcU zzZ#32BTli{sSu6`>)TJGyJw-?ojJpks~TumwzS4`MZpo+Muy^)3gF2$7gHQHgIb#l zK8@G=R?lD|oM^};Z5%UT2}ybT$y%@S(YwPq0RkHpT5v)1m;vwa^JNZ_rJ0hW~Q11nLSd#|Z1O+uisy-hq4_uRM-vXFOtDjbCY0u;cqUgb4J z({=0P?D`;>J*?<|k1env$7pp`rURYePOL#F1V*pOy3!J+9{PkDa4AD_C)a?#;2k#BQ|mVc{l{a+XODti)1Gp zUovTWrqWk9{!HvWSZvU)?i5mioVfxFV}vgHBc}gjo1MNE2(JI~!~7QXz-r?HU@eUB zOVa}=z5morU0oeIcED3C4p@jy+CLZLkJER%tCsimpUn+2aW>rwn9od~;Q|I_>;Lt&IY5;H5RnkG3dA#bR3eljZt|NK z5MI=gY458`Z^rTLVbfauJBPq|(A*(7(jAH`v?V<)$;e+IOX5r%e zk4ZnNCSYB_+hoK0-j&!0s*XX&4PoILP}(7N>G)zI>O$Mq#tg0T7%%_V6`)vU4gpmP ztXG6RTYmIh$R3~#<|@?RwW~{eBaL$da)2OZyV#BV9WP(Efq2g0V54*~afcXk_vGhy+2J__ME=PJf=AMp43VBNW$K z%E0LzAl$uPy#bL-%TBe;ggEbO)Lz8)pD3*f7BwBq{8A1R3?JI^R7->A52!+L!X3fR zHob#*oX2yUp<6nzY3_;4KYMpC?1i;EHLUj>&plI9Q;@m5LcpR^2NwJt)`sIPr9&lm z9cX`R3IJM*#HtE+&k>^WoM?FseJ-bB0vwUs4!ny@yA4OeFsZZw8f~O#%e#TMccR=v z>H8TAivRhyz@aHPTKd?K_x1?-@>^O8{O4UONW;*|EqRgmqNgE{O@NAXvzm#`3|^T1 z#atD6m9ksIV2OX{tD)S3~%ClQm9Y-=iA%bRG{mF8kd5!h~KNm zLY6PAc%KFcOi%Q__>}~`Bs6vJ7Blc!zXO4G!y#Q7<+8P)K%$F@M&bn$#!9(n9P{jr zn27$D(;b^`N6=220qq4?aZAv9hF6=Xfq|Zs)8qA@4Fyk?lp-scy!dYWFu)S0{TM*d zfq@Yngb09P2o76F{P8-(a1QB?#GGXTX?JEXa4qnNcPHh|C=rjf&>?a=kY%>H;|Abi zdU4{Lx*2~8j&tG*Quj6DyO46d1bPOkpzBngue7dg_9CWVzWZ&$Jz0+GW19x?Fb{#2 zK}|6bWRl05&wN(SACdr5T?rHc(yusCOlSNgB8CGy!{-nVduGB<3kbdcOkV=EO)NtR zJZ{{9UFu;R4r$?kqO<)CujxRUA*F>gF@2wEn#AN=9rm8sSRm8>bgKb`gVb{6q<7KJ z`%PeVg!5&2O>j#RoQRyO#DiT{T$@X=4X&hUH6XGb;J`-}SqCO!(*S+N==KZ-US4O& zE1;OA)84poW6!}(!HgK=#5$S2Ay?zGgL7P)0i_)16EwKLk$$$f?JD%WOc*l(GC*&7 zJw?kosf~3f%VtAFCF|wn981I_;TH$MN5{&5?AeMo=ww9hHB%0gPak&GIc>AIDE`CgEsj1 zYDxXg0LdR!zvY`%3wNS3%~pMn+=iM26Zh09EOi7fb~ys!N4);4gFQ(w4y4|gxj5h% z?V@?vuVm4vxV8@eohEd-fz$Y}?^YW+!?|2?Nb*nORH#axI=o|X4 zVwt?lCGh;u2O*%*EZEG&z@ZH4#ser*=raYH z3b)%R&%-Sw)>HWJ{y{l|rH+LEJfe`>vbyaXa8pA#fT;&n0s8d6!(r+P_O(AK1(%+8 z+Ixv=3x#Ctf&_Z~`t_h8NchUh0%0#BJC=YEAh-w3BLdM7w&%4#1XzEU^sSn<_5k!* zZR=Wp>ou21)IU!w>arj&B_##SGo+n1AtE@Zs|Q(>^$VoFHle?GMP(uk`!7aHkiuo3tQB-~-GFT-Hj*V14|GUt{1nHg+F?jt6NEN5M#Lsv^OKFdy>Afpd z6)l85MLZ|OhGPlgPa8nrVzurH(GAFYnLDsB9nvwH=y0tNegV}j3mT6y#Pg64_(6+2 z@VdMCeJeBj44MAdsD?trCDkQ@^1mI;+j$^<+5e(e?QRg4fQK!FMi;G=g8do(x3UX- zuiGB~yf!g-EM{-=*%2>VI&&s_ZNGc*<^Ab9i?WayYuI4$V+}_7`lRgdq<}JJQhl8t zIAO*R;r!yxI6NI#iNTq13q&xWBSU+Lm87LHi4GyZBBLHqsEVedFcD4{&|7-5Tiw(F z^7=dD##8hW05CI7BYjwXau*@CtEV=MUzqd0{ACT4%qt0h5t3$|x7s$qhXUkcYc%In11 z&AjP|2RJ=2A6|i<8XAVfItxQ^gmtLI*m$%T8Yp3xg;4j$fq+WUD9}n4hc=ie`aEK$ z0)PVi*aR?=x5<==tl9f?Nd7?N$jj5J z9suXifF0bjuu}{SNKdT6FrSG&wgZP1I_Q2l0D9^`A3Ft=bO}^4Pp=H=AUJ`2!=Xr8 z5?z>3{;rpKpa0#j^UUjcHqTz(2uYm6r;gl7}u{{}x;*xh& zi{Utr*XGO(+^&+GrO_qd~;Lpt)jDmP@kj0N! z1E($&TDGXd%QCB=_MnmkAL&+V(EJG!PCB|z9^#a~e9E$h`ua%^|1=!Gtlbc0;L(9< zQz6aLopLkEO?9WL{SO$6g%f*`;C|a%6d1e%>rWcx4e0~@!%hu6b+IdklaL_5U0|Eg zh}n+5I_}@A6I{JYJ1S42L@s#AgX&L9DuA2?qA_e}pg{f{ePm-psT`jxFd1+XXA=yd z<_fg%t}I=1iL4{%|JU7HM^(AE-NMUOTsDo00VpjYD48*rTGZPVTD%efy5KxMO5 zNxWtpYd=D00@mznh>X1EgpQ1RWQN=80!inL=oyud0a2hRkcK;nbAb~+=I;Z7@Kz@C zJrVmG;5Por78yqxAf5yTkxuzoV`rl8V9Y!b>bd-qW6huo+14B2C-icGA82_lurK1@tSx4LNOjBmSji{*t+RV9NLdqQtMwCexnrA zaHoQ4+Qa2&bs`oR61SL`80ZTjFHig7{7H?I6{miF-0}V|#(n|rf`>$m>a3`^_1a+$j;HW#;6-1bpUxbJ@XRJ$ z|D8Xs-}k1ZV(DKL<*~6<;h1|fYjZXT+c{RB7SU6FP^a(+Q}PETT9?K$n0xw1dB|Tu z${lZh^FhJI254$?q##p8#cZD69e4iE1BsJ!@(QV);y!Gt`6iBJ!HmCdOGXw z;dPIDeX7jRKpEzG4jS3ewHzRZ`-HmgSKu&P82In5R{tg&dwY!Qto6|@AdCyJk^Sgp zT65a>TeBnjvLwVAg0hYORvTcB4@kLVLD8?V^LK4Wg-tNwr%t%H++Fr|fr4jcX(#L-mmX?3g^4jjdzG@ABINP64=il`b zR2X*gS&YfzgG61)i5rMQ)W5!Vx&C7#DXS&shqu}Mfu|HyvfU;R;69oZ$5F;@wLL%` znT2LNE!mOnxs67mQ4O-(%tJH4&&nbXl3t}ezvwPm{dSO`zS0Pgirw!S1V3z%JQ@QI zMDDRQecM}JuAWU*IQvuJtLU+G)^~leV4fKP|3o&ysgczS0IFPwB*s=cJ+|ue#Zj3F zDxz0bUIIs55<+bJU2Q-3aNfz;xy| zfYXOy3hh)1zNaBt2-}+|7Z&L6A2(?ec&*#v0CE{*>2rv>%Y;ZM`WkF)9^!p$rJxV6 z1H}?H$4ZqdU|Omn)EDJ|yHMxD>l;-2OHKH&Y}bFs#<1qqdQ`jlR}YQ{J-7OFT&)_n zX4H%5=qG9=DBY_{(I^>QM|rqX(mbJGL5G6B8&rol%4Ldi$L z|8@gG?BinANL9~7iJ}{46R0FX0f*8F!ogZg5TNuT(N9K5&7qu*S}UQ`(n(KGf4#q! z!FCRYwKGO1+#V5@xigx}sP%zgbtWBx-H1PL4e#9z7878cif^R^D{xTuSYp<3{KLh? zg@uAsTZ9al!~sGebVcRXxEfswqA8Y{IKCnttvs%xdW5_gr+$G-r@rP_UrnMzb?Bb* z%aI_0pcDTfVx3-qXLc>9*%;b{kwMXUXm$C*>4MfLAdGM-M!9}IZ9md|@yFWv&7dSO z+@6?{DMTmBg;&hZqbrJb@f)nKX+}cPW(mz1gE`O$+pZq|bYp7Q9MJ)Hss+J%fd_}a z*LwqXKbUn+OZyYA-ZF%8=@hEdY@cTMvlAmh(fF_2fcp7!bB z;E}I5nczhbW%>?vAnij*uOh)r^x1pvW=R?1h^7PRPTo=iF`nut$3!*I3K}&n<}x4b z=6S?MciYS{G%1<`b`{kMp%mxA_#pL^)?eq@r^HnXQt8(~w5#^GRmnnLqaaL? zr6~#rqgQWh$CfD6y)WB;~x^fG4=5pZQGi`ZTrt=G$8B+MX~1WRFySq zU7bWDJ%wr1j<|U1n>%@xzOB{Y2jA=P(-^nXvk2)jBBfFQe26fg-&}HeYD};N9xK=w zp%3*20$75RXnVwAoFzBwv8Kiw^gUGsf}o=03H0|JNaWnUqL;6u=sJA1d|`g(*P!|> z$7yuPq-z&sA#)@8ScI2#P+iF<9TB8<<$4|VkWlsJ-}n|l=4%)V>T+W*ey_msLO-_Z z1r;zTx4lja2VAzn4s}zChqX!RiHvi%!Xq3-aj57aupBCQ7;u}w1iEb2x zZ@2s+`ZO5V&-X)uX&{X!`2ZUm$JBw1@UX{0vG8aI5c@hBITdQYc;t?Kkw-5`UtLj| zp}T6?Ry=X>KsXW9dz1_04|c&HpGqPF*u})=aVR|(VB~N~E&><>1r!B3yWUDAa4)dC zMD2YHL1YuI5|}v8DzzZBRXo$rRDR0=aRz$M*jGwf0V5Ef>D@uKd)F@R0u6T$FysJ5 z1PLX!7M>##oB{PS!AHo`bj#;)nw9aM{liHbsfcYRs|mqdn2FCp#yg+T-yLX8ZUJ!| zi`xu}5lYt{E~MZU2K27v6)qkXN(sY?gm75X782z4aB*JoHXT8iNnXC$<%TE3?!<7C zOu%kD;Z7!oMSKWQ2;thYl zhBVFg_c2#~4*O%nvUR@z@||95$cA7F=gO?RaE9oGe~}>nHo6Lpy)?poHa$QL`#yQE zuon2|)*YPy{@I&)7Y|SJO|T;%y8%NV1{DrO zUY$6V2D#bG&9tiT*FX*kxI5@!XyIXo0uopQs2wB7n7ZA7@DN$Z1tM*&tVCFVV}CwX zeeBMb<2duhp9>JUq5MY8p?W{Y?cO~?1}kT8{MHX#72_*Qz$N>0rQ7&Z=?SkWw3$gr zw$Ch!4E4=aBm=|A3Gg4L`S2O{ck{z7po>29%Oog^MIg{`W9D-tKkC?75^y8&TP$EjmW_*-B+;35^`Xze}iUI>L9>4RPZ|M>tS`Noig zP_q_NatD-Be;iTA`3G@XKu-)FczP<&cyg&{xaNb-zDH}fxJ#;Psf~>dOA%OP`P~D| z&AxQgM7JE=`EU>x0>F+sT5ShRpOVE1wTl(0>&awQvE;KLVT>Ry(uq283k0y-uzjVp z4oVUnpW4!hsP*%mElJ(yow@3J*SJ6*8>U5MjupCYXU>!XCgyn&>S*yCdEZih_FlA3 ze>`^v$C6zr5$%O8NUq0kMSl#btgslNZ>PU(Aqelv| zvee;n{(4U>JBkNtv7?l@|a*%^6H?(x&CqjKWHE66JPKKubUr05gGjQ(&QxS2IC_(r`i&8@nYq zJD@6o@e_h2D6g~xtQcnrcJffelGRB4*O;UwaaUf?_1B}W$Mt)x4`vPSm!NXkCvBSC zcf(oW(KrjQYR>lT!n*N9wG!!H#a69_(!ZuD9M}6!dQQ0|hu@xa${%6=I@;+LQSNrp zP^UrM`FUi6QNKWBQ^-4)t42oh5ivskWfOsK-n|P7yMeGgaB}0!HVXZT9jaGZ7~fh$ zr%IRo$MCT8*cS?;G)`*CirA=!`UhUeB)HP_oyzPIQA?!|V~W5bYHu--gYJ?Q8W!}a@Q5ITOJb= z1N-#_fGhZy^d6+{vKPtG%IC4QBh_a&5?{Hs%XjXi7YvOM)BLEw_Q zxJsRla^%`x_q2Us^zsS{ki5-R4J!hsSQXSp9k9y-OB2;YRF4>2iI77;>|xMS8u~p- zxtRowU$IUEAx8P7@8~o!@Y>D45g}QdQ5nH!l)afactu+9<+EiD0;gRmWBx`ONWp9QL9bD!zuOhyyBoHiigC*k3^0p(b386)LM5Lc_| z-fhNH!2!nhJfP>hy+|}N2B5PcWbydrYU;&%#fZddpds~8_r;4B-H@%(4DT)~f}FDU zVJNWP=%)_^3RU*{^{p`2PuGp$j4Kw{Ugq&1`czWltU+?yT3Rqvu74eTTzi)uvxn7z z+#u7-Q0*S9IaCGUDL#Mz15n!|5*Cv>!Hd@cgQMAuy^RRF9Ki`7mjnT;v`1ehd@6wsCNS=o$NNfooW8-sUsu-X?`)f<}xvstT#hjWs2N#N9;nBoO7Z&P!o-FP( zDZAKX0CA`I^jTCV`G_;uMaD%pj@pKeow4hOheb>wDk5^+Vf=2l736vrOO_URmrh5b zT>SW>IoWCscS2`1pe5BU;dqwYlR0qz23wD!5Wa~?NbLIPiPZMB zhj)2UznFgn!(0NL*8#GCW<-xyH$hKqLlTf6Jm2z5=4-w#r&{Xgr|`cw+W9^}cF~d` zpgBq{xlOa~Qk!5S>a{{bd58?OJ5Z(3vms>@+b2N@mfxvdkO?)%xP%00?>;`R^;@=} zV(?Pk^ln1=IQL^e9(_Gpiw+2Ol+m6=n6qE2XoE)iD98@`1K^x=^H$$)wP$ILE?XKv z%7<8pXTeRJHDSpd{=feztnlDvaXtRs)4&T0IDq6vULng^?Hjb%Sp>Br@shaWI3W~B z;6|H6W}~THHSYI(xGYvs`-HlbP(Pun3E<3!d4SewGkbz=iHq_KVD$Khcj=!Ydq!~z z+TGdkTnIv`_#moVgoFcaSxxU9jEsyN&f|%2Y6+D{#Kz}v>69e{Ejv(4u5-h+01~uc z?ySjAr-j)orYZ=a0dY0Q-F@NL73P1Q{OiJ<2W$gbtS90U_Ky(>`%U{<1s61YFTV8i zYr*%#-PI|uWz7vgq_?spPLM{8P+75|xv3Lzo-D&WKl;oG3E*_R6w%~NyYA3NzVMf} zJ4<2P{iz_;$C@^ZD}wI6K3iU1-em3jJw(6b`7^sq(d{tq;DEM`xThdP$V6^QOiZjP zQ4Vlf)EjhfVhPwtblMGa257OQ&;u*TSB2~}o7T!RrXt&+>^IFEF|HnI(`JtdYhYv| z!6$_v;yE5g^6SC?S&h`Y*VZ$LUC%f^L_hiacb{R&{P@NqbM%^*2zqSh`Fkul-}1>O zuHuPa6}DZzxSp^mKOoOmpt!pfcIeH%ae4rgxrG2@l$2@eT>H+C%P7TdNQq}+dt@Og z`4-(YvkmC~t4H=+8C0jE#`gg&amm^p*S97Oo{PH{GXJQ)0+Vphmae&e2?6%b8=CzJN_sl}4QRLvJpdE;xo-obL&B0RaV`X*g z-Y-3}dGOc}toZK`{Ue7D2lRImbOrm|Gsu~)_YvYwzA)fv>Y2BNTyNf&UFw0>Q7-yV zI^9VJ38Q`VyQy(ZJ~69qX^&3(5zKS-pY)x0@TlxQRlqIi*khxHhes{gL1JP)1? zjMWQP?DoJb@GB)0_AF+39mrE-@trmX@Jn?%pG+m(u9l zwTksB;P&LRn^B;y5l`)R+I<;IlwcoJoYKzlGr?d!AbZp+nOiGw{U*Uva*|(QIIg&l zn3YqlTI6lQiFDYzAZ(2GY}Da)?t-u*HxV^Ut2nH3AisR^0*fwq*^LS0j{3+WS7sSsTfP0e5t z1u(Wy2U5lnqxm22j4e82=5vi3af3O#<>6ZG8m4OS@XtP1?hA#FTEmG zrmLXqvcB`Yha_~Q&EK373^si%D05lAL)b26$I3B^Bi5uvP*ik|eBi)=8u1mh)3dWs zc-1it58^X#at9`ti1(!I4*~HtbRp`<9DO^}vt2_?vG*np?Oef4C;@ub7(b|ZbcXcO z4OfJBd#O1Q#Gt`)k*%Ix>3hZrCwhv5_(H?<@ZgVf0Kn zD)PU^mE8fe5Ue}AFH!ayvXum&jPpV?J*gSf{E4u;O5po9?L6X>ehVOBk$S{NG{h8$ zjyFj%d$g^^=|J|bf{+wj!qcwt6JF3+xDot3-2z*Q)OHpkI!_uu_W{yUAYn~6iz6ct z7cWtYJ&W{1-aOIOUQ|dyL189-u6aQZd78qF{xFN9M~)~G@^}P_{m{e1q4-%iBPz)0 zA^g1D+;XY)yYwp_K@ow|al0cF3t$E`N~V9*4{MF{9AQ{L%s z<^y`LxASh{!0a5Kx7Hvn#oM~^)q=B{_{}V*40%5Y0&->tRT63e>Xn&>ReRGXOfe7@ zqBhHpjFYMD^vg>ye9nJ&C=dkd9Xo!3DFav;B!T)6fke0widoU({}~IvN$>6BOZEH6 z`_X{>OW&iVx%prWEtmS9VI61jc{U zHd#)ska@zdzTCMMHMv7h+e^@!f%47e4dY9*QzN{&J?i7*YDg}MmpWI$hNMKi#tAQYjom4G@!(nf)TW9VjXA&S{$I$B<60XgMU z`9Onl=XmXsn@fv0z8AH%pO;NP%V~8_xl11yykM!`vs-9#dQ87NvbeaI5)t_K6jLAo zu#VaJRpefo@bGX#TF$B+>NU?a0qCd(hlkg0i9v%aOVBw!*_&2C%yyJ8 z$v#Eb$($cb*&G}k2D6Hk3Tc-gCr$DTjyU@@w~#TA2o4@5Y5~3HZ{u273pkKvBDc3t ztLTxp&>wOQ>$|i?*tpR(sLw3*K@)W(6BlI4;i=}YfpJP5Iz0XgAcZwtX+mdZnFHu_>fSx>Q%8=S_Y3xw_!Mi{oan^5<%mjDi?76 z2mZN#MSNb0mjfqSBI4p$GVAwnb9);FPCPn-0VfeXemb*$v=1IW?7tj8cdz%x@~cU0 zlg~EFlowdO@NWAF2B@<<`h)#dJBW}Bvt~wqC7kW$lO-Ze9cj-=bBg+l(5lz3_pVcD z>y>3i`xWKlT39jDQG{kWmsr@eTod!S4Gv=vykGeT^uq@KYo_oBH z;`V1>fhvIGXly(Hd>VE)Ff=sO$H#hhVi1Sw(XS{Jy_IW_NLuRwO9g$$2D~4XdXXoi z_Ji&t9pnZ5hNs9ay{V85FWe*WQ1AN`V)eV9AhMeecLZjqxN*g zU4wUln8q_CAoNHE)SI|KY07m#d><-Uy7I>>0CWJ1rDLjc$%wPw!UUNgx_pMCwsGo- ztbZ*+upC(>c!Tf+`e%#)+m&Bk+gWDq>|E%aJgom5wNdAfwwvKJld!Tm+l7UNL)*RD z)`H^vcy<=`>ZD%G5K)!_P8%1f_DHX&NAEr*wl;r6G3%Z;p=I_SFu<9A8v^tMJA+kI zQgRa}FCsi#TU#5cKoiayQWy`pO~y||fP7KWBLg!X>FI&k+dEYEAfjcx>MMPF&CjrC z0h-ch&z`k9A@M&*m;ItrV{ey`j*(i!J*1f_t-RvNLEWb3n8Rv<;4J{OM2g>BWU?we zsloY?SN3^GeCIK)sGBE?f2N{*LW!vOgNMZ&dx@5To1-#EjhxqQ{~bNG6_H=>5khgR z+|--cB0y!L`PauN595){8vbs|F@m z@E-M7zT%&ncmDZ_V>f0&=R9Nde9-}Cfs)Jd0mH3$`yByw$Q@5U_4nuabkE7j{Xmp6 z?{W9L9RzYvF0Xj|){92d?SI1Rx~j1YsM+Yi3-34}fB(v)0W?DhUbDKXaGaLqhABob zPfydX9QLI^k;uQ;U)VWW+1}AT5RG`OZov#JSqRW%ccuAHGYc&?eIn_0X0H1iZnTSu z|2?T?V2R8xmDWX}qSf8qy-UQtBhMnNqxF=a5dnx46&6A_<;}G_jNtd9#K4-TVh*=~ zf_r?ODB{gR?HA06V*)e))MYQKoJaY6{IhZbP|fi`RG#(|F-YDPHvM4qKStCHMh+v| z7=JpylE5Z`$+*!6#y<8;3y7A|!zr@QK6~`$&6}COH$wMQFkK8wo8oX|^*uKrFG zV{ZVAnnw;w$^D3&P0HB#;DsW%-c#p`U5TXx!iBSbYvmfu73AfKIw3Sv>rLb$Thqf6 zKE63ULNtt@2fiH?r1kM{II=5%U8Nrvnv#30ip_oS<7sp(6kJ83b1H4&^xwP8&X1jt z%~tpH!w$Gbst^RA{W&*LqpGvN;p`k95%KZE2Z)B7&>pn1Mq8w$q@>P=_wSt$0p(Pp zo<_7Y;DVuCSHl!`F-&Lj{u+ku$fi*>_s+cf8|R273sN_Bkon8u1)KcO?kz1YZeK__vB^C7%}(wQ!WR2qhI;t zwn6wl8jZW!tdf56$=(y&_#`e(Uc088B~)}9x7Dx+q(mPo2+LHt5k#TXv}TupSoR}* ztZBD|4yQbdb)d!#*#H6V*?wD#1&|5VYlp@HhdCD5uU z5jXx>VLv`uJ2f8vbvllYFCrtEgP634=q2_jLlDl|Y0dWFH@SlW>$gNCr>_9#8}jDxL?Wg$^l({hU&0imS+#lj_n;3 zJ3OvTZu)!l`uJq|tg+cRO}|QAckSBM_h}RIf?Hb6g2K=c{of%nOZXE$9)L8Da#H!+ z%=}_~zG{A@Pnbz+9YJNbAOlQHZCk)W$(J|J9)?<RxJN6+XBLle6afofW--MU~Ff>0c9F)iJ?SFIy$e`?by5$RsD~%#hJ$2%# zK5SYMQGcASC@QC4oh>_fSZ2T|4obWaECp|r@kGzOXypo%$zjQ9Njiyf73Ea_dJbIh{>MaTpl0#!b}fo%$Z~X3 zPlj(_tmUjc)6>`N+Cafub#5=%Y^%-Z$S|RVB4(k?Bis3DcL)SNyr=$v?-qivH@AD~Q08 z8nY1hP(&fxMrWy(tjx+2f@)C0sctWv~RZ-rF3>w0lgw%uPhdvM8e8ejvgb8irE`8Vvr zY&zPx*G6<43Qo*gKL-v;C|RTk9Mlfep5&9&%rSnw3C)RBnV@(c?YSU)bl_bovdJ+K zplNtn407TH;U3H<>%;w*!lRakHSbAK9(dw^9Kb%+1+37rGS zkGo+1mFb$VmEvag`vL;|h@wB>xt!C_&ls70%V=$FC8(FHLi@b_edWCMp=jc6q=Xs< z4646ovv$bHqE$1F>vM*>;{e&&)Qo(7#>87&puS2wWY8cCtcG=n4$v~+zWs`qd zhR#@ClNRe&=G9J)j^pJ$%4Bi0$Pg1VL*4G)wWiZ3KO`4X(1*OP?B7@OkH=M}Prdk3 ztzDb>PYJMEJe2d7o!_4VAl2T{=H_NFf4Y!gKy6Yb8(}`Ytn2J{OAeE(7J(HR^PbrV zZWWROf`7bI&Jp^ly6D7m)QvCoLZJoXCFFJvK&rJbUc7PR2B03}x}QIPifE9BHd%Qb z2XzK&Mxa0vgs24+hFe-%2AL&-B*+M%o}}AjXQS1Prv8L7+a5GA(Se77h&5D(=?p7&lf|qYSCF>uaZlT0RsY$c#cNk}*sX7JyciT-mO;~7 z$AG`jJnbMbS~$3dbi$QB?{?wQJNWynmBX)q@cu?&#w(td|NTbN3Cj~?t&{cjpHUNY z8t?TORc$iwO{JqbBx)V;nlrtSAC$YDOII7s@!16{_{`T-H+%*o259GZ5P(6|nU3BE zuTCT@C;$M03|Uy{448`HH(c%-FHR9zvKQG49wLvW`KiDbKq(m_T2+)B@qUjr_>ArgSi``H9BXC3h7e}_j$De;!gUZY^9oC=7S&5GHt*yg z4Cw`1VG(V<;IzoR22HOSg=q+QKZ_TKDnBcac~l(T`7GnlAgi5rC$mE!>KUX_AZrM}mO%Y5(wWuJ$U$wDfe|+;INO zp6ZEFbh(a?Z?!Om!lA!=dfuxvSed|V0gThUFH*-EA9bhCTbXS%iWErc4xozrG-nCc zoq*Ve)2WJ$UYZ)0iMIh)X69F;gz$l+HPp<0WnbapO+$^Z4&Y;9xKLX(1q^8uJG$}*9Lo!PhKA-l{Ir2v~`&Cng-mtl^7rY1Bfpw zcgUFv#l2`aCy_?)L=-zeswZbDi5}h*qI|Z(*RhcTQdDl(GsL^ghAKF()d~-p% zb>wU__?t)}TXOI_ArklMs2>f!942uMNAu4PkXn7HV4A%iV_3E#E%`jXdWH)pKmyI9-Y){_{?jEUlF51 zOh>_TFkD(QintTdh5|Ps^W4rpu!gipIGETKoq4i+OD>32HY6}`EwQ|mjb;p> zKQS=@Z>^T3coJ9U9~1Ym;WHKjg~UVRD}u5N5NJ>~hU}d3{S>n6d2FmcV(ju}F8H-5 z%#ouc|Lgl~udWrh{_J}1-a3Wb=vIs%Bw@MvYczN{E?xZ${?cKGTRBpoUE5x{LL)VNhEW`3oc(J0U)Alx)u>OPDw3krzrnBry5J}hygcJn+KjNq@p|28!>Q6d1^85!rd zJjIZia*;K}bzX~eckt*P1o88P1hsk81{AJ(6S3@-2xs_?(=aJkjs|sSoqI zasN_LAcy@!(ZvX@n+Vqmsy}@aPf7O)z*1hig2g!GwDuc0C$iHTvmHKNT^+V8 ztHHcf?yDG_s?_qb!QEV!VY4m%FPHS=Ei`r|08gMK=i-|b*TWM!*sX1VH}-jR3*}Zq zQ5G^A(|B2|9+%qCZ}@&HOe8hyD-$UvE@qI&uWjnmGYn#f5C0~B=qe~Tyn~YgcOODo zM4&sad9<%6>4EPpSY4e1AfmQ)bf`Pr0YhX|WwQqc+ZB4U|Az})L|qO_vCj(%s+dRo z{Qb+e4EiZr5R{~XZ~hZ8(uV7bfuDCf!Fv3fn)>!u2y-y~&A`7!rv&Sd9>hM_MI)Wp zCAE`(k1hP_ag4`#xMId`4SGJ$vHs->9!Et8oKVNXo*upNEFIROB;U?k>(;GPxdS*e zJ|W@zrUETUFmlYmN}!^z<2cCL{VPXb?s#QL$PePq|8=n-lDEEk{aOzi+e8EZ8}vlb zuq)rKhGy3*R=o5xbU^WVz26`9Ln5tqb0MlCsD~#kum>Q#@TJ|baU&-P{tIrVn2rRr z!C<4uzn&aEm*`_rCer7sjqg%5PP1q!L$1NF2xfe$~F+eeo|mOu6b$Q!6V=-7fo zp;^|{^ms89EeAM&eYtG^`LyB8$<#D7th^W=7G)!(3#L64Oc^h)0pWSFS0#3#>fs=T z>00$sh~e18dxQ;_jXrS=;lQ!QDt9od$e|}Cy;$0fC{)qex_}{jPN%XK{?2fn<<3&= zfiGVm%CRe2jMEUuYWeo#z+WT$@i-Hs`1MgN*DWd{^EFE;2u3lfdqIw1sk2pl-TKKBn2$i@}!yruF3(5Q`&fS=oT!Yls! zwG#&{Fd3}gwd4RBKq)^)$TlUQjca~Y7)U~$F%5tAFMKLdqU`*^zrKX|R=j`sYY+}|%gf68n*!@PvYQx@_aZIFS4XvK_*K01`DK>~M{?)yRh$Q>8R!L+UPBd; zrPzh8Mu)7n3+I+w<2tGp!>_(_9U#os-^lmxegA(DeIK-1`$QPFU_D&KE2B|p`bZUc zw5QjF2jBg;k@VdSak`XycqJ2K?S^3;k|H0BDTXpz+3G|Wm;G$p{F08OO6G%@?Fi28W}=f-B3U>t{f^gEox@TY7F zPilFA38(;}d`R?ofoI??4SB8xOzMR)F3J@g#6+Jp2B@@I1GLlC)iqU6>H=sC^q}P_ z>iQ<<5n9EX@SP`uN7E9tTXRH97z|Zjg!};A?}6{%QJ&&xUqR|My016aoTRkBIt=4L z*Y{D2@7}!|gCMrcX$|&)w1GhfZ>n-!79689yGO_P>%x*>-(38~oXL}&lam9|v0@F_ zxM$9u^=xxo0Bm+78Z3lLR**qePwg11-DKAUda1&(W!Md`d;A`K-hS$5z#9F9Q54?( z6a*fbKBBR|Bpkz?>qj$30Zj4>J3nMpVFFGa+U0qLAF%%6#91t(|qm|YJrP0 ziKxVYd><8_5QMOjNG%UTu-v~eWDh&Lhj=J>cJb;#kR~la@*pCUkcacb38`LgLimR# zI*kK;M*-R|>^)>a9lC%ZwkgG|XZI>c;_SoT$nHf-Fjj zIx1{&$nBYbx^o0w-QvfOA5D~Hvb&+LfXgsmHVd8G9>WLY0aOHt-3I`S{VDq#(G@&} za7OS~H!5uC7d~qN?1)o$E9q7JUiTm8fM~Lx1AUW)x6tORdrFGgowb|dd>ywngnB`?Rvj1~ zmi=|b;V8-V*lF_n$B2!Te4Ii59e)GdRHX<1iT6dks91%odO1xtMLC`-H}297PX5vS zY&`gO&MYC+4lnk1a&XJD0xM4_X(@Y$sG`4GQ13?ZgB5iTk~PrYTmq$C{(C8n<6yuf zj1$~WR?vdmE*Bs7GVGXWzf1MWY2=uRCz<5Kj5R6;V{e9z(4Y^X!pYHs-YZqjw;vOZ zp$sZS%vZV5yGc;f=+d~dX+_KB9?ti}HY4LHncv@~r`Jrcb%ZbbV4?=TNRx=}aj>9@7LBYZ2K4M&s zrqOe65A{u?OP{uuPJY28@#sB+@AK1Do1fe0OYiy0q&(hNCs-HoGg0qBiC%oj5K&%i zud<#*_X7adGpPcfnGBl@%gW1}YS1X)u9JrqH+?1CZrq;AP|~X;8-7H|^inhE9Z4aG z#S|gV{7d{rt?;(5%r-<#je}h^X=?ydl5k2ywOV-3AUaCHKn~J;esZ<^{rgtLZ^df*eK|h zjq`=Mm0<;MIo=n@xiqU|sY3Ae$_mldq*o|u{8+hkKd>=X-s--xmDLD(8jle}T1`jc zM8~h?5WNMdMHpKM#%x^}L>nX*XHt27l~;5|)qb6XB3B2_CN7M!vNFI8Vxp}z9X9vz z_Rz1oi?ytam5v}Q&ik?rIBzlaRPW*lH6S{DfRlhK`LZf@KMY#DFZ8I+4e1q*jfdU0 zsm(WOIXgSka(~-?dv|7@me%l*8<+>WD17LU4;A*DmwP?Qhj$Mi`{v}(@Nn%Tp3h*< zMAqyUvA=(yu>#EifOwQ`ZgDl{TtOZC3kMkrA?GxB>de@qXEcAJENkase?z)^~b%)m=iz|B+ zMB4M|B>DXs7xhy>HYeN#WBi|jKQ!NYn@x#_2YbI~k86LKpdIFtDdVQlaciFGqo@!A zfe5@Bd@#CIp}#jX@I&23U|h(VJ0Iz=D&bN%Z0ziI9B)hrMA=d>EHMpaE+n4;2ZoRX zTnUNvtAu*Vi9SIphaQwo(3_%Oczq0b$xU>=YM)bC0wJX}O)JQ-BPX|YjojnDi@*Zt z{{J6i#-IR%K`UsuVBUlaDui~(SwCInqixDeHhd&f4Qf{tNFs@|3w28XZ-s@0_SK3q zrV27mSvhNM;z=Z`J?Pz+@8x|0(U+!v2tqpQ3-e2?I(t!bDKoUElH#-lt;O{kXa${K%e`*zNANw8gOM1WkSSe5aXIWH_}H7wfc6o zDCeZ+?AAPx7ShhkgXo`%(h&+euKuRYyTvfLu^ZIWv3w6(zifj47u^Rk*d*gl(W?(G z03bV-DU;C%DhIF+mUt+Z78d||0!(@T2iF<1sNaoUafelSTZ1|_llCCy-3Q&Bxa0;n zO!i%1iOMB+rt6jfyh3v}WSO!NIyYAdZ}G>_0Z8 zZ+j*IG|PotB>?3HJ1zmRT6uZXVQ7^^Fn{*;n{kv(5<9=al9~iMV>E+JQ2{&n^a+I! z3XJWBsqhi2@^o#_TthN`(vS84c5u@$`7tq>Lr%xdKgRAdQEI*NUGVB~2e1&Fsox-* zf(lRfjxuE&?B4=pJ_HCC+&W6uLtnqDK!0AZaAGnBuQO3WK~4cp)vfP*8=CG9e585g zzbFR)sEpR(_H2B&gMhJ9YkuU4&l1M0?)nr;F%%34P=gxaVFtLKplLvxxE*yr^DCkC zGDyYHYx)tCg?C3K&ke+k zg>{7Hw7QY7@%!uhP0Y$gNR9O8QFiTu{ML>q8)oL_kUX9@UlABJ!&lc4_h_vO!l=CU=i1x}wuInM9Lspk_ z4;4Vx_Rd?#GJYM_h`ccwqHgro*zAfsXP)sK@yu%j%B=0}1taP|eE2{?-Crcb7ZeSo z*R`)-_h7Rq5pu__kGkFr16;l*v;OB8jaJUW^f5kHp=1_!%qJVvZjY1Nk-0L}llkHa zC#}>B_F_p%oWvhLUKq0)hs|7&6{O1COcFf7$~O- zsdo%67l=8jkmh@@KpegCmpoTKJaIewx`e8R_nPzyQi|evvYl9?hU|;4tLu8)w=nMp zrpQC;j5!x9WWm^Q2jn4@3WUeH&%Ak-g{6XFf z90}ORKI*+htD)E;s?dDm^%XeY`}Xadesl55=g(Uqrx_XEsX)fa9=E(+pciT_;s?Fm z&Yt-i)8ui56s7$AfnHTptR#x^v$$24%f?FbrO_{rxUU(PoamsaI-eKCO$qMVw)65QJ@NfI15y3UrGO+Yl0IK zjUa*#t{AKz0U9lWAq&{<)5~kTefCf--Mw`DJK}P9^n`BkYKYXP<<+3KMC9=su&@1n zeV6AaPbp(P5N$t#Z^k2iYlx(KXC_}inSBNGtM85^o6Lt4h%`Powii6hn5@2z^fY#m zUg)gADJnzHgH&BWpfH}MEmgyB#5!*E8V7s=$QG*v6wRmj=nq|bWnhPsSMSX#cs@R0 zdT#E*;i#&Vj?1**OrFx}PLN|_T;qKi3xui=ugl>U1I{1c=uTRN88dY||B}l*B{7{4 zEvSscsJf0X^_;Xcm7D`SkV19?7WyHx_!E9&UToq6!01mTR+!KbIGlfcH-hPA)Cw7= zQKm4tUFDO-TxGD|}$RmsdV%gTzdTB)t09)FOA1;WI1b^$GNRoLf-Z)NBhcOqw z!twW4;1vQKtuT#6Jz5!`#_O$#Ge>ihd}odM%Shz4XN%B(c|KKWKx3SMs7EMeVzav$sICV6je=yqOIz$=Dsc52`n()!Yj&CL&Y0KOUCXXQPw6nH;PL%r-|CoTsXMvHr zekWexYE~pr8M!b%lV+C8Q#8iff+CEu*h*6<8f@&SazQBgKU%J`L+Y;B}-GRd#PrSg(u7>~KPi!Ps zCS_qK6fK!la*UhML6ynF9HZ3u$CGR(wJ!fe{sPRRQP&fs+#WPDTfJ{rociPFmRIL} z16(KCvgs{|=|Zr)Zh4VsPwGc+Bix56v9P;-euGnPVhiBX7$-uoq45n}wp}hTAQGvo zNVzIfWb{M^)KegfPtE!2DYVAU%n?pmPX*{;Vj9$}^h`~kg3@IiW>RA^XF^TcMKX4k zrc5KQa9%2b3--8jiDqrBI(9 zuzJvqi(V4`$Wvsa-H`>KsibRDFf8_GmP&uYoMm(`Mg&x>Cd zN?H!E78Vw+xa(%rLK92e<+w>cDBL>WyL4gFNM8H`m)d(G!a|ACN5~ezJ0;>`%#XOV zc#8yXI&6wS>yA>#Qn~v?AHKlo_C9fCoJqw8|ggm5h3pOWU<^q<;Rjq zS&rL$!mT_FEEc!Yd2%*pA68%5tRGssiTSt~2Zyxz>Q3|!0u&Do3ZfgSzzopdAO$&O z44mEl;J|CAW;5kj6h6+$CD!c$m40wzoWn?&o%- zA;g%uYbpmqb;r-xVgB~yU`*Z zy#j)Syuk*uuK;L*VjFPkoaa7f=17D!%{v{DnIca*3iVwV=e&6oZUEUAAeur(g;mOf z%ogrc&)o6q0A#j6yO66A(man!Yj-bb{fand?>;C0{`-clt-POj0T)WHb73ZvY(u^D z3Q+w3w-#b8EYQrx=m&sc4XZc=s17M8Gyq*iSOK~xp(HvFbB2(h7}VS9!H$j&vBk~f z|0O2CLxps!!J*|~;MoOlAhK^q#4%@X7L-Ypt?qN% zZu%{8Y~0Szn8`X_nsg}PzmE*WWmIq9#t5~iXhOTia4>InPHIZx!1^pEgU+ zFD#@mHujz$=HCe3Ml?4((k2*)EW}O&1IBeaPmmMl0u2~_lB^j025#NX>AS*EqD~U; zPk+a^0!*Y=EPxOM)1ui&0GpdkV!=P4A5P_LF_1g|$G{7+5AI2e0gB+C)hZhCX$~U^ zw#gs35gG*uB+Rf1-~;tK(sDO-yw@s_%XjSF?Sf8)%%+X8SDHgj7CvnIeQv?in?&LV zp@0|!l1YrleQ%b5u#Q - - - - - - - - - - - - - - - - - - - - - - - Application - - - - - - - - - - - - Storage - - - - - - - - - - - Hermes - - - - - - - - - - - - A-TOP - - - - - - - - - - - - A-BOTTOM - - - - - - - - - - - - ADAPTER - - - - - - - - - diff --git a/doc/design/img/adapter.png b/doc/design/img/adapter.png deleted file mode 100644 index c1a1eb13e216e8875143f0d1241ad2fdd9a79dbc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15781 zcmdVB1yGe=)GvG#5dj4SX#pi9lt%E-k^*vQL{dRYN)Ih9qS6flhc0PpkWvICq(PAs zI7lcUAsu%e#`aM z#1TRGKXl?K9JyiU$A>_O+bGIj({eRhOmg$M@oD!x+s9+q64aExs}%I!^P(t!R#-`- zl)D_48e~XOQ9du7pYDiC5`>c@qK+bu@N)c@f0$ZC+l{^Y z$%R-q9jUZ6iEM)d8uogKo-jVsch3|r5@YxDyINaY`4E;F52oLRYpUv^=ta-Ea=n@* z!w%HQ_FHHy3|`)T6GX)mcn0Zn-XaveyRq14fge9p*WDu?cRr_! zo>Rfq_39PVp9_Pw!{up`F|e+s8sy<=CEym`gWN zs;X2?CZ?vjfwUijs(gHW(&d8WWo0iLRR0*ScPESBMp;|2!G#4HS%JmH_BH64yO*r$>Q@; zf&nK^p3E#8eC70Q;K<(6xZ7+}b8|C1B#uM#35i~=M%Lupr^c@w73r=1P7^D&m@98Z zn)~L`y72a`0$sj`9CtIXFmu#8eUmTU-r5pdZDqzHW<|I(2l;EM=L0d;9}74%GEc-% z@fcy&FIu=tI-X#CqSximavuiQ9>>$o(>B<6uoHAN8FUo> z?GIP>XO<66bw*EOkPaR;n5z#S8F{U=P@R=xi%t@C;iT(xPg{v3z8Atq{gA(_~odymDr6|MuS*9P^(o z5xk^%5#zq`JuOnqZ51Qtra_;}GP5*RJ2>Mo+chK3&dM!serdKff;l(StRq&5u)o6k zQ=WEyWbFDToqgRkX6#UjssD4Y{hi)Q!9{FbQ7a7vg<0G4YcD3v3Gne(2p>(*TkXAP z^nUSnEAwOiO&+JTh`D6taLri~r#SP@1a-W}*dpzGN{UbhaXrIU#amT&hE8dNv77kB zl$m*NpOQKR1VwyqS1gm|yj5uvuQ4T3=a8Vc)?tI&5;JEs#E%LH2CgCw7ON)o1jgxA z%=}++XlAP+3rqabZ~w&p0;Ql(-(L9fHq9JVZU?ekDVqHnNB&u~2FbarY7|?fmjwz* zx3h4Mng909S{=EJvHn!FI93~XBfBdz=?6>HJKdsA9aI8~gU^^8bD?IDABi3O2e`he%>!`7BgsjZhW@b>ub3_iyp6j z;5Tyq149`ZBjZG-BKg7fCu#jEZIP^8ORb95raPj>4IWB`v~?a+^`Wx@Q(MS3EPijFqpli+GV}b2!RV1eDNSJXB}hzfizp0YSC$(7oQtXE0TGl zYh&6{<1pL&Dws6z#LTn~s$Hj0&y1veyv{AE(MwH23eDG7gK3r_6_;i0h~v4bL_O>s z&u{jGl+kN*X)HDMe4yK6#iz&5Ncb)yj>lN1=`pj$P2KOFddkWw zL5ux*B2>B;ayC=lpT27KC#G^e9TXHq{*6C1t<9tsJ-0BJ|1mg|g~9zhUkl`*k1H=q zofq}YWT=?SVxqDW_6Qe7>>E92A(37XA9tNT6|~6Y3;CAQRIkx%KidAHRerWgA{uXi zapcFY_Qj(gFDs1JySsc`M&^2Nm1VU>u{jW8*C`#B)u__6%7*0eSc%J4;f@ai^CeH@ zYUM^27xRP!vKU!fvJ;$Gbm8OIM;2CEBP>ZN^);;~Z>GxQ~$wgnWMS;>EslOk;pL_Y~4`?zI0(6Z-3i zcu4G-X))%N9`SbbeOXo-g_sl6t@B(ET*#*xnI8hbMMx?9sb62u6Xjm`CUmoXXLuBKlkKWce&xW}8 zzw?I{ab%U<7^m2+w!QW5pUVJgLD?g_gy20_m7F=1jIp1LH$Yh44i@b`9~B|M&#&QH z8X2DpK~zy=(h>+r@Ho@TI*JB4CgSsc5|u_CsC&f6_j)xRV8Ww>F7P(;=O;<9$6lXc z8TG{=sC^%c0#alSzfS-!dyH&J3WW%kYNBr?^f~jt^rB5t@rUa#hlVPw<8)`AlS=M| zSGC7N#v~lM$ifm#dxcX>OpKS87n16=Yu8i~1mi`VV6)%!zdp6KGCjq>Majs>Xly+3 z<;xeS-(1EGdjLA7%r}W)#9h0O0h>umHbQCaNn}-Sx%a-cEj~FJr572ll?>UG_R771 z{F}u$;`z@h#=d+1KA&V&*m>sc+<1NM%Rx^+n1MBkJ0A;ftqWyjWUS408~;UV+~jA@ zT&qn#v@T)7qn!|OS-QdJm|8XAwcD4atTgD0M*xGA)pksHF794=FT;BV29Wt`Z)0%; z@QHP?QGNBsTDRoQPvgS4saz(vv{STq_)plb$*eRc${BS8GTnHYMVPLI-`e8=an~QI z8gtiC_&bv6!%t4p1yXF{|9Z_Mw8cZQ`c-yZw@M!$6dBa;cZ#)K{dAw3t&lB<;*$Ik zL7gBcnx3BF7wXJ`R=w%|8qW(cVsEl%@^v2=;}*S+5wO@xR?|1~UZrAyAIYZ;-qv7J zJXQ$@DS3>>X@dE<`(*8v1*d~ACn&uje%hKp%yc;~S<<;N& z%$CG(YHKt+P0dPe5xVzz@Vq{zL(y6~k6zF+TsOkMr$;UD93;`k@=I(L+rt&s6M%(l zofoTxzP+#8B*4vwfohp~=iDiP=Q@VE`8r&j=foeDzv_|0bkI;z%2)95@tMd`(h6Ei zMF`G(jVtnqnQFIl;J{rudMZR(LPA3Dda{U z3Ee1YJ!&n6#$NZbx{12y5y2#98NAq=@l?ON=z|k(K^=sP8xLNEFC7h{^_*KS1>+4Rmpw9`U==E0Te<2Zy!B+#Ga~} zBJt}Y#Se+XVMjeYJ*`~NezUoOx!P+|D?K-x;;ln;A*U8MN$n7otFzsz_FUZDsDh5$ zTR$c#M`C20y)RBZvzaM3-#hwb9dP&POh;VI>f^`9idI)v;*k#7_lR5PD@DTvs`%Yj z?*=VyGFn+#nHU~#+~1(v<+U9aEa~DaYE=woluY)Hkv{yGryWFD1+g&oRz1JeDCHQE z%MLgAOR)QMHm}@mJzc^i5K)=m#-UqqKR5`jXFH7roXhCBuI}x92zWd)N+nS!tHKhE zE~O?yj=B6dC$e8Dvqq*XQP@6|nos(&)2UX%4?d4YlcNPVIObvgB-}R&Uc^9nhukZmcC3gP^jcMgIP=bW9qw-)92{shxUNh=E~RCm5D;}*aQym#W{_m`$xMN6$&^0Am846iYLrh8 z<#l-!7|-L?64Un@Zea}KLq_S2^L>zEDVtcnppLvgSv>;}Sbb|tMo#X(y1%=rZ_};( zbyF(Ps0J%vG zu}8h=QZJL-MZ)(6Kbweoo;eNwK1{EQNfqyy`BSPK4{R-Q=s%?&%hna&+@(ew|az=C>j1Vv-ug$z^FTfM9qdPE*eIiX1OV z%j%kY;*lRGmfGOfpI>us+}CJ%wu}CKXIFU?ve~od$%YW#mouXsu8RQ-MMoYO7p47r zNXCp{^5~E`q5w*1|1b5WMyd<8#-52x!TnQV)V)(dS-CRiGC?>KPVUOKWL5`5WXj3lY$dEC|8(G4%66LebA0gzZ!M7_3UNY*>;0-gxksk~w?3Tor#cWD3Df(YA@hW&AJ1v% z=?fna9;rKxdl-Y{SkeS77aQsNtwx?Uq+sQC#V4a_J^?N+l>)ri`0Ng@dzl7g-!Mm! zN!N1f2^d^`gx=dwTKcX&Eq#V=D{P~}I=-v49kCpF`I5KfC_*^xa=HQkPt#e4I1Etn zEZg1nd1f8Nmy6MI(aO>4CjO@9a|A#qp&#A|5gtE%32~piovEF*oxP-qFT9_oJZ8NT z@gky?wY!z(^ywd|ySRwsvW=ZnLwsk7P)?4}B*8^lRwo%fk&ht@x(${;a_*${##~02 zkOSj?{+j^Wq?54jQ~+S|uJAv1fM@)C;{dOW%-U8&s}F!DgX-7b64*J5q|LCfqmVI zd;l41wZ~+W|CG@1uTlRzAKw4rIySNw-LN$kRQ`0TC5Z9reJMjP7%@~&O>KbF12eGu{(V@w94m}|H*${)}<@a{sLLfZF^--pyI*vft_jv^lO;$)(I z!hGMovswq+`-fMfpOTd z{$b348Ir5==ZQj<7mx3ycptcN0v#t!3KF&%dhI^+%E=mJja&~AL<8n&XlT%?w(7sg%_sRQj`tYn=T^mwoW5=Neon^)@4~gS$TGPIho01H>!l!& z#c$pqcqtJ8?~6i~DjgDxwFpbl+IF?#&)<^0&bsgq5^(df7` z=OO*M%05NJ{I$;!!Sk1D`EieuN74_r|Jt1k*I2Nbxg7NduN74?>CWqeGf5&&>C$@stIAYis383yTX_|~drS4p4S)9L3xWqO zdhUzES3=olXf1q?E9U9dI&;~#q9OAL^)K;bZN}?rUWI>6shiErWN~u)RAk6isTxzT zbS?>l9Xvg`ybQwaE&%eivm#U%B9wn+frV?MF>CAd{pXgCnN7Y&_w~eCxzL~^c^9PR zP}eS0VC?Ak0lqAa)|92Bq-e;=SUWndgHAQLD=a`(a}1eu0fMX95XocQ@L{FML#(J4 zu1HGzyt?!A+tX8YIu-Y{$2TMox5KFK>cEB;KGyL;E{4;cCJLUTfIA z@fqJ!YcV=5p~*3%5?el%opC%V`eDac&y&?)X^=Cg|G7j~8s+797pChm&pK>;wQ|c| zpmP{emr!N2H0ow3D*xkq2KNHMA5F`#poZ9#sMu+0Y7Wk1%Q3}4IphlH(i;3#e130u zGPFkE8r73-@y$_?NTZv=!%uSt2L@^geIP^{iizA(;sIegTg)T&krknmzpBJGOWbot z+@?cK|4A8SXfjaKcywQ;wm65NWMz-Vzm8kQsn~Jch8j&UsC;XM!^x+NAku%}(_&y? z=%S(!-}>U$E-e04fhi9C6cZ2-&=V&l=KRm|3lq?q8lsg zLM+08!dxvu&$N3SHE%xu>EEo5HF{^Q=t%Xs#8d(9;YOdXR%vsqwvpcf|KHkU9RLd&c5&4lP{{8{KPL62lq{~iJM_uIN0s^;`o0B}n=%6RI z5>+DVvh*Y42xiZN#>XaY4+mh--<UM!Z z@3}n_2a+dfIaHmuA#H!kP#_g)2krK;{spCs`cBFHbxiV0=R9nleHN{Z-JXbW#oG2BqB8Jw*&y9X+ToFtJ_@{g zsDlp_zy0RXJuW~KyI&LS1JjD`NF^QtD$X;ZjAA`^hiHm9>CM2ZcGie6-^SsW84;&O{wE#~5 zbSeCCk}?yEqqK)eQP(g>V8SMpLO2?--Y=MLejM>E5w zotG^Vg=`13?}3?<@%S1_(^Lv;m7+bCGec)g_MS@8HPeiZuF5y08XS0N_0S`#-bI-D+ct~q&=V_@cS?=pVX8~vC zq#ng{LfUpfh_ZaJ?@xcTx%(}VcJ>W=Q5~C!&$fRp|Jw)sy-or~kJq&`2fmo>cNjDY zgV=MuL9hXL68Eq-46E#tsAmzgXcBsn=sc)X+0ieHppN9^7IUiI1vEi9a+jZjL$RQv zOZdp|8?#h-f{I#_;KU`h7I74xNDaA33e$2iI*NbQmw+Kv$M*lBm0xhi+2Z0(3j+Wm z*sa?(jJ@}@bEk5Upn52Aln^hbsijF%5~TQxY^uC=duiZ)7CFBRGI!quR3!EHjH1rL zMBvR9i;)nc6KKiX+t!AF)NvTQ_Ppvap<^=e5DH#XwYGXv=Y8nqBv$;!k}Hq9Tp3JD zxllS`R6*H-Z=1_u3MKnkg1*CMl17gli{0HVs^@;e}%Y7D@6*qRdO+S+hqI{4!HW|UFfA1#BADEI?G817pYj1DwdgMo|csx0r zGXM6B91!7~|19Vk>>ljVdDi{I)eb;d{zi5~FW(R$9f-3d1;oY0uU>t=_U?>e#Pn?| zWB`b$>b5|>wr4^2u=hTD#mZJ;osEW$+(vnaOc{(O0q~YtS9KZ!E{J>FHgt zuzRD0E;4RRri}th1=>-76Ak0pGfnT2x7u{Nz(mqq1lzZhC&u0yGE4%Jm)v%wGVA#- zLr4g-d-02yZjtySOtV9`xB_cQNy+F1Xz{_KTjND}V=#A#uRM8jrKIUN7p>o4)!)z@ z)8Pi08IrdizxSMo{kRzC%Fmx!lh-MBd7z?|4{4-O+M6Vunr$GwOLw_9xd06d5TRs= zzuTwG(+gqu{mZZ0hheC=4ZavF0~>%bpJbF61#h1Hn?3))5V-KpfM|`&%ug9pUl&C> zd<_O;%=#W@N1znz4Sznlb> z7k9dN)X^=%_*8yl_uv5m&{%PQ4=QEeS?R7(UpH)lQN2jT-PZ66V9&hBJuCml#Gj}Z z-G`7a`wXOqltFB3t~X;!h^D7n5RGkBVDucAn2>~!#!B^Pt5_|@VKA4Un7@;op#J1% zXdh;Xj44ucKY?^0_!>d(DgYMKf0K*)Kj+up#b!EsU)qEuf*-W_<^s+WZt{~2?I1*i zGZV~=f`;&XR37X}OO*0qAO(V&goZ_FhI$iw3_%TM#p_HM{SBi3(?1;EhYA!W1dH2n zMSz%!dvAabvfmj-nRx%9A>9W86m&v1YK%*;OrvB|AVHYO)E{qkGk&nUVXHI;xJDze z_*U7IRWDGqu6ec3rcnq#tcTpW%WR|ik+ z^C5KA^zDv;z*oNsLh{1X^>xP*KQu+Sz^B(!unBDemPS8|4h7)4I9el+Wfekuh27fI z!=pLa`6(2D`x^qMXa#Q*JbUqi-FbvN??@jFn2}SloVapwF7$`T!D9vJX_%KIT3SZA}k%K?y8UOM5hQZG#<(DA>~=2lnE`$F{xgHJp*1pqG=C z6~M(y**dQn85t=A%*b`j*Lf`ah*$B+9&8GCeqSa5}nXL@=X z%u2=WolnNz0qz7smV-+BWxxqL6NNRyRO0z(fTm?hu+Uk_o+2Tk6xEZ1+}P?MA_r?O z0gUy>S zM-0;##=(~YrB~4~!P;?QAfPVmjP~AgiSFEYD0C`BsZrAqEe`7#*sig_C;Mq!=6cdV z)6DuAw$oT68$bqj;^L>`2@re%c6q`dqodD0aQ~tXO2GPbm?UVt!P&KjuN>XBCZ9s( zA3O}890~p!Z;>1?WIGH}NXufy5#&!caq%QdCs_A&@7z7Aovp2bBE!Us*TJ_u(^Aj!$Poa8JcBSWEVwK0;XeF?4Gk)sr@$v8DV4Px(>WVTkW29#(Z zj^teYEm!Xw%UFK`><3K|bD%iAO>)#22$~O;XLeNzHrTC02gb)as`O<#H8Zur_DNE zsAn`2y_Z6{8M)0+7^(NTo%wd-%ZUW--qAo!Gmv;pD!@qd>{pvTB;Nnt2wGEq1Gn#Apo}QVG2P#R^peHm2H0i zUSIn&lif6RD@Rz*Smqp1Z%dMl12aWS%Q@Jchx0ip2M4>$@m0q+385NJz`h49IPr>g zwt<^|>D@MH$zlKqOz4MsGWDX?N^f_2HpOLeSbOW1%FK|ML@U9H5jqhXzEG;eEp^rz!!&rul$D0o&OO6cWG)uZOSX zz$Szp2`6O(sf8Abgr8trH^QEs@(yd6!ofiZ>}1fPpo_<_(PMbbrbHUWb=|Qlq1I)Y zLBivCS9Q5T8leWHYsd-w7Sb0V`T4btfC9O`F!&A92CI`5hw@rJw92||8!ljEW@Q&r zpyw8vdo;s(+ah!*2cB~2SDqv#?H0*UPc5^!e*J`hL?Fu|h&fS_k(}zUd+BPlW;=KBKTB;z!>!YY=0D0 z2t7BMxKiykC5(pyL+1VIq2n!}_PrIBPY|>st5-;{W)~*1hL8aZ3?NuDNvQ&?Eo5x~ z#B*mAcW*(H5kQ*6=g-+0jOfTdA4SeY%-L~eW@l>{;p})gZA!m|gqk-MKD##$=?Zm77%dfuuN`LLrzO88hv#`>Du;U%sEgz%A_U`l1(flq21?cK7=#UA> zlul)w!PW`VL3WjtDSa(;aalg}6EOdfx~cq4{^4>tt)Pb3&G1%A*b1^Pi8Kw$^fH>( z$p`@`&tC{L4MErV?9JAr#$%Ul0xsRc+cWRO7-wTK;_f;&mP>d44#)0G#r3Dl0`D$%Wu_}34e-%ZE`(=DIy}mqhCs7 z2RsJLUgepA=isycseZS-<>~?S0BV!~7Xw<>1D`$nGUDH72cqI-*<>-d$k`9)9C8J6dPbLKHij5hgK32I8nYEfeX0M*)!0CX-s4WnhQ`&Q@ckX2c;W zS`rVYWVzEK;zJ|ZR2gVpYaS2xq{$GB=je5Tz~X2BP*!FiAWluV^K~{7PJ`0B0c2J} z{{(}~{0MaEq8>at=8BC_U@rvAH~yzYmGca^Q0Zi}`&z!<0xnAiL5msLfIF<8UUKZGSK8?(4W9^W=r;3-P@zp$ZIC`}9bVtqf=eIRV2q_qzQNdXI0rqb`s4SJ#LO0du_{|aEI13uJ{@3t{o2`qKWf#j%L+D}*HhiRBz|$5ZoqBr>n#B+wA~g`N56r> zwgHMcAj$=9qG>Cx&R2z%@p&!I?mr7i58-ozBso^L#m6Hu7QXLpeGwgnZ7I~xeZ%ih zei&iF^>KrEu=g@{du`U?jt+8jd7^uGuQrP)L0{u{hd$bQVL3tN$2IFQfb;;d?8fV$?ne+6A4s-I)C+o5Dt45gMN5qs~o>M zHK;*5o;^tZ+N;2r<&8sm!&y#%h2Yzm@}Z*BkS76Rgl#T(-B&2Qw|A zxG+mRDjnn^XUPiwf7ay-<#PQd!%to}b{(wr-&FOQyHKwpU6G?O(0{eH_4>jcTCD%- z=(E2id1z)P&Nw^S<9o?_80%YB!R48yEdM2tyZN8aD*byCi|*tZo|~4Zpec958@_P@ zF|}%ln-0FnsHaX6jrfBc@D!jE#;NHs;pF6`pcW;k7CTtFU@b$BB>6P@clDpN?n(CA zu_`(0bGh7pVmxv?TLlMZkSD*BwAw?gO~GT&XN7WIujcM-#FV*w@&wY!?<|hub>7)h z(z+-pxDH4k*W~7XxbN}+@4oSUFn%F-;8YG^ryeRy&?WSj9t1lRwB||*7Vht@b%R6H zUiD5LzP_`8Z#01f+H2kQPVK#bwR{eO%|(x`T`ql z#@PK5wmYQ}K0gw?{;s8^=BHx8CE(&)P=tYoOWtXE3}5m9^V1aH)lPj=3BR1>wHd;F z+I1Of3_t$^|9f^9KNix}?;k%!s|5|g%eSMsAS&xi}n~NhZ!HZfI&;tS(E`D$Q7;^a%be?le zg5e1Y5$FOToYBv|%Y*Kvh^MDgf(Or@{yt0h2ulPX;JvX01(>fMpZ41jw7x6!(_#2b zkJlUP3HQ0@45qfVIN-Mu9{B1MidnH(-s>d4ZT|MjDKX!2WgDASi*DiQ~03ig40KpR^xO>nb!JUTS?(QxL5G+V=@8It43GNc4ad+3oVNTxP z`@Q#@sXI0E$DOIW>d;hm?Q_oVwfC~;Svy2QP8{PI!7~5=FeD{Jl>h*_2LKS7o+5(Z zgyc0YfdA0!B-9-N0JG!oA3~39z7YVt1tdj3s<@@@FZgPyIJXKO<89zp*Denw>sudT zTA-` z%^0ltp%Bm0&F#sSIlVIDUfdoO%eNQS@mIJMRDXW)2oC~9e4jV|gE4jF1mQ?G*?D<+*YZP)cMd&Nrz}|EXU+HXCGn5Gj|*Z09~gZuTgG>W*DeQs z4Mg7frvN8|RKk4xU*>$tUSCW>=MSAG)ylnZHj;Y;TaSD%kys)HmS@T!7!{|4mX@c0 z;zS2RE{^w|>eQSE@Gk6`W5TD~+h1A6k@mw}05m@5 zL&K{_74!XSx2}T3=mu}w62ev=R)ICP|MRuvcT)CAGcKZbDl7hx0+PB!#P-r}%x&gX z$0}{(8WHRRyV-;-QZN>w;52dr<8$9K2*RIZ>@mHBPKM-eCBRYG$M~jgB{mly$i!uo zeR@p^xFHEiXsD^pF2^1dTs1=MEy@Rk{#lPw?QkCmqf@o|LpqQ7ABhzv`H`YCmoW~< z>Q-Sa`dHaFpXGK@CNr+fpFItA6J?#Iv>zC|kwRkDC9s-NPY9&YE?-aZSEc?(?~Q%h z>Z)SfPH?=FX9Ka?C#_T7qZ$`ddpJky4DS-27iQv25&sZqv+{HDCW}-yz1B|KJH?bRWEG-^wFQY#3(O1T5ooe9#LW~^YEOB#YjMu=Y z#jzskA>Pb{m+*|Nnwy%g3jHj8K2Sc%C)d<33zcPQ;A4dIa^-Ytq70r!^RZgXoyO0O ztI`Rmk?gIkyog5t6gnRb%Ebr%?Mh9*X{n0UIezR!@A{B;cHaM;iVViC75t0<-Evtfdjc7Z!Xm)M^i*-W{V(*=O#wbthjJGP^egN6?}qUa8s^e6XUeJS;Ed zru#^X7VIs%*8KBB*I6V=w@I?&Cl71qmDS_rtJ4vEKJ`@W*L+k(^(XAWvXkND)a#uN zv<>Lor~1)mN`YTo^a6%WN|wXCG_E-G5k$mq2NZk`jM5040}&)Jv78^RskF_#er@lZ zoeriTbLXhOW(>nRORKJKob)k2i|?aVCB}P6^ex^WqN$XH^fk9{FLi^V)sjJ*B1~7e zWD?1v#KF^>a4_(|tZ)fAU?e`mzUe&Tdos*G{8lcsZa-K+dYip}ta%3^E02jmf&J;+ zPe@CVUd5p6d2{A_(*f71ExEG<38-d+ScV0Qsn7Es2t-pklk|CVZL590v%tS&QcHbh zaes?F&ceciZfxi0+(CNty~DqL{Cpj5=S0jVNN{|ijBY=n}<_SY*U+3cgIL8mVtCU1bl`x+#v|%^IPxEgY3}vM@d&o{MgC$>SIp1Eejq4`MN2sgS8OpGiX+p5;!%A zPdZvfW9qi}{=e~GH#evG^8Ea3JAdIS_luuuj6>9wq#a@Zc6xOsaUl4n}isQ;W{(k46D1hvw=i#ZYgruRhnx z^vM7UR!c=yi>w8-NZ8Q@7Sxg{{4=^nZN>x}1G=s<+{kSjiWPjULi^Kdhv4ww;DFzt{+P7v#p2g56yN1=jOih}U&@Pba?^I~l@U7y1R^M{4#`6m zueY+5`27;Ccx z**0`ZF?mu_Qs@hvT&5lJdYT)^`+xbb#xKtcilwRw=e?+IotzRoP13irDnE}yZ!%F1 za7yzlUNiQnJa~WVW^W>~x;Gl>1pGc0v9;BVX)#dvWja)uNbi5*C3v5d7DtE#Od`j@ zXOK=ZyX;@?p{2*LKR#q7S6J@H@ zB^nyrkt%d$b~46UV`8^&+j<)$eKJ^+7kv&pb})@VIjwrh2^ zcn-Xk4s}UhXT|U%cDT5inN-15L3-1cv7Ezv8*T<^@Aq%vRNGOYo!UWo(w6*#x*~1k zReU4oLWXw8;<2osQ>CwjL%&szeb^_-0H!7PjG{-~fPXeG3QVED@Qo9J^qJoRS{A=m zc%qJ8dI8L=EPrk|l8_byR7iXvGH=AQ^({OAU?!0Dp4!zAFhlXHH_gsUEiq6LT@?mK z_iZcp94Q1Q1YDQhx1Iui)NaK^%|Y%hc^tx)_&fK}n5LiR2~mI>3)!%ZDq`87*FwiQ z;RSr0Z(jhhVWypobETu(ALy@9p!g{x`Fp!AxyffAKgepc0lnRlAJXt?FcAQe3&p01 z&!2x9l5%b11H4y-91~{NvtDZ>>3HKfpehCv!xWFXiuk6RMOT`nozOi-4! z>1S@PYL--m6aI6^-B`&O zj1~YuiVxM|#GFHNx~#dC5-#HEG#!DmyOE5C`TSzN(mjpXBA*xua20qj@ZQHF+p3UW zFWdUX9spdAt={Q`x&5}W<(=DCMbgu4VTrpaxkUi>@*+Pk%{dIgDXYUz)niD2_t!jS z!9HKuzGz2`hkxsU7*od8ka8y8o>2a4k7p82Dp(Z#@fE*K^R;gu{hWtLruDsmkNHKG z)_Y+u%9v8VR^pDC_*OhbpgN5UtI|tF8!_0I6nA3p)ce#kS2Sxrm3=7stp&t(M3V@Q z#I3GpTivqId9!cJ>7LQRQ}9zCZnsKAGcE7Hx|OBx66tsymQ&P*fD`BDrz`3ARK@ce z#&QJ`!tGhO#kBO6^m0v&d;jDPnU=QHHINEi<$2YUc@F%O^%Rsi#HW!CthuMZCezt~ zmw8nh@;vxYHlMjKQ5H*UxD}iHM!NT){N$J~jFl~ykjP}+KsU(#oh7=+zp%{?=OPsQCq9dGgZgCc z7o7fcx+bD+X&cs9R`n6`{K@_MX?vNCFW8dlC;-uTgIicuRSO1yeXxrBqRwscfba=W zxEb$Ln>k7?%^?)MuZTzUJWy_nyP~Qk!N!tL93?=E%Eqm#^r7mo^o$B#8vYt-G@(?P z9m#Ji_zP|Y9L4I;;1mbI zR#Z@~wWxJUb(YGBwElNFB{B|Lww_jR)n;{QM)`URUBi~M*c)Z_2{xIs6pDvPR0O|F zCFkLpJI5NaK;SJVyeS#~%dJ0|C{UhLJ+&k2wg=y7#R?1(y!I-<^mCWlNQxn9iaH@; zv_haFAAS?bsWK}!+Sqk0OV*^4u+{6=3z-?{M&fUY`i}0Q^aRihvrMx|8t)p5 zUN)DP^?XPZI5Wv*iPL`WcTnrMY;FKh(-*oVW3wdKt;lKH>XST|26dpeuf}V>;h>_f z-xTybPsW=iNPya2YyRp-;U9b2f2~Lu&$%Z!ZzP#N z1Dwqef4)yn-WW=4lr!sVS?J3c&rC4)z*$!n9{|7A z{tt7hzXH2ulNxFRMr&v7_NXArmY@)CmN$Oy**K1n=AJJjUnB<( z?00A4hRBYw=Q+VM1ekQA>ie8wH5VmTJ^vse>=YO0MSseC7>m!(MeI+Sju@2N9uG5} zQE{fKZ0`FG;+^LoVl>m_poyta2nUTa2H>%g+&gAzHB|sPqW$cTef@j3IwV(KtFn=f z#6?qAcXeAFf}*z)lOd1}zc)(^v$(|2t0E|>6e!m_fVvQC7fvL+moev!t)QTB2wi$w zT}P77=*G~r?S~*Fq3V;_#*^6aGl7XG!1cL}$)X&8l`!t6DcyTx_BT>cV(mv8IXR8k z^91tI3}sR;Q{~O~aR@@eeB@IwGxp4?k2%2C^yjA>1mP|jzSrx;qOo|t%XhA)^)1I= z<;fmjSlROb?A57|v8QEq&~SE^9mzdTWmO|lp|G7Qb|+h&o=uJIRq(B#`fkk-etW7` zR*d*Ixntem%l6$c1(BXAgr|qHl zu!i-^jC~d+e->Xcw^fJ1X3ONreCAVwc?7LzbWC6`Qhao|+rCNTu~ZY2jSsectkXt& zMN%$0l`wo7zJ&a?sg+l|HwHhv+iz)W;E^njRMnL4g_n`_aN(PR&YtNhg$8qLsS2Lo zuqq5H{v^g>?JRMKfzcfoA*)IMotNGO7g)&{8PR&M8f`Keu@l%|wkQhWoF5%{al!HQ z$?clz>IQBO9**jK-#}0B|1Z}P%Y(vR$rRgu&Tiw8!J4k(2VXv>==XT9HI!(PjPY|} zTNX|}1kFaa9IJ*2v>I5i83YuJc*mJ&kJ47fKB_H>7q_hF?;K4!4~6PF3Y94rr)GUN zXH%n)rK6QjSw;gsgtU7UfMa`&*eKm?@fO2(Lop9 zjVxHV$jwy#amuj@CnyQUsR+eUt?KT!n$}_8`4udg^oy#*BXJHl{1huCxUj=ai=S3SN2Y^H*HCCa$bsc?)ox`%tU{t%M$}r_Sl_w zB6-i-5#1uBsqZAz&O981-GV#WWnaEbkGk^+FU?`SR&!aJ6px3%Zp0R9H~2L_-TkCD zS=nrXn(tK_$*JnezX5n@Z&(rx)pYS~%}rzn3JCTrIDfrF1bpy+ROK~fXr(jH=sp3! zX`!7u$ad_ZAzdfhsNH5XeK%W3ju@}0U0xx2V6*l-R z7ng?`Ym4|M*#7at+Apxyx}<>!@6(eGEJ41X@|YXA-2Zc+cjOAj_;7pF_SjiQh4s7k zKVU2u@c)e61`Yg(8%gJHG8@ahT=YEhi2wF)w0E1}hg41&E<-YptmZxTO2D$(UmLq|$IZ=+Th0iQ{c&tz;h0d* z<$vbUqdpGvp!C3sT;lP<}=(rCD7j#7b2VH&TiPDy9zKyiP_D=J3qMv6G8 zvVMzTt+5@a4`~q`;olJBq%N-3BC%X(wsX%wR2$G+$jb1mv448A?2qMhwGss0hw1XM zG{pDe49dX3AUI_!4(nOz2tp{wRTM-6O`ur4s#MjW%UD(HZJd;@&-DhKF2)Z!A0Hn8 zlZcyn8aX3n;qh+dF<*tY6)pjrK_&HiJKvL=lXIdPkD_K{Q>sAdPC@j6J}9_u$;Y#N zz(KZHqoyz?$G9(=vYGPzrPX4ydcEXB#lxrV-QB}Ool2v=hx^;WNAPDo^DO!&c_hmu z<-x*=6nr>|WK9}mHJ&O~Z*nueB=G)+Q1YwPgj9_74)X0QHM!ND=`~l(E+0|o}e?uYX zYikDwE%s~O;w0{!tds%*4=tCAXZxh@tjWBhtp5w=-`;e9X}fN0ZK0x~YUb9%dY=6U z-**~s*h=^0w3)ve6~R)^t^e^^|9`yxli=g^$l2K$5)u-FvdK{L#YTcIToCbp+#GY$ z#pF`~z8>7GORO8?@`4ACJxTw?BV?u|K%q<0OfBn+rvLep?EeFD;lJVZ|DCybV{$z( zK0a>JU9OLb{^}nid~u(s-RRV3#tyIabAAZ;kBa_(q(c60X8ZqwXL(+AXUtAeVye~Z zb=`>fUv_QAyx=ltot~cFLa&!5sQ|j|W<>tGLB=>H#>P%S zQ$R-fmn32kb~lM-wqBg@Di)~F#_cx72H03A=gQ#klO;MiJ8LuF!sl#ln+|G012if} z7lVy_$7e%ndg>I@KlE3K_m!{__+Da4I&KXU?~`f0dc4~bL}-T3T7o{d-F3;^b>~k` zmUtuyzsCgYNd)iV)sDQcN6<`3-QtH}%Ke4Sp%iekPG_ggI?Yd(r>RImAKWp;=5n-1 z{OT2(L{4UkgsbZ%L*4wJg$4d#$sTgl6# z^1EE}eGJi+cbQS$*vW$~)>_YPL46M`z6>NVVdZ~DvKH*V`tHlHs@v=~rpNr|ISPm| z!uHy3cIj3`4|h93A5A5nKL2=CI!`?p^sij(@zZzToAaH@%F6TebIaEtNcfk&d5Glj zQB)km)Abs=Wd&z*Iqx{K7AbhxefyE?2CjdXLiGR!AVHJCM19)BXXzUql@l&YR+pP>Hv7r(yjfK~E|C0vV{=E7dFu;g+x8Sgqq%VXd+G zK&KzE7rw_c>83|TmD?;dwzs!0&ZUuX+bx4i#k<*SJgA*3HGLNnoGa2_5m z+a8AoyvfKc8BF4dLPtfI{df=SJC>Gw`(tSy9v%=75Wr9HSGrhN2hy#ZLcLwU*MkAn z{{$o01{f?Hm1Ur&&Mz)5&dq&KMYVOj(&6F=W+e~g;t^&gM1nqll~+Um_J_pX{e4e3 zq3N}2XYdQyB_Z~Ug=Uxi=?eW0lmXwS#zvG1W>Hc9J>AC}SYHg4;Qje{j4Hi6#!>67 zjj!+HOqHpo19gP@%RQeTfnHK>Zdd(rbRF&OrAx54jR+S;ka;ipok|_`#I(Aa%$r1# zRbZz2DjP4xfaKg>%O}4Dwd-oi$@UW}#e$Akp%t8(>V!9f<{76A^=zpP@}00D*;+I! zZhvPjK2Ok9>;!+w{jXjCk~f67Y#ahKY?AM5 zb!B1%_1nV;@{l=$tOqZm`#rA>fXdw_-O3NIw5I6LtI(y2a4<^zxb^zauP6&7$HvBU z!StWDe|Q)-`3F+{#grx(J^wn0pMT};o~_5}df#}SyrvSvE&QOWS%lqUA|KEs`_VfC z-{}a4{JTym^SuMT!f*=Lh=-K8cu(&Sa@*L`f5=T;$-A=FoBq-lp>laugE;ATfk8o$ zgj|PU!br+xGtYlHKfX2l_oZ$3@Gv~4XTGK;CJ%?MZB3sxj$l?G#{Nxz;XGdmUd|H8QZFUQ=pP+xm+1Si!2>g+ti5vOwQV z;Wa%5mXAaDjj-)I=N!6MujRJhr09lWdKk%dyBOa}B!Lj!a|^MRRl6lgOM?AfZ}B

V$Iu(pSry|Oyc{J@l# zMmaaQYZWvYQ&7#)@$@sz?q{Ga`oi~T=pK~>NfIu0U8ndS*Ruhpzfy*p;}~6+|6C65 zz-lI~$*!_o?^m{Z7&_*58e^UW|u;aCE`(x~XToNekG0#Qg0&`rM})YU#mrH5ix?niYeGddU_smX_zmkyn^ z+Rj;g69WU5a@P&52`zATcl}N1I4?Z#hpiiorsF-ny3x4!D$ZszBWV}EKNf{u<3 z2}$dh$}O16hb64-QCgjf{ot_}6NW{H(JryRlfrpCD_?XZ!c?PQP9GLpy>G#kdus&x z_}A&Admg?B5sjge_gM+V)A2m0VQRfvT?rzUlb83sJ*X4B+0H`8W2~;Notl_vzS%7b z2?>$sI|(q9>WaHPY&t4!I=mT&KAfNQkY1)MRgkBClxga?csSsOXqQt@71lP*5)+qj zAS3aA#lK`aJGkvd`=wHos(P^r6V|@(B_ekvQZ7!y^ASnq+L>k5X*Ozh+sTR)*d_RR zcvu_SU;yY#(++#V7|e>~u#o%`qo-_mks<||zaN?0RdHDq6%`gbI5>bBYyj)AMfa_=c+#}GI#&sa-l`D+u-2Ik4GmrEiEMh&)^-b9IMt23 zI=Glo4#1T%c?uLJEM5FqO>C#a!fNb>oLdSJXyw3=T1QQ zK7|$?`{7lx%FLfv(wFf$9*wL`nM3MdG^JOIce5AZ=58Flkox>OC;lQ>(Rxha5gBG8 z*&Yo=d$bGGzIMv8_8z}2!Ebk;xhRyv`G?DYY=H(u4V%L!ep#|BV}WKLs)jLb=hsCwINo=*@*k?W225B+Ovz4Ypw>Jm{?0ljIi$}6c zaMkIXgs%vnNw)_PhB%0#WU4d6h-91V_!Z~v zl2IrdzbY@1zMZmC94Q8~pb*^V`{)BAENW4ZF{`I2%mQ*aoePcvKcY|yry&I>Mm8@Vl+^0JPe8dx3bM=#XYuPP^lZ#}; z6)DQh<3!5Oo+7HjJ1T`-@;64Bsc(erZZguHlggYrqF=|;Q92IWWHx6kv_`=-LQTVf z;ONV0UYeup#&ZRISDvB5?f%tN6*ttzO70&Unwrnt4T=;TQkEt-79YMJ&EnzMgWjRO zglcx^#Q7*U5D=1(%Nmb5*esljES9nip`=-jg6yb??NU}MoKCdOnsmR2hL@IZ;r%Rn z9letoOE4fR-OqcVoyL<|6nhCi^FFxEWQZUS)&%A>O|&Ov&eylWM+zdT7LS&VJF4@N zs$HoFFT~cp%mi$FiWoFsBMR}s+!yP?Mc7klyvCgoD9WUJff-eWE?D_w0>X%e;1|M! zp;~04L5Q+K(s#<;J@aDD=C&G^lk_Uo9|~c4v>TjjzoC_=%1Ah*{90_=rm3~{QJ9Ne z9Wi|=Wu0A;JW<_XwgIWH@biJV=W3wmT$(NEc)?ULR6P!Kxk6<)HAO`O4u`M0a5z(I zL><8J6LM57{LC@H;@jwXzp`o%=lBzez){^za+GV)eD5#dV0Ny(R211Eeeonp&oL` zHGfnfK>_r4Caw7`E#~O7bk(&pU6V*6Ldl6E^)l;OP2JZ^OV4s!Mm+=BOMd2A@Tgv( zH1uxB|N0;?H_XlAR}nHN!17qwYFWmW#x)+u1xQ1nOM>kZ+UV?-4v{rxwtrfcz zMd`psr);Fo;O?ROQFZCK@%Skvg|)nTa)*9S(_A!6qEZma)fHwMg3w!=ZvVZEAs7&? zTB4Oh((|>|Yt#?+klw<^E^o5qiDp32YrP`;mCK_WLy!2jwr|Xd%lb`Zh6~JNb1*46 zIr;AH4o}DJBW+x07eAhA$2Sbm?TjE0U5ZOex*p~|BI5TrU+oG*3imA;@D)1|oO}Pu z0#@g8GH0m->N<&bJfO-LZFtudrmNglZV@p7Nnk|rOC*M4L3iWB^yi5B@JsAgDEXo* znM$so4bIf0DLgC=|K^XtkwVb>t#g|fm@6DwwOck0H}!%Ah?h;_PnZ4`c=84r@Ue`O zyK`Jq!yWK8fec%vpd|VpV5-OYKq%CHNSh!EeU9JI7gt+w0HSREOb2nRzR|y}s?2V- zUgMQ^*IheUGWFVH>^Nu!lLj@CM~#FwCXF6~)$Q+OeT6iEBC1{0zW#r9_em@89#H(a zwRiS4pGL~a6x9wCvadh+4L1SzBq%NKWuLswrw~|N7X*b1Pr9<%e$Zf(L-EOx#TE-jLfU_xoyjh_>E|n6vmQS7Yz;VkhNOQ z8ef=ooCWIJs$p*Ka`s~c(W)@bfCIyu8AR-n1((d0zg9SiDgA31gMex-@BZifhrB;W z#i6oQ1_^UZUWY~1w#LhL-E|R1nZhoA`>7^#5dEyGrhFbP(8;WPUN=_G08t7qQ}2Be zShb%Er7k!Ul54p{Ea|x#W_j}?0v?C!7sexqE+q<0^mx!0rKM~P!)d%~peQJm8}&s$ z-jqEOa9G4%xRA1$$QT(JNl8iR>t`qx@VOppmCd!;woIh^>@PiZaS@}E5$#%t$YH-d z9l0K@Y(fx%Z@q#m3e4e}wK+?j8S>S&Cw7+2WfQSMAER;8Af_ zPWw@`57Te*mPts9aK)7+>_%Gk!w@fp!Almg~OYv}m$?56DxVQ^2 zihL&(d;kHc%gf8ih={+@bBH?=B?k{Hvi2i=a(qcdW(ij*LsXxl(uhpfq zbCCEw#HhB0^G<*Gp7<91A`JSWuXcb{KE#*WZxfwUoN28=!(CKiJ5-|+N7ZKcR^4Hb zFbrDuqQ`nr+UY}+FHRq1{YqQN%;onfvtwV)EUvoWn?EO*+M?k|4CyjtM3AY zSo;ry_we4J0Ns}JxpsKKZuR1^kZP&>L7QR##dN>rjXz|pqj6>|~Q)C2)JU6pwlP9F;B%pqKgH40HL|L|%JP2(f&{i7&ifR{GpI)x8CO^11+@k=J-;hgx7iN1|KMa<>JQ1Pl=?>`(2#?DUj~t zSXCukfGgXF5G1$9$Zn&-cqAXG06#CGI`?xE{?*dLV9axgeCZ6a-fE=tnSNhL{ki_FlrinS z``6>87zU^mQfCx0?>t{bT$c>;PcwdWz$3hp;MYWUXHOc*@8=D-#kU6R1n|B*jI6w^$m7FCACJ(<+DY#251Eso_mc9~RoU0c;txk)G2x-<#z?2+iv$j~`;91%-O zK-~{&DBsZPBM*#L`>ZsRQWL!VCKn5A!1av1^f$^@{qiN^`vm2-fxrkoR?Fk^3;P%V!l81PUtfpYsX?XOB9I+ zg>f#x>eFMtq6Y+m>p-F} znCyLXZaff=gNIi%xeJHGe?OqyZ?GAPq}|9|%2C9XroKa?GQ<^jy3+UoAV`XDIl zOCBbb2ARk`bt)daya`0iw{%%dcN%7mAZb5y5)C;0?TdPX%*ph~|`TlNpfVF@BOOaF~j zqy6~w_PoE2q9tTV4DGkvx3(&0b|xR68aEpqIGfM#p3)<2^ea;w{G>@R;k}tyPXCHZ zbx_JXHg&*_O*lGpU31}!8g)vd1yTh3U;+i^Q%CF-4jrZtfyLM;l{rm}FRe~v0b6yh zeM(Yt`y=u9?bk{-3=9SV|Ur{G+gU8y4D9WkNE)>No-Wq)Z1Vf0g>HXsL#y~ zxU2r=FeD@-V`F0=*0_7P&;)|E%2{J>=i6h%ye?pn9!%vqQ)|1KJg8ebbh3PI&)~a+ zs5iVm+B*E|Bl?4qV`d5Of+V~;Yt3uKcUF!DVL3$TRv*eB&TQ;B^1#0jPXhhZv)ELv&=`^)|epymcpPp8|fPbG3Ld@N={AOi*KQrQk z`|@RUR8&+-iY5ed0n#Kc&dwYh9C5L+pw7VFg^zjT*7NoDEA4@-N&8zMznaKmm@5x` zSPf$W0n7_q-`kYmqo$WPBH@Z>;F68`w_gwb5s;}*qtiK$R&^i;GPbcm+byU~e?>un+wBvsM zp*vRu2k6AYsJ6$|M(rLRfuyYFQBuE6P9=v@R-|v?$0%ow$L9o``*0%zW~aD}%3*7# zm8(Q$uaOT`DY8ySBy&81@l2Sid@Hxw&BF`vgV8 zn$2Xeej{EJc>4Iv=D0X?r%#lL%@Wd0G&Dq~AOb`>|!#H1y(ZZ}a`n%GA8WtKlQFoOtS=ZH!M8vf=&iQ9SAx~SSl zGS;%0cYgCq*rvl)s+gw4!KvdLS2Z!f=xVk&UA{sM=xHcTl%>W`PG8OT%eMQZCz3;{ zDh9W&W}5?hBFh)ETVI~}U*FGi?g?G+^z9`h1aI2@=#gf|zL|OttOB|thM^qx?bOs+ zR70A|v!f#>2`;8yY+SV2Nr&VRHorF^qE~IMqQdDaPayqwv8R!N;&h+xzi^y{g#vBa zH6i?J>1(g560Q)CL+mYMO~V1Z29dMj)%D3CY{|1%u{cXf3Y zQl#3|*~xeO3u*1;10ry%J9H0yEGR3(3lU{%J{JD?@uRl(LWAR0>-A;|NO_g#=bK%h zZv0&!kb*(E8u~_`cE}ABv~|8pV}R`8K3f}-Bnct;RWFg394n5{d9{EJbPsq>LQVF_ z6HO;(2^PjG*Cp?{u`aVLcI4C$q9RT>iRPnE=K#D&XHCt>-xJNhEas_~f^sh{%NcH1 z2sSsj7@cYxQnz-T^9J0VkXUr=_{%Hu5*8?HE6kN0OtPp;rw`*hKAH}+uC9iD0#;Ft zzqD>!Knmq|ik9f}6Fi!y0Vw3K0TDEU$~9<`EV^tn z#Ri+-Fz%^qOLLz2xyfrNm}J33&4ubeXDV|XMsXcd(+|^@X)X3Fe64uLWY|IiwpwN- z*kGGuo1Ie0xQ#hcJeMq>9|^^tN#zuYmKIL+9{!%RGDLmE+0JVRXLmD$`5*4kqZrv4$%No;M8z|$0< z#Ky)lXx84r>)J{R3SOMja5<=U-5ISixP!)v-D9(YSU`s;80<(hU$-lzV;y&3Luu#QH95vZW*< zR-$ds=`G9td<|aSI($EFb4!4wPjdMn0=c$F0xl??b z2hWB0+OJ#E$IcBFTI>m3xSR(^ly@(3&XA-_P)X5bXK%Me(Ng)@xV7VBDEE;R9X8RY z$b-;iy%DIGsuDl9z`vqPxog?50b^eol3pz4rKI_^$ElVe`W+ok_)S`rDX1^2=(efm zczAyExZTXG32iC5=?&pdJ(1lY)C;UWb#LG+!o_9?-rL!uHB*#bLux=zkD&^%k}Rjk zPgbcE4roizPohm`ETfSibleF8No#j-Z{37q0n6c6EZ2|a z^lUcs^f`BQ5Aj1}z{fg5zq*sF-0d7XOAN3B4dG88c{QqK zUG|g>jjloixnO2OUI{R&ksPOp5jAs2-q)-x=xtP@Woz^g41j2~rhYugP570t8163x5)(lam=Y;%p9r%$>M|dQW{q8N zq{Hlr_A*ZN3xpQOFeq2tPFN5DJm&}JsmBkU@xC1e7+t;p#+n*@y0q{D5(_Gw(2ynll|V@ zE-j)dsnq=3M#r&Z?fp5PtY!p$yV*{_B%QOe{ilFHp3CMYO(j81wOPW=vvNt?st6Kf z6i>-3@V?5<-J{c7EAIU5cnWonfnpqMlN39@_=Sf<@UeiNsq{(M z3v}r^(K5`*_;UwWJ(aeXq>r^EtM$z|xu{61;;@8FuK(%6Gt%kp{|v4PX{ftwhB|xm8TE7eQPib#z^CVE+{@Qu863l+<+z0FQFr;7A$5>TEvcyz6qTFZyx~7$!iz!SaXY<1VicvHWb|)_qn_ zgdL>NUwNiuxrOE#C+r6!OyHxV%V4>7{vdqN@Xalu=x$XEH0`y8q9K;cPg?nA%@@;} zqeL+?X{=U02?-g42XVKVpRf!gElzhw5Z_KH%je|gww`no24NC|1ZNq&2yah!Hzpyc z)#c#=ooX?dg?595`BD8^L=dUZRomlj8?0D8OM$vt!+MFXepjGOgIl|vgqBf<<2ZLY zz49tt-%sFZX1W(G|{&HMwq3JYgxWO~}9*?1l0+w5k4=v+2Iy290u z)`OVF#FH}G!hr7imS0+4h)e)2FV(Y`|B;J;KhV803PWD8;DN8#sMY}moFdK zh(`}^j|tK7pd?IW8S6U}6FO*jr-9@R-5yqEaI)#IDE^QgAlEkE{qqN*)dFNg6lI6f z2_{+e-)d?@EGE9DH6q~sY2JNzayZgZXp|B8Qqe&&w-|O>O}iOhD_;OiC%wF}JI{?= zxXO7mxz~=guS5$7^rjqgnV)!;3=3ow&KpVTUU&?Rqu!ThcM9_Zic%HxX%jNyx5(;# z-1N6CWP4#B+O~%%c?|&XoiH&ewYiQ-{KvD?FKQA1c&cK4X(*l)<$eDEm->CgFoZHd zMzdf$y0mb@y9jbnP@tS{#=BH)%@})K5wjUb(q^{;GSD}FS%W+O?Mx8rb=sFBoq+rLHPHvB?Y*+LhtmP3z1>|-+eJR`0?$q6C?);) z@q&<$@Xed680g*l-?|uVPSycC%)s6@d4cQoKRME{QmgKmF0`1f*U)d&rJpYvFT4Zu zOrr+mowdu&Tk%tslukSaw~G=2!;ihG`R@9TB`?FDrk16*D*w2B0~BmC-abu3Evf~c zN`<6*QL0`1Mu$(BITco;!9P-rzrqEWUvs6~ow688D|&sl8lwr`il1ofGbIp7p%#8H z(SM>xElc;e>HL{UXz&^*9DaxeyzLDSov;^D1oZAIr)j*WR^%Wi0qn}gLcz_Y{hO3h zMX`0x!ab7xp1spmrdYtcA_spq&er&;GOAZd%fuDAS_0|;9h#eb++=#E)!>08bP~Fd z#TCk>24fcBDnuO7jA2NIVKT^gEALywWUuf!p}YM@a2~pfi%*}g2QO6ZV%Mp9Lw5Sv zVg_VoIoB(72D17hBNrB*fz+fm^j~MUYb*zF0|CDbmyL0UNkDvS&&^hS&?1Kf`w+Ux@YG>b}?u|v0BZut%R!&XmU4}3n8^P z=i>W<2VCSVXnw5{-h+KTTF?{c4R!NFm0nww(;cPjk4W$PAx-1OmMUk%&KoFTVMjCY z9UG%VP1oD`c=z*|WSTn=f`|No0 zd7qu`6MVaWONoRdJ;oqmfAvf^)#mp2z3t0?7rH{+j9!Y+D3|BX+U;T5e%Dwt-%%yK z-kDcVNH0V2Lom`EPJ^GXnos>0lN4wv!%)ecsUSTF_E3ou5c?X4(n&rk0*U0-89FIg z=;T#^^^?S@su39pVWnS`T89m@K7Zx&7jJnt;#I3rX=s~IQ=Z829293W3OO5?>7tdE25uIxfQ@>{o1~J{jy}zk9zaKZiuh zo&Lu;^5pvwAu+_ood>_-_lU&<2+c{n9_?3R;`_DiW`gIkY(5vmA5BhVckpN)`4Cpx zo^vd1Z_~$gM@L74?V+Wnc7_B72zet0GekA3t#JaK1U=4wy{R~A3&w72Yz*L+GB!4r zkmz2xMVui4@@mlEyr!8@lThn$_kRU>_Ni;i zac3a%A-`^c_+)hB3E$H7Iq%va*E`;IGsvSpTJ2jQij~v=1?^eg%HELC43Cm0K9&k! zx2wSVLt|_YemgSGCGs+A}JO zd-m7fr+b3M9ap#a&+A2^Ue@t!tU0vu}ho9KXB z9%<`x>2jI$&VBfa8(8{kI50C==|?6Fj)EfD4Weet&T)@7cgf+%P4IM*d}5t<|44Ac zF}41zws=7EMs3HqcH7vVZ*h<$N7DAJ{^^{fIl58qIQOejDRFV}_1mT3rHG$}0=X0) zTPG^1O<(=H5yH@XUwJQ6PKW}ECRL>-ADp?66V&zl`+ZCtqTsI|1tpJEy@?TzB7*Y# zg4e|tzC`G$@|9f#!Btc$_qAY#EK&^3H#+P%Zr@izxJmF|;IxVF(&qJDpqJMTSnKFH z(u;lJcG#P(Jxch76SbZAMyLt5tNBcTA3tY@SjfbCa#+BbLx7Hsje_j{!ucU{**|Uf z#ea%>@WrVq6Y?`%GH7}77pi7NU0hsTSXkHx2IRs!)66`s!^gsO?Vqi+m}oyr#(jb5 zS`xuiuh$xggkOVX+1lC~491B#5`t#ZH*3SmplJ`&-*+>I!(X-=xbRhb z)HT!SyJkPQ#DjrO8N%(LbNRZf3el_WbV*xak!t#JX;&Z+JebaH2M!N4wq4MXx@*)l`=0AJmm>A_s@eHd%TkRIci^!}*KdB)J(oY&(VAb!7;OG!#eWh-uu6{;oC zYJdQV*UgEVQ|!QcdCXr{Z-@P21q*mjn+;WjuEP%>hK{1p`71fK+G;+uCZ8ahb?bYW z%?SCo2uCOsGr~QBDpR7jT1G?1roCYY4AP5kr*8LGq-W%qwc+kERAp;vDuN8u1_m4p zMN{H^gw2cZN5xU;eFsqvE@WXW4vSTfn#EZiO+r%i@OVVM90GFtQK1_6YNc?$_apa% zH~w5G@dbBzUcZi`Oiph0WVaYzYYuuupT;de7hUal{mThiIWb*nakSPq;sovc+tdB) z0dyK0`4ZA__x~xVdz8PXr0)FdMe}Xg`HjsM<2C&{KTFXkG+E+lC1K@XcBbobhg^c( zt&%2>6n>{&&&4RGxmmX2VTp|dq`1R4D@!J)9$vs$J1Nd^>R;=OX^Vpu{haLZmds>f z6jzm_x3@{_%wh>*-B~rA`xH~b%at;UMg3np%oYaYapu9!Ug#^ly7;zk+Do3fvva=^ zEs*z9^#_wG+TS=y@!yb}NYZU;Jk3-ZMqP%+T~8*o?y;+j&LJ}XyS(RdL{}SZ?yQOa z_UZBMd3Alr!ofZj?A9S+!&>yQq4M8Q#`K%*$G%>ePho!Rw+$Xlw;cz5;;cF!sWp1| z33rz!Z9t$4UQq`B692g|#{D5NMK4!J-;<@)-HXrfC2U5fnTTwi_puHb83JHK4x)d) zL{Fz~4@zATx9HNQePSOX$uu#)wM(*J@e^JMLW7Aa_is)6uHOvoe?ep+u4lbhwO(fq ziO5K*5j6c(25JC4S>s!t?H1heLd^UyFQdrfdLU~r%eY2Ke>UU5r@BtK zUoZna939N&MrR*cr;#0}uB@!3Bfd468#XR4{|WZ15AL$MhCGHsXWlN)Z7MJM&V6C? zTN}M1N0S?92<@MDB$RxjT>O0jOU}X`CVh0Uod7S(rp_y_$?frLmJ`EPhabM^dS6u%>;Yu`9T#&=6c5?n6m3( z%pP`=(v>h2nt9wt16`6iRb`R0>i=`*K`mt+8L&TX+AR`LZdRasXiYXfxGH4$kpVZN zxEcnq9BWU=9GfdaaHm%EA#7Q^_9#f20f#tc;BW*HiTgs1y{A;|aoMgUXoS^rn9G*u zr5(GfwZ<9Y@B=8c^cRN09R2AZmrcKqQ|SiWC6f%9&N;pHfso5%HrWXD(sye}4h z)si?IZGQZ{7ko5qNuYR!g2;oNj_wM2OiUL@I?Tp>8L>2{PLHQ*A1i*-Jr|#Minosf z%{=Nhi}P)$JEx@Vaa(u;b`WfYKNV*TVktHc%MA(&Pmx1C`{`-h)pTuQ6%G)&j%QO8 zmHVCz|BfztjV!!e;rl9;VRlJl*iGNqt&C>14-;?8VT*iCwxp#l4vjjSun>$cVD15a zv|IezAUcKJ-yNqrpTa+)J0C*GSi^MgUR})Lr+D&YEmG#mqzdFwk z?h*v)CL*vLX-3IK_@l}rv5+JTHd)a%Y^(6vNW`aa@GsFRU%3_6MI|&1{y5ub3#er( z+0g|v+rwDHcH3qLe0d(PhbJ`%#JJcL3zeCa)(7H2barYCY&HuF zduX)Dc{*i5=K3z0UiiSV+l3mDE}T4*tICLFF)dA3NU|-9M?s9WR#!*JKS*nWs_Rkw zyP)78L0&rVGfB?Yi%-x@pSju-JUp?07nte{U*m$6v8sptf@p-JTjw`1;8zvM`-kkq zj&_;lZn}b)sV!y*VX(bzrrtJYp4!p4AR0s6w54A>Lg(0+MPHT3jn+P7{&SH@EOka;?wWEuN zmyjX)rK#bZna`VJX@XBAHUuNVH6TBZ=_x*(dZH4~Fn*o*o6XxFe;uRA6I@F?4n))x z6cnP$%!p_`$QXk)*Cb)=7P@bDwRe(RXZh<9?;>5>Rckz1DG_$*?gAJ>gUT&Ry)`@osOAD=6rlzrQz<2eSbjpmub#!~*cjM_akqXhhx_l#hTJEr$v&_I8VlW@=@~6H$ zjSk~mqkI1g%{5PP1>x-d791R&`rN#Op6)!)qC)RH>Yt7(=O5^PM86Jo6Yn~)S`L2c zvCha#)Z<=ytzzljHsN%QO)aaeD&VT1VkRA%S@Uapy|74zl(wZI5|1^Tl3z8@v4c2ID4h3tU^<=s?gRaZ>6{4M<#Fx2Gnq^p zCV1g#T7ZuJK1*7q&{H1x6jfY(y-)%+W|-@Nt<2&(8XjV*VzlmBi--#$DUR9eB8yvd zYA-vxL*M(40ba+XKZVD1@HII(`6*juY3ad7E_VxPW-+2PRv6JTJ z=BP~8-;Cd$G<%PG`4^q-mcF(fVGzVTnCD|qYt6f1*ltuoh&Gk&W_?F=%{p2II>z9E zs*2Ik*dJC!jD(HnPc*M^`;$-oj*V=*+iV=JqbCF4xwjWW5>w}t2ir+DJZXfn@py-4 z4_;sv36{L;IoayB)G;S{Z%=_~!2PCxj_nHBQ@~xb`!@ z`jQZ_gc;Sz8=%{<-4(}~_(PUS&ui;RSWhw9`BJaYx=L}*Qj0l8CSJCGNC>J}$z(`X zgw|Ve_nGF;a}QxFx2>UNDt`u!RIcCmqlLSbGGk59H=0|#$$cIDOZ9ZN!p$O3yZH7= zzqbb0kH4NpoZdJR8V&C6*SfX#I}Vs7C3;R#)^jK;ucq~@@6d#BnmDCiNh(9rMa z#Wx{RX}!=+GhCNyo|CBavlF9B$kJ0S6Cf@^|^Rd<(=cK3pr9l z39twp8}~@0m`c2r*3>3Ljtb|JvNYDD{E0THNTJY(INQULk96|^7UqYP_h9HvcC6;Q z+lJ2_r_voQz=B9yBh~3>5DT>P8ZRoO#Yk&r&bF?`)B3P&@fL>b*gV-7y&G|^5xzjA z5(gwjSxT1f0BPr_5`+Jey51EN9b)IioCIfyi>|4lKvGOh>`%L=Ub*=;?e!nTQM2GT z^0VaJ=){o1_KSo1R^#9RxO-FMm)rUUB1r{@$ygk%b6SSBLMO2qvU?eX7$&rrFBGnf z0*|^CHv?(j&sMX-w4B?*-OrM)2?%Mj{6Nq;TC@DP7_yYk(J4=G305c zvDMDI5)-&fW42{++}5epdtIeZrekyd(i@HMNx)!sLS^}rf;|OvI2;`63gs5tUR9zE zjh!x`=0_(wUf1np?ylPR*+S^f`)W^i7C2uvQ`kz)89hHGSsK!}S+ASB*p$G@eZ^5* zpqRDhtBePuJ0OQDp378beipbdhSM)qF-^Ij&~!r>L`p&;3&Y39XQoB;_UksD%ls@D z9W~s4Uf@1D2pSnr2_P<)J#stA9W?ARzzO!H4T&m&Y!fRQ@UC$XOUN+ ztQ@O#KAm2&G1W#$so4Y%d8#7+p)|2W;G`@wQ{?hUq_l-D$g``-A?f-kDSat8NHtw8 z&AU6marvdc^08Ibd>BU|xQX)nv%Z44SJ!{^4>sx|oN$-2OCMTZ2QFL@-M`L}-Cbz* z1`2k{2w=AV9UnhS;vEhSGVQ91VU{JKIj8;M>+`bTIyI#2#L)yX=@x!qEJQf?_u`A8 zbKCVaYC?cK|2F}L;Ip;MURHM}xw(UD`YOegl zB^+jL#jC91S^C5tY|msJhe{}4RYz9x1UdETPJ;+##~&;7>Sr0t{eF*@~A~U znd#ZlLcK!SzM;dc1T?M7{q%80sL1^k-P4o9Eu!~7>v>pUbm5DHn^-b?Zr2@07jNCG zZ04G_yYg$4b5;Zd4B!6czUrv5_TX+B0GdtNnMA7t#kXf+3d*ONT0#)gE;%HN4)aOX*xyeR=>l4UjKb^+SlvUmGUoQTZoo5duA-cgiXe- z-Er8vdB*J@SCZPVgh&rhZGi;Rgz#Gq=#o%9#;vVn&-G72b`#rwE}Ga$t`rvt%^y5( zlc$!`g}5m!4_Yp~TH;TNpO*N`8jWE?B_*18Km8RBv(DeFFWa(d4~jSkFPOvrynvY} zm(9GP8upV1(5Uw`FL}AS-`JlM5)^Qx>m^SKf^Y5eBTlv_u1>b`K#iqG?eGGMh8+U= zJ_VvkvznEawOG48J}IeN+D2Dio=>zmJUm=p7F677C?mfGUgjOQpeA4Z>#_M-GzUaj z>3C;KNzY>a_}C$*EBf2F%U9S0!8ZXl6qt z8>E7jg@xtj>de{JcE|||*&PLC(<>G+1heszfH-I{jTc1LoNN8m`K<%j;RBsWzHnCiHV88;AN$E-kVL* zG%PJI$3_j9hLb>6VB+Fv4?a)mxXa80Ln_?l(glr}=4)-2A8Cr_=jDBTCnGKGFS6+W z6nipBeiF@(j36lZhxKs*M($+f0E9J`W&Nn9G508v2%YN>? z9?DmuW?X$fI6v3ueg;sux@CsJRoM4TJxTaoe)s#@?M_#I{rYu#qNF?vpph|A1Ax+f zt|?ktQ8B7t1%VVjFQ9`AdVzB9LbVc|`T6-BtencD|6+^?P|BW{N8?msv(Utvjmx3Q zcV8HL@sJe40SbGQX;XiJ7;$U(>F$i$>lY0G(gs)I2P;5)pC95wB?5Z?_hC*Y+ZmQ0 zD6q3`&ZwxU(DlBFBQ^p^BR)Q~0Fkw=3_!PGN^a}J$;8V$-O*t@`AZKu3r1kB2%K48 z<||+RTNE|!`u$t@;?s%s|6u`Mzj#I}n3DQ5TesPZm0FustqTP#_3s5JknbT;QA6L~ z`T>#J#*|-BuyjfoZMHj`D@VHbl2IfhvUAyt*4c-13Dl2%!64Q7_0boM)L;%UD08e^ zQsqxKM{?nL`T6#$aEc6=3YJL~rlvdSCzaJ_LdWdgzmo31$kw(NB(!3=)17r78jJ%E$;< z?Z6omH_ijxC#*MtgVXpv!2QGoyd@_6L{`)&gZvOH&HwM$jt|H+B(>c~|y`1wsy zDp(KwOtkd$Qj(I2bJI!gC-MN1Q5nVkTS@`IfWRESc<~+F@nsFziePuta>QPp?f)kv zo;*MbiRaIa>j}Wt$4~uw4>C=n3_7a8E~Ci@ZZk796t%m}fE5geUZZ4VV*}*$o?=s( z@?o0pr}4YCG|bx(5|-zBCnqL8dGHy0@=H4m6mS;K`QL;5je84r()YM1n_W2EvnyT$ zXaVI4CtTD3U=3;X5DT*bxeKV2z_)Ztqoc8s2zr1DnNpCFlFE?h29;Yme#u~XK=c#0 z{2>Igi`0Jm1?K8arK>@9c1TnK<7aF-|wL|C|BDo7&9-{dfkBO zyaa-6Qo&dF@5aWf7cYLx1_HaKCET2s5acf_1o8^NPZ>H5J~r^S)81_NGh0o*a-|AW zT3a6tz^T*deJd!d?7P-}ytGKi$Y>PE>%0d>qfNgwGRvSdd|Kue7%Oh%i2eWMUE+8@ z+%j}?bJNz=c6J=ObVqG=pA9mAMgUKZyo&ET_v`ZmfP*mXp#kA1gLg7AG7nMyXxcQk zgAQ6)giaaW1H0_5ce>IR>mW!|CIuk$ovE3tfITHVBt&04X}n)nR(5Ah)O7b5vz&uN zMRT(^AeyO6eNyopI6x*lKj^Qtm`I#51Y}!)tUpNsL=nJ>x1QzRGo<0=<%QE2$2bq-Cq^i;$6c2)B1Blvci&Bj)(K9O9 zK7lpFL``inRb~W)yiVQc&nQVKl7LQv&)md>`o)uZmt}M(*4X(*cMBFxa7c_Ra^ikV z{i$*?9^j|}u|wyO_{qshQt#_1{G}=?!}b{EC>Kb z^w_POn51OA6Z1ex{i?JtoEO|B^HNrRi5W)76|P=Bwz@#Fw7$01lkz1x zJ$)dYb!-5zNR&++z9J-~mjVVMX`8qQ@@1oTU|S51?x`?6EJ~fCM@v+cdiSmaFxXIy zOefu~Jn)?Hc3(Pz!orAo9cT!|Dv36Uq|vi0Dq``shO5)h*A!9>1NpVQ+&V4~@9V2fWCWwz>ksrW5M7oQCQjYp zTiJ?cc6MbgzT31!NExOW+vkQw{QQGXoS|iWT2BtsG;TMY#Myc1mJn_n020^p@9Tw z(R{hdzzVgf>-9Mc+jNm8m#^>}r=1V?z90=wh-o8IB>0?4=~_S$70~NVunw}rK&*WI`pl^U7;z#Q|3Ed6@fuDUSm^&V^B`8wfSm_{FawXkegmLD zzC1Td;PazvEH5fztX_zGPl238PAnTVph&f~v<%le@8JV4VKzw~>3bw3 zbcO@BeMu4#wEL20PSVme}y};v>6ClO`TjL~3h&;{c z#KeSk3+VbT`rQ+J8=9DKwnttD37~SHUqB!=|JRS4oSeJjMn9`vOLOxcU_`xs@x77_ zGZjD&HiojWsign?`}bN4p`=!1U>sv4;Ckqf56tL#;n)9wtz;rnT3t;Ir$PgdVTBYs z5cW7=_GBS1a@*oxK~RQpb$4HA$T#+= zAU;)hy3z+%Ca2OKs6V7iguPtGB;D zK1sf+rX~@8YcRtPhzIUZAAxw_%diA0;7)Tu8r*o}ROEtaC@%_PVq)G^j~-X!$&_Hf zyA#8A!kA+jD*_*?(6%U5*9ZYTQjgOe{3|)2YRSp70I8TAAGGn??(NPL+Xb2{ z9UWbwV;xwX@v^{jPqfNYxK2nF`k&GOs-yYx9N6E)EW>E)$R%Fk25bT#`Ec3ffXpr1 z0@l_oINu_fJ_$flHvnCeTDqVoftHf4Zn~xl@Xu>u&?p;LPf$t>$Rhx1heIt8gH+H^ zPDDaR#%L|pNZkGZQmpN&iZu}r)s8_U5WKrhYo&DOm+bah@%*T(WK+_!b#k)Sw^#=v z%UAf{zW41_jGW-m*qBPGF<9fkc$xAp24(b1ZNalRtZZyRs1XBs3wR+n3VM)9CA>qB z=Urc2EzJUUH;V7yKCT5r6O=t*J1p-#0j*AxA&oO&P6iaE)+$xBwYTriR7c>OJ6NoP z9LUzz764Y?&XgK-hTzbPh={}%b*E|=0Xq#;UD93+0P`G33t&sJ-om@O}HJoi7})Pe@J#BTVw&H~vy)7``Vedwsc2LNN`S->O!=KmB~ z4&x9|6~Jl%Hv7MB>U2Qg_Gv#fzd%_{O^PBK$aAbAkabcm(MgStZqY8PtK(&41VYi> zQpjK};^$8WCt_d+0;LH~SzB3Y8jUtC-S@oHeF#$ZGzCtH}A zbzc_&QTyUaN@^-#WP!`lSQr|%ebq;9LWoNovOUvW3M{0K4i1{}7wZVZB!3r9l>do_ ziD~b6a|C_9w?L&Z7TptZv5uNxUqa&wQqC71O3`PS{?(UEG5o18R02f z43K6+CURC2?2oETOBGlOpTzitTnC#1W)x0NPSh`x0(qYk5+)+aL|;CiYjCv(L9xSL zqv8#Pn#U(6Cv&s209Xk?$bUgjXk|rNNl6K8IN$a2q=r6AUij^)6b~vD) zbS<%3{s9^V5JaMmD0Dgji2`|-Yrr{DDt!#R0UMxJw4pJ8(XSa8{#*4x!a2$2mTf)7 z%%;$Fw6h!Dh-WOK~dLvqcV1$z2v}_e+L3efU*(YT9_#q#%eXqz84KP zD;s zesOH;$Hh=cSF7}mvc4iVDDOwZ`_%e=I_VK^?eOczTybLZ)V|bXaXp7>LcV<2Yp7p( z3M2^{TvtarWoA*%NY;A={p+sBru-;Z#Zaq+wGADyesJ1x($vww7^^0#W&7L*kClX24|wr+fJq)CSBkpef*31`p(O^t|Gck(Dh1UH^aIbczEmPy%;SYmU%C4-(0`v zXTE*v6<0LKzxjwuIO6kTk=8V4~X39ex$Qh z@f^Jb%tw*@E#jsyO=aHQLmY&qkynK}x7yuvfWN=9{3qm*@0;F*I*@-sAO!c3-u~U? z(;&#*`h#DhhKxY;Y@6-aBPRkWVA9n7C3H(cvB}^tFd!!if{v&c7bV zK_}4pe~Peo?~GRsRrndv64z|N+Me00@M2TjLH$+OoX5aL%i`s2DeY#vHy_95{vl0g zGovIGoSXz)KE69$k_;s$zr8+^gC4%(eWDU)JSe`x1D7#QTK_eX8~7;v@?(ei2!r1;dYfF{RpgjMNj0Ia)3V5g1kuUITQ5*n zRW)~z0@SZz96bOb&XZ4{{Rwnl$eIE~xgGB?;_l)#g*N;>h zOsnA&-v4vPz2goJk_f?6nC;=LV_`-`6n&1C+&p2sQY%T}na?e@j~vicQhP@vN#~a1 zbbY06>)x!v=@?j3d=}|)&DYCGB+Kx#S;NXE6~kcfg#f)Qe}$3y?_^O1Fxm|sD9ZB& zdWzDp^D9UEkWTMcxCicQq5*~Rt7)@@rxA71f$+qOMO+oeNYog8&Yvlpt43hf_7B(K z7}Z2l%t}{!y@HwNR+iC%HnySy^x+G*f?wR3CkkJ7C^_qno*GO?zlULzhg*HP7>}Ip zyh02&9yqUjwy0_POd=dM>#QQSuCw#};2fj1it?pEVLAqTe6|W{M226RK1=$*gcB2p z*JSGHfR*M$)h*z>^a^t*(Uv6VCxVsS(T8|WeYJv-|d22$W&7XgTBlq#?^;~1Nb;;oU@GBRb9dtMg|H2XS#jgx;MaasTH$vV z-78Q@au-U!(rY}gO^}7@@M6|Z8AKT>Z3y=t*VmeiGh92&@W`T5^3e+N(R%H7p!*h9 zdkRiv=ot10kRTNP&Ex6rpp`G3S$|!q?_?Xy^6g}hJH=e3+?zW`%(I1t?>woLnSOJW ziYN2UVt*B$NzS)LxlrR7b#+xni*{zx?9$L#785Q(q0~J064x~Mqp`lsgw1&r-GcL~ z`c*=6_1B}IMB=lVsE5fSW3|*^iJ6kz?b7d21te>fd+4_te%P=aujJd+iTe4EOB1AC z6 z^-g35(*bBE&bMbr-DzvE*lO>KKQ2=*T!omlY8F#`2ZkN%`_p6_oXGZ8oO`bsg7Y?J zJL0ftoyRg0(ECR6g>=!tgacJ7T9x!p=n#-3HB(o@+-@yTca#j=r2o>a9Vm;5t-6rb$m@#2&yx(&@~QziBBt{T2;oAP=kVvA$=o?B^4zDx6^A6GTHV37G9^VBRLrU;%~#vF~sTpO^r(9qSJck0#o<$9u*QNQOa zdO`?!^X&3I$AES-?KZvZ5A&V+0((B8sF&DLSHpAl+awBH%MOJgJ{!-ZA48`= zZ}{Z=t_2zmBBSI&zBifFdqV$ECBW!Xhh(oR<`#2_>P$srq0H$~{IDbrbzv-C0)d;$ z^NN(qxAHf=PC-G(?&_>@{=?#SWGE(&Cp;NT3^sYpo?$BJDd^4l(`M(&!CqoTnNHQY z0?@k0Ez3fmUSeln&#)G>?$$&a#)2txbEqU#+iO_o>d)|;GIQyD_rdgtsG?T{zXXik z@nEMex+k@Y+Ya5^;4XfB;RFseL)=;zj;dR(n1dYsaT&5={ z1uK5#XiV>Y8 z+;-s532VGL@-3MV{JOrIh49)F(O5x-9;xmgXrBt6bq9_?p48tRE@#Lo`K_ty)q^B9srG2JV=e5 zcX_Ycp&T0*{|>f7@74F-%cyV>+hYUHLW!Y4CdaJCmLzuh2ySrfAV2PqpIERxZne_% z2L1FPQm7W@J=v9CLHRl4Oc$=fEuwj-u$68 zMOe75uD-Up{MbvtV^c&#->J0_TY*c|QR6=DI_rqTWKuUS!^%tCc`9fon_kuz|NO7r z9=ij*{L(Pk>DpSzc-hu6?1zGiSUb7hhCzYl`}mHlme#iiS}pO*Wh z0?lk_x$2Q?94dIXi$O1?Hx+igq0kR6c9S=(X_7%G%m# zL(XM)IbD`@NVD5pwladlI_lJ5>Pwk#c$}&*m`Xy`Xytwq5igfKKYVz0GDsjadt-1u z&`*_8%-^FyD7kCaV5{Z(M^X$a`z&kAAigi;x;>%_zei{NXnbi5CyFT3%ugO$s}s~S zhmZ*VY}Alsp&z*%Lj!uMAk*ClhMG2~5+Rv<;x(P`f8R<5lLd!%pB!C~NA~O^%U{ zUHjK7v9N|8dGYyi_c1YzaBwhbYo5?hdxwJCzJ^G^-YV-YM@GaDG0d(ZZ2my(`7$45 z`j2j})fS{7>#RLf>Rk4=fB$z{n>JbT1erV!~%nb@5pBeBDrVB8mDOtbuBueTeo}39tXA()PwX*mSHw1?e65pl;j)R{x-KEuL2dH8j%} zYJ51_FiH-Cv9}Gq-Lv3V0Ay={&1Y*8Nxh1*sh4!;tZB)yk>g#47E|?)u`sjS(&QB0 zd~-UGR&W!!ZnR$O7%eJ!FAbvz6HNQDK@eYX&fL(pypY7B68%98X1KOQMg`?P&=e`X zp3cS_Xk%xzN9Skl+ScdKp z742w`8*N))a!GSVkt^Mn67z9H25n6CID*L8=n9EWC3`_|l11LZ6?CsRtn5NadiwMt z9ttHO-XM3L4$s-CZilyo@HCQGS`?7Ep=z5^c`>N&DRQ%S6j_{%l^S9 zI{c|aNS1?bOA(fMr8CyNE`z^oS}ccQkG)wtD)v|~&5qhP z=~G)x0nLe_avNT>3zVSdu(I!kdHgBC& z2;etw=+3m1Cu=WrBcDB+Z8A#ZbMO8dw?FL{_vjD(nF(H}jXHj_7}Z#}1vKI66KN3m zs4-i`zBR(Am%?W2V*m9QX3nOve5d@US5TML=-EEqZc7i=vnMx*dU>vg2}WjhUf8dK z)%LWM1l^~6m(H~p59(W{?vp&fIl;Al8H+^ZXB1ZJ2F+DYPh^zLK}?`@h~3X2PgReLP$r0l zs;m{hfAMpjn|m1ebIN|_{12s<=Pr>BHm1VR;twk6Uzz(aYIvDk&$y(h-JcsuisU?R zrDC3LxDtV*+TO&_A79Dp*q^qP@FhIB8e0{O-_wb~vzqr@+}X;vi)r5AdyF_c-qcQi zd^!5A%Ha&{8Q-f)i;ORyyKQv~I`+`Y$;f8pZX2i^D2`l;9)&+ecqq+I5r;OsJR2K! zxS?XHNWS#6oK3^|<7EvGFC*Y~9qQMO5D)wyAZ>}L2Phv1r2}1I+6x8A!!Z!KHFxOk z>5&W@=X&!-_`kftwHki&milUAr!#vUBV-Ub-pgEALz(3H8CxFQpoIhmTfY%aU92!& zyy}r^d}cm#8xSG*ym_zUf6+b7qV^`le`#iN1bxsg;JS@?wHbhf-GI4-A{NBAKy3KG7z+DckOIQ}sK4IUd0tAZx%T+Q zg@5tCr(-Pw)sJIxK9{RIx2)wyh^B@xq zEY`l^B$u-3HNDQ|%G(~u=KZ>zsYCe2IQ$Ap)nIch}AT$+DDx`Dr z^KgEd4?N*?6n#R8^$dj`qUvuF1_5hi{;wMJjS`SM^-7s>b;e@Wk*IT^Btz(Hl*Ot~ zj#P-30~(t5+_%;295?kPP)X8wOa)21E#&!vAi!+O$Bb+TfX(RW={f4MeSM8j-2X2k zGIo%zJYq04bgNma`t4gWZ_A1qHnm)O-X|JHN`dQyfaj$Z_l^-(C*??Ngz6 zQMsRxm+qzDX&)8yZ{(lcx9Hy?w^waL>XTboc`jOi5T4cF-<&37l{OPmLP1`Q?{U+N zFc`+5UMz2!j2+TDY$@|O=8a=yjHd}p3Hnz=3%6V} z8>zYg1E;mMm9PQLyu0M~KR|BKZNbv=U62|6-2g9%65niMrCFrw74K|$OxjX)e2BOT z`n=YfSGtgQKe42o6}PV*d0>x8e;%E8rYuU&A+^}TcV+6=G2V#Xdh~aLV?!$O&EK9X z^A{+dzChi3X&&n%jWDN@S3$t0MCGaP;SZz>4K&2ZxiW6AH$LX5BTGoCRevX$6IxjA zgh2)#&oK>ViE=p8j421jmvO}f^eN^JPrBFTN>*MBlW7}WSj76YT&wVhg1UWh@{{28 zzt%^~K<>P`oA&Z&y!fYk6f>=vN?!u$543p+k_b*!xeYA`l~U{X=28a`j7OTjynT{k z&!lTGYupG{^J09;uE#*mh3b}xI1I$bbDuu~;qsW$w{k`pf4Y9dvL3MC6a@~A1m*6K z<;DE&jZBMYG2i$wd`f2c@#FXYm>?NpuPd%435qP*0exgd?RiEAvC!y>XkL~M_5Zqy zBNz&>ctW7o5M;ZKE<8B)ks9N!$2LviXbb>F_>{o!dI&Qn1|Se4E@wx_8*q>rSP39~ zZG;txM9|_an9t768e&Bb=yEt7sznKd6EwhKPb#xj07^tfRnWv<5Yr4Q8_SZv`1=R7 z>*E{*zjs7tGL%?)v`m-R)^d1#3knJVr#`sAVFI6d0cZm7EB9D;Rf40+6tdpQq;T5; zTtKdDDzDwjdp|BfK^F|rC1GJQAU)&h>B+eD&!qvN(|T7*5M)0#4@5&iMn{A(6!T;$86#&Hwv!04B zu@L!xQ_9zCA>JU7zr(Apt_~-4-fKvFq7Dvrnwc$5$S&>+?W#LR@B@*^kyBFZu4>I{-wDAQf5@EC#2FPLA4s`t)~#g@eQD zjdAb7|Ip~jEwI44oG#$*)b^e^NcI9)K(&qT z! zgow!Jv@{Y-0I28(a1p?dURG6=Cg|zr-;^pjo z1On4OPX9Toob*CMLgeJ4Ie~WD22G5FH@-@m_^f+HeIii@c-!9h>^ zbM?R)`2>=`;LxXZ0H^?n5<_e2k01X{fwVRt#s&y;080UMEs_8ca32LFaR^}R!0gB6 zms3zvm&aJYA|L=O%gx=rzP9!RU}pyh{{d#7o$(sGRY_2?5Ocp36fq90&vXGqzP-U2 za@s0v@04nV8$j>roA58n`m{6><$$y^fW81AQW~!#oszKL7A?3hVCV92W*LQ_u82H>UvG62@Ld?1^~j~u38SWDR7_4R6SumS*(5rLx!!HKX}fXIr+ z5pX3c<-EbeR{!t;98wAB&sbnU*vW&6;MmyzhpqRH=dyp_fN9v33Q4w*$WF48iijkc zA%tW`*`th%q!1}HWK~vH85xnth)O6c*|L)DIWG72_q?7zp6m5}b>Cli_*~cZexK)Y z9_MkKZ|Ai8N}N}QTW@0dVJY0a8ULT7y0ml&dIY;0Xg9CI;Q{@2kH_$g>+D=nXIB?- z#GmF{LQ2%Na}${8?_CNuD<~-V^RvA1RZ>oly``n)_3Qi$p-}wbn8DaR3b0dj-o(&L z{K|RUs_^@$fdNyCQsnIL@LgIb_d~t(@L`t3XK2)-ln+X~OnI&@=7okfV*U~i&attv zk#Qbr#n4*y!;eJ8QlSdMi`O}G=B|f<#RdA)H*fyM*1#5*gTmaR0q@V&#zt;sLhN@U zgsetIHhPubYnRZTY_$o5=2y#YUeSOg(db^q!{B;k67~@+cSQ3Z z=WJw~Sy^SMuM_iW@zJ2YJg0=i1)stMT3mcg`bk8@g^L#nT{$NeCFS?m*Y^G^Qd=cP z+CaiRVFu+E^olqdn9Su9^gC#CpFyxqN>cEy158X5Eqv7Ly%H9L)6kASFMIYZO@v=a z=nu|aREy{4;~T~PZ@OV-Hi#Sh_op{8vtS;7#x4c?{nZc&{p>B{+|JNc#vZEBv@~v~ zh|oumwEcoL$B)ZFEdPK1ll3KbNDfS$-cy|u6E+K5_H&9#Nc^5#fuOhQ0ov;UN_ zv9Vq!7Ceru#HoHun3IOX*U$?!iGEvB0#o;!(o&K4bYT`WPQp~=D*Jw0WLWuT9UeDtUg!g(B}kxSj{Hq3JU`ZxqR z{BcS}(0@aHh6^?!HMb$;<#n7!y|%W7hl4ogOuwrZivSm_HbFpysCYVq?V**x3jX+7 z_A{jQeEBd}l_P>7y+%@gPz&YP+6V~?yQ+by6Od+qXz7HOvHh)!J$)vCfdchiC++_X z40uGwn#k5NvHULi1qE}nvz_!%mYchii1|QMR#3GFaa*i$G~aU>&&3CNQ^+25ZGAO5 zCDnKz+bypaNt2UJ;@1p#ZOT>R8&-IYHE+WLO_y@8SphFcAFeJetZeao2DBE1_l;lqLBsSu10D|)2C09 zNt;o7i0n4_#mNbjP*hz#Jonwr0OyW&xOKc7S^Qsv5z<|=|D?O`RA>6Cw*I&OacErU zP5f1;`=OS(cJ(SW+t|Aul`$P17mEUKVE8l)7#2&uq+1b_tJUs_n!F>K;^TONV>2x* z)El6aef`=n&-4_Ih>%_ivZhi!Of!{w*busD;@DOwH>xjZSf~U9!EhsP)tUqmCozaq-uM zg;-WN9~++ zV`yk7PV_E4gI@7hJ!d z@cO>pJJ{r8B#BEmUfQ~Lq4@RdGM7II2yF53m!a;5#Srn8g@q+{-0%sj@pnjyn8jw! zV%H&@#3r18M%wn(;tMEGEfg7kZy-Q5-GGX;% zyi{ioAj?y}l8n+T2H)oP{a+B?(EdMPChRx=`(pRCRcLZC;k>`-!(>P1-o%#Tf`Z6v zmu*Z;Bi;&8_t1L#OkU1bh#%bB0{Fm8N<|>HLF#HNm`tg6!-Qf{k-4;!eLO z3R-AtnwotQ6zlKKAO*?E$ssWheg9?SASv*8VD=5q*-2>9OZWd3@5n5hE z8Roh>gL-(Y_l`9*Hbxq9+>47d3}5x?o@iJbcuzgXjN&vQgQ4tQ{#S+fCO_4s zecAyB$;r&aviGB+=%O_-C*c{cl$37(>(E_eqYgM?Ok%>?X&|X%n^##^XQ%S#0Qf}0 zS)>D<-qI=}cGU&kaIAX}9ewFv#qPv98NU)5+EEpM;`C@>nap;?vP|pH*x3C=U(+Zy zBPg_sVz4z#@|Kap!IHLb+e0-&0p{AVtk~VaR+c&-} zFE3yJ`eLcYXC0g94s}j#U7etD1@m?ORGYKmVsI`)LW)Zi6VmrSD_Dh)8d2G`=iNJ! zMt)t(QBUbqiH8vpaQpV}YPe=?{cKD?a{P##+&bz8^ZWk(N>61gckI|9U(ZdvG1{#P zL{FV$nH8nc!MDjtNhFPjQr)*JDk~~(u^gw_FY2$qrlh4+pkpS2^cRCqw4F}iIgWtk z^rxezk5&6UJJcAf#`7vnQSP(qot^R!rb~WLfAy-lt4m5)xD^W8lP6CK_x@ysCK12~(=H{U2bqNO*V)l1;$Ku3gSj@iqMv-PHcHPo$7X zGPP7xe`7Q>l%d{?Viq*fI@`X!LH&;F58U0g+0dt6d^iLnFz-tr+>fpR!{`vs4gczA z_o1=TKu>SR(gKE>nGK!isi`RR9~?({@++s))W!xUoPGCWC03#DA(Bldk=;|@z(D19 zGcrI!ioToM@mMZi-n+(u;rqn`pZKx&4ZlW?KYug|{&JaD&z@DA=7IgNv9*m=dma!F zz+P@0i7F6xgDWOF`t#^KJ~%wnI@`1)ti+l4`VLba;^V_H2R878Agih7dj}@6SkI`aP<9XR2eQSU1&5OIdoOXe z2D}&@Uj3tMHgiCqEGiCl(;kRa56vnKq-ZKNI{9B{=v^>4MEHO{a zCcP^_J>9s(;Y6p=>)R;w@$5HHqYdiwG9tP3M3kz=N4PV9dV-M~tBA{=Y$x-Feg{@> zc3Igqz)aUCP+n_+yh*~|1VUp_Y}GLNeHO2Y;C z$58WQuT5X`2T71f=67Dd78i6M`1MQaB}=zOXJwz6EQo|tlO32qxL4*? za&ncqDCoA31FVFxwllBU33v>`{*e(Y{|DLG&gvzqPep5sigGy;d|NBU7Hm0d&=ibasD& zb^6nb<9oS#nlkPDzLc=dkN%$)fWKctXNpyTA@sfJWM|$WYFN-DkiDWjLzyAze5xcY zQR*Gex38@=htS`(5Qg`T3a|MQO_ZsuF_Dpjqaxs-cm)z2a}af>u&`L{?d*Q@byW}D zu>V=6687jM1;YP~`JOA=N(s8fw`4P{C{#Yja0A$V(r*dGG^RCI?GhtCd8{;YtvJ&; zcq4S>TmtcitZN1l5C#1Xcby}VQ33&`)jng9s~Pf5z@FfbMWW@j(p`4;)H zBxhzG2fKrkyZ_7c=RDS2!&vvgv;F7bWbWuHhV!R-ZrMIHEiF&ER_teIXAnJur&o=m zApA-0OS9auTzq^DTFm>t+3t4WK~IXL_1k8!NisffY}{hJ>|fI_E-wDyI1>{Sk{rBO z*g0Hjl_f4!uS~n z5B@?$h>cUZIva|5xYJoVxy{N;gCHUM1DR)$H4jaFPjfpWYy z^6Hz9s1Pd_2SV{$;$++~u;2!`eE?Dm(O6xh(UUhlGcZ^N*SRO+lsR9&ebe%wVL5)| z7Yb_Ey@gc|fOi3|rpwm10jiah>}lE1!Ky|&{EeC$OgoPgG2bKdwEymc$o+V8R=a}h zo)=RjH=WrJ=soqB_w6!&S$5@?D!ESb6%-O?*4M9R9+&p->e3fIqo)_U8g)!ejO%5n zbi3t6^KcD3=0b^&Fw;#-#;eym9+J#V&*qSLw(xJL7`Mm5zz`T3sGcbI1BJd!)k4tP z{oNsJAgRnyA|n*BiJB|hcm5$c(Iv#sUlV%*cY2cI++$(VZ-p!{1&4S3q1lqVG1A7B z)eKMA{bOxb;t_&M?l!?m8}YH1E?o-Dsp;rs5RB>W=urBdk&!XItq7)U16^GjCUNDF z_5X?>xz0bkS3sw-PEKseBLXU?(v>S58vUJsTX3t3o;Z7!N|hJ;JFS8#tzz30e;)5D z;4@}wSXx+^ilhN{ha(3P7RBtGifU6Tjby{T;2$4aS`cq`QB(6DJ}fFMjOipPQT1U!< zC0YHqpt#g^#?;l-RZWdX@x!N2la+9nqtD|%8Kp4#_S2)cY&M|axNN}2}I^#>M6kW3lFxp?+XoQlYG-?72VbP#{s!QYFiq)6vd*^)=MgYjzZO%NaQNs35>SbE_`$%Zhzz!=~0Ai9a5;E0Bkb;lt?*Hcl{51kJH zx)A7XzZhAL1Y);fGg5p=aPaY46O2DGW&G?HJrGy^zvnGMcjJ=8w^|GIgLhuCr^VX4 zfA3zZ_a&G?(+})c=NlMpCj1T*)SCdtoh-tlg)8a&FmhCz)obu$nkx*UF2&7QpTB;Y z?+LqrgLk?hfc>(7AJg6IQ%MPKDDjm<2Us`okJhZ=WNc z3HOv3!b17vKi$l$Mbzc5n<=V5KE^ll)M(ib>~+nr2rXOne5Xd~oL<5$&5Jg9R| z9o{Jdo@3Gm67|l@`$bIsb6M!N=~@A6lNr6t$oP=tfT~bN4yC#Eq=AQrM_#5X5(mcX z;tG_zuMG$D4vjX3op+r^KXY48b7hvlk5oBGCi@%rqr|ZJg1Q14hh~cf1^M_m_L9*s z`d*JGJm||SkC@yz7SL0o-NB8+F;^?tfK`b(R5lCfug0ob#B>=2vEhmBproYSvBTER z?v{fCT3No~;o(Tidva)UT5eaF08pV~KMRt$dtblUe*__dxeyG=i2WebJpN4;78KOX zQVE`H*FcrUC{47i%x1FCy)9g@RZ^iXX+B(Rr!AyGb1Z2y^<^fhDlzvA98n7{uf%V|yUCSz2*bd{P)|j3JNu>AzQwQqn>%V{3 z>+}xh#Lq!AKu3R-a`)Q)9`_}X9g@yh0XB$kvhbBH(K@x#(=X5AW8Edvd#AAI?EI+K zvw|MjJkim3HwYw{T2Ev)#`L;FhY!Ob%b2dDsSykpWrT{7vYOh9>})P4EcYdTBL0V7 zJ;#QOZM2!Vmv3Lb)aS|H&$1iK;KS{T;($44;H5S8*~(w?@|VtzDlD_g|# zT^Moh*G^!M(8yqvQ;5u)pLS-St%!sKLnQ?}?x}^Gi1W3P=r=}`qL1U}BgeqN5J8E* zFS#E3$wI3p8T)Oef83ZAP4Tmn9Vd-!y1KhTjQ^;rA|+3KciTD=xCJr{L-edU#4+Ls z5D`(2qB~QLL{r;;GDsO3H!Lv6rP-^@<@gk5Q`#JqL-Fu&A95wdGl0)C)>p4`PYL=r z=KK5mKOkF>ycf{w^GpnY6j$I*mJqr7+yeakDVJ>eZf!?b|;?#D26Ni_y=e4CM?HWAMNC$EUE_g>=1UDy; zV+CN>eL@pGvnq%K8XAJ{$#}0m9ub(CnL%CtZVsNTG@T)Ft|A~b8bR0f_fE#5U^EH++cP& zoYs6c>C3D@f%sh5Le;lYg+vg`h~h>_Xy@I%STpqWL&L;%%&d+EWsF{d1to59TMRoU zHn18UIdtgV6#-q#QiuL6#Lnd|w9jk)Y`=XASaDoEcS5qV1|Kk3k47Vsp`ENLpPiCY zqm{RP>%Xg5X_SklG$g(sG>wd!wf=2u@9H}Hn@Cfm^Vs}a*XzedM+>jz%Im0d6HUIZ zX~y9e6l?{$FJRDqXhp||i6kCfmWS{es`v~A(c`Y~?d^porK5w+)a}gQ*uulRVu}h9lK~uYR{FLd?xj zzdQtD70VA@*3%SWXiP&Fg#C`x%WNXdh#*N{w9o4RK?I+vM}Gk|CaN(gM2OKb1iw9d zR=&x46ua7;K9Cf+~vp~A_KEiDCd}3lM<=~(Gv(@8?HB;om1DNtqUxv zln>h~b`g{oroId;-@%YQ8wn3!O5KxJ@^q_j73|W|)V%Gdh3Jv>;zgP1dn&9mfPpJ3 z+;Xj>^C(lYvJP8C&oBMQX6?s(1xYch%>gHNETDTSEV z_>S>7H9Nwg1qztVlmrB%1^)<(z5j#A$aa*qW##1@23hqSw!RA5qkJ7-zEl#Y3gV=2 zXoGZ(^vYaC>>4IUM^%`HNaF8S;my)qD0q&IEzXERREv|vz0)O_^Ps_HiH>f++mSc_ zQN-Y<`n4lY=;`Rx_Y=|s#N%HDc+9Kx253j1x96V=c4yw~Ak)*Ioc^M0SHa__swPfY z*ShM=HEot%&ul77OJmO*t7-%nX@39NGt;L+F^jzqaTvtW4cG@rIRAK{)TnE~Z!0EYfdO@0cj9mRR2%BlPb0(wWd#*&TJRmsq$*NA&`Q;RweDR*955v&w9qPo8U!D; z401hSQpe{6W@LaBe8Nu+tpS;UZZ1KP2pN^Vo29bj5XXQAgnhtzitZCLRmdQ?YHGIT z+>m4jbp=hzpd-SM;ooQy`r0@CAoH*XwOguDeofCGbVsKu(Pw>W>W8%G8L;Q+^t;I- ziV#Q+B9JKf`15M+r#^%ndvGsuv17>#C=9|*Lk?K8{d}L7ddn<0UuP=Ndaj1x2GEt! zDwHfugg_DC~p3Nw&w$;g=@JQ{UexeTq?hLj+fyM zJM5GU1^=O3221F(u`auW#to0RhtkU9SI-ef*cXT}GN1+;N8|TSqvz{Z@BZU(*6muq zeM|4U%|V}t7*W#%xgEbrcy>5|E|qc9Bo9nV#Rb`Bgk@Y-frDh+ojZ3>B}DRSTo?xTGxP0U zN}JW^&(kK`N!I8VhsjAClQJ`dv!2rKPb(^lZqmGT>3RDoNI~c{ASYz6+=|~7FkG@o z3?o_wJ1WkZf-+bvwh6e;WdD1pSxUSYPND>Kc=`(|1F81hQwe;`{Ra+Ecjv6{j8})G z!Snj{Y@oeduhDchNJG0`UrU|-7`@qk5(UfOBcTkwxN~R7aV%zBNQavgvHr>eQi7`? z>gV;Xx|aPa@3+4pZ(ifbPSuz3Ir$8-j08!?JIi|D1aT!iJ5pi@#&lo@nD5s5cOuyx zZ+J^0MPk{xxdXOdLV0jne_#8kGZuv?`B=g{D_+0SNz^r;K7OnoL}LWG>|+!k{si3(3$OyAfN1yb z8NPAmyVd{gsauE`rF!_ASN&N9(|Q06nR%m|v$& zQ7+4bF=$KV4qSs!>3+S-ri|fZm{>_kS@o2WhKvty{W^vT9hAv=&w#{c820#H-^vPh zNvS7#ig{2JG(|mpi28)*c8z&*S6>Y zP*9-wYX(tHb&wJ0iy@+MZ3{AXT*G@YBORSHhl~0Gu)z=xQBI0lLXxxk*YTP-(?tz( z5=}B-dRk@-FIx>Tb7N(l4hhYZTa-*6a(0g0NkHpNiNaf=PwCp z1+$S`#%V*oA}wit6C2GHxi?}HbaX$ia%K^|Tk8c9URID9ROF=G!Xip;#Me zIEaQW@>y0+4zCz$1Kdeww_-%_MQ!Z>0(f}R5L;^w>t;H2(xJ2mUCXf_V9LU(OrZ4v z2iVuEP(nf_$i7R%t2*u3vn}<>CI}7FCCJl}4RM%4>U~T(;u*&!mqUbk4FA;5lG>G{A{o zS(5QqG^jnHuP^J%VF83c%R|E`?0Y%|l|JNF4+#wc%g-Z0ba;_qoiMV|O~cO0>VvuH zJ^@|dt8|Nx_y)v}9=#$0ZM`1Eb>J@Q>n$Ps!bTcl5@*_Bs=Z8$MTgcP6JR)26=s-1 z$h)bG7}+ay0SzXgIB;AX=D*O4Oh0ZaO>?TmejasBuf2MNVJ7{G+q;ah1vW_Mj2xHz4Z>U{TA05Rs&X_ogZ40xfN>uVVRa5M>Yo zV4Bl~KSq*qq(%{;Okgh&0##MHgzCb=!cd|>R~xO#{x2)%!ILK;kzQltOjZ+ z#M_A!p!mm+8LxPdI>%xl7NHZxe6A91Ck5j24GpD^*6I6HYGrCiT&Pa(>!6~`lno^6 z(t3J2y4*4hKMdn9fc=1~WqkcUMm^%f{IM2S(CV3;cia$r@spv&Y+%N3PFOBsT=d)W zZkT98d)7kqsrjzCd(s@$s4 zx!-Lho+A2&v2V@RAmkpqccVYI!~a2gdOC*eVjx#;{1!QCsdzMUAWwt*i^s;;S1<)N z3`FGk0hmOL^@Y;X8gx9mxYb=$ugw`8oi{H`JoVKeKn{9G z7c?|>HS%{Hq*E||2>6P}d*cQt7Xspbwi3u9rd|fpiXwenj+=z^7*q6riN2#=I@&0< z4luZrjS!&n@g=Ks{dIJs?PDcTU_*~VVHH#uz0YVQ!bXDazNoC^9ptF zDTdEx6pE}$JFqu_PH~NyU}-_dbo4GoI^E?8&r)<$)gq&!YP+b`tJIzqsnGk7jMyuS zOO&6YxGMK?#CmY~)z{#WvA15RUEW$JG7SWoX*euW{*_?o<*iS7YJzHt-Fc(RTW8V< zz}l8>_SXBJ9yDofBnvl>AQ|UU?--y!7qhTEk@)_y33NNTy zc`*lcxbg+@p^poKM%!S8!E6B9Qg-tt?L%n;H7*a!CsL7*;FUpWB?50D5Y^` zz^>R-{T$eQlaOy{m_X7~{=Zf&52RG|3<$>7rxXv>Zw=aka)pynO*?#O08yC1^q-ao z36LqQ&Ri7Yn-hZVfJT>h$df8PfNDLWEfNYta2@s7%9V=))Gv~Abp?sD3FZSwowpC z$k?G}`1^O=DoI%ntvyr>(8V#K3rov%)cT@>LoQ+^)Oy5SoWS~m0P@8P}S&;$@>cu_F%v;08# zE=FMz9W!W7T(7hyov5B;8rcbzFD{v(Jm|G_FI}pOR3S_OIOPzD_Te?G+r^7jvcVJ% zOeCDt9onQTcZ!VvUw%h_U!~VdbW_`xFKN#wTSuLtwiY~fcj3oid+&t2$AZAoqkj`V=bzLv%udAVdiK={WDAEPZ0{^2p3?`xWSy zA@cHGUv)coZpZ%G3tec@?N&rHullO<8_UYA%e6D3It$Qn-}fN9YH7*kpIamf;b203 zuZW3p&;s%d66V>-A+>yESOYSos7DuhF0A{lwd5IJyja^Cc}QNqGA%8jsjaoO54mt3 zNLBwS`OZJZ3(2pjTHm6#QmnV^uitSNh9&iMbgI?jBp)750d+K|>-UD&v<4|z#p^pZ z)9kjZ*RM-o{Yb|X*?^&_YCNSUwyfPoIJ_b|gxxK`a2?qP8e$M)fGM=hn8Xjo2DrkG z6NJ_Wbe1$tt9V=28 zkzyCdT6OnDPZ3rbR?;Crp^oDRfA1B7dIYcW!X9Q?GsN$pK%Ay?_NTN+Y!k@p{^~=h zJT5=FODfWbJ`pH=!~y`qQxr#ZbKfE7y??)5J#f9DxYzr@?%h@Lg52B~sCv-Vha}l= zJRzbfX){H5Iedg%zzmqCnrxX*%m}0c^0-fafA^mDR)#pfV+v9^%$)~h!7_c zV?bd+_=RhF4ABlyQdADK<~msIGRpc~+W1pIYpzCHWy7H*y6-T|Hmt$*=GLS>fXK}! zK;@To(Wh3H+f7lp3CLO{n0MZ-yd`kWJd_stxA)(r5W^tf3aV-iOJf5Cg5t(cTLPLs ziKnvmIPo+^S+!>SnbjpIn+FF5q|R%@v9}+z>SWhZ|JE0J1pu!VtFx-4QWQbr`0T?v z+O&!Y%@iEax`4}DURgq?AuKnb>u@E^0I*E&<1T@cVN+|%&Hcrq zjR9nI)zr3D-rINM*45FqYT+i!iLkllk5R;=@#=}_K4!9nUZu-o2|7ALJmd4yfBSS! zLNTDm;{=Tg{)9eDfB_8N+0KM5TdqQ?1q6GB0_qU4Shca|Jw1lZad-=+=H{>Aw*gu1 zDzt6d#MCq&Zi3I`{s6rS+xh>>V_qV519BUn98M{~87d(|k0iQ%f$|h7>uJ8d}e7^KS(= z7*u^{yr@rdUxpdU`P=-;K2Wj+8>b98NP;{tzYiZl*y)12JoraLphwCi1{CuvwcKt= zFDXHAc!?I~RMGahx;W??uhR9bAzuYzSGl=8RF_4A(N)ge_3z0F*k2$`>Kj5Q8fmeA z2pE9SA7QCeBtsfPwtf4jxZ_$@Mq4;ruzcZiJ!88JEVeZ*^tMDSC>1y#cCkqLpVZm1 zdBHpyHbf7STvY7vVJeQO*jUs!$XF8ItFHZSJ1BM!xz{ET`Ti1qbfJ`X$31?0kiP zq_jJd3i2H$h^g9y0}teueW#Fsz^{fLsL$~)2lqM%a<*{$s;5KG;_}(Yd-?0kADsn4 zhXtAlojBkJ)NIY2n2wzGH3k~R8m}BchhSfbxY0r0-iL_4cL8p#Gh1HnRz!q`!nHn} zZ5JgaTwzf7;FyPn+1dQu+>rz*qSgktF1$7Oft&%U_FW^wCv4-eNR6LZk(n>Wf9&p# z0>Lo!2tR_$Uj~<~EP9new6H!mx3rw0!1hF6VFC*ijhxGcy*yqci@sjoE0PP*BLaXx zu5qX>J!rt^xinOuzY4cHfJV#_gRj}^K6`R|3l_jPc6+LxJ0g+^DL-qD!w&}LvCpxb zlt1QtzP`}I2}6@=KCdghS5LLeO}(JW3Q3xjhusSvcf8F_QBTB~_8lu&W#CER%yMKV zJ#Aocp#CZR$QO=A*{$@ldJQf9<6rT?*+>v-`rtgbGWXw2`&S1edc9RuskI3b4+@FX z6llrzv&w?dP~&NDZdQMQ+$wiv1BO_Xq970KoSx&Iuue!5`Q8pd{^h6z%__@ zyu8*RI~n`}GzQU22KnS%>+t}GnOVuk}lbmi^~Hu zq$`tE{SL?!n?wintCmgt>J{wRtwZ0V{ngw{vyShSXBOX3XzU5d{P>{m2LUe!ydf(h^{Qlhy0th@Z zzgk+6)Bm+WfmSVLjvb3hw`#<H4PqG2noK;oOlNdc^l>-b{!scknA{)R010l%pIu?0`cm+ zu`DDGh}p1LgO=FIq4cT?;TfGt#49A{=|!AyhS_RRayz86t~XsEu?h!u2s!DjVUjR} zPl&zj&q;p#hW;MbK^GKlcxzbLxj#sk4+B_&g~w6A2Y0w+VkE`0q$Ku(2bF+1rz=RG zg@b^GGZ7Suku8R@Cplm&HjZ{EYiMes85h91EXO%KdA>e@ED#DU&J1F|8kgQkd*`x+ zmZv1Y1%^3D&vM(D}&bj@*INlcIc z_V%w##+X0J-V4~!AYm|jqL%P1HwgCmcA6pmVs8E14VKD>18xNg4<1azTz@`YGg}HB zkxJK@JH6}R%z(ElTxSMg=*Ni~^Ha^h7S@Gb947;v*UeCf0|iWcKDL8VI7xLb)6L4O z%^P(NREuz*xxUKUhEVx}4}kzC0YXOWO^8BtbczQ*_Iph&PIowsj;T&Sg8)Lfx#f>j z{jk_1jE~I7Z}1@eDzlD6!8(}AsySDAY(iQ}YEQvsZ4Lls@W0aXl|rnEEqOj*;eCBc z){Vnt;0ic(<_ytPc(PG`rT6RCXu8#@)2#N`zJ*){_4Drs+hte4*yEn;bFxN;yNNHM zZ=@(H{`Mg`ItS~BuIkNyLhJ8-#0er867a3+Jzh>$Vc|_A3z)wr1J^3NJ*BQC8VTn3cZ8g~r!Ybhgg0>ApY%9G*?Wvv?Q!4aWZ2$2N*{gN^5FUj3H|1r(|l@J z$kA(HvCQKiNEG4%Njb39%XK@k9=X*F&hFo8TqQb8CpOkk6np25^KG{02B$b9Vz zFwN$^&-=^e?uCPOw7x+KFGYl;rd_|ps>b0|f!wm~wGg#rVsF%N@1$kKP2h7)E$+En zm2M2q&#b*${e2d*l(QAW>z1Bp4aDD{TVnYS=5R)srhj!5ugr+4!S z2sd7ZA|XeW7KPxODLlPW4OerD&*E(CorGYXf` zd_fP%F zyP-x-V%^F{_!1Cjz&c_jF@#O%v>t7!V^$x);^@#LiRbY4OQ1{ckL+o%*Q0D376i|8`A zX7y!fW=cAl_n50+Ya79UJn-MD_A(m$Y zkF@k#!i+OS93+W|r#QWfRtWPwiV>M15Gs3;kYHzTA07i@vl1GjvWbC#SqRQAoRTFO zVMV7k6r@EX&~B1ZJ*Y+G&FLG^v8Jj-`uDN18!^keJ0rlQ))`&9#^**kgZRNF=|CdX z@>qoH&>>~4O-Z9IZ0uPd(RTvyrBwt`=aq3;|LRpjn1!x;kPf`7!58A-<1jPxi$T`{ z4Ba-B{qKegFji$YDLW%$aqJ6!{xxi6B-83f$WEtcW_Fi(RHv1ddBQ2vw5v~*YWg@5 zwFy4%uScDz-W)mRBu4LKL89b$$_j83q`c7#%Seo2g=cJ;2Jp^-=E}~ z`5V<1Uk$}q1elYw!qI0n4Go#e$)p_O7!iPQLjC=e!aY#&%V5T4X0aS|-Pdz`))LxTYryy$|p%`5kbm^uKlQyIOW^CW0prq-|jg{( zFy_OR688cL2o;E_iAn2k@vN-RZHa_gmfg{d*d4y}J|{ab6xH#sw# zo*bnO8u~2&0BR?vh{irR~bUN*#R>U1S-164^#5I5y@IH~Oyn_-vpG12?c& zCBSBjZvv-`46_gpBp6Wb!BKgEXZrT=-Z3n9Kgz*z1!2xh^TG*=H4HUDN$%=~S3!{s z4kdEIZINc1FRQ=*kI9}=F*&&km{Hl;Ny1Ao_DDlzJIA(&vqE`SYRGsZgK2?$^bjU2 z!TwE|NrTqNmt+li~kt)x6(QQB}`Sig9U!8v9!&i8^p=%3i z&+a0wA4-~agL;~(J6KH`%#SO*5Oyf!a(tfSK$RmF^&Xl!e}>GLD)-bgIQsL zL(m6NfN5LR3qu7Cw{F!@12`expmcSmZ$6e3!~VxKFY8VW&Yxi zP2HyI%F_NSOL3|mo+H#QvLl=%w90Np02D9-m#Iy)le88fgZCTQqin|Go?$oa_5E2> zTf0pNjDWx4z_Lp3F@@=SOh>7N`hF_7?iY0A`~alQbl|{>`vD&4RP(8k3Y($wXatU? zPtfv}Gy{dQ4~7`Ekc$si1W4v3b+u1W;ApyEk&)fn+8@>)(~)oSWv&18X@?MU_-xS* z5_e|sL}d=3s>&4XK9jT#K3MMoH?>7URW&s3Mry<%!vQ5)&!Z#bSyYNa4d!rga`=Nm zzkLRS(NOsWjNz6aSboNngJru+fNbN8?*~OARp-;%5AEzvqC;Uc!aF%cnyg?~P8hA= zD*dQvJ-L?fC{}om9zTOcPLO`;4DN)%G!Wj&STg?%?6zwB-Tjq9Ee0;f9mgDWl-`V>u4Cf}oea`%6 z(nqrYs#f%>~Jmy52yc~=y~QzxGQZRCj-&9@s7zw6o7jKk$BPNRRz`FXm>V8T!y z(GOqwwjdUJw0eP(mi_OyZ=+|GS08sa|9--2SuEseioc?)EPUGaMW@eTyL<|hy}gC1 zva)NOm6P>o*c!jgX_L@5MT60+uD>Za_Ur5|R(!@1^gVTQ@*Iba#=wuKV)BCf@3t(c z|Jgac)baa0`HGtc+m*}^wgb;!QoWgeFLYJ-?{URp;i5^m!k)#Nbylu* zZiaE&u`zp6m-)76b=~@Rv)Uzk()Ls5a8E*MQn>v2#&V{KKhj={cjFT(I1eqT z@FqO+WldW69+0HnE+_L`-GJViBV|FWC4@!ppSP0M1>aCSDf@ZVpGV@4%f4~7kf>BU z&hpy&Yi~<=*2$i-wL5zkD;QhUBq#pJCM=C^h$h|bOIWoVEmd#w*v*nFRv~*JkKgd| z)OwYV+{sJL&96?M^C4S#D|WY~J7(sMhw+c7JtGz(pX3(}shGwlCdLE|Xb+Tmzj+^| zohP(-FV0BH)89p5Tf(UCR6SWv;*<>kp|q47yR$1^-Br9SOJwt+dS@(kU)NN3#F^Ql4d3?TU5|DDt$wTu3+3szzcpuAdXzKs*TDfD=&?c&$C;! zOo)s3NSEvM$2pBDs@9XQFCGlAH(2`DTfVk8vptbf{ng(dcC~(!8P&x z`1XjH>dr6Fm@$yIFBNtC==u87r)5mnXy4e49OlmX{$9(t;W69Y4T#8&rxZcxTTel#0n>I^>Mz^#){<@KOPuA`?W;zq$XO->)inC?wJFb0*{{Hn6 zwYq~`!Uwk-#pBX7a5o;2wpQzsYyRlI5Y6@Nz$o#%ug~#_SZ{>nz2WknaB%T$3G{G( z)JVH)BPEaW-09|5&CMst<=)b>_8pDe$V(pee(?0Y^G3={p*7xh^G9vUe(AEi;X@vb zuDjn_3$&YgIjrtAm+SieJ(q7iqq6=%KP$Otmm5pBMAzqh+a1T!m?=7SzBXH0H~F(9v-}vp-zqI+nw29}5>NC$g;sH};XpR)>!KsR&t{fBn(l_q}RL zcSWjr+BU(z_xm^}XXK5BWy%lib^3kQ-OasI^lrX<<=k|rt(NTgVzzODL4P6F_G>NQ zonj(c+4ef|@EjLg>Ch*S+Ba41%J#R+_AIsjn}cuEJXXWM%{1|k#{H6YDr~gd{-Qhn z^UteUqg7S5KJt%*Up}Uja=sEIcMlC zo}IiFPXB&=e)3UKuhy+uwKmvxh-#;har?%PhTwTyfxjaz%kv87Mtxe;2AOhCOG)iK6!f#a*(r6nXq;vB=+n=NE-aIA z&ovq6+TX}`cgOTJXNdYpA9wux@kbl95s!~PU)!Je$s=|6_G`H{{+$;e89keCsu=9=g1%7C6hSLlUnYTczKCtNd|JB%-6 z&rKPX-Z~K?AY$nrd{3_aURlS z*Q4ut+uhQ2Fe+#-_3!PPLQZKdTJ5xo-BU%gPSzDW6MmIshCwj!GG~fRlGDF6wZ`Sx zjy03K>863$r#Ug7_pyD-^7h`BIQb>GQtk7JU;~ToJtUXYimf)+LR={G>a%=RHpAfK zP%b1I#e03UKXiCyPC7hs?|SKR`ubP8j1TTGNBgJCm#@s!y!#i$Tl0r_v{+o~xSUK(~9FgZT`k6wIY0PEJk^F1z~_xP-mt{&c;7%OD<&0~Ad z;r4{L!xlS##ytL&9{20A(`(JM(=r>=GPCc6QY!yZRD{1Co>KXpYK*&fl3Nlz& z`Mn`cb=T`7q@;=D$9n9qy})$J`U0ccWe;`xWAcaoT>?vwr;M^&DnO}hq#2NFS3Gh+?KNxn0h&Trv)?^&Sl>ea%MTlMHNRHji=1#3gZ4= zxuo^G>ZW&FKv3c8Kk5BCoZ&1u=l56Pg!J7QG0pdX6QidztcT?;GOo5Ieazkcb(zQnR< z$E$y~@MHz)6;ICJmG?;3*maTa?dbSr75bo|!hiZ-3qPdP-@jZEdBeRt^t68M>`ij9 zZW)oMH#BVZ^WHkSE3f3UD`wnz`Y3=_pXGa#1hT-?KIX27gIu`<};#eH{7JG-Q ze=nxZF`a&%Y#j+d2S`EKfd@$+mySyCEb$_OBy0+3? zxBl$?RQ8r$w~7?>v%8CZ%s3Sb?(AI5ySUD(m8a)NPbVgAXKzuqWMH_eAxG#6b1L%o-<27^dg+#W^9c5iE}p8R+q*@9O`<)?0>E^@MHTi$+Qi3279RPU!}vJEcL8knTa#Oj9C#GN zFA|r&HT-blm^30f`dyWs5x5&WpYsKo+{~Z|!-@N=CW`Q&4-$LWRL0&JUV_&W8%bK3 zhyH%EFCu^BtlyCZHS*7;)8>J-9#V;!`Q14g&^jL14U*=<(PO*AuJ;SbM=1R!hYEE{ zIbv=x*F*gceXZ~MFg40Jy8C34q?OrqD-MrnEc{lhQy$138>O4vG04|(Y8LtuPvo%v zbjT5=Me!Yz6`XO-8$1*ge;t}3`SYjo)j;Jdtb(|U#nwyWyFtI}O16zJqQbnw-3~#q zZ{+3^kKu>%kfRZ_E!Mi+#B{ai{Pq%w@ekI-+|A`Dr$=GPqeedTH%fDo`e16To_BN^ z7xoUglBt7YQP( zQ@u>dqK-880y)0c_H>nFmR&5Bb;jc>mR-!0O}s~c3aNo|t(*^b#Ar<9N7h$)TsJa(#*ooz5Sn?G!o>4EM)B5!0Q-g$m65@Zk{nd6^Z zZ_f*gihI;pGy92IcS2FE)bt(ba?VgpV?8BAzHjjLm@bR0oduy9kt^4+9x!y%e3mrO zpWiH1-LqN^S3Bx}E1ykNa$P&a@*6(sh(zNjAm#MDKq2;HNUBM0zl^`xO=$ckNet&( z^~0pkuqV>l4>Nr?-~B!B@T}{5;oO(^O6RHXA0?=Txy{SjC*ng}$}mRbo4va3yeQ&- z?;1PCyiStw)M;Qr?-Kw!WE5ufs4$Jf7v;u-o3q_BgNaV4?JoJW!c;N0)%m}$yvX3l zc`g}=`8^4iUFw|UZD>iz*Bhq$gnYV|;lo~mVCq=1c)Xn0^@CDU(K7kAn&-CM=UFTA zIQjKvtz%`5O%9Tkgp}r3<$rsjlE3fzAs3!@vZsDwVrQ{#Zq-@e;e4iS>^IECK=I`a zH~ObZK>0PEQqn8ZFjl&KOyS(ZnBVZ@K_Gi{PQv`-t~)b40JDT)X6FXRu?xD>r1;9A zuLl=2B9FLZYwjH=P{+zGfD zlT58gpC6MdEx{Jy#HBxDwTC#Ql_^4_QgN8j&^}`!1WO`TdQ$~!3YGd|e9TQ(|Eb-H7_wS(mpfVS!tK6= zxGob@Yx)xptEOq3O-W*sZ&AG0z6(eW@lwgbF}0^Jpug)d?6-#yaGY=k1#tetzhk#n z4ZM+km3>9fhU%w-TNk?LuKr2o+?%HtQ)`l{gZ8afjwxjOFRGEI(ljFB1bqRlnJVI5 zRpjFmVRNd7s?BrXB6e9~cn9=yyUxsQUN9Tc)WrT2vw>18@4FV)@`WKL5(X=U*$gom zm|?P8^-t>I%Fq66P1~&BvR64cz!LBec~w>$HjFE8Qf$FFnl--EZf+;aI0WTcrXfk> z!^Ee#FS0qHqjTF|x;lbxzsCg7e0&R6F`C0Q^6nN;jRzNgKd;{Yh9#?=uj$0iQrI$l zzJ^Yk+aybvLq1obl(Igynm9zLhrhj>o%&|_!T4~Vx?iXNKru2FcU5KQdH%<&m(U`7 z%qot-q~Tl0=jbA2D)LZWD1Euy#Mj%n zW@e=wg0$S8ke_i7?dRFn_V~f5$3o)h-WU=d`5aW&RsI2G}IKwwxL4go9F4x(b6v7eo(yMym zh5eEhu`Wll6$6aNMTZ9Oj7#F-DPy7%=T^By_m8UicfU@}42WJ&q#J4XosTHHQChCd zQ>yDtd%ifJJZIv9WTle;rUvs9I0(m~t|q@3R~v^%R2ro7$udTForq$ zWYmi)^QL?U@|zF{UwZt8o%GyMC*Lc)X;Ad17y_i?K?reXi%Q1Ei5fu5vmUj4f zVcyI0QZd#%YnfP=gS@*c)ghxFYzhw;25}9j%SJ^t&je}fGy;_0=ewAkFWV~{QtMT2 zyum?zvCjnxN}`B$zhq5jz(SRky0CHl7yDfZNK-EN(G1G#l?QsoZyn@e#uuq_A*Qa( z+g3EcMW5CdAh)pAzBgI7(6yE?CA@G?|Ma5XJ{VEj%mwvijiQ0i0uXy&4T z3k8@LL?ma({h)_K*o`TXs+Vw@yt$Ix`xS23^4F_8%odhhSkAO)8mDZByPcI=i+oYW z2g|O?{8%0XKBC<)lNOTy*^78mQ$++>S64MsMQ3YE<$W(_W77MrF}N6Y-a@neL1>c( zHGW@lRRhA$FKGMy$k~S5G7ySwDl$0vlNvpdjr7)baX8aV`0-d|;Ul=mXavELZ{QNQ z>$qSddafM^CfoCh=x|}ISz>fQsgnHsLCtD^@BDK&r9N+@B)X#(LYq+(dp>7@z63A6 zSaWi^iZ~le5v{He!TcIu!|Gza@-X*LLUp{^_to!zx=UzM9uT1-Wt+B6xlza;WgW5DcxAxHLee1KYyJ zJlQ#xKkxU)pJ5DO2XkuSb4v`)QG^<3?gcY4NuVEuZngQSMJae{k83&T*;!E`saMugZP*OiVm`F zLq(ipW`f6i3Ei}3wRzd1UqNwO-_*^+)+R);U3iPt9c&b9E|o)->_>*a`c)(UVmE^8 z$059bd2uSw_6j;nd1hc-K7FvVm4aVN>_Ydw4BMCJM>gV@h+s9wZlORWx-zqJ8MUs6Icx+qOi`MRyZ1DWV60=T5>3|#5D;e1F zJl6Q?C-cBxyDh~MI?GHpTHMjn8p@NEx^nMS8_Ih+Z4p-h0dyQ?=zj&Xff{*t?UDB#$r?CQyNK6w7 ze4eD_KB{wjF46HqVBxT?@;tt1A5F6$0+L+A<)r7G*6}Vp$2;$Fw718h%U)Tnl4Bv{ zED-UaM_S96QwtGuh=$~yt};wRFkV&@*ws#ESThPx(zLX)(&g+n6gj9L*Pp<&BP2T# zdm>dWN`9ZG$s|2-QX)6~f-5D!GgHnCS3h7n!{f2o^JqTu9`(P1+=Kb>b<#b4?id$8 z#%d?u4uKdKbM(W&sO4H!^QSMM)_<%=I6N{a9}1YOq|XFp^{`O8GQvV5+dMv$e0P&# znJ+W^J>Z50a=w>f*3OrJUo+JTl}KXpxa_?&MkFfQ>;0H+NFr^(4vq6gP;oq@9dJYI zysoeL+K(63`X;Ej9L+Yi2x<^WRW=~}WK!0#FB`EA?E|%1QfqZ(6$`DW3;fcUlBfoJ zE>UoL(^k=eaK?4;`-8WaMFyuWc3Z{DXC1zN=-CUe%WB?6VOsFnlXS{+1Yn z%jVz?ZDXi>w@G7RFBboA?4Lv1SN_K;8BUZE^$~_??Zi2h7Z+Pl;U5_N&;HTK_4-3lY~L2l;|VJ%t-aKodsSuds->~Q0TtJ>Zo1r$J*@ooR8~+?TQwrJ(dmx( zP=7{!%rz7JHmO3E_eAG5C2=ED7^*C(>>v&tQ|WJgGae5*xR&$6bw-r0Jm|-7?7!(s zu1$(LSi#9|{YWHU?}KdB=(S_d1?zZSE;u=Cd-4-bs3Y7)D^-pvLlL^amJ2ufj@BPz z#hop>K}S?(b1`&fms?+aLW#XGCr5qhy2?>QZ;REqPr ztW*cAGAfP2qeDOFYv)bp6#^hX-4t2c7?=(52qQx1{gCPS1%TE2_p2^PH^I495`m*+yEM z813aRa}}km|Mq(Q#m44`!XJ6~q^|i<#Rk#ZpME!XOklP}wW!tay*$pxL}p~1c;~A| z;nZ{)7|?sP!C5$Dl8oip^iW$-R<_v-pPN-cGmghqR1+76x@`4O{M#QcciqOIVE-j{ zXTzGVxg!yIQ9?j7V=I^I@ga?c^`~{Tghs7X!rH6~!Xs{mo@D({&a%WgU(S zJ3Mlk(b0!nhe?6H_o!e+P_*pLB|=fwZF_oMDU2BQ8PS`__8eAgKARQY_?S?jT(z9U z?r&6dXm6gbhKHm@8y1kX`%uEO{If>Pr8!fz%5u=zHq!x`7==u}XA%MQKm)M~DdRx9i&Fu@R1$NFr%Dz1;0ZVmO zkBTsQh0qelnVs za6ZMqNreK;zfs`@DRF3Vgc-0iqhO9s?NoGs%&_wRX~L)W3gFnyuf~v=gY@%2Z-tRj~Jcadw?!{ug4ci75G;p~6suySBb|po+x! z;N0KR*>?T~bk&EPW&D7H@QxO`qWir-7}klrMIT@LrTTdd&B2n(*w+hpRH#;sM%kSI zrj+b!sM0tNbGF}BCV~_09@FsMq1Pj!3C_(kb+*;f+$+nNO3yf4oS4auMN)|GeVFAZ zqus`JN?FuMQg7zN;VB#2CayXfu6C!T7KSj(? z8kO`^)F*vO3AGro`1~6mXeHRchsl>J_lMl)U4okt?9FkDmti<<_eJrr-dLzg@hO?V z>zBcT{j{~8-aAml>h}yX_@V6l;^#m%Y^op-_$TQeyk#tRoKrywSc~y3)GJQh6&VVD z27x-rUdFp)8+tMiBUETs$(I$C#p$X}<7Xj#qn7-f`2j(sdN`K(XcJn*mQibb0_4l< zR(hLbHycAGsXnc7NXZz#iR(s;wncxfi|CY$8atd~GB zj>iXX#?07r^`0-Nn123*koa)pof9;Voi%&XfC7;=lqe@`%g~YvXhJ<%8zl$dDr5bE z`@q0^HU2{CTPWv87$}_xRDh$QIYwQ4I19(Bg5e3>ijkmr(c{F`WpHmH20iBA<9gqJ zGSYRp$BQh5jx6@WAg0wJ1DNGjgN+~iVY`-a-mjF9njsU&&uw$8hk5o7Y7dNeA|?+= z*G{8#8*i=&kr3XX{e)3D?JrR=-8E*o(LxrfA7%_gc&HRv9Mbg>OBygH%|^)3+jf=A76Sq~#9?o> zOWJ7o%rhkE^4b=4AvS~`sx`{G9tcpE31^lILklh$MaXN%T zSXt6w8wx@%#5jSUb_iwiQhbQ-rSa}cM(uL`dUN(hyvlMv@1xw*Wq{Fo?%_3mDksP> z0QXzN$gIPYwB(N#Gk)kuPGqbmRi($>6$`Z8oFyE5AkPOmU)4ygamdIARV9NN#m3k` zWvrALOeA&HTPqk>58Pn|oBc$5>qJXt*wpt2b?3HGQ%$}aSB6GZxOiRP=CU9r54wBK z*f0CfDLhqdt9!;iEE1}@9il**A*I~oIMBSQR!4FQU#+g<1k|oDnGyKhhk54J*=JLa zo0nD?W1dC3F(CC5ILYviVrlv1byPm!i1PgO5ciM)E*z(=4@JRqKlyso4JzbvdH2V| z>bxX6@LTL(WCA@Cyx%oLu~{ak0uukAWqJ#o=B|p$Kp4Se>}Ptu@6bD-qwIXT3Jdz( zWN~QI+CYq~x~5p@`m%McZWo$vf8@mfoTnp<_EYSYV_)EKv`aTWdnfggQjTD)9lC>4 zs?dZr7fs1NE^IfNO}DasTm4r2A>yB#^`s4pxAVZ1GXQ<}^~v~O#O^k$Z>K)A$N6VS zMuj^>i8w~FC>e{9Lmw30)d|5Y+jOG;F!GCqiJ3qlqXFiQ1AaV7$( zA7V1{tzf-xOdMj-0UFwgbj71qOxUxm{l}t(eModEuazKojjvV{`?8NSe+#={rROeH}In8|E1_+Qp9eqcg?Tg;{XtExkPo{Bs=p+la@xvWMKlo zkS(^U*-p=2Sm+zYpto?O3h$pH2yF}y@dE#f(%f^zLle?7_bSPkxI3_d9wvAsaUKLeDXFKha zZdKW%SvM1o4r^*<@uyFyan?1Rvx0$?+a+y&UxezV2$XM4Ajs+!K3t1j5sfM^k&uJ& zmO|o3<5-97xXM!L=%1MUg%QC-!sh>TzV|L6+U$WwO$b?8wj0dx% zB)zB!b6{J}j-nE!Vr?Bq)s_1TwKyTizs6LQRWFPvpF!@N4rLgrtS}z4M~5_2TH1rM@r|HdDH5uoTh5c~65b|mj_SOT zh!mnIu*rkG{flxC)E9o|6(-p1Msch(>0JU`GU)pH+c`h@ms@_!A)KVLYnYwKz zbk!=57E8Zur=)$PSyV#5?tzRIXddu4f<>^IDO3_amB^dvI8ZpS=?qIH z-9w{vIXNLjxg<8Nuo!I-+&N8y+1|TGlCnjTqro9|4_D3Ov?s=B787VGASvKo~Z@o@|d_M>Ym>!JW&WvoV;HC0{H}j*_B1SiKf5U8z(TvnytRp`pGuWFiwrG z+cB&X@8pTw{VKaD$p`z<07Jgk?c*nnK<-|2gnBhgNmFlfuV~|hgn*4&BOmf`>@rs; ze#(T}WYw&|qInv4&NO~mi~B!DNOiA-qcC4asGe77>vd7XU%%!VcmB0#FY-X}-0@w>K5_eA!_@FbV4G~;9Z@~^01ww}=ZJLdVhs#y^O%iiaE~f-GlxAVX-?+%?32h0<5!fWMt^?(E+N|1+ zI^S0vqQAxti+J`+JXF9!#bR8q3dE&h$avL8aho3AP1P{AnGt7uly3SjHyB<2c{u)u?@eFihiyKuV_xOY z%(r@4t22qdeNL_TW_`~u)%Vu8vT{lttU}L&`l^%c-ZE_1AH*UcK;ORXu5FRNT;^8o zsWw`*$@SyoVfNIul@#(peygN)FL0#0Wya%sci?HN&;7Bh#HH-B)A1I}*r=GzK;T+z z-D(*()JZaeG&NUQWK4N&>P7>TYJTZHXc_%$GO;eB;(0nY%@E}#B>3$&1)a%K>|dRR zQ$1~QL0i@7#6oR@nhY`V>X)s<2Wui}4Ht@)j=RtOgF!O=0(S4k_5gw0!AmtZX!!`+ z@uq64p3Oe^yQUJ4my=PPlo71)@6wC4+m1^D$Y6*sfv=}M#B*b)m`82`p>R5>*`aA7 zO+o8)WwgF$TMb2mj}?816s0+}3@q|~Ii>}f)s)z>P_xuu z5{g)~6_3wReCQyN9vZ~)?pgJ~uUO5%IlvY_Cj}96xb}K*{%U$(LLwI7b=-{mA74`Z zrq7~s9E#63;8U<`b;X1De*N+gC4=Iyojv>R?_!+ixgBT84sUzPaOax@Usx=BI?a@y{QM!jMauLdBm#A)kZYaxbYLfe5iYk8v8=f! zZ0p~zq0M*#+uxci-?=x?{+mpAd&Xup7hwGIj*pe=dU_|37QyLwbu##OT;Pvly3=Fr zv{m-%!MbF8`?|kum&#fv@$-d!2#~>^eF|lWyZd}$H%hHQDN2mEuiK2YRpD-3WAX0I zA2Ytl=Ig_&1c6X{`j2571Vmc|L{mx0@x`|XHI7>deoY_dSdOOE9J$^pDwdrFqZU90 zp{z?2?NbwzHGCgZt7wX>+_LihUJqyz&iSX%xeM&}?^G1tt**LxKkG8}DcA}uc6@uH zk7o791Dk;p$B8{CCp~B#wE){>g_%NIFSfw?++y$~(%(yFCS}%f>?eKF zRUfa%!rqVDM!bp?#7c-6-A{Vn0oou0xE?@_$xGIY<@p>4bf^;nH0j#gMEHu2 z3Fth5&>jL%kF!?5tq~RxK~kk{)7t>bWWXSRfTtrW3d(lmiy%Y6=+|gQTH4wrI*lkT zqoZYog=mJsZq)IBwgvcqpwxwRIQT3E@b7?>6b8r|fW8_!I;cN<`wsh(BdwW>(f{AqD!V z{`bGfvmhPH|NZ2QJZy~rz3xetkp;)5_~d<4894xE{=Y9C=bMNtR7{Rr5B#pnnap9_ zHg!a6>w|Y_YwzX}DX&tPN)DVkE8+HQ9X$U8CpO_?9Bs{ac5gEzaQ^frhWh1;o97mh z%@#B1aga8;ft&j8Btdxzk;#9i~Oh)ZYOmsZ7`H*}@bW$oLG2tbr*&-Sx&X zBb{CUk5Jjfi~sKu*U4f(U0_7?zXihJn*Z;m2QS0evrZ6DsoFOj0huRWF&g+<1KGo6 z^CNl%ir*5J*A<*gfRJ-qICbdJYjP!`6$FuN&bE*zm?$?l&5Inzvy^a&@TdFu^qbB? z4L}JBfmJ<0+EGnFSkrOauk@MpeL6HrrA{;eZ;l}Wg}RGEFQ%u{`g(;5Spt*3Z`K#h z;2Z#7LrWC(6ABflNPoWQcF1?L43zP3IT-rkr@(S^3<;nN**g}RJlvMwgD`yn);j>$ zKGp>|>1QzUt2)BK+5;uEuFf?TXNMl(Zkq2e2qC}-yXgl`F5(qDX({1F5&%F7^rs$o zzyil->^oEUj)|={yXvkAK>x6K0RKDS=9R%e@UoE=ytC}}A*?$<1hscgVs@!0uHf~BL;ui?0p(Cez5RL7y@ z2^nu_<bAkE3i%l+C{a8UDrukr0z&X_Rv6H= z-+W+&74iYTQ8#+P`w5sI^2gL_Ap13aqQ4&li*dTPXklfpG?~LqKW1`5*x618f*u zhlaxV0hK=q7)gO}U-EkNxakK>5dgwOl;&#F`@6K1B5TVG;0r)Tp7E{u3UpV1=7cZ4 zf&4A>iH!+@G&2m~-}nGJ9VkVCSIC&LVJx^Op8WZPzwn3H21Wqg?cG<-?gA9^-wm0pB`6^szlyj45C~Y4xaE=h2-<-Y1#tPXF2Fp= z-cug{Y#hL1+c98eEeAqypOG-8ZNPpI1xRp@24I{ZaK9r50i(<}36t!W^Z2Bs0(v)C zhaN)f5SfDs1)#5+Lu5}Ih&wE~FB_QpnZQj*LgfaI;LuEMqeGy#AY zob>?uQIU521wg`(FwsBzh8!VoXl6!xD&PsmwX;@%$7?x20lSIoT3e{GG0;1EgT@Wo zOLet@Pk=h*22;&+J`Jif8ZI@%G*Zys#5i(1#R{3-!;R)AytS z%o?yE3?GdQ4FO8J`#~EBr2#mTjuOXbnBEh>p8$oZSff(%0<*ofbp@Dh{F@FKFzP$m zpP8O6)T$kwoV)<&B=?cJ7l;T?Ob&5zfYGbjlLiXieswfG5CQ-i5Po0paqEbO7YiCc z#HcLQrstEM3&g;0q`*dEYm)h zIu9mtf!|N&aXJ7dQ-+#{y0TQky@P{`wRZ2cF9SCL3W0czPP^_5bYo}7fLzdIvnC2y z6mQ;S;N?^4|9Qgq03C2Bp*tWU)f@J_m}zc)xS`$ukbQi4>11aYX@T(?v+g6kk$YjX*sKHvbe{D<^R9YP0@4KV%+WRS`lLPsp4&hOt}H%Q z1Mog(25!xA`kkXd{&ei|nL`Esq5LZp(8^(EcZm7A~%9=_r~Yd<4ft9VYfRu+EF$;Q0cKQHO{8P#v6#iie!6a{<8jvO0(YyvK_} zP4c@zU@rsS@hT1gLUzC|N2>_+6Y_6v7KmR(W<)bKHg-6j13nsHl+sjB`vRMG`7~qz z3jROC;79~kY#rPGhE7!>gkB8?{tm#D#N=R3d+xH8ktArBMwI5&&)CxbP1!WZjCIILm;EEVmCPa00y#N6KTwk)M z@tg?&-vN4TdU~2+s~WfiAs`_|V)6Rod}PK$?g$8uWeBKm_aOv^8Lf> z2yuHr$g?!1o;xcbU$|Gsyk@L*2gqLlqXg9OQ9?2Y(Bntl0p!PF^J5dBHKwHr*?H*H z9+RKkYCQk|2e70D?(Laat^}`N1OeBc65En+Zk>*&&s84)Apv+0fOB4QUPB+);(#7+ zrkbDb8R%pj=8Gn|N>;BhuRUlV!9$z~a#}znSOgc;a=;2a9cI7-0Z`Vj4r|GZ3Ad`lNK_F{Wvi;vrcmdcSh-9PEPk^qy@aqxqxByb+D}P60`kKi4 zWvx5V0Y6odp@vxY96IXYMBDGGy{W6sCqD^tf1T3(?*H$_%XDK*Cwh>E78~SJD>!TM zHFxtPn7V;^1{Eeh-g0wZ1^i8+O0T;d096s%q32trKQj+$A6#|2c~{b&>JxFa$C27@ zev0L6?moe7JKg6T<}6?HcT54=(!@r}3$yWL$#Vdz7#%&4%(~IS6IOd^u@lzH7(Dsk zdz{Qi)fGFPJ{`0ybJE9KCvbPyVjFe{>>ge;f6e}hyE+oGvQdO69%C_Bx{2g5Ryo#Q z+0S)VWR@UQdx)F%jLrzI4beR%nK#5Bp}|D<6A~V)1v+i^d4}VA(Lr8jaM7And8F{b1#=j zWtLKAt^c^ER^rorKmA;7h}LlXfl@^bPmafwJE0GPY&O4ETX5U$%gmNk&wsZ`DC4}4 zR+#(X*xapmB{%?kFTPeZ9uB`EXeIlxDH8EbkvAYi{e^gg z$%lFZp=ofbAKiHl@3!8_E#WHpesW5vGjI`Wly)O){wXR{!6R_Tf)hcSb)B#sa!f6N zn>kDMW8bs+{oi<1i;+8_0MXe-CWzn64ldRhPZm{2(#)}D>*9>rOjWm1$Ubh$ zix9b@CDtYV*wZ<_{Vn5vof|&RgY?B%F>u~eniIbT_n{R`9s**O9)w7%*Jsi68x$HX_C z3BWij*nWWu{EcCtCPLvyepyG=l6umW*x$IvMZ8%aeB3ty#1Dk|Rv$>?Sn6XtBcEm| z_lPqvAHZt~{}7hk_zE4jBoJ7GZ=r2;Jjw=Gknt+*`d&?^e?mxjGT0GOU z5XL)u>?iU6Az@+it4Pt-)!cJaLDq!E-0}n%m^Y#ZoDkF0k7t4@9X9aLWsuQ=8?w|> zoOyn8R(X1sCvvj<7yYe_GA~xLLfhv%AcF=EB{-K0xu=Gp6+YvI;`EFVAyK0ks3raJ z4VX){NkLEbb?|KkLzu%{gax~V6C7osCYHsuo-1u961`KotK%lRNTwddRo~R)@BF{I zsDB&*mVbvwtPq6dSmzX<1_YCT-r=c%(bG(^plz0M-|>_8S{-Ub7` zx^v=IHHc8{oJr4*l7ENe?%CjRe1+)sZU@=K^k;?5(b6xCaks81R-2w3KC|l!I&JMr zg93O5E`AgCIp77a7W-vLw|+~@WsKGTZPEnVXiG5nsPgIg2)dN9tu)9q9UNr1SYs_Y ze=jjq8a^#j?0lYGW|zzA$`M(I1!q3JK?36Usa4$(d$Ko5KWiu)#KHpy#CNXT;P<;R z^IpDYQ^U3J#ojY!_{nu%G|E)5aB!VNGU7xos!`MXec&{VU*Q8a1~v44aY$C<@Q`T) za6_b?8ph$ZU#vtaSteDLPjbf)=*tkr>c9$-88Gv$pURB^1JU&d|2>8=W9~)OjG&}h z<8Mp`A&>pk*`tI6k=GuPz7@&xvHzaBW47pry)O*u6IR&M1HC*yC-7G5!CtbbX!-PM4+B~i%b;}*KLkUcfKI_J`L3D2ES7GpGNa)cXFCCyg0 z5=dbv-lwgQ-X~GoR4!U(w4XQ3G^f?KsAs4TLBw1VAB2?m*L>)d+#YVOeymoF6f+Ed z*F9`C;`)Hs_iV?c*y5i`^g40aCv)TcNi8piAeW_CPX)gUYaDQTFAw;l*IsHu`^V0RO-4l^|fxBQ+@7pRhQRREHikU({7Pv`Ro__X^!p7 zE+jz%%bNDTJtPT{Crx;8XXroU1pEo(Mm984^<7f{*&XEWIgRmx-auqjVmK>`W>+ohn$vVu97a@3Tu{>W^C}a;uu-UbGS< zdc2s~UGyu=KEHodS|%gSO76(&$SAyY;A@pB*8O^Jcy7SV^Zi*W-ADOs`_hlro&pFh zF0~24--)?C<1%t*?KbV3tN+YwIvHL0yX-F&N%2C$5g%&dJI;Q}Au;MM3(L&GVI zacn^G8jB};5QQ;(UL;9(}#^Bb1&8vO{&Oi zsKs5%rV4%$ojAW?HYCq9<@ogl?B>Vy-8u_?NwX`#xTaG$R%m?V2#0ugGp6IDQHi=` zv%SWC1up1=utNH((%if#3>DXM?TT^^TSLO2*L;t>pF6(hI4sm(D**|cpB3#0^0g}$rRv}w|0h-HF=9Xr*8U2!im>8X)jnNYzFi2v?YV=IpFfc zu1;fezAks0Q!kY|A3VH(vJd8DW4==b=Ywx(^%*GoW*Yj~-lmH`e~9?%-J_Y{+_e8N ziF`Wu14%_p?cK)6B#rUW?w}*v2-eeBO8GxMpFUG+J9=MJ*;o=oypX^}3P&VQsA+0n znQq-m%g!$X6}iu57-EG3cyR0rYL=N+C!0SFz`E9Mg6+K8TOAt7V~cvtgG>rH>a1y-(vy@ zKRpA?LUX1|l4ajf?$t1m$3TA6wZV(uVrXYZUP9H^7}?qdT&8n9Q4@okXdZ7vF4IVd z086q-XYhpRRo1wyhXzIjgXfu7* zGZHwWAiaK`-(KYUwR05MjiCm8@VR|LD`Lupr1AXJQ}8_ZMt|^KUD51rJLVYIi4(e@ zqx}H5t>4@`WjFH9!(SvtXkq2;xL>boLO)!(P8iH}&8-H9$Holeco=W*J}(-XL6iyA zM*Oe=z%lpL_9%zORH{WyAY{_uNS+M)hlToH$<#?2_pm)_Ysk+AiV{nt-|H$^rraP*tf0G>Xv>Zp>Y=tV;YAJELVZp^jcY#5wMQPT4}9`>qX9Jnw(f$rKEFG zWcoL2HIj5FgYA8}{d0Z~YMZwZvOr%9df<_*Bo@?Fli5x?Pe1VF>u@IhZa~N5gK(v^j!~qkF!Apdf(o*v6A79E*|_tOc{Zujis14%fWgZ?tVAK5Q(KRhn-lAI!JJ4 zaBsIPfH-j5-7mH>FFBdSAo0G<0`DYj(BOMVqfB^F#?o;)hcY{=2}9b)KMRZxO{RudeZ+555~HxE3h#qYIAa&hvpYjs)?KH^KWm* z6=JZwV4!({5S8LjhMU-gYZwsTrLiN<>okneTU!!n#dzk#RF(k2*Z@Xv1$IloiN1qB zwa+;w`J}vNN5<{eNd^yzv798|S)r?j9yMz{=OE1O0uihXsna(xI6Z&iF{Ubh!KsQ6Q zQZ3IU(2gs}bNZoSr(NSXqwEj6Y}9~o00~a?v5-~X724(y=&d$zwv5lV>XhU~mYSIU zv!`+nZRiYett>hC-C+h-h+~-_|3>$iPdrfkBRWf@i`xamfj_Lr2NB1A7${%!Gk2Qt zh$=B4oRsiIml{cz3!f0^&(#{vMR4`xJDRWApKo#V6}lpfS!Wq3hDCc2G7Y$cRX>i- zS7ZM-q1EC1pIwu0-^p(Z<2~p2hp^eCa_2gX!}k9z>*v+hnDO_KXMe!0P^4`5gaI|! z82!v=Vyc3HXf&Pc8666nTx5`;#k*HDPD4ydjj=F-*B|v&p4T%atn9;j2lIH23IDb)7!JCAt%IwuZ}TZ}IlUKES6f{EY>LXZ zeice(Z^STOsFZBY`YWg1U&pQ%z-?Dqot$}xIQJS$e(C>Ib(L{ZJ>UC+fJ&{XhzN)- zDkUJ@p_DW%C7nx32`eliiV^~XbayVW^wKN{lG3o$5=wV>{ucdzU;NLD`?=@N+_`to zeD0ZP+-}zjt7aq#AnNPwvqMJ`o(ptqF0iYzEm=_^4 zK86~BPQ}T46$5hCds64Uc6svtOrsvzfe*~xm3Orc^*^K2q}|wV0{Q|Ls?ba}8M6tz z-#*2InPoHaI55`M`vn$IP#1IoffRj+RYS(=#l&I<97_y}MH4a#F`P^OvCql@z;TcGy;slhQ-r(ff)|Hu~v! zKvT4sSEM|qPx)F!SbXFYcNs+@U~b>Dgmb)kQxedCZ1BcyXxPi&1}tKCdA_|TUpjDI zJ1@(|8qrS{yDSWG;^tyCL_`dXyB6CcZI;t3aKcAUZ1u`!oEY=OKK^uB1Bs3{ffv(i z29a17+9bh6zASc!k}5ytXK4sZjOHKD4R`J81Y)RT6Yw-@Lh81k}%d?uF<@*ePt!PXn(jyrN%S`OcI|j3PA71`?u@0LAc_ru512(o5Lr931nBX;FkpXm? zJpXE0&qjq%mrKwY|B$xVmUXhpe4F;%=P+Bp^Q*e&+@yg8=TP>xdAyP)%smegwsIi3 z1V!nBMjGGrt0okaL@kF+Jx7qKRhG_H9(gzhyL_X?4~=n+Cd#(l%czcr)88{3uWf>s z?@4KP^c&bx2|LF+lq@*ZzZtA{faby&CI5I!je_V6c!N11W|B2EqOPg?WRQ_JF&ZO> zFihFXoeKv4GRx!EPI)iRRxFl7zkYSpm!!nrYl8ak>gtVu{#~1vMfhjOC%eoR@j>h0 z=39fuXqP7!pLT?S)WZs9piyjlYPaXBvnF+0pKvJ?YDFxsV-4kY9yeuzLy00>ss~Sf zf=K`YPg&(6=wW2W@gN40|big&y9|YV=Bu@+LyV|P{6u4^xtX0MD%Aq3+Du|6#heJH}ZMdp<_z&x2 z@;$D8nU>hfTIl(16`*Z(AGkiD=xYJpC;JqqIy_cnAsZ}RP}WZNyZM2tHB z?EtlJ|7RdCx;bL0yO5%ZCC}r`-Fx-=mVI#)E`Bsh_#f+foNw+4Dyj?Jot1$ltq>`x zED;tYSi2JEu%tI9pb~FBVpiz8aieUe46jT7&s0wSGiiIR{5Kw#uQBZ@Ry8hM>9l|u zY+&rmygjY@G2|85{sRp+O|s@MA5Pc`)YL;Nw{O<(!1&c~)E~*gTD#)0PXR#PFHq_> zkjgy~a1Fp$LIzmNYeu?31O%URUg`D9w@lwcMwV7p_CWh5FSYeB8#;^?@ef?QVc>VBp?4@k(IP)1EcsMJ9K z@YyU+qa|B!w93yBpzXNFbAzzW7tj~`y$^<2GA=C*h9;$?=`8Ci=_J2`#m38@$T$kl zYP^-nW2ekT5D)TZ-+B8CPhw&06=oH^Hs=NyEAw$jHH0U6DM`zGjjwZt1I2xQ8TQzr zQ8{)+31?O=Xg|wCft=3GZ8P{(b$qx2T!xglRt`}0!3DXlCxkt&-We8?Y71VaLOBrv zsc1wQ0rmYuZK0O6uOgb{BqnNnFoj2MrpB4mS=E6a&_cHKq4|g8A_oPM#K}B#qRxwb z-L{N)64m2BWoGJPv$$(%1C^b9_wWWD2llc<(ySEzv_za^y+@YH1lWN`iXD(Fur2}3 zH<@wV!c9wF-E&{gjvi`&t7ieRrORhn`<=x*TlckhUyh0eYt#w8&6gN=-TC0ZjX?c> zjUaTwj(*GssNNNG)jA@K(qT!kr#Iwtuoe6$m#|n#Y1ChmLi3gku=M#H5!J+C18L^B zP_HELdCbR`9Y`e&tAAZ?wTw1>2AF%^Em$i;D3gAxKB;U{P?tc@oR3{)t!E98d-=cH3OzXu5zY5q$p#l%{wrzOCAJs0p>r}Cw z3IpbaAW_YHw-O;ifg@62w7|7QtxBfFZ<8e)#B0SoRwJ-w3k0h=>}y82tcea|fp|TN zHnv7;AY@a>QRzp~KqOVeJ`>Pd+Tc7dT&F!PHl=utE=@vJpU#6x647u^ZJ8HHA{gmg z8mmTH08G=UMGe%eneh0`-2GPo9|@e+#iWoIkyg*8O~;TFP5J|>c5smiemd=6o*q#@ ze%SP)kGXf%hN&x-SXhbJ);$@d^Ay)op~0HPk9nfjzD{s&>FCIM6x=Y!zhzGzQlEEU zZ4g3BgO18$zo()6CwKoLKA>pHW=IOer6s5^&($KtN=#cQd>2kE_S)7Ap{SW|yYLTH zLyjpAj$U3xTYXx~wFRt}Exui+OWMdfW%zG(=2YRLiqbzcYMevpX*NMWyG(zEf!1@! zIGbkE6@V==cS-l)>(N_xRPQ(WH)jN!KfI~*SvX5g{QOYQ#Uwz&-y;-({1{KE@s2+xg%cOx8n ze7~kN=)=eBf>P*5-wUWHu2v7!cqNu!Zt?<+-$yh_lTWre;}@2LCxW(xP4OQ6KDU;{ zu?x?&8u|Vn!J?_5IdgL${`c3oUP6;D24?AKu34=vF9XJc1#w@<=GzZklLxD2Q#QD} zNOe~n^j`-~-DmB4=j7DY(_=>^&@+r7j2mML{o^5W%Ckmaw=E;I<3Nv#mwW4BYwXWA z9e|IPmuf>sUoRatFU-aZcLhQkI;_{;DH!)>O4pORERGu!t<{x;@8&+Y2o`ioqQR$* zE_qRm6)2_u-5N&pL$hseK_ZcSy(i<-`qFdZ4{qdI>3`Z335IACo~b&!M@8{hx@glb{wDe6UXd56|_hA)dhEuhNd*1_CP@b3EAh*&O zvD;5g?;M{Z4mr*&W+5%24WwEVS#UlM>&@Nwf=SsJgBSBzOIrSH6x8Q=gWE9~*=C~P zbgyH$L3=@E9Qbji2#OuEQu?5LT7cNFT{(b6Z|32~apR7BznXtQ0T8CLnh z&MhEV2pd0@`y5_tx8j&IEO{~8HZnQ8i3Cp>2YQYSjp#j~+4nu3uS0RJ6#w>^D7*+M zgL)idrKM1?5pIg9x0|DhOJ}i*yZeNQ3Y{8ow%e|}(VY#W_$Q_@VW6I56{A7}@D$cn zcV*5t*4Wj;c;3*wsHQ;3ghT>or3d4MWGqjk^@kofIy!mi2>H zmnb4@5@{(1xLexiQaQBkj!Q~OED`NRWDr`rV!4rz>gE@=Fez%9gr|19%Gek?Of5FgRYmB2Hb;?YGM!+(LnR&R z>bznl?Y)9w?bF`N(Y!To@19WV26g*6N6msx6s@cMgozXRC`tHw#Hq3dO=tX6U+=(W zUgXL`@1--cFaZ817&`s47O7!t_7k_~ch*->RG?JD!Ez3qq}**cQ2%sVCL-BA%cIdS z)`lZd0Ef3*BZUhgT};qjD#bj<-JsTccAX`p!2%)k0wGFbC&g)bXp!Xy3S6foDhT_N z;d<*zyQQhaNoA5b%cRSA3 zId;&?Zo`?}!?J|Q^nmveHWEJoOYTtj^8niZEaDY@&LKvJPi zO;(H((V(1v8A`-t2h4oYjG;j*=Jzt^FVkLhG;5|Q7p)wZR~b z*Wy@3%%ck(Y2|M0in~e|)?*T@$2|x|V=N5~4^9Y*I0X~Q6~Z20sP&Si%5@>E=r(eX zkT^OjBf~d=^L}}T+S*BidPV9UwJgsqzWs0+__K!OdJft;{S0>3K5y}^(3)F9C3hFo zhsbrZIBeMW7;z)dvTkr|9Kw=#sAuYoRH@T`O}Jjwd({^40QHPdUP^zTUa27SZfROU zcMapV+O+Gatz8cxP!&Z`eR=pfz;pZ95{KB{J?YXrzDfJfMRhP<63x{X=)D?L4tiTi zEf$rhdvr$q-P>c)+ir3%pC_UHM`&U6*#^;EukRJ&8|T#=^e=GyV`oR3vu;|10MIok za=G|Pc#I$B*EWn)_*Dk#a_sXuHuVp9eKyoxhX+)tt6uRI;60$(ReW#^V`^Y%UNy{G zp(poW@0KJR%r5YnhR;bL6r|E`YqQthRJsQIwv^DkD*4Ab@)vBrQ`H%^?+*r_KuJ)t zy!PIK7@8okbmpeSK+Sn{_EZ`U8L3~*HRPQA?sV{^iJd}?I*6Of%0M3g>;>0w9}8fy zxZ%d@(rq#=%VM?=A(H3g6c}sJ_N&2-y>}te#Dnf;Z+V5*+o}*`6980+ZWZ7t8)7H> zz7LjZt5or2WR{eHx6uic;0ee*S&NZ9H@;6h8^NnHNH?w{8fNYyBvt4ULbx{2aXp0Q$4u> zPr~8+=5Qdt|99~Am}k8kxGf*JNXY&1-<=CD3?Xqlre_5IJz3o#KXWFIc<$wu(Q#rN zZX2a>Mx{ydUxo_{X2gqYH~uABI7e$-`JgL#fk}KyX#xCWrBI}NE$92+>qQV4LQIHm z{ms=#P>EfSo4nH=1r#g%m zAixddZ2Ni}zu7plHGK?c=peI}`@q?X9=h^?y!DfVZueEmQHN*5b?=|dKfcXiJO47A z@%I(k3E9u(?&@xQyeGy*gJBM$cJi$Ey3>u`DZfd+3xZja589NDJGmKf>kXW=D;+4I zrRkxSnOmnxgr}L?h=9S2lI3`OZ}2wWPxwIyL~#4E79pT1XE7?$Z-Ipw-=Qe8bFulw z>x@^60vHDoku!BU!+$yDpWN{#bD3v%{aiM3-LP`?K+cX!07763Li)7B6)2cevm1z_(KS5A9!m z*bSDvAd6^O3>}5`KUDK~T6p`m%_1;l`)HVqJ}lm1{&~8k{8oZHZ@!TH*bz~8=_97p z`kQ3DzHrodV=7(Lk!HlJcy1nnx(DBzs2~aPrGdnX4uU?Ev3!F)sK>GKN9HHn@vplM zMukf5q$+DWv|qT!rfc9^9%-gc{hHk@tNOF-vCNot=5oBBOlm(o|4~kK9h46C)6g%n zPt8O6`*S@ri;T_v!hM7K)M004Y7&Fu<{c14{irO$5O&(q_@F z4Msyp5dOoa8efNNf(#nR=Inhdm@nKgix}T^I`$@slK7B^rOQ#@Dt}s+JPR*$g{_>s z9Zr&3*&q+Qp}(Kbf2YqY3nJdl^Rexmc2u{(JS2lH*8KdOr|!2FtJaTRcQMTjZ(X|@ zqNR_u`rT4YIzc}!1;uU7>n_fv{0_HmnEUbX7Ris&=-tLHQUSTl@!Qeo)W zJJh0DQK{5{4$L}l#uTm6``{v+1z4ozV8gSi^F)b83Ug%#(jFrCw3)7}Yrn%O6b+?u z1j12;d9>^zqpQVm*A15|b0^=V1=)io*QU2>~o3=hI=X!EEB9;9#{n_%zy0#}) zU&FD&lJ*su134BORttJp-+r%*6({bao7kll=lfQIgF{e;7138r5RxInHy1+!u7bi1 zEL?@Zx=M@GHd6_a=F3Wg9;p`aX=|D&^~e7v(C7JzikBI+@vL|Bl>96p9fY~op|%}s z(r9BeV84=8R2p5nZHwW89>aIwMk4ehpOP)>Dgw)K%pV41IYwI8ipoehpccg^#Dl!C zy8>$TZqJ@yyRLVja1q`ddJ|eHLL~XhQ+scR#fP4}yNJ=y!q<%O-#Z>dbH;=8k?(Ky zaOQ7#=*F01R3U;MQE20-g0yvi=}}Lik-T|4jMqFzx#jDZZ>`BBk>AhyC2rE$&si=H zsw>i0+&yKAi3`@xBZUNFw@M0ciW1v|GHFb{7T2;T86V$OD@r6E4d!v#vw8p0Y>27h zJt_6jBPe^oeBAhy>-dv|wz%dXT?73|+m5e!-=CJKnh-jUHlH|2H$RLA=5SNFt24-@ zTP32cmk@7RT56D3URcWrmrAcRaZ;_Bm8unI#T>!NHpv@}@4}5N?Q7l^g+Z3G>Y9aq z`L5aS#2}R2gP(^qGuJD}j&F*_z{--OLCc56oi&vYBDw-Ao0WAvb-7b)?*p0nqXKW@ z{x!kzB2n9p&!JKc>;6?UloeC4Uy4|2S9WekZEVkYSULK(%Rz)+I@o2^R^)p81(zCu zN$j>|-#d`79f2t@JS&IV1>5S@H?ZcaWL!_Z*Ey0p`m8Yg58ozWh3G$^0u5C-!Fi%80neMlQ~H z_N5Ov^~D|J{@~D5h^Nx~y=|?-`n`osbbXOP zUo7}RN~p5MMNK*y)i9Zvwf0shKq%{aw@RSi#!3a5T6U4gLcCYkn#kq=RNK~v@;a(Q zFRt?;kPnLZ5Kqyf)3O|4&qKVSMdOrI6jA#8%csmvk$%>#S`$%r_t;I1P7+7%9Aa|g z|Ac(tQhAExbGDgb`}_w${$AMAm(zSW*-_xXI4NQ4P)4rtu>UDH^5Y6IV$TNG(WED2 zZ$)40iV4pS*Dd^TWP7!D^(UQg%GRYZd>)98u9k6CL@uN}+{ba*DL{GGn>c=dhnO%Q z;-SOyD5x&R(sqwjmMwd$f=6mj@=~T&$knbq;6}EK_oMJ2V^*QzU-y94#badiNg?Ro zorM6dpeIIlzHh!ipNvBZL{KU^)o#aqT5#PgZ+i2%RmZ6B4!2#^=(8jt8#wJnN>jbK zpWM*AN51pe!MrrV68qs61zeNl1CY~>3wY4 zU3B1D+$3w)`aSEzz6K>+Gqp}%RIVq{v@){Evh9&}-rOhDt^bAg2vBxwIjicuy_Tb6 z)u-)lG$L=DSF!`IQp+ods=7z-F0VPKn*J{Cu`qGjOq*oY$IEU4U(!mGo{qJ~CZ$1u z707*uMg0`9pB?h#yQoiZN*Z8?e3A7WSnE(!>ytOXNG5V z(E$y-m^Y=C3Y#78j2CYu2ovOdXy|n-tT>q)SRNd@l2$~Og$}7yi}Q+@awM3S)_LZ! zc{2R+34{d9D1zkCkJpGhdw(&DFDq=vK6X+aYKxkoPk(AzQknpoEoTkp6 zZ=UStvovBcI<)UJ zp6l7KVA&Gd@*6uE=1la+vMz)>$GM%BGga<#d^kmK^%m(T4h?ls%LER)H9y!fl0z_`|7iPP@)*>(u(! zrmghV9Acs~e<@6l2}baVjmSIJbmjdDFHnzn72Vs&>`v%^SA=CID8?Ia@Am3Uv0c%G)B9)mkCK0hulMxL(g^hX-~*3HM^0uJj= zuyq2D@Fglq+WVV0oO!ML>Yd&8DOd4a9rMKAZ0%gikKjIZw{p~RU1Px8+%1T*FYTfGvKt?>zGSC^9WA4c3iT*9lN*06SKlEcGUe zi;EA%YUb{pk&=;_Hu{P8^?Fj`lkRvU0|Nv3O=~l?a|hiC7Q2{$lg%I|pO@I_8XJiK zExc%=!$c{}ZMiQw+pkf#1GzV#n4gzdjkJuWB+AUEG4pzmV)}8|-WRgiZB+^waX4M$ z_O~JCO#!D_s&TVE>xDL5nz@JY{k0J>r`Zo>r7;~}bbXK!Owz~GZvCz1MkRj?3D}8A zb!MIyW^5YwL&KWRPjR^E8X@&|0q)cnpL!F;z`H#{L$UpFdjya5(d&V~t`+<#X#SI8 zEAw{Xi)iU9L`1J>|Ch?M4j_`L1m+G;tjpGDpm1ApJQ7mIq4#B9*y0s>K zm;p&}#56LJUYj>p%?mfzJ=>hBq6?&`2#uu&cKYz)gH7*prc%KD+rRE=3~g(Iz$`3s zpr}mTk^CabhJ3Pn5yi zFNO`6`u+LV!RC}p5$Dx*AnE`oJR+q!H`aC_r0>9c6R1hA$rpmM+61`4xBpVJli`rx z5fT)vem~?+i0Qa><#Iq1*dLfzzzJ`mg6i$e$P0JXbZLByEc;I@=y#KWucqqZ&CzOs zFFu`^x%72>{Jo4|c(%ys>>TrT8qSk~wEPr24NnzIVq6k{V^TY%Mjk<|FhDWeAWdf( z9yI?di4XTXP{wb&PMclcs!Bs|h=$#I+(PE^kihS9K5n&ln*zUi z61R-23QK(W*To!0U6~Ag9drEKfZtkrX4GNR(O1gXy6p}JvR`!(tV4VDz(5w#Y^r$B z&t)hQ6j&Fr5tq{_5>)lMtbJoZrqs`|^csC}qgj~2?pF(P3Ni#;Id!NAM zDHu<~IBI@#;a$r+?I$?X%B0J!vLMjT$w@FL%L^P~D=2&UTtn}e9u=EQiQbUeej@D!5vLzFyr|X-uQKe7lBEK@}@ol05ryKfH4BrWGXMBwi(UuW-L=nEdB)+9d;r${TfxXP%@)%b`%m!z`7|`(r_;D zc9%}%F^d|?EQFr}hM^N==sfH7waR7tN;mww3WLrZf`d(x_o1l6N2D<|x(4UjJRree zC1>u-fm!jS&0lZjRWg4i^`ns1SgyO9%PtU9&(7FgBzQXtD5+o*zlBl;j85ihkH48P z`xFaWHq1p(@$4yHyCcwG_oyI|N)nXN)Fi#PnwnZj#5NL#PU0va%JqXm~-*nvg<86^Vl-$D|k%-`PVdgV4uHF@+ zII&Fj70J^P8yhP}oK1ekh8={8i2LlWq@+&OOFPN0Zfllyt0yNnIsR0Z@RW%zl%%Rr zJUr7%wy@J2?aLU&RwA{_lIzAeJw5&I1{@^xx<&}w?I<8#8|WYVH7==ma@}F%0}LZYeSIuGAni>D?8aI_0OVIXsloc_ zqH0)VZQO<+0)b%QKD9MyO1$=xtzl)YrGA2hR>J8*G^X%6aWW+;5 zEAynTQ^5SsX*}}Vz2&<-D>n61f?vMq+@>F40QqnhE2*Fn$t(^ZPlyxaf}53JFUoB1(`Xtz*wZ;H#)c! zU99gmCVGU)8yX!)RY=1Rx!52O+kM1vXghj?5G3~=$IM5nX^8LxIZ0s*T?HtjY?E!b z5oJ|iUla3}&y<5@a1uNA8PKV5j4p@+lzv$~q?R4rWdG$j5nlB{jm@?qy5vi^I2~3e{^YOm=8GmN-4d0@7?dG5OjEcr*_Rwx) zf>#eOtJ*Twh*Tal)KG#zsJv&WU1|B&2$uC7@}Q!O`-!V-stq%X;&hNq?cNjOlTEpO zFlOZRQpb-04C}7KZ!#7*~!(L=2O$&l7G2f!zn1DdV8B^>il?JoKU1L}u zkf-5JRGprAyI0MJEF54YP8}9hXnB zRCfyR18I-YR;RoU4qnItb8|So&GP*eTt5@eQLioModdrQ0)^~^Fxsv?Y-|P3eV)=5 zmgux9k}YJb-Ijl89eSc7sg)uXQIys{fSvfn>`leCk8by#H%FVRE!4XVF$cUh;}+9h z*fINMaT?jx>6?xxtX!uWu~yfpo{P2;7^57((B$K8fE!+hM0e297dFhpBps$3cA@nR zBfcV2+#M$>22Ngi>Zuts`Mo+V;;)0-cb9T4nT%%2VD)t3(IzvZ)E@DDb@J^L6jAGZ zmL$0dsqemV3zLlf+h{|hS$v9bCj6IvUdPC7{ORk?LN_VO*5;{>safCn;9$b3h7k1# zePvYdXgeySxP2oH$3so^{x(GvFHdUDCb62%MHc+@Aq1Csr*+YugWH%%{FJ2}W6$6$ zqP0S=Kk{}b`08CAVaTSzbdcCWb+J*fZsXI$UK$_IgN=y=<^)x%vxaWXxeZwS?vY*R zeUPiSQsYj{LMc;u82gmvERm*oxb;hQD}KX1p& zxQc1#=)}7j>{&5yo>K78`TP9z7?2Y_TtZb97z78ej21#63*@})Yi{No#92q9p$)+3D2>=w*xFcT0R1}>Kw#+6g*@r6_eo0LM*!p$E3Vs1Q( z8uD*GnN#d*EOJ}6htO{qKlzpf^QM=0!MueAvu5!`SPu#pXyHeH9vmKPzRZfY>x)n= z&+x3->aK^P<*g3p@?HY2dewP*MD(arF7|;JoK3xFmuU7?YRCH7t6+OF325^;o9qb| z0&z!LFMV2WZpQ^Rq_&97u16GMlYYF_Ld^6eiD6qWyEH!0{^0+9erEsWPZ)c8D$Zng zx1^uU3w$v?`1df^w2_EKmj+G9=&}0fafxQYZ$@@XjObC&JEzn|=9iCjH^eRGBaQ@m zMhdtF;|6wvxT*fI-QvZk!)@__Jl2ITA8}4=etp7B0&zsDT&BEyv$b-X09={F+Rt| zP!DnUQpBrzf81ndLOnw|2|7+Y)&Ez_3t*iwnbQ}E=ew;%XiS8ZKfWn_vF`iE8f<&7 z^o~4aQ~ESYjCcRgT$l>Q$0VJD`>=73U;s4pMxsfAm1>{J56>QIK`xt0S1g?o$l80 z1EwMuL*Fdd$uKj1`DBOo-agC;nM@*%iZrt`IscruVui^BEnD*pLl~vwKujK{o7d0X z#?J&e>N9O}CeXe(2IUn$6vlraGJcZ_>} zO%%La9c;y06cngSLwYUdU$v6Da)~UlT?X!rD2es}dgB;?pk?jA<|rHoGMi*7Bp0rZMbKx(HF} z7t_(Gr;vq1q=k1@myh1Tpjhc?0C*tvj6efA-c6O|VN2h#d<_j~G6zLTeckYdIJ^~O zi+)d4FTC88n#Rbn8yx4x^9?Cp-+uO^P$RlHHe2I3yQ*F1sPo8eSb=S81?Cz)V$bLaeb6MFo0PsT?r zH}&mKN(wRhxDDM=Mhx{NnfWu%pN8yqGYRLLh3%7tqZ*cPHS~N26QOPkK`#A+@%bHN z(aRR+?+~pu!$V60OFX|n?K9yo_g1L*hI=-H83a8eK7j31RZ*fT@efNtZb6ue~-T4mD7fTfNp0u3&wrv{T`6qZCW{eld)7=#q|aXdGt* z?CgoE)&9FCs3msB2(pyH5nj3L%yup59~OXk0COPryEZo#W|Z71IeR&cn%}^Q!wwz< zHwy4vSs4?wrr?116Vye^#y?H?TAbMgy^@O!^W@=H`AcfPg*T z*F5j>#~myk`b63-65e}O9wd(G45f(Jl?N4dAS(BvHO0rjt)yp%&jY43@F=-S5xW~6 z;cvO$c1~1?89-txo|x80^)!|Ewwbr=PmwQA$}V1xDvo6B@*m|{N;Wdk+~hVX1Mcqp z_`b?RfinwB^j2iVa?x~`BVsx24Y)$@E_H!GK#8KZd-?Kdwjf@&Ov14?AW{wVYxS+K%Z&)htXNhEK{IXKK6S1Tgj9 zZV5YFsM*w1!U6|1HMLA?wd9A)K0qO3miF1JSxy03D}W|hf+uZeknuZ;ho>kHZHINi zjSf%-W)px2D!jPMCXEDCMsC#Fdm6J(_susZ0(Bo4x#QA(QN@ga=q| zyZ1A!`OZVeB;E#47X@7d9PugB-QiyF9Z9p?bOTKfKlXag%$)wsP2#M;4m!u7ujryS z7ONwdRpH``oqu)n`cGYp_RBvW*P&Stz_!qwXXh7F*Sq^b`%H&w$(!agzvG)(7I|Fq zE;x+mI}m?WcLsg^Yi|WuAG?`QE;gnx&9(m0UiyouJ02=n0` zKy|dBWk0LPoCI9mpCH0ROB!k{uL6jefL!@$)K4RH0d8gGeDlld5l0mBUsnhL$m0m$ zLj{FvRaOZR;4{Khm6IDP`93rJ-ykOc4>n#_vZ?L5S&&47q;oEtNo&r`ShXV8+kcy3MN`lgCeQ5xt_@MHTxk z_9M^SN);c;VRLAFzdWKP6{z33e2Dvm#9W|4+B3lQ^V>vFWQyq@+-Zmh%x?1E7}md2 z+3(A3RTk%ZG8_GK#J^c8p``yp<^6X@`TvSiJ)u9^YQFWaRhA_EMMu@6>0}0%keC=F z4G?Ot{(g@=Cby*I(L@Fii`5xv4n(hX5yyM0-@kuHo{k~a51s(;V@>ENL#%$YUGQL# z?@#`WCKB9KRdqO7VFz4Qj<&YPqg9QlEkCX9e^4=G`_jBU0O6EZR*Hr|T!Hn35WMfo z<=0lX073Q_`TGl(8|&y$A$sNoka$(#RT=~=ew~9Zn6ljbjEs4pWi#DG{QA|vOEs+5 zg%ySu?Ng76%1Uc1s}BSx1mf`5ujiEQ7vE~X_!d{!@-Fb{PB9$lSQHi(Dl-7^-8aUY z7c3BGOlQkO(sPO?%z3`~DYwx)kd&%pE{1z&|9HJPvcY5U+qZ8(`z9{-qVk^Z6nsZ> ziRhC{YQXX8Vs~8AK}GQ+()q)a^Sv)%pc4}rOdDw9;9y!}Azw-Y$lT8Grx#|gr1)a- z&CF-tW_;#^qXVH^M)>9`gFmpYUe<(^6q&{1XR%NVw>W!(?9ibN0?Z^D$HFu?}>?VmGud zTrlwGG9|iXettamcNG2WAb=hJG3S@t2_ST&Ks(_aP;+(s+F~~T_;TQ404lXVbHwiX zy!rmCk-C!!@COmV<-Ll{Lc!U5FZu-E{y1=1mbk9#iM2mF(0y27DjgL{^6_T z;vb*DVOhBP*Bw8p?0KM1AXfZtxo?3!jKWac8sYTig)VLVC(wzF=kKxsVt}1ZX%L_U z{OilWr$m5Qd%|~x zTHC%u+CzVdtQzXWLyNXAp<$L%GTuNd=b(?f#2+jK+D>J-B&4Tr<|+3E$q_6p1S2Df zn6%ns<%DL&^SE#dZ8sgv3lXC>3*i=AGBKHx5m7*Qki$$C6cwy;=kMF?0N(tx%ba4s zps@>@(U77=r!=)^1D$xowH)O2A#vdtCjDKMjbh?Z2yd^T`Bg=Hf4iLnbgus@_;Qt3 zq^TH+e-PU7d{=KN8iUjPb%J|Ytq5oDwM2#*T3Lg^CARgcS|ib;UKyJ%6S2D?!BX&0 zq3#}!1Avg^1gADg?NB7F_BTBXq1ri$Rc%>QvkmN?%kz;|+{UOC~__RT0I*6mmi zTl~PriQYAkL(2`Sc<}3I0Sr)V?2b6|`K%g%8uYU5wnHu2DluNFQWb|7%jwu$yTY5N zve-g+tm}*c5@4k(n6}E{1w{59nPkh#Lj-Q7zEQK7M#6AOjuMbnQw+4*^{t$WO$^Ye>rGuD` z!>uu)XE`lPm59dre?bj_yK~=As7m@$L;Ft2J77lG;%Z@kvGU)bq=%$3G&4byCb>}1}rY7a)1%wB>N<@q$OHA8iD%I zM-o|cN?O)QYYE@JxPU*4ZyqFzsg3>Oct5oZtf(^%q#!7-p7KjZMye01HR<+q@OeCQ z^E3|Al)fJlEz}-`8vUlFl3j`uKJi2Gd~sW__+X&#=%|oWz)}qQgG-tjnZj|oiRco| zE&!KIaJNr_g^Qnb`1i`*>dG1t+vxbu}yyf;W@0#~XeQBe=he^{nh(>TLp2>)ylvtPrtatMN;7ku`>tC?WNT>BZt*MO2 znUtsR+DEF360Cl8%fTy37DTySgDG-mK4mdPFJJaq`RwJWbkG zQ9JNq*HKg#tEQh?&qJj#Mtx!%eW^=DGdQK4m&YdOt*pH!ZDu4552b^7BulScn2KOo zU7U7=)k7UQ+p4*m5)j;6_%^=$jMu33+(&(-wqe7To)L_d#)E#&j}>kv>r%yyJI{(u zQ2LtSVm!6(BG=A8m}fP!=o(e^14CyA_{mvx^R+=?nJ5W3eegxbHZBSm?E+v(=E{N2)-pyHHW zo3d5nHZk9xAe!LOPcIF*;0*Hug#7ID$Bfk{e=A0NKG(Rj9>9Je74vj{&e(-5LZA8qaq8S;pb}tadS+=l-WQ(}!mAUPy*8qFn-6aOiX+y`x+?q$D zHokNn3AJak35ePD9PivM&fyz#vkofqaCcK`-0r5IWb~_H?JI2?cP*|v#O+-ZXDtWRjgRYbd`IWwTh9D-tUHga^OMxI<;D&e2H-A8oC zc5PzNdAlr{IG=e}^;LEAmp0ithvg@?YA6=(W}s~grZJoV#}^e-tZkUDVIb#ma8LbSt}&ep zoTFdT;qD;oSAgY?uhq{hQ%tQ)cmaT=b`FsVBJGT2^(*w5pV(4fW}kF*{d_%pOINuX zISkN~0PCnVV{ArI&GK_>`VP{FBwVtq=Wlw_h*v9)P0eJK777rM)~Wqfh=szALMf;F zWsNS=tI#2*3wly_Y>09|HnCe{qTR^-Sj`EhX!4bgllolaV$CG1Jw{ca&Ce-mJ*=JY zsJNHLUKT)s8%>~y!!PCiFnFW!xc@=IdY-t;+tfQ)siQ6xDB1hSg5BYTU8lh~BT)(C z36|E$zwybeKBj`Mm_KObR%eVB8pss&6L*v!x`aSI;tBLTikg+-d@c++k$)z(!`Z9N z$xQr(QLb_42eYYfUHci#3jjU&NG4zC%kUm6C3n7&0DR(cH~kn|(!A(;)z40lp~Wnk zr?LeCDSs3OPW)JzB4q0VKK!nohi- zqHifoRL~BL2!GwPS-DQtQ)|{t3v_Al^X!{$V zyx^{NPqU>PtT_kTviYX_0Dw=_m$70^@NhShgFai|BBH&T3+8hzjZ)`mk~(E8nP@Pl zRAlJx#h~qNl!RngVj_eV{{QliOwtULt3;VJXi0g3e)2wIhc` z-m&X=*h(3MJ1cTH7H0DaRMKKYyHYRU{Ji__>>dQpb8f%Ch7O-Dtie{^=8E>ZmC{kF zo#F!klq^lf>R$_Pc1?l~ntaz}89hRKj6(VIRWj0y?;$(bV-D_nlcv0Icu z*3T=AyqE84KgTW#>!RIeHq|=mRQ@2Y=+Z7TMHU)yUd}1gZt&Lp1vgtBemoFUUvoS} zuUhzB@T|8F7KtmGHx+q-Ni|R2TZiG7;uypW>EfRyo}F`sz-s(VUrT6&a_Z1cS`O5NWr+f*GVH%zu-=MHa?9{isn^EjFisRcsCDEv zg?$bdTU(vg$6%#n=cqtl;Q@7Sl-~DyMu~z#bc_^Q1yl9x zaqc>0oAmj4dL>r7n|wAm&mw);Q?Ji7K!8=)Lsuum+#@nmOMd+)H$1-M+RJb;L$~HlNpVYhgS@?+qIoEsS3v zG4wu?9!l~&iAFZf0R3j^*}FY?@L08%^wpIEn_<<(Fb1e&Uf8s&8;`$ZjYdl6N)4RD zRG-@J*BMx(tBaMpqGk?E;_PwO^EAo0M~+>?e4BLmH)a3J2<2K*d#dN(2ql+Cqm@tEe~eH9BHj5i z@(eE3jj8qES(-mpUr)hP4`lR<85E|p`}zUR!}K;GvokODDBR)6Jkp@O_goIqn_Y_t zu4M3CNAibs(1@hac_IB|n7d2=K_^G@4bGRG&e;D(Cr^*&BWVQ8q*@8vl_nZ%!-s=e zs@-Tl)hB0ANiw;{XUp{djWjBlEHEw zfsgykff#sZi=f*N2UHzRpyGXHI5IM#r>8gF^ho(S5s`McDN=el3{1}9d*>k^>9yvY z0`<4gcXGYQWq)b{Fk)zE=<|o4vgc_M`%5=?it9@O3KD=5a{vvAoLPp@Bh;^^BV#_e zJ5~tgppAY|J(}4>`bQi%ZxdraYdt@v(E=xCCulHB9{pxoRy-G*AdbI|Gxb`=HJLjH z)fc4u*>!SAhFJ!|1JrW!sHsVsfu7^NC+*%KB|0x&0a4{2UD=H`i=BPhbN1j{# z2jbybM>`;@{lDV={^~i zA1t}vBAvLEe7c^;$Mc&p@`+5;JW+n0$haSB`o@*nt?eCRM^$xssrSJ=O;ssc`gY-B zlT2~fBani4sWie|Bb9GgB%y1LJ$H8D@!6-?;NaH zZMrj+*tM`-^de9OO!P7E^6OVQ>Rz=-u$s<)$hJoMefXbr(#!CB$C2wQw- z62+wN+xoqcf>-i#m63;?KOI}I^Jv1DGAj1ppL`#v=25}%@#q&9{o~x!Mp?C{xgulr z*5aIB@Xch?@Wop}30JDNy-!C%8|qVDv?<3Yt*-#s!Sql=&M%SeZr0g zLS_S&<8*N@n|Y6XTZ%8bC5_X9?0%jdBTgK7erMw9jYX3o;*a~sK2T#WNqYqH_gIw+z3ta z>H7mm5dE5$N8CV$gKLEiEC7_Q$K6{yola5}cV1v+TFAZj>ebXy{j5~Tn5gwh_)HAs znO{@w(IOuShF7XufBMBOU`R)i)*79Lah>tfNu^R1?d#uwm3TV zhtN?7eC|IQyl&JwZx?N@*xHsn*|%ot6e=3P_bkh*)|eZeD6gqAf! z79M{ql^FWANOer@itgpl1AsFzR4K3THwqt5)a&3p6lwFFdnC=*0GYmL4W$JZQwa~@ZUNDh3U)ZMm4i)EbpD4nFPu_ z90SKAnVTPyx}}yrwis{N2M(RO{frZwtv%H7eRb|q_v_6(5+gvsY9cnTmN1*BwYFhz ze%N!0haM)LG^=_+RgCmu_d#W5eWe9`3v8gK!qbl|Um(;Tc`uISRK0o*7r{nOOu>_= zL+@FMay@VJmwd&B8QC%3{i!)W3k^Dqu=8bSL1gGi1*+<`Lvx92XhP@1Lx-G_z_#ko z&kl(rs~Hz>a@ciVmY`E6OS0sZySD0l& z@{}&r#;1Bat9N2MGJ>=1K=QccI_a%)Df0_XwZKn)V3&_ri|Q&0Gf}uwAhUm$pBspT zS6cH`WMSnqqm_9TkWBFO$izgJQ3byVjUR*{^nxCbd;tiU?jRUBh~eyR*`Q{J|lX5B9f5>=K+>FW>p z!MS5-56{x~SP&XfHBE+_^X^sQdd%~LZrwPir5KP^ha0#p($jOb{C*tJac<^6H~3q} z+&*W01zTCUX)G~(;2v5xB8@ZTX2n$qD<4nSBEQ~wOX=wg`PKgE2~81% zXGOagN#obC;s&SY{GtL5^5hE7D(QxnedFzDV4kKS5I3gTsR<85O^6+LVx5E}B6sOF zp0QG)xO`}??AW8+hA^O zaV~Pzq}7(s!*hJT;4cbT4;OfI;{>xP?q;zmlz;XQ3&7ZYzM%k)IylS8euCD6SmBrt zV|KlZd(|>g-QJPjy@4tjzr7qn{C=;#vkadIP?Qk`Sl&=$#B2%`0PIemeft*Gd!tvA zrQeyiTY$Ac7wM(|2juB*jP``&sJgO|{AKS5swk5Z6_hH3mCI0W%9o@b+!aMhu`m9C z6#x|#e9s?+>&imR6sy8TMRRH1$@!eN_Enm82P>&VglnPfQH9Tz&(CNta7W!~?CP?q zVv2*i-My1Ul?4bS)A)8G4NCB)1zo{5OU)r;OKfw(yRSb?kvMbuYj zZOK<`F(eoB=@%v}ODosQ{^i!FjC$d3AeZnzy8VH_s0HX<9=owJMWvH zO~~_*WwV%22iBSKIn`&|Fudfii3?UL#nwgPUSW>YD{Bpm;9w|<;{nLHaI{FzzGJp# zaq-95*bTN_H{IP+06dLJu2N|=J)1_A>!T8NW{4LGn$?UHy+zNd98$4clQgY={xJawhpnH zMQ94AA{(FFYh0`CFKO}HeLi4v!yhE~4^FCE`JBJi5?>jIjy&IhGx0=;vgBnv=X$#` z=p`nzB^kl^l6AGR^vwWIOEHu?RfH>If1Rey!$|fW?UYmS+tDc>9yc5aq+JN*?d>NGJU@*HCS3x+@b;7z}b=a3~fPfsOgiLR&dRk-{C%0tAfp~Rj+|-+>qXmtU>y6-n zK!i6oL}GjjqY<$c50OUotFzcPDzn@)sbL`i+GI7TQkOT)-gR`OILwksXN%9^=`e0I zds-+>Q?tr$P;4k2IwOG9e-?8}3@vqoo6!IbUZud zWW!9BTJm`$m4u4VJ^QL2Cpr0mcW~1uLW)GfCmTVpbl!SPft^ZuSODqQYD_aD1aV93 zQl&q3B`c0~t3$arLm}-smA_**5B4+X_ax`EnCN{2zjkzs|&sv51D>4;D zObD9k)h|!oQ?}(4pmS)&YSVyGsLe*N?XWEt3?>~FDJ3n2EJWvEn$-`xI{0v|wTeF1 z9FuM>*#r|PsN#if;?ITk}q z<80lOu`BKOIW*;l5%bn9idtxI@IY1=|MWAUf+PCTqVC@TFn5F7jgjOu1*}e~z3hB~ zFWnpoKke!kicEli%K(>rDo)bWDS53)l;(l*VF7R!2}l0i&^b*(H(0H_dTW^P@JLdYq2N!2!4?=Vp5|^d1TD1Qi~#q{<^ai<>DuRdUo-wjn~!1F$F z(_;0+%%*GW$8j3>v=&e;u0}nY3p1s>pfUljGW;oV)3pBAW(X|2lI3cKdb3e;Wz0?- z!{YU(p<#>F`bUv94XISyvwrbv;l*lzw~1t;gtD2t_}Jv?GKyUniBECyN<)oDMfNZ{ z#?^bY?D%i*t-ErA$8V){xB1_r0sdD3s8nRdp3E|p&?2eE`k=Tf@ulU0PIs(Co@1G0 z{!VEzVN^*TUWPM{KKGYc*kDdCIGbhf@>bZteNM7e;)1DfpRZMdjGnfRo?525d&{sG zKRnp3{z?Z~NNSuB>PsEtN`fwL=avK;_E5wC_n}(@MBPK>_O&5=_C-!T(VS+^W3jmcfWi`c)?*|{i`@8lsrHzMyj z4=Iwf*f8n$Jh16&=9ZX{eB$45w^%nuY@+k0Hl7$#UN|!!mX-@qfv}I|E-mI>K&dr+CAZD?fsh(Y{+q()dqVUlLu)g=PX^-Go1vv%Eq*j*6yYf< zw$=xlWa_V5>$;=Eoq1Civgyw(FXL0_a{4weJ!U%DY}{KrUEg4Vbg!nHc|hA&6N!`0 zSBy&KV@WT_vZeFbOR5=nu~Cr&8KDz$Hnoup8AUpnMuGDZOAANgd3QyVL5YQOXviI- z`nLHcX1wH{m`U8r8%gbr$w-V2qZ|YhT@!|yWM-5;@k#z5b(&%*7J2$(U~A!k-$r@9 znA)&4gH2{SW5eIx5O9di2(> z3I-t{A}!q^-6$X((jX<>-8m{bq;!LHw{$7pBOu)!0}Nd=F!ve!-rv2y`~LCXd)IHR z_kGqZmkbZ*IcJ}E&e{93_fEa|=93jv$a}0uIR)*$U~nzS)*u^nfHmUjkF{7#*Qtom zA#f1~_frp}aB)V>$q1sEZ%;nprKXl?DV>hQhhXncFll6W*qa5jl{qA5>ej3ju&S30 ztCWNa+=_0kv?|xv@7ffr;S`C|+OxplQv9A+mXaC7&?B``98Xzpk@jpI@`yS#T8Mocjw~(|dQ?^rErd>9|8m zeSNwHvRqTLu;A7=y`REK7x2a(+ya(xxw6hAX%lIMZ4gm5AzU9*Q4lmoNbqxZv?-mk zt0W537Svlq7l}(f=a@EZ)jk|LAmv<;ZmuKH1!^a=4vILb+0x=gK-akpo;nxxE~Q#DS7S}LJ1skSq>~SrtzAsiUZI(J zKnD6=KKDLuQ9NiC=d@LvwD~wbCSURt4g+)ZE2JsZDlGh9WGao@PB39mUSO(Sg>^Yg z?EI7Qwo1W$U2h{cJ1`+4FqBXT?(MpPFX^4%)4*6dqt##M>p4R(RWX$Ax0VUgY;GKh zDY9gHc?P>t(9x(&i-T4t_}11qjIJ+Yhr6<6DEP`-@$>n#kcr=N#mkh}o+&Lo{TX}f zRIuur=cKGy-}IJx`OGi3VnJ6bS@0!7QWRX6E{P)9PtIx4 z%ytl7tbUUCajI+xe$<5&aQuxVU6$b`;_mI*m>F!!Wcs$m=HQDeYl^4rdzF|-GW~XL zp*G=6f0vtIaXh98D6G^4q(As{q65Zi<;RNK&~(aXy&DPUO6d5oXtCI0d?D8C6mP#>z$u>#se$0pp{i}t%H^L^&*vE3HWR)()F2onW@<}-=HGo z<35eAbp*>;>BD^b$@n3VHR0^F_xKvKSf)xSVof&xz6Rl~_*hky%MEUc!nb^nrlV4WhhJ7}Yg&?1ju!?@ zbskz`GYv-7$_U0^{BO2$JbP;*IR9mA{U(tN+nzl;??C0 zp7=xxAEfy`8zuFe9j~c&pHhR37+hOcx-Ziz&&-*R7rRfA(QDU@uqILHox7~fn6Fz> z$>$|AX~)ZZN9t%FEt5V9Xq+7v)o*IGGVs*!=|pN zx%w!Z(Z&of`lxf`Q&1;mUF|kQFU~hcKF4BTKMEnPp=>-!FZL+&J&t1JX4_yEQYuP* zBsd#R^Q^@l**Nl15XW`5-evaz^4l;*t!+q(>PynR3Qw)BR%)Qvba`LZ&j}ynuILB8 znSV06($nt3zBLkhI%%iy*j<)b0b#L4UUf-6X)?KRlK`O+q#^;miVn5+W)V~nF4-Vq zdeYzmz1#w|+08@^jL$W3KSRsdwe|Zrdo~zLQ?rIR8E@olW!;}HFI^G*ak!>B*YhW^ zfGd-mAb`W%+GmkmBonms;6&edEz0o5_UqVw#X^;*-WDB_7%I`ziB7k8(`_tQ9ve@S zecFul8}6|kzz?iaG!a@y%Ks4BimtA%{*9zo_G|xV4`3$&(~V*Q=erYZULdpuV48u5 zz-_*6l4}6r-vBcA954(?F|<`A^%_)u`XWK;xe`bq8Isv`Rs6;v;vPWFYhC%A|KiQW z5yc&QE7r0>o^{!pP%p2_;o2{rq3{i1>2<&vn3`~NFnvuk;|Fh9%S7%3C8iRzSE{a` z{CPC~Mf6s!c4ot=`)q5=37(4C&-tSzTAy?8QaH$8XZ}4hBNlfyrE3oU;NB4x0|P^; z0w1d0)HF1nFBN|yq;T3RDj*w9GPxvKAEm->2Y5<2V_C-z9+qy^lpE^tlv2Kn1AFUIT=oi_e>wIYd!jc@l zzRf1NRck*yo5d!N4MT)B#9mKhEks$lb4lE=KJ0YXHZT;-EM`c7)+Y) z`fl{km!nft1e3IB-ti*nX`Yb%gd2K0m@CKf)1}J-Ywe4;X!cwNL1_FO*bNV$4?_ z4uf7C!LP4%Sv6OId4u>8{7tOf!G zRE-#=v18ytU-UqL={QxcM+{}|k#W4cN^4%xVGE!B?t8|bCff6ax9{s^NYv$&7NVo0 zW1-nS_$>Gv{p|tF?s_w|!Z`hrM0Y4YlNzJ?_Mkzd6HVzQ9#OZG0{sg~wzdk75ZD~g!y8|zxzDIaJbAuk!xLQXBjmFm zDSQjM1uV{k?FhNk)nGf4RqylqYy#2JjT};>Keou>p8b0UKSYbDf!~#>xVZnQ1qPGf zydUzl$SPGEIxWNfxBmc^(=6tM=LXFoFk;cJfk**AtwAY4KQl8kwY6Rd{A5gyja1N$ zxpev7bEJbT7cdMZ#)$1(M(5_>@Fk?kB>3K2YJm9xrbGz8^X5=8r@39j3R=1%0aW+i z>zKQN`nHRUhv)bT0c%Izx&w^4rhzX$)aBl5qkn(fa@0edQBo$K#GcjGCNjBm?`_vz zP|nYN?5V@T!onHIcBhR{jgkmUfqTzUC=Q_c0)?<^82FyWPgu2XBfx;me;*JGU$yaW8J(`E)7$zD&w}2$UV&#XgTtbN}_q z-!@7Et|gNU_uj7m72@Mgiw8ez(eGxw1TN4+y0L$ygc5QC?cbZE0od6=0Yy9m#RaHl z-!K1HDlzP{|M3f)pt}3Zv;KEZFqBVnFz-!K{}qso=vBXx2z;+b%l!U5oP-xpRIQd% zoSiy4!e^;mIN+E7;RD({b9TkPwk30~ zS#Nt^-!|8~9jk9hNTj&ACtHLc`OD;9pdaBNQ$nIf?;e%fm^ZS$L?!_LqZ-FaT=#Ty(jVbuA)HO{<|j--`#CM<+1Jt z5in-ORZ8)IpFel;*8kZ#MHo0`j3jUQYMr*sk(R{Wdz}r)^X~e8S#iP3*&@M)fPiku zYt=X~@Yav2Dgnt=$jYs+2JK7WXF@6jR*;{fC^8|~$CuM*ua2*-f-JGU8Q&mk zw%&d&&WRN-tlW)Fi*yLL5`2h)_dg`aO6n>)Eo>e1NIk*{wVc|@r#G)_1_TD2=vWDQ z-Re2sMp1xL1|&*nc9_$NXMRr>TQaJeh)aGo#iJ^J-FO=VucV~pt|K2Xl-M8&n=9zz zzYdFj41h!QRj$trvdxAS3&wX`$joQ1s+JysJ+yE9!v?8HolCQ;HqNKzsmC+ivBoxN zr0CD5-)D8X`La@#|2_o?>TpTEmt{^Bj|$Orvnv{pQedK^tmEU=zl#@(JNg##yD*8U z>ifBP-~d|V!4@S%^gX2By3j{z2$U>@2FjG>q$IuqB*|t-Ma+qdb}mFY{j!dmP-a0t zVFqbGda_T%!PDlwpU}|zp^Wytfp>Xc#raXP{N}53OlA6#s7P7I_v<;4GA?zx!DX@Q zGcgR&VXyd=862h3tKXHwTS}30{57R|3O)ICT6RsNaK&>c5>^#L)7g{JP|>F-{BOm> z3r(R}u+rRtC6#nrRsuqXsHXU0=l^o(~eaWJK+&;tfCV8Kkfvr;6k2^C|u&XXDwN_Q%?%{qQ}c zIp@ts{9+s?=$od2>q;xlq{3`Iwv3ccYO*NaV_eomrqYnK`+t|T&Pe3G`x4%$`%;r` zZ^uop^#XDO7^b+jx)+T)Z{mf@9*Gt$+_Z!od`xRji$mV}!Pait-vHjQDblLf z&mp3p*B}KUX7b52!X$c1|GDZ8}Y!Nv_Q zEFb2nnY7~%1B`Q`kdwUi*!B%!?*7E~t}S9X`GWtzAwIYUE^_xE({^iHGnp}m_jFT_KqC9}cNT(HjO({t`#uO6PDIWVS zss4~=_mk4;D>tDSaniT`l6+a9~%p_ zSDO=KnDioWiKOg^%SIcHU6j(={P~~mFab~h>celsr7Tm94brA0^CU+0m({b_`}A~; z2k+m9m||9M=s^M>Ni*+_DfJwLV%s7uPs!xJ-w z*LNzbxqZu1G{kuI35<@KzlKQ~g<3|#TeTcUG`A;jNMO9l|G1@c|%5YtVJ?&(1j(Zdpn=`LfOl7Z8$XwWY#A+4%WM ziDOfbO>M28CSmj|PrlYUPkn3yXFow*tf7Uk5Ou)hJ{nvQSO>d<`iz3U_oo$388 ztzLnN3bx$;+BjF%=4>QC>h%f+P^zfx#it ztS_z+gXU2_K7k*!qpH*St zGc2Nmh=}G`BiSJo+I;@oR65I*^+@{(zHy-`(M1-o+Za3k_%KBIOU5~SXS8E7hjhyR z1iU#bg;GLU>ZxKOL6GY4Zd8ql+LQWAGNnJ^WnE-g{v38*Xyv%7fu7P}C>)rl-*KJe zYLQ~uey(R^vRkayQH}lEb7FE@l;ck+G7Y$IgR{{c;?x%o+?w&)O54Qf9hdWoy}&Z` zaZ%UVVB)??}?HTl|wk?MbE0iqfN6k+^|HfkbU$-kImiXT+uS}v@54;Gk!@YVU|@A425;u}>^e_Q6WBKL2 zn8uq87G)2+yA2i&3|x|KwM~=q1+oQE>aX4sCMlR@Rt54Uc@qW>ceBGBHD=7Tb#|JHhK`G$QF_QUUaZ>rHxW8EE{G>gx97|~Rwes3xa$WA7MhC_z3o393lr5xAWo0w=KN)i|9oP>aJK6ij zg!`0iC^|N}IG2By9m%^+Nc-v+#^5=yDu{RA` zCGv$~r%S{m-_)KLY0J!$YGCV0?s(d6am8DG6iKfzIBJ29{YV>4yxSg8##(&@eS!Da z)-^PCmfM5VtsjVd6Q|Q~Owjtyx2L$!mFuy@32X!fi7vFCMHy(HI6>1p7;k+Ljz z4B+C-G%HE6t-SerVA2(w^1V|)K!yCIXs85syp4Ar4vOwwV)?5Sri_A&gXz1lU%)SK zX%t%!oslrj)rUi(KQ6`V<{$TxOYW_r+cic10{$;Jd*3N-t2lfr^E@>S5x4eK4$%zB0k#vvXJ_FH6~{Il1>)Otbrc8DR`BZ8RaOEMaoq&?Syc z=jRLY6v_228xH}%_?0>4`GU|YdY340Y)2i*2aZuX ziR;T=^p~Z6>C!5TmmM9TQ%}S|Jd@!Dr!Hb8oU(0vXZ_@AJ}i<18D?y?F?j)2YxjhX zaKie{cbdrh>)W#m5Pp!eUU~E?&`r7%7c~%(Q~~amg`ZhR!E;W+ zs|9W3ewQn)7mLJSE<0@+$sARdR0|Y`GlT_Ij-==vh5n%aqWNS*km@<8Ye4d~-(xW_ zTeLLVEr=@zQ_nl@Y9f@xpRQcs|mYY`POu}|G|ywa??4zbDg`nt=)V) z|J5dWUZXxaoT-61wYm#`Vayeu(3dU{MIdbn-+#k8n2yweDLT|UQpsT)~=RZU{4$nJA9p^SF*WHFbWd6VK(T`ycK$A~v7+2zS@Pq+4p9Tu&mv(NN|t=(F`mD8|GkSC5o{$xBN!HjK;NK(s-L##pWu$q>GI=4Bs4y_>G}aL)%*lmM?sm z_Yr+#;UObBI^RZpH)}7cHT^}u*Yq+N0u!LNZ@UfYN>FYG+ zI>cMKHC+)dB@PU+u?{?5v9Q}6o1hV)Qw;R@G3}Zydydr5hN*CqjY->MdSZ@1r(<$* zf4fa=JZX)>w`CrC$)bH0$LL$vxUIjG$9WPKCiyBNIZgWmhsPOq^Q}_+wW<;1BYIMv zR%WZSsVj7~>{+K^9iS|VR?*kj*U)%*eSE|}+iwZz_W&KAt3W!S?`$|C3@R!w_dB04 z2Na@}t0z|oARgMJqyNu{O4>hh!G{{RfDn_JgV*1!40L~Nf43%z`cfmW;+|Lis<0w|q8993~eB2xZ z9=vw4#E(x9(NnJ#91>#vn4%NHsBxuKx1s>bZn(tnyiS)YgA5U6x{aKH-Aq}n6-LI% zW~c~)j)KocxyfmaGKuyGLqhCQ${86KOAHKryFR+uyG$-8&|E1+uWww(KT{VQ38cW8 zzxdgrxpZj^=r6IGZfXU2DxipUka2pS$K`TYbGwI4ziV~1M4aEnQpHSzLSr`jBse@6 zk=ebbzIFc-b5OgzxK!T7!BS}eguFqodm~CgT}fu_5->?*@`$#3)Sp!&mu8|S%Kg?e zUI3aS!EPlrBFG~i@m6v|rp)I{{p=XAP7tb+i#qg&hpsbC?>n}hse9=lAFw*F@yIoa zrI00=G$t9o%6D=o%&M1bx0?}Wz{q5iM^2puMM-(Yzb%StoAuh1{>7CU+}7|%K|<^u za-QVffbc)roNe`v?Kbr|(d<_*+dq`!apzfS?u0=TgZu(Sei9g9TGE-DnkgTbomc#0 zZ$?Z?fWbsi!SNx(Eve=a-%N1%uhwKa`m0Tg@^?MetJ?Nevui77t0?8q^FN2Som`&Q zI>s{q=B7?=Yst@;s<QrLrPO@--XA?BN zT`I)3efxCH3D$+S)4y{infxlRo1Elnyby||qLG6!)Pn~NtZir8cMqJoi{Yl+vvSw#(-Yw#+sBO$pi|`EG0t+5+r;0Nk8@jV>l{s< zttuLwl>OmtdM;3Q6+74lZzsf8oI{_G>oMHCO*A`qoGA20`}IIuF63M!zV-I= z)qJ9#^-lrYbna1Ie{asjZ{l+z#WRK|F)g!MP8TTy7QgqPiz{rf#mAminwtVl64NrZ z$;HdBTbXh%2M<~|GA77d+^g-ur?lCswZx1={lr6?p%`R1(ONAoEB*sq6Od{6NEI&G zXrffTu<8`cl-tg=d07a7iA;f9HVR%aXdl z)SNvA1crH~*6h;5$cYmj;M-pH>3TWVvYpS#W~_J*-}zzt9_agWZB4UZ#^yMAgFs2x z43gQ>R*xfD3NPkblR2%?f?;6SYtY~rs;_VA=ZXJhEEik0rg}p=tjn_g`kVp}aEZ|Z zUuf|A`LKqZq7*67%F!M4UzrQ58U{Ih|5uMWGM>hS~eK^V)wY9JyO2^vL=%O0m zjIHx>b8v3Mg>3%PFh;MlDBH}xA>cqDs z!@(I72QUEq&&H6bEm`W8L%*fY5Yg9us~2Zyfv%W89M-(MWd2u+2ts+jJ`1`o)tXJ1 zF(V+fQwJIG%B(~f1_!Rse#OX~!Wqb*r!<}zy_k}Fc}!9^+Zt(7*XN!usC4R)|lYD7f$j>zcVLVCIc$Nh^pR$#^{x`>%olI&wSgT|1Kd2|^a z&mVcG$NZfKWqU0yspY1htlEbgtahHV^lLA~Vn6M(^K5NFfK6Rgp(TT@){ez>4#w^) zsl#KbLC?s(3Yt{7=`mUqmUshebxuiH7VvSyRNg%%ZCU_mez1~u;9PEzpQa=wC9FHp zXGkSZM8Nn^R`!9WzU;Fk$q8?#sd1Y4I_9fRx(GIc7edoTIwC28AM#m2XEkzYQ8MV(pj|gfr%l{L->KVOy zrKxvbt|;fdM!p+fp%V`6O1xIAvpr;~(rT=u79O(m&P~LZ?u0F(P@C5wD-heqLGB=J zor`F_l4`r#>_)eACvC5%qNXb;;(Dx<*#`$`l*LT?)h_#SYw6YdaY?@Tbb{(1>EYuSoOF{&PoZ>zsBi8JE*-5MRjl z>5;fbiuxX}4Ttu%M^+rO;70*)VNH`mgo1MO8|xtj|ESc5mdVFoZ7b^Bee=EL-;!>) z+Fv*!C$_;{3a*dFJzcC5Hsi&<=vC+zz{qqYGF~gh=~b@Po9lm3>Xz~wC!%$bBq zDAC{147Mtp3~x>Q~(Kk_ArdHawLayhbPmU)We{XlD0 zdTcGAoolK@FuiASK;h0Q-K{cYiUkqvk! zj*s1ydLm7ZmQ4<=Ga#q1&Pf@G4joz?WvA+ZJ~QMP5@xeN2Yx6cl+ER-V_kv-HT$7j z>RB#>AG&Um<$q2T^+@nF(iNsD;k44Qk8_fX--LS~ct)uJ&T(#L?)rKW)~vA&oU$*r zwbkIgcjlsmm1jJ#=gKFs#ORwnk5$IU z!A*Oktp>lSmGckNa?C15c`m1?CWYBjEMBGxM@dO^TzSfEySPUDGJle|vCK*Zo@7>_ zOG&GH%Xo&gAK90P6EOdx#sTsQIVlJj%=nlr@2W&F>ASwff9f{T+K>~w&i_m$_-dTK zL8h4h1Lf;y&2>H3t|1_*q1$fR`xE#gnp60dXyg<79#ydkThf6zN-m#U;TWTP{CAeV zd#{=+6~UV_phnfBCY46|0v)~AxQAY%V^enJIYXq6u$5>j_043;u0tA(EH~+x;U1F5 z^;;Ot^;SyMTl;MBo|`A~{c}z2dB)w0{fU!jN?Qt<+m&RWv~%=wd0viSoqw%)=3eZ& z8X9wqR9OrK^u_{e#drV3@Y4u^V_ zImRhtUH$o=(}#^QxHx&Hi6#m;D~B&F!y)xx-YU;j&Jo*$0H^1mv1*o{VNim-&vAx- zeD;R+7`lpmzMLM*uvDb$4Ck{b8`VqLSUt23=3{PgE3QF7U0y>+EdBkZvw<@KYKS!i zc;45@5}RU3s;R0zXf)k$!yW1yn3*otp*cH197#ax=QeN6po4H(P>%q;=XLeou7+El zhXY6{sYLi@ZLgJ0eQKSh7U#)UF;|Z(&5UK|p8v5#x1g*W+x!(%gwwC`QXMcyUP+84 z#F2?<E zg{AL=L;c%vAwHG4w_SK;B`m(Gy6R=K>@b(J>JM&fp4pgXdV{I1Y?Fjbwxu?}J8$eI zvuazN^{}zZyX6A5KkASB8NwpRri7TELZeVZdQIQ6;}Y2-(Xvp!I)9L7xvR~R0q>0v z@!J|XZTex{9HhT%p=)1%WT;@auwXyIOlTO)x44LW2!>hBB{dz!ZPsjl{S4N95+k|_ z?>2w6*@nK^dfh|bc|G?qZ_c3E_@rq>>6RCDw82(vfrs@b=+ZICN$j{IZ&D)W;+USI z&O?33Ip29Fq`vLg3|v)GI(Q_;X-@8EnGR>ei@W>4Jd4{3ce*B7ihQCxZR z1iM98&x6=f&U6ZMwfH0HKqxur4&qkWe%DEOVVhv-ZWWwriT8EUZB2V)Qwyf*XK5*r z2=THP#KigI=^(XmoNoPch4Ky~ijMwl`yl#5mAasg;X67^ViFT)I6Qnmn3S3S=v#Z|9_zkpNU zpnUkmUP)1Ck6h@hRul*HvzFoA99jtXVy;XTSsCip@Yyl;TWlPvkb9qmXdLz188!(5 z!~ojJrwjda!1#Z_Mj6eeO}gcGgidTsjE9i^eAZ?TB~ zA2ub%C>_<1IlJcJ9hV+~XZIds4+7X(k3g7X2fKW!>MKzcbJpx;i(<`ybqBamM4Hg) zZ*eNNpA8EK(TNU?x`wUPQN7QsJ^vzo(QBdr+5-P(~~)Eu0SX8uSW03kmHd z^WIy!-Cq(j@aqV8V%QB(Rk8r=#daG^?&kqyCZk26n*Z`>IRv1$LS5|+D;=cn^Ej=S zZL1V*5|o6O$nU6pYtK=@-Tl>eKsTGkOpXn#r&0DbGXY|T0J-K0Kw#qqO4tjX#K9N6 zHSPGVmxe6l`QB@9{|~3gfO?^J1;>^6;b# zZvrZ5ZrP*YHK#_g(!Ij9;98#_YpC#=5mpEZ5a{0Pk(D?LfWuq>V6>y1g3LAmoU2j< zP^MJ^0E77qP3bq;-QV9h@;Ey?s~Q2qY5x2DED&@PMW`hXx*jh7(%J$JZAYjOpg+%` z(290D3yaHJ0LkA*VRb}AL`La2_wIF}a)REQV)Osy(22>5fjEiaT?{eg05n4o)L_(Y z$08HDTyS6Vyj|LNYron|1;OW*j8cKT#}+yObfxwxevVE?!GVT z48}&+1taI|+kJ1&ri|l(cmrLWe}8oYzrb##&@?yS0($3FtPk)?%(#QorgDJvb;xZf z6qQepFz5=EX%Be|L$(2(0kB%H^4r>OfoyjXu3v!)8r)Sd>Yy3crX+IJb%F#U5%a#n zAon?ZX9Fa``0Npc!cM0IgAC{G+)HbK)w=<}#sCuvD%|KjJ{nA6a@My$U3vH%|6TZ` z5DlTo8{jzF>gBp!0|s8Nf2po0@xNTCA zUCm&3v-wU+p8<~T-scB*ok!ZqUZn^Ww2F#H3iL5xf>6p3B7`ynQYc3;a1D!h58}J~ zzX(5uERYa;%f4`EYm3;+C>=O+b@ASbF%R!!d^ERU`pCq%5rBlprZ;T7}+ zP?$1`2y3`aD_n(`_ZQ+GpJ2LCQhK$A!m7H7*5-j5Snr;2JH+BjW)23h7ZIhm60{Qq zxBvnrR1sf9qfTd(4sgd^-{{Hu&m7}O@_+6oFhB}^zx=x=|DQNDlvq&L^0zzl;p>+igY$Nj zlqtm)E=(;^p)Dxg!{x94y+xV&E15W;tp!c5APZ|lAzSF4KFlO&G)pA|M2!uG&_a_} z`PH4sUx2+i=m`L;>HXPaM-gCH#Aw0q`nQ-czi=;hY@PTEiFnZ!Um~C-T?q<9pZh>g zeBqkfzDAnwF9j*&*Hks^f4*z<@(wTxwnWIzlXD&$z_SR02i(*+eyx*)D%%F5UawCE zK6WQoqL8u4Em|r1{Lb$$ip`SJQGgkjW@XIufd*PbA*4~yl3Jm8PA`FQ=6AxtTzpSo zQ20aW%Qgvx>KmQ{RyIm*H((zFF`rpe%)|u-040t9W%dW7L0U=OEKB!0^evl#pxcjt zde!s{o7M1NSWSPACkB9*ynh=8z80(MWqnHH_A1T?oU)H}6B$X-cx*gF;w8~ioos3+ zn&>hJmaf?x^(bNiIzn3F0boO3j0<_>`j-*wL$-7BBx!SSw}SQ)F0=il9$7Lkeoi*sx8mTb?XI*B z5!3cfD~u_mB6pR~uGyGvRvJ4d+4oRWn_He7JT512B*k$gexT6O0n%a|AXcU z{QEV6VNQfQT1bcE6{k4vi2*G(1UBxpkoR+mt^^X0`YUI1lI6>|eW}i+PaZ|zh!J3{ zWlV&!HL|1>nu-!-2xFgn{3X`}aGrDXD{Z;h^*#3Zb6UhQwi9eWjSC-1S;H<>{*8;C zoX?>?`s%8xXq33TG$YygORTNfq7!4 zw}>RYZSn+x!012LFOL3(?d;cg$t{FQE2qOHJ@Ox>p_X_h>lIO>bx zee6M6+EQG(P26BI9lGc}R>Rz>*Ych5Nu}yOS)o68U|kq)e#l7a-?)P{c~k^re)Lkh z?^3pmI8#jZuzc zKgqvwI>p{-9t9^i%o`VK#OfMAh1sbXYb*-IbZ6d4{9{>{25>Rgkc--S0HtKX+!)mI zNMy#fjCxmZP9pSp5GspTWd^G{S)V5=o6r4d?(lq6_laCromZ4?I(rTEUhfGp=O{{sfQ__q6>VX!4E7XX%x&1R2j5~lUX z?o5a;oyBkC?0pzY;ChT^xZ6Qfr0s9hNYJc=OK4tx zH1ZkM!^ZBiE}|p2-l85k^i{)V)BthQF1bBt-Mu*<^qg_7oisAL+JDut@C)#^dHc_s zvJX{VDG+PZE?GpdJL&&}1Cs;*8~1$)34%EyV(A+CHmntl0^KYw)+xPei^6xwtL|?{ zfeullIGT}@P3SXOtcnIe5yGP3j3s(&CaCBYVKce3|0Qc;QU}VCP2gz zdt#ZC9+7V|VfnuYXN~DU8isl@_(pt~tfMvlY59aIk-INJPtFNyN&tX4{EGw7d;+CD|-s&oJPHdALt|wzK^?&eaeVj&>^TkDcrA& zo;DbEYt+4erPOkSmYw^g4g2=&D|q|Mm*AeWo5nFI3F@g?tUfxtWTpWc{8a-8`Kz0D z;9ILFqKAk2pgMPH=Z#z+*q>)7oGh>+nQ{Xk^eA4HTg~f`HW;X$G6wYcYYS514v5@V zTu7D7v+{SWawqypFDoQ=3wZtk4$WnK>kt-p5h-wcD^LTp_-_@**i`L~l)#o)Wwn2v ziCw$xJ(KkZb<)fF>-lxZA~q>)OEFxWs*@|5pX3F4gocA>U?mB+8?<jg$oHuQ`u#7&*|#L6$Rs5gncR_)%AELTZ0)k*-x^Hecq2W=lvj{ zL>A=ef(Z7oK{3Tye+HX}5X{c&50toe0Bd+@Rz{HQS}P98S@2?%kS7|)UC@2qxx zwNw#NQ<_02=s(MSL+YBQTgOpZiaWDxf*++EKY>hJCQD`T8!-s=06$Rr@|v=`MUM6Me;i>t97v{MEfMa%-AT4$xAX^geHKf!6=hyBFlKc$wJ zgn3`3rYz9@!PVvg53cQq2y)R>n6-z8Gp@X3!Z8&pkB5_uK?^Y&q$AB;2bfkJzuJVf z3>!w?QjV&BwK zjo{kD^;Z1yL1b&;WsuK8cMp5~{?fdpvUU_%D%hl>p(wxN&L4{XW3p-DDV^$zfjHeJ z7cB~}Mcd{z^A5|!IF0HOLaaPxKx@^gn?91*6k&wHH|_<%ol z(A7AL>*v6cY8*!S9VfZKZfg!?uSKnnr;g$NYark?BV5=0*e&n)oaDMix$T!_GZW~P zrmb*E`Q1CkZ;zcQx|5Drs^C*6H==*>G(KR9@xpEeJMA?EKx+1!4=79AH&qv!-Bv6! zk&s6UqXQWCILr(2FQ7Jj2hSNk%{dkl+^v>v*?QI`rR@uVx4a9ZaavuqEM1bn2e54H zR)EaxeTLBRk8Q@?eCZ?6i4AUwMJPoN0xEP1X1qlxE)mPoY26z3)Pzcxo3ZyTWci7k=q

zh4ZQKjvM!7=5t_TF&T)sU;Z}8gO~+!_Ws#$!kNK%6qDaQ`BNtmy>m3e z)9$Q*Y!AWOat2KxPbl6DNWnI*H=pSWVL~eq2(O-lLV4_TNo6U-LP7t!BScI+x(x9W zw9aIjziP@$efX$nM6*VC{C?9}R2Xrh+x21=j|T$fIgY*^*2|?Dk-$eAT-5dsslBgn(BE}w7 zRbkTC)bMW*f<3-<^2uY?L};pg+f@PRx}d5T`{Sxd2`m$d*b;=baB>{^$Mco%&P_~k z15n(X9&)?EvyjZ|lwmWl1nz%8YVM1kTUo(mp=t;mSrY6sR6te`VvStt^=(Le#9PGk z09+)PE<;*I<_LkbkR%>y%^sNXqp)^fDo}0M z^YL2@^A{%^e*O7NnjdQ}Eq2lz+(2w@?zmt-{e)Pz8c0s8YVd7o^l+u!hCX`LeQX2v z!p#y4Y6g1iKSN3L@aQ(Tn&XW8x*p?-0rCB8HgfgBQ0&K=2i7qQzwm~n+YoQ24CK}0 zD(2$V4dckZ_4aceJm1*dd>YvCA;G12p;O4Ym(=~a@A!<%Z~GDYPIg064j;s_B+imu z{Su%nsb6rL(^`8LD3%Sm72P+#J@RLro0}tcLwxG+1+@X`eV8=M5uKt4aq3m26fW>~ zMXx^SYWk=6yAwb)H4yLwuOo_FLJNfzUIQ+8;s)9Wm$7t8v;tSnOKGPf&$*EOvBAPb zIIY)Bx~##SM;nO-LHHp=uifM_AxGZc-b(2L_(hlf zxH2506DMXTS3e7~;{e)g$#W}xadELfmQITPwvr|?80hp(AVmvXMg-uICRJD}mG-B; zwA3cRT*{7^bxx!TH$L9}o0QpGQ(661PW#e#Z9*Sws&Wby1(|ROfi*lw2Pay3^v~Ra zDcFQ!GeIDIZHCCzz16y4FP%mdLez45Loh5jRhvli+Y$TI`w12~H5whxi8If3%_riw zRGRLZ#CfpG_5Plo&VyX=W_GsI{s6*ppIv zd7xopn-DblYg)u}>0Oi>zHnPttS3G1pggEl$Xw@FWPObr^Hck{*PAvO-sA?nthx=) zQEsgc3Ua3ik88a(M5x5+8Pe}$N)CxcHbM=0PTnJ3j)#EcHoYhKzB1G?1o|EnryFa| zKVrx2{Z3K=QWNDD`mY@Xv{%)uQK6p>=)3WOj1Txs8bW{;QaKdHw0Q&sD;^>=S+D!$ zdi9KQH8@)BG=0*3fru3SZK1*<{8D~Gbis<6^%4`P z>_y$CJ?548#F;X2G@kapW5Uk zZ@KC1@m)I)CBtUcy6tN3*TfeR$YA>jFSgU`#yV*ye+3~pv+#qb>{h{@fuqa&=eeZMovSh#SdXLbjjoo+Ri*q?jNds9jIul~jHKYU_HOzhNH>Tmy zRmV?`(g812h6|zUrgLFr&WG~4{Pam}Ph7}{&JC(=pSIr@k#v!rKn@7djy2Cx+$Q_b zuIOpZg}$_2fmJ-G>Z{EWQ+(T2f-|5RHr1!1OpJ7tZX!#tTfRg{a$RZ6&@zxeV&!J0 zySXff-{mAQ?|jG9odd%lDMRna)UKeD;AtC+#3YN4Nc(`S1Zg~QezW=hMJ&K$Zg7x1 zwKyXt_Xv2jsI6a{!7QL3df@(Xr_kiD5G6YD#dMS8VyKiw;oOkgI zRns7oiI?O2FkC=58QyLx{v3fvi(vy3lMTZ$<=@aSxKI5|PB+tNJ&OlUW!{?axsX(7 zbBmTC)r*I*ke*)W#dvgbP&lpIQ7#2!wCN~4(~b2^N_MTXBLZg@ z_6cjwbT3j9nLn3(o-4IAB~(U79Gt$b;etSQ zdSG@&e&1C3{q5_RbXgT{tne5K6C8Z2B-_GNQu0?Ju0HSQhbLw-boN+qqz{zhJ!UzNsv#_$}#atu0oB>zT(bsoSD66s)S?Bul^j9$o-bSpX z;{^s%2&ym4RONpT#EoXJs;nx4$Wb_G%2q5^DWc&F@jSk2Q(rE&+hw4BA5cQmk| z9@HqwxITU{GhTU4oc?(19kbtf`_?Z-dXxWD-@g=Q3Sz@ zq@}k>xA*+|&Jf^)^Wr+dJ$Epf7*M+N<Re`c zg`K45DH>P}1)`@JnY}tm4DUcY6@+A8KIjQf`JCg_eij#T>7BJEn?zq&JN!;N3)Xc4 z>W>5lzHsez>vklI-@L#a<_DSIH#Wn}7OwK^4%X`i(P{ZSC30)akvK0E{gN zTBOvE9w|c-H^WSB*ewJew;zD_6&Qh_qpJMml$5-uF-2evmheErnCe-xTAL$a<4Jdc z51WbLj}OccS#wN3?jFq?(Lu#PHkBs3#cQnfI&BB-XS3|rUh@rx4CsOq2c>ZHf;Uas z_!-cyysK((D^pcM+(Z~6+`1dKr&(2T_Lb;WrE9X==MpHi?aIAhU15Kn-S#ytypL_H z54S8^xJSB6f!Yx9D!LDdX@q7xNw>>zFeKi%@lfWg#20V2R}7YuoMLWag9sy58a^uc z#;wUJD9t_#wa+0{B;Ken&|CjS(2QOc{dSg`MEd|!E^Z;v&KXPcsu$eGA1 z_wKFfI7Y<%_pNQvY;w(Y0t;3U;ft#zwc?08nUg4IPcknMDD4wAABB^ghtKxcWP@35 z`~6p5?FxQ{@4qv*e8$yTDGH}?O!;v}X)$ar^>%xErOcJgI&xW1$kNA%SA=}PqpLFC@lglV(7|y)(N8Cs0vRvvByV# z{Wcw)Fc2Ttifencb3l-9li%>!&0N0S!=ao7QN*;YJz)WQi-q~d5GEkU-aoN3OW}_> zfNml#-jK%!9gH*6O?&PyC~7$teGL<1_A-1RrbHvunfD{Y7ZCq|L?b`NaKw=`7Wb@o zf&uHN;>Al04EE$4z7H*LFkC2^%SlP#Y<1gqADe4wEs|=YiG;6Bl zbQv6r-Xy(IW|Wv16X~ut5}B^z11PrU+j@0O1dX#>G0Y}nY}l0QthTZ1QMQ`bGPjKu zRC4w`SeX<&dFB2ovBVjTSb4J2ty?uaXr@1RaG&BvhFb1l9()uYs9vC6)Nxtw6Zl>^=(vf58 zu$zZl>^oWz3uhE@f`bmX`74I8zV6+3Zv+GfRa7oU46T;+(CYjC6fJ;c%F}4y5AD@t zv}v$@ZxOFKh^*XEr6k`c&y}jBDf6B!f59HHWs$xS*m3?lnpTZ4?DzHzCdfW#G(s;x z1{(`!{QlQ3&s@}AQOJoIp>Sic1m+gkg;`q<&#e>)WI+K!L9Hq5)f`F~z?vq~w1lL| z*_yG9k8htD2cTO7lA+HcIcWQU4wsIDr07$UPvvFuFM0amX&=mE8uSj!EyK208=Jn` z7JAZ(o+bsSD5^HS@q~yzRbaBUitYT7MFB#{^M_L&`wYDJtb@yLZ078GYqGZwT@+uQ)Ci&6Qf+_qm zY_@Gwh&ZLJ+aRM%-HFD#`k$lhqGa6o94;$<*;I##hlP7fby%X<=Oc#A*0xJhs2KA+ zb9djjkH;;~ZQ)WO+Fy-$E^si+C&?`)o%^@@U<{?xcL(xKMHO6x!kT8j19s4vm|ZK5 zM6-<4TRCeU;&O1IcG+GSJqqu&bD_RNv;WOaWEoL$^x->D3YFmr$JN)@E}o~$q++GB zD_9tBE+{*Vw?SnUSN4yIUM0W~Nr7zntL*u9r$V$0nMa!+-XLX~x-kyX$n? zSQiR*ERLg38Ttr|yxdHfvdV^zj6!zpKSF}o7JVk$!W@E^E^}Q+oSdAsgkoaKKiO01 zVlLSuFe8L%UG6*yV|S0Z9r%u|{X^@bC2j{v3MUxIPd_Izp+CAVjETW5ZfbdYdeK=G z95e3m4W`RU1QWgwI^5he^}QR~gRX9<56ByiPf7)0g4Dg zZM80l(e)sn^Y0Ir^*mYX7VB-V>scu@FHL7p%zzSvLAeVhbd`-#K)vq6`i({hHF_ZM zjgM(Q7s+G66$}OGYRgbw$MG=EZK10kjc)g|79b}^Mo^P3hNsY>cVtRWUe~5liwwH- zL7o>{=M!ImI~yLH?MVh=iMl0MjmunzIpkQFK3S|!fh8K$j>(XnLup?F>x4faWABT% zq;WJ^ukv=ncPlqaa`YH5;KF`dG{rN;jK!I4)cm^NLwV&SlodUD;KQTA+-WT}+EulY zT-EkL==f;pnwJ6ckWZ4v+T+NflCOzcNTI(2MEPj)=Ukx$8(E4VonL`DHCY321>xqy z!w?_PVq(16saSx9N5U!3%ZT4TG6U_pd-5U#FXhS?-NaMx5nBiyDoYT^0D&YfsWazd zSDYlC#RJJVj%HBVD5Eez!!|< zAm|CjQ;hJ{VM*V8f{3KquP20*eVK`W75=2pg&y_+gmW= zuxx}CJs#@K5^O5bJ4DEsF7pP2dvWM6LAC)Xg~z~b*_WE|0o;+WWJ~GFxtglkWD>NH z^ckXQZZMcGY3(*%0zs62qtR`^(tC{EG2p4*pQ&j)Gpeeu8Spq(kom`!QTl3OQbTts zsAq6c)zS?uxRiu8ORF68bpbvtW!uf+fi1=5>w81O`dsJf^Fqc;ozI0jL z_*2PzUU;E5ktrPmM7CIx{SLLPYtIjLy{dyr9(^c2@zHB@Yu;n>I7@f%IJl}Btz~YG zkRJ+r_2YD|fXa5@^G)XqOzRn)Pi@H<6oJUZj%qE}^8tCdnJA3PuxlZqBPz!;;&>VB zur3;prrW0Fl0YuvZaFq8^7GB~kISW4=F@2`QnHqbl4F8oJYmNN!OZLbfd!C8i+a1O zBISgnCZb$ID|l_;Ao>XlFAsnrgDIuYGit)QUxNXBKR@lQpdG z5BBmyHj%oD30{G3A;#PhtHpe()e0G7Tw6=k{>fHoR3Q?U_lb33@K4I z59i_QM8$cr*et7_&vmNdV9}SfvmSy){PwJP)sravk>+Uv*oTaV)uw299lM(><%~e( zZsv*XCn@`ww$!fmUpc^FFm#`0YuX3`tkoIU1HcarPTC9Fw>|#+Chn;D{iUC8!5;RTxoL?cX8!yk?^#nfq64R5c8OAMe z@r%owBQ>x~E2>RiH3a9SUyA?P&q}#s3LDiyq(cDN^!xSLH~5%-gaN~NLOVwDKIPiY zQfJH7H!xiJi(2;9Uc5PyZm#rf9G|J7P@2N~4NTuA=y_TVk!JN8c^@+-Je#aHIEKNy z#U{Cix`mu?#zlnEx6kMC(~lM3#n~w&l_e08_Ih1xkE6H9M_U@}qj&|FU3V!ZEo1xU zdm+9HH5Ql{x_JdyZLm^mSIe6pFQC72qV`sPI7tR3#>*sU;CfRRcv!dRm*(7=ya`0#*4Foyd(1x%bMw%lyf)zU{zU5>$UL|D}EgvFx51f zeUCF(Zc|o5;ErqN!1Ve}(q0bvB|6j-3%gR^-gd*hXA#2e zS{sB%lYkI{cjJ-2pYKBfuv9&n`{Mb=!x+ibgKA>}Y^H_-1AS96PoInNH)hl@2xIPP zm}2l6w11>WRO0ZmVB)mBOd+`NYAR*wfO2J16&A57K7-ZcB{~JLpE}tWWOL@1kg8gQ z4)adHN*&uFEUa)++$>z5QuUjkovD=`-c3P)r+LmE(I<{&T~%digI+4y68hVYq;s|p z+m@lQ@tK*KTin}yxucFo_=Pbt4VpuQDPr69>H>UId<;S*zM--OG<)Ym_u{3TmW0bx znn(O-uq)cy;wPqyJDD#X?@|}w%1YCy(7H`IBN?Of6QxAFT(_fi`g3cqQF!4@!4kJ_ z_s(5RpKGP}{mqRXSCTlTyf>mTywPL-QRZB7YQ0PCUDTmi`QBXB`7R^6uU5mTsb@tz zpZ5Z*Z)8Rm-O;+-$)YO5V9)KKJX%4dj?3i~bWF|HCg*fr&eu76sjzV!k|lGpE@zFI z`Xd4FzAaBBpU+qF{^r)+a< z-I3(p*o(xg@ND^125b~v-`g{r6;E&5r>jp^M@Gy-cYfw;dItR5*!f2OIQ#{|OWmt@ zIV4qAhFg?`vwpE6d|UdsccV)wYt}pYhjCA9>67qH{T~%t4P{?CI!vs7Vp~z^Eg2}>+kaG3Or?Sx z#Sg4f%rSdzgG&t{(3okqfmj?X3U}4W+xemESFc`J$-zr_^9R{>?^I*s=6g3UD)Z=3 zB;~HrWT4Ob>TfyjHTe>otnBdF9oB3+76$%ohd3YgzJUJDc8I>%+$3hIU94b?KW{Hr zr;~HV3>l~m9|daE>%cti<6JM)PIFgjelviRHhL&J7n#jdG#M23&TauF;ZfacD|wcO(l4jM3&kwRQag@TgSs71 z&CjOl4ZEE<#?=$Sv|kKgx}Qxc`0_80e(C6|qfHy?+Je6*aXg?rDXhNRnXE=X+9e@I z6t`cW4B{O$X1kh-J$-bP{CIkHNVnt`I69py|8f5P^pV?YcLsUSEy=?oWhTfkAjuZ~ z$0a~F{tN7``BL`KC7{mxhv`4RAKCn}0zSOBG1;@Oo@>$PL0!}jf7^}NSI+`qqg&j9 z)ggV%O%_Cd@t|k|#z-#A$zSDzeDJQE*O36vL&?X-N5|U_Y3ndNH8s_G^>KpltCi}C z3iZN?FNrWUIIlp*Q#RkhRb80R{6yJ&o4D=&*b}rN!u;z6H-A7PI*kPG^{+7gwjTxH zFMof=DYm`NSy9R=KT5{ZP4+VBrunaLf!;5@==qDIw0rt*&xwDeZgkn`y4dZ6+jKHp zT0Mv@_E*lXm>&?@Xrh`Eg+@HjQxnMkQE5!>{Ua6u3;*-2N*eWa&ti3%fr0g!)YGbW zrySLI^yrlEp=G__yob5+V_v`=nnv|5?RyjJ;{meKUz?vEaw!1xEddq5pAWxZ3UVaQ z-{Pv~kLL&4JO%w>PP|a?dbA}727?=}QCiYqj(4+?h%?)dpJwS0R(fUF)cYse@Ko9h)u|1~bZKerE(u>f<$|GG^# z{*Lf(cMkwy?8CwNZ(aNRn6XgH^Jq2w!IjkAf@vICz4%`r&s61_Wbmf#VHkZk@6TQp z0-6grn*R?{r8oa)T>STpFBZIkp}LfFoBHm0GWmB0oc2fNWKwaPolpeY5&KhueVBj0 z(f!8inuUXY+E{z)eO*XTcGxXIs!;U1cxzZn9$ee4_e8RyXpmcJrD z4fkCrqA!379M5%31gZFg_Ww9KA#mI^`zsFx572D1 zA+MUmZEP=#jnl0cx1BNoP3Vy0*8fHxqQ`+5zxy-eCs)2IgyYPALl>pfdk57e;6}~% zQR{!P4GRRndfGLjHYspIDQVI3Bnt~3F?hB}C*^&%=`RKRmHeN3P`JjsD{VZ8ZN4bHL;ZY6Mp?~GH5tuCC_-}TC=@+nh(w+FLwnw+&i6@K3 zK(@x`6yXQkh&Bb68-_E&41g)PI;gOpkExFPccqSdsHXbJWAT;Xjosc9^I$azw?l>+ zyjZJtW&GUx?fq1#;Zq)8+f~cTna9b1DYPb8{gUWAxKjTeFv8dMo`;Q-2Blv1H(09RuwuYF)5ZC!Hm6Za?AmAg%gXSqBbVqf=BcM-h+kwz0_t<3f9zo zFgGudA1ubBZ&<*yZHfN9mJ&EM6!OQ$qk(}_zpDQ%^Y}^Xbhx9`z~#%GCgt^uP(x>r zSG6zyBZuK(VgrE)R;NL!O~v9}>Ypr-i~P$!Mgl*{4p_QRLH}y(1;8@RDD~yAWuPpS%3I<1po?YTc+eEkI;z!-O!*>)UngyhO`F{`m(0r6!Y{t@jPNu z|Jz4C8H5AtCN+VkW=Q*;43Nq54dw5x*OxyK|LOd<2Pa1FC)0k&tDkw)v`;X6^`xbk5s0o0K;OZ*jc)SDp-zEpwr8 z)2miIT$vwRsA{=+k1-sL3#wcBDRkfWhd&x&?Il<&1l-_*Z%4AOS-iZ6h$}fZ7;}as zOkKZ}DF*8i8Vdd&Ch152^2vbz{r6QGIPv@CpW*+%nKIz3|F&WRXa2f&{~vGj?e7lg zk7r`VF`K{tU(DBk`LAKAo&Ou(?fByCoHr@to@zERayX{-m`YrHyvtxa+8rBj`tY1x;H}#B3mo~1|Q`m$|BLopC@Q=N! zh5EIau<*31{U?CvqJ087k@ll4_=WVcmknb(8y-0vB3ucGszIq)xsi~GJZ*B4O8a7R z28Fw^FCs5az3dNgVfeM2>FBswuUYv;Tu_$Z6H9zP+6$ZXIF_CC4( z71S>(NvK~{us?Z~Q{aS=+PyNAHL1zrppyShY#rbMVbmPboxy$Rya*AyuzrhnBkov( zuX!1`KpkHnVcI5D_`^sO|Gw8~^ye!ueX-hW2G2thF?H%{Yzt?a1g`f!e=aTgmX~~0xfM(&_ zS+ReUv9vFGBZJtx-dEe5A?D8ycHLQf_&J_!#iuP5$)#`F_s{!P-*_u4-_#9ig5-Mv zmD!gy+YJ^vjRWqNvS&Q#l(}8)-3H@tRh6?u#;5Wi7aHgM`n)GUiJT9lj`M;X!+1s4 zDX8dUqA}#cV^TQI0jy)aKGtk_!m3jPX{_v?L0k`)x&7gV?Acss!Z?RRpL!}f0VE1t4fqC(i5ZeKykvB`@pgFo-A*;S3!|rOhG##t zo|-$S&>=AHJ!zyluY5v&gpm>Yl7+kTMf< zUTkyFwvs?y4bL!5rLyMmDuLe`Uwke7e@uwLMQs^Cr{ zK+_UXs~}G#%ey&8WmU*`a_9-xQ4$s5Q%L%bj%_(}aKUaaH5NsGyw({fs53`kzWD50 zA<5kH`FhDrve7DC-pzqQB?&1=EM9z&S@0(@vV1dGm9rKtsr{UC2ScB`XnokKM^bk! zbdK&YVX9WMIlk7+-GRD$1YuC|W7B4?4JcS_M5bzG2Fu_v_TvaKV&)!Ss|#Aqp06Ry z0W_ubNrpvFkx|CUvVXtSc(CX>X{Xyrzf|{^j0$gx;-=b-TWQF6vM$obe2vDe91^EA zFgAFLbLIY6B0B-GmEoU@qfA9vU8S7}s%7Wyns+~95rkUMXt!;y!S^lcO%AM&ei_d; z1@N0q%iWgde+j%PP_?|!ESa_{2G#v*X+Ctmd3fWMi$@+C*>EhQcQ7y6;&(BtyUk2= z|0{|tnEEccKJGH^$JWWAPl?>xb?2?9Sj%k!3E4@l(?R+zl0=n=UL~05goNFID`HUs zfs{;qybZ4%5fX_XA&aE?Asu=Xm+^V>u#7-Y>DC@s;r!5$$JPF@Ku}7;QB9G9GkEsst<}M&ECV>94Mb& z)nbE>m2&zZVX@N1pv}tO)2V8jCd2#^=BYJtciBV4ytdx|x3NJXt`}SWFRQuroEVn0k;{x%zY{{$4$3 z#GuquNnG0<3#_;amcmi0eMig={2;vz3|xzD4>_phKdPUkEjJzxSr}#9 zBP1Q$+~2kDox#*EpC+y<2)POOO_C%F!Eids!oNa<0pSDu4{Ond`Opr@6BSI0dUa#@ zWI`j($}C}+$fh>R#-NmxEF5V*z|?~rph_!Jjt4Y zC@nt?t>Je-P!k82>nZPD;W6{{NY!H+4BAi>eWZ+obGU4L#%kwW{0B4O8R~rqzP8LvqGHY4@B8iYUfj}mc~eUPdTZ^d z^qqxNJ1%v}(6)-<9#xccxx<3fXxJ1bM0(lndF|e1dRb zFW{*WKW1$nOXAF-&bx&Qx6)RAKYnuqfVkJ^!^q>HV;NY^8$xb&=E;e#=ajlx-f$Pw zR$?|doG@&(1$*;HrpQ+Y*E>X*VYy<4gxt=G{GMP}*luO5^@RP1LBPqOl)I+rynI%n zlZ|Scy38~fA?fTfRuFdaER{9pJQWe^(HN9$3najd5Dgf4pv(xEX(T#|@kdyR2^PEs z_6om0;8aRcQuh3?NsJJ#qno{J34gGZxgrsB)7=CS4VqdB^t8@#2IG%yn_1k&KY-?U zw)*ZW2GHlDe^zVC>=@nES%8AobAC?gj^u+^*(AgE_l-;jKtoCq`EOMypJYfrKFPI*vF#KW zc6g?)A=yV-51N=Kn`~{$g<>gDJ9BPokL68t{JwhpxPS}jqQRL^;_x95>@4QKa*&Zxzq&0`YLwZ|B~ZF4 z>!ligAU`o{}DGEJv&;+&dn0*lpZ{KiM}xD7rRQ zF!H>;@QbCoLP3@O5~knq3KIzoX>smu0G>!t+G4Oq34aPy4m9KOCoOnSmEn>v6gP>bXY%f!%o3~|lGcf=v*@)k~C%$(>rxde= zm7G56UEIB-7{B?P5kRYX|8|@cp3F?LSQ><+bEnhkYPRw?4rU6k_j2^)zI(}pw|JE} zI}6t~WNFH$AeOlDe7oq}d#-bHa4Pk%oB#buLHm6=o%Q{8=b@u1dpiE-tj3lZT5J zkgZI0+m1LO001&n2re;Ftvnf7FRIDyu#&ztc2)He^g4wF*l0%(9+(T*4HZS6h9k-$|`No@(Oh6=Sxwhnnd-onY|2OOiF$=Z}?4 zb#7q`K7Qay_~1kHqc(<0+qdjzkfWr|`x#onFiS z`c974P}JCEveZ-cY%Sx}N*E31!I{&mrmGvTtMype&2b3jEKppkeSOurM}u+y{q=ml zUw+o2$;D+?1ZiW>rWo8d0|`!#iyF{cJAEmrX4D_+D-;#!yC#nX6po4xe-#$4c37Qw z$&gqBtF#psZsot28|8&-v)jWg7XAJGk+CRh7K>&tIj=WnPE*edb93Xl>Wxi8jbOzg5U;uT{q)tT(2b2|72b;HZ^$K3{ppbBZt-mKg28c` zd&oBU-Aa2AebJC|64xr5Mx5ompXH~(Tt^q0RU77rDUSVwO*&j@?Jsdz1;sl8&YQ?3 zHcLMgg`wMye^p#)X z-SmYoh44XhDqPW3&1mHA=5Tiq{FuRg{L7LzFZ^ow^tyY?Fn0$cvcVm6p?vTj)_q18b1$cX?JPXfbOR#0R`;S<;x;Z#(aQ)iRQ zp1>Xp6qGj{BD{Ep$cJD>Xid%&fIy8sg^_un7m0q4C-nU@7@b+|}2&cX#M{ z@hvJLG&?Uy)wn3?R4$-L8`&@Bd*VxnG`_>iyv*mpB@11F9g-oMnT?pajO=fAoEoQJ zE2*l}(Rz?anxZ+U#!=ZQb&QrKpDd;_&Q>(A7>18{BdMm=8M-)hTzBoYe20;rG&<(k|@%!!R#p5>`R4jMx;QDBYHcK|~9NtOd3X?6qI3G-4 z@3sDU(EGyj#2s{8ftcd7P}y}o8p-#c;0~?Ky=VW{HB_TD{jMvoXso z6G@}fWFRgI<)rM9bw;d7>nvm}OJ}mGq=0-%1};^R#D82Gftt->N@Z4@Wn;gbkHE=D72-W zwV^oudL$i^-E^ChxH#1Ov6ffRb>HDO_x{o&raHQ}+(D39?b;!m!#E(EUdnsCv;J;8`3Xod8VDj$A{<~GCnWcyhgpJEYwPU- zv#V|g!ZM8-ARe8w_9t-Y;ea>$oNdi3my1sIZHaS zC8vvNO(*~y2RdBK4o#O3(s8TcHM@~l99Z!Wu@dL;j&H>sehyq-A9qqbc59&`=mF(L zGv{=^3e*bbBlC2KOv1z_g<^M#YXRVnC~-$^kvH;*gk@zGQXDD)I*r6)5{gn>X$fFL!)ZU2smExLW;N!<%Vn#FUr zW14#g`&Wy`*=NB)!lBT9I+Gif9XM7h5VA7phDsi6l_;DvL#p$FXgPkTXEp zMgCc~WBc3z|D+C5^RFwv(d-jHQ8W~qIWH5nw6S-aaU%2WzIlW7M7=6}@Q3L{&Mq(M z6c~1wPt*;$B$SiTs;Z*ztT$m)HyiLwAKIG!LJrF7Nqsl(5cHVQ8i~RRKFkxs()?}u7%i^=ZJf>cfK^9_aR2yr|7NO!ZG2yo<6`*{ zZV1lN(VQ`2hY)6n&CJQaM;gD9V1UW_+2ncC|TS2@zX(nK|)wR*Lz`Y1%)c=Anht6u*68yKv`SfC{^EFO|CW0LR?UU z0ZXj%qzj*}nv#^r%%H{WBr#d*rm;t)2(sDS)RGC~ zpC?ST$Lsyf5K_{=Uu>={6>0ON5$KtSwNV!2br-rx^hfrkE*c_qvVR7k{Q6!7^|BsA z9He?);WWKAn2V=!12x_tfk5}>@i+b^M*cG75@Lza#pmL6U?_cPnR*MTuf z_(X|Ua-x14zmC_h+_yY4B7GH^YAt~eI@gvWHwXo5K5$?t}y3hluC*$M5hzBq@WC~6U z$F6g4)wF9)F+iA@7|tm%rM1f~UH(KOZ11JUsyHsM%4B`1nfjD`V7eB(j+p%sfo0J2 zOlc9wtE^`j7~|tn@sjv+%oVFefzzy1l>qUz0e);?P8C=bpy?|Z8D9^nwF0s1qFb7! zlsR6(-_AcfYoFut1z|?Cn%Um#MdV0|i`)bW>Cd7h^sgtz(#VD2ocUQh_O|umc6NPd zjT|)uN#P#z3k>k=_Vqxf6|6v(%BOp|&Gu=z(ykPPJ-BrwxU3u^Y_}YvE@hVgTv*&Q zJ)#5)ok*g8wL~3EB?f9L(!a>?fjp9a_E<*ySs9>J(w! zL*^|TA0&dVwmO(UDWQ_(-c!eqDS#32i3)`ni-(dQ=F}`#x_{Gjj?@J8jwI3l!Su^7 zwq`fICyCPVHVEvL5Gm$SSF=YXsY%VnuZr2T(Y8SwZqzYgnVOp`hNox*zHh^~K!{W= zBcaxi<>#5p)mWmr8z_PgG6)1cn+wP>5!@=+gvX?Ljl)Ai|Ct2hGCF-~1)y(3XIxzi_NL_^qr6SprrI?svx7^x2@k zoRTr+<2s^JOg{a!g4Ik*&H#$O*UE?!i!EL8&aD2svuj)47g(i!mb<6xESm|<@gL8= z8p+B=c}e^4S40lP-~gcQve;=0|0?x!@n0I6b;B~-WtmEQG{2A_u~`szwX_MDqNy2s zq0id+abehauYm`c_Yp%~P1!`c|23&=;VY6rcVHFB9KqTqau!-O@C=rN*5BO`me3rt z**c5~%5VKZ5yaG|qW^nc95XtJtxjeV3C&Z9EdQ)w?8((C(c!OLE9->dRN8w-J=fyX zl&v@B%A1`ZUemwNGjG^BQdm3w8fbAQHrbCf#5-Zq$mV#h47>>@YHodRyKW4;36^iH z@ait8y>A6xmSZ{Hd%N>G2E4r3>9hFNGaHZiaC9qU+WZ+n46N<xw}%Ug;E;{_^7wynDXBZy&?cGNj&>UQpQU<%`b-(?cm35B@GF!NP zHPikhjw4)rwu#-!mnBk$QgHZqqcB?T?#lh7)RsUu9KFKl*j&J`@D*yw{^~=Hf_FV^ zets>sSf|`KL=C)$Np^i@MIDiICX`t;B9MJQC+GBQnbH8Me10+BC(|!5>shGf?9YA^ ztWpweiNG*#OQXRQ3AT|z{cI7Vi#Pxr5^3yer2ry3&T)?4hDbRI8Lr|{v=3khQ5^HM zhM$r`|K=-HHx}#JIK@D?SxNE4Inhyj=SG%bzh4b4{9>yNlk%K#@poS0s#RoyLVT zZr6ahTJtHZs@vtt*%A2Y;u~1*U2=Q*Xs|@ykPWVLh4@Kx^q>{Y)!P{yxQ{XGQ>1fl zvhi&5uz3G9pB;k!+;#^TQT8Nm)5@8`JkLQcjNx(1!h?&6Mb*mhkBub8(L}I+Bl;o2 z+k^>$UM?^zL3;#I^*xZS3V?)29ox&}<-WWDm<}&K(!L_#=-W^2)wxITT1_zt!UpPo z8+$Q|%LrmdNR&nH%-|>y7a*X4ZRE}A!qAK;HO)nto9k6S>==dwv+70|PPC=Z(X$BThRJ+8h4&*VHo^#g@?Dbu{FD zjbu(+@S4z7xO+&q-PgZKB=rL3Ov#ix+cF-6f}oL$`8ODzBQ&8OKb5=}gk{kAx@!nv zz(w99R(IolpWB)zxL}m)DhlfwIfxrPbQ`i7!-{axhPr4^s^QCQ27$XoLd@6lQT8e7+mG%t1xKDVsIpFTtt(wV{DKFDc3n<92dO z`7kglx#%;@(r}^ie4UrF;{>aiWfe(&!NOO_H(>#cQN1HBN9Fk`7d7Prjtz@7B!TU3 z_T@7;;sP&issVP(&+nUii`c3@ToR7^fNe9B#%VC+aelnFnNohgpO=?Clc&fEii$Q#`!@2;@(T|v$YGV?Sh+NK?+DB>#X zTrGG1SJH?!y}JTkW6*RpeQZLYNJA;7$ASgiOmgm^8gLd_yZSZA433~s>-EK!?!>sl2(L>4xeluEE@Yst)0&cR zj-yr+7P8)2m8$Pl^w3Kie&hG99e0Qb{QA^0&Ey@l^znNUi}ca2=oG@p3ZFjD)5Q6g z!P-#DD{j=#RrY0;W}}ZbP}|DOituek=-zw+*CbS%+BP&P0z1eLookMnM?kHJz&BPW_vwX5Gn$Sbo6N%#xn zuxNhF8YhnVITFNfeQsVV@$& z*m8%EgdO|lu4#+%_^e2GHF9B#Nm?GFll>m^*$fUbo^PpOnl0OyTWaC_!Lcui_adsX ztcZx!{SsAo$SnFn^huYpsf9!7Rpw5TsbX&}axgzFD9OfW>DW(%Tz8yTz{2&}*Jp4M zi{w$Vlq!v?j4-92E=zMA@m!p?(&B1{2W5C}TSm_Ot#bgKLP{sja6*kE5A!vAAj zA$>nCkM=zYtMhu=u`v2Qwc0;H-)T{4*>K`V`I@-1vt=kkFIIydHJrE^>?}YPA5eIt zsmiEX?EoH9zVWDUWZt&16If%EJXi9~m+u|-XV?Kylb?C`z|CB+aQ-D0J_3C-5 zOI59jT9&_}BC+kP-*O>2{O~QbY@EDU#EFQtt%+6!pVaA+5oClm9JeGf>=dD=rJFjC zh^C7riv|ETPN^YRH<{A+0J3edRGl%^nh<<~mUMsB!U)rHfX(aKU-d+%4zMfIj=Eop ztk!n)#xeqMx~BlP$d_l*?W_kU3rpX>y1W@H6t53GT}_ZCdo#tBH< z{@`P)Ybk>J(k~*3Re0F;rr(hbzIM(Wqtz%*p_E8k7R)2& z79iRNmvDi8qz2Rk86`Yfq8B|{!2&!h1HF4cY#*g-@M}!HSP{xw1A>0lz8Y3bZ%u#3 z$01+$(ftULVVZ8Tr6UdTFOyBhH8RPSjc{#dj$DMyiUv$!9q7@sOl(uhn;YWqzAUNAYSjUb`sz~}|?L&EXM|aDyku+X*RR$#= zTh>*;u{Xi7MK(oWT`zqI(y}Y>JS|Og=7WEXvRtk z=i1mFM7m1mfyql>N?XzZ8yYdPe=3v@<{bLqnXiIJP3tAALnYY--#uV7HnKW;YxVfu zS02I38fkv~0a~3jS^X-JV2-uuUkO|~$8zvFGSol)Sm>7bJ73A^yszS0@mQv{OiDR( z+EC7G#~?vn1;nShutLpa%nkgYOQlF-QR!F#{c5>6>}5=1YhEZs&_occ@g2A&{Vjwn z&+wVVf2FiY1$9tv*xZtt??={px*rb`sL6s>q@$g`UUG7*+k~bLf566$j?_+yU_iEK zh2%oUpp1Sh#f5IpE|n2@An&xx4B=-fr^%4Lfc1D=ihk`*RChk=lR&Wyc|~U>j+*vZ znzM$Z$qDQRKF(2*hM}p;Nt&o!?czJm@kwPYh1;Sr?)s!u2vOy;#g9(5U!fdy2mKB7 zwiE;0`#b;q2m?D`8g<3UUU{-?FmS?!L>+ah1g@ZD8Hs6ZO5by}W_&8ZxJ1M8$6liQ z!j2ppdwoCp**@uwjYKZPM|zaMh=VVgJl?xHxt_Tjm|L{Qhhb?!JlZ`Cax30rHL1drQXH(7PsHiFqSPf zhaa2pj*C7e_g&?+e?P4suwyJoyTJU@SGai&!TS8|H!qJ)C0S#}z?~G{VQ!55{7OZAFG#x)59w9LBH7P>u28+vq~Ul&P^db4Ytndk;d*_# zEtGb2HmQ3^arav@ij0YYK6D}ny|%8>^&j&h0d>qHvl3Tt>i?^+E02eA>*KoK8@*ZU zCUQ$$B}F46vW`oU%JN3ZmVFQ*dkiCMwk#ErOcJF@Xfd{#p|Qlo6fyRpvd%D;7|Sg0 zd35i6-}}DzegB!y`JD4Szw?~$`Tc(9JkR{T=f~OR$Y#5=E3Q}fb4nu5Ic7Pk8yBb! z&(YnKEce~o6{aJ)(=ty_`G|;E5Ea|aKS{QtANobN6hx@-Pdy6r8;L=1Zvdu)0&?c- zGg3UnqPVWD&Yhk<2`#L~2NSqOi=!1|gr`!Hp{uWm+Yg!s5r-d=km>8jl{S` znq-(ee_C2c-+skD3zeVzMw$NqmnxG@vU$lR#csK8sl(`WF|#aQck3FstVg6WJCt*A zUj6U6+zDr~YLB4@TvuOctyt?(&R@8}0-^2ZKMw60McZL0>|S(5Kcis*C6O|%gOc;t z!F1boi;5glk~re8LsJ(gi>duPr)FxYpI+xNup8{tP7IqUX-(>xnF9_FXRg_KWZ3Z| zbH13q>sRfnr&9U0P0t1ku78ear=Cuv={2Ub54CRadO~=89qq4^@ax#f7pDob7B#;n zD?$k<8caleljcN!D2^cHs6O}`?8WHk=9oE!{nCHQrr``ZY->(S$u;{SA$bDLQ@!%( z^vcu1fRA(SJ=mP19M;NU!UxBAM{cB7lOSRxqvqgnSJxt$2pbOB$e41fSuHQI^V>VC z;Syb7v|{h=?cLoxVTaZop!mJRhwgcZ7v}s3)_fZJ94?_f{QzHd4%e9dzK&T69lpRV z?9n}OT?CLHtj=_O#^H`Y`~)-9Kj4aPGjFnGD~ATZZcE==z8*8JB0n81ohe5C3U|wH>P2_dss7Ma61v2X}pg0=@kCh(t`i4zT_oQLFs7 z&}%*Ym~#-NWD;htv)=z<4|U-}hlWR#4ivk;#GF8*L(WkL2HJz<2=o2N90D!j-8L$t z>&tY^-fqq!^Xxf=w?a_7YeV#FcPv6DC=S@n`~@H_2Rn2TucJ5qs;>hE}S_WdAIc$B&m}f zrmhfSNe7b71a;1IAS9#Qw6<@x#}8!P#!o+ywNc#+@&@?%lhZqpp~U!}E6ax;4gPl6 za3o06!RPc)8c>vgY5_XO#3a>bcXev|ACxLO6x+|gLq2d{!8{*8M1(h|&8weJhdf{) zcYb3*bO?HjAdPcEoX_m=?#sBuyy~2+HrgrsI@?hIK^)s|nbpD#QR+@o*$g69aq3T@ zr~r4ej264J-bI=sJx1+1pc9}L?xYbDMZ9tC@dT?>>@3D{*`7h&g z(l252_V!+cxHF$n4lMTBa_IZn_|^BkYK9D?#7}r@oU#P+%sB*>7nKXtcIK69t!yyz`q*2do|2+8kio$} zX0R!f_c~*?fNUTe6_1}!JMRAf#?Sv9zn9rWf8J^Y-PwWk1mY&6Y`H`Is zSqNccRg-6~=d+qoy}ypjD-#KK(g6zL^!1%MsE-om`GP(DCS}g4;Y(l%9N?E$cN;gF zjs`?BOdc0ObOiwJ*%+)u+lztu&|7S6DqJ*s0B8>=NJ|YL9o&qLkG}tBHla{1&U2dZ zITiw|^)Mox%lQF-P?~c-=g2N-k<)ghqTqouG9dR!3uN*et^25G8{i9ss(tV}pkaof z+Z-hK{x2fzqhy*Bw=}JOF*h~kAtB@ZCPPLLWemuJ;vdaD(6wQ_NT>q|?DED|e&M8^ zk*STXvExJuXoi-1dzsQgIe50+#Pa1Ajx5gF+A^JLXKSma9w;UA*VG`OAJsR$Q|#Jt z>LXEulF)Rv%8}Ah754$gR}4q;KH|8^ps4U|Y@5u*pF}gqNAy-(H&`NV2Pj5qd)Ay! zT^o%w^ejCSv8fhYME|w`8GlqHed@zD7NlDXiG8I#%EVfC15Xv8x$iz)e`1y(EUoTF z)Bb}77Y|V#B>mTTxKLu{g}b3YH(bGxjYb&wiPVw;x ztUx&evTHQNOiJnpE;D*|7-mbw+Nru7JxVsJ(onr-BzUQ!Leu#DGr@i<3+$uQ0vD0j zf@46UEhXJqJU$erb+{^LU-`h2eGP_!?Y>W9q5S-AFVdHWBDN-;DNb08+Yl#_#?|(w z4$D~^8X6X#R}4vEqcTK0bo?HB*&fYt#oa8*#MD)8i54U*E)UM3J?$70YimoE=9HQM zFNQ>oZ4FzS#R)eAwbN1D;;I^|h(?PFYP_DlqPv7*NQeswJ-!-P9=SZuUzyV63^4^& zB_YESY5`ieKNgalB_9%tI*`V?V|d!&7u|7rc@t87i}2D(?CC6dwNK_NHbXdYRRMY? zvu{_xavBgeJmTSsd)HIds^0t}0}C7SpOtRCoH8t@fM=?;!}k`wdGp~!`C{0`e|2OS zSZI;Enrh2*$dWqbNI9jy31u|IPBorw0V)loXdT@{!`ma9MyC~GVAY>$2l;<%cal9akB{?sOC=>G6BDe7m}hI?Np6nT(TH>wI$RDw%!qX6X;`CRAu0BpsYFEo2+$to z#qPlQ56T+lN}?J~oLFm=_xKHKkrrR^#?euznxV@!V#r zd>NRUu@FDLlNDYZimUazo*chwB(+Gi#WxchK`l#jkhQ*Qt<3$hGc8~z$yj>g-mQ;d zoADnAZ%$>^To_xQLa(&@`h;1A)9~?mju%cx^e)ek1zd3NC=0Gkz1Plx#r8=K@}kl* zee6y|db6+iuccPuo}`JLOzx48%oC}?d?)8qXzaOZJPbSmrhsHTXTOB)Qw~@j_0!`1H4801sz=o8>m) zO6h0E9%uz$0c^IN7`U36mF2`8MN2(YAv{%BAAesnaH63>pCOTt;MU@_c_}MD&nS!L z6xru3ov%4Ia`I!Y$jZ;)h;1(}$T8s7)}$bR_e@K=Lte>ohnE(IJ_P}tS{ygeM{msH zz&?ynf)1M0f2yXFE7Ec_>$X>SNr4`R)~7sLWLxbyw!WbnMK^7^Lh5%l?4dC) zN=Jv|e1~h@$=Q(^aBqjW2aHoAqB%4wRl{$jzKty72MB2wWMFsC93R)vpsr{aTOHo3 z8Q|6!^W*It1h-KNj(q<>8#l9do3p<1?b~Y{&iryz4|g*B8tE&Mkg^;0Vww$nI3oPp zRL$j~^aE%Y%}ExE1u^>;7vXJ~diDkEMR)EP`X$fD`a)LCH%AOQ@(v6#l|^vAER3f^ zX3otK00k^`Zs>ZqsfMW$SHS8xO+j%BpT0vMAu^)SBJ68sczs7oC2(>xZznqLcdN}# zO+5{kb`}A13i2FTTT_#EDi{bhyIdQrx>`w3bqD?+dR0qX8&*9UBldQFem+~-&V?#P z?gM@zb5}}pnH>&9u4KYOX9s#{G#Ol%B;+=3K7aKpFf1(W8(9nG`Wos^jy^SYBEsi% zchHxXH1Um84@hwCEg8fJI3;?b(J)aYI5gCF+VU=RvC9mD;p62cwd91KwR|>fv;ejb ztymzE%U(0dx6c@x3H^QBEk&&O|$KL+}RY!f)VPfL9t><|5PQ12Jh?z|mvehAOsfko&M ztq#YFm7Ke$y0EM08*TvH5L{SbEe%0tu^al;-|7nJj2Vp(C~l{-9@o^==(Q$H^g;h9 zh#7c#c_~VyRN7g2y!pLHFml~`9g{5iwP{YMLfdeJE48!}x?5bA)ljD0J6C3P7+ap| z;QqrVn%@2F{>me6$G~<@Ux_k2So`%w#g{70w#CHscrk;pKytD?kYz(CMg-Knk^(+; ze>R;d+4o94?oGszL1N@}L0NWK#HxGo`BI%aWb0LI)z2xzes?xuXvvcq($bC-n?NIN z7>AhOfWbE~Iq|eJ|we!uueKUg~k=Z*4{3%BjRy|^Mjo&8`1j^Ge&JdX36SAe{%7&X zA^3-4E3V;ygoM$G_jBbifsdYW^g^?O@y<73U(z zj3!+gpNo=h+j&;`UHsRn!g4_uVJ}kT_0X?gia8|68c(*8&W+c?=(QzI_PO8xUGQ+~GBusC7zbE7B! ze&U{tL#O(3h+!@X{oi-~PQBFr_Yx`PbaqQ5`@Eb)fb(^dyWg-?U<%j&-V!N zGxCn+s#{-P)<>OkWGQS)U_$wIB8@X-0GnpCT@e>xs6QmfyTW;SXu zx<+fo0ND&m#z}a{5je$V2>}|L75_&82ikwWQxs>|W8|~JJJsU_EJXVsuv&L_N`Y=+K*CCY<`9X1?+iZr$AlJ5J|D6TPYtU!p37m(jy`5v7oQ5x zd-A`2!$-KmqA)_HA$QHI*)M=ZGkCg>2Q8~D#4rG=AAz^w#?I#Nt)z)RqZ(Q0oEVZS zp?B31ZfK_*+jE=1M$>J4uaf!IseKhY@q@41rdHz$gzyFRF)kI=cOz0_Tnbs;GSOTo z9ZZ?4Vf47F%$Ng9V}-b;u9RBf2Q_YQ5LUiuBNP0j>Id{-!t~3PRUC?M$>iuxntkD= zs&}DkB`2EP%|!g#-uL2t(?urM7eqA%VZMDCzEiL`xNrxN-j6+}0RgR_NdRwINlmbP9p-EjX* z_@|Rq5GR${Z%j@pks4T)rb1d#4zd z%osxfUgpeEHS<$}jBB9wb?+F@oWz_OD?|$?LPcT!Ah%Atr@lH~%ugyvh)kcGWyFaU{hpS3?+}n38Ak58XEH5uV6F>y9hSD)sS7AQkHy;()beT9X zsdP0Jj?MD)k^=^mHV+%bvuGA|tCtMtj+DmWFVK}dgt zdIl#Z#O~03?3MmD8>CLszxK7RO#0c&P&}TNAsV=z zqpH?>h9}grIq_0ZxHoKCyh0}XULJa~APg2ctNgzPPhK;V>aA*{ED5xV3?_G|jIDf< zQ*(KH$t%tHUMxA-7q{sGCRehVGWCdaQQTjKN>)heW^ckm2-~)%PrmEpc?1zFT%RW@CHtMYK_SiT zZ%P+7^uZMR#^xC!hx7iEi$D1wqZ+T#337Z-%!dve`>4}uG3{32T1cx}H}D=aFc5e4 zU%XB8mihZWu{GtN=I4G<~&BISc|-FF77 zLFG%M9b~3IhANgZCJhR|j3Wn|7<;bU%~WUOPnH}^k~r#+sAAWI%{QNgz-yMml09J= zA7V}15D2+!M{B!^pQrXFuoQHal=+Z!6%ix1(10$!s7K{R1wZ zaQiN~$jkLJmUPML@Z!>5y}JC*%eWA8Jmio3T79?=ANj%I5B=2HVGh%6Lj)(+a5x;T zPPWbo%yjafwLAJ|iAp)B{?ahkCs@ z{_@4-xMJ6Z(9rFSYkfZp8KEnQ+X?<9D2sX^IZL8uvk2 zd6E6bIhcYm=c z=ePa1e)y!4gWqcINu_zjie@Tz6i41MYSrOfnT<+}XHIdha=sJxU&n?|Zx8cDSVQ3A z*JbqkC4GFYf4XR3Fl9!j$$lZ6bUO=8b(rPviZ2!Da|Avw+q2`)mUWPQGJaOoe3IcO zNTM``#4W0Kf$%{)z(~D;E>V#eRZBk`o{S%Khzo?&@D%~vKBW-!l#Q&H2|i1-4EbZ6 z=qvulVw0jUa0#7 z%EmA)!{Oby8I3u^GXn4IH7gDKeHEcl##1K?g~w93>w$3icQ=O!Vkx<<;X$5avq~svS=Jz!o#rPah{`Q( z@9sU(c*j^Gp{JTyEumpRK9o56Q%16KPVk$=+I~i!Qk1BH+4`ff%z?~o$^hh+QOCN9 z3u6DHnlN|`PvGnn@$;(A%+6!QNHeVf3`1idD1%%7&C9nRq(xi0!zZ~a43$mf0+f4Y zd&I+@`R0zPr_byDoYXcThb}8dU2ZOM)}yWcftf?cx_cWQuzqCr55^JudgNJj|5!>j zf3M^n@E>W}o_ z8O{N$&OGFHKQ$y|kJq!@gFUm#nFHU}X^wQ32j^Tm$#;5o|?rF>0AqyN1Dybtp36TOaBs`jACMH2u zrNemYNw;tBT6$*#% ze|28^JDrYEY_knxi9}QTy^V}bE^W%FYbvt!&S>@W6296P3sj>Q6V7npAoIKb5?I#6 zieC|^ZSepTkANwvP}TIgQ(H0oBH8NM_r#=PvkE4G!<+BFhwBKPthc=WR)F;{=6_nl z?TKdC-s=Ug5aX>JCMjvN$GP>%|A`X)X0so~)AO`j?S#yaIQf8Qn4HrZL+ zDB`2BODg?6pyQ&r%xlnbn7Rgdvn^sm9&NVfft5najR%X<*qsYsbRwTjW zw52>B85I?!^CO(=|6}652R3Cz5@42eu$r2F*RUhwJGL*|-zN~0wIzJFUfzE0y;m1e z_Mb1hvE38=c52^_oEUQw%^umEZ>%p3vuQW!QI6ynz*DbjAtn?3zF0`-Ec}yEb`xLE z4%cooKJs|IZ$)Ou6_yk>d@J#`j^5~8^+iYIoR&ah=1%<95`AX6b%A(IKlj%B*5~c- zcr-TaLhr#^k}xL-v}*b%TM&EZturfTyQtlk1!CHnjG9@vzdIN3I8#d0!#-aF+ zQjxfE@bEUEHJ@^+5$FSqUoZam%ybEbu>dG!Dv{tn-$O!jPVC41d`SFX_Yw2|Px^~Q z$;%rL_x1DB$@p)koanyac+M;B_uzlt@qebnO9X?E2zf;IM2vE~m3C6^i={Y|WC9lb zFXim~E&c#eyXYqW=X0c=Pp4|^R+0rgvSgFCz@|E&|9iBrbT;pBIFU&wda5N-*MApTwA3=?{{CoHb7OyhE_bB6qU6)3Plju?XWIib z5mVJpzc#b3`piD=6n7f9)!{;+E5=ojP{8C@+mJIWWzIaz6imJ%|7ptI?QVXL*fv*a z{|m-MIu*+gQtfH>=-y+W3$wZtzjWqQcwzJ*HSc(OWSz()og_&ge(ImR4bD`JBO-`C zsDqgN?aSP5a%nsoW6x9*hW*2~GQ;2raawu3FP$BEw_oI_Sj61aUT=s zo}rsL^W^b>SwG!7x!Vfs70E+T@VYW!?bVT)=t;Mq2O}y7^GB0L%sy+59{D4;$Ik9d zX>WM`3eFTR&fDGNe><_Ax3AMo!)BGQ_R3l>e4V!Jhsv;Nc9En`VgmFvYo1p4L zSf-=7DCVf9+(a1-GNmWycxo{MqN^&wV8EF}O8L!1nO_J7Lq4D2E22bI<9-#lQXM5X z=V=3EO{leOYJBs+J9+UE;?K;w@rQ2yP=EOMZk50@InQS!dNSFqJOn9@ZGNx8dXTuj z2fyM10dM(abA)mD;4xk_lU|eOrTx9(Qc2#=M7ZJdQ)n9vCY07o&ws&jaFb9OnaP5N zmyLm?q`v&2ZNvy3gRD{*vqf%idSCq6hIrOmEEyVUmYDmQZNytvF;SRfjpRa;jq(A# zhBdpH9;1BfD`GJ1rx{KVc)>iOerVnqbZUR$rcKD;sPZtdUQ2JL>#3CScXYI8&z`+X zC|Q)r9KSo(dA?!n-BVhTfRS&XT#5Y-W_h&iJ3K-x6gTVsn(9CPbS7g}|8V<292oke zhrbn7U@)z|^vbE#yj)|g`EV-zz6CoN{es#~q_tlIT%232cfl$(SG3YtqOA7ri%0<8 zs-?rB)$#7)MP(Tw4XaImJJZR=rztk?0(Z6 zU4Q)eF}y4QPR%#o_Zca8D@_fL{5yJi4#j}1zip$ZO=-_;&(-UQ+{4V)3Ri6z{WPpd7kc;C@n#~<;8(@f2a;sV_NNaF(w`Jmu0YV2&(fIxe$3D*WC zgc;d>c2ZPt@^njBh@%Cys5D0v1B4xG>3qb5K+ZM&b9L(4moH(kS;^oPLBi&62*l35 z$0KX&bn_uq2UIh+bgEo9>>Z3;g^=EY-M&m~WL#A*3}=6}7KKqIwYn=X+KM5f7?aQP zSAg1~C;Jp}m~I$J|GDte=B{8TN}nVY>MqRa(#uR0ZDEBLwJ$!?OffNCR6ytUx%$55 z_3bh5#@dwe@R}s3k^v|boJ_I_K--d+Ly@$~CEH~)cFWCjT@{APnkZHnvs%Zo?uuaT z%DEU!yFq)TGh+U8?iO*t3PueNYMY{o(H?_CR0M55uZL7osa8aI@nUYupSBIvDdm?A zI;<0N4pAj*SJcllS?@Z~rzcv^QuHs29cIxeCYnC?I0ardP}RNG*%ZaGr6<*+i=yJ# z@NMss(1hBg(D^Bvr^wY;D`Imle7RN=!c#-KX0IyBY}Qum^^sZpBSkmMtAMXl1c#@U zv*RUBnDk`+a#vtA>l5kgqbX?m)og$#(s7omK}!ye*XxbY(kc1@Pjkqh6?%zqw}csh zIRbJ3`Z%Afso_rRf6f>?x<2{NuF)-cZDX;L}J;hc&pNw3!>8Q zJDoP#@zGU9A(zQQE8Em2So?6B3C={U(6(lO?RQkF=F~#VX=C4$Qx!`*P7iU0rn88- z^hMuX^Qg{-nRAYH#lm96*aTdtR2Xk$wr*H=>9FYn=kcoerKY@<%0Aq+3EQDOFe1u% zv|^~6=Wff;Z0jY;`i8~HdI854b{RetDoMLj5;k=abqT~KMbzD`5$DkpwD_=RMhey8 zmYqYj&|m>mVqA}pyWUvn9wUjxr`dVqLjaFBUWDH6vXW~oJDo_u!f>;jsqsW4wYlQg~thoG~IMtjaMZdHQ&- zn?xgw^z;!9wJ3T11h4lNGF>^a9W>(Ra}-a@3WwL6RgcU}6G|EGdOy^>;%-p1}Pg=RcJT_yY5rmtyeH^&LaA1 ziDJJ+U)x--c}d;Z_3zFoYCcRy3Bw})zJ&8=P@~`lV6jQkldj0xrpk8dlR{@0Uqshr zhS6a#g7QBt<58*lU7k?rSXQ`mgy`dTN!=Cr_sg|i5&`RJHO;rMA+u%E zs&_WrRJfrK`mn&=lr&=IH=O=(ah2^BOYHXYuRma6uPLm)z9+pCe}l%Tytzy2f@j>6 zINfAo|I<{#m={9aUrj2zJRO#!VnYx>ZD91Z^KR**@U#>GEW;yv^sv@~WGF7Ke8%Sv zDQsuV5jyOZbEaP+?Uh*;=~7MpLontI93sYnkM2BEVd`K!K+XMjmdJm7TNNsxFXO#yq7O1xf=MQf<^THaD}fqJZ+MrdKEQ zGVwRAW@PEBF%G;Z>vwi~O~s&MM$V{`+ZkB8UMJDW(IE~i?qt*7k#lTG{Qexuo2b>z z2MbbaQlXVI4{`P=3V|!K#_*2{9+N4u>75yEVG8NpCR&U|uHgrmdJ8)%mdtthPQ}Fy z!7@~ooo!3aZYjmi%A)mGk&6R&*ml0;ipG&Nzcph&KNxHZ3v>Co@=-65MKv!_PsC}jF9u3cMpC|{&5WHyhxqS znwrX>p?eKI)X!a`s+F(GPg_>kQT-lT%eG>?uDUZk>Xp3ZY|+U%{E|)1@XCc^< z@Xpv6O39w7f+Dkns>_MAC?qbo8g3boaAW|ikUUD{ zbt*^(3BVEwQN#JDAOB(q8KIW^A!{f!yd7=#JsJ%5?oj&;WbIK&ln0x)znH;r@{Hr>l|8wbo|lyoE|JG-$dyKMu47ed6KFO^^X( zUltgw4)^YBw1OV6fV_$&Prua{a!L?LZm_jSkEU*kFoqBQ&Zq^(Z^-%+T1Sqfa*4tYYgqX>-k96U zzlH7iss$;33r-#tY_uHzT|>UX$GSxgR|jl+>qHq0q{q^h=26KN?CeO@vd2 zqb*G-n8wy;YjoUroz_>aw2P8KhQ2(z>-}o?5SnJ?zM=Ove@{o_>;YzBufvYVa(pe@ zDALFFr}hvC!SS#zgVR*)YijaI3_Jod2^~%C%zjKv0r3FW znJO~^g#z2KEC*k-b=^OPo^)?I_rAHxzauHFmgLw?Lt7>Ynh_xQ=uHWwQL%KXKA0wQ z6@CNyG=S)tIG|RUH}1Z)WaFE&dh^B?1tq+K!Xj|AL*S-wYNjXjJmxi1HQQFTTa$vc z>Nqr{Q@uofe#lFt`1wntT{hFmqfwRu9GikE-5+uBFFCmBmyC>pJyN(^mU5q^a1hAV zdm0=j8~oQwXx}6eo%SnNU#LRlV_FY=1X5a1$ z%ldWAhN>C>@2ug}shUb4LhfMdl70iFohvvqxkT9TY~*@hLbd1Rk@Yjizuh?2{5L7t z^cE8skiK4QT@TjZH_~6oMJzq8rJJ2vI!nGF7aWo@Vp2&ONjadY#GAi&`WU!DU^TQ3 zCri(WL)NNvUq)ouOM=>cI!Q;T;QBKDnK2_5Oebc|WG3Qd8C~lRy8sev2-!qr*BV`r zAyF6$)3FZ2!mW8q=!vQHz7_o6WU zZbwRX!X_~}o7{(}+IFp}MS@1s7#=;1O$qpsMKOIdD3IOjnIF+Ks zw>Gpth^w|o4PK%I2^n#IG`|n%)$GgKl z$sj0fN)(Czh+JKQ@D<0Lx4Iaoa;&UEE}IY*i;#@I1KVH@g8~F%)Tx$@x~nzhek&;F zpqYLrF9utVy3{yK)H&DbnS6kOVPQ`6>P@A28z&S~lEn7N!$Z2@J6t?d@bC8IDz89V^mbr5?(5flLb ze$#>ng_;ko(|3@Y;9EF~LhqjWx)B8*FgKO5Ahh!9mG=uq^(>Z^B$bwT+)|Wppuy|v#1|@2 zb!RE#dAAk6irmQHa#1gbmS(lDpC+8<$sNJPFUtN%kiGp;rN{_PD?^S;pr3Jf&!~Qa zS-Jn>H}cbcB?}9*z^WN8E6ssYt&ZF}2V9$j@3gyZc;^vJJ8!wwapSzv&mh8(2x9f4 ztB1(mK{E~)7o0#*FB^-%n(vTYljwu=M@qnR==o;$z5tCEIp&ak{`?M_rg&Y_W~R~- ze`ayxd0M`BIseEY0#r0;dl+r0eQwuhXIo+9-O@)!nlk6esCsmr!9YDt^gV}$#@*w$ zm$h2pbU}F=MpU0Ew*-tFSM(@2_@5%K?#FUjM!lw4caPjM#28*%>F*!I%f^C)9VF4l zY(7VIwn*y9W1cE1rjv%U$~iF7nOQc6F~-~`<23%v$>Xvy9=1T$!_>3rEf}5j`l4*+ zC9i4FWM`~9qFx4H6Naiy#4(o7aj3?uOBSQQF{|wwH*VY&g(bS0**OXD&C|A_6>Ktu zTgW(*T1CWFaW?+uUorY#AaB7;)hM5>!DKMwP+Y-;@-#kbLQqIbWzMx0QD9Wa{!w~} zroS4GHi5~NH{91zmFMdfo7*IL7#_Ci%OJUPEm|esQ-w)bvKOT?4f$1jp6O>2TPBW3bBexn!;vL!EP{GeuW^scTbZ zV0?4CXcCm{ZhFF%!DfB#DA;-FAo_C9a(hF=KN4{5Ol!wkVZKv}b!aS2@t{`Vh2|S^ zRJkMXK#B6{`|^h%l!9u@W<%cRJP<78Y!q zQf149PJhn$^16dLQ|M83(l43EYKLumR0eqm{Y=z^>MfESZWcTYx}e{Vn&s6>HM=!8 z)1u-rn%|K)3*hY>FM39z>lE=cnK?w2Cu)Z%DR2^+v$sVT9I4?bi9CUT!qZ0EM}+^+ ze&@8DECIFAM%{sHYi(^Jh@Nc=UNq-!8QK-R4C{Ic6>S}q8fgwl_Zj@ znUJ!S+7ziL#B(mhJQXvaO6;06vQKscE#xMh=WQtCA68svXr1(o;7L1L4Uz_0Yp#B1 zR#udKw8_NJ)_cm0r(!|}T9Bz!A|qC}Bv+l?j#+93RiiAFivMhH5xF=NKBH%_>d+Uz zpMAsID%dvsiClJ3pCr^mZ^}RK2AeAB)$b9UtB~&CYPrI`wWZVrtN>@ND2@88-d=?mIiIxJ`65Vxfy9`M%KGH}9vXRgp-L|DUROD}v5Xdf);nM@8o3FWPID%}{ z#f0?{Ba>!RG>!`&uNquZ>z~U06*NN1#FUwjc&Q-^!BTut3PkMzlkq|7d zLX@vqAdQz`l8pt&99feLPR&wx_M!JUs8_Z)|7invH8ZP}OeP_F4PAy(1Rc>mkH3i{ z4z{c;FciOi4CgDP(o~WX#)4;0pXRjtCtaHAYd8a4l zdx;d4_-#d&CPlSp^jmg4UC)B7Rf;zYZ0!%bt_l5A-=r`Z=hm_;pG38daZnHp5%*Ex z(RO)@Ny6GIH;;}~Lqpk>!?jeYrC4KNVDM+N7&|(#AlFr9Rg`oS(R=1Cd}p0(@#!(k zradT2T%F{ADEV;ihGdQ%s-mEzV_BHC1noIrJ{N2ACrF@ME$>?^6x_~B@%kVg-TqSn z2yG^MNz%VvUhOYE^n=l8J%{`XQtHmJxnCW#fpU#ayq>p2dFh2v%F9GgA`#_>xA`k}TBG;izp*pR%3 ze`+#w*TQ@927B6w{l)lgUsG*4IqlpMXyj91SeaVMzP04EJM#NlaLqRx#!2;h{6NLv zPMZJFi;d?8*6VJ{aEs|O{uFVnn+%mC2Jy}}8eOHHD3mdX^Sril+S;?Ar$|c&cJa0X z@1G|4UVm&L=FfV6Nuzv!!(Ij6kY|5`KKgb_It|5JMzt0^D7=;m41f>A3}z?g{DkG+ zwb;UNk?J&+;#fPx)6_;F8VLJ7V0w(V*iNn;nb4vzv;~#hD0VQ4<>v8+JH`b?Q@`Mf zBYkLQl}FPfA=@k;jc*I?^Hy@;Hj*2f=%O2z?WSB7X4ZJ5|j~ zQp8v|)p&4_{a|^=iFNO-wZbe4~d;%*Nwi1HLBf zw7RLj{`O8m_s+cU!kY(vDs=C*=h2VX3JmJxkT)wcipE%eVjNG4W@5xFRPMsM<$n3k z!RXja7qFsj$)B#3saBQE=k9b=@ktc%No)JJzk?Y;TMd0I>-zc24Z`Uj zj=@1h(eL<6Q1t&kT|Elk{=aW3mx9#4ef)1*7il6DG~ezHT3-zCAAb)*Ap^Svo!urM zXD_eY04%D!{QRkH8(hkW0EU+e&EB_Hr`veg*rNkbT3Yjyze77t2a7FDCao6Paw%2D zcc8#u#be@rzBe}@Dlb3yQ`oQlI{};X&LoQg;v!3}Sa&Yg$3?r&aZ?N}edr=R7@S#0 z^!kP~rDBrE(&qb5H%GoP%(`tPz5n{0fYlH__YO>2SC4d4(A1P1H7_T0Ygj)xJKL-E z$+^u!&AQ>9nBDZHjoXu_PhHN3^zZ<@xe01(3&NA31TZk4uNd+3%YVl5)ri$FhYB&= zPq)@;!Z*Rmiw_?@$jkrWE55xxZ*ENkdY#kB5l{)udCrJifcVitVwYJK)4`v=e*M~G zR6dZ<(t1_DqgUtHRkye^=Jar02%3CXUATLd6jup$#dwEZcdu79yJYbVrLLa;=eNVd z$LIT9-ltn*dyFr#A3l88AXq;qG&R@!5cJR-UBa~Y5zR-Ff#h7JYD;*)kd_X(kLW8}9@mROU{nQ+Y&1GiZdB&>1&2z!)99RcfsGUJa2s%3YDT|cO z%AWu%#C`5S{pn%@NqimePz3Mf4SXJ{&-N*KU7r~#d<`QLD?1pUEHw}dCR~&9E~1NJ z)Zz8HYiRjkYHE6Mxsl>i`tf>UeSN*C7omCHYo#X`M5gHMC8AloKJVYZZ~guID+aN< z((PHJ8Qpte0Z-5g4TCu*8!<@uFDMJ1>+~A3R#qT}c7tna+k<_{nN!M$-1^X;9bgN2 zOhU{z)}KV~?{1%R|Fxy)?@nF=(-aMCW2iZwLr(CmtgH$I#^wlE&`?qB`D5n3u_Yb8 zw_O6#r=dd(C2H)dZE#>?o(0$M33}O?{0B6c;SBLFL)P1)9;Sk9BqcpP#9<&@YjG(l zDM`fG`cB>2;`g+trw8b<2Pkg3?x9k|mSCX%)Y~w*MRrs;(!OBG{L((|@M^=(L}2|2Zn^)*Pg);+qb0oV3vG z30D2*dnFG%MuY-?1KVLI4CWAe{|%$^3E||s3G5gPp}UI}pqIj-FThrgzJ9(w-yhJ5 zee>oGF|IKPUVQYuz~#6&Igg?@fUyiD@;I#b#@RGot{)EwnOts86sJ_u^#RQR6?<$X zxj0*Ox&4~fU-h~LTAQ1aDAsLMZ{;H=&+xgw`;1ZG;fXms?(7==g;`^d)VgO6!{vcUG$ zFPoC?MHgTxb&HBpQbFa_o(&I7T|)cocIG7M#);d}CXNN_!1h{`ngfXI?AHQM85#Gz zK5(gBP3rHAWDi&|qT@6Fn$nu^;hNkY0WROj-9fP6*QT2?QAO7%Sh+iE-*d{>q^qTs zFp*ugBDo;v9MNUS8aG=j+~0F1T`iD1r_mI4$NQANTD{6LF8fuAkk1`(8I5Y|xy$2? zsdyTtEOJUp*(6@4-dGkv+>z=9@BA@yjY;1^0$#_zzx~ltJoai+-+#L*Gs*Y4zg|E_ zM%ME@^mn=s2nYa5#zOUP{r2MAXdpN^xFeL5iHT{U(qaO{*vamvB(85MBLt3n*_zg8 zP?BWTban6ceeUZU8fFjYIy*aoLPH7IjDg3senmIfotNzn;8V}Pn5}gHv4Y$C<^ueR zxw$z}guqcJ*+{mW&&AR=D(Tq4s3|tWU~{xaU7PbgAbItVAp~se$sT(j{im9J?u&X; z3xn{O%8qXCuJ<)uw}E4aSe1aYv|uXf1kT;mZzOfQRWo%I`LNN^QAcoUZF*XBdmC8Y z-TwXEz^N)xhrho+Itf2FIr*Py(+VwoZrcwNMDJqEtgVgFr7X}!oKu^c=07mw2a8Vd zU5=#OVM8Dg5|V!1sdw`7@)Jb97L*a2t6dR@LnW1!mP_C==Xt}83V~=aA&2>B&cT80 zwuPKeQTG+Trh|inl9G~x!*Ox{r9E2%@RA*8N8sUwXd%V@s=8EE#IB#`BC9;sbh@si znLg&;-Ci&3eSP!(t1S^8CgyTSdei*P$%ve2dMQ|vJFk|U^Gta%gUjTDDSzEY_gV+V zde@#%F?1bZ5j=dAb36-PC{oBzp44%-JjEcwCnDmvU3|OEHssB39tzGm?t*iBppor@ z?SFA`0kVk)fp!TR9sl7UEd5e-dfadR{+D#;szA~8&bzcg(Pc(xmo$HD7k_>B_4W0+ zJ5GVI1Fdb2WCyh^Pfn^g8%>MK$jFF@_-hs#Tisn;xW-9GCnoO1Bvn^b>`a!X|HIdh zfPBVJUFO}gD^GiHi|!ZCfst$EPfqjSZDyww?9Y3h&mA5fz7V|l@a~=O+t+_H_;-Mb zARx=xPU)_kFl=1kusBI!;I{t*vwBZNqvpaESi7R)@cS16*m!v7zaG7Cbe08%baxIS z705hAd3kw-g-*^Jah3QyA>}$Hj4sZU9_;BuorKZfou))AB1DMv_4Q#;1xLpd9hbQ= z4vn$x)2A;;Zq^fQK&<`?4f)i)$iYP03asyUYb&Yi;^(KNp0>+>0)QpxB$`ZsXcY>W z&gSVA=}b8W&g#%SNYQQ9xTU%UA!v;AjbVHA!fLE>(mP*y40N2eZM8Rto&t9=loo=X2FHx;+%>ddEQP>*T%_5j1mp)>ghvq^|A7cb=a7wTW!{*Jryv*E0(RAN@go@*d8RNV0#t>velN zLF#kEvVAJx_;)Z=@X8c~OWhqRF!=Zq;P2kQkD=EnGwcd)^18m_DfSe+-YLJixgiq^ zd8r^N8CZE;=K{2S52Oo53Xj8AILLIJ$&|7DZU}W2-wVCjW5$2}4TBhj5P21qF%Ul{ zpAicS3xmu6hR-p#2nNW{c+y?J@GRg_SuKFG&dl+qv8Hj7_u>5hjNY)4-zUohEcZ*m3bZUCH(#xR0U*}M^7;fx!ej|Ja%;LlpyZ7+l8((YW4SbH^&3?gX?u(!__u=;~HH>`-(B( zVT%s7#;U6E)AI91_fvh7D_|5v``gonEpZqEJs?x?@lSzdF%>tz^h|KPKd-9TyDK}U z1^Em=xg2Q;Sg}ce;?(RceV8~dqjv0%AB!;!b;W6E`g3(onVFejC9`0^tuKUjhStGJ z$y8}dvb2q459_XQ3ZRaiM9ZUV0E~{0kJs!kqv+HcLYSzdx_|$EhkHu^;2sF{G*OcN z@0NnSl-7>cdQ?&xyQ_(2_e1W`Lcgak0B*s>!z&l+dLeXkpd=^2 zVbqO{=l63Szdw-&A>aYQYfIpCus?R%_V#7kjOV3(CkWaA9NC})!{R_60J#;A0RXT< zV6ae9A2M{dsS%J@wt!uVg%I%oYy~a~UJ!_iiV9x+O$T(!wj{jsB$at?oKQypBV3K$CS0|eq<;GqBt|l9IhU&U5`2(aFg3q!X#MFc@dKvI8SVad;r@2oq8Ks+1f1-7)J2uzStjN< zbWY~iQep%|uT@pzeP?E72C~e&%y5=sZnDqa<=3xIp!)uCv9W(Qc684tABpY6rFTV8 z`J4=i0Xx>+F#%^PW~yz39RI$A1QYQbwSa&FT!X}Q@qth9kB3RSH@%z<>pF#-<2@6czfp!4C(9j+@Lvg1BJ_QCPr7;LVdne~So)@+t?>RcM z@Arw@_@V*Z+WCt9s!pKn7dVj&<>GR=ySdz+D0Xp(0RB8SHs%GyC?{~LoV{N79@sD% z9^=hk{k|ZGv;Js!b+sz)?pJ`rU7v1OTTYb$pK~$CDE6@!FVJXocLFs=Y3W|@^N2e_ zIrV^--e3FNhr&rbc1oM?Z})YWPkx>52;E-}#B*5C($enLtc0YcK|@J+5DZ0{GQX%O zncXa0nbwFnkB=#T5IKZYs0nZ@z+C|(=B1~%1P>%z>t@LlSo7L%rHs$H<7n z?KlN7)SR9Vv&SXl?H%kz+yoB zxBy<|-sk{C3NjHuA6iwGVdCVvJ|)e-AP`7vX=y3sBfc((ENsTTvA{P0=i%kE+56BE z2F%(W0J^=5g*vBgEIEDzyG3MO;MYk>jU-FJ!kC&amYeTRizjqFtj6+G_H0}am)kzJ ze&yicz`?~8yj+b)ifK(2$yFA-zc~h0h)9^Tvxa~rg9!nGlarB2;;>N7HhxhzUu88D zDY*vR9K@DFWm+97QKOntH)VnX1a&*?Al;uVB?ELE+zWsv*v;SXNVc~I<#{pSA@S&F z!r*owD`P`Lgy%N|G~b@h0NJ+Xb}kAT~ep_|(!m=pj3 z60egXkPYYSU;1^l_}A6d0V4x|ucmbSWXD3Ny_@XW)c;+N#R5D)#nAUr?|B$$Ih}e-L@fugC?+GLFrJgx}S6p~>ro zz{!tt!<^?_kGSak#y&rKdUvxS1e_5(AKxC6@CIoxI6fgEC@^r|btSl_rlzcnwd1J* zI2VeAiOEjbH)GR$3vi(UKK3V)t^DF9_q8ZBz^>#K6qZC%tjARw+iuaF_huzv<95kK zPj}2N5y*aze7?zx4|2(RT3c`JYjoC z74T?sXrt}|fyUO5x|%3giidvEr|+U&|XH^G+7sF%H zwY0KoTTDU{nE3Vi(UhMQq5xHH5^CTJa9cS_1K`OoHIV++f(N|-ZE{@tjm&l5eSGw~ z<|PhJ8}6iG;M^Y2iJ_bg?{>No6*>7AI0g~-VC|IsD~knpbHF9;`o#y%LVa#pg9_Ev zVgS5PlHJ%RI{mQVd5oSguN&S*C7911!$zm4<0B(0fVGE*hxhkEkuJstgzg*6MzXS#-OcxAYw2x!J%WNx zjqsU2`~u)K#eFLm(BD#xMx>wFlhrm0Jxt9Y!E0)11$4%k#Jh^Bd~_hHlh@STx_gYC zWC8$x|9a1=5jrENd;IRJQrf6>+#Ci-rE$_0Y{$vT3B(z9OUr@B{Zh>IFm% zrE&5M7^VptB4a1mG|@zKfg;uJ#Ox_E;G5w+Yx_;tMR+r^01g3Xy}Z24&CR8up^2cB zIr2FBxlPRPdI0FL8f{cxUmr*^*1E+Yf{D+oJjpjn@pAt4=>dqq3l(Nc;^GKoS!mEu z`ha+d?8Tz{;OOX|ul5xoo32Nz5)obG*rEJ2# z96rW#r7Egu6YV#^f;z_lUfFcDl}}>zyFJ^=+B#Y?b9=lnf!E1=!?4f*eE%BcYa({jA9@bck8S|o0})h#8e1%s1RWJsQe6DN@z2PYFJDyhRU41{`A8A= zOzPzboIVH!1;waNvd~$(4&tOptG~|y^{Hn7yjlc?8DI!x0YX;a898k$dh2=j2aXTF zK7lAILm6RKISoLbX{BbAq|@n^iiI!G`?r~AXlMpXBXjmaPm|~#{E}(DzvD|t-`klg zujHI8Z|?;d69oKq5IyadVVtkl#tXIhW{-h2Zw#c^2#nkVa6WmL_6!}pQefoU3)24e zRG2H|C)*8(tCsI1{aQXGkuP8GH{XNTg=T=8fJcV7EXL`bvlc*!4B-($UL=B^u0S4O z7mUNM{G`&ZuC7*A3_(m8nVC+v;w1pJ0=dvXqGuzqx&v8f4}@z9iZx)F$wEHf8>>T6 zASBgldjn_d>=X}ee{BeQZ?+RUE?!^}pfUuap_0D-G61M<$GtB90;9;7nDTqUvq^*f zhGSV**BTd(1XiPNus^5NNd-MGfEgM1^BDKy-y2Tm@1wjJJ~{XcX#t2?+_ioSO^HKCZ48 zpt!72?gOL*YE!4YkDwr3V>X@)OTiFfYHQ1UJI)&vnmalo0AcTxb=cniaRxx@ijy&t z^@nkXSyOh+0mtnw>sO41fP;$ylnoddNN0#r1K3JwX(@m^M>a+L7>r;1{rfFs zWDP#5=CkhZZlF$iRaH?~SXkKoMTbzsHd_OD)Cy3OJrL|bwU7x^4Z$ccD{KExwG~uz zKq$Hd<-Ze``Fff6@BL)6^8ppl%NyQwEFd6)Kn?+IMpXJ3uU2c!0PO(53Y41KKn1fF zNCyF-5HZAr1#A%zxQi9;B0NygGg#}=4WK3`Hw2GwA#P8(zcGPMH~TbC@HPK?&;3>L zJ(vWT0{=gJG)gkFOty;!aNnGqocg^wi;Dx`buUcNK(syXN&RdAq6r@;F930YB3=(J zqF-=)+?#TDl8tb8z*GU4!9(>I^iI?<8r*c`O~Qy;;vM>Egb#}QFlazX$eEOp?P$JQ zF{rLd$Fa6K8|moG0Zg&=&p2@x&;li6TZn&km4N!6H{9G8AdVw23{bM^H6b?k>iK)5 zzwQhS44_tS;owky8pVJJMpahdLHQexkAT~D5g4(U=Cj8S$8~Zdj(WXirixhbp99D@ z?vR+YPk*v#Nd*vhl|q}a-;+7(7FmZZy=L#E|3TJU22{0n(ZUNyZb^|wQo01`?yfucdB1!AeCr42pt9NPS$=aG+dXZnZ?Noq92^m4kdH^936AQ98{xjg5^~ zciH#7Pc=0)LnKpw>a&x6EXT>car|GK$zcwS4v(9|veDfbGFWesHipSzTe>Yv6rjyky*t?q6sxi6^Yq zO~u!%KGyeUVD4b!DGrfw!^#%mejO62>3R9r?><0v1XpgwKaJL5fP9Maf5BrvkKoID z+1ZFaMD(32cX~Dn-r$$$=yDsV)mMj#@O`z}P{6LnMeT;E?J7JagIb?A8x!pqQ)`1c z!~esd`FL7Kom&9}`5mB>nV+AZi@GMN-xTN;46p7tH#f6%K_v;G;R&33FE5h@RFP@C zuDhC}V_OT2?*&9QqTps&Ugw{-^-wh%zWBz;$q9)CRU$x14x+!w3fts$D#_f|uUtb7 z0Z;1{miAr8U-`^>c!Ne2HOuJu%n0FBhma1AH(a`f`e?Vx%FD;5vh;CaPYQ%|u^Bctl=q^~Uej$5tKuQF`w=^@kCj2lfGq-fwE*G~3|?UrW$0m;`t6(g`v-3}cu{8$x=G0QdHYda zK~myy9zE8IT49Rb&}Gmh%DtQIeMgI-8x_8w!iR{(|G%j+O?c++58)s8+1YzZe-@oUP=P)Z!18sQBF%B> zd^b6Q=MbLOHNU)>u){oc+k6I-+;uMjw~0zfYN>+(?Zy^P8~>Vv%Rfq%>(_p%?9EW5 zY^^@O*ZL8(^{cbjH}nl|R_B~RmQf<%FXOm=)rZXJYZKf|SQtJxfuAKDiBidXBM(J* zZ0U6B++(9^-tfVMDbyM#4IV|U+u;TzHCvq3;#20yIh_EvXb=i@+EhC(N6f+>t&iPN z%h%qfTLkoIex31Gwf=a1C5cbQCv~h!2d(_FvO(^l-We<#9n_pw<1hZEy~}D|#hAoB zL7{?L;n6EFEADjY`F9b!G24Ctv2_#!XJYe7L-0A)QsHdKe*F-ySSZfTG0<$UP1TO4 z>cFT5RU<-;dchqG<;Q3~7vNui|C`khKYZ}u11o;te<}0QY_ogzTlPuoJFm~y_)^?< zD%=vcL5Hc&&f`M0{|JpzyUjl{t&OIOb_a*_IIL4qhWRs#EUPvLr%c2922n|gE8_RG z@ihpIn!-rBvGt?PzNwaA{}uqgy%Fg`#xd3Yk5AVxR6fiu?+ zxhwn8cPgJ|Lys*KBDeJfdNpY2Ei6{x$J*xZ?R72jF)Wvtb2BZ{_R$l4wRd!+#S3wC zxceg9OrzZwWAF8qW&M|$noQPs07KBUKHTP_dN~eP^RQ#AR{u%*H!O=~?r)NFbtG7v zNCVU;5($ z8S9P_VRqP}z0Uft{jBf=>c*swB)HCW1~chYNAh5BQN^#J6d}e%+$%RKD_*GJ$h$JkQ zHZCp>dU#b_F9LdcdJaz+9Gu`OinZHPV+zH;9K=$T()>q9Mn(n(XlDZo+t&ZV%@4-A zbL?&Y)oaPKzXh=O-gR^t7t{5S4sf%l}$sNgXPqy#F z;_;cL#MC~xdfl+TZ#JEzv`tYgedH~E=Xr8UMuya<{0@(r+FBjjy|1roU;Zf0*|yh- zZ`b>yn!9!d1NZBDEZwmsZ(KdE^L;>NjSriK+qO}K2aO$?y={F}Rje$6EaL!eL7I*; zoJ0aOVFr+Oa4<@9cW?Pc_;1+dt1Bx(#&vg{)b3k9Cn4*_(;^5pnah??RQ{l1Z1Uz@ zUZ|xBb&0p=yTH^k>VT}j-JxkZdS9{_3#zO zz`4>-f$`rZ)E_LKWGWM*GvgWt`7~q|+d^K^jF1{ZbaZs=8As0bSc{}W+ekkKbgI&t zeJIQ*3A~{0xpFb-_=^7H*2Xbg+3MeVk>$Vre@K>v)nf-UVb9MIlw>^iN$c(qYus4q%*jnnA=?>m+L*`NN|P1l~8z9y*8 z*nRgyDe<;pk1n{!rnF3V=a#_&*&X2Jrx(P%CtO`4)XXr+?9m7S*v6dN(N(< zg9g!FX)=8S(J`MMc^GyUycdE|{8;sI9c8=<)&5G>JU5S>$H?x-*aI7fUqj^6JA;Bh z#XMP1^ZFR_@VB?rw%nQL^C@>KIOr1C#*Zh~0|iYE-iHlkDrmNRmUS+(XxBCCF>WD} z_DJ^PwoQtRYNJ{#^$V9huC(?&b)u5{0&s}?N7++gXwymi=H97)c z+vQ+Q=Whag#UoyXRe+=l8;EX(F>4-;3&2^_q&WINnV zJUc>Xa7JJJipX*0walzGfIIuf#P_>>is!8L_6C`L|9 zI@flDC$NVH!*hALsq)ZcUFVl_b~WEDsZwm|@4=}zSM(aME?nQIsVK;JaQJg2t+1Cj zOKo00J#e#nwC6@}YPa^8^2WQ$!YTs1yS1OZs|SY%@B0NmxkxGmm;wdtprEZHdAtMd zSB&S3R*c+5uD;oN-^3!^_z68EUtE+c8g6?A4-uplX#IHCi8-`(!+ieG56-5`?po5F zd*0bi)AL1lwOsjqGf&}BX?~kGT-%v$VnVuugv@NGfX>j^5{L7iMo>e^Nq=-_UC>p- zs{O)n76^70`KBVXS8kC=6psz(S<78)W#Yk+*U_> zJFubNg;={jsO;5^BB*-3AzO2IicEz;!k>vmQ^HVlr(Tm#YvNCpsIn^gCEGi^23@x$ z(MpD3puH`uHH_4{p(9ixrXnpJdKBu!VbS=;7x)_k324Yy%eCpd9$Pv3 zNZa2qY35*6nQc3FWJt=KA@k$ApZ0iD7k`pOiA10LF4rC03@CZ|;5N3uC7)2@Ex!S| z*NLsy5JfyZEh8D?2w{gu2C)?ddl}RSQ>3%!FHHSBeow^2GcNBrADSVI-7EW+MF&F#yUV)$WF!Dr##%a zb?8{(;jzsUzG^uulXL;*w#_{JYO0GS#^~N=d-jTb<$?0kZnkKMal*n(9=V_En0&w8 zyIFna;RT=mG-*Axo~_Tag*Ud6Sxs`XYeI05txq@iiG+|5HrI#52$gRzCXyyINugbt zSZccQ1NBcwK1auw8-=@bVef}!CpZ4$cvtJ(no;sz)}@-a`#mZy{NWCAE~dHP#U6Vw zv+QD9@PhC#xR+P>Xmj*vL0xCN;u;C%xCrIG>|X2$D)7)pw(ouVwaEBGs$`_l3^$-1 zr=V!7o)e-NMa%2!XDj-e2T}5ddHxD4z01$0yr{MoM$pZ26aCh$$j{IaO-%$@dED5} z$G|I7%J9_WpBTKJ@yS~MZ@%}q+r$Y?Uc&m>p@gVllUNhu@5R8sG8cS>f1wqE^(HZb zeAvE4eyE@nM|zjy%-!}`c_=+b(Nk%mBh?y1CR(4S5WjCww-I$&o?Xp8Ln}1Ri z%G_>L_%1)jeNNS7)|tnPU>mZg1?^A72CJ-?NFDWd_lOXXEu5br(}eeus5A}qhJze0ZRftF zICAgR{q050q*`I&&({#f=VULIFrO0qL;FI0Im@)RlB&^SG+Kfn93|r2W#~4t=tzSA z+fCzZL>Iz%kfU5xGeR@K;qS}iI)}lu-qKkOHNQTBWSUGSc{D^Hd`o|uNp0eqpo!Y6 zJlnxpOlbO0h>Lu=Q-ApMW0C{K>twtyaf;u_RIf2&-Ku-B*u4GOq4F1wexucHQ z&Tj1!6HiLgrkneGgXuEJbYuB7o5t2o_Yv3w=9@EQ@1QNDv{> zj~}dB$zeP0wo0lE@@MqFho`VG^*2ljd~i^dS760`uj}pi`lFUovcAP3ce?%WWkDyV zb5l&ReMp*3)5Ysn`Zq{h=&1PQ$*UO~tiahTg)i3wZ?8rl7Zaql`klAE_ZEtek>1Sv z^ERvX(nMa{c`>DobqFaMsZ5>l_*;#PiVJ$)(RfVgPIU|28iw2P!Z2K#CyW zO8hPbL%L4w23#TSaMnD2c|B_{ zxQa~c&C=UwFD$H(X;F>UnR-<+7~XnMm-08KMzc7S0UdF16&6AEwqlv*XT!eq7v=E7 zdw8>Jbw)shBv)VHn5T*r5|Q7=?mPq=xuZ>1fD6S8u3{qIF@^kszKWF1FI_elMvtiA zJv4o%vwQz;Ty(apK0^0`*Ay-~5G0AwhIPwPV#5K&Y0frqL{Z^XGo8;?8U+;q_+ zm@Y-9%eGJQh?($cy57v_>zXCkEMM}5dL#ODt4BF&J=R5)@9jdfQ>`Q1=Wp~Yr4k-Y zlvohmH}Wm0y@rYc@A+9LI!vyLQ+6rfIHlMI``3s5YV|-!?!=N;kG9%i+ z3d^|vO70^63$HM8_5I0nGs4-?*L|*3oy}i`WhfneA3wf!K@WhZo%SeAm{K{`0v22T~w_Mi{ zZ@kN<;YQ!{Tr)HkiEVa_KwjH!Q487yNsQhH!9T z-w{Mqb~tblebv3Y0}sl3-VhrsPA{u#s-_Hu+>d3i^rn4PMO;nKK{=6{E{y-=*OyZJ zR*aiSnmbX<^Yfa=cg#Fuo(INYiqLVeyz9Z9sOIba-uY4EB6$<;@OG=@#%xrCfOKvH zTPlIH$JORhZ3@M`I*mJ<%G*wTQR+vh{4BWAs{`B1CJ$#%2Q3NMoITX?SFXpN#owts zeBtFVMizY$Z*)ZismTk+YEGw3=+dMqiA)?TymBr9cr76>B&v2*>;ogut!ZII)VxeO z-fyOAro*4m+?;tURxC~R$o*Z$q6?9k5Nn;Jsgq1s&}+1527fD==YQ7^veQCiT5cnS z=!fDbf80V3=VCkQUMICx5X*0YH??$@$`-Qq;n_!6jMIOv0cM<=Vq@E-y)sO5gnjOS zTx2LD#<3gtgjmBWHI;5TdlE$AB`oDJn+U=>>Azq9;EXxVz~Z>~arR23l#2A^IHA(xJ->(S1h(!Q!#tl8ywtFfkxIm zwP7`?0r`+oW&-^#0F<|Uunpw7k5Cua@?mj z9Oab4`CdET7&;rF5QEL$J+clPLbq4B(4i~9ovtw&7c-`N{Kivj4VQxH++eW1{@-T= ze9DwP*^HyDBN(<|!R%I#Ahmdmo}s$T=2PEKc!c2ZJE_ zcOB=-%8d}-6iUo1h_UIMOl3Rc^t$F!iC_`7H(`=9i*xyaXj!^%>n>TqQ|r6|_wJct zZMDstbdUdu8%LY1f4q55ai`~@yrO!=#IUtrt*x=L`HQ>BlG4G%WRc5%A5N-RQ%Ima zw8>IzLzUc3+eYJy(n83qo!_czP6~Oakv=mN;i^exW*%RkE}~J*wmMNeOy;(ned3&vwXG0TNLcpI{cLCB|P&<9}c36m`!@Lf6(IE)hfKi zox|EfL(Z%@y)r_Yv9zlf#g1#^8gqTC)vi^Gh#*a@$<=J+c*rECFOKvL&2C$*k|@lE zK!uP)Oz$~-jujSVS!gN4ZX-hxM?$Nk*t#p!T=;^|^^I80JjjSJ26ox#kGglAO7LPP zggqPTZz2uc>|gV&3S_bOHI8(Iy{)&Va#JabO(uhL6@qA9Xl_gquA?j6t6s=)KNcH% zN?wD89C0~uGu$-WR9##L%ujk--zPQHNIOFV9Kd(b=vy3xo|H5>wqfiu z9OfL8h(+zWzbw}f{Spl3@xZVAj7@F^VrbqvTS;rFIoihSQp=C;b~>}*AYWBpVg|3o zLmi->`8#88FvvhKV3x9!QTQ~d@a&bJ0J25s*{+mFcwO1sZ@Tfn*@4^8xyF+*Z)lZ9 zcg@q!54BNgenzufB-y;p9whS44)5tM0kbQjvadeJG4w5eWVwPTd%=~g*pIWC z+|y4kU7h@5Urt4u139wQrM!1E^Iet2pKU*kDMSOz1SSNN(r`>Jx^fVNrTmx3%o|ut zA|i}5E|&Mj#Z~ENexLskU>gl`aai8>c-@qphUeaPWPPjEp;_wn?NKx=v!@LUOJ5tJ z^l}XudyPIdC;h`hwv4dV#(FVWCqz8a5&dJup=rMxgN!pj>x8X}j)K`N52`n_*0-Jf z224Wa!1I}BKEcmaa8rc<579;!(rNnp`Kgoghd0s$lnp{OtXl3@TkmGsFOc@UC5Ph1 zQFagOiR|EV+HC~!!n=FN)xa^GiOYRdkTN5*8ND!fbhEZ#;wqLQsFAu}4tH z&uID-11TKt*Qxm zYr=X+!zh9TTGG%(??#{58X6uW4Z2+K@6UM5FNI}1S|QDfZo)U3b(lEo@N2pZT=FO89#xR{KJCn)MMxW&83iX|Rb^LQJCGzL8B+;O=2F#C>j zROzFj1Oj2LvorQrf{QOWWYm9aR-twr@#CrSBEM@5!5&64Kdv0R+vi)|+WO?zlX$1| zioQoFW9hB}Iz;gs&>@kwT-@3&LHnvd>C>RTu}*||A4F&WvFkibs-SUJGGbSw_q-L0 zhuoy}AQhyz8sWr!B|FlPCh`bLxRHEqOz1NIe^un|Ch?7k)Yb%@X)`ijq~VgT$;?Jw z%)*mVj|%_#)wBMy0l%$lHo|1PXx=5BQknLL`gDe)5K!mN{}g4J>`tk zg$L`IkKfiCLl-1B)W>XQqz$q{&xMNv70y>)XLJ2$9XHUBtbrmNnG9cz?Q!4F<~P;I zj|_Ft5Z>IN>&{`$$WqN0mE+&dTkbK>I3p=%*UPOel_l-&90z3o3rWkPwwbdvxV90v zr>0kbZK(G-7>>K^sxCgm40J``SKpJQ;(xp7mnGWSw{T-#mZ}}HHnsx zRjBkG92-OiqaT^iN0(VaH)H$^TX_BfkEi{!frqHF{(Dp@rm@DwD$nT^(I?O5-UzPM zel=jZEKq2|97RtiJFE-+U+vDC`<+#>4YTkD|Ch49-D58(B>ujKLgGgCv`}nc63cW( z>(Qq&eQ51p;AB*-Dl@&~G8*iI_BCtPNKvdqyl42~L_lRjAXK7rQ@;@J` z%-IXjG0_C#HJrTqlTlFPf?tYt=7@ud?C|7<&nn|(wX{*Y9QyANZs+(2CW|Qc*l*zm zO-1T>npAk1@wGNy+BZNS6|l{G%iRC&&3wEAcj3WrtPE6+e^N^PTY0)J78?`w;vVkL znB6*?%jBxxLQ~JFJ5n*$bQ`Bjwz0?bKKr8R+;OX)+tkaYv_{afr$x!jR>AC!KHu*x zyva(2)sv{w{bKrTC2Y7y^S93L$0GQ`SwPV+6qob%*c59+h%*4?7GJ>A1@Eq;+X59a@biD zon6=^B#!K$#ew_0T)^>QxBDf}@f#U#^uyb_e57P6BB{HGe(ScE+39m#1I{9N^_)`BHyOMEY`el;+ z3lzSkGJGo?ROVz)6YCAYql{ESbt_4dC^e}~%7Mwpjn+ZUE5R@BRLH1giL4&B*S zGfe(FE&r~1HesDf*L|eZelG11jKV4}BdrDRALxDy#e- z;eHji1_s6FQ{?N=P&&+$(M3Hy>r8iP zD}XSA<_(0ggz@_QAuZ(f-qQ0flPsQ~&`?Zo;GZ(?G1}VNBA}M5l-wDJ2fjF(cR@jc zAQ|ShYiJGmI@Q3%;f!Y;a0m(xmSNSZe4P@~Rmfh0JI8|i8T1z)x=d|>HU+8#!K9iO zv^0LD9WXzh0;MuApZo#>T;IM8w1ox;ezk2PFvvlL0A1tC0WOW-fOqSxHIT``*&G-d zKy`ZQDuR9WE{wB)@zf85*|@muy~C{MGBPs2eS%3E{LS2`Cy-^4u%x8?8TWQ(R+u}w zySvX#Ty*jLz3L4l3%e@q8>TsGC=wfxS`h>d4Gr7lH4xrz-n+JC?(OY?U;~EHU4Q@9 z=jN`VNRvQSO}DNG(kT3YU0Pbx-@jfksRp8`jJ$lU$Eow11rqACOrUxsBqRVe8w|uz z^bz26!!MJTk$J?;{n9o0oZImP7+uK)E5mpQcoERoIRP6G*!aMfI|N#8C>f8!vaRYQ z%dKJS{0K7NeBq+SBF?{zApj(KClohsX}X8;Mt>? z`yhUJ_^<#5{lF{`yXc_+K?A&_dJw0=KsGt~FG##=I6)~PM|$Jz z+VvDRlyr={3#!70IzPmJU}$M-D!{6bj|*^es&v=V0l&}G6f{ahgM(-daGM|mZ7?la z=9+A15cT!K6O)br4$e@oHr#k+<(CO;Qg(gsjEs!;?_1#$3IsrJl3xE$@A5=VLE)3% zv=1%}tYPDJ{{Fo-dL*WyF^U@A!_n=E<5~q`9WKVxZakvfw}Enh3C6n~yInkQ*2mKO zo^@=&PR-S;r$FY;MwdaVu??F6^b(yvfBqaER=bG*?ICGkU;r8epa1~v5zz;VGVm<3 z?5ls+%}T1Pj{ubc3-R>yoI8X8!OHJ^XP7tvFB|kgX#)KMn>9~eeI8_|r<=gfX=rEw z%a+dH4p=cKQ&RH=2ui`h!7%Z6u(yX}F8LCUDqJY(Qb0h!&Suc< zfZxBL?abZPH(0W!B|<&o)1#vYn#4_9JUk%T0|wn=AV1Z@y8>G8xlaF9A;>Yy%jxAJ z!b-eo#H0g&EjjVSDd+q5fVvvk3o!Fvjt5$42hgorTg_lL4h%e%lhvJ_9pFg1!Rs-r z<*TTu^!_CU#g?=@Lnmx3#&f6Z)ZbE4-l@EHl`qK3x+O9b(OClOjMu#P__Y4$>gvMQ z0SYWZa+oqbd`y|U@ZX-Eg0ElofshG2TDT@hiD8AX{=0zP2#Wx##X=;PB6J4Y?Td>G zpzYkdxd^*UrBt9cXlHj9o>Wvq03;7nFmlaY{Q z@wJZ400S6^F)U0>9T1<|+o_pRLvx-f7hl2kZ>7=E(O@NTkN35=-$no)<;Lo2=gez> zgnHlGR=iYExqatOqKKC;E9=L5mNhjsK)@9f6H}RRfQRYq?ELcOOJ`?p#$*Ic8DOjG zXlsvHm8YhrmX+=IraVz24c{Nm#4@s1F~(^aUJ!LZDx}rLKzbS&_+a%tCr9oZ?%o)y?x-rR9nFATSw%(gfL(Y8NYAxiy!ZpG zhn*(`QiT;23~~`l`4r}~@H|msKrm=%%j*X6CQ1ag^Cm`FLQJd?yc`ra)J~6&J%9}d zyw2x_B0%L04h*FBmAiKJsylEu?%kB?dV9wK_*oFm!Z!%OW(s;3HQ~xwyG?tL**HB0!@Dw1@#Mi@?4Qunj>V z3KtL@T`OcQJ1v& zu(&Cl71*G=sQ>JAbapP=8W=2L?A*l=5U7UG3+VDxf`wCo~B3hMe8IK=5`Wg4B6ohz$SqmH%JRXb`T=*F+ zyguS_*lhT)iQ&-EHt8D~JqF${oY`03)ZW_L+rNBC6w_Jzc9$MGIzILf2CB8VZBXx6C^0&3uLaa|S}lr<|Oe z(9lqb3jxKjHcz*w#}~QiFaQ)E&VRaxf2g%r0bys1QDu7{*c~-tORW)c=7P?6cu|w z9BcW@VbfhW*|P*b`=|KF4^7(ZVrGzc!3V{_08DPEOc_Yb$;lHp&Kw3ZQ&VRlAT}y+ z^aDk<+G#a9gnJM;|JGJkAeE57)3=_JR0Kou>g76Tk>2CMCT|%ffC8r zx~k5_^$!j@K|cWW+!%)qW;!7IhZ(H5jg)_e#1Q%5iV(0EA)L-9d>w1wOZ@`XAoLmq z1-7E1qCAH=1$lY3pd0}%aQhBu&1oj}7*VUyWdz^w9xy& z)!p32~J}NzMrw!1QY3YxF!Lg$T(`byEC@Tf69@OLLY%ds0s3|nuqWd#9G2v-6L8>D7ZX_KxI$!N zRQUt7EN3De{OzcdwxLH%hHmhm1x(yz_)sJ5+Gt5l_pmjoKVCO$zyArRc=XTJ$d;GR z7l-5fx3zw#h9SoT@ky0{UWtDdSaNBq^{;2dx|(k{LC!WYHV!+B%wYcvRe`J+6Wj&# zOOwu&)pDpKQ-l#NoZio$Y3Tz?5Jvq}dYC+dSiqSD>%ptt8O4c2fj82*DNx;=g#dke ze)pIL@5QUnKv~>>9O0LkNRCDK<;xdGE$$yuaG-$J3JYFka0d74%0#Fls+AU7^t(YwS%#)id>~bm1XyU=cDqwZrM6ls5Wt zvV1sqrL{h32Gykfjfs|S(Sl7VGQR>rKND}++MrpwZnj>%C+eHeFRL+e ziK&V|SI&3h6Mbf={JJokn3{TDh)7g(8Md$68I3(pz{)jI5)n_qK*mfSwVhqaNZZ@g z%iq7#6VPPk=B6DMQb9kLX%#8bG%=CTbh~sVT7aAc)8(hnHJVjQAy?X+GbW*F23EG~ zNWWpuUK_LdiSmtUxXb;fNN(!igox@iQh<><#pQ?FM{LsV0|m`Hfb zu{YWk&NvbY5)zWCYSbm-aiT)YA?AZa^D{h&+)?dL+T*Rv=c$=o9G_m?MtUQ~)5Z=b zk_R6B>(}_oUi5N zt0#i2EVy%axZt$rkJcA{{qxFcc*bB#m;BAZ#1;u%!xw1#y8QoL+3!QMq%e$CoUJ$L zQ1|&39aAV#eh=@Y)V2TBT<nr9TDApQ9oz%ZZ+Y3_~7#oYbrbS-hKoIHp zXf}Go7`IZMz~YQ8-1GEb;FM=_c~Q~xv>YF4vA%FWV|FgCsB^bEm8bVa+zy^enz>}> zN!#)$FDD5ND;EJvnlU1J`@vO=FSIMQ{wcsE9q`g~mx;m*bRAALet0Kai>nsX2^^f7 zbg`_JZ8V)n-#%&SOu9h(ab>W*sxw+9Ch_i%2~oqluLMa`p|O&27Zz*AX0w|{0w0!D zU51mk=TFlNY6oH~u+j-{dMC!;PVb%;1K$EjON4}k&3`%A+3`2isQMFUAhkiA1G>Ex z&?IK!feRc|vM!L(Eq-T+1A7~u8wl*=UcQVzs@_Y8bcFhQH~5M&Hy2kpBW{)=pMbza zjpC-^;o;FyjX%dRs5)oh%O~2td8ww>kXw3J46pD9g;w=XO(~F$aMTUJ;-yEHl(x+Fh*386wYO)f{k?mKvZtV^ zDBOY*hFEtH8d&n7g?}sqnanygTsmVOsZ*Yh zRXr_;dmlkDDkO9c4krW8#^xx5&$;Uac~oR#&|G6MEkr$`&_O9~V;@VcKM#O%0E$S6 zcN}zd{+fJ-O&_)srFpR5Iy)V1=x}IVHUFgtrK{^|l$tIyp(tGe03c~oXTgxLI=i~k zr3jlIR!e+;jGvK@O(m@M?fUldO=4UOkM0kkA>6aF&dtmOcQo`nO3-)^jEcJF-?(Lw zKqqcYqiFXp(k_IQD=3KTHE+VL`;wW=k5_h}00-!=qeJ$@5iuT}H*0^cHFnRBgZ|pm zC@-f+9kShyzO;)Ilr~S|rkeKF+|p8RfqX5XUXHbGElr)6Q-yeKZ4FAJ{T;`*AT+AH z{07C3bTIjKe=(agON)n}$5$9u1(o*Q?lI}JLzdZ?*|DC_Cz8TP(QgZ>AjUK)qt=4J$`X)wyOPFh-8 zUY_pZ~|CW20cpW8omEO9;qh&46 zwMh;-5{OgDN&NDGh~IB|pFU01Iy{OP-GwBq(7!30wYV6LY>m$gb2?eDxXnoNH7!yH z`IqYKxm=kNW2Gn_Gp~R^J+$#tQ&SeUk6(aQ3kZtAGe_EWIETWJ%D^A$ShNOkLV#{) zb$varD(4-n7;Iz!5QN;1dPhdS_z|Soiu;N(C<2+9epC`ic^A;GDaTV7o?z| zxOcM^yt{t=N+mPA0f-N+sZ@t=TQ!`-Cl`&=b90Jg{|3Eg%p{~4Zd>?AM?C*f_D77*x&r8_w${6eGQ+W=n2^xzI?&-c^eSd;5bw4Qj z9_i@}uc0E(P(s=E9e|I51s0;t{{DVIy2nu0kJD3O=^Jd0=HT23PvV+4#3Gm>)B)@> zNtkA9ZG8@4DfpALT@Ti(utE8gM2#rI`hvqG+B5*vH*j#=_6OAl;}y+ez5dP4f-jO8 z^4ihy430+>MNx5D+8mVffacXZtr8*d*z9@l-F!a_%QZKb*8R-#U4OrF|HD}bt#g}$ z!^2*)A9)4_ndl$I0d}cZ`TAWhwa5ZA{~(+>9l;NTl12E<#=U*Fh$!e`J>VbfdYwBt zd2~W}E-onn7frY}rfTK#z;>WAGLjI^S^Ofl{DTiFDq31{`?nr2GgpIU62PbX`LVGi zfgHc%c@#g^YQjSY0Wu!J3kaNFhhX5KQH=8o#LW|vlMwRf01*Ow#HOtW8l|^_Q&8i} ztooF48VmM-|0m?jF2HT&<+J>x@BN&Zs8Pe?_YD)DyM?$m{fCF9ybI|Nr!y{X#{~0G zYis#t(h)1}_$~@Qdt)Bs3z$eOY`X|jga*+AcDo9-MSnRDCuiEQ-G6%mi*7GHiS+s{ zxb^SZd{JNFkMX^Z5)F-wjVIe%S^@-Z^ZzB%f`=npjN8fpW1u`cbRvKPf$p9H5eFl` zfBiXgXtsSsio!RFqnqv4Fq!dDsQZclm7ktcYqnbe~2+}il2KI~D`?c;Ks=*0 zodKM55p`wE`^si_0@$exxW|hZfPSMjbak=9KxNrQ-pI%ZdcglsS7&o`Mh!lTfd}@3 zV3W~kTEfc0G7$APjR=~`*=-jya6Xvg^K~}M4r46PzKVG?$Ob=fURMF6imOEZUoHihs(Ah z0%oj1At9edm@O*6zqQ|b2KtYkzjo)~c>#XByx_$GPB7i$<64mi$qR=lYEAs&JAvkkT6i<>#rM`_7 za@nHFAT)=DP%~e%Og2T>P@VV$qwF>Y|2xqz7BEZ0NZkf;DSwVFdex$$r5j%U~?WaE!rnAtfyUZ;t zzKI3VNXrlR^`V--`FVHN*52Y~5U;&kPQhv$9%)?$g-oD$Y~Aba8{P0&rB|wNbVx zP-BAYsO+wNOY5zEQ)vI8RunT}KwjJ1Yu2D#x`M&9m~Pe=u&uQ3k<0y=77CJRtKTQ? z`^~+8DGbbQR8&CFnhtm~%Ik)N!*5^@@{Pn;#sH;uVPRpV*QKYkb2&7NOkSZV+X{q^h$68rpsyyuXdeZ59Kcw~ z9c)=356oSv$jg5$E93J0S?PYvar3TJ7C5be6$#AGv&>DZ=VoTYland$+$l3|Ctyuj zB*^|+%-+ybV9fR*rxlz|HzR1US^76f?}bm&=m;zO5 zv&D3RdDtAx4#K~Dd0_YWtq%?XeKmbGk=`W?ohItN{=!KiqX7oGm6es583=99L3;Wm zY2E8=TMB^}?0a1|Cqcp9-rZdX!OT)p@KR$j6{xij=zhJWp zKYWOX^OREZ4hqm~KYrZ=pf4UdKPRM{xw$#8;R#`(8d!xr=y`Drlg+adfB$B%yo1Ab zzpe{qBIN~6&M;BrGm#$X#!DT33?!qhXV}nrK0k=zZwLETRS@u#2=|r zvq6dc;M~MQiheuVAxwP-cME2dT=5K5;pSX0cbQveH8nGnx)_0}^2fzoHEIWD?5jRd zdUjqiaW{{pT9B#dO{N z%5;nR{29J`s9xyJdFwqc35}5v?A3TN|94^m0f1X-f=xV(LO>o5c5H7qCirkUsnX_G zR#zddfS2J87)g$fjzT`|f!gLk3W`@SUgUAzdqqi^48#POi9=!kM$zV$SjStqZ&(<3 zohoo{1?T{5;}b&U0QQWU%HW9PLrvOXLZOnIk^#pW)`-l&05XjRS zfp`&p4^Yn^oc~1Et5$Rjjf^U+$A|j+zcrcNz{QpRm-uE1f)W@jmPHG${dr*z+04iY zTz-K36~gF12R1Ml@cirH=t0#Dreom7ItnN7=g%ikH6-F*YG}02+k;}*0~}hxU;wO& zs;a6IdF{lKdJGMgj8qtLZ+m%N!dp3pNDKka{n;~IzE`m9)nL)e&kyrTxMov1ukU|_ zwFclHsx3%=Kz{@WLV&Ib5g627bTAj7d`V{-{ejvNDMS&Om^caH3c%EkdHch)(WeU` zl<;};IXSD~!3S+FYWxX{njl4#LZJe4TWHtdrJ;6!cnC}r1ZB1YLklo@A!1SA0{en? zVk2e-hK$YvrOE)<CqgyX{_*8;YSMxS~Q-Q8tKPP!V^mGx73PmX>nD zR;8nB?&;})X0E44F-l#h;QnoFsD%E&?s)M64L;YqVQh?_jZGmXF@#)z8$K=&rmD-! z%kULO0>ns|+g;cK0Jwt<$Ds5^_=z?+KfpX3eklXPB7_U$d2qs5{2lu=`%A*Q3~W{J zAJ|tWp}B421K!3l$_=3>Sp=8?C?3ooMUn6ukRjJl-T=R0aT*&9+rp?JfetP(v5DLpF_}fdTONf-6!9-N4+O4{UM3wAR(drEt{Bzpim$U~dv^8zDWQ zO@s3b%7qDj6-?wMHjViWo31PW$;-`^CTEa?$}dEK3hKGdzZn)M@cLlzpY;X3>?5>8 zk&%BQ=;XcwO=)GNL!W(Me;r%R2ov)?%of;Us$uJY{rVM>RTeX|JwMo0qpTI7T8G#H zG$klzU&|8n^ah-!{6mXwJ znnesvO(mHGYMq1CoH;P=i9L)zIB*WC{9CWcbP5@#S!82r=?KgY1>p2UxyjFO zVZ-#wzo*I7)fIdrKSBMa_7S|c!JZGUe(?oyplg65`4#w$2s;$&5G8Z9tCaf^!9;2X z&J(IH0!tuxT!)Z=-za`~=Xd8|V^Ry!2e7EXY8CP%9ef%IG~K1vrof1UeXFq9HJO%DT7%P7lfMJ6QWhU`q|& zx9@&iv;%KMF=-8GJkUg(+%@+F2oUH{(Pv2mvP_bx(FI=s83G~~t%0|d!F%nbI@;-n+hKkiH#9zgCBV6cYlUfRuTj+W zIU5rbOug>PFYN48i6swG1^A=EuD69OYI8)WUvsfuF95agL?o1KmZzbBZDxB zD&Df?rSgDU;ClC+sA$0e!D=Bv9|`14(OXgM4`T`;6XeFr5N*b|50|3cB3 zy5hiwnTCTt)-^pNBbnEZhLtrBK49giey(O26N*NLwrdP5Cz`x}@H=5|K^FP}RS6$o zEN%t3`zz%4wT6&9WgXleeuY11^qvJUv8AVI8d_RzaHB3RHIUOR3aBB%1KSHO$We>` zbC8OUiIIvEog+aD*RzK#9QT09LPbwc(evqvqHJwWMg|Yitx!&I$-+9Uk4WwI;v=Ei zMcJOg$6Y{?c>%Zxyz2ixhbQJLHY3!oz_Xu?DeN>sxQkcg|-% z^O-0qcSw~I&~XVduPrm&AR{~Rf<3NO_Q{ZwhrgJ}p8JD2UMO+W5Q$~Q)FlHP;Z>R& z^1s2r&l;_}0`N1H+g7)_u_3eQ6|rodldwZ}qG+0?5&z>o2)ndzz=kbu1@XWBB<)CoOU zhSr3$z~y;fEcfOw4qc$$yE+OWx55RU3hTwuxd>O+jkgiMZ5I-}&^6!Scv@IlSfur( zoA_y7kH%YE%$2?Itr-Y)J4YND4?+%#lPOqZ2Z>Q&eK3GlA8V61Asu9G{RR>fnYl1+ z5Lb{FY4mh>#@he(pjrjZ`3;=5eC+4On%p2fgi}-ZYM4!m(kNsW3pE9dLMR zyxV1E%-N+ET_3w>lmTsRfG}njxR8yiiO}?<6-X=gS~lD*Nv(ybE;cI(JCUq-kC^&t)O@zY>&k8wp2HwTH7OKy0 zGkZShFV|+~M1VTk^l0I+2&Zg@#MRJ|Xt~1|J>|3Hj&VLgoBD!iM1i)S2uqvfp>kCFn0>}X+!jv*{6t*^QL z^O@L?EFcL_>$Ox@kDC_j1u*LmUai?H1#nFI<9qO4rgfdS>!i3-aOdo$c>739tzBG! zzwtw?7j&U|f7?XQ9t%8treU*SJvsRgSyY(8%;3eAI{4_QE>7UILbD?e4y=n8szlMr5L3fqHS1k(;Nm+DtH(1qLF6bDlP z98u8aZB&oesWjEJKLVB(i3iuki=4gVT;R?B@yx}R(}XZ%6!8gkNMLVU{VGMn;fE+E zzb_+E7#7p?u-*NpCQjCdpwpM7J77?^9I|OTr`ugNgm$SZI zhG80fVLwiF*=9Ld)07n*w2DW*?*-S%!)0iA`J!FqWqYy4vrn9@KrYGyT@dQe6(@9y212>nL<;hLh| zR~)hOcYKYFGr!f^E?Pc@g|Hazmf(qO9o`m~W8RB4U44E9z*4%OcDMVis4ABy%UQ7T zPIO6Qvnu|uk($0#YiPJr1YT;?26J_|G-!OhU$T<`iRICV-^OL%t9S%99vSM0KlMe& zM!?wR>)c!9-FE!(<)4KDPVpuXs?kz-yjv zp?n|=Dvtm-n47;Yo{WD63HkJD^Xt;A;Zd*`Wo@sdbAYg#*M||)n(rW*)Ef)u4_u4Z z!r07>VFkj}mw9{yF#`Kbl_ zpTGQny%p5c|NT4-cJKpGP}|1U2(y3Qd6rf)FL8Q&`~s>#;?;8Kzcn})*_awo<}Mo48K@(xVdatYzz zw}|ao1`Y)d2Dxiq`xk!5x+cx98r`?d#wUOv9N3wI+-t zBrMJ%e0UenNnJSS3Y-mL#lj)M339dsV-L=78- z`xm}vG;e+#)^!?TP$GHx;pb|wRCnlp_lFe_hLm~{TKqu;1=G~LP)!q3hi*rJ8lZHm z;V?RYBL|M1I$>yi1OUtfFhbBt<^cpZ(DmpjmsF9+d24@va%}ASX1+TRmmk3sDDAE} z&RfAUA6cy1%5-SPURRPbroW*6PxuFDngW^Mw^hS>a zRLLKP7tS>fz6C09+E}@{w*jO%|1|r;%E}7l51m@BflK6N>Y2&}^9$Q~dv7ls!Wcl) zPh@u>n8L%s2?dNH^4QMz}XIj5<Hy$1ME)k7{aBu#^@ZW;wtK>-yO0wBp} z@YcO15>nP7K~|}!+OjTf`S7X_lmtOzN>tyT)3dXJ)<|h+`$GwnaT(%aD7Oowuw?}O z*l$oi0Pp_1Q0fRMJeDm^u>cckj!Q{H`uz*Q*&@QiutW27Ho@gWS&znBwCume3I{9-cD@aejS&`#U-cUuzPju8vY(hpYxjM6dcwLbe1%tI0fx zQ;5PcZ1cDuUz12i$~a~-GndF9F2rlTn#E48}}dxf!e$?z`XT~ zq{Re5qY8#{h3Dn=F>x;ehT`lDq{n~@$A*p)G*(qnL8)Xzw(0@}pCFfYsK}+@9kT6Y zo$47=>z*=#M(k)j!gLK2Xif)yFl_pagn|N^)+%dDOVqzYrx*A?i)M!?U`c#wKus0U zSR$ZHkJnF3$c8~$49{2~GWzCwxK%h4#aFfBUVIY0z!vy(07inCjLf9~-7AJE9?%&& zK+m9Z1NOg7t4l(e)FUW1gajb0%wWLByP@igoY(BWjb~!{9VO7WM+MlBwgtZmc>;I# z`=luX+C@9hQ~!dL#wp0!YhGL(W(d0sAEd>rsLxau=q%+Xn?Nkt+IM1^JQ4^rA)F+V|08wReSO`cvgd>l?DrpP4$tM-wxF|SH3 zVVwI}FT!`zQtEX_3?U{F`V9AP>M+z-+6f!j48^Y3BCf zszig3dF+E!FQ^H>cQv)Oh28S&D_}kAHS;qwnI@9_c8}(v5K=|kClq=c83BtMqTT+%ZE3 z7;)H4ta0fO5(@oFJ{*GzQcvMqjdTroIjZiiR1h^ls|@vKVUOza@>K}=L3|P5R=9vz zVg_-KzJ98eqr1C1gnNeP!OMetf}h|KB5UJZYDGSckdTnDurT-$9sz!b(btfRW?7%8 z_c8q&wQ5x10>YfahSNG{BiixDJ3*NSQFE*nxh1+k%iJzJaAPT;|`zX*B-Us~&e3Fvz zFALCFOt(mSPQ}&7!$T6p@=@K8HYcbN3kXIbcj=hCtO5wJl@$%O4XQ;BoEM4l@l-9V zbq3It6P81!*Lma`E)PF{_5q-BHtimU8XEDB|6IQ&k1FC&gxfzC zN(4nV)M&~Yk`jJkDNt>3xF1>dL3bUPWdN3jz6Jm-Peo-Ag2$gfe?tCJ?MJ0sDWH{* zx`AXj3>DiOzJ$0qbr+X)jcpGT;BG32^fOu%35L1<`-FjA}V7ORo9*r!`>yl|(39ta)NVtrMk~ z?Fg@mY7+r?JS1siqN5=&dv5Qb3IG--V-y@QT=*pb@OIE?SI>;L#I5G&NRB}k-AL=D zhiIEEeRLXx48aih!4(w5#V#!)^JZ~zUHL+wHU|U>?d#iHTQ#B_C2Y<>o2YosQVN9j zm1)5CH(v0zj(QuVr_BRX3Qu)qp58RAE67BXX;Yn#!rOB@vC%r9zU>!p?TW0NT(wTV z5)awLL8wbsVAMKH&+o5~e}@Bfk2JTmBqS%>bV44(b$lfFS8{bzQ)=M}F!)%*lwh=O z0FOzHZRshL`vB59(t(LQdHp(7GR8Sun*x{{Bp0oNs7f zElE>Pfr8V?Nj)QVVDhu;k8}mj67Z0PM;}M2RiqdzdftxU)8q3jhmiHY2NSl^$}}qf zZM|1VVG1NpQ8E9yocoI_{P(Y^@ZjTIE>&1Cs zfmvGCN~3K6#SLy0a!Rt$f8*?8jnFFa#h?fuRwO{wKxR+JJ1!l8ju^mjP+XCl`|jBj zUE7-v$Z2D3ZTf2$DLhP8NM*u#0)TA9*-VL!m7sDrK&rJKfRjMN9ZWt)mzK1!DJaxR zH3wa}Vtf}SzIw_7TCv=F;?b71zqd!u%S4he=oA3v^-o$uM9RZNck6~E9NuA&aJ1kH z{0OLq7b$GF=T~f>Rsg6vD>gef=jbYE=<3k6P^r|9S4>b51#l;Ourjc;hYdz1857Ei zF0t?K?U{-Z96Ca|X)oLjT!1tfDH&S@bKdd7tAc~1tE)@@wru0?Nq}qSh+|hDC}sY> zQw%1AlYB(635@K+hcZ$%X`F?Hg=#ySvZ4UNrdwfl_mp*aTXsZ^GsmFe(*jDbOQRO( z?eWm;;gw3?NmiwZh{t7nR;Hy7A6jQ7Hp8}f`0!002~f7`#Qj4H`W?8oiJv(Q8mEk% z7cNHco-2XBI17_~eY3EDN-)Ed-@KXG4ho}2Uh*rmk)+$!07_9cTCYLP5loC03AygS znyT3BU(*MY@jejlECPjf%@kYLUCN=f*i#THMUL6EbNFnbWqbxZ#T3ieFJCgOFoi|U z0kwt%Xvc_*q-0u!lXQ`9#RX(*CRT%u1*>=Q^`wg+)fJaC21SZ|-MItVx{Nh>Z)2t0xjl81}*Sr-@bXn%V^ExJIbI8-EcUt zMD`47O@`iA-DvlOtQ9b<`IU>!9%k+?eI2KwqB79s-P`^6(U?wXOekm|ad2TFpH2iN zs1UHk_4ZDuV_Uj|>&s}E1PDT;cCL}}#MDAMqi&7p^j)PfeIluu8y9n|QkhXLWX%p6 zr`46aShFFP0Jkhgc-=FGq)sN~ZNYnW!$M5X;T$jg!?_&-m`8sq&zo=p%+^t!T;9H2 zpn^}xdbg9iKx$jEHsZS&m*bY%c13aKs@g9WFL2sVV^qBw?0rC5ImY$Xh-OIv-~ZFsAq~=%AZ!uSz%Zc< zEC3*A8Zi;c*;=IZgZ@VQ`}8+QBc#xTkS3|Z9@*O7u7yzt=B%fTLIL3jX=0hBI-PJ% z>(0;P!^75HntHbwla3K!%o&}{gLPwkJlb`y{{7RZuU_TQ&G7gFMGy$A%VY*;@p6Xj z40pz$FqqZcpNnIiT%phO$2b2VGX|R3JId?e!(|$~tn=khHZB%{rO4?U6T+0jX>hGi zjkL;+y1^;^>5><8N#f`4g_80TxBJFVrp=|EK_MT5-8pkIW3TNxM*i^Z9Ct{9f+6#E zbS$%Cj4+KX&k*GrD-qa$B%(>4**;C;LJVSRlN66H1mr|C@b5LmObChhP0 z!ir+IPKK-o78aIu=O<|}1g}XxEA9`}U0t-9%7UPj5Fz7SA;<3_V~*={p6XQRgfkr# zg#jaU(GPNd8vN&|uW6oO@|113ja};L?oR(oDi^k4I$l}tPg@A9pLR2t`YVJ>cBHGh7tL8CN!gUn(jmUOPk*WAxH@z*NKW?~93xX2Rq z!f=0v@c5*hdQ6v`3l%Mcy(zsstLT;NxBN8k_iJthmxEkQ{K}ej*Q)w1RVstd{-Y1? z-Z_+89B)P?)46RM<>3g6F0u-r_;^7A2r2^(<0ZvY$!kn_w%I(lqnUEBc+Uw_Fb{u{ zbW=-u|a@pZYqRagBURTOY(!tUze(+ z*!ivHOi&M^ngwOoJu=!apFH`HET2Kqz)+X*t#C^HFg+aKGh-0-yKjZ*F)yf1HFx)S zvfvl145)8zWR-|qXSTm)#G>3dk7{O2tp&@WIHTZI87j)>=d!f0s9~$M8lB*-HN!*L zc>d=bYnd4Ey5VY$n-Sia++yW6RBX%(W&12uk-E(lhKdpKTvl&DUdu-Ote5lOUY zhB5hn`|w{5)81WCo2>Xzh~il-{!BDj$t=_KjQVGq!1>LAc2|(R#@mYj_qXwerGaSw z&Mi@e!Up=kbIYU_HFo~*d4~n?zfb*n9?E}~1rdf2u8F|EuZ-AC$ntuw<1 zlY;%T8P|VhZDzq@FvvzdjECdQYR*Q8CE-pge>EXgijy8a&r|SpCT8bIG{!~diED6% z9@3YSd0ZLE={i!3i6dI`@MtrA_SksmcLgo0Xa7!_=5*_vvMK5zvof@9Sourgl*;$? zgEQKXbL-7cjVZ(H>DfVH?3$kXM}0vZRl+eYZ}+U})gPOax4G#C+MPC3v2CXYY^BC> z?-LjCQWWtDrB-s^402}Uw}7>4)p@-}tDx&wQ@kJ@%*WOB7-t6z7^!mE<47QOk9z1Bo#Of1zUyRy_SP}4M!WbSRg@c;buqlyWO zh8V~-Bd$s1IJ#W+omff(Zmlpq!nzl;cp3C0CmF?LpUx*~3X~d|4Es@J{r`RL;q(I) z*PXCLEjht8_POAfGqX8kco{!ax@NVeW2Cg`gRW@x^k!5pDw@CfDi-^H@G-^4Vhj7p z3?wczCW^R5TM1QtZp>%rtjAm7F#dtY$FZ%RkcWX*mr-Z#f~y&RQ(Z*+&v?Ouha<)L zLhdnoC(dk!L%zgACKb{;+pBGw(uT9z(v~JhCw-w-CPp**wZBt+lPk}owh3%gQ`CcQ z#%3)8;ldR)X9C`msuvfGm;Ig>$gXD8ZM_Yn4bb#{<r_XDQ|KFV)POq+bbEh&;)Q&xpCf$s;SSS? zprkC5XWA1Q-te%azYQVNy|tTXoy7&(YZ;wyp64~5`9K3qL&NmzUD*G%g@~X{tr0g5 zFZHg+3`-2`E+JkQLA|+`cmH`AXG+*c@s`Xn0d==mOl}4FnFWWb{k@Nw>t(7lV|{<# zB$>~nJM!O+8gE$BV)I0mi}>HWoMH@0`me34!n&4miE7uQ?e6;~RZOt{!aFq#w+JFF zsb+}(V>{9eBbSpGq+Jc*mEDF?NrZ6F+Z&W|kX-f*OdQlgeL%zhw_R~Y`||E}c z;r;!=LLdBXc?kKv8;*^(uJ5ni?O{XR@S#6YKAXcF#NHJU|MdaFWv49gQ13i}J~w^- zzwXrKc>3=Vut>uD->g$I>fSaoR9>Qm{N%E6f7$7P@%Jr_P@(fv0%B#NiK~VA*e7>s zjzuI%$x@+8vaCcfnrY!H}NmZW5vrnv8=^44k+=iq)T;S~7#Iptf8%H>)m1Zl^WkN<=pPfArwUc*nk?!u-P zdKws#6-1MnE_kKUeo619X5%qy(&%h6BnL2BQ15^m)b+9|MRfe0;0tbU-gZeO!LHFu z?Ns6j^R+n2l8U&w$XX&&a&%?=$71#y*T*gdUfOIyiyt6c;rR8bsoE?p9C{Pryp>2~ zBS}!nczCDrCMOJ)lP79=QB&Zgw{;(neO~eXqsoH|txibf$S=YArD;`BY_Zp*BOATh z)}QbRk@%Qe%MWpPPr~6-5HV6dtxOMV5ZhIOyW>lt*%+{IyouUSn-_8aOv4B(o{U#G zbX7PNE<5i-sYdW+>>CCWdDvR7W=CjyPz3KAKcq1J*Ljp?h?b%@IrvcP_cYsOSon&aX$8KlrJoPAU>hZaOHE= z`osX%0!dssH<_6Y0fJo0QQh=0sZ*I)ZRJy5uWU z)+}_3fA1GNCXGHfO51&t&ogggYf%5Z%HzfF6h*d#15ZcC&sl(q|H1ZPvXN&YPPw4S zX%)671>fWNj)d*Fdi(@MW)57D^{@HlL@l3Vb`QVTIW-PAQzv-4aUy*_^#(<@7b}IRu;*+D4AKw$JDdkx#K~ic>w9XyQQPcYcrv`Orl^kX`e%zhoDvrk8Nc zZEwvY38bX)!|Mr6-@7mI4Z5GN{EeRLJbSV=ezKd=*ASmOEpnlL^lMUcR>|k<7~>(% zUxLL}Q=}3cA07YDLXWJGH{!b|gv9@N{nTP9Wg81)nlIiQoI%(ySb=E${V+++g%0-U!sPXN z!B^#EQ5;3Q2$CMbWn3W5#Py0Z=&9=5(1QiyqT4_2)41-^xN~^@^N~Eo0 z)+0>uB!=1x=ASPk@1t%VJ2xUvY;t+iMcsx#OT3kq9rkovcEc#ks?|lgFdh56ds6Q$UUSqAmH*py^S2=u=x}BR+En_jl9?xefW=yCj#V92_V#zTooZ2kuMDo8xas<33d`Wi-rtJ}Ztq zz3R0{BaV-F;Jx+4LmU@cNDz(1VhY-kH2EISgciDT@7De}8l_Y*uM`gruhjs9+=h^1S?5Nck7WC7JaP7F`lYA_B;h4;yp0H@P_jBR=a}ch7hn z4!G|np(Cpm8HZS*_l6?w7jjCrcKdxGKS)Ann(lH*ZcHo3j!q`wbLduZK2z&08h<;9 z)`$kJr58?-=3BW$)aQ{{YEs6(xd8e0n_dwIpFxuO$)$z`%LbF?T*S?VEM#6)Jxxes zR0fCoEOy&X<(aQ5dL+|Ye5YP)yALLZdK}>#f5ZFFdB~t$lv)X^l7Ai{5jlCSc_GX!Z8ucY@vi@rcZvyBaxMRnsn+_2lD|Dy6BOIwx?q~2PaEq zU7CFR7)VVIkBQM(o5cEDHW}G&qLUZztq0%c3D6O7HqKiFy+7|1e$zGVdvCKCu7szs zd_f58am=zAw{ct> zbK@YB)f=f!UmY~)3A(CIpL&IVk7LMP4_;EomafD=Iem2 za4LcR{BXLH`WW53o7an;zBd6!VZC_PUv!(4cytYaW{g6SXwX5H`(3SdmxEj9W(U`l zg=t1^N6|2*z7LzE1n(jj^kq8e|c5K7sr-*|Hr-EhQ|f~-H^P(@b=eTCDw z#wso$jfn^!>DL99auMH9Lqv)njy#D?+Rr_@G*3m$lp;;nDK4~@*D6noL~@P(a&X2N z)~1-7)aOP&@;}5tJR0R6Iai2UH#K)8P!}9tZrnW3NzTz~{qcY{got0(W1BC zy!bdUN(i36H|Q>)J4?t=mJt`g4}0myEMeV+PUNv^3=3Q3i(!BhA>V1mKBm7N%DPq7$zsvP z@@gzx>C5e2%a8ep%1K$4mO4lbH>PZM+}Dgxd--{W03{34%WNveG-lC91NBA2Bt^WhSe&IclzhlN z3)PHo{zi+m_!Mp>gku`m*Buf#zHo{>_pL#yd^PNWJK1m;ojSbKFM5U6Z1X| zW59FlRJPkGgm+eQxNEZ1=mkbcHIfexX-Uj2ZNeQYD0%sOt?!kZbl$DuU=@rDH(+7V zD@G81{pOgDABVL^v)p+3;6K(eRd-FUmihe*SHIO{f1HEYXOdAPFrmZDN?Q^Ae(G7U zqt>GJ`ejq6dtWqg*`k&}W3POi_fonk0kV@Ug%-&$rt9K0t=mpf^6s3`IgJZ?_DYOMeexA(1uoRK{93;uWf;!J zDkVPPC(G+};m=XaO2$6Sr_dVn*&R^Y?mfh)!UkvSnAl*+jKyG^%xdu@7n+0Lyfq(w z4=?^YuPhOLm$mdUN$$Pj-Yjkmx7oP!!St=A*)Bm&kr_V9OUow{#$(mo+_Ag2c7;7% z-i5h-e=UNB-Jg@iHzaXx8Q;wdNGo;dB}!kERXtI%5_F0em!8$ zlU=Qp=JnAUYvvKokmUPvHAOEnaB0X^C@j}~0yNKhQt~^VWMR$RBx0d06=i=v9vE|! zm63jUC-SG2*@9fU=>FF}3l79Eudw_8*R;?G;?(XqCydY2)ai5JU3D2)`5~>TZfkkd zY_Z<3PX|F9^#bR$8s%@V<(*ma1?B9%8_PZKdUpFtH*LoB7nhCf*Bx|kr6*+ZIXhH7 zK0m{{F35uIBQ63U0w@I@ln7AgO9o>=t^L%Nm_Ohqf^6e+2blrJ6 zxm+y5UP}r&YHV2Q>b`7=$BgnhG*ltIq`geftVJt9FZa`tct;f2-wmYJ`koyE9vhcxr zOb^!f?x%S!1HW)4fr&TTh=S4O%6l)_+nd=x>jWFpTIp}Pqk|2dJIPFLdgj#cjaJagTM|+;5W|5IN5`##TBs6JGLSJg43l}5I zDdt&vbaH8~NSj7mx-&cBA)Oj(^Xcg8KW8_z9Cc8CSardPs@#!YmG;B9C27eQ(*Rq0 z=an~f2$TGaiPt@OU6w@s{=HB0GBxe3Lnj8(pQO{1QQtkbv@>*ebw-G?9!Z%=ZTadbWk|%)0=$4=(|+u>iMYy= zT3^`Z>~DdGAohKaGW$ZuuyyWOmy=OT<~?-tDE@FtJ&Z)&?a`Ie_h09@{2^K|1yA_w zQ`G=Ek7TON?d0s*{3cJu%^z@I>kg=INTtjmdlqDld%r>B-EfkZKm zzx?CN8piBzIxtJH*XmmyNd9I8r?ZQVYY3;BKUqKictfD9gTx6TdUc`uec!I`l+A|! zYP$mgA^1C7qJsL=ilZ%BFVL($m2WYJ5?m~nie^&tzorm)O`+U)_vYdy8oT!=vnB=L z7Bbh3;~ge`abo*W0u7dRC<`c5!I}Zc$U&mWt~ABZz5#BtKm|Zo3@lnNV~(gd6_L@taD! zS#G8stz#6=+o=bh=1CXs9HbUuB8oV}7jSuYj)mH1Wb~=EqfMObcpfXn@2mzQ$o&#& z<11GYN`$GpVXWDIc;qP->rPQJEXnkuPXzY4t(tY?tn#FI+nriiZzcw-O-(!#HF67y zTqt?Y&^?Y4sG_`XeZRq^!?;FpPNTiy}6 z>5)WlhT1pDWq*NH73V)6gw4QBl)oC?d*qQEfFM7Vstj5K3613(gsC{svv*MEt7q-K z9z3c1BJwJvVE2Y7f-sl0du%js@hyB8VnmpP&3^r;mcm3s=(x>xUf^62o{kG=QL~t> z*Fi>>vg&r;G0Tz-JDsZX>CTqfg}A)Ljw)?sOKPGM)TY2lr{1k_jm5IESU2N9=rqQ; z=nE9xHNj@2Ns}ik&Uim)W4l5aVG&A$!OABbg@a&`7@01oG`SDASlnESAEYbHR|2nC zr`Bd(e(RonSo_6 zQwrMNtcXH#Nz{Gc)mLWaJtj{srDZ(G$<1)KDc0*@{e|m2nro0K%}b0Mo2jIgKQu*5 zy{xlZwV&vUPJ~5jl)9By@1XLG1Ds^@QitCQ*P}}akf>2ZX%kPQ7ubrEF&-{fFb?H3 zA)Q(05RA5J&x?13;f+=U6W;l|jotHD=`6FAjyVdNTX2e_enAJ|F}|U8z3|`$$*)O| z$Eq@8Xp#9@WULW(!UH*j=U{Z=ziTKgY9}1zy8EGyj`Z3O%^A;9RtC_JUl#%r;>KKg z`4SVRlVPet+#g-OHRf`PAX&LtI44()3_|4Q*yCm@Vg{}jVj*tR_G{~}Ny!)}F^~_L zq;@X^I2l8Vl=>&P;wiuBgg2>^tIg&dqZVMFtguj`W;F6GhKlzDRSm|e#xN{2_wQ{?EJw~rkJ8Quca8rAfuJzx_Vhqum0!Y z{yP>ccaD%UPaJP5q))L49l`EoA9Gb$vI^&uljh=69okcREQ=r0BFr5wM;fV-ny4jf zz|w2^(+NDI)t(2JqigOHtf~@x?*ep)c(eEG3Tu~DOb?ZcGll2lE?coWJ=<5;4-JPj z`H5?#OpJ2wN!K?iO$Wp}`?-Eb6qxyRm*oWz!M6fwxY^a`xo?`Jn`?gt6B0k_HTyB~ z!k5S7FHT;6BFkCE`gxS4Ro5LgX0Lf^Co(r!yJTEpqW|l;gWVnbVDILEO7|&)g`G;D z%>1+v{qm05MBEiO0#EgfoF#l{1{NONqtD5sfsVv4MwouKI8}EIU8@gp?rgX8Cfs1J zYHE2`U-`}79-rT=AOS5jr_w8n?kt;6pMDFwHI^x!`NpEeH_x}FdM#(UuWiDU7s9_} zX=U04*sud?i5Vvby0mJ9Ib4p0a)-!Nt zzr5CP3Fm2SUVB_eQxI2k?IF#P(4=?h5I6~uxs9LpjlM_DeVQfDKG>*Dp~ic%l4kmv zlaeoI<=*8vhWxWuH00Xw2VAkYxKi}4p1gC*A70*)J$*mWV9{|YR>JQfUskw$?Nuh` zCihSN*`LQ=k9WA<&{jBhj?2hvm+-2>#M?&MIPV$un=BZAsfZQfr92&yAP zAKtJ_d~T^!%p&1vM@3D<{J3(dSr>i0l3LI<@{^mH5g#=Pb{~UE5~ChR$n|{{Y;J zw_VP>sZTSF zS<|#5I-05=CZ9Qm@zQ14;8D>w?c~Njci2lFWk6!HFRCt>D87}&Z?;J*WOJ{4nTM@7 zFtGY=?TS=#?#3d_L zD2va-DZ|8vjbI0dvQ&^<(o%usyZaahzc6IxW@0i3W>lD*A3q9-U_*$ut&IXZhU~Cc zeOA$l@;MWS6tD@7(3ozro9n;OTP&|(LqpD***9%vn-$TI!CrqB3z*HJ4E5YtRPG=<<1ND*11!aQ7peCLrbgmL?hb4i zlw85|89Pt<^v-X=lyk_3(;uW(8w1W28ZY9l6nzPhE8dqYR}3pEi-~Xaw=Dg55qzpb zWuM{+yH*5^>1Yk_lvVZmkAr+_PH^CTaY)aGXsmZnc6&?keOj)Ao5o54Cngc8gTzb; z!$g^Mw?)ripVzDN1VzUC4A#S8qVF?9~tDZ3Kx5u<;S1Ul@O#c|D;wG`1l8 zS#L}QI@%ga~VG70$!Q`fBdJ{Fkz*zM?Zm<~Pg9V@Gb&QUJnN z+^WiPj;YTsqg#rFPuzM)+ptL%pN-2gga-?`!yXtab9hmEn0NL2+$p2#QD?S-4~`Fe zs&gSJ>tQo-UfSm}25+J37gLkan1oLsV%!j%MUSU=#07nl?HBUOLW{7epP4(;gcz&w zk(P?aQa)7nx5YzVUZkMdkH52ENbqDK`w1e8SFB|#jwv&%C&D|v%%D?STF)Fi-<{YYj#Kf zoW8cG^z=}aX){_>Uv4cQl3=K59-(^TBt%*HGCJd`FZ(&J%QrZ`cd_Q@Ba8HHRmn+- z;dgNmeUY?(y~ z`W@8EdQ$n{yJ$|sCznFWKu3X%+)xayR}7^Y(?CIne{lkirv59d{y!KIn_R5XOgwZ9 zX`&hqDda=OUF;YC+U$_9_4K?J%gMw0`3H;no>$iU_wS)(E2j-|;OZIW{S$@#Sz#n) zzJ`A@57LCtY2LBKTXtgwpJZsuWeM;Lw{el_<4Tq-04piJd#kovbbi72gUH>T^~>9> ze}+B};}0R>lWGzR@EwSq{2pT>;PnXy+V4X~pJR1Z5H!SQS2oofzK$i`%vf~!0qqi3U zLT8;!qbW#&7%g)TPfp&^tDf-?4ex}W4#0E-@dV(;Plu3AC4mri=yEe&L zuT#@sm1#je!>h#Y>30v%7P}YhJGbB7`ONXq|H;GKO!2N%pcTAw$#}W8>dL*&k7{M5 zr*VcFY$P4wWb8F2b(6#deU&Nd9jY}xl7YyRQVtY_bzA$!2q;2X!I;8N@MvpGU#q^g zwW~Thgwd}VuK|uHj<+( z^%X=n5qQZ!a;^374++WTT(_o3f;JKzYIG0^_a zAtEX1F->ArYf`0F#;#aBldUEUv|~I>0sZ4hD2hULz3IRK1q+ysI;T#8S9+v-w|`9) zax5-9T(DX&Xt-Q!CQimnTbG?8s@PFxwP3W)h;@dEb;fosJmy?)Mt@&F7UQVO!8?VL z%7@)uYkK;_^Z~;~3`37s?yrCZEwo9Dh?YR>q}{EVS>~qo#QBAV+MTub>TI$I=@}rxJCUzrl4+%}u0Y8?R0p88 z%gdW{K&0n;4xOe_2MU)omRZVTM%iV4SMtg?hUmEd5{D=x1Xp_R^U^z|4GoS`DtK%=Bm zweHA|DnLBYgWcowI)w?_R89$|CEdNuAks^>%F4Lr63`M6{O-z#wH}74`E&r*k|U*&Z)BaM49IhU8r-8R;K;?R!6x} zBjSPD$dM|-k}o;;4=BHWO&A?iD0%Q$U-sq2{ezF`>?V(|r)`mUeNB85rn1+&iqaE- zHX!`HOzbwtPv+*b3(Eq6f<(6)1+g>Dzk-w-kpF9;0)dSr7#I^@aG{J1w8Wu&(z;T@ z(|{Q^5Z{~xNzxEVje-e1B*{1rebbEjm=mCp7-;D68R;oQWjIK2OfEkRfr^A+>12EY zg1QrK)7C-I#$|m&OY_Vd)cb2;MM2R)Fhva*wGZ#fc!v6ypRGK91z-Kk@Ce`MOg6Wa zSueb-c_qp#?5Nz;dQX#7T(nMXIq}Av6==oz==zeqd@D15k!FjO@Jj+#CC-#W$+c#C z*_WK(=P%}Ps5FKzKkr?B9&W1(kEkzO-SBn1XHY!VUg!!%sfnbDP;LgY+%q#XP#OxK zXZCeXEKD2JUZF`{N!(@Cs@0O0LgA2;u3q zCaBZ^JKNrFauYjqG^?H#IOdc@M1n@K)C^#SVGX71ymVLCfA_9bdNTmFfJ{u?Rff6Q ztD;b#cBWI90VQX(1fH;_phyjhWaRkWK~^`LmxiqCvUnIA1J1K%TOcbBda-b9L@Oyq zyp0jU<9)Kg`Xpa$cnBWJ-|emn@m!w>yiDJ)+zVfH_W~GXYZrZ!Xn7a?%-bdGEI9{%B(Ery!*L0_u7|F%(sp2ildOzT(9nw7zqW zBeS-E%9P%_jadR~MnH{5GA$8vPG!mBF?611LNrwrO`uO3DCR-+Pt7&#+5l|0aBwV} z9TcImkh@8ll_h-(%6Y*k(-1$E=Jl45>1pGCK9sY0Sv{n)N(z?zz*X~=a%<$ z{X46=4H^wtKh-Fg88rIr!IwH)mt8VsbMSpi*HjjnXhZM}TqXp_Y4{BDq|$q?hp z2gdZLFJBn?1L`3j)yq%!lY-$gH8lkr#nQDXLtU2)RtG*Csy*XX02o;9l-4ecI<>5; z#X#+RUtixnE;a-dU+tNQMzAmo$X*oqLy6(rQCRzzpwKC&z-jOAB8yirQKOVGPURr2 z6jTkuDY6Ss;9M!q1JzXFWkPp0PAvAX?9K67)DlXG>7e~Em%y~wwVbbd#cj$))?<>d zKxAC+o?>%HNEI;GEGX_)HBy9DrN01~HdA+9UAYubSreY@SzSVzKkfnbEEo-Le7F11l;-DI zaEH{j&`@2_UW0`THG7BrMS6@^({;u`&FAJ1P`tePt*p5@ZAs+j?F#qO@%3S1k_}Q8GY#J=|{U#7yY1&6Xg*=x7O1foiv)bh6X+( zo%^oPYgTY*Eq9hRje_E^+E-aQCPx{p?bV>$MRHH?Og6*XoGu(`q|fgRYmQs8vP}KL zDR}7bj55?xF)RB7MyfFnD$MT%ZIK7d8tBBo2^*r9D}TjcSpVQ*2Nch@wf!%$-U6zs zb_*9qL8PRW?nb1gB$NgT0fSCyqy%XY>5vwrrBe{;l5PQMkZzIghCBE7opbK}$G^rN zd*d$mT5rvF&L`A{6o`RM;98)RDmv|eDj#$Fx(?{G?XM4j^hYk@dH{w7zR(~Q`^AsA&|J&1qlOS&4&igl!3G- z2gf!@$pL5X<&=#~*3U0mJcS3$udnBfRlH`O$rOSeTRZa{f9Izu5>37+8N( zqNCqloY-X*Gxpir$4i!$SAH+AtSCX;BW~Z*j^5!MygNR2#5w4`3hahul14WW7*n8S ztp%hb(BM>fGXcoW`>d?&-9Ru5zGrY>i`AS${%u91U|hHG(-*=@EH7icMH%{4yB>3_ zi7`ZO$5el|D3t))9$-2uJ$~FsPGZ8DRLz8U^mxn}V-lO3x9fi!lebmfexU!oq86&b8)6nkC49i%&k|wjD^? zLSrVi%%j}Cjnnd8s(*wQYE)|tnysK0a19a$mph9%hLOZbH(!?Cd|5J%IXyi=$J6o9 z!n}IGW3w_ln2PMarx^+oX>x_@$Y0RcR!ddVeDb6T1O*UAhq}7#vuoSiVz1g^a{k8h z5nndMACnu%q04%90!cZko~|&P!4%fgsi7?Lp?*lE5;Nw+lIzB*R z@Askn&b#NhlHGIqUdm3E;ko5GwJ+GB7-|xPCP{^~Y@Etxx6D+m4IP9${BMFl0*}d0 z&cSvXVd1*m+)nl7X5jxJ#JNFQ_7aBi5+&)UX1njnQk5y@HasELBFQG7?OH#kc$2E= zK~I$)_Npj7woWaxqy!m05|UGp$pBg=v1TXmg55z)MeO?GEl^UcP18S)7=`ii{2Be{ z#;3N4mDu0FxYfY+w_$MFNn==VaNvF{izMh}f(~E}cw!rbs~~;{Fvz($IFx^wo<%PT zuL!vPv^zqx-Tl01WgN)h;+9dht9n~yVn%(!bKT0)D67n}PoEQHCQhf`_QR@hr#LD9 zznoQ7)x|32MumC|P}1AER83hu7+ zAiM+1Zvw|QuW-kd`7xf_U_OAd5zLlNuii|MMa8ihKfDL{E+9jYI0!JrW0!$&U5V?c$L*f1o zs3ZM5Z|?y&?Uh5*mw{w*ZFL8xSRbCA0l($O9b1P;R>EJuke*1#_Ki+v7NwPbO)L0X z0=L?XlG^%`f<^E9WpQz@VC^DM7A}FG0EVW-t{5M?{am{kkUmQe?AkMoV1R6o6wn$W zBdsO2W?1=aB5aX2YS*6K;qY>Fl+J8DZ(|D?>0Zd`_0_E2!7DCMdqarcfd1FgTh7+5 zBXrYRzrG63BRau^2Wx*W>+4s?DpKyJdm@4Js3v#=i;I7F5}O1~*xlI)40}SPD(6A> zIK`xDhvuIQ1ZI77D>e2F6$NLjk6ySq6(v&E&*A03G#r^Z(Mg&#^u)h7VX}FZ$isv=Q&nu0y z0gIH^&6tm|Al|xyfDjb-_TG4Sh@6hVHF#1%^!BW3_w)i3#(~z&p|BI|?lt<^7MF(t zVt292&A~Nf{u^@{AXpFLIz{x)Ks~>*;qqhus&XP9*o3^hx;TZgX~c}io{mA%y9$CV z6B84(k!C=e2f^N;Y`!Yc5&8Z7ri)kwe`yon}T~ z8Y&B*Pkr6J+6+Fgy#IUa`3zmrjrXau8|9WpSP5fex!U6*-$)(4k|6xSwIyPOMohKl_avO;!PD#BgXp>AZ(!#iz)A!i2wvW8-^xRZj z647*cB%*IFV`04K>-cHz6Z%VknS?~H~dqq4L(~vC+^#aHDnH^wJd4ux|q|Z?8%(Q zGsL=$D+8H73J>$=JU`COjC?rvLH5xds@MI!GYAnuIk`Q^vOr{^SK|=0GPgoNEYiEp zvNC*Xa({>U$<(w$KC$%(;mt;PE9`g6JL6(wuV9iv1cW#Afouxu#XZZSh$#DGZ4j|M z(D5U&2HPA!pMmWW>XD0p2WVx7>VeXU7%LqH^(W_J@ZX&yU=xt{T3=~aVVjz0`bQl!}G!4xVRD(>`b zjNhjj8OIIBHD1D{%pk$$C5((Egs7KG38xs zD$|e(bfeU;MZ07X~?}z4(gIezOjNA5&#=_GQZ)Gcc`(&%F z4`JN-R|4u+>p9b#PAP6%)u2_JhXET@D44r}R5|R8VdKcf#YJ~_ow9@tWFSsq2tb$% zJAFM!#T*u2rr@^Ui`ihTEqcDG zcYQVnsTuF0Imt}>rn&eF=zY;XM>ukd1;UIMXrH*j7_-w&5GLOPoj$C*L9UGf2VKkk1F>~IHYY@ zO@OG|38<$*Iy02J3Y4i~_lNv`mAqYTmj@ZDuPee!1BoKgX2?1M#R_EP5zi?Y*q5jC zL^Vr6VjOOp6Q!%=W4fi!n;@K{Bq0&J?6NDq{$ZxctW+Z^ETCA1>0)lt`BV*7*PkAT zv=chul&*pgk@d~@QU#(355-1rW|4cG_!S;wYNXaOEx`Jq2+FyRZHO(`d`rNr%W5*j z_#Zz#bj9=d3>FQb8Uzk3Aggx(k;y-fzo}k)(`%rLejDj{y+*+)m`mQW9t~mcDX4odKfJ5RFeercUVg<_w*&|ouaYRW-=_KK8(|br@`=hlPsP(|VIl~OYeeyZpQxq$>8Qz;fS zH|D&q&r@$b#I$F;#lv?+EcABG7tw7%AFWg~9;8n|%G&~DTG+(mu2bvzacnY#&ac8v zq|Z&dqKsW;H=TP|aGC5DaK-UE`|!VH##Gpx#~yg!Mr(+z>{l~OYb;(e_r6y%plkCU zpZOO({bO12mvCEnpu5TR@?OjCOZD6Zf`{ZZKne?hosbQ^{$LnRoLD;GZwJ)W|!2m+6g)m2``RWq+sNwA5 z4gW&_w%#d)xLF&={u7NV4Hk_Yz2kF3U)#~@CILVJrD6YurC;wNbJV55Y0 z2wV1XSY1J-D8=*4{MkKgd2ViQ_;U@k9zhN$DbYwk$+jT+;Gy87{_sT%0NNc5U0%ML zMoR;2NN+cU{vaeRm8;YmCBK12Q%;Jjg}z7*b9{}ZnDg@JP-=sjmDr>Cq8mt@u7%m7 zAssN&>TL@-G4HWB$V%U;pD}LTa&x%Q2*?Fs>KXh`~4BL1Kqp3H?`OhYzjf;@f{pu<{e~_%Mx%M9pd) zFkoU)8qcn!824s{FLCXAuY<;?h`87WtN8@I>+{3NuMY%n+IgqvE_Q^`_*#b~;`g-t zjNN4*-aaWYB9z)a^`_kRLkK8UMKCck>S)YWJy1Gvzm3o`uZkuI)gT$PyHhEgaz#>6 zZUslfv&EjUqw0y{z$w0lgbAeZ)M}nUx5b^j_r=U{p_Jl&isI!3>_XRh`h|IdmsN}i zBwnJeE?yV+jY{=RC-M0|4MtM){$a8jUnzwBgSa|{S;pR;+jQ9TaAia5`t8ljgu#P% zo-J99!|R85wN=Ww17;UnP7UXc$T$cwIs6Oc_GnmV5wc*?8F*4;!c<$TF_Tx1+Mf^F zniLKKtE*~t)sbjso2~ZI>0}+@GC!4rqlb|MnlbzHN)O|p-+Pc-LMW{xav^V5m8>bc zVV3}!XT6XpFw{Y{jzL5LF;71kdzNsPyX??E z!XxrZnLhE3}Ng&uM)T$r>~}1CTm3 zV-2EKL-Es%si*&On0xwY_7s~xN_dz46U)`5dT-<_%^v<7lb^3v*@wc`Y3UlB7af1~ z1}NT1CF`#nc@n8`wTb@m-0ikys6lJBL<9f3q*SN#{>8(^&$>Lpj2}+%frpwgEx$4nn ze%k(rulpZ1qxFX43diCKhgR7}TC=8>(zxFFs^21p{_6sejG(#xo3GI{%z`}0Z0eIz7vGX zv_blj^VQu?toWr*$4ugiq|j7EE8E8P*UY~iXn!Xr!+Q1O$8THeIxd`+Htvt=`FK6N zG!nGR5uIg*)h}t4af1!y*pLEmF29Wo6Rj*A+x$aE;bCn`wZ3tjz2vRlcg>_R}o@nJaHY-cJWdR!#%fc`t|J$b>C;Y zGv!j#ASDkbAcGreO#Q{{e|EWaL{W9N>BX7L0?)GgLwR1VF*cRn(7yPFDMfa$=()Bo z%GGzRM?$bY5|foPGvGQLip$E*p29{&d8I$IE2ew3cUaNyH64hH&k}fW`nS&bcM_{& zHyZ;Ea+(3@oatpUk=56ZpS=NR2{?xXy7y>eA~+JSTSD_YN>i!d;k?tY`+?hXE4f$3 zZL)v0zaA$xT@vjs$BE}=kQde{LZY=_^nBwjT=ogmR6CwV@N2pUlLts(f_)NKxY*71 zASk@TcHGV=2V~CCXhc)JR+F7XA-$RkYfft*CR&={tX#Z{vg&4s$U`bJ3#a4Z4Gf10 zo|S^3)dG8)k(Jr(wp-EHnY@gggl?x*r+aZLu^B6)8H`1$)C|F646%hHM8%4&x5if+ ztOGspMN=CfW`;=>BF9ZI@Iok>g8Rb11p?w3Hc%&qq9uRmfdg6!l~a*!{cmz1rict+a)*lzVBb(aUmBS+}a=apsXD;I);CQ zCp*N;%A`&XtiC&n3KQ+-|C3~yr&_GU7Q=F=3dZ!mo_=zzW__ru7ieH~O%i0^?)Gg~ z@=g8-YHMi!naJ#b(p5|~YShu`C0ZKTDL!L3rhA<>VL|jl=$47J`zTdJU~@9oY9dZ; zPGFUOp;Nm4g{^?**CUXAa&&VGtN&edQnJxy`=)z9ZF{`FJht>9X)nF2>giBncdE%l zs=t=Pzs8)uRoZ#km=!+Cdmbr#pWQf{+XZg1s?0vA<1F4|a!xwmU=m%wXdiWK`mOT@ zXS4frF}r5s=TgaN&z@gT(tID=pF7P6zn#0b`Fd5gQqVBsepuOhuov2Vw`=0l-&Q%L zMeZ4AIZK<4TS_l?sve-FwT8O*?ejfdLg#9|<@?OQU><}7Yiet8z;kD3t_ijRh~P#0 zrtH_3T963_y*{`HpYr58vyN(hV8B#_si~vm0m(>^MJo~obz<+^lvdC!eM@alG#jMg31Te&6zU`o zsEvVPd1^oZHxGOJJ%j^IZ(@fBzqDd57h`9P9Jeu{%h+N?l@)5-w5&byw?5IzA;Rx& zUv(r>;^#tImQYFbn3pk)-MwS_CvKX5Q|%`5-To9N6OM_*TpY4j&0HLfxC_~%rc=esJq9N@~Z}@wriIJV!*Jv1WPUW457SW6~gno?BP4OXx{4oG;_3e{x8p z8SJUgqGp`#)D5r65-o%fM#sCP#bMtJG?G5g^tGrU#A{VgjG~GA;rq9Y| zjOczoW$M!HiOu6=yCWAoapVy*tQK7H?NGERq&Ogk{Y<{jrQ!U@&?SZG*`vNmp?;0pg3alXBc4g*H%mfktcvOKLoR0Mzk1L~N3#^O&aJsf6X7#FN^{c`C@^t zV=j@CXP0c^Xf3DJrCZ(XI*AEdez@Fa$)tUGU=cy-`X?=b(iM%y0rC;0Pq0CWc4zKM z(Z^@Z3NXPaDpFKY6yLuXg{8ZgW(GUFs>zYSi6i&%-hFTovGPmk&lYd+XV zxA;}wbBl7#uP*~ach~WiS-|oafIRAKAg@M4N6%m@9oq7O^~Jjgs9E(r%VQKg_Lfb* zCpWx%_GfpPO~TIJr^EQDijC5#q2mpS*)QzDP0p}`ujoIRwSDF)d>`YNG59|2#vv4Z z^o-sY|HtF*#c6t#r;ky}`V~*LV-R;`{8QM(@YijWn@*oqadGjakF#$^iqpRcp|^;X zyvv!%x-~=-m5*45vzIX$&4%^%mx=?dcou$CCN#3~p6}nU;D0-ED|*<`hH=)4aWQmth6 z60DgQy2thd5QQfE_)~k4C7AbI-ww7Qc{MVll9}HVGqMXa0uKur1f37rq+pP zTUenB73|Px=n7}$n&ISmaxP!F$FJaDEW4p53KUoNuRyV8wq* zbXnyv5I*@=blps7aMJEaLuKBywUv!<>%(WcL&G&Be_L-l8hQ)a`E!eQYuc&ZN$oJ; z;!}5J*Kj3eYkwM>;{U9I#y8?|rmRA0fg*bkyUwG(z7pgBeS#wi;7=6zq?%e<)jMy>9W?I&4OWbb^@e zwcH?&z{`S%+E3=NVm}@xb)}715ERSg3_&2Ee2c<>P+veVUm;2w(*><#eQWhOYHA1G z-^EjLy7sa5i0T`!bNbiQ>(~?ae#Hbh#LKLbH4)uLK}+-Hh*o_G=(icc|n@{rg zBl2-8`G)FE1E0`zt&bTkxeL(X)t5a@ytrv^&3Vp?r0CfT6u==zc$9gnA5bKawU5y z&q_V}j}smoo6{$jMfsv@v$0NfZLeoKTIYucFVJ^nH}tw*d+9hIyRB+rCw4c)#p;lF z9JM;nl?<6yRHi~H{qW<1l4rW*4_E8A^@s~gxTP*y!dJac&z3{3w|=j!ydJC<20>r{;SemRT4-9mQ$4qZ^{rtHUy7p*;JdQObK0$at<#8k} ziNwe0XO^o=s1jeNC)Uf~MT}zGf8*J4Mq7nYr{zRADSY1hf&acs6}7&;duW!r{~ixv z&&dbrZ~j>qq2b2N?{Wh*>7U~M^-=xsLp_Z{CDg#V`3^pS#i*#nKsnjF7EC~@7W8>Z z>HN1I+zaVh26J`BX!E)G7T=CVJQVLm0le3;iP`1}*(CmHKCKl2p)Qp8S*j#yX}Kd7 z$x1}*47T36vwXhzpPr?wrbl-DQb`|0rW>^GUQPGdS`WyR_~cERKx-Z>nK8mBeJ8x5 zmQh4|h&nK^8~w;a@bwT#1e zCn4days#>+mfVI^r%F7#HJMGdhxF4_Ss*d4c;Vlm zqM1J{z;y5Hj>Iy+T?PiU_H|z|4&Y2m?z{yqDKI@S1A!-l;e%u{|jl%R8 zZQ1vkbl^?YzPl2K^KMLY^Pgx*ihYg*%!;Fr{42(S$F?0&2vJ~!RQo3qzD%JGUHl-b z+Gb~N_#$8Z6JyX*f*u0KHw&M|le3@YEcZE_?h&5->w%BTXE>e_R&qr@R{l3fbleYoN(QgpTILVRtBwEb#w@!nD#N$IED~yxa!$GGOIzO}@4q&bawk?1jaC-VWt$n?&jf{*8Y`x6LxPKxXB!Ac%JPV5Tp$@bi zcmke1&4J9)!&p3A=eY*N?IaLV3luY@x8wOwTj#79c)aGs3JE+xYkQC87hTs~?%vC| zJ9W#~Q#&2))KGZyOrIe&L?_1s3dO^z+6=guwF zACH5jALT@3%d&H(kR6zq;Sl_#2?&qXBNNNH9hw?|Q!#`kajNlq+w{_Nw4Zw()lKKl z`Z14Z?BTgaLuwjJ`0Ofn0eTtvkk-TY9cSY3fL+eejus!(_QiQ(r8(dC9WBTYi7CG> zKSXZ%_N4jo$w_%&_zgEMQA$dEakL`E06bYV@0gBS^njuEBxW(Db`npL^-plk+;xWZ&&H@s#8=P&cqzrT-BnleV%2>EvU9lN#ctOWKpf5dDGK4z7+pk`0UQw zMkz7b1CcwI?a~E#pVsHqL@Rp``+Vi#=%iR0wZAWF+{=sJ4-sasC|5mW{t3mkt}c?Q z`@EvFP+1xqBXY!=owtpk?}9LN=G0D#8W#&|XwC|vlR-SNreT_?OvP_G(^D;+;GIeF zrHjDd)sPJRiRsx+#enY}jHMiiWiclc)7*{;6?YIpb?;(YUwkJcfD0;PPUpJA$24 zodIXjfLMG-3{RT>imVDS-G@7`u5#&*pC_?dV2@Dx1nNAF!n+^y_bn3!3w z`IC;VS%3ShE=iwOK4D)J#&=1{AD@4P%9}y%N|CZAC{uadqhoWdX1Qm<7kbZXx3sW? zO_!HRoYgO?)703v+wVxp$lFmn=C94=k_lcp=P13RR%Uxie=kh>$zgSTifcd#pkPNe z&+Wp?d49beUFo(y_0$|85F9s46>OCkWeTD8-Iz{UY4L!R6{L?xGv4o(c+Y>CX+%=7@rfoNyx4G@5A;PEiet6U=em7sQOGkT`g>gjA^s=D`{$8yWSdUEC?74qfB)OETp{~~!*BkQ~4N{K_pdCDV<9o=Xv+C=1MNZU+D*ZRFn+r}7c0I{)q2e8wU z!~tFkb|HxBz3H-idaJUTS<`U{YNLH{NtJMzUhkY<G;!$Osi(Ojw3&=}u&OOh}xgs#?5VY(NykGgM zaEXbqGpai#|9Q#d5Bm&NkHu|K`&Lh3kw)rNZWMn3X9QlF^JB>L0IAZ9v8JWZcxPkeD2P%1vG zOB-<}5%D=J;Pj95Tu~fwEC5OhowF@K&4kYJ_yiYsY9wPIV zMUqpiJ@vt&jbbc+#DO;_(;dYy$;tqt*#}~`y&2eEC>1RrH=$@gtUto(D!wynsj9S z48t55cN6v|xt&ZOm-u^P-SNOxC;Ra=Ra&Rj2P16%{nzN?)k@4y=TzTfE|z9$&v>R? zWK(3I{&Sje_nEl8N~yR6iEljK*hNA+-TR-9Drpaf&JvT(+vax@{&V;QpN-U#oH z`h#&d$j{#UbrkaX-5^@2bhEi&e6~d=fAc%o8Do|?EjdxruNzmg6TgRw376f##C(H> zDgZk?CKb#j1B(yOdA6R)-HE2#m_2`{b$9!A^yi6x+XsHQz7@$VrG3%o&btI=7R4!# z9wbSku|&6ZzqV@%PBa~m!CGZGp!K_&e5h85qqB7|=HgQF(jd3hS=X00Vi-Y16XS1oBFQ8(~_^RCuS&-=e zd4ZU)H@boH`<*3ir1vZxcB2i^yNMUkoB#X9#+~<_w~#(R`|n21B$SU)?C&1kw(lt@ zn*4j%8ojSVLL!j2(zpRnd?sA7Jc#H1b0|Fcztz^@rr(dnW&ZmLJfgOhZ8p?; zmJLikC(F1Ki247#G=&QntmKZvFdza|-Ee{_m>58~G(;I{gTB2U^en`3}VQKr`|(1XtPO93=)FgIvphkM}Hr zoDIY@zFnWVen0NJ_1~xA5moA+WEi5^^t(%9cQPx7w^cRg3@=U(fTX6n{>_6@o%438>B7Z4!+qrJyQ4~b8P~be$o@XK;y@?? zK0ntHjB z6{kx&ZV&i*uk%U0tK;jd0kF{P2Op@to^UU3T>Rk7-EipNru@s*GIeoU`%yFn?M*55|}K7S0}>be4D zZ~FGGGe(z{>!Q^nY{GtcW?k(=nsdtQ`Vz19HTMJp>$y?d?|FSb3vi5cWJIq(#!(24 zZgk!OlSKsL08pky0=A^!YZr*C$_((fyMPp72gmpO_d`NL5}$2*;~AD~BCL;~_=Ny6bVkxySCY}h zBqd$cZ+qCl<*A%;wG1^{{m0~q-1nKXazA*NAb60#lUaFsT~wkj{sv8)I^-#S4}q~= z7q_wMNi6PTXGWwcT+dj$-!)02);Z-p=9pG;wN2*y&T=xj@vsHz>CyUr6^1}c0IVCV zJcqXhKFo@jYktK2n=_%(F!6cO@Phddaez{e1ll2wTBqDrnBSu=@V+@aKi6SQeNOio ztAW-o(F2VJc1bF!k&)iG-yK~TXR=trrst&4)(+2tyeyNIoHoWw-~`rBiBe|I9WIGD zDl&V+dmzoIlijO{Mu#*MRY4FN+mgLP=kXvw5iKn=^nl3ov=v&9w0!q|aUCro2K2#M zU2_5Mw_VDZE7nm3gN&oIEUhC?&pF6Xjr7Lapy2xhs*5U)AgF64HTg2xx2UYIekF;%^f<8{&kgwC){p_ipSUoJT8x@uaX9v?FyujB1r7WZ(3 ztGr66Ph;{2_t*xXt@14dqYW^2%AR`!k+Ot~jsI4#$JZPv1#I=NuP$$6lb?X&8lW%V zTH8QDR3ifLS!HFsCF;U>W5|*|^j+ZK4ot_8C%-2rCEPGEFu;D%7J&ZYychIbT)P0V zL~jezV2cHAF}MOk{jtx?+8U}^!2notZsiEslj*0%ea&c{M*6n|P#FH@~j!jM$u{lbwqpq&51}dh}^4;wTYVEzd zzQ4BP56vP*Xz>Z~Lp?m)Fdp7o=z8@hIn-;EUpo2aIjp5*+vDWaP`D|7ZoG zM(L{A46fn3y3$^kfhEfZZnh zLsuO^x1yIP2{U&eZ|l8GR*;tm`&7~aZzJvpw7wHoo`45D+6JrFMo*lzCul*07lqv? zF&f=V@!d-r^0M8W#wFU0|0!_S>DRjd+^~CwMpGhLXT4bi4pmUD37dX0t%d8hP zA8<3Z6;)JJfI0!o5a3%iG~%D3J$bB#S3KqhLm7TE0uuJfsVFW(6X*J28?Irlzq>G%BiTNBwDK+wh2L@c& zJ~X&2D72@JT500cQJ8dp+c45(;VC;qQZH8TL}DmCAC7xuPOp~SwhEtKwL=MBdYI@| z!bWiJYgi%xlx9F_l=TAWx4WCPJfOr2q^NaVmpUVvNt@hfJbaSv*1j7L*iSj&25Z^<^89EfR8$iE3Qm4E3IYv48<_; zm4!ygs;x~V!xpkzki2Ai^;c|do>cbv#ZtUoS7}TQ$#-Ajcs^yuA@NyreA^?J^Ms^& zMWsas)3IAxWBf0j#P0r11P0uslQmFibYM~6xY20qxfh-K{N>B7@e*J;QG9*qrfGQu zh%h+~Ob7GQ!}$HXf4i;?NpM_?lD4W{f7>oQnC`0-nLv2~F)|t_a8)#_?HR@}zZMnE z03FX1j61=iJir&#_Zb*OL+_H4lLOOU7+==b?EX#Pc>WUzYdC{Yg2=oSm})wf;gW9Y<1`VWOJaD-(h%PF@fW97#gM}A7Ldnl zVf2!{`#H=v2GH%Aj>%;a9qYE_hnN+lKtRi@f4eqH_PC4*|^qz#J|0@0CE2 zQZbPaNGPAClMjBmEl1Dmzh9B?*cNr48C^bYp?y6u_Uv-? zP|dR!*Gb~$rTimu={3nwH_LD@WEQNUwmr7p=tNq^rP(F3e*8#rd$uU;3zeA@-Jsy& z-e`uUo-!9ij}wwe{}n=b*7a(^nsfs-e0r`#+wPc85YhWS_WfWqt%`ZyOqo_S|HXSK zTiUm0`8V`5u&+J_18HB@7w9fQB>SrNtRq2w7>B$myz~14R@pC&&RrxXyS6IS()4D1 zeSHzIeZbs@JL6N_g7X90IPmQ)R!$WKV{nQRei*d;{QQ{@{~VqH&?Vk#@9t3|a?a4i z(2x-)vABgeVBUaTAz6BHaRIY-AFQ-6N`Z|<8;B>o9&WkR{1_$GRn3ax)-+maU`w}(=BfKV@F(He@98_*-q(IU^~zU z5$3cMPE#eXyQ0wDl(Lxfp`Q^T3f*wP&#bMj?KOEjd_?RI2nKDaG%REIiBr<&KZsOV z{482IxrwO51x#Bzp#)jHRlV6Goyt-;V0Pf@Aulg4t2kFd;4a7?*0EUPXHYr$^<}}d zb)`-?sHtmy1=z&Pf=>qi#M>W{m{0_0Xiw4`V>xJV`bSVp3S9p#3gq+Ff!f^Z6ebQS zRF22X7X+rssZrUUmSVMWPsJLl2lL#N@;4+2&KAbnOk=^3aQVaOlO}9XTj#S$n)g&b zi<_NYDND7jt&JtX`#CE`_#*=3^OR$Ai2(P*?A-m^)t!EkL72>JxW~|Qn^1y+KmVag z_Dt}|8S8cYjp$6N7fVbf$?ob*pYD93w9!ru&1bXqUB~RXR-VZ`7+1c${M&b3m?pat zDMLx}`LAG-*To_g#2D^cT4ds8?W`1WZ5F}wp~N;tNo5E;#O$O|ZaJMVn8YH2tT6W3 zLzF;FJX@O9!^1V!hVtVfk-`9*!%<|6&2+ro$`=X2HN6;_8+DR6Ss`0-rT5{E`6@^=;h4 zs3n?TkrTUKe}~?xWJYZE{s31=Rut?ENz5cEL_wd%&(9ASn^P5ab=n0Km(M%U?-4Z{ z5NE>yaTTnNJTN_>TC%C^E(=8}5d9*}w7AjNVS58-(MTYbXv1fxO1>y?*M)5vM;Pl| zfKEO0!s^?g@vfc42S7d$i{NRR3V;{|v?X8?;YjjcJW-$iNY=5s@95-2hUK3-*_0XJ ztxwE|A;ESp1XYHa5dVoSIaOFMClI&Xq*NLX&6!j-SjtTtc8rs;h6!gsa8wp;;{cl7Kp4%weeYpl23C3ozcaBaa zq`yQNk|YFG^A~b!b9|9-eX{eodpmqswSzof;U4%?4`QO?tMo4_Zr_@dW?`rHt$N{y zE1oBQ@#O0gbTfVNEx-P4ubi<`hz)FP$%?^PD$=oFhRg9 zN91+}OpDul8FvJ1IuBMd8kg+>KLrys=#b0{N6QW0G#A!Ozb6Zib&u0mZr|~i?8PCy z8EHrwfbF_bHo5Rt%bmTn|4rHl|9|g{v#0LqZS32&Al6jq8*NV*uegBU3mPyB!+C1j z9VYtrxIw~C^1ASQ%JAc*0Z`|oRV3iqPo+C3Ep#rM6XbAIotz!9`R(0=!dEtl0Di`k zi>QSKfY@l1nm{aYC#mw`4S=rNu}Z+CYGjnrq|Zr=MX6kfhhCyy__E*y>nF{xVQ)gs zP(09MR&f&9( z*z~lae6V|r#?L-%zC$Rb!FI27LfDo}CG=8EuuX-T(4t)EAD8+R9@I&tiX48N*ThLv zJj5!;t^i)_mr;2hy7^jfp%U){HOkE!sr}*O{$vZB0yOrV>06f-iUs8LfXx04aw2*_ zlY+RF+83j>Rfn|8y;}d`yD(q$0eZl;M0BlpCZ4$EXyNHVxF_#Yb0R3+xPnV~wPQF` zqDCwR#Qh$3P;uP(;hm{0=uUMj*&V=j5S*y$iXONJfXO%HfEL4xuLrD4pFaE`75>_U zV<1$|0@*aR(m-$9&&v^z|ADxyY;5jJkuvZ(?(}E`rxv=?ztm7OS;;taKl5d~GgT?o zAXU9|S+=C$#dM!1F2vTsEd0}Dm+PM01c3=4+XbxxhtssNfOxw$_?FX2Mm-GYi0l_iWt!7UrR z_bnzMP1ZzTct>BMW1c#str;@iPoCF`jBbdgsdt&*nU|n-Hj`aCQz2|fH75D1=}SY# zCy!g0S%sOvxB#thKGJmG*fF0rIdkWp7+@S*SmH zp*YrcM3DHCT4LwfTF%X$$thYQHdVB^4)jtfBRan%$1`m8dM1jQr(2EhCv}{r z%OTYO@je$(XGnA0B+s{oF)pnYau9*aAX|Na3(aHgu?56Lcu?@{a7&sd-A8^2;U z{RDyR7&%FTmIegX%iR#G0HWRYgA5Uqu6E#^7S5y3w$Y%Osbp%lvPwv_2tG*wiwQ^z zY;Qd4dgpEh@^Ln1X2fhNqbK#n3XCDL0<9!0EG#g?xAdC|e%f`d_wjg)>iLcwTjq}% z&tQBb0eW~xbkFjXjQ=F|(TyPe!PUyQPgSgmEk3;7$jkgEhuW(D%x*sBT7>XzZy*|g z=hdt!vVZO8qAcE(4ZO)Erl&v{czO$8{KK7TIs4Kp#wcsrHO8m~EFKDC{4oalLahc* z5Fl}yvg-)sAf)mA^=P}U6y$z@Hdmv~Uc{KvxYsI~^0|{>U(o2Y5?|FCYLo>%;?8zx zYtGD6Q}Z|Qr9lLx-eM$wKrjeb2xxwg$~fAZb^{yjXU3{|w0Y0F9EYRjKecS6lUZzv zlw(UI12$-$ri34>`LK)jWK2T+EmNy{2UqM`fjj1jC0O!lEX47brCSiA@;vR&L8-gHqw^%<`P|?_v9w! zi5&XG;scR60m>o@NvnCBf9e{I`dF;&7bzOyxUS^04+nWXabM4|)3g&92;1#2G&`fw z;H)$GaNw*TONYq)VCs}j4hkMxn<9B#m^5t0O9ly+Dm$Z4sS24UQ0@^FyS} zYU78usC^yo*8cW9qWl!R>B2q$U$+JZ6L1xb9wj?+TJdSgn$4lHJo0I3afvi9IH}W* z(qYfL>p#!@`CB(tcUSX?xJKcH_E9WNY1tnKA?v2z$G{w92%iTP#N9wowAYW@{9>_23SWQBBL{O(6 zq?)G<(RR92i1}H7C}rM6Jj`1@YjDj09UXm2f9bN-B&FjJ5Hi04Xy@zbC8!9QaN zaYxXOl9iVjV|^+ce-FOQ7fkG!pXWV~-s~LrSriCc1IY$EPcjrrdvhCT{vKDV)9nAh6!gs78>(mj_YgK4+2oH%uBptu#(tr9{_n%tD7e*d{@sZB-{k*_Nbk8& zA(4(atTm%tS}d|B&dArlq3;KGOi&=6tuhayWl!2`2+;5Sx0sVHMz&*+huK`0zI%l7 znf4d2VpJyJ1Du6@NYc{B*0Ta`v~bdZN$H{h7ZzMyP^JuNYnd=4{%Q6DefQLzP^xX* z{@RGG$G?JKHFSJNmf;>s$1d?C-owE0N5QmDaA9K$YuA)N7Nw1IGY_XY@THh0j&a5J z?Qs87n|yDMB&7m!lU95Q^k<6-$AN$!a;-mU+Qn`NduHTozDA&8TQg5~Q_ z>5KuhjkUG;(5ov?Ct|>BTBy*1R~x<}nW%*VW_5-E(uoLY#XZg|q82l%w0rx}N`stgnEID%`>~5K&r@ zQb0mLxUQxK%35s(z68ziK=yOHjWx5s<$TW_s5Yt{e*Gt4>XkG=O7 zyUt(um3QvFt;6$8hpH)yI@qAXW4?#oOYW&l6=-2=-E`pbYy4~H7h+`;Pi*UX!b_a& zk3t;QBjiTci81_;*Jl^Y&xs3DhdL;97mNSaV;x>%jsT)zN~^(_k1>gt-UJ+yBcG}W z=wj<$2Gn$mV|`^l zq-7`U&%!P;4aQ!q4`(dyXPi%b+mS0$K`Zj&@#WHx(t+$iOUv>fi*(Ekg8(64atcM9 zU_lX`e>a&vA3w@ZgKdzr!40C=K)VN|gJWYV{}T5JV5`q%O>AN~c> z$s3Yck?Ru>E;XF7rjiNE6uGjiGxPH&t@n!u&vf_A?Vlna-P4<_A9bWllcL5#p*%lp z34wZgOH0d+rA9YwGrO+@Z~uZFiF54629#v~NiE_*PDYqY8M5UwSr^%+PuM4ett#Z= z;_@%?)D#hES)IgJqYg5G+`-&@sT2$x6B84Ef5x0gp~F~d##Wy7-T!j)b}+fuAs6N2 zY4N!4fswmv1jBD_3qP2rtI#fBDk7VCBN75+o-jCV>xU3l0Kf$CVCv>;&5ltReB$1E zzs4GUZ$}%vxfF3Dj?j)pS`lMP!dVgf5&lukDhX#nj7jrE6T2l?`aUuZik=h62OU9w zA3xJH?_}M=zu5c&Hq0KJ9hC%i!G~2UbqTxh5R7<8Ft{yZRTy(ig%8vwD)C432=r5jQ zp_G5Jc6L66Yt_22KB~_`#AP)LGF&JJvX=}Ed#Hu9Fnb90v-StyN0|0lRBe_b0>6Ak zK1$9=#@QI0n2?#S4C=L=x!(YSWFveq^Z%}G1og$9W1EeK7@n!2oD#RLABTj z?1l(K4RN~N>aWrUv>Y*y-0oYBXP|p44ylmQjGAeQ zY5{9xcsQ0tMpo9!%8JzPXSU>?5%Bi_`mIN+;PH>TI?sz~)o-%CXCFx)JbXCN-@muN zpH+TfwfF~HvL_$%`uzNoiB0ks>V+eaKF56df`f z@6RXM9x;{UZL*w#lmaSYgf$rHty5Cg*JV(e!{!HKE6ktjr(gM|wOlB~k^tI-VjDU- z`qz|RkRK^8An`}nw+gA{&sNTtpK%L^&+!s`%0k&8RXdV3V{EpX;3s*kx$#YT%Q!1D zlX5;gyj@yR(E-Z*ICxS~@1w4+uCixpfTTWI60%w)T)!Y@4TG!=Mp|ozatbv%BQg#1 zbKFewS>BsQH#zShVPf+pp^*BtEMvBxe1@S#Fx-XAhN7h1Q5pLI8JniEfL)tAAvidY z6vYn_!?k0%WZw5=iapq2oBs7G$$6QeVC}fm#h-_}M`zy!6h!Yg~wi5DCD1b1;CdQE!)4zb?*R@LJ8*gRO#KRQZho zeX)gK6uV>n<@!bPd{k9myuiF$71!_d@dbk6t?>RW&Wlue@-K+D`M`=L$Pn3Mu=nEJ?W@lAZaia^ zc#H<CU{V4d~5hg#A$$FPT_)#3*<<3nZa?l}3 zMvfTlRh-=t@#HZ!wv8(+r#H8V-u3U6Q&1=YzgZ~E!)?x$oxRz&D2RP&m&uE<=^nNu zigj(kQy(rZpNE28SaTQkxKx6Nw}?fw@jvMxF`jY!j4&`3=+r$5yOBmP-YQKT zf5cMkpIU8a^v?D6AtbpqXq(@F=qxa49b$}Q+RoHjza;^gQHYS$&6Y#+ChLp!7p$@u zX9XCf;j>)Ht@hepi3Bf3hbj!uC&jbhJ)D>Ty}g&0*9Fso%Vq3z7&aE64#rds=|Z@- zyP(dGbAMDpPWnM^v~iuOx}AcjMAfb#JzeGEngvq$=kBa5ER9uOa{3piutp$w5x`Ms z^rXZRq>0J_+ZW!KA?IVy#8@a)Y+SxIXu}E;x*7`J1^ZCkt^*|4Utp(q>5UP-sYvYA zO3ZZT3m`((e(wClk+2C8b69g=y?S}_PzJ@g`YEYabU0&hPxQZeOCg1leXm^G3pR_e zrK`dxH3-fhlS@~A!L-PG%k%+*PigLp%gfV{_X6?c9Af50o7316t1TB~4(WO2Oq3oR z>^L&AcL7wWyOv>YOw3j^LNPHF8wTUNb_xC#`%D~^dydkn((>~1Ut(fnqN2LM{6bym zv2ye#N;<^pfY1RjHGr*)Vrz3NQjeFrTW&V?$x54$+Pvzw4VShXUUxox7x5XM)xL1> zvuEQEPs-e3@3{Blm*2Ki*ZTR4zJdj3AsDMWGnb(u$l#wKG zg$}>m5j?HHE2@Ijx?yF%=4jj$SO?&}O{K)D*AfE4YdWo3URKt9kqis}-_~HG1P-5W zI}LZWJTKXm;vypUM)kt@&Q{aO3BNHDhW<0DF;~mZ{Q1)cQlb5oo}NE{%0DNI{TiDb z9(I_m{|I@ACOA9gic3f!Y_$Q{e-ZR>3v7tsrQl^U8^n_TV`D4hXH_JdN`JZhZuX@< z_3-5KR;>}wwWiYMmZ{4qK=h>+Q!gN&nM*dA?dr^y?x|(uHkGM0jG)(P)_drr`Ydtf=MI3HH;BXn4>SH{vNSd7(LJFu>kBv3 z>Fjz5+au&H(6}4aK6%#f^Vl;9%zgd?sXT~!%kEJtYy~Ja0v-TJ+tkEl-UMox;KnVO z>1T{%oE66eLwq*QsApO46WJl>-1c9ERNJ?8*4kQ;=u zu%=>=@Qcfgyk?VpBl;)Ocz@|DeQ0vO4kg2{Fgb!VLLvZByD>JG_D^#a^I>0?fubA) z5!BtD|1a!KTFw9NwSLeJ9rowv6hB`(Wd=w;Od%J)g&S|a-y6dxo)A;JkvhTKY0zUI z*$Z>S!%?Q}1XM^z$n>oJpa& z!6Z@CZ${D_Z&Vtldq&N1qAQLw@#dG}i?i4`NO>eAx;i`QP$ObuI6EDmXj#tzpy~oH zDX{y8VF6N7wcpKy*nS}O)_K9{22-ukGYDKVE@5K0A6Lx~br&Hj;AQ3$Eqt0v^72-a zhVS2pRYU^ZHVpw3RkB+npClZ-`Bdd$H_!HyWrX1!Oa@orc3=z2hNd{>*E*T&g4&Mx z7`zI;|3`WRN(*ekTQ@z|hqDlK;p>~8orLfIuiQw7P^QABT^5A#n-;f^#aYu5#%5;f zz(OjftVaBl+DCd<)qS_JDWb$>m+K&A?dWLGV^Z$CsKH>5)CW;U02e4yPYWt#^`~M# zOG`N@o`K6HCdV1g=XHubq(DKc&s3t^Y6X$h56aK(S=kfLIUH0HiImlhY%*06)iC(? zZLZo+gsQs_3V{nld6EK9BOpu$C(_YM{SsBoyAV&I&wLZbF|Ht%tfeEB`nT%F2mLza z<)9gi`)h7@#Lf*`TQvL4tj|3wh3XYQVMPJ7ABuNRa*xrc%7A>vQmvV>Xj>NSY78Xg zQx>;1zGfDj$CVl%l_Cf9su)D|Ayx#`Q3J{u0gZVg05o?zhDsw~oVJzX1_nz2Xj|4C z4#PV$yTy>h8u-G_2&f#BudAUT@}HuFFdebH7m1Sf9HRJ*V8(4$9{u^ktd&?Ho#>dr z;dv^t*EMD{IOsAsXfrtJa-eGH50-<_DBu!!2isQN+8hZpF6;T=E(WhfBc)h*XiIUz z8}~Hhc}UE9y)u?8L3d?#q-0PAflse_*;FZvUD&e(NOj7uS0EZg(7K%W^a0jC^hNdi zf&U^=IhM3(!P$(LWB^NVxn<^%uAx6|qxtM8!FVCK2nPr$nT2kI@cTrvE5?e0@tmUz8{6h5Z77quL`1c z#WF9PUbjvs`m@_Fh29o8p5D+w^|^6V|6%x$hK%+1^6hms^AgF4?}<6AdS!8Co8npa zXAY-Bl3A=KZxSuWpAQ;0=MM#;%CGy;-g`EWr;yEYB`^d59J0ExS(yf3sD- zQ_1@*CSR@~=bW|lgp!idVzL~%8-(%^D=W*&{>L_84L|sv#2zvCJAC#LC7jLWxtTDP z2bU5G`7wBGCFWQyt*wRsT(=Jy7>Hb!yEtbiYg`?H29VH&Fem~8mNQqw^7WCz?WrnG zdU|wH91@Z`SRxPw#ZBTdl7>4qn%8Yq$r+iM(-pRNNf!zJh$$0eF$>}IPi1M@DkH~; z=+M=sR6Eux+Q<(pe}97#3XYzaXNN$)vHpD51p*Yo_w`3M4j3q56{Y97;^^Ki+T z8w|aQcEaY>X&mbP>0gm0M6Gt-F|dl;znFq}*%LUns4Lym5zAyglPaA|xCAX^zq#b8 z{ju9K8@bR>NNzCI)1gGIbv<*b9MqnOj+Y7T4!{b+Kuu(KoX%*A1hYqQT75z?^n2FJ z)TY7>T#R5L*MPkrjyIZ7YnrM~X9U3-1ZaXEEFr;83}}MG7EObiZ6DESi`CkI2}bWFBTgc;Z4o)k{i$(oZ z&$1H2w2=T>2jFp93n5fK(g(aUMiEc#f{uj-wAM0e#g*J%skq6gIEfU{^^y5No@o`3 zs&=muauJDtRu@A?Y$&PN5AWy*rLce@;SxYalH^Glvc{^as%tS? ze-=w@eJu3w4^TKBwHY8e=+L}{_Bz>|YF3%DNgw&MigG%mTSxe6J${S0qRQA`sQt>D zbZ=YOWDp*Uob_tZR)$0p?D9!mH;NGaGprSQ*R}6i{R0DlEyX4!bsLtgJ^P^q#k?p; zFYxmS!}pZZW$*R<~+r7e*=K+PWf*^qJ>M{@_Gn{k2Nz*pJiuX1YbTwBiztfvm8Ch5cWJ z^ahR}6|@zZu*o3!#&uVRj6UI9eXKgZo{d2_taWF?WC-Isk%Jer#a&)ga;uti^rECX zLe3M)IlbDh2lTWnnD|e9cyCwS>$rTVi1fPK!zVhn#<2jm?4EVm{x5Kpf(w6a6I2_c z>2gn?Es4)nF;BVg4f|#iroe?2@Z(TY5w2696R@rY{ z%TMY2Qszz}WQLqx=k%k%0sf$WaToFJ>-Xea${qS?+Mv z&05a~7+;ZkU&P$vlaq6($vKf#*j>t*2ZCyNcrTplqAB$O2!9Z9B^@I;BJ(ggnsLnGNjWeW~TlC#*AY3I#XvMJJ^h zWIS*_iuwF`cTs7fRp4H#TEhC>jymVF%(AjFamrs(73M5)R;S&{0-`RS0aJak!;=bR zK4{3^mxTf*gV|6n2(7K=d~@KY=>oHpCv&h)ef%jF3Y6w}hysGVQKu3T?!PCl1qp;V zvC2!7%Q;NXmt6`rb_c66KD`LO%ByTSQ{AEykWjqbROqi|{X5qCDva)_&vG3%jZxkT z6nei#D_x#8_MoKt@*sO^Sogf#3Yig3Ig(sP7(HlI;j#_e_L76CS)XasfX73>_!)om zF0y2ecUQ^5e*~2V2M0T0unc5Mw2QDAUzwVgC2N)ctuG&&ApObYb{ZPX(X1 z4pxH&M&C~taA05qF25K3LqL=`;V9w98;+kJuym_fGp{)=Fu8w4CZV@=b-BZ&ak-E` zA(;ZFNyHfS=|xOF=xz1X2V&P@&q5n8&4t)DyzN#19asDhX63J|DoRRlfplh!r7SK} z1kJasD%hB+6(0W$%Qd5Q<7Qa$8?oY6&UG&--j%N*o(3rlkJ3jK0HVel8e`Lh37Mz2DNcqfa6;7%pn`VQ z`SCW-)yj6|f7*Z1a&VNs&wX%I0hIBSqgz#xqPntpF5SL`7wlHs#~aj2w}-7hQK3tG zD@F0a0+SG>3D?^1Lkt~q8->Hv;+uuR$bRW=iKkQd5BuGg7;_vHH=#KiO6vOyr(^r& z8C=fJj(o%^o?(yAHA(NgPeO9TFIzD2^oML&x_Zf3fsK&P6sCQp*15f_dy#_|s#2aY z+|7>{WRQ7@3$>I8w{g82K46_KCs=N{S~c=c3*Z3dkvPm>Rp|j~MxT+5jSZII+<<@k zh6@^oFV=XSXPrCdRd52@LCVl~c4UA3-RFbL`tu4D$l8IS4EGb~L$MlZ{SNL%wjF@D z22k2aPoan~i|zH_q1yy7rY*4r5nU5BM=@QOqLi=%b;QCBn6^trx^Z0mcc7MGHDj;l zbg27E)IawT|4$$CQ_<_8$&4N)gBVTN+#*hFt^7&i#*)4s;3rEp))4RO&kV!~WJ_C8 z=LJ`({36pjt(y0Pk*qanBssipX)Hd}l4|q)UVVB>w^`dzd^ZRFm0a$W%6E@%8P;(A zV<*^y_i;Sw%&Rk4T6SZ%eVsB}QCykyBzfo`R^u2vSb_&PH(tSPaJ3c?5TG>N*Th<>x!wF+3S4TK&<(FbL$kI83jo^o zA-B2I4C60Tyw~>_c$VXBD%5?Xw=mhodVDe7Hv46ss{~cvRrTowKKcaTQxH6`vi_PE z(W^S^AJNwz^Tp=O&z%4&N}3nWc^bT`q;A=F-}oUKyPDqgBW>z2;P;3nOUNYubzb;Z+~5 zV4SgEoT;AAEoItn!NSp}=z|qQf$M5ZSyi-rkBc8|uW*S5ujhwoy$Z$bQ=hfY=yiz} z9zwbANesgYJ)DFG(TXZ)y135_Wp2_vz-ywT`5+vq@`dLPG^_757YQzD7gZvlR}fJ( z%*Ztph^8Um{PGf~DYJ1jM1g-2iP$#nF>S!qoPhq&u4UKH`^V3&2ewSGl{w>|9idJ_ z(vRjlE2yZ5D>3jVHQyI&@90Pb#kgb~az0GoxyKLzH~W5494^1q7?MU|kYCC+*&9S5 zZH@BT(VEQuW_ev^(>Y#`LBcD&<2IPx)z;*BkVfY#o_1dpfAPDwQTn%ox0m(RidQ9t zx$g>vKWPxLJFtg@I2lpr?r%+<)$R-*Bb)PBQfO(-K+rw;a7e&Y7Kxnv$Ip$kkZal| zgVvWJv8|X;QOulove!_G8`8as$vhEY7_t%x?xRs%ZD|NlJcZUbJZ(r7HGYk!2 zq8WZ4wySA#o<#De;C>&+r|@SJ@06VJB&LD3_zCHm(0HA*7~IEG(fmRQ&C8 zDMu_lx?!u=w+6H3J5TQ0kr~P?OvaL$iJ&ChUy@EW6T$cI8*^)=A8f4~nyT(368L86 zh3Wu;J_f;6uu5NV&^vF3wjygTTLu5&@n4>e)Di!nDLI4+&7AF$;8hsAR(=QZWC{L# zmPO;9X%-V>A1txbNM0B1OZ(9NfR*P@=$!b19kv^bkTtxGXCgYJ+?Xw}3ik=y?Z5X- zRzoc8f&Qjs_`&V|vj?eWgPStql7{(L6Zv@>BOWf>4$BOED9ZG@PPVG_jT%^NTSYTk zS9A1PzjvtQ@f!s8ezH+%Am<8?Dh1D;|6psEw8{6yz6f@GYtwIFI}_kFM8+~rYZmc% z(oIF8q^3*86Y>nGwg}kgBIBt$U24#v@UxyRalYw$|Hryx=}y}|q3 z56}DS-!|edJ<7ef^i%P|TK;_x@%At=35y#10=cGFh8d3xgB$g~Gus>gMCKPR=e$eE zOWjy=D1nLm_Z2qr7xVS*r^-mL$?LUc`xM#DW#Y_9Z$C1Z=~FBj)1BD<(R{30$zdEA zE{0r0jU9YnM~-|n1dD^+OCnU8HTLq_WBcga5C|nf1{OBa2|@(olZNQG-)umnRer0S z#XKwyirom!>wGXHl-U8p9{R19;tF=w+z@6$5!S&j!!wUZv)XEFUB>&T_$8dg49l2U zJG=Sz4`1z6CT{EEoi1G}?_ZXFrK})ze#fOX>d~rw-A5gQcZ{>}$ICO%)n4=bH`{#7 z9nrA1tD1|o9H|GC0t1nn<(f2H^a8ssnoFB40db9Kc^C_*al@WU%C;jB+ep|%l8?a` zOE7zYn}V#hfEb&vO4KHA{>nG#9FJzpV2bOQIe2r6fmP($8~up%pDl0s1daAsNiE2R$BwrTm$?DAiA_wbG3<(mlWYf7 zS9ke=Erd`R)Se%)f(!vYn0PK-yX+T(k?S21O``}$(TJzK zV^l5$jKSB^z6Bwwl_y?5PtqP(?smgTvtGZ|%1_+1t9?|C-UW%RhwAMA{%D z7Y)}{W#rX*Q<#{qO*R(Cqeep?n`|Qn){KRX)vE!ErkkXLqG~vlFkBmokiWIz+Cf=h zu(%|!G}}0LSa+yb5nV#b@pOrcLI;(T$*T9Go#jLCSy?3imro+JBVlRdqG+H*!Oy+9 zk%vM+r^t6zRQe^+>kiF$|ASXd6KTj(?)12Q4_Nl~!v!gy>xc7;%La6a-^V*!7}-d( z@I1YPgan$zT}IE-4umQhDsg7h)u*Q`$=lQqODqXOTOi-snkXwQF2?yMUxIMn!5bA4 zg=A1x6zIFSoI$;A$4v9KRvR=D*_f%D5w@E9UL1IL*kI4Gb)H4VsGK&N81n6I7U(Pm$%=rI08SEMivgDkph0QW=6T5F8+2}H0jn{s7Cb+=k z>INxJ`0Vat%z}eY;#Su~$!!}&k_4)N#Omh`dzAT=nK0-B9hEJ^_3ngbaOz4JL_%sZe1eit%+njP!^Mze(KTl5{iA_>B zhLTN*HY%o|xW-9gTj0!zL?9lp-69s{kdnzxC>jXH;|^Sc74%G5)!1?0Jv28@%a=fW z9qJvX9PfXv20WZh#ib+GG-k)3A5!nJ{xsga1DiGB{7OS48Q*i3#-*=&#gD!xu(k{p zKf;TTKlhkm+s~uXbbZV<`pmoeR`p}CBpO^FxVFb*^{LafN=qM$^&d_ zz>c92>`0!)cNkyDO#Pipw7&xP(J{8GY|bi<=H(c`&i2u`TY zxq+0}+}OBY;oCx(uLrrxgU)F}CN}5Mm2HOunCVBsdCVI5;`F z+-cCyd*OGci`Jx^td**!l`8*fTzGl2M>_7wp%!(6iVV9`5g*I^yQnT@B zCaK-0bXkqI$H9bElqMU5KGHQ8Nog!ur7Uk`p2^6+%?VmJ zeHH0Pa!dygAB6_ZBMV!nr)A5?Y~a6riBeRd*&y@l^(#Gh@0I|s)*ls`d`_vW6=`r+ zVGOO_P2OAAidiV*6AV~rKoQ&f^maEic8)yAX2A(#0r}TYW%9^utBK4#$qsHDDwhyX z7N$8yfb=n|@{T&jT zb+W?`IjW3|Ml|iRT#SbXy_C$un}b!nC7K?d-2UdSZj9dZ-qX}JMRA!p4wCqNMAV->Dm1z z5#7jIa?JYZxSN*7g5!&30d0-?dw~|(Sokmxg{uP7CaoK2olx>M`lSmZ@z~l>;}?BR z)QzY|xQQNe4Dq-mQNxbOJLbP}hBBo&8~GY74Ewijt*klQf~1q0MO}@MrcNXJwnZV& zOY8ut2W~-&kuZIoC@@}(reRKANhOS^sOjD%bUm(W&Pk;w3@$m>NZ<&Yy$HbhSdE{7 zcbjx@$VxMkZ?*Spe1XZ9a;YOrSuq9Ci*FK&xq%{U#MjQ1fp#dIT;>Ew4IAAK)U4!c zT=g5h-HzjW4k&l+iixrX-l3M#XZyte7Dw`bI;r}s8EXr}!1bk$(e*2rG=F2)|Hv@r zpo(rCFc;=|K`uzm%xnz=hIi3)e{(;^97B-n3|s*CeHM}p3a#DJ9Q;NFOyEJl02UgE z{=&>__f4zpvkX3FRTYhwpr%)5{?A% zC-V2pULl}~rJI~ra@M;8^LLfj!m24=q4T^OhYpmS=*NyT1<|J(fB1{ z5L5p4YxOg;#k>MP)yx_!4jsK6#^mUWpIM7*E10k?epqh~iG3cTu^97%H>I$}ZZSEh z!u4&EXO4)qmNypJbKSGjoke<`S{WhT`uvwT&#{C>K52wn(&SLsv)?(~YjioSL3U29 ztQFMsc&F*n@mGecseXTFp(6OCd}FJ1AM4d>D*Y4w-ke2M`%svA*`~!N6^aVp$7SbO zQaa@?ah5jm=y!r2AUDDOd8A;_6i7VZRL)%}Jk=vVOA;qKQFE@%4o z-RehL^7Bb6tlM~GK3qxSZk0P>mR}O6$3qDR<$pW$Q3hYyv6o>c9E9exw52bfh;;9v zzdOQ^+A3`NCUE&zN{V;42|QaM?YRd1AVNe4B&4s<7#O&G&<1eOpYcmvoHLM!pp((c zGN6F~>-|>>@k)y#K{tXK^->r2jMPVd=nN*E!5DgpuSCgl{My?R^rcEa8;w*U5AL#XiK#F>+Vi3*vpbR`%Y8=c5^IDtl`XyUdeUVVt3x(waU7)s=Icc5`=NIG zp*E-z4_ik0)+8}s4jw;9}@PDjg1bogVBq^K#@C?rL$zLiTfG zs(&G0%HHVRFLQ4^oqs)H47-e7QQ$YaKE-$NsoqBC=0xWZt>i)Nl+UD=+TpWSZ+X%I zFD6q;r=g^$QOYH=k)m^0Pp|9kjq0Je8 zncE^ADTbEGmll{s5Hy!dWLvV$WgxeD!%1mGsoe|-9PN7F<3M;`ZQ($l|H`c;e!?e+ zEcB=Hz>yPj@BvH~u?I~I2dK%UPh#hYp~EJ^fdIP3p8~5S5%L$%^7HVVTY8=+BqwV@ z(b^4K8wXaVrKG$Zhl=F8tY3%OQc1kr&z>!TkjSHVOA$gLVdy!+e5M%eXVVV;0rm*Kt>FL7Awl8yCE(u{v@ z1d>_5?|3)#982=^=6(cHCk>-f{sUXZ2>Qw0OURPD8rYN8O0h;CuNM6}^%- z_-#tJ%k4j=F9-A9c8v<^3t$o-;;|I+jX)s1u72MtS%EZ%Ui8UyHz^`b*P1|DNz|66mRsK@uJ#wVQi3d z>n5l+di)($7ekJG!=2dTz}RsMLa%((5RG{fEP+3?T1Pj$+YZs6Y(5?}k!F*4OmuEk zorLy3F2Ho6%kts4i+5>5iuucS$)vAOMT6Q5nYeJgC78L#TVxT0i-D7XYczh=adO5(H7{W|zMo*SZETe)W)Uu?KqRqg zu`Rvi!mPrJY*GKi3FlXZg0~u%II(wgIyGhAUd8YzKTnqgk=j`Hw*(V3*k4Mgopm{= z_~+bQUG8+3;{PeS=SDI-+IqW{-$KoyZ0lp`>7en(WzUJH=$oDoJ&(2iSe&-%3K+Xx zcxHZ^bZaq6AKp-QFrw}!^ra7&tz14LHkA32Z?HMhtWtN}|N zpoqNh-w_QrVg9+geR%maAS~>7cNhDp|2Rwl8MfUrkky(k?OOQ(g zvIU{mnI(l`vfdBTCk~F2#)Gj#lFmr^7K9;$zh0ShWKYDYq*N)p`SFFU;$Z(4z#Kb> zoN9g=PUY<;Z`gF__d~I#9It+03!;p7Y4?^hx_{--p&?KCZBH!|<}X}q3WN!;t$B^I zw6ZdTD0EC!m4_!MTY_dDS;*a;r=yeyL5pHH!I_>HBvHg9ZCcs4{*uZWW!LVAf{wm9 z@m3eg7hY6O^e$AH6$B{--3oI@A7q5q3F#|aBpC*3gq%H!wyyr&J&K6on>_wR>7eHK zqppTI`-AMxYiqm;wy(^G)jD0$OA8{Ypy;*}=7<@vkAnE-cr~nzu13g-wd_sjB(ct5 z?+KFPN{+s4iFl4l#N$evzst)V4qTCle{h*dl=Fk^wxD3z{vGnO`@%(&HC``NRU37g zc@CyQ=!lM-FZ4>OaAf?xVdS*gU|cBCQ8T&$UFqWlDs9^uF5bfETT1oI&el%1I7T{> zYMDpbH<2UAAo;u0RYh0tXOD%bi!k?qh}{@^LLB)eSlas61#J2~Y!o)*6e!h3a?D15 zv4b(^?7#EmeQVQY&!NGKtVl!5M7w8Ly{x=92d2UnhBMu%@f(PK@^u2C7c9|k3a&y% z`-wXc*s@2dpETySx5Mhp7v|?FCBNbm5lvYt5KGhX|EBsnNA~d(hsu-RlSleKimt_m zPjD?CQLpf96UZjhfy(apm)8DP9==R1-63f2DrtRla=&ok@5UV;EJ-XNQXdI=${Q*J zF1bjlyYRPl!?hE6;*P)A`>N^`3Ea#}7BADPSo}YM2G+k_*-Qw28TR*!ByJL-j65W5 zg2me+*9Aiv9b1r&{}GD{7K@7Zj8Fod`TAICguAQCa&RN|$I(AS0U@Zk|Ckd2^{LoN zlw!yu5`XO1d5=eRnFlCuo0mSj!}Sfh*RW9yV0C#-wtI z7M!mS2t*^+o-W;-X->hsDR%I2^qqsYH$(5MwnS=MK$N%nIoH`ncZ&b0j!$ z0c9{8jl`}G5*Q!@Thyt5W!3eSyQ_;ApTftEC6_X6JiL|7sfkl|zvn?>Hhf}RR!QNH zz*jWg@WX>Ss|7c0&D>T$VOZGFING!$jf@!MS{7x*iGBT!()@8EGkDu>CcX2-J>7U& zJ9&d#3>g;1F(K>Pm%YM?4%uD~GIag4u{;^kH}K=Vps%{OhJidn(5fBllJRB?q3E5R z8~#i7C3YvOPIWMJBYR7N=byH;3IOQql8bxx@Bbqr{g7%bJJRyyNBaC(1sS}oUZ^0k zaPod`aU%`UNe(>mpPP2JjN5_X(ta+wZw14rk2?R?X!piycnL zs=z25(v5zk6?k5~!tm7yT_VYFRO&zNz12-WVZ@)$rrLMu3Tr5b|D+WX`**ar^#7Uf z{(G+!rSUd(<#)BLJBR;%Mm*Fr4M|#&1_Y3txcuMev{4?^?kNRVekNL?QM_S``0vzt ze?Ej`;r{P~qut!dE|(1ZID6~H?RGq0@>@mNGtJC+bi;3HRmHGB{?PufjjT4;2gIM4 zz~$0b`k$Wq3?EJh+;)1`?6BX`S==wXKN=>H!z1|NSKM%M`bs47@(L{yu~u znRw%d`(m;JSWSY%D#ncdn`Y})sb^Ds>bx0=op)}$yxtX_3wUYPNO_Dy*2QK3vZ~O(!7>FOah)Q9L1?LrVdiu>eV`I2)p8zK{TgAeQKE)^cVYlHD)+6_^5FXAA00TG6qRfWZ( z_}o*4{3t-Qe0t%a&$gez-sQA5*f!ar)gp1Ka9wjBU;T8XR>a&!A~GU^g_SjTiw)zm z<9V-)^{<ITQ%UGhmzXPlXEkFz)IS!2!j-dn0VT|b>6f2B-kmSJ}NLc*mjaT za6|SsB}_TVFajP2u>7Q8+5n#f+5*VBqWH(zXx86?_kk8EokQrjM>LN+=<_34jD3 zOb~e!2J(8w8U!!4MqWk5`R4iQ%9QN)%Nv}OkLsUZ#C;J6M$?wPN%yuht%**yGfjD? z`$meC0>NJza(Y(PoRH#9QKf`clk{)nW(tH|M|zEdg1C@hfR0Nnk`f#@4mW(q?=f7A z>`@I)$74@VPs?RV)&Ttv(gz?uPyHHM7Xl_O$WP$;LvS127MGO#6cE7WdSnLT4I`t~ z*`e-b2p>VL{5=A3NukE}?TvvDMS1wS5W1kzd{f{NAH1!Y9FrV$tLY;?*rknNocR4D?%N@L#l-Dvl4Jf9I1Pc#rc!kq~Hz4w$h5o*wGz zUk{8dn!Ef`tTk_|*O2DYRb5S2s16Qjufo*_|Hy4aqH`dqJ z%ROb2yBSJ=tPJGRi}wSeU&$4IX$7Li(XDp8C$Ruh1?w~|9T)u@lp3(fBRb@**@Je} z)qCyW4FV%WGMV3fZa5#6W};Xyy&TvRtkJC@OU$~?;{0sS{gFePO9n zZ=!qqhoT0`v)(%ys`sBb<=*J+G!sTQd`E6>%%nzWZ0bGMuU!iG{E?Ah)Mtz1kJi@K z`)|s2j*qw94dBwtD}OV0)P>;>s+cp7cIfKq>FMgaT<`G>_xA1#t68QlUs1n-GVAGL zq|_BO!Gr*Oq{Ebnkqjhf0w=S#2Q{C`*nWBmAO&e;4RDk(?Uqz|c2zb1E>%94%M5r; z)_N%ynW<4>clhUB1XV45>!B`&;vtsckn>uI<9%sYDqFCYMwn2$oUR3q>_u1O@!*%3 zes)ix+g~8_>4lQ+*|vgJ!C#W=3uAL9<}7VvR~jP zLschWBHyWXz4q0Vkc8wKAQh0Ub6(?lIP_Jew-lg0$;-jO-m8)jxIEkf08eHgN0&&Y z%E8>iVrz!dfZkoeqGkK{^JQ|r7Jg#yHbryHZUdJL6-p9 z9fT^;9G;nbGk;5s88hQ&btd-oewSC5~{= z5RqVKi`%^R!-{9pY4j)Y;4&Si8wq<0r(aCW85BaGRf|tdbO)uzV@?ZEiioNgzufy< zZ{R*^Nd@*269)NLA>Sf`S4`4%*9!Qn54nh@7=Q^701%d{JaqOTLJ=g)gVhIU=S^S5jZ^9w{Io-#Plgt_CLIG;jz4@ZYYFy z3(fY>@Ll4V?EBKe1bTZ)HG*rl*nFe^=LD3y1!wX8vWng9#=AAS` zCI_Bf2sS*J(}1Aj%kMc5Pe8LwuoHVNB7*Vu84NF=u762Rt^*oBm^ziXc1~EB|kzgFnozop%qDI9l%U2Mnt`_7i46 zOLsT)@JS6NlaBKTqruq=smHHjtOB=U;yORCdg_lp%M6%##%Os|wDvdvsQ8Fh5z&4W z1RNxjCyjDad{IeBwO|nr1U4r8%js?~4+LGbn2E_6B>7l^z+VNF0-Mzb*kGVF02@ee z;dCZkE$L|cwC-=&h3E9UN}`+|NVPV0P*qgsF8gJo2?f_^%Mb2 z7>}CLUW>nNqHtHCV3;*$^h|#fG^zhN!KgZ)UDYbJQ8Z@NIC7}M;KAd^utzGpit9+z zp;JUR?^Dj}GRSSw?dw%EUrzvNs(|}Tuh{4fXndKFkdS=a)6>)SrvHPp zHKpvy>bH*jHnY>)6$@FVHOJ6CyrZ*oL(>ZUe?bxgTieElS&=fr6AAV~OnsIN)qI4Z zE(C7y@gT2tf|8lTx!Oh)8XDRp%w@PE=lMxS*Ln0$!h<( z&aK|PtJgZQX;zjj-t$s&FB$>Q+#VCqlTKjVdR|eURbC{!eWniv4TgVzesf9WXKQm^ zJ_@*rG5kgId&ik;AILSZ5bW7qAm#JCU>R|d3|%a0GQ*@;Ta7Kai-#pBlD-R60feue z&0+xfFKr(mg<%IRxZRrfy=l!q&qdiWbcUl{9~(kwpim$f{T~{T{m|)A)y3sy3|o4( zEOW52TSWuqk9%MtDEwLfAx9sa>0x140RWIKPt|-s$EWt(GzHVphCCR z|Arq*ub}ojfvV81#FKb})7W0xfQhgu;#=}8?od5;Rbmx1f&i}W`f?i~UhR0_-Ps8a z!*Qzl9w043K;isiIUCWyn0AL&5I{$SK%3j+qCBkU!?G383p;8M;{N#@{VubYVjDOV z08*!~IGjj%I8`Knqi&a-?2*Mac)r1-l&4$-1M~Vc`TFm~2)H+Zr#5(^9GOp8d6T<- zxoOEqP+DMCxP0M^x%L_5WwwzTiQ(;k`(Nj<$ggJR_@|sI>b$&UWMm8s4F8-DiSa~j zvXeQ}`vPvG12eXd*)*YCOv&m`wDRp_iVC3XU;}Fhse$sI+lQ16~7v((DMbOu@Qb;WsdyXv!v#E(K~Fx4eV zhTcVx|Ag_mV>AIHr|xt)0XX(+tJ>T8^7Bg52kXJv{W9|@!)xl2b;$T=@+wT*9K8ZgK7Fdt zE~3dZdLf2VweKr0BV!E~-cVqkD!0B}cbjOJuNi1MfGcJ(J;9s2vl%9N*IiUYYdV|S zLK=sYCdm5Lfbx=^I9PgvXd+W8xmLgJ0Vbx>ySrufU_yWpIbNKlWQuuX%{ws(Mu64~ z2@si7xwFeI0c|T!@34m4&EUxQa1tY74 zB9K1+atEMYHZRpfVM@5TP4=I}?e8cZ+Wq^g;ExVswbsz}Uw>6U5%Zm`YhA0RN}rsc z3qZ$!+qk;r5QZVNK;WhP)TiYW(K(oMADS7qPj_^8^V=?q15NsLe?=lQ`t{ZaBus0s zH!uGWS#JSWRo8V5A3{QqR#HMzk?w93L6GhaK|(-EIt2s-MM7Fq=?-aW0g>(oX%uOx ze;%Lb{l54A|8;TQ-pV;=pS|~5d(An=m}6w-3AWPHwzjFy{4(Bc9M{Q7P|(mMio2gz z_9*k6f?lqaCHDsaydj=W1IZBeH1n(0#aHF!pq-?pskvGUGN7sADGV`UKYkd{&fcBj zZnu{R#@YV))9gj78%ON=aqh>SXSSJSNz=?21x1>}4Hc_KuKWI*7@_{s+QI@C=jW(Y z@bx1)mLL)a6(;uMk5R=SpGIZ*1pJ^AK+R}-H7$fe_$#g6oVi_~$uc6?UZtSk(JpZI zFZ;QJMNAkdd1DdLy|dP2BFy}vPW|mZmTqBpVBsy+OsC0bRkX7Ddxl!xh*{l9yDA?G z+*<{$fsRtJD1Q5f)*omUv{83O(>+Tb999&i9%$r&?$=tAXj)uJcBAn_ZATOlrzIqQ z>lkL*9!7|)b#a+@f`9#)=vz85u?x_#fk;Kr6Pby?Q`$Z0@-;jxtfBZTadpmvWCd`6 zDiVkV4&?2Dstfc)UjZGC!=M%vY+e!QfvEz}ZSo!`k99m{xqBC)@qyymtC1~S0}iWp zrN!fCGO{W4oJLP*gh#ZJ=c2O|HGwMfx>vmZ$h#u^Ysl5h**AV%j5X z*r*;1YmG{BFxiSVBEG!jj*?%7K-hif^y%IOQ80mcEa1uVO54jqHSZaxW|q0Vdx~v2 zooZ9;^=s@W%Ng$%#+E8-2jVmyFSMe0W!kv8RqGWwc;PB=H3ly_1e&DPhyB{^dO^B3 zbKyaKC#{}l;DT<$k&;ZW%AlZl@P9+~FV%fs24zl2*YFtevan!93j)Zq4Qt0!f=@|R zb#UC+&*$>|6vDkT4p4jHKR9zt#h+}L55(wh14w=~+7=p3%UCKf{_5=~4oSY0PptMhsK5mXkrxunFm-T*yuhIm>~7F2-pJfHKGar93LBlR4;6}Q^mJ`Fk2^G zLNSAzn;Y8W^n+tULow4hO0zK3gzZ5lef$3497Io_Ra-l;Vbp}r z`tL*tV;1fL!v{Oj;5m~@i&-fqU797oPUWGzVXA`u^LM2DJq%IERrVl9l_K>GFtZH6 zHU*9i7LB1_`%>K?Pvx3Tac^sLld1-u5sltAQR!Yw;s$15$k>aE} z7MRb*g1=SV9Mod9m$Mo%aBK><9w~1=W+UU^-~dbfW zwW`M6J-X_)zqNexA`A-7(+iG*_U>*jC1Ps+H!gxRlnjVvu;jxU&y>NICMb2Xd zf||%>r>sov|GfvwD?4Bz0}s9vD_Plg&7$A|}I9wZSeTrWu?x=i-&HiS+Z@c#$=?w4na>Xk2=v|wGJsc34=E)uZWdi(`Y zukTu%&!#_!rgWIeftxR1+bRmR%}@>CZ_{Bxp!>OXSSz)z}6Qs+MJ&+3hy#my6guWxXEzw zLd~y6T*?+ID_F%!qLTu!*i9hF3k3}<&x3VHh^dYvWbaT}f_j+rM}jMX_G=?4`Pmxx zoTQjy7wO!T$J+t&Hg57h&DSh2Fcx#!g#83U8rW%M9UO|6eam#HK&Oq8l9GVnhYcen zBaTX7K`bKP6G_g4WRZbb7B$E9XP{x2o|fhdg)3kb30i5FpWr%Wh6)b0F-H!NmY+z+ zCv#Ae>WDtd;^e;@nWPs~l)<@>7}!ucTE#>?g50p6Dtnmw?sh8V0uGW5UeH~Wm)J&H zeu{i24g~WQA*xxNyrUFOAIlyY$7SVzv(XqQqe{=lSNVyw1_9TmrS`qeO)WObf5%lP z+36%Lp9!{W;^@^PDCfOy9{V*zm5i^dTKa{O)MF7`UC{aQusa8sw0`u4AWLhA+ugWt zuqr@U=HUMLd)kNm0}*XEOJ5qLL{jFXzq80RCaJjAdMGSY?x zrBhVg`PppeXA%Ym#*r=6M|F^40s&3D8_B3eT5BGNSj9bbrDFdc;!+P(rt#^qY#WE^ z(g|wZnfz8ZDyDQZ7t;p+(boVLD@@On4kkwF>1jV78l$X&`oqV$mCCLS1yoej%-c#( zL1_7_*xTC+UD8(1uS1|L>_f$Z(PKZ0wSui<@at!wf6t+rFyB?;wcnR~|HB2CVRt1< za;a38Q5|1!?A)}w=0&Wk{XDb3Xf7Qxfg=o4klIpHQ!`nhEo25^3{T*ULA((E{{4a& zY5;8^-#H@~xKH!Hb`idP`8}?8m9hVIM1P4={_+@u+H23o{0$i2h;`f~cap%#vPS>oQ9*~g-4 z`(EX|`X?n_y<$prxvQ(IZ^S;u#l_Xy4B*^X30-xVW*{~&0bynjZnXO@gli&%6i)|ikBHZK*t+H-26UuNN?`X z?g&-IW#{qvqWiao2L|TL9t-k%B_g#*z|AZ&5;t6SNVQvymSBM;NGvyFcwup79gKxbIcTt6tMLcPhV@<$d#mZOp^9~;wuFZN}C>j z5RMO44KxV<`E7CUNLTZ>97a=Gsk<>&h_tBk$AK(tAAJQX^{_W@sJnrg2I^Oi)3y2e z-#(}56^$9mc!_~TP`LhA@5>g(J-momU1Q_u0Sgv9Vj9D?%&ZT=U;j zEOwWW2m|%5b2ZXw6tWPkx?ID>k?b<{&l<{XLp|I^d`T13$${cA0mOcnyM z9rhTg!!C^ARUX&WcP=`nOof8h=+RilGLiZPmyNXFKQ@L22-fBdef=tZ);v@FpnTFU z-ReZ-4$L%h@uvFW)TtW@|6nxE_vF`Kux-k_X^3vjeHY-yV?X$H9t#CwG^)Gn0j8=7 z!BFBu4sA6_%)wH@52yhqaLpTy{fn+v^kXPKsxnY~tDq>IgaXP;GB0sNhna)?aAjRo zwFvp`Hks`df(wb>V;O{iE-^8&h}e(?x6ok*=v-8%=Ab=U`zQ&G`sjg)iHXY5**0J7 z(_c*Qt}ebq9p(}2&)Xr1K4L*j=3tneOw$8xy28enDuvL2$cyaPb?(i zMk5Y{kT`J0QrP7BM?7-mzrdiD0z?f)i;OMfXpXm|NNLBc`J2KkyM( zDj?=QtX4o$L;dc!Am8zC93(Df39zOc+q3k!I^p8}S5X*#B^io)O>MTn_m*o`ggI?3 z0&vo*Gh%5>hWhV2{V9CPT39D#(GW4p@_i&hJj!7?K|4POSp~} z1Cr4H9sWi(%r6$@HoIHK|F8FG+XyGwEz>H!$4oGhxGAD?-@Ijj{{hE{;eg@CuQM^u zYBuO%d;W)QHq5`ww+za6Ef(CEW$)~5!w8<~_ed4eS#gq8;z8isW8qv#C;u+y{~Uqk zKWN$cLZEyWNT$Mg+q=4RYjbPsY_sabyD9zOc*yXq1}4Yf%_>istigvlYT>eX;vPA{ z_UyhI(A3lfQxp`Cj*3c3o=a$Qg9Q;p;2a$Iq`*S0KWPEaF0d|w9M=E~2WKmf!pymD zW={zircv_4ahR`Oy%J9AnV8TKPoFQ*tm>_y<&(}bQGW2BtJ1>B^`rG#Qtpc+1qN_8 zO992!aeO+gz)1>_;jIS$jfe&erK~}ac7@@7pyw?vF(@T19hjH*)Hpo|Pg6tV9CU5q zwBX_D`XwS0gqWB1Sg500msLf@JC?aB^(wSvY^y4@Y{#D26vY=86#;4qqf){VU=J0s z!w~>{4rjnt~XQ=3h~)YNwG3&!&L7m^ob<%Z86_bog1nXXCY+{her{a9Jez z8O#d76%XDv1z#K@6_8na#6Xl+pQtqCi+3vex+^PDNv$AD*z&Vyx8(C8Bc~e^ykPDH-_Nf0P9Uu`uXguLCrK>nN3~yahgrD6W}6*qlsx_O8*2|HYS{aH z-GF5L?^XKU=}CD^6hid3xn<_paGo*5lPSIq&rOugQfRs>5WXR>I+y%ck$tY6ZbR<^&;It8zO+-(ftPQ1xfI?d zvJi#~UkTfc{8UW1DU@(Cf#WW_v6ewoAzUA1j~J8$m)F_Vl|`<+@f5lOrIs>av=r-1g`4SL7m)r1U$&yc19{>DoVdwb3J(xjZ zZ4)iJYaa-P;j-d8juR84xxUBOO2^q609o4=-hc>31^B;U5!DRXD%9qxkt}gD80O|h zmMSSN4ta3o4CJ7dkeJt-a)ZU6%xg0Ql~vf3k(nHO!gOpe7zV*|JKW*{RXRk=c!G$n z+wbj3sR}F1v%g2{eOl$^`Q>h!mX;v5xQRl`%j>cFt&95Q3gI~8D_}y3gUl(Ylx*jY zer*l%nK4^5&Z9iuEt;#y`&lv^W!DQ#62JsgRkh zvu00ag5F**RJdu9NAV(LWo1EL?im!!#l?3|Sqkm)60c+2BAD=}?j<0Rn^eX&#ng;< zA*Lv*W}FW;d=|OBZoOp@)7|}0pn;K;<^KK3DP1V>B$2eeqm-c~(BUyJy*l~(cj|k2 zooj6V0b!q2MVnM~1oh1V^LDnmn-!DWI|FUx#a_z7v)k4>!=J}H^fI3}y_m>h@$wbf z!61<vR!nrQeNg>lJ4bR)>JneU> zN~|{i2of8j(Y9`lV*ui+|cBQ$b8Cf_=UQjuQ43~)+ zSw#*nPh`*Pke%248%(dbCMy%~=X`MZT<`PN47Y8H70)bh!fhS%EeRC2AgOetW%rit z4|OOacRKlvWSs-i?gARQ#yTNR&B@E_YHu%n76{SeIOT6Lt++*?oBcZT;p;Pg| zz@Yp4_c{2Q05KLD)D6j)%idA-_C80FQb3F;zK5@RWp&jax^qFQxVNVVgEcuXz!xmRpr~6rB)6OI``}U$^$(Fg6Yc-Ofbw^D2pZ}P z*kgM{g@{X=)r^h(04KjVJ5C2K2Kd6bd2vzUD7*`1Qb1ky{x`#nsU|X+2pahjr9s?7 zg<~1{e4m~jF4R(man`1sC`h%g{J^ueTc6X?TDEQ?tpOun&{V3B9=Ta+6e);4;ITog%zLy_@jmAqLW$=Ld>ZNB7~P1;(! zUu_e)Jqq^g^0<#_pzV}bTMF|9+P59x`MhlfR~aeD7)=AUhPel73}4=YX(p9~@A<#7 zLyAerE0~4<5gfHFZk_~d;ODbnZX)9EW9uKv9<+yFHd>e6n4CP^y7Dsbjdw;ogl?U0 z->&21yHD&x*UkWl1L7mXRANLgT>66WM#aXeqY)i-;K6d+0@T58{X zuI9X-Tg4Tu*pA&Th$1B0jlH3dIJ7zxYni2L8oSBSVmtVKw*BT0&U1z2mEg3mQ~?R z#e9atA4)d*H08f}=0n{Q8f;KUj;o1x`KkL zirmNRm)f(;*utB{#<*(^iv`(j5=1%e&+oZV(nRF!T976t9t{M|VzzO9xKX(DgW9%! zdb!%c@Uk){HPwEa{WdMu4)za7|K+bH;cY$YOS*9e*E~STupV*>Te6wj)sPY z05%r18DbDH(#RW5{f`bmNcxyxyQWVQce}N<6*4jSZ}m;v*(j|;G>EE@P&k{;7HpAO zh4MnGXXK|1Z}fZsc-`Q4EDuVT&G%?<3o zmO6ey;SvCfN_S2BiPA;r)VtbL++y4^n?u(=nezCoZJ^R!Czd zyTb%0$H`37dFIJyW{wS)5vYTRpRN$sq*PEYeFYs(q|5d0jF8)lF#nW0tB>Pe8KKA9aE$#x7VPBT_A zAwo@vNepgzDcIS?DgI{9Xlo^mw4EKAlnjcudQVcs&GYE>n49V+S_SEQaS9@B0|U{i)AMb0!9FoS z;z4SlLyrb>D(PD+2BNfOqjmLw)}~A>gS5b7=Y#80^5*cc+_rnM@gcXoqCbjJuUd?q z-2vKL0aI243een#fvOIN67aHs*u%r4whH&g4Y@x7UZ$F_pnwiD7pmO0P_=kI{rJvX zVBbW-W`ne>3dR35`r{RNp@85XJaD11P_6dl$qySW3`7{Vfs)B~$;ZT&_OW+%_6Ihs z~Mhy_sT^8owfRF~>|IQrF*Q7XTa zFtZAkYRpbQNZ^n5gLozoVXEtaNmTB#p@uj)xw@fWF_w(t{JhDjBfQC^;J)4IWU}Yj zr4Ju|v7ol@XDIyjcaV3^TEc(B2@6_n5TA@BhN-~?7AMAgW& z!MjOOooU*Z3qe3g$kq2XLn;WU5e^-9D=#oom9;?+)xf~JC(xc&0o-swst*Qq-l-H1 zG2WI6Ge@Bk^_M|+bhPbAw<|hsCqiEu%b8Rf|FG`KWB4?4Q>k)QZZvsan8*GQYbsz_ zuz;^!FAO?#1RXP2wYLqO7J=^R0pt*r1!#${2Tg24u1|gyepi!Xe z?E(-aDyES_PGVQ*`xRE#cIK8Zu#XZrSaWl!iLtyv5kB>J=1~dj(?7QspNE*_m6KXcuO(mng#(ATIxQGVeTSIf}3*T-a zqg_B?hvv@_Eu&|`;n+gO~;KjU+i@SyhknxtEBNqQN$2s5KteW<_lW3=o z=G(x9j+PhHj39KoI-9=&TNL5Vc_4M})XeKuk&TgygO0e?liHkwPbVTjW@h^GagpNU z-Y2D61=swc0p+QM#f;DKY(T*G__1m=LqnhKf8OOt&EOr^U|Mm3cWAy0yIrjp3Nz6` zXK#ZgFrVrNKa_hw`{hlzDTR_t(1&7y!5dazT4l$RZK`wn`esI|b+dxImODB--vGsV zs?r9CjuCI(xOpgRw}`NqMQa_Ht@PfNK!+gyTVA!|8(EG)ExINsDWuqop{q6NGUo;Ua&t`E}hxWhh# z3-5Zdph=WTAj4DmAL#-we`G`P2BSJ#9jR9F%` z#FZLaRDK(#9_0z|^l~)cPw3Sy(i>I10=9F-x37&TUonGZ2}K81snge36WP|D7Y_Ow ze4Z+|j3_Dj1-YxCD43zQet8xdcZ8D|@~691e1?FX`WvISf`R-JH&FsnVr0ASkGvQSrMx1b_Z2Uwfj$bN#ggKTp5al-_57!8rN+p@*7$BbM5lldJcC?;l3p<__-1 zw$D*a+u0#OL*NJx?`+Qdb@U|mJOcb3&O

m=Sw%K!;T1s5KxvkBIB&wl{qU&sayU zn83-(8sph~Kiux$kl8e!|4|HORqz8Q5mMD1P75d*mq0qt$<7v}qNV$LpZkN|xA#Ek zRRYE=o1E%@xMMBTb7g9_M5N8qLBiOZ&E#wePfG^all0C z55>uQ=rQ&b7-(n-972OkTMh2i1}Sv$N`}uF6y`m`jW1uA>P$(2&>mG%`uxZHJ>vg} z#SV^cxv3j?EQXN614&I?{cX8P7c5jjzUMr**g~6AL2H{5(U(ZRVpAb3$+U`SL8@;9 zHMfEdnZzSz_%lmt2|M5HM}-l@$MYj;Jkg?m!m1!$-vYZ(RDZ8u5x>)1?H?vn)WV#A zbZGj8(?|6_E2MBJtO@Ei<}2g(g$Ybjg3R)gP2_apD}=w|3&2`}E$-26dCm(Ew%S&s z;CK~IgZoFikB794`o^srgfSfNSuT&8uM~3Bn28hM+m=^U=>B?bPy2}`npnwf`K3;2 zG-t44urs#SW2_zBG%WO`Ul=r9CxV0{P82ls{TdXXAJH4yWamd}#D(YcKg;Jx?-lcs zw|r)JX^*7MKqw6~Vpx!W0uYvs*{X1)xh>6J{++GkDc z#|?!JNBbeDn<$zy1TG|4Is)D}`(i+h2JBnl<~CT~135bMw;=i-AAbZ&H;{82cG38O zy$&

~K9LrrFs1kg?ZPuD^_O!k#g9g6!`wG87@ig~J^J9|RTcYZduD1w>j^S&tIw zMt_Ir=A(=Ld4c(JcvSLOjXPD6xvn=&Pxr zBD70f=+f;=)aFePi0RNzQ&W0~TJO4E-Br)%AvHF{#Odc(xa66G)`*NkB~q&G@H#}Q z^NHhagkVq0wtN#*i-4c~QdPI~k4a2-LVG%^Y6B=S<8_391ny15T*UkD$0l}iUTQfF z9A+y6H(4(aOZ=d0IPo0~)gOW)j;0f{UKI(W#x#5cTE|zxKqFB}QG<5uMWR*%s+?Ey zw4!LFU3X3`va^!EB$>gj`SAAJaH6wfu}}TwCd4tAh`Y<3Y(RN|*pr$Qh}U)yi#5Id zDk8Bl7`%TO+?`&3vcIT(JO<3ZG(jh`^OLdKXvhm;D>p|rdLTQ5(<{L` z0r^{qvq9P%zo;H|V)-bFYlxCpin+ZWVoDQ{lC+{hDeVvbIzIS|@LzbDB=n=t&~N3( zZNKelB=dz)(LgjjNT(?;{B|mnfyp?B1`7$?a4FLM-2tSlH7tL`k3n2sx6$IZ(5>6M zhoaX`pOxx#eEnWlr<=)E#t?GEZtSP|Rco8XiM82|?Kp)`S?EPBAwpV4M$D1T?NPUj z?sGe$oaQYZ2(?|v@$aL^YJsbB)Y~3~=Z&X7 zTc(DN@8n<-qv5qb;?A;+#)#;AT}LR2dKU*v*0?3K#SDE#jvp9a=~7$%*DX7?^R zLc%v1aH4@d9d_~&5`lGl%3Y}3AY$+jE^ic-EK2C0Biv5i)4X*=TE`LU175sT%LbZWNW07JXK+PIwTMFHEx*CQ!PLBMODcS1{h6@PD`>Fi^zwD zb^g{`7aNBs=$|0Vq(+T*>IJ{|t3TcpmMJk>ACuP2;Sd_cLg01=^|*Jnhi0lyJ>nw{?7G)vy!Lu7!3GBC^#7oof05%_Vs__6k4P=-oGOi`#tG@}c-^X05 zjlAi`H)D^l!`v}0KjGCUy}t9p02BSUoruS^*tfop9h3WOQ0IU8{bj|6m+P$nsPn(< zD?QRl=yM^}wmFHwV2*wI2&%V|Lr>MN-CFNMl$wY7v=4(8H$rl>RZlQ2^rFR%{yyl% z^B`h~Jp_kH;P%E_s7Bo1J!h_6x=H{0f4Bf^rFspMlF#+LbC=X;k|fsU9$h_~@)$RM zA0B{bhTHsivjFKK0hfKdK~eUKmYI1C_@2nB4D550z8nhg5Wh`gN}oJ6QXff4(>}IP z-N(F{Z%pZuz{g}rnouf7OF*6R@VWSe-ASh2{?g-)ueLXiR|R$*M#N?gU2Mc&f4&ti zizJakiw07R1_{V}9RYmRmnJ5{#%2%9^V@7@gZSW61Y$O@G~3}EVZ9?vx?|{&pbz+J zxj}ZJjDOYq)J*(5W8rP+K-_!l_>!>5*MF?z>of24=SDF;Z+5+bpwKfqH^PB09w_?hSamF)AsOJZ#=~hUHZCcZi+fy zeHbmsN+<~*iHEv_Wv`;UZr3f(W1DN^t_W$ioN-~q zovtJ>>T(!;?2o)MQT6%QX|wT7#BanzUwW=m z_4XeB6GqPpdz)Fxe&(*QI`=aiX2q8QROhkxToaUcUp24%Q%pGfH4nTwh*rSVsTwjF zAd#ZBKq3vy60m@H+Lt)EUvO}Z-MGoFpP}SQW$k^Y$4N+;eXza!k~RTMGm8;x*d}ZL z9O;jXUu6FdSbf{8jY}ooFivV+>rwV{xHEa)?w;zserKVlJF!7|O4yDp?C9u!s_n?n z{I1-EpmID`oF6G7Ey~1YH|4UQ9L4w`dJL@vYJU)c)byvN?qpwK(l9CJtYH2kEqu9@ zdDwO!`MRso3p~3U>JHbO5n+_~e3zq6{1McWYfI9o{5NMv97`Le64;l$#3KDvVw^q)n-k-!j)~%uEfp!0MR>v5Aaq9gRZTZ zhzo_OV0>xlqecx_*L-wCqioKgFW&P-ctAe4T1~j1;`z0Y24;^{U#KX5{|7wjOxX+s z3rWf3UZXl!X?=6md$Aw%HTxxOvla`e=);+;CePCm=!H5Id&QsM+H1}+$>)fPOqL%uX-b_d552lb}CY+ho^n9;WYk{sGJ{JEq3v6CLI!;S{lq_ zW^4((@6{(Sek1U9RftlHm?I?NaZ1tek{Vvjqx#>ZX9I(H)+Bf9G4HgQ*_14P$X=KN zkr6}sLn3GSGsia1Ba21;503EA1R5aHy%G%4AV7rg`F$>Y;?r__dwXy&y6xm#+a#vy zy*Q}BL)XLDJHs@8ONhi_K=76-?&B$-++?AkN?`2*(iSJ*9#gxdH$jAjD zaym4O+Lw2H)MK62zUR-oY3$DE_Ed{rdW=jlV|L*Y(g+!h5325AziaeUIzO+-#Ge~@ z%J}L%NwW}n>kDq_cd;TgUaoNxSB2B5npGkN_auyOAk@{+(q8ggeWyxjVlHXACE?fV z4{y7=tRHgaSoJ-Ty8L(S82AM5revB1oa-aaJB2{2`s3k-hH$Jq;Th-@-?xi2`0o10 z^oZzGcpr!HHHZ0%kpZozO02Ja7n&zOClO-o7M~vx<9=q-lMEgS-4TNfchO%0M&040 z&S{ji>_0vXCW*Xek)pi1OMgE*+-N_5WyX5u4U9HI9OxMR1`u(7|7^J>&U8Q2E;Y%al^mcKw11=Vgw>3f!uH<)*{`B! zfjg*C_dya9Xbvihih$yzKw%P}<6gpxc~243%fKJVS@cp-=**gpXtfmow|VDDXXjrR z-}pD53x61X$)2dIa9;cVZ7S(j=+4&=v@vlZk0l1}D8GZUPp3H2x`Zn$J+ES!DUFO0 z1nfOc@RvSq*t*CsuboFmpB^A84j-B#rgO*H7Q>?kcNxxHZ%4 z+Kw!_al96X$T)Yh56@q(Q|8cK^qxlJhiXbPCoa2BF$OCfRQHhR$rHLfMLz>H&oz9f zAL#Q^#NKNS#OY`HuDjp!x6W)QA2=WgPMzB1yYHy4&?yZ4V8E___m%Adlnc2Z=*DHS zh)o-~YKUpCYxN-I(+Z?TvIcE)-t z1M*KuNLcuT6E<)tR>A;=#=?Thv}Q4Zp>&ZJ_=9T$%#Qp+8_1~us{j&fZk}cM)X(n< z%4t5ph=B%YXoYofzk%TtMvKR#HIqB2?DUCS?a=DWoNO{#?x2!Dk`41F8X^o!$&QeM z0iDUA{w|#pc7Q@yyqc1kwG>ORyQ*5`>&&>Hw|XQM1|Pjv{jd@5mi%sZ%dC^g4<6P_ z0R~?wVklebLHRQ8xcOHjUu$^RG623X(EOG2dIttLSdhbn6*}Lc&SAM`s|zHB0Dkmq z*UE0z8GN_FRCwpcMk4oN$_)Hs^?-O?M4yyIm5mk|9v&VQRTU5IDUD?WEuU2Xr1MAn zT%OW};*C-Z>v*j{nH|vo<~OH?hmewLsQviv#G}W)M!`HZEJRwGfdT%*0z!pqEDV); zj|%u~`QB?O+EiK<#ozn#vCdiUC8NsA)z?FAY(0}Tm;;G~?~^tq3b$f80s~L3UUF>g zX{v_ejg`q$i$3xCEy;$!3Tcm_TcVR`4_A1%VyY$~k@nmiFPh1m0)AqR2h0b>GTKPi?oF64Rq zby>a|BfWU;Lx{koV0cY=Pg&{xoLe%lsDbqbF7hlGgs7AXa`ePiB3iYJS@!>EY_I7) zCD(AOJ83xmXk&vy%zSz*kNzOx#80(+?GedY(jok~pZp>Hi_QGdAT@cIn4QT&B1fpW~TqVa~p>g00#rnBKXK*J`26u_rM~L{kvPln) z^gGhvCrrXul7@?me*-19dItoI=Z(F5{1h*x-^5@&eg_;E>DX+#o_9YO!XeYY17bJ)O!2@O{CV4rzmqQ-6LSLgHSXf|rjVD`c*_woOdlJuD z=IQ&8BPptCJjinvfvg!&RCqiD1O%X< zpaA_xi?7w3AJJk@Z&4E!Beri>IQa0_yzz?zN3pYQpA|>Lv>R1XMOmX;Y~nLrz55~cwbks>4w|AsFC#RZSX@Ff`Wh| zNt;xr$b~0KqqJEJ!wQ2CccLcwrjkbEXG;hD#_(bR^+TpF-(}}{U3_Ks%bbc%2okZ7 zEQha7gaMQGucsU`%Wiz#d4l`NMh|(b2TSQ)-6||clx?l$?I$&+*RhH%{1b>g(4&9X z;0B%FZ_ALxqyEb1-P7dkzJq~|Nh2&`y}nJ0c%rqnk3TXiI;s&@Q20%AGwHE_iomSs z>}F?72u|Qq`x0%MVAvXQw>j$)t+@V%7`-MRhGvwMX4X^IO+B~aCodk48d#{Sa!wLX5#%>25GU03>VWL02?NNZV^e!Ee9 z^Rvdcwf$4?CP~_bY5!%Pb;^gsm2-e-b!m}YmxGN)Q>wJUx=L>?4=jek;w})i;4T)r zkEM@o=tFFSM+<|>n)I&&s+X7mz~s}fo@w27u#Qa1-BNNX`rO;A=foKqtxtm4cSm@k zrHe~^?4HbTr^pBt%(jQP|F}s((`gzke zJWLpUxaQ^CEt;YIj-hlyVWm%0M5i z8B1Sw-V=HuU08>al&nXwEDU{D&(;96$TpO(;8&`}XfBhx=<)?=NS7J-0;6u(wC_wma4M zo#@Y)>(7?UJ`*`sn*>KVa&9lus=i7Yc!c|a29^cYm4TXr=AUk{woe*K9Z5b0>js%An}3w(Z|qH8JHr%g47@@CGa@K8R;? zmX@P>Wvwkv!0HGMk`-^+E*FA$2$`x#3s%RE!nCP3;N+pDG~KSEOqkf?CWIzPiI5TVqiFd zT}S?Rpp)E7?seIr`B_v-6!arO(a|w5f{-d@VF5j~$V3p6xwa_z{Xf>784La6iny_N zxskY*nN?BpTC$F2Jjy0?;qMk?6&_{^GAIzeV|g3$@>F%WtBV=hV}*q3E{Z!lxKcA~$V!f8(Ah zk8ERRcSuLKkRrS1CCiBSppO=&uM~R^sz6Pl>fMk$pvZytEruN2DPUE zBN!WgtopD2DJIuW3|dt(B1>Dh;QJRs2)AlKEIx^l4NeL)s%(v$pJ{5k^^gX47r|f3YYMpid zdx1xNskA?j)EtAiu$=A(J`gk>2{LQBCou6hRKH8Ouw=qYU4~2ei>3E)K)j9~$~t^`=HBxCMdP!WD^V5h++kj9m(1#2b=Lf8>zikwklfE90J{ z%!4~xt;~hjz-qtAYrbin&{1z@GS!VatBzS{g4lT=J@z=gI@v5!PRcRj&4wZF>B~yJ zo(P4O4u?y**brq)ULgwN0{jY z36FA=1G$5xIT-jiF-xOivoh3VMEILY+78<992mH%tMJ%+|8&v6s*4(E2r>i;XA%lp z8yU1R{F(8N2nw*w2&}==+YXfK{@y8SfUDQY?KXw-1{-_(`JDv!xF!~MS^~cpuiR|k zr_ru3U}X>~6RBg1Z!-^<>MTciG{373^pS6-ra z)}))LC$MJLA_UD~ghocMS{=8x)Jpc^ti(k)Zp0A%#!dGFP8D(+ErTDdkL?hs_N0*} zW_EW-tZfa@<|t-=sI{^U-tfi_9VHB-UNLE^iUD_ua0YJfa>WN_gy?Rdol1&EP5TTfe zlhe~;9w`6Chz}>4lU3|&PsaF0;s3-TJE_m5;zO~!MIJ$y1p{?!8=9a;p?~Df7#Z{w z9r4LuY2m^sn7#hF`kZ46FBQ)l218Cm@u;?4l4U;O864nk*{AK0XH1E=sN+6qsdODW zxEXz;z<_}N-YeF_BPlBMEwDEf#n#+6Ie60OXU3qJs7?YreMG!+2cNqndAtIWo64ZVTJyRmcoPe2j8U4s_!!yZL z6XSNj4%0)g$t=E@2e>PQJX6I_&8%9IvL(lee4=i>amHsG{KfXicmA6=8X__XD{$-t z^&9(GOK$Q7`>T&AWVvmESef=pvS&}Qs0v?*IYiy1#6m|60BpyoHK6u4Z&};E8+62| zeC;YDU?&_(R>gJ*X!kel8k~MbD|!P%(nJz5Hl`5d!uX#zg^6E(ZSb$Z>fdL;M-9@U z@t;rG&@jLv@p)6g`w6k+SbOc(;{SRP@>kSMH@znOZBR?4l>Zu-105a1BoV#CpB_2^ z{(VxIKFYtBb<`DtBrpM1Nz1w-!3v?^pl(UovIxY0c>h-jj9T4UQFm`2RT>`j}dO zNssq!2ip1@30sL~&nf@c$2p)ey2#RsKDtd3$MX_HX*qjptGV&NKD#DB3o7RS zzk62yJ_r~2wH+kuY$NLO4N}Syi79sN_5U5UmT|*sMOsIP3;}6y?P%hqq>as81XN{& zW>{l;0I%#>QB4+>FX9U7xEJs5k3d-4ZL{7&l~m6noTBLYTFuK((P$D&KXIk?dE`cK z;;A_&0?lYZ1$h&0gSG;UUhBk8k4O;EV$K%!0cI zQkWgLoZMX0mN~IeBhFsJZNDv0yKJz>`oA;Fg939&nk=5+6{Dbp5(~*3h#Mk)chg7P zfi#e|QvGOmPfy)eaXo;8TwqfN2&~{Xk6Am`x?=S1+JFG^yn!4Iu^zxdeyI)zX@*6m zQ>`mKpLg7WQ-g;%+TLKL`FVL!#r@%+dfH*Xn(p6iys5+1bWmdToN{QbTS5mP0bM=S zQ2>Zw_|y2Fw2JW4iGBs{#5Dwzv#XsyUn6a_31sts7?~Cs!;N1SR&6D6as|BLRPg&Xh8SgCY=HyjGk9u`+;{cxN=e)yc z&FZIPO0(>D{_&t+px9%oyn1Ae;^&O-dFPG8gAJaCM5|md>%uNU2MxyNXd~a6-@W5% zHlMH;67i%o&p}LpoSeLR2RiXc5DjO$_31uGAJje{ymJkk%PcMZ4Q6W=R|z|5V$|U4 zJ|J;rAVT|WvFg>?pJe_q>*9?*BW~gVNtdYW^i^3>HMyUJE*`b1>iq`ljNi#(=r;RY zjBINdiNUrVo=3D+#0hYLKw!-iaRL%3pya^^#P{b-r}9Nw$15o{w*Zea1{etX0szds zT`}1JF9CT=F=)`rx`bYJ{c1afhhbNiwx{np@>svQrd5tTlB;v*Kzl0jx0IHX(*)Rb z@Um(zW3QbmU~0#|xBk^Fc)7)2k~3-|aR-59T_S&VbQ>T^AZ|THcq|_l097_5Q#f61 znleRXn)mr`c%xT70H$AThM5KoTt+^rA{@15wf{UOq3km$`xXz_DxxJse7V*{q71ae zYsgfw2`kD%`qlZ<=f)0jveF-apINT?`fxuO zpV4-3n3JBio;mLhq1lzdL;&&SozDI0*<{eLdpbZ3aX(1!I(!YrMp9n_&@fqE|9Mx2 zb)2D9;Qnjw0vKmeG!mE4mq01(vJO})DVv_fxIGYV)=CU6fTmP-viR$L!EUXZ#O1U7 z*Oaf1Sv4?|Z(I}J-Xo?!AV`Z8WTiAJrWD)w(NRq!-kLtTF=7+9S3uE!byHyA{*#=a zBVyW_EI6;d!GvO9b!9?446Lo0_8l<VUp{ZJOlAW{TvPF&H?)>OD^w-Fm8GT6 z<-AJ8~STSLZzE8Nc7}xxe4<=QjQ6xo`tuAoT$1 zfCvdC12=P{r`$mO%a0PQ3!yu*b|ra_+|fp(n#NvwODij>$;mY!ut!TmKZ8RsT`2nc z${+x#37{#+$^shM3ykg((d!jeh^uPAz~Kef-D#DHRg5tTK@Dl&?>_q=;};LMfM*yu zA`sgu1M4xQPYuRfnM9HTNn&+{=&62CLuEJ)Onh4!f=#Xc5qf8Wr1Gw6f?E^15HsfS zacDbc=0~|*BccdF$3N|RD-Y@yE)@mjh1Hyz8U<)w1+}e7HbLDnxMzVj2gvSNxkXl+=a6$|02_e$u17i=) z*NWKmJoq&ivrQkA)9Y6MImkf5^R4^%m2;_L@eJfw5fd(L%(ycZX<9K0-Buu&3?*Yi zKe)CM1Sxvk&(NrNqgFq>(~XVSKAxq@&bXH1el;3|rON0OfN7iuzRE+H*3ozCeJx@=E5x|pHr=yR=iI}%kI&OOt;Qz3#xT;ks zp2+Z?wgI52@eJLqN=mkNq-Ad;9yTYHnQzoAu!LgpBgb2b5Y+ zCDvqnf)VmB_-A$amR@$RtVUvaIiL0It~>w0-Q0U)e*)*hyslx}b~O+R?~g+bBNN zTJ7&g9n`dBbF>+T&FO(%cMO%xG&3ukZ!Z(zKWyrjD^Esiz=IQ7L$y5FNo+96s#Z}= zs6U?g&`TMpCBq>=S%Ap9><5)%zOZ%?+EcRtU36=G_E2osyxO<`;+Z|QhW>Fae z(YU=8GH>SCj=03cwtP~D?8Zj5Wq+_EK*s7`lF&o&CAoUNcw-9u$&}>ej?a4patZ+! z>PmM&Vhsv+dbB4_u?1Yn^S!A$dfJtvCYyV5;)R1_HyR83kLnDG5IBDwMCsFZ>V1zTS4?Z1e<=9+ zHa3}-PD*LCQ}77O((cbrpPahfapsS50VeXO$JAkhS=q_|nLEJdZYR{gYn9^2D=d8N zkC+xPMfDsURw2kYI4}TaBy)O!IcoY=qPb!LSXH`wrd7A8j!XdQ6r38q-ykF4<)wlQ zX?JOaaF2NoF?lkJobzEgqjK_Mra6V;7)9`HMkglQ3p7vy7BH&a!J*)j=sK>$C<5TY z?$d;jR3h-b>QF*LLNE(SS-(rkkSFQtmx85(f_X-P=XSHWr;7}$?H;#~Z&TQHtg z*bb(to~(&c(uD@#FOW@byP)kqT5t+-3kmn@-#s&)kbtBLrwKDao@>3#TMOx0K49G! zag7%Xi5BKeKYbOW$_?&)7HQAY%IEO6kg4Ceu7H-8zX-+qqAUlME!vXr7a+uL<3?X>0OV$FT&kK*BjNU*Q&3&n zA<6B)QN-WR`PjkIvbHNRwd7}Bv5C&yJYVn8S4=<%kh|O4t0C+%7B4j!jhewnNbmIB zblB|j!D~Lm5AJ{ZV*b^{)Z4>G^(T7fv}Xvxy*>`UT&ZP93H!q-UUsX6dG?$&57hRx#XwUm&RUlngEr@M8jh;i2_~aFUnRvLsZ~)Obu2zu1qI zrTqfCG$sgUHtwEUlpb@}0u*dzI075!#={_Ii^Qn-yDj6*5tI#EZ^y6X6&GXUsR}RCMeMyM z!k33ZNmKKcw$^5Slj(gXaAZSX(*7IdPauLflwH2%tg3*E;87aWs%{)U3W zG;LRL&(IyhO#6yE1oLQ+{)PEOiyZO{7zHF1z^6Amd)DHnv$K!^DFp?yw5K!GERA9A zO_v`Wb}|TQwN>XsF2Df_8sMIK;bJo86#B}B*}q@2?0T9)Z;utH42Y+Scv5UGq!H@0)j#078A=Q z_-GpYT(EGGQj%2ph5$7g^vaMwV&BgOumON^zBm0Dr0jqiQkArf8=w%LU?UcH7)2mt z2tk06PZNC4Gim;;{Nzx7jWnJH0hp5C5DA%tkW~-WZ}`67pKhy(33NGJf2R0yQH&eAouT#>Qp}po8^@UVtXmLev@dP`Ah*bWHVM<+Kdk+(OC*BxD9C_vzD1YB&kV1ne$qo3_GE1`P2x}(`GitI7Fv8s?0 z22`i~&YvtPH$_A6X)N?W*4ICtfs<3_bEqh~JC%DbczeQO86L4po1HZ+y9ZMcrT_`X zEj_@M>=<_X(du0Q*gA{ z@G7Z@GGgCdpimsYOUxMIlxHCJH9?JI@HOEE^L6dQov!3PID{aWt(M%w%{)%g>MPTv z$tZTc_@?<*dXlZ6M#F$W8pE_{Q$i9R34wEIfoBc7Yn792IzMS}r-+DDggZ@Ipp;TX zguf|e<>Ocw<(dJzmO~P{%o=QToLt;xvVGuNfUSX1d6^wf8N_M$1q3P+9D>!z=8)_$ z?P#~nvL>g&;xZM{t97C3xmpFd27dtR9Dg`sgJQS2%1D2OhK`P||2#UH6uI2$VB&+U z)SrY#>KBtro;+zPw0J8iRec)hT~x&d#SYHLGiTpC{{OIvogVQZ7%!w389~~pf+mGi}7zgEp zGroKuJ|mRH)d*dhT~;Ox2p6FF)VRzNkBj)Jyi@jSxs@hyiATLpjmkLBzcxqI zXe4DHwWCuh&|;aaiiKxJpa4Dj_;nr$(3-Ow5wM$=ABgQZTbi&$gD@Oqu7H0QdxxsJ zFJGP-`vgoBE{<)&0_Jop)w!0`ft%86ZA$J0Y&RzJ>1+hN{c&7~$$?R;D~`)eexz+K zWfZjGpL${d*ap}}BdPR_x%zUSAn-!!RuT`F4>*qX3pm#pq9?*5A~5-+{r&w@nB0q{ zCthg#_l1QIu)#iOtWi_g3Z0KGh6BQdPd^5n5ZA#jmhw@;(as%Q+VwUfkr>M!T-X_HzYKTM7wBkD;P*G;NUL>1&-shgcfNp&vtM)L|Lp9#&}0odZ*-x^^dsa z*z1?Cr8!cD#=^Odi77K}!mm;#-D4+j8AVd?-Ik9bv_a8$)=3x zOqM2?=Z|xtjlO_5sylJ!O6)-(x`sO-)Z+!il_El=?_|TA>v7=t zxPAiT#3M4t)biiVKan4x&oSG6064?(E|C|~=3KAW(SCtyGc-8*IOjTPlTY6y z^ql#&`-XE7dfeCrK7sifBRTQHEX2bdEEqTl`s9gaZvE4@^!^STHRMdfZe{LeOj2I_ zr)c!Xi;RiNRzq#U9^KVHb~H}FTI-r~rKcBFa-)KS&&CK6%Yl>vHK9E9WlzE_aXg&f zm!5SI6ToaJ=+ObGvIPEz;;?gLx%(j@#C2n+VctvFas?prj;fwzzh9Coo{nSU+76C- zYiXUew^mdPp5$aJLU*ND3lQP^r>Ci#LUjT)b%T(JCD5gkxWA$ z@mbTvkB3@-zx3@@fk0d(^?igVq}@9bOAEXPK&8{-n@Dw(N_PA0MYbyMN( zFaPWf*m3nOD0&xkal|+lV7e@IHtUStZ+>4b<*nk=qzoE}jSb*Lt<=YfWzTsoimZ8l z>a3W}=n9&pL8WfVY*%L&=aL17cjQ0P)t4Sa5%ui#4_@@BEvc0@^gJisvA2}mO;e7k zEKmB)^XIou?$Jq<+L)NcfS82$N&D8fh1tsZ2bSNq!s1EWZYCzc`&PjF4n~vgC#P`j zCwt%Z;T46iJcWKkG-&4Zb~@kv1uDK>y@ac7P!tteEL%6YeP}fY-`-)v6<=X-Fyb!K``fs%gq_E13d@=DhEmz(Jqa(Z#%F5aNfQywMa?Ir0`;wAz?@#6Lad>v69tE2`frRxl0tAHKop A!vFvP diff --git a/doc/design/img/hermes-buckets.png b/doc/design/img/hermes-buckets.png deleted file mode 100644 index f7b76f8550e3f164a4c7b84e8d2b2925be0d5793..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 179583 zcmc$_bx>T**Dgww;0!Loh5*5X6EwgCcXxt21b0Y+4IbRx-CY9=8r(g&yZhbbeShD% zx9Xfa-yiqX=_=S$TUU3lUh=G`17xK|P!I_a;o#s<#6%(TaB!~z;ox34zj+RvS%H&y z01gOWMb+%!;7~iB{+@MP=NZ7kk-~{V1QeZ<_7=QM6fPh5k5mjvpnu?Z;2D$Lr?2qx zzmA1@3m^DWy4TEL?!cfg6u3V8S}=8mPq1=lm769=p0s}Ht^vpXG|9uFq?^{`uPh{Ui)^EE%QVoRJjb=#^YNevxkxMOZCSAIuI)%El z4&I7rFvMok5QOC13=o`Ly9;fXd+m|jc0c@24?(v}!^1{Em;Sba*#33To9$e0c(fWW z@_$d39o%htJs5+-(=-`2kDb+%@&CP`n)|T7^blfOKc8sbGP9HP@9DN1-+HQlf2YB; z^RL{@%*>`rw2FHW{yken&dbZ|bTALnu^u*IC1g)Sd-d;?D&*&w|Nb1-dn5Dj&rd;Q zJkS39P49Q~@;^7@wg2C{xrQut=<#N^w0Q9Kzv2|>Z1Mh|l=8@SS!DK|$CU34J4o=b zXD(?~g~RESqVgY!86(!jCHynZkar}TgxsPjrzSy-_)wl=M&qg5KGaeo(=@>-qVnSX zWZC3%G(g%k;=e;%CFz>5YIP%mdb#f|Wz0t`G1aW=2xhIoBYx9)U=(ZTtQ(wI!eN=R z`TUJ&Tfa~o!G7bnZ**{N{cF+6#=vX-)E0L1?64Pj-p~z5;gVxKzr5BS8~b{qL0fs| zSvkKC2Lg3`_(6*Pss$Bo-)W0rF!cj_FPa14u4ufBB$(?NqyL=@*^u1SR@A4C0Od_U z2ME4BO>SF(ixaD7fzj$?tcBU=A>UpA0tC{|mRp-jG&j}KX+`4mLL~rd^7!Y^i!kE| zlXjALhq?xW!L%ca?o?(5qSPL$x4x zB#s*j9_gSme(WUY#SR34K;~sLH)>$8_>I-q5J?@~@;wlm>eN-}N)KvHtmPllrzv(2 zat2I5+CuE_6eih$!2Wn|PwZO|2%pYteB0GRukF(n9af?B4@oeXS?v}^j;J3g z5jw;ey7zVFCU$4$Nju8u{u%p}!8U@d*d$r*z7PGcj02$1Z59WRLs^aXvG=F172l6> zldV;=uORQji;}B2D^`7c7Ze{sPht?131p5~On)N<#!L1hSM|@@x0yOA^^dlV`G;F3 zChD?8%pd3(ZCyjDGWNpAF6WYGYiignbFa&|KTufqq(QfCyBsTOj!=(G0>O!8#$RCG+3 z%cpMIA;8$AL^{jgwZ6GK4}YfI&@($V@Pg`vri(oVHz7XnER~jQy*#~5TrTxN!;;6e zg{OxNXxviRZXkhGCA^dx*fn4ajm7pXG5ZTnZtLXIU&-<*4(5q@4pKc9lT zX6@BMf!n7=<=ck(!@Sf;Gi#fVy63)M2umN$?LR|L!mfvXfAP5tjrw8c?=co6DlW42 zA>}%o7c5HCy&p5Z4jH2?@9( z+4nq&_zHsv`t_WV3LiUs8~M!r%`Rs2m# zb|V?`WcHI1J8Vb(Omm^Lv?b#O5(P_0dUUwPI5@56+OV6drkmCsm8_>uAQf1AhK%bH z-RcV1G*%>bq47;fjPkRe7FlqY+P3=@(!#+N#gBe{`DCCOW0l^>WOc05V+2pDm`P_skh{sG#Qe02W$p+;yLK#5(On*Az37rT~LyhO?}( zVtudjdn`mGkbhJ`zY2z&>Kl>>nUzfE5NGkv&U#EGay7MmMfKtrO(#d4!?iG%PuQgu zhIbF{M`?GV&TNhSC1s=7)H6QNHAH2#ZfIWPjbzYrK>44q<$laJDLVuBb11%%CBXyO zwDMssO=<(2tqOHjr(_yx^rbYIt7TKv&5Cbi=o%kNPq3-6QYe~u4UH*rOs@wWLJ1M) zXck$snxaV(KH@RdzhlwI#?_)hiQ&ch4A_`IFdx7Sw)GdzEC(0|fu8B9s+bt1uLPpUD(o6&VR`vOE}Z7q+74r$@QQ>;AzKBvWR0y2Ma3fUNZztmo?2 zeu=ALgLK)2tDwjhAfe$W8AcmNXZmp8^a0Zrn`9w|Lm$yMqUFW-KHXfI?0bV1ie{R( zy1pU@Y^-ln-dm;-_+joG`@(n$8Dgp{a653=>@@IG-v16e48O>o+y`GaVN#g5m)DQ}Bq?*0md|3xvj?&g!3^=%| zH-EK|@qDrWJnyEQ&rZ(D`$Nqa_3dv%8(nLZHtUhsqenhGQ9@+h6!a(^`8ze2E0JE{ z3@hL6rusuITaBy?<6pt0({n3dWXs{*7}zK|+OuBQSn$>9^hMeB->K>4N$?en3Rydx z`%&2#q>oo7&KV6PX5S!{tO7nOeJ-* z)e<*r1I528@RkJ6pzg=fj2@p$e+Is96W@6(&<<~Jcar|NBz&j^D;*L%^J#i4XcXC& z>d3<@qrG-jR?+eSqBCu9_VOFaKm@q5(DnKw7t<2nU6|aQayDz?*`Q3euxLm&Z87}i zBvWN($7#Bp!nIMOX_zGwOf0wD%<|0u$qe7NA*BYhmZ2xjUSdor|4qKsWneoQTdUJe zOGE{~ExmF@_0JZHmSI!uslIuGJ-4a6L){r61|s@#u3~~fmQ(7PI+s&+p`|}lCwJZp zgInk6J7bLEix=IUDsDvC0oQ@r!~xSBnad;*`%Vw zdQAVH+i7Jj4P;)AAhgb~MrD+|FfyvihS0+ijpMm-`H3N}R6q__O3lgu-AZKO#3<99Xlxyyq0;cA9e|6EuAxLdY%W|1HQ4zo}!zU{VOtM^UcO*c={ZA zL~E8(hdxG2oJ@I%9FIz6S8Qri=o%)b?S%|yacKdu@|3Q||&_m@B_$S|#a3 zjY#6R&G8j>s=1Cwp&6ZqKg@+XrJmbpMx9(P;#L`v^!+Pu)*NkrdlX7@){6o(E%W=( zyM^iZAkgZTrMCT#?Z~+*?89A27#8pAY%KMC$8Dt@IWg}j-Q>;g)urxDWWXO4b|C!|4#`q|~s-Y-NgY6&_HYf!(ry9G3MCNfm7j0?1Nt zzZ+XZ-}k%~GFIC2f`OH+MK2jX>IYJW6lK4_u@W<9c17Jjh_=vvj1){yj$3#0>J{9# zSfsyewQ&76uN=cUIaTw{(u`W+dH$}ukn4J?6g9<%?b^xk)&1FQJ-lRzocH14(&D8> zehD*&TTuzdYcdat?kdK!R45k9{4CasElx**&|9&dLPp2Bx3Ku@z{ftyi!tsN@!y8F zzPBUgXsW{7+m{*JaWaS`oO{iPu&nKEyZ*2W501J94GXRsogIF;3CM3M%IvL|g^OJl z(}Z067}SytuM2)*oXptUg4g&>jl)D@#QRO6xku4KnQk5*yKy{M$W?f~aey_qTAvj~ zQ8T5w8jKPbeL7T~flj-^& z2PlQUdQ3L%-Ao-T3MWeVGXiI~l&?t&cx0#!PXb6b<@t~DMNF+49&{P1R?0;AZWSU{ zF}D?I*b@imSEC5$gP7Z2Y)7ZeXwoiEO4G_2el>)hd&yOn_WU4)XNpfV=1`&CX1RX8 zAekFbP^mlHptuI41fD|ZDR6MAwH56`*Xu7*++@iOBbzF0wpq{JXs{Afwzpp!X{c89 zaCUfVOcVRLjlhC^ylK|xvYo2JeTS&aiA1*m0R<5eX5s@LHD0Iwk}E5 z1D%ArQjF&uuI|H5TJ)4{TU_t&zeI1m@5w1~TgVmBH`B>LO+FB4CN_C5Rlj5ZAsL0od?TPGC#C_f< zagxQ( zE#SYi0b~}W0*45IG=<4CG|kMKN%0?ZIL{Z5EF^4m7yl6gw$XP&jpmb zvnMC4v_~>oF}L>@OX>FM^A^Z>np+8+A61cMh(zb~~nVr4+k>Y!Dq&snY<-18iD z{CGAOO^7KlsCw)jCpZ`~fsBW_llLnp*L!aK@Ehr~maP$lSD=8wK+0hD)}%LaZJV@AN!1(0j0dhfr@f1zF7G9 zSKueFohFSp|Kv-y9nZu{OKbwvX?YFE{i~CUX56QFU0KfC=~i-C_F}bP<(e5ini_nX zGhMVK=KM%-EhiqAj2fImyf&TijZDIM?Z1^7QtGJg<`&iC;Bi0!IPOQK4aMj#t zbvt_4nq}+w^8;Irn7791Olo`)j_=)Z0w)spSFTVQSfeTXC@JqCjIf^2z290clRcQy zyUNuD*Jt+$cB~9T7GS#1!m68%JC*)C(zGa5rc;Dj6l~*@#pv0QNu#;61&@V+Gc5{% z6;i@xGG>uYWp;@p7M{k#c526OpTT|c5C9_JFZX5~6{A0$bvwNznjMZv-`3G&tXM_yQAGvUJoa^B=iQXc1hmJF!I>#OWuq?fLDln=reqUPNJfOGshljQ$LTH` z{}b}o>6NFvnaEuPsyh!UZ?myqQ8S~g%>D&0YUwy)U>zT77FH%Q+KYorpvwk^K{Jou zFnY_~OV5(|AAK@*y4%@Gmu8X7T9$@Xa^=&*tNG#=-79t*m> zxwas#n22VGW4rZRhAk{~e*k7LVf%OzJjZKk(6p`zLjq7qBaW-3U zTXO)%RWSE+Xw&Qdd`j6EfL+uD^?6nt0{6+uScfa8Jb2^p4Ubpv^sPG02?O#F%#5bTF6|Tz&;01wF6=ESWO*>a>+#zsk9I_~Vopx^Z;x-z}WTtJQ3wuvK zt5u;LS=b@eSsFe7+OW45#JZe;I3S(bMAuc+c=(l6&6UWjH-}RA7MXI$emgPBeDW8! zqxu>c@$LSp>dCZ`!gv=wxU>trx=Jug#!P0o2|y{Srv| zZfamFiAQQ#l&BnLbM@CcRlv$_nzQiKYCLAEKN(GV?o?8G>SQ?=Y;=~8Q|q0w8j zQ1a)9rj##@sg41;H!SScqiJeY)b^O_7R&YZ$us5Lrt%*!xffAq3x9T!tkXDKgwS6W zIMIpiRv$X`(d&xq5KXH!g<9lZ=&(Df&xgZ9f?-bjl4g~Q;vCiqFT90N(SNpus8$iK z?-1uiR1>c6Rydmhq5rQhWi$P+t65yPGbE8M=B${3XO|GN z65No1_~6wOwpQenz@Wm4PqaggdXFQo@Rp<3hQDwBy`O{cMxDYA%VTv!*H^lPV3cx- znN7-%RuQz)>^*07Kd7zp8o6${=3+IEykRrgqhn7GM!Fvd6qYQ-*6r)7v})yEGlm`d zxT98*moP@SWb7@(Ri(XGE3zIZNhaNyHb0EjX8#~y1aH1i|8OiHyoZjjS-VPst=bZ* zAdfwKev5ILQQ|o!O^v{zV%O5;S9N%5qw+RrO1Q*xw7qJ@X*0s1^cSTkvDlyzB*-5A zN5G*b43X}x$0jnVii>Xhtx8?nNb;!heM5nG8IS|_)u?w-Zgc?}rPjbO7No&}PWJnWfIH2LOdk`&D2(I5M3;g6fUA>WU3=v>>ZGfI2@iYfihIpcp@CQAq9b+rSAH(f zKGS_Rh7=dHT}~!qDf-__U-755;w$~`hpi(M74N#GA1%1l+g8-Vdsx_6QLZ z#E~j=U-V}RYKMjsRnQm3N(^*&Y5A*O*MH1BEaw`6U%d>%m9yq~@7ejvYKog&goyP< zX!SH6a?~`Yes^GxHzQPX+#|UWezu^CT%=er1=dum60IG=l^GBzt$S-rVc=L6y8H#q z!tW_6GZYm=L0^7%(>gnQ9!h&!FB0&_CAV6ng*l7mf=e^@N5F6QD<2z&8;b3>91+K- zsE#5`7LlU7n$hIX;67niuiMu1-*3x!-Oep_w&v#KIPT9*jM6;;xZfaki@DXGp-^cG z#16(gc^cqc$kz6Dz1?Q$6+ts#+5wBF>k7GTm}zoryJsuqJ0a!okf_v9uhl=6f`hxF zcgvpmTS8Hd#Y^56VK^bTiA3Ud>!>M46gwHqTx9Hw#R)3ku4DyX)bwfC4VXK)H0&Az zdbds$(CznMCx!w6p1qW%jh6?PPzlWsRGJq#B`rYOaFp>BzyT7*4A028y}%?+f#%#h zdM^!b%}iB8AVZ@Rw3_H+t0757I&0geKwp$mm;c^BIc$#x^E7dKix&(<3P+@t7LBIT zoz~CP?_j^r!7%_z24Cp^slW^~?|XgqQ@9v6;B@BhqKdT5MF2oOwtfBE3yqM{j5$EV zije-P+fSKlS%^Z+fvrx;9z=sy4T~Smx_d}it~y;iD#YNrt}!|e~N3ny~*58H@CMc z1S#mrAMG}ZYZuLgOa6Ht?n^VL+r?f)GBLp$8T647t;U#S!hbKR`VXe?HX0B}hWuv= z{u@O&%fFR9Qqh0qgQNFu1C--$tNOoLxBK6h4=sB(Pg@^u%nK|3Bl|9{>wm8lDRs^{ zKBe5xp6T)@i=UOg>O)w&19cerSfN)@{z$k6N)nw{iP3R&wW$urKLxw&loIWd-63P0 zEgRaW&mARx|62h|5SOw0j^yo_={UFiX~(6|uCgHIB=gW>tl}Wvo#m1S@CqFF&MWWCvgMf zm@q$Oxp1m>!LZY9bRo#Qt5TTyMyP>JWRKl?W(Sedi(#Gd`f|{(1Qe2lfDCmAia$J zoR-W?lwypC7Ne=aIRakI^HIDR`@NuFd^HLhc8~IuA;iZ+P^m@&B;OAW$)km-qZ7vO zCH;lNF5-1?(TevCZeKI{T+LF1OLdntNJtF1PZPz78Yjk^16`2)t301*4RD!Pf;_b< zHyk9FY!e;SBL^_!*0K*?qFwcH*S$3P4;R3&b|H_=1FtaGTi~W);mk^uG~*!s%92ro zFJ0(nGR^d~rkUi1T!A{>{c&#bm}<%$JL0bL)%JvPj>Sc&#pvT8d-2FY%iCp7EIT0m z0Y5}KMmlON;fle&3f;ua6)n|V9+{YK+o*`Z)6Ja~ai4shkOrb{IOv-v>-sH5bm{Ci z0E)LMAaQ;c-#@eL(Adk>hXCo^LaXZ{Kjy?m)3H_OfeMD$?34%W zIz~S8&1{7L#cQb+YP(vFMDlEFkrnioNeu5ahz2V!~kG`Dg zP)C~;xrpN#>r8dLxg4l#h;sVb!fT=uCN;d5cbCuN?(w^!`s$fE@7l5ZzQGb2!Wi-9 zro~E>oKTj_L|cn^l^j9pXbDF-uM|Rilr!E@iX;Xoe8h~-@l$V?v^wt?1k#1p{wqj1 z`A))rLEkPp7@2tMR0h*)cIQBxo*_=2hr4T~k;wi&-TAZXE?L;EUE{j5B)Z_m3-0r# z{^7MTpc`-qe+@=aXyVB`R6};FNymeGPD=ESl)&7UYBoayk&}dOMs$!`D+)swC1k%V zy`S#|NN&_g!-X$sFRS~ZCY36~%33djhFxH$hk`=}Vu0VOOpy2!HSJM?)cFTl#+s?l z+Hs(a5yExR&p;2RFYKrgC=~i}*?GU<`%Ycn~rKNI_>&y4zP~Mylq#kj7*Isf%#_I%9Rc%nehTcNjogPK0f7+6E5x?tLd2n?s(Dn;WrV~o~WKSFaF#6;8Nrwxc**$rSqR@Rec$sNM^_}Z8`r3t^ zO==?N?vmT@gg4&mb`7XQm7`$Nx$2PDukUhsPx{BVOubTVdg}6-Lw=@&%e(V^d2r9? zU6epFun>1|x9#kvd5O$AOXQkdGB*6ngvJZDb}~Ajl+1?s!=+fkJ=`^kOz%D8`FkeI ziGt0~g>K$kl;J;n4SI)zX9NLR(w;wIU~uL#nREO&ckGq>3vWP&*UTGDdX{R5wQfCA z9rgUHrn9Rz?d={0ssdSA@sfuPnhPkFWLeg1Ke?`XWFYOCisb%12C0~T1Hm%H4@FOe z3O4)Ox6;QG6P=O@8)>S3Pv)(3w`yrl3exGd_x9f3Ot~msQdf*)oGZBKnQ(3EcK`=M z1~HXx6Mpp3zBJQ!0Vekw*WXtTgZC5HmtUGLn>G`!d*-|DVuy?`EqmsSxKR5uIM2{S z2)uXOe}0ETFWo;R(4ol!*4vhW1?9jq#ByAeTw%iu%jd{M+C6>5;vIO@0&A5zz@Kf9 zA+kjRNE*9vv&;dT*k+CPcO2*=w0;tHa^=*;7&A@&RFhSI2TpCl&36wQ+1~?$Gt2`% zMwg)7)muK{YR*xNOWVROQ9#9rhtKp{GVIj0f^MXcixvc;pu#{w@d5nWAgQzlzWBVt z?u&@zctj8=TzAva51Q98ECTRW#wFD)yDK$SgPry`SPjWkfqBUL$B&q2S;Kc?Gv#@0 zt^^VWD<{;4%TQOvq}qm?PaG1cCQj3loG;1mu>ai5(8*CycpLjJWTy#%;KHD-7aerm zot#-ok=oREagGTZ$(r)gaM6`o&e3XIFj%y#(Xljq*)gmii@0){W?Qr@FwMw7K5J+U zi<55bwe^8QDS02nU}?fBNY|a;LNcd(T%KZZWrTkQhg8ye?=bM5^vQGdx}&&yPphY# zUzfyklvn1r+4$>FKYZc*1Il%WzI6(I$OQlRx$ehH{u~s3p4B{=5vM_yBC%rSd>K9x zL5vlAUN`j#aQI7ygx79{KH%Fhxo&^5#yG^ zT=lfy6Ew0rzm<_M4|so>iDmQ!^@jIWoUnO+&lA_qMUj-ufCWLps@wBLW^WEvd*+lN z_4ivvch#o0^}bo%3Gf-482e`??PMQ66|mI>>&WSb$;0J0nunn-$Ml z;+`d`SbCS>=+;sr{0K%d9JX(~>yR5WO?f$Z>c{8YcCb^sHA~HyRX6wW=RE2Yq;pzO zKQuIA!BY3h#4zs>FRZavMP4{4w|-u(8yK_tNODXmm14Tmx;4kHYtsVI9lnpO`L6A! zJ=#hJU*hq(RF8~)f2m)^iv2NJ z!iEJrN<@(ARujjJ8F8Z894Xik=hc68U!ggmWs^75lxC+H->LD|qK8 z#%Ar{3h-4FkUF=DHZhS9=hij^r8E23pXdbzzw47^%#TKFPk}X8W(C#t-}xqF%K6;; zDH@zq_7C58s4MX?CeB{PE#~qK4JKF>IUc{?ng$ih?F9pfgwWzqv*I1Oexqx*$O4}Z z4;xAIitX7J{0G)~2Pn&g2xIAR3ujDEwckl)(VVq}LlpXLMu;=fDm-HP{&?uQt zmobI2K?DT%-^)7%4Y$kvHCM~8u{JC3iHZI$4;77c+5hDEm*eA#re}io7ke`-#)C;f zG8HMqr?z_cZ^qCjA%)DX2>({R1N6V=1omsHo;_u4kB5&BGRK*L*VM#(?pKcwOOJ)p z7bK6zB!|tH+J*8h=aU-Pn+>8N1E06eWd~Vr}UaYHC~+b1f{-WhLs_0VxdA1NADZ#R6_6 zt0$Gw=8a>*ZFwI#d~SbND6H@)h%@0Auihd*iBKW?nQ3M=OLOQEWtp#XFN$&8gEC%n zvEI&DSHSfC;06ar%jgh+>5E)gu|_S;WKH0wae~~U(Br(%9+Q;oZimgeadkJr%H)Rk zNEJnDeUr*zJ5j~O#Aeh}Ij(uxMU(A;KL2wW5HI5quj(~q`pkAO_`Fja&$Bu`as)ru z-ve`Vsq=WF0)u_p9gpc>e>ltKzu&j^!hGy%&j57#sSW?sYIK}S2y_MgN&jf_^k{soI6;0&KyTOU2ke?3XMv$F%vxVX45Jbdye&Lmk9!^U=i{1 zTRzjjV+v4}wL4 z?slb`KkyOCX{AU-Ncpxv+1zq7c)VPV6DJ3Fb)uVFo1%6{8J}OOk=DeFWMycH>3(b254=&)+3*i4kwaILVp)c9k)IOz#D5>7XwH{flZhonq zYeJAF@qD9}^+OR4MtOjPPSr_naf+Ct-P0GVF9dz4Mk9Z{315beiN6CXc)XSlet$kk z;2PCUY;y_>eLTGNOrP8Q*j6<7%u^#Jha62k8D7=R++ksCYW^azfA;7ity$SZ?|p-; zuBRW;?TrKE`DOWpP37K0``e?d2H%lLZDQVnr|FrT1e7|}{Y30k8C%M3N*~gB;G}+m zE=T&6dm)RLF`#?_;NkgxP|w|!KQSN5Ype51_EX+mdu70*^mIX?YpG2x$^i`<5pZXD z*xi0KXhH{00NTdgggMrG<=|2 zIjo0TVkH&%8kydQaara~3t9F7N))B8FCczSYU?GKGm2~NUs4sls*?BGdWs2DG)&?7 z8Db{bGpp)3F%4L9QKL7UiW07ExAy=Lq`}(5ATp4p`I*15u?2h2iVA&0OrG! z!~vSX9vQoZ8CyYL+-vekdVTz5yJDM`Hq|Z>ViphEMj%aeop*z|G`oX2 zq~LLOuvvZ(X1%0WvUW{`opTh$S3nbabKrTddeeY3mVV;aZnDRA=l=_J8>laG=E=a0B$0Hdj@(*OG#lD zK#3>-zpijUBEG&NgbWHNuaJGx)azqjAfFI9)>LcoJt6~xjX33+!+Z~uAdp~{2RaTX zSh~VToz<@y21D^*_(9=pQQ=Es&iO0Jg3Ds=c_M~q@cKwO5*6)18r;pP6-tC3_Hq&Z za{^h!;faXE;dj195s7}*i-SV>j$8{_ewS&tRLMpUR^5>^>fCI&pS^%-A8~buxSXHR ztwr!52|*0*-nR^I#q%Mm7xKm0MknIW<+{t+`?sI^QVd)*Kzsg3(74xpozn2Q2l(a3 zlsH(UD!l`mRf>n-KH?##$kU##M-(KnU5W0KTW72!Ka{5l-oMM~=1`Cq$ZB{uA(Jm2 zRp5aWAE)3l3~}M5;}5*c<=XJxTEV_@;d2%}4l1$p*~xo8v0gbUJ|Q~b5+i#ZA>6!P z-{(^leO+{w$FFKjvD11xQTIYIhq^lMVe`7|g!j9y3~cu%0~*0c4st-DNXefvzE+lP zqF#5-&x`k7wHu0Z(#+iI^mr$Y4~3%oV%sL3;Y0KKITRiPo)&kJH4u8>F9Le2kA4 z1KF$E`O9*%;9@EsvTjgA@~^cYXRCa*A21^{bVS5wCGJ+jTo%%g9Qa!-I&H0SvmpHlLhjV++K$=x3b4^Q{i(r&v;mUk zcS9PmKcX)0v;6RNI6(M*2OHv{)ez?lyt5)u;`?#B22CRd8s+Ff^wW$^S;K&*mGz09 z(al+K2UBjc-3!C<{e#IMObY^$ye7XF)Fs=Yy9H5SR&s($?`-ZQCG*i;^RQFUQ`5r% z-mKxlW{bfDK>7dm#f4niO2@%veQ9rEIY9G{y!?1WO7Y=O3|BOk&6Bn_{7P1qca2URh(8 zWqa8ah-MG8G0?3qbvtY858Tl%3|J{JEkj<9ap*mBL89?V<}5BLk1nTDeC-yzqFfIx zg3MZr69{y!>}z`vrok0#QA$fgl9O`O=`I?af+x|a)e|nI?~Md7o^Xkf{_H9U zI)s5s2}r7vloz<%e?p=61u33S7(d*w&rv7y=6^z~*xLP!T+KZiP`Ek&oVL3ilKDp< zGpfg-*Ju;cuW;`~N6?Qn6e_E9()R@KR8(rM5N`k0k6B13tg4944B*&j1yF>~#H?k^ z(4+U2^;3DfwTHz70cjl2<$!qm^)fH-+q&W=uj(%6&VAC1neskUHk=_JfHa35Za!^T z^cGTFNn!_*JJBn0y{rGhP<#97`j`VhnTiM!br+%0Rh7A&FDq$Nn^pM$%icBOPDP9l zv`hh5Cl@!l0Uz-s-8Hlcq&UYcE)YO+JGwAkw!Pd_J58Pkf&FUEiu`vXkYH_g5# zP^+B83NvZGR4Q;othfr@!1OY7QKqx?k{a1VZf@cXx`2ZLE%*H)v;fmIu1^GjjO3$+ z2HMuG25$yj(7=sepu$9+y4uss9o2e>fp?tH9#mX5YG>!WRVsodBM&`yP67fz-=V12 zyl%6!d}Bmk8ylfeG--O9%HBu)PM4%ytJ24;a^?;i+x(xI#-#5=nJw+n%yrm>h-oFH;$yAAZry(ayMgD$qR??zzIhped;F! z3Z>p;7%gk!1SK<_KB>^@Oyz5l=3mzyJf7B`d2YV-ksof|j}I|VIV>D%bH(*3C~EMg z?*TfVOLXcUKgB%+GMVh^OR__7FHH+Vn4kB~4lCtc&~N-+jS-G6od*+96bo^gH7-~z z&Y9vE>R#1RdQB?4SmT-#;K`45S5Z(s>BBNXc<}*(<+jvxna9tZILN{7V8R8%1jW}V z4|Y&xVTggjH43KM$P2r4ppk+2kLR!DBG8Z^_QQI!^5^l2@amy#cP%IM!nKtF@Fr7) zbdYz!vmqDZyP7-OwuC>2Tn<4t2OVlt64{>!Qr;eDQ!>fEqwRx8~wVI88=j=pY+GBBX_FyzEu$iR}Pr#I;{yAd=d5XitF(?V5Oarej#0 zWSh+Q9>w*{Zlk1pzkAM=eaZ~`y^}Agx^rD#BaSV+PusO7s?opjm11Oy@k7z&Y{Gwt zk7B$wJ#SjaWflWx01`OI0CQq_+BKmqUfscjQ^5pj4E@{BgYmf>9!@RFpVyWob-%@LNN){3-vxn7lbE}zXAA|u11yZp*1GE_X+gH zGfu5AB1XD55D0fIt+8_d78GigJ;C^i@49}BMXGCgWaojgT8yZI9bZ$&Ipj$ZWV*~8 zzfRZHH$TrjteKYfHLfFj5&m_6&smB0C-aOd+f$~fS_@=}%`$w~2D_8R{fk1o?U?d9 zCK!Ay)W_x;{?;)e{qxeIOJ?qw2kGhL$l-symKw(^2MlKn z-ZmtP`83rK>An3Vx^U5GxiPq8I?yeYe-m0)X=c$KXz1(D#&7LFq{`Cp;@4kfD`?!) zqYa(``3%2`u=Xw{xe+Ous_peTDYUS)}@BYX9aHvl|>gM_ST3)m9@PZ5kt8tL6 z()~$OC$les0s0-!!f5CW{)k_skKr>i3{jKlsvELYeY->!vR8IinH_dC$^lj1l60;-^UAgEHDxj= zHi$IEYWL@vYxUz6U)OG3P*OH6E`N(8!w_CbJry7Nt>T_j`l0KE&@Z?6`N(>R!MsCH z9yu7JZfJiGLq>su#SG1$EPp4V&$)5wh#8K~b%4ODDJ+cF{GCh? z7XU1QjedfK?U?R+dJ)h#%n`};ye~E;Ulo&f`Hmdw_R^Jc%?~4}WPb`fs_r*a@JEyQ z&mX=V1xkiLSY!}=Zv(C_m&H3n!w>jF7^e|G;hIKwS4Jj~t|A2X6+j7Ufn4Kc{l6xrcK{(CVI@Da~RLeN3=O`Dc} zeU8pt9JNE|&2!ZyZUbk=A0|T@pyysniQbVrq>`eg+MYkm#V)H}Vl-;N1< zrA3VU+;6=_0-!I~k5B-&vndi<=K`K(A?C@RJk_!05`=h2%CjUKl*)# z)@dNVK&-fQF1aIenH=5uqQK|DD1YNrdFaWuh0+Xol65AK*umU85Yy{s^^Qen@oXGHez@OrXkUpk2rhMV3#iMV@{?3d$J%UV6&?t4uS2C78Cv_+^ z0vk;o?Kl5afI2;*Rv%6KV$VRoy|XV@rm zE~!{K&G!u5{73LsD(s;rVX*Eg_Iu@Z*`bOycKaHA7@C>)l*_f+&5@!Lz9H?Z4nQ!xjj%}a@-4bmlI>-*@RCA@!#>tI$V$x)qLnT?cRd3U{Ew`lR4MrPqAK!gsiROQygI%K5s$ zmct9+4pPo{O6#dD2d4uV;Umq)6hT>&GWu6Nhr5*El`w7*0ySgS$4jOqDKNPISxrF& zLi{ZRQY28yD)JY?E&8F!aJi9ajq{k4kWk>6Mo~iM3IMz-+zJW`fz-Pv@AgQ+;0{=P z!Zx$O@i89@)=+c{KYxqAsQSeszSjGt_uH6WAVU5(gFNBv{n?nS-P5p*;jbAh=##QG zZO($O&Q#5n=)B;KUn}cR$z!8NT+nqwNQ=jn;o8AWvyD-llA2&ik zWoX4v^LJvEMz$J%0bhi4ag`?k#sol-h2{m%QTuoT#6On}5z6#`Zc=>{b9@^p(o5l2 z*h+1-;SGaevr&%@6!;n;=krECYUl-tz)gr!hF3f^l1cvHQ-eO|7rhowW1F^g`*1cCPew10y6%G5WFc`pTtndXNK^DyGQ?k;YX< zaYMkPCOk_f@Mp~Y`WmCkd3y#@yds#+SgF2XZG{00_GYXP{>xWU4J*-Zeo_s4uHvh0 z00k4J{zKZrsHW|#N3o`StcPisAGHRJV)Nf9Rn6KX&JUK3s`^UvVXyq_U0^iS#7owS zf)3ahY$~d(toyOy%Xq_smvSc=m_JxD!R z_@|g711PWrDGDO`-n3(y^Vg9RJ#c-3RCTM@6PQF_@YaxJBQQHOJ07d#n)7Q+>?`Z@ zzyR)a()~)*ZRg&XVv(+lONHYg<;gG1=p*zDYP^i!5{hg7G{^rb^NyY*7{_-bT9gGx zB62@|!GLe;z6yLclsKbMUJsobDJBQ&oyCyrA4}?4ERCLanl>aYyTxB&$7YHg+%M`@ z5g)(0{(TocO1*1tX6#S7(cXcYc{T3zKQ@|%OB>&^v*8~1b&uksxpZ90kLbV7E}!J; z&xZJ+xuTWp0o$#1@lc1dGfGObmH!{IzA~(;Xlr-TQX&mfDjk9#-O@;lf+!^oDka@W zhtl087=U2VrF54f-6cp%#~s^qzWdzox%0;c-J3PnnsfAc-?_0TOt$qulf>AM&SrEA zdA)heRUhWCO#gOoAxz5otDW|48Ly=jr(NmoeX%x^UG|bpXEq>y+?10!Ecs^a4Bg<< zDb>T;p|P7%hLcUA>XC|@g>wwlYb&e8ygpwoFw#vRN+=Xg!ZE)}UF80XZ$^*aqQxkcQ^04?PWy4f&2(KHfblD3hw|7UqZy^g0 zbTZI3wB-}F>o%`+7|AIYoxx}3d%Jyg&AaF^lyQodK7abY&7@na9X4J6D_QM{ftK2V z<_%H?F%yH|G?VK!S#pcj4Y{8TQh4fEHAA#SC+z=bxc{Jjtt=}ZyZm8eT%|{O@QK;t zySc-zbYK6SM5E$K?Wuv(Qhj}xu}sJm8Tf8lM?Xy0 z`Iw#CzKQhhqo(T@2Uf|U?Y0wIZ9bV6gJ%78!For?&qr|b%Z=#m{O|YL_`3f^0B}%A zoVBG z+7~YVtul@@{#L^7#R+wAzmW2@q#qd`IJ;ns_<)AS8;Tcxt&@TrcjKD)OP%JBca@3n zJ`aSC>4cwE$KMM-{&fF2@gud#@W&D37`c}y_>I3{-wE)`_45mri>Jdq*^G)Ae<~=W zq}1qs;b0V3v$$ZlM=1E`I6`l1x8Y2Qu}swQ68ix~4YN@;PjP+1XD-7;?ULwkfR$go~{&vtHKZ6|?; z*=o18+IhDAvE%3D6aikUNd2OEs=q&^So{Z`*pishpk^b0N!zcm=ds#OR65MJ5~e(Q z*~I2|XV+Jp98)XuW4y;`hSLI0j5UvK#^w30WB2pSW_Ao5-z)CS-Gn%arnl-;z3=`4 z7^n~zEAClHyHGJToNk9YK$XoaoK z|Go8pKk|$}iu?c1!=V000V=NO|GkrcKO$c?{qIXrVKJK9bgzw+cKy8kpNs!n3FBMd z@$+p_l$4adr`v7!G^VZ>h|m81%`xe3#=*hi2;;?zfgLs0SFc_bA;{Rj@#xW`v5a3Z z#>R3B%E}!2dZJTvb8mb*eEAZ+eDRXAv-5Kj_>6*rf{Tj_bi&~k#=Cd#@ID)nq^6}+ zWyB{Xy-@!AZ}1BVRfJ+A!}7Q7XQb~_@tIvpxLa`Vtrr$FyX1`!YHcko|B|V$B%Y3- zv9U)vl+l|zI|f2I46h<10}cD`1Yoltk8on$ym>PrgCB;V;EtQPaw7b@_=eTfr=wqedw0S8xh42P~DQY*-V@mWp2Pg{XUQ+H!9+DDjd<`=-{yW++D_ogOk(L)HE+I z?}eiytaSB6!yD8Hu3c(w70F2Y>^tJXCy&6{K`@bOvbE`eAkDb^{X=TvMer`fpc^`Yl&h zR;2&1%hT*ofC$_%N~B$1jqO+2k%QZlL5**3wF}$V;5vUqdHMKmx$*{wWy{4*RR8!MHMo+tsMgZj=;df!vk!G{4Y}*o4ZQnMg3w)D&qRwLY$lt{fy=1 z<$-qxKrCf=-uV09Nuty?FsSQ_7#$qE_uv8Zo(D)p{J$|#=PAU(f<2lCK#RcLU*o7j zM9r7)TBtuD^urSF_ZLlk$c(QXMunI{D~DOt;zrLC1FLGVvO?U zY$yF!5uD4kO2iQ*B_%~_oebgc(?5UtqC7hvn%UKJ??k}8V7R0wBs_d)>BjoXO7z$l zEZzNyQJQ~)dDVmxZ*-_b{igEKnY(D0K_(j~=gFUn>10(4Jw572w@q6k0bx*6S@~;p z)Y(1f@4wOUR9r$LY$s1K%67V5?Gi(wdIq0>fcY(|ET3bPW~r`Utd%z8=H{LsG+g9F z!Q+0i?AzbkO1|~vTPQJOGsAlz@Z{t?F*cSqtg@e#xetb>v$wb2<(GAKabs2} zH{#l4S&@OkqerQ6ad80+}c|4 zg1ADBtexiqe0(yLL%qGW-O0QwCcjr!_7-RcwB@QzX|7)F|Lb*n$V<`=mff(yr^n(` zM{Zg|!jD?_U2$>oHyT;$Ihr0O&ozp`bg;3t_w}(RVqszRWl2%`#)pM<^ykQe+!GQI zgj%o~-OlP;` zr;3WoXE`v|#fBdnRIbv{WQn`SW}!wDo<^p4Q^Fm#@^=KV9rO|&p9>0@%huQOzK1qB zefjd`$B!TIDv-OX9CF&|kD--R`TF%`tt3U2D_dy*+HgtNDe6ijnJyM|PV6*iKGP-Khc* zRJ;yn$IeZ)B-7pp21=;WUNP;%dGz)u5ZB1aNC-75a%4j~TU|(C9%W@^0DzwC_DP{h zv)pKrx;P~zCwJaiP)2}42XT?Zix+94&hNS-S+-%v7Zw)c*?+8?(XZObe>muu1*T4-1xI~jkimxxx0(lq}~YSUKWD(Ok`{GjWF*(kn;GfpZ`9J3R#vRc&3|+W zMikc7)n#YTtPT}UxL&`<;=J|@WuLy|wanpBG(UTg>1w4qom*B$>^#O8T5f#D-n^pT zYNpYDSY1_BwTKBW7(&FrUC6-3c~?bccXv0&m9c%%mM}*?lY3<{_h+=O2{SPxoz+9l z3fgowCm~UdqBQNd_ra5aP>P&MBlgbD=4B6Mn6VJ)@_-qga$1Olc%S~{x+3P=R-lsf zv-q*%CDNxOB?UvPlNbJz!95lXqE44uO43qO>BZgb&rgr;rCTodW{A10BYq(CcE3J8 z`5sJIm)Cy*njaGrLwgwqq2Mu8#hg`XoA$l%hQ9#EkrW?~_+@6&UnQMgM~x4sMAwzw zw&&!*zQVAuuuxFI?!2E{Q1Hcm_FE7>CCU~h{GeQ1T+E3`Nl6L&`A|_ACsjgSX&sLbWi+k>W z$j_IvSEm0X;<0B1R_N+0L7!8%6e3xbyLUTR0>w44bc3f9V>@q|G>5Wlzkzuiz6Ije ziLekOput$O?;#p!VJHAMVc11LK(M#Br>RIfFyCC!OqZ5ik41c0+Hl9)>+Dz{_IGoI zJ|;zx0fP?`39YRu+#Wj{m!N96L^ojEqh7=Jf)lbGER6qHS4Lg36%$^R$90IaR97G=AXH=_-u;wa> zTt{2qiC9&W`y1dmfk%({Zr|q7uiRSB^3C0F6%(tGkDzeo{H+)b(=b5T4Ewl}gGy;Y zOnQ@fWq${L_6~}TC83D;+s5FV|K<(4ADB+qyYJt=c~C87q^4eFKn;y{28h`Bul6)G z`77iNSbaW^|M;m?TT`=f)?S!aT}Ou^4y&=TAgl;bC*Eu@U0Ekm$I6*bBA+fwJ4&p)J&tzctP#?^Wy9@QtMR+PRGW_ zv7%zbEUsg?g8~i=4Uq~+NmJ}#hI~KT^u2g||Ng@KJVa!ZlUYo%g0>TGr-$1yjEd** z9y}EZz4Pzkx3DSbjZmk8x7jo>T4Iuu4@WJt9^AjLl%sjtaB*<))pimOfiwXYU$5Lu zIg85bnk>YV+py^8N6T4OpTC8Zb6#eUOpHGXNuH~s zqGB89qDl@j*V#W3^Thm`wvka*B5JtoF7fA?-Y;1zgi=cZKTg=9Sw{F?Hx@HMHrN~gv;6Aws@QqJ}r@tdpogY;xYo!KSXRE^M}; zmFpXmg)o7qKWW8WRAC@J$W)~*KqdLY4u*!*iU9K z5s;kn@N9zXCcu{cmUnt~2LAOO=njNOB}NT~B@`S|v$L~Or&}0NZE51}J7uPAt*`0E zi@*CG&W4mWoL@z1>*^W~=OPT1nzFl3&km;H;~gCxpreYfX;*(XrqBr4f{HCHEe)>j z5lNk12@2}!yRXU<4S`E)s>k`MyI`7x(@KAI^c7q??Ml8qsr(WGv$^?s&{ptc0}qd9 zzlS`2`0(N5$5GU`u?6lQdpBOi&g+H+&RuAR_(QJRB>tkSyZaeH{0OkW4-Axb?V6dH z-Ad_(#4xW`26g^1sB(GvRWTQ?t12eu=0O6N2o(IUh+@S=(YC1EV~D!VYtw*SkAAgM))iUO&VD z5-nVb9LqEk#PtYSzLhTf=!g@p|a41i3=p8kSmfe;U)5uPcDOYepd z%?SZ~B2Do57`QfY{WKI5ELGyXc2l*Sx+T!@UA{Qqyzuh!TCibyGa{|$41V1g?EigE zRDt4n&w5*NwzHonriAZGb6Xqu95zYGsn*t|PkoYUz#d3QAio^iXdI6xAn*k;msd1G zuWQReVugebASZNia8PB5Ycj^f#(qtGTVaJ76==zL z5tKE~Ghd#LynFv1koB*=;PSyfX7QQd1?%hORm*f@RB1oknZOC5t=ydlJAy<*Cx?fJ zv{F739UZH%WI}dRWS>||2+rs`I5C0ZM75ZLt*_kw;|G=>gtX3UMPRWp8o|>nKDn2Ljb(OM*aQD96*%%0qsBcI zh$c0Qh26HYlPB%X-58=j@?D`7cl!wWcH8_ce-V}Hw7iA}5n)5M!#s9G3(3SR?6sBX zJ_MF?_rJiqEuEe3s;a6=O4Pp<){?b9%=}nZb_a`J;LW#L$0{qb_eDy#CBZ)6E@@Gt7p}Y`a0uYygozgrJL0Z(s++ zutT6K=`hzkSfFxtdK$gQ^*N59fOX`@kH5RSkE@+l!$?{C37%3ByeW-NO#JG;y99ED z$SZa594mKDkXp6gMQNt|P6yvYTThyhe2)(YnM`JTM@P)ZzsyolPuiSr;A!!1drim5 z#WnHu`8aHtx$T4L8|>`XeVG!^1RsZ+Lb|8*1ac)*+$AmTS6yALTD)coN)i$j^k=%k zDBKiG6{Kb4)YMg8M>`;(L#s|HX(6lLgF{26CMN1xk_L|-i-B9g-P8pGrjq}j0Q2e3 z4+B4bNO26lj|>mXWlF+jK!5cfKYsk^(Z2nXBUo)xo2O5+oMiT$Al}9f&@wV& zASS;4`xj6B;mhx{>u}rE*5}eT?lLQKZ_G3=%kb9~5Ybv(*3;Hy(2RRAlh}48Fp>TfX!ZVwH1nC#>plHvHqq z{ccvBbPjn?VRpu0+E@)Ypdz_?;22|n4d(D z=?44A_HknPmy~Qk(2aqP4$1i$%ra5$eF8xg)CHR_Gb%)WWa4#N(c~Y<4iAqEkmjW# zK==g&a;R~#Bs>!}&|f7hP{CeSS05i8iHM5kR>^}Xj4!5ur>yh%Yx8aB;{3#Jho)tw zDWD@$(xmwk|9dtHHV6y)r>0)Nq4drlO9kT1f4(DNdk%~DNgF6{l|FY_6eK0@-@lKF zBHD4n3wrtH@|BjelLI|29cSk+!#W-2OY|x#U0@#bHKBr(dvJRi3Nq=U&Un;*k*hyv z4Gj&qx3{5;HwBdha6OBrdBP+lBw$e?uLp%bfRt~8-e>(6FW6Mskjok1hML1j*{bGG z;-uuSA}*3Ww2E?lhK5rRC6bbmoFC3dNu3{JAPQ0RY7n{c zrv?OIyi)kCXOQqJJRI+GNL)ff3o10dY!igu_}O|$r_@lwvX>UB7l?E~I5Vbhuw=M6 z2L?5i`@mMhK%fL$uv?T3fHq&Akl70DbWE^n6R03~6ONyLSEQjtbJ=+1c6C!z_LVY1Yet z50IHhr;)1st|6cm*0eFbmFz<>I}c_xu#mJ@or0DaB6tc43OH`%zm};ot2mCTv9P`b{1h6t9?huH@vBp@CUcKr8LcP&RiqhGi zKVOH1jlv>8v=rUX$>4dUtfVCDvi?v;1|5l{5!yJI_D$m6IYLEfdqWx!F~ESGgM*2_ z{28wh6aF`#mM10#l{vYeUpXT|HX(4}xs4jTV1W#rGBM7+8<-6kHc+RSO`M0N6e^@{SD;Nd}rv^ z9GDL8{dGNMQLN>@JLrQCP#M8TX;-?oU>g)N~VBO^ooppf)FCL;&MWL9WneZ-^miHhGU zy8hpQ%-H=cl!A7%{F?S8!Hf$)ob%;NGYL6VuheV{?mu`CCK3jpdZqAb#SCg3Y-Y;p zf_W9}JE@dBh6~Hfq>`^s>B1;;)lHyE_mdgg276+6Y+pkWJaMSB(bxk_vqE_qX|JJy zff^g}C0^En!NKw`U+5-ruUebi*o-}GkRfx7VMUA~5rFVY_U0c*8;yL=oT17EDPM}1 zYl*lmAsD439;4S$QBh%GcV@Fu9uUk7l#jy<_iP;&+KDYKm#2MB2gtd>LBSDpKa5@+`S|(s-tO)TS68Et#{L8al4@!_kei}XVyHz4 z3hpm~AGMq6?(CF#{N?j!7PZtsZkFmyHmv(uer@wP5n&~k3JR_8k$x5^Hrd(PQBo#> z47S&jIzs9%7qTEtqu} z+>EcG(%4ZR#Eo0-ieJ)<$D%+FZpC?ZM$Z3(9IT$oAvp%zzz#Q~;_LhL%ra~{u^=1& zX%sMjL(@D&6$?q&K`e_)xpg zTO9o8w;w*pmGyx7%PVVwS)wLrg<1v2linA~$s#XU?n=d^rFou#gB&RZ);Q?O9wZXW zeOZbpKyS`*5h#OpR!jn#M1D>V+L!b&W~1($;NVuEi84Up5lq)E?iOELYb&;z#_sPQ zoD>l)CS@RZJG;9>gM;m>*_soKIXcDcjS`2lB5D`dg^r&4U^SpW00RR9BX>)Z|2pIlu%6K6f)OSo zFOOBV9cIBtf|ZjI=e%e5iNRn0*nOWcs&*#scX+BJ1sxySV&;4*~ z7TyYFPMyDksH&+!;Zs6f+@de@=42F!oTB0~l&tjYy^f$J0pUvc)qT+Z@vqN6L4JGN zA`)`60lb8TK&a?Iye}dmQibY#W~+gNL$xRN_STP#F}1P}&i(ulL`g-3kqdJ5x5dSE z;E3HDKl=MOT`6m~Nj8G&uu9pb;A>#>V`(nDjy*PS!OB2|5q!ZF1eocq7N2m@3i9(Y zgRX(g&dtC|P?0oq+nn*Fqmc z_P(G1%fNk=i&ohFS`{I*zluvrB%!hVS3R_RfsVg*3 z7uLk5R#ul@{Um;LtwMDQ=jg!|CoHDbU-R2;W?ynua9BIl)YRA{aAz3Ii^4ZeK^lq|9CSISA|~$ELrd z2IGwzb0(~i5G_3=f;k~RLJe!{9CQR)FetkW3tH4aJj~<Lx$|mO{yTH~ODw!N zaM7gB*x)$p#jfobcAC|pKDS;*JR{$WCkB1Z$7cb%;z+JIH^YrFT za_3+)3~Z&H$e4x9@kP~Y5Q0D||HMv)@*?P;q`sA@(9zK$c|Hui9Xt{a|#O&4>$g$)bN3z9Q_Mz{uN7iar!D{S=($pmpokC=eD*RVbueq#~$C`Kzoyg zhssd~?9NBVjM8r1YipN(Z-)2`pIi&m3qO4d50NWw5bqj?FL7VU<0Vt+T3!0#$Hz)+ zd>ozPD8u!AF_d|BUAW-A=7MmgEL2ud1M&OD(C%;rwdeE?PN`8f6^QFT%LN(o(jtve zg@Up^jo`B@SDXFKLZMm{ArkQV^=mcOB8MWTx2w?jfYHuqa#%f~HB3vr$2SSh{9U`{ zu2!zYGCPSv2v@B zhhXeLcfb}B5)zt-793T70B*a-loeS3tW@%bha@Oh`&CR1V7wlsjOeZ;CFH0a-nc_R zNQm;omh`~{$N2d^@j?OuF^~C*X!1fP4<}{5>dKBrYaT1^ePl?}@RP1)dNZ9}Qu2DE zWRPxZY^;;j&}pQ!z{Q2SzJ&93tfvQ21|+~~T>Nn0b2O?w3VxO~4~@*_fL6$sHmlrk z4j;M!VdzM<8zySiv#)th-vN6KRWWc*4~Ia}EumzT2et34ZEv=f%U>*7)}`yWaqDPu z59pnr@ZFs5oH|y+Gwsg?Oa~!JoX$rG8HNRz7&|0%Q^ab>E&;?;lTAVyO~vVH#P97T zP7J@~4X37r&I#9nzQzQ!JaLd)l;|?W%Etz)CC$3w$-#&PP&(PDxyjY-8G7#$>3b_Hf;E zg*~nZ%~5`T@R=@|n=Nd+0>A{ECYhXLvFAusU0Vh`O6MM;9eYGrlP`ktn zTB@H@u`tO&k99PSN7JXi)rp?ea_ShA^obdk<)Qk{F_$=t!NNG{rU<8pV&ixQdt%%Y z)-$P&xlXrod>3KV+UcZ|pO7S-yYQ?|G5qpmlc7&4*8e$W^_qYAyK#JQY%)1nOsFfT zVW*1<9<#W?f{U%qp37W^xk$&_r#}O4`D~v#^xfF~-csM?XnGo0^|9IL<>lp?s{&QL z-AofM>v2l#8X$#xI^US+J^NDq@(qtTRHN8lkIqI3FzGeboPA%q#ih!2Q+(gB)Y|qu zhgHwLsF$*Q{xS}Ct{Q&j!w<$ox9SK>2&l|+4o&|?o5x=xHQR?5D7S;!P z>Cy%Zix~vT{69HomRBmoFobvnMCWevsI>W2#u^IhUnXn)VE=g!Zx%1*HGROUthg^3 z>!d$3FIwcK7`sJ6_kk?6@^s28cf_kI-9N+;qwNMxaUatPe2_4F-Fwb8p_Y9K%9Ih% zFr94hHTr#<&3zA(XVb0GaYAVgpDb&1$yaMYVUOnOFDWnb*C$@_iK&5qx?~^fXL|%U zFY69N*!Y8dQoQ{2Wlh>Y6)fU$g;p56bvW=u%>GH#^{qVBRDSgv?LE2SpgGm=25Slf z^oumT99vTBXdMj}3cRE?Ke@V=yl>YCGZ983Qz(5%-fOG*iztS$G-4i){5b~pkZJht zF9hL?GE2gnO**-(WKU(IJi!N~^e|HbNntQXAop;7PPeXaguSx8MNRpq(UQ`3v zqh#Z0buM9(v6)H_<7IqAKRu)8Z%_{Vda~*go)FJ~sVIVHdik^)E88}_uGW=Ttzj2i z+NkPaMazf-5nB&QuFv1l^3GcWE2(~y;`&eH z%$!@{Ih*&R*b|T)NZiA5W1r@k{*IQFoqZH8h{YJ6s+dlO_+e%4xf&648+du4A(t%| z*&DecZ=lQz(^jW@>J+#7l1ie-R$b1ThSN4SVr(I?q5r^qaQRbkuhHBy3( zD5);a^j;@W7$D1;!L9yMC08@D(DLlBqZ4i~pp zfG<1?;**G!x=*mUo}zaT1o26`5$@$KUDb!vzSv@-0kRs2Dt<%n~8^9&MFBVby?&-wJk~G89 z>t<3fp4uS@JrOeFY;w(G>9A!>QJYMGCl`E9xM#jsrU;wf8zX{)=fCw_#o$zsoL^PN zm&Dt%8g?g#lh^I>g61dyDx0%$N{#t+hg%-=xK4s5Xo)+@$&V`X*$l;ta-`9<_LeSr zptpUjFH?ky^QhC_#`HjzSj4a5>Nl+|9kl7Kcu$Q|qW+>GGuJ7#C*tr${)T_jw^}Uk z3&yFet3#_i!Qwlj+s_zIp7_McMyA3)C2sqQ+1I2uwYD+~L$_9frlq%s(^!$89w+x1 zTY)%QDx=h>=WFd$y1EXmwpBI}J>sW=EH7pI>v>kcO^4A6{ux)pL3%TbaoWFsoc{XV z|M(~-hY0LKL7}jK;AN!9$0@D%0V(x%S;wezlT4co z*>VdSVr7GlP`P6>7;B=<%zf>T*c5+}8-PxN*2wsK1-nY}#XCJa?Y{CD$$RPaZ3H{1 zU$4DnY?*PR4_RFkwmx!MYN@acka3Z^ixax)8o$tj_+j+WKJ4bvtsDYx(lR*8X{@22 zc}q=NwoT~J%JDq!Fe9S4MO@u#ZEjw>e9h%j1DbgCB_un{qn+%0;oXxUp5&!^7JkvQ zKZJhXG1S6#sUv;&CXw>w2wK+SUrmvoo`P)Zmhu<0Gv?MI+IpPiAj25g0DLs|d!}}j z^khox;I{b9t%%f}?_q}~Vl-BFq-=B6LMOt$KABdB{5fQUsoKcx!{j-!KJxnw<72Fs zD$}|)Br+J6Uo zHp-Tx12m+P82@G+I#F90KKx$wVdW!v9BtjVp`*tup9vLTQVhM7F|lI~N+O{lL}uo8 z_OY{PSam0o&U4&Ng`Af*Kd@C(e~H9vH2-sGc;AKK_nfw~^N}JjNNb~(prHb-rU5iW zxSI!^v}(bt(V1LcC{HI9gG>=eo@tK8Z!kN3fHb)tjn#2?-znR>&aR7W+m-d|e4DW- z!i4oqOq&3yjo&=;T|)1FsqQs@3GEA3$slQagujJ{Z4q*=?-RbAr>Z~Nd>$pL!-J*@#l&4F=m!Zn5A&^K}$kIRsh%; z+}TWnJ2{$>&nM3gSR9*f3$xFMMlm%NiMyCtVNxQeRJ&n?VfD+pC#ba!{c|!ZeyDrp zz+yl6n)=D{Ue#pAk(j_=>-V|2KhY2-mdRBMITzmLt2BtnI>SBy*4*p7(uvC-}e*WUbolF=|abBJ=%}N-uM3q?&*UW*Wt()@{@&J ztPJ6bPq%#w-w^lM`I2EE*LZpJ(uDkE)bEKR&+yJO+i_0!1Mg7$nbuN-EV$ z64S10m|RxDYR8-Qh>Q#IC@ZQr_rJ5CGUC7yxGX25$*2cPQMfKZZoEMYT=TLykr%v7v+|BM6a7A1L9Zt&2|d;Js_9l z-eSI4GU9~0H1R6xm8I&se+Cr^syGcrE2tdt{G~=M6CV9?qi)W#Y@E$eHj9TixN46Y z>y23v?iVq|x|mQta61StftAACXnCI2XaBa#1xqFQ($`({m^$a<{8wSf{prajl9j7e zDVnqgqx=WiRuq+2&_bLSM>}SHmVz8|@F(l`cM7gICNwah){Hu*_#8ioJINv)KT~CK zsTH%@i;W6T{f+Q-`<_%PZp0g`zCSDR56h8y^ZUsDzI9ymwfd)uo0sOeKl7fxFJdh4 zT>J8-=MB2_T&sITy;-X{`7S2kn8~fqz^YCE!$Yy74kLVVMyHjmi!27Du|tY9t81Yq zoHXF)Vb&~qe?XrcZmkB#Mu$LzMNdPx=%CI!?N>SOnrPQ}8!feFnuOKE$T~Q0k?=xS zQCD{a(jMo!_m@!Xgig;E{{ zbD<+nvLlUq$*K9FJ**LED@~T?gu<9~FOI8=x)i5F^m<1J6_R=-DA?J+;>*) zk+ITW*5~c%{nh9gYiHc7r!)0b&l?MSNM&WHKd{N+v<$h||A=;Xv{oKx6me4M9=Mjn zvmfr^sAcAk@yn4V`7#m|)}2j0^!yd1B^2J&#Br&*0SR}RF}g^TZNIBXeH0128RYcN z`cMDp;u90~mb0MHx#Ug3Hv0m%@w`VvZMm^wqe|i}YIa(?8t8HO2&a_gCGl@PlsGt^ z2qHFj^+MINa_2zN-n?-S%V{N&=L|t&Nj5L7S{2XAaXT838W2bmMK)a7V88d9-r~>c z*P+2?ugb_Qk%>XYa)}&!r%6drjzf7S7HWzLE~A*_Ey z2o3Q&k(Wp#{}fDGHZgf&#*0X>?SGr9x?zXxcsV9ePRFhu$z`vS#iwU^nT3=(FB4wd z-g3Ecj|-0aJLV%uq^QkC#34>!3OVh7h4C?o9(&F+M2W~*D5v)OxOMCNX}SL22om(WaZ)4+Ob2ScF4Lm&k;DtU8_92=nwd z@Zi+I0ll9!xCNNZ#5pJnbNSIIk;ZY=bMNQ(t2aj!HG2ds_GVVt?2yWpC_cnAKUdfy#FrLv)*FtY`)pg!Wnw0+rY+>s zn$X~j()(>QzB&pwwQ>Tosc$7ql2DVdwu;vIO3`*}f|AY*Bl(0*O|rtDr9|p8lLbv` z_TC6%(`7_&HtpaOGii(3he-sfELb#|I*ZtyLl9bQ>(D91uU1#}hL%tMoC=84a9rqL zqeCh~^33@o83JOkR%gk=0jkRx*q7aLvl9c!*1}PtY1br~-~1Yya@ly+m1%K;1;=4I z=D0&%fI$TdX(y#z(7NZufFP93!v8)&l|LDN7&O`pYW z$18cqJgF}XH0r5{6a4J;z_*2C!dC1# zt;JHd!T=NLE4pIz0#U2DsgIqgibl(DRO<`kekm#Ab_S+CU zNB6YVq}D^;hDR$ynBfBd2%n|)fQA#uf=c!-qWvSbn! zQ{rA&4eZA~bys^Ve`F z@P15CBQw1XWooJ1`1RQs(zK>(GkWUFoAVAsDgIf3j&P?Z?lW9NRLnBdyIlq_$%n1Q z`W%O-r%1GR2#<_E5xthD?%knmZpovweJAAO6xo91&EMGl;Ckm-liqX{S`9L|6piPT zdzz4Z3Lg+6vt({v-_8p3!T)j4u46aaqOPf?fPvWmN}uc16XCby{srH&>2UrY-L(9? z4(%fmb_wo0qA7HmsER`94p3C>l?QNY=dcK6m!&38QiKRl}HReMb1|d!7Vy zuqUt6I5jtN(`s2^Q2NQcgVzRANHl%*-)lIEn(fr@4$ix@ID7oN!VkMA2a_S1XKVdl zSo)+Kty?$q?&TNmCw;If7ymZp;US$eo=Xn8(DmV`A5f7$71ekTs(dk=lGaN{bb|$b zLH`L0IwC#Q7r_-0F^A9~el#JN4k6i-Yf63TPo|otJuwimXKVi5+*u;2pBwRoTs`lh zRlXhY@0w)IvHOy~P%j>xa-Trj`AGEqzD&x=R7Cp!qHJ!pmw4=*|}Qj-Rvc z*PW&RGMb)3Zg|O0BR577J`YF2UCCI-@L5cx#o@_$#GBC^yN_vK+oiZzqVSDpDV+=6 zqp?gjkNMxh@{2%o!6w5q7mL6FNPv5J0ja|T4^FIZmup497eSoD=b6{+}FG;qWFDm`sNvbx5@ zZuhK8MrIV+Su}6#ogVziN^7GKxs*=ypXeH`D6M{4rikX!uK8mZQ8Yws?>z=d=-lyV z!sVUaxTrwh+R2p+Zs{+Xt38**Vo%w`d_pRfW%FNtU3K%?nyXOn;E=C3-{|UYmuyIy z9a=3mRzG;ME|HbtAQHNX10k~C8|^~vJQ0>D(Z2?(PuF*A#q>xvIBHmvQVyo#2iJGC z(sZhwrZNz&6>d%;`HY6gslP-6^d0Szl)YP-qwW|V zvps#w4`^8#B{kpdu)VP?VjuG>$>ElKQ1Ng;!I-Iuy{H7Z9uGi*7OJI*PiWjXdP>JR zT|XVO-Jf0&HL7?1UK_PALPzX?JePHx#CCribiF^ncW+!|$gXgO&VJF??mvt8*wdG8 z@$5^oM4*Je+$`fp$|v>$TkVQxd4t|kg(p7TqrEayY;v}_z6QFihjBgnFJDg)igs4$ zOiu)qjRagzA*PBX>Hne)xyM9M@KI*$A@;Q))};EXkZE-yDcl2!jp1)XxOIirgE|`$ z755sYydrFKbTVW~iDp5=DqKM5H*zG^b{ufbqlhw`szl`bs{y)|b4z}z6%L@+B zvhX=xErd!aH0)KpvAHz<*So9462vT5{=cg*n!0~$YC=O$%pYirj^9@40Ul5G&`0hR z2~R=f@**>Uw(jT!FcqdBU}gXo)G@*pIS1e^8N;tu12=eUW;?eeuL0Wr5b#LW{9`~5 zaM_&dnN<>ba?j~~|nhi7JH1~^Hb<1(stI{>7De*zry z@}JAzaJO?!dR zWd*1ZJf}vM?4nF4KuRtwa}{MbU}50IUTj16Uxu?+p+vfcv13_jY4IGN=Y1 z#Sds>-nemtg6rWeK#;&ZL4kqIEiJ2SYh9}}=2@O50CR&q9{Xe&_3Bj*P~**fd^+Bi zlw8Key~CpHcIlPE3m2Dq_<}NfdwVOZBHCARe%giVbi(#XHK_zp#yxsK){C{al-2_mP2n{SrhT@4TOXjp<8}>ouA7-a z(3X=6T)+WpWGO%y&iBXd;1Uef)FQs;r|`TrRaJnG?dp3Jsn5>EWmfhOa1mu?WzcU# zK~G&b0YT=RgaaIoi|A@T6x!SeD4@`+T!jrp5jlV&0BA4dV?_lD-Vfm0keKN4%|_!} zH>_=JJk)QnvMvL%iA~%cs9PY?Kwp#yXj<73|Bt$*bm!;i18gHPDG8&oNGq>3ia{f9 zpgprBC+GTsQBafdQ~>}v;^jaMfV%Trb@1m;7Y7GiT-+nT=s{qyXw&Kkp**U=ROx`> zUGH;NZLImzb=sQ}5TGO^J3z;9aaW*^ssO|v%miTIv-6Y9@Ox~T;_l@Dv;`ha8vqCd z1$BU+{taVjXz1c(`XVtQ!5f(S0LXO|xJQ5jh`y#11F&D0`Vs&Lvb^`98Q=E-&>yw6 zOq(kkLO`3)2h25Kv4K#6?I+>6|7^5ezFJ!X*y-1Jxb*b&#Kd3}|E#K}MhtX9SU8eSOeaynbfFQ}2@tJ( ze0)G+0Qgdb4yx-_U2O|<0pD&%JEt=`K0dIu6d)bmggFUMK(977HYSS+UP9yh4Y0`y z3MK&%0cH%C+Y7}-t9)LETPQ>#GlvotV_1`M^A>OufII-kAT|m3slI=Ile<>{;s%hF zl{GaQ&|n9w+CAS%P-WYx+7ytGj0^)!P3b9qw=H8y4aF`%f!#fGhGwmno?aMj`uZx& z`I1Np8 z>}@g%*Tu!j32c5?U6O>me|}Pf81vZKZGy;&X4C^0aN{!G%JpZ0Vtjl%aPQX=1pQG+ zB%q0Cqzh|3cz}gKD-1Tv1r*HDaS+<(Wi6defWia%8K5)%|)H zKx^J7`x)ZyBpo4uN&;xHlc+aPug)sFDu71`0MZJ(X@Ddux^hB!2?$ugy*SwFc6WCN zy8@6PcrEfl_X`fj2GzE=2lgo-Gyqr)z?;=oyT&~~usE>5iwg^2Uq5`{x*ReDK=o&T z0l*ArI{Z7a!c3zc7z0u{q$bQI1f9XjoyNw-GfPWL8ynmWSCRpZ2em&bz|X=q7DZ=e zT^#>?T0aSG;$R{MTWH6_r>(4%xQ_5_Y}~%_b)P5!iVS-8FwD)FtE9c;{=|Wqtj7Sm zW@~LNV$n^-$@!i)qo}A|@{^CJ=gGW!bi?xUa#CXA&L3ETU?)5P?TLC4$=@S*6Ez4R zhUIy$UPAz>z=0M0{f(~I?%e>I=l}_M{w^bP0xNJ8=G%e4bZ4fXX^r$hnwG|XcI1X3=oJM=sXo2)>(XJ%$rl2pW+V>I$G z1w{nDCye{!i%+^iPkueyhgS##;M<$U5yp`65z+6N@S%> zb|@M9*IX--USe!;F!o{YOA{PD09X=VR^LD1CK`qVI4H4aXUxDM21?N7r2{DDnh@pu z}GnaHd)o7R)FjfK-Tyz{XhK zq9W<#x)u)dSX5Mm;vvsiCK*=VM^gx_=+ME;(7(BW>-xf&%g@S3ZK71twF+ z!YOP5cotBgb08q1P6wFEFDbDFsMfy)3&8_kLt;l#X8-`7%mLdA6I)v5q30-GTX6!E zt1HO3t1APqNnm=nlBz1xRV}aC*C3$_5!R-rjI1IICT1r{H3_N0m=xbNs<5{!(#r?gu*3qMmuGJ#~pr^TkpuJGT z*XgAIAhp5z&EX3)a&z;7ehI)+a)U7iiz^q=($Nv7Uf8v{F(&(e4_}!ssI5)EG!jb- zAgJ<3{)sQK=JFO8m3gW705KFeAu-=GnWox0RToA;Kl*O9H{eu7<=z{tlRg0{1ORKh>VauLuuQ4XOFTK zmC+zsWn_=CM#&htFZ z<9Hs=<9YDzvJKFNy*D!InQhqA39Cf)GldC*v5#IT)WaGT6BENiHwl^Bv=B@FWNUzeMgxQA0y!}U_U~uLT);Dc z;hneZfIYmip|#@@x!@(gzOmkygXbwdz`hXrpD1{0rg3&-R24Glr+?ZQjw>f)vv$@>$H z(@ow*VMI)+e;ZP=G2M&L!m@GQym-`H)Nc{~UKv9n;;cuV`j&9zJ9LtmLfgjh}8BD`~A*Pi_;hhu80Rly1FSH!GXCB8C@t-B)z{ zhvmv6Pd_F$IX(`M&K2&ek55qTeewS22&FRU=`fGx0f5r)=oi%2M1}aHdx7U(!YGM!M6S7%YLYFcv;?=!5fWBOk4zM_`VLfV9L|ZC?%-B z4KC~TWb~dm@za~is*bo(9Y%%qNTkp`Aqf-|fuN7V6&yz6yE>l3{TLn|CfaKahU(%2 zxcCHY_T(hkU6v$&Oxjk5p>@(5593{neCs{MR{LyO9}N8 ziI!9lF&*g(_+D)DviGo-$s@?HS37@n3xAhSy|-;&a79&>$8rEyt5Ug&{CQf7fxbTe z*$kzNL7bg8cGf4pH@An~*Q%1f`|4ZkM_x>wIa#ru&wO)E3JG4f?bOsl-F!u<^r?6x zqig+kKvE^MfK6fLw6aylD5cOM<3iJ<$@vt_ACP|0XbR#aSFm58P>lBwq%|b4z3}nn z)f|B|6t|^KXST|`As+>*hjEZ_!b3c8fNsFr$Y_CBQKOsQ^zxHMm7?`291YAwM&4xb#P#&biv(>)YR4QmhWWV zLs1aNWvp0Oz22IcMlAUmF^4=!$I+#6k&n z$%OkGoLHX}gtEJyCB|f}`jkijp-tWql@^{SASz5fs9vbH6 zdxQ!8ust}y1HMCsiM{=0aDRrjpDHByfY zjtWBbxhThs|`2xm(x7P??oQ5!_A|Hl4-gS1m5CzR#9bvP(O=6^< z36u&RA#v4}=X#59g}*QCX*V6gzk2_li)<0Ary?Kf%+W7Q;o`;jbgRgfeI<`> zr0Zs@j?{boA`gDYGs6Cw!5{=av!tp#7#HCjVH%j^M#3;hB`D z6GDTw(VvlIU{+NiJ3*^;=!|+YoUF_z-V92%moAB0o_%K}uRVXoDCVoj+$~w!eG7~A z!*CAkFH_bldR&nRf7aM2zqWY2wCe=?kub^$0khPRBmsSsAKiEKL{ODted>+>j(z%+ zd9xF=A4lUrPv|x>c%fm>8w~fQEo|{8tP2iU;3R)nMXMlD{EkF zSmZlLcPNrRof~BC?(F1pOHkX%ZgGCg)~vmW$H_HA!aOd^9!-9Z$@@-!&#^|$#|w|8 zV`E}|LJ$K54%;twy?aMXo$zIQ;dw{LHUuN6v%B_tpG#&&#%F}4kWmr2^bKYTg|8iH z#}R4ndy?pUT(}B_?e(c2%2DWYLS2K+{?ere=K%?EadDVguaDyJow&KB_=VlP!fbxC zKRe7B!)mzy+6fH5V|L`S0hB*Nbs%$w!WN#_d&P-ja4nP)sH&(o+8g%n@LYvDq0Mn_ zypu=&^I^LdT31PJXnn{I9u>LGQU>DhdH25FUJakK4?M2${nk06>{_ps;HMQ~82hmp zCMdrVyVOIUcVs8EeX5>;7!GFZrAx`z!=GoiIO!?)s%CE&b9^YdoP#9bsAbDaj^qs6 z%vJ1|L2Z3uSmc!PC)_-KXIy+X_NB=Higv&T6pxzX+^)K_NvSzO?7+H$0(WUG3msPe zog$`?E><*FSnBbrEy5~J0I&lN)O*OkxgRx5(UL_L=Qf%BaI5Qk6nLw^i3Ot<-ly!^ z&le~bZnMZoIhWkh=0Hui|GlV1+b_Dx`{@)+XtoFo3R64^<2<=}de}hEG1!r@$EtYo zaqtI~3e!=ukJX<{%9b7*n2U}+rs{Pi5ux&_Boo|HLdKwCUmg*c?Yt$xe`J0CWmea? z7@2(?voGdG_NQnMYrB3we((y^Zct2zU_L^UDC&`a(M~W_`V<_&cqxw{P)`^Ohj zQFhnRu$iqA2??tnbKh!c4q`{@^dwtvvi#9U?>6bExFBU6QCM6og1Lf6@S5lKqKa>2 zsPhXgEiJwH@d}D{5Fi)CG(pdWE6LyY^R5GqwPzyVUpU1F5gjI)vJ)nW&K7Cg0MMX2pUT38|U=o*|Y!eSNYsh4uATW(!$mW%`!3 zTP2*$EAoY_izwn-*^W3f8}3-J^9(31Fu4O~v2$meP9OKg;{a4Pqx3h$^~PC!ZT{}C zkjdWLsi{Ocjgu$!q*U)%?8?e;1Y%cM$aNv@2d<$RDGQLL$mxV8c_7#HP`2=6Hw#a7 z{SorUNymxo>AN>C=+6iT2%?fJ+;;n;ODOE?G~8QDQ)?#Zbw+=A_7g-YCJ>wE=jO(Q zqMm#E?tQoA{Oa@b^Ua3j9UO$heGm)9^B@91k;KVb&$7uBB2uEJ=C{7s4`7m%fH`XL z2L*)%#d2Qo9uP67qm1(YeQE*}bx;dE1l$e|LNT8owrykJY5%y6z$275QAmt>#gQ*x zBW@o z$2qh8D$B~UWVIhn8;WFM`;YLAgX41gs2z)(Z!Pr5N=s)^{r2h8rwWT&xNbnQx;A;u z>2Jr}zWw&B`KeQ71J0f7@XI>W&so>=XHvrCSL7p%ebQ0U-0~mNOs$5uYpAAUE_N<&Qh=^M5p8mq{)B1W@8Gn^& zXtil>mkeWX;(Tr+zj%(fq+Dbrh9iLC)_&n?3e1O!OqRiJ6zLvXD5)%kN<@X*{9DyZ zH~J7xa~0A^Z*T9e8POB!P9IKhI}o;?Ii@kR>7J*YfpvM!J-;fVGCm<6ZqCfo5iG9e(2#%~^j+6_P)AP^Ic2E<+B+pq8aA^kdh z^31R0wI4s6>E*aBEG)K=k^rDN2JSN;DZ_? z1sUJq_^w!@8;Uvb*BPy(eSjJ#K7BcZy-gbti3+<$N~y7;Aw4?}H}vPvchf=2ZEy-^ zJ~!yjTwDM0jI@j|9zKw_(Sr2&f4Kl}Z+tR}wD97WLYTU6k+L%IE={h_g4nI|pH=Vg zwZdf36*z}7>g-OcBdHg<-}d!AE*(@NPHZDB1njx>rRwpQV!z4Opqwbj+Uwpn_JX9L z+4j+Q1bG!>NJl!zyAq^FiR;WD%>xzs2xi+zEW6j^+&m=`D4M@4rFf-iV$oNXxZA@0v0#C zbUaT;ObpmG@hk)z$TdL$LeG1;1K&fUfE^MSz{LqR7cNYrsuu@9Hz<|0D=v`$A>lux z1d-U*)x-PupRDt)tX}EmZM(8C-igFd4A01m1(YCNK1S+X)(SB=M05rj0>_`6@65&& zYgm~qp7c({Ou+0}{#);cG>_pcY|!6O1H6(kvZY?JDpXn6`_E6V!#Y_&#o;Nr#f7D$ zu!I{zCrbGikCPZG;-S(QSD<|1g6LH?k-&?sCDd5bK=dHM`Q`F4n3q09FQU@X$&;SY z_Uw}5@?m88RSo|Yp$5ve@L9-SrXKY4saJ#5^=(|?Ez0Dbw^fL4_yy5ZcOyQ;b{3;* z8r5rg*^I<`g_07$ab|+ou}9(!+S+n4Mn8e@0z#VHymMtIvted$Lz!(W4|&&lx}Y#T zx53-A4|Es4*@emmXPicr(#H!B-LdPdjp)zD48}&n8^RzT`ZXM5wYfg)x6L0plnd`4 zKKt~^;>J?$4!|tHK%p72lW-j=4cJw5HiqcZ+GKGl6fS|6+&S!Y$WZkZG2uak5`krE zstF=XM$F1Ula_}kr?HW!yR(g)+%dbrM-SR=u#AQV2I@}KR8$dBQ6{6qlaMUHZmUDT zcw5%T8{jOCz1Hc5mPAA_j(E5(P8t(_E-ZYix|GpHmx1_+BRV-fz5Ux95hB~6gH1AR zNgH~D2*ROEVHvDO^rv!Q(jlA|`%6WFhK7LZZb*DdKsN%&G5kA~@(0DmJik7)r^{%u zGTGRV|H?HE!6JeE5lepk^>WYQs0Ab>1_lO#2O*9+{_P`%O#O5CLXPu=3z#WNi@HDW z#Zk2Ob#*B&+Nv}7E0eV($@;dT=ni)Qs0N<^M&Y50Hh=lGj3YONY4`3f9y$c@xi=yo zdk8b%c)>;+_aG-Hkm{uL^*=F=-|f^E9??EKT~@Gl0~QLExewk^SvXSYLI`Cal;H59 z1fq?10=IWLjhb%4Lkt({G21rJ_RIh~wi|B(I5gTYx!U2kaRqWin2zXiN)!5`(~iU* z)!yiLdK7*q!!13yf3@iA3lj+~Oo@~siDRWl(M|d%pePj1VW~knY881fG_5XFmd%|f zZFDNF``so|dfqKc$vP<|#S8i%Sfgbp?;~}ya>;KsHTyUlvVzu|D@-XN2QPj)2BoR+I*7K8Ui?(>?%3h`)TDuC!hyuvTx4``06PVs8Dq?K6Sck9 zP;yK?kwN`1)`?@`%H5nQ5{y1LP?N1;|4dXsJT>UUFXYF)Bz+XVRdCZvxj@bdF-pf&S$Dk@EF z?F0cIC8ZZI@yO~z`n3I=JWvrxYz-sXOw@sc)ydfz z-xxzG<nEXXq>xn!7CXazmEk5!i% zJm;Wm=K|r*NA|B7WTF*(x>0+J>x}_r#FmB`ec^Mx4;-DCbq2hyMEXPpBgm#;kbzJ@ zIpTty-66}SFE=0Z^dt`wJ!LqPH6;G!<-Z_WXLA4K0o&(fEV@V!@rc$H6i^Tf&gz&b zbsJ33DbB+&82z<;UA7@J)4yX3juI3eA}ZW4Xbv3)GSXQaS<$>*S(VMKc0A3Q0pfMU zx>3BNI6f^mj#L7Hlr#{US7}ln5PCLm|EUc1k&KR6p0cd^#+DXAVPQw4GLhow#e!m_ z(go7NA?KUcYDZK6dGeEkj+uEJ-p9<0FH+;nFLuG`5KD`pp=45GF|cjcEF$5^KYATO z!NEkS5%`(QUjlcwu;f$o9=VZd`-QHU8c^*962BwYFT$`vlmz%)U#WG+*UQ*y51SbU z#YDhiJLJ%n3;DNsq6#b^Eiti+@B|^G;E9#MCp+~SgNCajrP?v->ha_K*V)<9W^SG! zh9iIVZv@25AK#ST2SHjWzqGZ@FD?@0H6T_~kkxNI|MGk8<%MsMqVrn&Q-<6%JU1he z+um?Qx=@IYLBe0f3alg3x{Y8VX8K|@jwyxz#kV<_wQi?jc$JV*VIcICSQgL4EsBgJ z&tpFo6B3HuJcS!15O#zHmQm_sjjcCqt~S69t12tE#LFU!he*}e*B4BH=9zK-L%wT& zU;|u_rNJ<_aP%T=?FGLG(}mfX1sO=8F@|WQ|M$zv`XYIXfS35Y_hgFe&`!BR%>L~4 z{qqd641qgk!@|R}@{dDET5e-*C**|iH?~OZFsI5Fe1<&iwWUE1*kf2Fi2ilL^f-(M zCs6DDSzgv)d#KQt4+CSAr#-21aZ+q7pw3Qg0k?I?od0QTYVMJvM_(WO{&r|6z4nxB zM(N+@)k#5!xMRnTdAZ2@`b6Ki@bsd&9j}0o5PNtFt0{nT@qPPxoXcWnh;WjE<3e7K zvaLNc^~ind1O%l72#X_qku)ZoNs%QBYp;eCQ;TFg^b(qu*>PgTCrHi@VG6dk3M&?T zcDQs&S4W4RUF7Y$od`>M4B*U=AG`tr#YfMLGB7d4n?E-@b*eQ!{eJxw5E$O>*1Cu@ z8V#QUehxVWgKM@cg&2jlr}JUVi2Dg}xry$IN7hOimMiuo0`t*}Y0?v|x2^#=4hRgy zeA2hDxIZw@H7zVAHdJ;-?egXQ6LwLohc#7CV|-E-6K^Q)9&mMc_eF#P)ubdb%avYJ zYv`<>F4&SKTXOH-cf=1Mun^M>RaM|eR6c&3w9eqhM z8q=}YNAWT?p+KrB9eaOj8LB^ZaN``sjW-`bQ41nawMd18kL}oV{XMOPY;eonm90B+uE>C6Z_LZL5{!-#fs?YjvJ)gth`1Ok>7&`R7*?K(2%T{ z*o+^PjIcLqYrjDVORz!m8ng=!2p~gX8?sl>iTPR5NHL)Y*&!q*aduFIUXJ|&FhpVl z4Gr7!mJ{{wL)C8C+#FYj);lHlW7wo~k=C!gj(CI?19EaW8DD3>3 zU>NC%72||NMKZj6$c9rTo>2KEG?BQgRXZXkw}p@3bqOo-=bK? zOW$c(YcW*NfrRbIfkxe<2gsx0FA)wsqj-*(;l+z@cRhyw$?+SH^T9j*?D_51(IHP# z0%Wn`palpyny=5dcHSt#{wE+hh~-=1@(4mgO)Xm$9q0bcohizb!|Aad8yN*?Ldkye z$B$ZU#-P4{+;q3q&N1Ja)nNqwGBT>f=|V$eS?Sn|J?+q(u*4b^xDa%#Ldgz7VgR7M zaWU-X4&C>N{Uc#b5Wr)tT=nqi*E`NfPeBI5z$^Rc!p9?za$>OG9+Q&Vgp?Wf=4hnV zAQ%ss!6%owo?1R2kN9Nzb9oJMJrtZ0BR+23wyhN21r~>qq_piBgr#E{P}m$t9K|~_ zhnN=%Mt_2ttZpB6N=nN~UWI(Y9XLo>F_@?E_Qzm?>)-%lDhNsnhBF^=evg4e>X9hg zo|N<(VaZUvPfc#_2wKf^`2p|0i%dFFJmux(fq^etVtIPkl47`75$QuV1`=!Aw?iS! z`U>h1aa^V2)e%1162p~C zdIoU_08M23FEqP9yc9^D58Q%Cxv8#3f~}K%l4~MQ-Q2jXxtWibw<&K=3e^KiqC}f+ z2lnnQ@=*IdueAvZ5+aDBD~3?8P(Q-88q%!`;0r zLl&~Wrq(`0abHBvSOmSjx9SehOFbwp^#d0T+fwYWc8vG~Xpr?58Q<{~J%CY#I`&Dp ziiAI_tKhUOVWR0D+69>+sHDw6(KHenz+1QApFMB~K(c=hncVGUTefeHBnoZ}RvU-^ zBqLyWiLM!TS>B~&sFL%_xC1j47cYe)1aUg#-3~X9SU!PObwB#(kq`kqu(+CeZ%@yH z4SM*v;MMhnrrf>j0|SC^i_||kxgQJyX#0h}&|eQ;2|$1X4>-OfUQ%7ft7o1bpa7yP zxCb?$SU^}Dl00`ZGOUe_uM$~Q$zuAgs<&82-`%?T+vvQ#VQvhPd=idTx#kcAJjn`{ z3gRqXqWWuv!(`FV?8Fq3x5#d)nr&j zk}oy`IJmFdFW_k*v}V-3TZR2uQq)Q^pyN(e(Xu);*;u#}4I_bRiGiAK;Rs9Exa4+iu18e=Dr2v(4=Z0F4u-N#f#PO*&5LKz<9>2 z!mgFx&u{N- zGSud0 zRHNwKa(=f)HRTh*<9y;GvZoJ~I|@h<4Ifv~_fdi-=qUl3_x=Xpk)djP~Qe z0Fms8@dEda-t>Ea6j)+ts}i3V*^I!)__Yy;Ti7z+z05ucU4P@+nZMH9Oo!5DDtFOBtK`^Ct~MBIh6pI_Ep0N8C(|CE%0>Z?;Qs9#1#a=n*@kYOpU zok!eTyM=%=vaqm7bBOcs1O^Q>ovqian*w_y%)}nR^5NYYM89`Ca$A|^TmKbkzG$JRN9*+-013|4ck74T^*^*Cq<0a6W5f123Iq12 zTU_iQD>HLT{RWhCtaw7PSmFdj=2pb-}lf2LnzHl)E zZpcn+TK&4+*wtcE!l?OLNl)*a3<XHV?G)ks!CrF!enAVnwFI8GHL z?cx3)fO#?Yt2KavV1sGce*um~PnGZhNi3a{T|VUaE*|ZqbjA9Dt-Omw!3XgPA~hEd zCA_V#FjYja8^8*dA1q$d+EwV&f2-+y6$zy~BWK!n7-^AeeUB3d_wT0wZ`()lM21*= z*%P?D9E6y%nr!nM^76B141*b_Z1%9Q(9DK6pAKgy@z0lmM-s-4mBBD2_xeeOMF#8K z0mq&KSRvq`)VVj6W0(otO^T^-p6YyW<0^idN}JM!F57x~6&*&8?%&U;G}?OJ>ydGP zkO3ojT3v&Kmd>iiNNulcidR~jn~T&mXhMNc-vzHfnQHn|vJAJ_v~~YwQBaUi+%l=*@enDJh+y%z9Ej#}nF5m;FZ+sx$B;3;8*F zy;$S{yz(E#7rI_8Mt>gUgF3J6X*G1^k3WuPDh_q5`dAZy`N5^48;p=i@GE}Fo~8F5 zIN89!G6?Bb9wXYS?nVQEDmABq!qA!Ll#^3&+5>Ik?=+(ao{c9mW8u*^#N7RXf!3^A zfRl=M0b0;vk_B#mBdOg^!r0*htK;|FBf>qtqt1Uc4_zn=1S&dTwEzm};>H+#Zr+ll zwA%2qjrpk`U}9nd$HrzY1W`^upaY_hgE-8)nV5iXJ$mxw58NYw^jN?++U$?nw=qC^ zA6Sc%ZRP~>ce_Q+$Os@>z;y2sT0`?34(uPPsm13n^7Hc})r2?`0XGQ=i6o_4tZ(6X z?7e&Z^hHD@|ICOWpgvq4hnXOAd~5D*M}8?J0)4JmlTqrtO*?M5z{!}85`e66$@kbg z?H=5*63{Ku!GkWuv;Ebw6Af1;@|U|yORVsU5b!6OI2Sc&`qZAD@{PSq)CpBBOWIy~ z96{Vd@UwzYyeiU19pQ-z*>EJZ*aQ^a$sO(Wq>wk8e=0Qh39xH;a=V@L#LfvcP{7g4 zu_&*}nZ+H>YbD1-D2zj5thC z-O72EB({_Nx+<@Zf()-q1pLd+ZPp4NrO|1obdWjc(x;qGPW$=mS5CG9sBwIw*ky8} z+0Fom4i;d#4|mn=2}6-Hh`pKCkNDv{G`~EiUhA(r9nVgnWvAejWQEkxDuWVA|7Jc0U2U=j+NjU7LlX z<`0z@!&eFKrc|zWvODD0(u;pV`fB(tg&lK2qQVEh!@}gxdiqW@#2mtl#o?H-U5UIW z#>u-5gp&9%#q4ZTPUFwE`a^cSLEPoC*;YR+a{e+};rRueYF&5}v^ z8??|`+1M^R?bcgLUeKhPC~h5GyL;l`NXw)XPt!LUy_KCCJN#DLIx$3NR{|5gR{ z-c3!)PuUXKWMWfN3RfPb{rfvs0{YC$$jLQ;wwrgQ zv6cPbzv((a{r`3w+O0p85w%0{+dDGgzfYF+%z;QTY$a$! zW38ZYqwDyWksbg3&Hvw(3y(`H^ff{;(fHEiB9K@DW&#%5y1M9B43W@ZQ z1f+v$k_z`$d(3hWkP{MG*k`~3e@ypQLZRF$N{t9M*e&OI(~r4u6VSEIG)F~I(gU^1 z;~P|Q-mSL*w+)^ej`3H6*lx77iOnR?rmC0f9tT=&ef_lziwdu3K6fV-)fRv7YG{(R ztblP4zKo3toLli2^4dlKSM#uG?#4KxR}uoyhv0OmQEwrDfBE8abOG=%gpxPGY(;p7 z^FVEIejehda=vR%?!69i8wHFCRr`YY_-?pzcDygz_A|GfVi821!W&JpK*YLynV3n5 zrn3!Nxr;i7nU7VomHk#VI5B_~L_`w~L?QXvA!0dGztuG&1`o);$M-m$Vv#30V^~;x zf`ClKf?#(oi_)O~{; zp?v|}W=M!NFxTf>1+agx?mHz1InpP98cOR|s2FzY65AcYKS#C%Vw}m$H}?dADF_Dw z>DR{yYf+KXvV0#WK(4xB?&TijqET_tf1WZnsrl6_Vnf8i>Z{-dTVJ13DV->jptgxH zA=gAo2_lxzOhvid^w+O*$l`5$+q-$pbwnLhRe&moO>Kf=Tp^wdsw^>u8x|IZ!Vg?S zfS(^BETpg3-yZ#(@caG}7Bhk{&=b*tWN2sz&*O#J43+fP=&7@@mt{19cG8eT+A}64 zWf|>oe6Srqd-e>DAYjl}U}0hz5;fr8*3IZuj*Wl)>Vj|{k8}I>?b^jYbH@&AUm(@I zCEvb&4`dDm>O*Gj^%K}FVVRL`784ggS3UqN>?As`WTzmDF$44)ub~AIgf*^VYUQP} z5xb968z>lhEC4(*dVQr5;VMQAUhA%vPcdCFEfAaj9Vf?fCy3$@0Fh4f#tV>KpwiHC@TRY z1DpyrX6AzM?B5D73K%XO4Gp-`Wd4AOmx--s(A%hs#TVHdq=dN23-JvOM0@y&X2^OA zo(RhX2T))t0vxx!J`0@~)5O;gfTg=tfxAGsjZ79oWON?=htC7rHBiga_M`qreM3-Y zyn;FolxWoF5iK6uzat~W>d;xTR9c_}XUOily=dK(&48_tK;A3!Ys;l8iYo;}2 zk;x(hlA&9g&@r z-@tuGd#T}J#mB@fKzFyHsfPOd8yuHuY1Gd1N|SKYp?yL*3o&ZKRFBc&w09WwO5Rf?c(!UQ`Gc_A*#;=vM^YZWaP?;dPk5A_@01cs4X9 z5_s+Q0H`Kf^CAIxp*auxFS7L_l-h(wNOD3H+Gy{z_DU~NF$*;k1%cRJFpm(vAbP`* zkg1%~3ygwL;DsV%+<2mHE;dS+mB$jyzkAj(A*cF)irV(}*GlF@?> zF1nC=xa&VfM7bVVXg^>JhmuXUOT^rE%ZiMMXsHtw0-Aaq_ka!JAQ}(BLvg6yk2qGE zBU-y^_mpzk$$j$&;ySKi|1Q$5?}|Ya1pN7ezJBa`(^TxtNLu}QzFi!NHuKcMXc(}A zhyeh(;i#xcaJEmg(YG}0WJgj8>3QI2;Y&yM@czX-ClRCVmGM??|5~siFcAA&d3U5r zh;U3N6=wUw{5&ffY#rII3-M~}`s*nc)dJ;rk=L?YBR^vwqdL(>%tQf5M(JS8GztjO zaJP5y;{$f&Lb5v}aQ;_^?1Hy1!yH9kJ=64C4e%yPLLc|+sN~Dn{hN1yZ_TYA0#w2= z^n(GfyO#2~Bdvk0c4qIwZb^(sC*R9;IDbADj~)~EhatqM-i$QA%FX*) zzt{Ngi6E$yf(bc&cc@EcDX}1LAxYdG*VOpJ^~b_Y6)5gWu@TM|1yGc{EMWE&}zq9wm!B#BYS0-3Kz zkK)+o78Zd0q=EPVM-L`?w6;bHy+vG+nS^oyT-jBe$Bv$gz|6{uT9srgZ;kfUvO;1J zIRBoc;8a=eH>k1(Y2kx-w``Tg-hLPEBMr$;6hoV(Bn2GIojwz%IKW!GIJZ zGF@3~M~~{)pl}SVI)i?-xG@4F*^n6pF__K&yjrHE={uI zrf+2xm8hVU_^iD>Du}mYzjcl{!Yan?C(+2^=x=46y_71tdz;Y^C5@vzQ}PIay1(7Q z`+?BWevEyrD;KuTt%4<4h4h6-&hJsV_d`QEv+-fSwR7Jp#_fu4J(SHgcot>4@O8UH zv)tj}^eld*W(Mm4kjmD5nBEgKX^Oj53_gJCp4%+gYSQU0I+s~3o2GzHL`t3RCv?Hb zGIUUQiCqKFR%D=v5%DI?|BbT9CT_ZQm+6xGSOK<-CgVYT(xT&6>!1j%vsa37Fz!kea)=JQrtS7f*oB?nmN;38GE@`^(M+pO zYe}nDZ(|=!m-Ey4%p`?(FZN+9SJ1oj=6*CIt-FYgzB65*a@vXg5~Hns%rhu~ym@nT z!JJ&l{#^z9KyTfU$)Q6U+}@qq+xhJFGXu^RycHL!KfZ>~Qp?39gUbR6^~A)SB--2tTWH&#XIP& zH<90Z?XM?=H=@;@2TzOMI`;SKx-%txF*Y=L_wK3w zutm9FFKO=MJH=nyP&3L|ce)ep6tdu&rBIZGdWwmhwJ`$3>?5qqU9DtHa1>d&Y?C%a z%}jqyA_4O*fxUGw=vbb`Yy);QDqYvECqhQfu^AMX*Y-blX(@}@- zb^PxZYkr&y*}-09LCSQ3zpraz=hbzws~vyyR9dIiNk+EojyscOp0~U4=D=?D^YVno z%XD_?&*A;JZTMZXvupa>3(QN9mV!ucI`WR_Mxy5b^fv|Y2~fZ1X8y-2B{lW+$>o4M zZ6b4Bb>sd8*slq+b$`Sq_nlyvkY0L4r&-5(#Q8Rg2XHhspxMvOCZQI3lZBmnH*fBY z_dbt=wybX2S8Zd_;Pl>s0iixMzK=M`O7KM%f=vNifzuD84nybStgC|CQhm)>nx4}- z)|NAMF}hL}cP1lj%I>FbE)f)-)9chq=%U}xeADWcTZ^Bbb8(x}tqqUNXUG;{>pUUe zm*YKUa++GPlv3XAWN5@&i^V?C6MKD*eb3a-nR z-NZf+?myluNIa`)V3x3>NZD636~H<8qRoGWhkk3L)lo%K4u^?jR`fLcJuP?q4^kEK zmGXZ;rrUAxQnDL=&Jbu*V-ElVKn|^w2k9q3yHJinqz| z`F6C#yT03?*3b(7d~iOuX_ke`x8t4R@WjPZp}A|S3l!-yzwfy2_v$8L07|J&Y`CnE>3fmK1{zG74a<@vz^^^RqdF}#x>?$DmUvBX z;)jD!@FDtm2kyL!#oLINHc;m|km7DIymV7qXxns0Ohv*U86I8Dwe+knOVsSdYtuKJ zQ~pZ4n{B_A-hdAxuc@x^p4~lj)A$fu*2s7`5DaKr$p6SZIYLeVs>R#K<=qDCOG{fL z`qlY^eCxgs7UG-^=2ehuRuy{P0`I$hxBgV}B; zWpeZIBD`>0)LR?QjSJ1WRrM2ZtVWv5w?mH{H-ZHBzPTrB%m}j=RCDMr8LWUBf~}e8 z3UqA|_%9qut4JUFkW}UiCu>w=-T}J*;y(2%x=Y}dh;3gs7b!1t3^m<=^JQ@n_Zue zOQ@lC;u_uMZysmBN^)#cEblPseHNK0Y1-$c8X2|j2AZp-37c}3i3yVTIl2rcAA7$u z#D}_8;oZptjrHepz2aQ6bFh2vh)kY}E;GKQSx%GM6xG8K6?altG}qbBvte$bW5G`8 zf%8pL>t18cKDWJnZq}VVqNZ2OOTIe@l-YI|lO zW3-&Ee`o${c&(1mx^7+iS4!K;ri3Fm3x|DFMDL}wp5*Eytrg!F_g=S8r!DpQg63Bm zZ_QM$_Ln#H@8}4P`Q;rU-t9lPoG>Vq@O~!=!HN9ElXLl_NqVlQOS!1ac#jdLicQVs&HsCMf9KqXLJra&*HrJkOxPv59M^HUx&5n= zuFwR7Uqh3D1DCy8uTQwpDGu4w`F}oWJ3O}Cwa#EYJW$SW;5ht$U!-HX_d;?+(C347 z_t@U_DkaXcC#(=dHLag!y7mw(q3m(%e)(Fd#NUSB4EcS=)$yr0}+Txeoe zeV5SavR{wuzYCM)-u}khT)vtYl%-y!Kc}62sjeh9(Ql^76Ec#ZDtSU<^V9vScMLxG z<*jW_6y^5V60J#9Z(uq;U{a=yqDg+s@Ei_pG(DAGzwiy?)tertv++x=GM~-SUgQAZ z`ttAQ-?LV|I(t4WA$|kuJE{w?RzO~ z3)5@YN&d60y3$_hOQjD!wJn>7w~fEowJ8bT3I!hCjW^f?R@-!E{!JYK$2nIOX*T~$ za?hTGb`m637LVJVGF;)l_n*~#NDsM%EZNX~PMX$DMhq$6Ph@P|`cGbAJ@)!iry|AT zfheWMNcS^xxVomgub8&4*yvexZVqP(;l&Ear+12jxrrt}kT&XX%Hmxn|FrB|;_18Y zacT2kX`y9P`ZsmMbyweYfo#Ztf%uERw=fwHFZFL8BbPohOjl~po{$r+CX(Bbr|A)A z^ibq!WUyf0_0KPQwo#`ZGHzx=_DC<|Tho^h{*gCpBDOS&X8pWDd!L2+Ol_6mKPN8O zBqo)7+KtG!?H~95=Mt>i|74xA6fP8|TX5M_sn zyG7cEA7#nfgq3Xn&(HD4jp=70nFhMWNySe!$Nc~I{>EG)gIl_y)Z!H6C_w&-*>13w z^gmBYPR;iB?+kCJ_j|2{KQvRlBSan2Pa8tlo~%ldbwlN+;eI8mYkM#L{McETw^%L4 zLy-5;zwUM?bIC1n&fs%&pty&Zr<++Y>yWOY%UY6+b!bUPtA}-ZPhjAw{!Nw3*YirW zkfJiM!C3mXx=?u1kesn(!sW^RN5-DxAFKF6**p^!MkY^<(C19YkGU>=ubeERB@h;C zb)QCwWl1xBqssHD4I2MRxTe-VQ_F8GQ51G+hM`(qsI+1#i!oJnl#}=}X#PpfvG#CK zd{vn3q!j(=drwE(OIF-vQ$mVBa>@K%HQ8}hvGbb6UnF%qlRu|xUXxo}XzB?3K_Fv^ zRk$P|wCStqHRph?DXUM5RERx)!bPTe_bO9x^;G>Q%T-Wnn{p?t>`KO-iixp>C-wx+ zu+O_*^$;#3kT}z`adoGD1L3%`jEM+m$l^ZG?wnZY-%9QPhd9v>3b3_v}1} zed^+8r|ueE(*Ha1G1~4V#V-g1s(BH&W4?+$KeqG-lMsIX(lWX(x9{mr?jYL9oDg*H z0thwvpZsTmhc-V8jm?KeaVUCTBuB?evQ6dLuO90TOTjawOzsDnsb9`}DyXS+C8_9` zP=_RTXIJI?;)s{)XA)2g7GJ8&Hz~a?!_EJ#)OoAi)`QF@S(NCJ!DYgGDX}jt;ioUz z+-nvnuFVO_d@GTQ_4rVEvp|=ZKu}(ZljvB8yL{Pe&3ZNq3-8v+4wpxhW|t2gTN6po zR2m-J&1Ox>*h&{^ai5u5BtQ9ij=0-{Dz28WaL>CR6N3|XVX7FpmT^0|R8UD|7j_J^ z1=nnJdv2WZezX!%G5&LOP+VEFMqHy!q%}pri?nvUiYeClfyI+4xx}P%9UhZk_N=*A zJl}N`KHYi_t{(aCO~#C8RgB3Ju)+KGK-b`+s(QTBbknqI4$x9*ywko9Dj*wyE|7+uGU6seZ-t?*FjXom{_WE%deGR@3v=Wd9O z_KIawTlN1r9yG(moi|ATV}x1Q%Ix?rGIV!R>;Gu(@?vLRoby(d+S}~UbPo0jgb#M+ zY+Df3!-C*dh zWP1HN@f^!kMUQk!m0_Ed$7O%FP`!rHrZ4Sw_xGQ@85jFKNzb#;@O?o@<)=S2N7d#= zEw=cc{#jiK1!*#?O9K3i=c4#Y2-lp{{e$0UkR2GaZ@*nK!pGlY%7KKxw%OL^!fSzn zvZJI@ycgACH0zdw!orl?Z8b(k6|)}Z&77T=^dM|xJ^aZ)@`gI{TYu~*Z?Ndq<(DGw-V)mS`_puIv*m1A9=DGv=N`>7iay+)!_R_q z!**s*4{7{ZxgD{q|EYpcTgg@HQ4%Q|rqbFxOQK1c79B18c-)!RPWfVl=*)wjfZQrl zZUeH$@6Ni_8dROGa>Ub-)wbl(6$ zh~B8_5-}@yzDkgH(0<>aoxCz>cyuM?rAgR#A?JO%pLaEeMBJwR9Ck$#{Rb4!`*&US z9F8xsra2&Le6-^Sr44Cg@L<+nb?Y}z7ql(5_?w@6FF|-V^XHJ4YA+w(+1FumBXjjH z$H}_N3aO(yVi;d?ib!YaFmb=Ak2xrB`<&8Z8{x88x~cF%TOn~F8p8Xj@2e6Hehk;& z#vcfikGXrB*VL9)QN1|ig!@^qKhy_U3lmMJ(i1rY>_;okn+v}gOt{U&Bw^LdWOHe- z(yx$x7vVO4)VC7^;rZ@)f=#!;s?sOz>iX&w{u!s8zE!>LE+vL8g_=xnczn*!zG`U` z;*?-KmKx{!t-{6pmdVhKtDRf?Q)=RLrm609EpnuMJN{93)OTzrXIR8B<;e8%s@WCk zf~hliPM_QIV0oZ(3!ze)Z6+cWJI2r&u!e7Zhu9a129xK4UZlZidc zSMZYg$%^Y*_fr3aI-esYh)b&UIcS)Ebc>`&|9DWc?2up>4d=0zc*!f8jaoWeC@6j= z7fVxcq_(USBs0$1VY5D z$)a}4$Pb~0RxMq>SC+lW3F?hnlJs}oOnJPP^g6UYh!FfwWH`^WaQ!}7^KJ=E8aCyN zJQuDX&L4Z#^Wg&-;c|Jd;=cQrs(x#GS$VJAAUwXWSn>Vd^ln1qhp>k!GIASqvQtAl zE~mY=mEB#jg<{4urG2T)e0TNvY|q_MGb8&BH)I=>NekIt+8U`6_~Y#5PuE6Hb!fd8 zCHTLsIr5gFq0OesyPKaN_&(!9sDJ`b(25Y>i(S^ze#?<#;>A^hhqN<#zbX(2YoAVs z&yo3d^-B{dQm4NjlR{_tuIzX__cG}u@@*vYTf|M_ zfm3|=f#6fI!2LeQodyVvo3R2iK6uI2FX-e_*jd@uj_i?-xh*<6V25@HdNFZlg6S01 zZTqMn&d5k+*?;Wl2`{Wp@!+3{=aec?A7uKm>Rsztz8NUZ)>M5rYGwlsCg8P?^ zh4QDPb~+Jv=6}3&@yO+?csR-OQ1JWSEd&ZzkGKWf<6iqn2_&$t)Op)p!Ypgh$66{Qt9;XvU4ef|yDiQR;2?B^Z_RAqU;(Oq$q z>Cbc8dj9k1*G`Aw16K(I1*^6Esi(5P2d79S!ykk^&`yVQ*W6mu?_uvWTO%!PW|qF# ztg-q$u*SgbxJ%!xiF2aEA=(tO*%ikcDS|~hNih(71EjyXXQ_3 z25DW8{qqN|jL|%F@%12d>(kYTMzV1<`{^)X6zC~`=FRzb- zBxGtzFU}ZGvF%{I_GZLc>%-w)1Yv`n!P{y+mM=U=v!aS#2{+y6#JJx}!&A=i`WBOm zy0vNbM3b?1#r;xba`u5?S&u6m_^HWX963Zu@E4%HR<6|CK{?&Want-QIW5B` z152BWg#CdZK7DFU{Bt6sltvYOj0A$`F~MUgd4=^GVj0Q1_tx3VEZqt$8g+OcN9#IX z_-OS7RT&2F>R@%SKqF-*MHa{Zgg^k49b3OnIJu{>3~gK5 z9}x&&_NpiNlI|qfTuCA)k$>~lf>I_l;FB`R!+uxZH;Kt5++oPcumOqUN2?Nj+_o7r)mJl^Ul=qd6SirK7Jpg#;z;iF}N}PcBDE zd1U-C*s?h0afSVEK$Wt<)jS4L%fWHlLwD~b#~~ywesnT27yGOoG;@{A;xuk_U?gGE=PaA;c8NHehRA0 z0XR_CmAKEekFNKgK6pCYYRv5IZtL5Y?YO7Rh^pkNae6kEkBOqE4h;E_T)1kmwLeLX zf`)_t$_t((5<>1q&y;X873nHu4vzNTRZpPm&L)CrHG_$qnI%tbCtJZUVQL397t zpi7i!FF19v|feV{tzBs%&SrCTdKr@fd8) zc6wZ8H0Qqcp+XEDOGVx_#)ZDxg2bsanm0~}I&bv5juPNoMEzuJFV}J#Ig&mo<+2=Y zR^4YK$M&2Y-jZ^}Q0|GZ#Kjp%^)WV$>27yp^Hi2|M&ieBC8m|q)hVe9gZCmwoE*=go?RQ!9OibBRbN#nVW5F!REE`zkroh;?(S}oZjkNtmkelQB&U(Xx7>B9ZgeR8I+0C%s$r)=`5(!A_+Q_8?be#--Y%~}ZSduZW*Z%BBv#Ep(H(yi36&!Rh zx+-Zd=yLhw zb<5d=jkRK8YS^hvryw>nobr63ZDFfF%^tPAe@C)-2dk^?p?%^w_wVuJi(4v}6PXl-J zs5g1!H*O`YpxCEprp4Kd^KC>YIg808{2+Wndi4r0Q{8nksLj4`JoAo9KJNmlw?Kvb^{A}-|Qa>nL9ZRKv0GsPUvcJavAB5tFcD3=^1(N*Q`#s#8%QsB$8vTKzBe+gsM*W&<#7HV)8f z$C#~2vIy#Swc@|b+5C1+5SKKoq&G-yui(D;*~`f8GVyP2`aPpDFYusBl~ZWZ zWgM{;syo}~SWc~W{dKJnmGM6D1=RW_hFyPYH~ZaE8ex?(aQmZymE3&7 zAFbMgw*)lT->26*YYH2N(Y-=&!muYDZ(r!T2X2QUd}QPKqZn=z-TkUFy0Y@7{!hw| zDK=+QD?D`NJU;Q2b6^9%g3dW@@%^Dn=R~r`rDd7)f7hf7j+}R!k9+JxWtu=OSgDZd zzj-$mW@ZQjRuSaYG)(Ul)X^=mV=Z-c%^YOEUE|eHNn;71eIZ)Dca?>_k$q+uB;&{9 zoR81Ij?W{;?rl0(J26D|=EJ^wF$Fod^bRiT37ShN8pL9Al%=4Oe@*+RzdxjbtT7R~ zN5szc?s?lEDubo>{M*Cr2=kuw-AC7xM<&RlvFkA zD!Th5z9htN=A~g3s^SfI)$^#!in|LFlP1V-*S{8xQPoU74*AEClA-flcn0j?88j%n z*6A9qk(MJt@XxN?GK zR!vQ#SZ|!f7*Q;vqtA_<{Zw4#m2hZAt1%hT#f2**q*ph}^ z)?d~dDp!DAq`h_(PgI8cGw(U+4uT%BOfoeRZgjJ7V`b&?R`AcSp1?e zyJ}5X_PT!*{QKG&zFxhZM-+dI7@&0Mymh5sp61D4%7UPYz>0fS>jiU;II4gV#2F7E z!!V!p^G|?us?VAD#%XhS?GWyiI(if?mDrT_Ikd)F-=jbS+Jc}hXa1F_SI%aaB{?VZ zr{wRNTsO9UddtMAQPfTW!ezXe%A5`H4Os~=cV3snJdGi*@7J@{bR7E<80tP)F6_*( z8wHVk$Hp>@IJax7)As&rN5mzJC2WT~PM(O-OPoj=<)57hAs8{Y`CD6})~?aj_3aiG zFO$CQ6HaI_o@Gw&8Q}*NvlU1EbCZKlmHm>{jd#Okhdiy7m30>WkR%6bd8ki$<2_iy z;J+=`rl;9;a5C>^g#-`zwSHV$n>y9PN$Kbbr|{74hMc-&MVlpmWb(W&@d&uOr9Ur{ z2b0PU9O>{n*K300%Ne~(H8?0scgD%YZylcfaINavJhA5?fmRp^9v0?RnC|7*o$Zkb zE4{IKnFvjgL$xD^0<_8{&VI|$tH%ACyF@1Y(QL3>k6u$7P3uydh5Nb}9(Hxe8dt7j zab#im3Oh03VzU_;61+wc_-keR|ortLYJkx2Bpl zM|o@pv-4hyKVCg+?P6fc{r0=2_nV{}$qT9Gv-WOloks|QY`1LYt~}`;lq5=jqbV-+ z_VDw*hEZbLzuyc<^X&HjxJ)$9u7^n3v(y(X*a!ZDcc5vry#_4RCedP(D{zOf-FoH7`8u z*HSP5YQ$8SO+Pz1uwv@uuKL(s@#xmzupEe+yB;d(pT}QgU~_Vi{Z``SpDoOMOtK8C zxMojRhf3mKOMZ1n9-AT~cA~>OxgO8nWD5_Y4_D65qL2UJf8=^x_$s7bb-5#+I9N}`wks?j^*c{X}R1&yYjp=tZp+P@L^mXCx(T;Oum?H zD*Tb~?+RtDJ(VzRK(fjcwmo`6Z>Dt5_xiH>cd4#wD^b-yj0diD=jZLzCL8XKn%bpb zi|@}%_fxy3othto10AEM!6&(XbGaEN*8wL&^tN%ShWi%TUUir$XRXnyFumdE*4qp2 zD8&L5TusobU~3LT>nX&0Pz5ErnJqyT;pHOG^p`JVROjh07w z<-EkjT<%NA+t!N2q)el;;c4++*L2X-K;&ds#g})lhz{u=zEIPNckYXR+ z2VybD(KaU*)cQ+pU=z8n7-M+I)BYvI_pBFnL?Z^q+t}XP;L!xE&-}AlS%KB$QNZvL z(cHL&TLFr_%vadVpU2z$5Twzxnm(KTBil^%1MYd=F#jxmS`r5`MD|6E+E@}xqW6cS zMnBcHqsnGPmtCKwk}Fk$*0oGZ%@7PE$$vk<!-dh7mwK0ywc6;1V(f_AsEB-0<@DRCnX!KIlz zvgw!q(j3404dp(+!p5oo^;7Qx%?iMdSQqf@+WpZCrf$=puHxT`7E78+F_Li{qFY5^ z8ee58K0UFmTyD1sjZVzDP2CXRGFl!@4}x`2aDQ(a98}x9Q=ubIPWd}J=LGWUW+eVw z=xnpJ|M$i00Qaq^ba#0O#!uom7*9Zw~9jn}quDE39|vzFck4P56rsULH@QY`IZ`z%$4@?h}6M^x{65`;wjZ z?2P-l?6mz>w)lt~%u06((?iaPh0_dE<^DZdJ|yX>UA=#yGp%(ZIa|Khpr>_t&JX!> z9knncL#w@&N)U9s!{b#J4BJE!~x2ue8boAZ>TB=ICg2(*!QM=7l5f*E2CMO5mdYeUwjNf`ne5 zeDV;`x!NoytRB56*lqV}Nlvc+J%K&nzeNoF9*K**2HpKlc_)%?q0Z_T zMZ^4^X{D2O+#MHW#mq+Q&l1~U$wZPp#LuP?>^(;8a~gi8%wVf?le8X^OkR30%^-)* zIW$EkS_piHBT73->wj@)5vH^J*Y4!%j{tF%fS<-H!nDHv`J~%f^(PoeaB6tBd2%*r z278K|)6{WQ2J2yAm`;Sr8Wev2*DGN0S2noXHY;~{0`s_nQXZcHW-bp{L*n!xIr=VrK>pL+7IlG6O(H2H76?6ZKZwfsA z^QQXq@g*zv^rjl=;+=WIJIV%Ie_qo4!vL}>56meQS@K4oL?~BA!ulU~=Ha`CB z!c}Pa8?nJPb1O?nuvCK{aArX9X3rHMrmz3fU1U6qk%!*@8p(g>aqT04?r?kYD+JG)>5aLp+WpawwzhGv1FNs# z;ZGG~Y?hjP?&L#r>bJ7~|JR%fLOTBBQREURzY1y=#xega#)Z7WfJO3trb7)zEaTGA z_(j&jmw&ch{JSoz8UEKIy#>F*!r~`{`KKv>C)TJq2ti(DT9S{ZxhUmfDSV`03wWy} zN>a*Qy^fXyxy!a!u1!l{68E71Ve$swArq7U;m2D{xA$UuOHizzRi?+)5ssdagKlQfTwP2Xl{yt8uVs zR*lUGOdb&Cf^vJnvjw*W1;`&L&Ry>XcZnpu{Dd=Cl*L3vMFDkNSXda`8XNx=+&@7E z)CC}J0>&7%|3;^#OcdT*gUNqo#>fleLkUjn@w zcFD3&YP#4yX4DFq4~#pT7=Z?U$DdsEThobAI~uJ(ltG5*>jaC7+|a&5lQOhvR|ga& z0~_sn8bmWV{(@v_!g&*p){k81QWv$vKc=+wf`0zcbU{2NYbf8oaP1kJ8(yyg<=2Px z2~Ds4U(ECS!`0En7ub7A6{%l6M6;v}kyK#_VdMp6rNDp7lJ_77U`s|^##fUKo*v@IbOUecoR~gir>;yk`5Wt(6^Mw!qQ6H< z*GVzSUyCaDFs2FaH&9Lo@C;MTBFA#GV#X9T7KN?TA|jE;EIR6?wOVo{%OO=kxNL zS*MUo9q`nkcpvaQzrw`@N*SW8$;`U-QLNq1|DrJcjmwe_tYB3n)>oQM{B>;(5FH)K5&TwX zUQ0_#f{tz7*m3tCX9ma2z=T*TQS=BTgoI+g z?Dj^84K{9U7)!5-(zWVz{Fci@_ob|{OoO~RU$MrFQRm6=@Qqs;;E_BrfdFt5@DKf% zGl3OU0qBb0$jWMOKJa9Z?=xOIGA4=a**g1zmSyiNoar*>LUaf7Q#qfc>6>mzk0Wxv z>MpC#qkExh0pLObW?N!>1`PplKYI0a1W*IT2oQ(xnb&ER4(vUlQglbBF)>*Kb-G@V0|aEThtlS$XC+}mCxd8U;U zmutKiv3=*;6072YogJeRQ^`^uV_eXAOZ66+r^!3y`S+aC4)fsovO*0~mO~=?CTXJt6ic6|)-)n$%u){OOSb zWur|kZ-ICZ7$X5q2v4LaP~lv-aKn%@yZ1Wq2Mp&iyVE%EClBXkXtUzKi7`VA5%0t{ zj+4y#h;2+DEF5wj`3~t#3=N61Ec5P=_@MFVcQnn|cRtCeniWbGtTRsGnijCJ`0UC} z;^vAqJMX}K!G{|BW2HEt7_}~5J32agkN4&!G8||vaBzT=bK~BecT`5GN;e5KHN?+t zcwV$*%j(6^X0UlE`vNVhf#y==FBhVi{i4cXTVxLri05$~# zU%-!IiGc%D=sig{fiM&}A3d400_Bhl6L|wf{t}soI#6k1A$&$>yq+nL=OdV2w)~}Y z>i-AFB35X77Ye|hK%W;zD)8i0nARA{*v9-DFzEm)i(%M96f6E)IW0SSKzDAIC33i2 z&KH^~_!M8eU+mR#k@_m1iTvMF)+=E!36qggBBzRGHso-Cb~!+j=bTmuM9kpS0kX~N{9y`hDkdU4knsa4 z-Sw2&;vaIupOTd24gA1P;R_H0r!iweqjPiL6BA93G#%b}IzRJkEA@&2FL|`<`K(u^9>iYVYo+`vN(?jL{ooULU)lN-r|>5Z@>o#PEqn0^1?s#oTGh1EeP5 z-zvdXr{YDzztw`Pt~qajmN!*l%R-y+*Z*5 zM(V-HRpbR`#9tJnn}^po?HZAqYlz9h)Mp~IccaC~`UUFwf2%8ZB2?}2L#($xG>F9HY!fK;0{Kma;71(3u6E__zjzSc{zTdNoepfBbEj-&w93KS)N zTZrP!a4PTHjA~cS+G7QHY`Ngym;~-9wmobfeILxL9-G- z#z#jPNq38{fqWe_MEC!6z$JkH#6toooG5@q0C>RY=-t@Y2_5*YZg;DJV8CtRXN>IM zI0Z_4pfbC~QCRuj1w@G_hEAyzY-|(QC3@N+txdf9Hj6-!Y%Nb7ZKfJ0lVhJxlPGMY z@|oUUrfcF)1d+Q8CE5ii^WV0>r)Mok_oN8b^Mwol%p0TdwwfZ{W6VQg>=Ayi9AZ@vd^`*d2G5`(gMLVP|YoSTrK)& z?*oQ}0Dw~fYV9)kq&@DeJ{X}fwUqE;h}WcOx)^7X+SXzMEKwuSX){;4CYbzW734k><2`PpAip!uJsbu1l(8^1-^HH zV*|AHK7>dR&_eQot`4-R@@c#~4!r^iT}W9<+oGVaux?k{sYg;j){k4zyt#5Elw*ynW;%>7e)5HtsS2+o~P8BN1c~v zvq2K|?OS2*m@*t+KkKExl$}?dKWxnJGqwX12M~qEgbUd&nU$&Td20$H=D~`8_;i^< zkA?$QoQV4+yufzQh%~a^{ia+we_uOLQs_>c`NqIO+q+gT5>HdaU4%&cXKCT9S{HKQ zdd{ele9jmMKxoahjLn z5)vJ4^DyI+8rHx9?nA{kHWn5@O~ifw&J9Qip6|!b?KUC;6DwR>?0SHJ_u(Yci|DLf+DsW>*3zM$TrMjY*sR+nr88qpN5g23;`NdoXY|hKLCom#6B($fVO~# z3*rf|)|zJn|0BkV^39tz3A12tH?^0uAiOOjm6ns!Db*Kpb^QmZ#ukY%AQEI_(xiYgyi+C2qTzzX61aKjUD zJAp8n7odOmsO`Sn2x9S&g%C&eB1sxz->$tV!LaD6iHpGss1;Rm7NcS;<2VGQ0-$UK zLdz$)#>4DvGEfnM`3LX_)?7um9!kCF2dI4`B<|WDldAFb9nzlntXi?O{w8lz9pau> z)ynxj{FSj#lCpjB8&DvE+;*iE5ETH2#Qd>feib3x^L3IhrtBU^ zVBZV3)|4?3m>oX3GaWUV6h>PtMq-bIKbV7nY3Lz94gxVTFLngA+7NFfWG_^Hg(A zUOE|!DNOGiCR{h)z&MJ!U%dfbttUfopgCu$3emIw4PTJpvl|?g$$VhRxCds-xIML5PdovxJLkL)RK{oj@#?O9YFOw zJ^4vOfi2W!PNFU#_NqnZ0afwK|BfF5UQ1snkkCG{yr+nJV^}IC8xh=wrf>H<2HwHR zTW}#%dMdc^72Q`qO-A*BE`l75Y>dKqV(fZ=wRq~C@~PYajSomk=NDK_G1SbcDFD?3 z@;=A+y8YM}30jahneYpNky_>x04M@-Oh^z_`g?b#yorj+$HOpQ)t?+cV)st})?Tipwh1PaK+^DoYANobSG)5)DbkS~i$hl3_GvC_ zA%L9+)WDY|?SPC3+T#Jqf7``qZT2k%5wWe|GD`|z#@06id(2AeLs4+1dCZvsqyTq=M#52OQ3)1TR6?I|`J zae3%->%O9gt9or8XeB;?-PdaXV|oAxChk`N6yO52*2Er7@cfFr@CfL-Jv2|&Himw&COY)bXTNIT0GI0TZ8 zv3MggaVjuf|NVOh=7Rl`-#}b%Gh4m^#%#ztDcLjjgMZ|Y^+*gpywS2MVUL@hxwB}XUb ztW95X)6}$ik!7@^!zg2f&gA5KYc?(`EqAA;HV4`GcBYc|JKd7iO%7 zp~=61Fc1C2*RG2BEmUtuEEonXXayF{Ig(#fyiW5vya5y;C;NVYt^kf&j$qM8;76Kz zs-dw!51NV)n@e!Uq~w?Atm1uCNFQ)00GDe!_YQ;zPd4YkAr}?`7Uy6_28b!Zyxv$u z%#CWMi0wPIT2ZkAn5f5YLZJFKMs1%wF_|LuBx3tV`^(^CU45bF*2IU8^y)xQpB<|N zIOLj|njoMCMm3-jEWS8m?p4gq$}$9eqjdLEBxnW5{6MG~Apt2xx-JK006q|nR5x%PhuZJ@LS+OKJ zcmewlzNTKrVdO~q{;}$FM2!UuG{S$o-vV5eNW!&yCDPCGdKxBmvaZB0-&+7I51#_> z2|Pq*N#S++?Rad~Z+gKjU-7XzXZMbC(j@j1AS3~-(V-FvKync=%C1zv8<|``%e94# znKj6F_uPhiuadDE*m$YzHneeBU~g3vZw?9C*_|3i@?FjtMS=k9>tTIUG<5{2z{BNS zjkWGuDNkPgh0~~CJIyb*Nm=fcICi8XrHk)0^gfOf@UU+BU2*b)lb;&8dSnF&?$_?e z<-gTvTxsKQ2v+)c8#z7VS)|0y0QN6Ht4c)yD^0)>u$ij_<2fLLFnuiV)Y(H9Y#ey9 zs{sHqfEc2E`3;~%cAijZ+eAOW!V2`j$R7uf3A7_{Bg@G%CgFy5g`U6aI}w!{r4X{!+lUbY`gqcEwcJpeCma`S}-kr7*E{ z>IfZ@MpLn|s<+c=UpJG1eKRwjPPc_N;K~EUAUcF7yEg1fPW2_4hHM8W@+WapJCf|` z&t@VMj*rrjlGeml^&UR~JRbCiqT46b-g%zD*MU-icLA0}AV4|V+sgBo^M`ABfllwW z>%S#J%}c83uGn}rnd0JS1I~hc*Ky5VTmWfT#K5I*yhvm%(!e?u{qn8aCNPev1gxWB za-;;X+VOgPXt_VId7AodKYqaPrw-r8O)74lWE-q-H*% zW0*uy*aJaNXF8j=%yfhV5|oq-o*wjdA>sW_`_NMD*Pxmthww{IkgNljL%_cc2L|GQ zyt7Z_JM>(6$Ul9;ATJ4BeB&@M<%qo0p=9zFz5~4PWdQli%dU-vdAa5s-PkrwwLiEz z*8FV+76RTp%d?VDE1*R+{*s$PA{g>GS?L7B8cVNw+8u~LO3fk8oTkh$GgL~D~MpMKj0-UpZ*JQ*!{s>c6cZorl7(*2>!6!QMK zXPCipk`Ys@Sa}T8UeH6oL5%8RnXufs*7 zFl(MiNbYI9JZ+Xt=GgDCHR47(>J*l&Din0e{6^kBu==K~t=i)8u?9)FQ71KJ%8$)T z>fbU;Frb}harktc<~7^VUvz9QWcNgvvwt5RP#a7acm&iC?6em;$0OH4yr_8nZv@Is z{Ar?_JjRZ|4o-l0_6=Zz{ZkeBf7z0HLiZG33$|JzRk2as4{qD+13~Dg8Ni_OQGAeQKjZ|A`KIPgiY# zaz1bmc=9m;6iR>&YpSBe=?n+^wqRq(9gN<9%B%&L3vO-=1BMegURg1ms_{f`>8=yn zaLR8$HMC+e*|#m+p_6dqGm+=BmLREv6<#j2q{(1Y6jP<%H(#((q;@(*m$n=nl{6@I zw!B=^Wwpl(TQh)-;Fr0wzy-CwULY@<8Tm|=9D#v6oC<`b#^qHb?*EaHN+JcNbf>9o zNmi9P|M&oStps4;AR|Li8QXoM@prZ{eBaP{izoj(0N{VWNeYZ6z-5pM6;rSdAaPY+ zBYjm|l2XNg-3hwh({kRE-f5n15fM9r=-&XtS2zx*ufc?g&OWHmH2c*OAGMr!UvEts zd;xokC!&2e^WvtBEWu7_X5aU0R$y4gD559G?~ts3&kQgY1BF!poVNe$mMqN}(B2xH z4uB5_Y%#*ITSMt~Zi_Qp&Qx@?{hiI@oWpfVP4T04`kl*4M`P)BI3F4s=U>SCSn6(f z`6r_c&dZnXU=Rl;T=Okn0?kiU*(duKfXzghHtrRy`?4O$M_1Hm{4F?G1I(RTHcgXi zt+-%{Ph;l%GA?5tAZm9M^K-NO=jQlg;S|Ozc-Zz6w5@AVTlWqw{#8)48n+Y5>i)}N ziEIfQ{uIMtG@sd#FxOm7+G4{BTo`lhFy`%dE-*Mul+1R@qV3t9LVe&x3NtI4Bm)4B zX~;|fN82gbuNczW^O_vZbsZ27K^OJgS{YzE3l{mr4)<6~aE>7H8U_ONl3HGt!<|f4 zYBn-Zf;vRG)VY@#}vpg(d_Khmu^jK1Lwz?Z^`4`NK_sdbk zmhW-zZ63$!PI0#h*dDb9+#NtteOmPZ65P9Yj$mr{`~14|9&D?uz@#wg)Qe`1f#nH6 zMDrm|qv7`uH?ePDyE*Tc4D6K*L>4p1X;VsT(|r4WIDqEbx@iB>&}ue6Va;!C4bG6a zhuSy##;LLqW=|TZiSBMjymZS9pEYM30p_GaD)&X(bJ7f&-P-R=Y2FWVBJijk$BypA ziNJ)(;_T`QY>VB1r6E{MEDpRg#J4ms9{27238bEWy05M3U&k**OIypN90niV>)KOK zmodYTXMqju)lgQC10?Q&$l;e6^f0Ml%V71?7$X=3)%9Jj<=C(9@A`3MXS0;5z`mjE zVu4zf;jX&uq&fTNRQc7LaK!M`Hr%co#C3Pk*Gjpq6XKUDX^0e^4TF&p5eN|bww!Wy zvcjq7BPpAfCaMY43T$QflUG+)fmz~)X7<&7{fm&LKN*}px8d**{IicAwrlS#CVIIZ zbW49@i#=LE#Rw!P)$K*ua@cyvgzklb^fAhxHfA{>L1AIv8cP@Jk!#-}wXGV}beBg) z$h(^=K>lxJ?@Q5~TZHg63=A4IR5*GQnR~~t8+FyEjR~iXIN8If&~d1NygLiM$EUw^ z<88(A=a^mOqS78Wvzv1kEJlg#0b(!^lG#Y2_H|}qd$xGECfKTbPU?OP=A$*XRj~Ga za-9Q%;FtC8y2aa1X#Cxi(F2KkB5PJjZdG9>VAK;2jIW9Xle*DN_dt-k3_>o zNy|^k$VEfTuYXV+B=(yy4%jiPv~H7!O9!C>PQ%~c6gE@wjAJA% zpI9`g<=U~2M-x6;5K)|ps^lZjG-eQkeSP2;+w3T6cR0`wW})*yGZA#D{L__`H!gDk z>K2+H`<214-vV>vD}%GW)?u1?vs?Kr{}M@5Cpu9Vn)E6>h{Pqn{F1$X%||(!ABHx) z0S3?u&cA5WpTap`U5bFA;{?TDX~JEE`h0&*I&0LFao&mRiZq_{#jD3lj}x0^6p@~Q zt|8>GAMLoVTtC9|#th0g>-TDzGbbfsp!U*}FYAdDA3Ya)!HN&0=0Vs7j9&ruEsfie zj*00A@JKpE%#h)I!5R@H@4+oNdL^%m3F&~f;#%qq!06(g%ASSKk={ihbjz=6EQasYsb04$K7Bp{%Bi43N%QG_&-<0gL5r%C}0`ZAw9 zz$^zlD(9{qTc^Od8w{La&gXrwcEH!@WMJlkk&Qs{(I|{A*f6ZhLh5+#qsBm>R>-$V zZ?Rowf`#vhL7!wy=O_Cqh8C4IEtlCc8I`r`gvRLj)-oyx#!udAv=EIuUZ{j)`wv3u zT*il4)=5{lW?ie%{BsU8kiFEPPdK!oQrV_aX*KQ6l7;C%&tZF3dxn|15fqAfSpUpi z78ZgGE*`B`@+J1xT|d#pzfV3GLVUTA^yPr>E9bKZSA~a9@SFq}o3}ZN`H#PqCOwh| zc+AC+_Rr&Wt&xv3Tt7}`!9fn{otW@25cB=7=jw?I3ruOv+;pA~9rI+Z#(^@A`FxjG z^9X~TC^_q8{Y~-?fn?kw$J-KL-X+fMG1g4$#7^tQzgtpeUsBaxGGG9a->L9+qhcJX z^j1Jdqat^6s9j^p8ef#YoroLSlx`u|GOnC994A0)F0>ry!a>0wK_MOiw}jE;`bf8z z$~mJNv70uM$(NcNLw=xrN|`gtm@~pb+m>E>N{tGFzFxg}k5Mcy%^!p60@2G8`V#`P z68=%RTa+ciK#0*qe7j{}(r6pofuJuJT}!I3Fffh>*+zKJ2H7BQkZO9%0ruShx{|gL zHC`I+zr@R7 z^2a72ONfH_Vr~U6c?AcPW+meCtI#erj)KC%6Mt<+Cg4%Sz~K1NFWZD1E5wDDBz{=}krxwkV`)&!9dQ-3oTXH`liSHK zNH9idgR#Y%x-&iyo)h@#TK(b5Ekg3sHL8?GC3s%cMkjK9l=YGzrF&&!+npfQRF2u@ zdP2zPSSgpO|78@d-(Sv^hJW`kQ!CMFTu{NQbJ+Ap@dJ*`K+Hc-Y6MQk^ z#tqJj?yu?=_|Vt*OL3tq%tmdX$|Te~Y$aB)er;6ijV#QkebUCM-NmwVyRk=tn@k6@ z>C}UO^iI788=cvmgn~0G7^xqfV(pz`SwW^&N)>&|uA+8(pC^i5W2sf3L(C@uit=P= z>1wM^$b$*$C`rB;uK2GfzJewNV5JLvb!W6~9JW>M z=-QDPoS)ghb3eC5gYEhBm6S9cWT*i^^a;lQ=a-VmLScX`H5EMu_<@Ct9cP+CmA`~2 zYiC8p(P+M%WY{Q3u$vt62zN1{cxG^;^6^16fi!+Yp(6J_U)I7`e5+2*eXNo&PEPYv zto-Cp6{kD1_@SvvsiOsHuyjGhp2>o*rtx{j-xq|tswenzgK4=nc^7vD4L_|p1slMZA z4#rM)Q1ro&YrU6lMagO&&-}}QH9&}RELfCPn37zS^+BQ&J9xeLfM!jx^`8WV(@QGy zU?YPV5h{G4!0^y1P!>3765D3U`h5t&qTQG|C7dc2M&Z3N0jMf@w|r^3`~Ku1+hktp zmOOh2^oI+lA|W`z7rLm6A{5!Fyw1PTd7N5EzNujv&2=HR`(4;vNY?NYyg8BT1tGxJ zKu+XM#z+3o@2!siJx*$_H*;9p5Kb8~kF1XU6hAwX#zi-?|5|BRbTaL9P_Z$d(|k_- z%%Md{``^G=@76tj4SOJtAIj@5X?LD~MwoP#FZQdxu@$YSp@sZdb43rP9;FLQbpE3| zr!;RX!Wd?h(}I67aW>sRH;5#x$IE_lmr#)95&jN$p3tUWq0x)eBv)@nR9Ayez@*y` z|Gf(BnA#7kBHnpTr}~_#@dDRw0Xo@!WV|Z}+ej5Fofvr+B?(0o#6?F;EryIXSck>9 z?PsWZ()*SzGqc2x->c2#v~ubvFN5+}>R{e-u;aY_Ax|BH`Fs{Or>svEC#0KdY{r-% zMfB50Y<7+vA5=R77OhI;Y;}V_bg0}p?=sc!9Hqf;pG(TDYPkNd7NBdN)juEQBsvVs zBOA|`CeY^P1n0L^!Y&L^7A_wW%eA1M5v0g;I7m^_BE`IfeNsF6kFJ?bePPrT-Os8k z>-5%8Oexi7TS`5=y>-$7+=xy`XS&^Y`D6UBxp7m{Z))bhKlM7b)J7*-b|=AWQ6-lG zG8!?AxW7mv^WNUx&wzimMk%!1#rzwTJ&hxzffV$90n-7f8pNds(j4F`RxwLYczNze zaPy*rIY&8PmF^XCz|$X^nU#fNT3P9ET!~RA`G+4L+w!uw2Q}BVeU`S>@5kCJhG7c{ zeYJ-5j8P+4=4F9N>TmSeNduipAcFt}e99R9vScxsS&pGk1$|xL1$%>^s^0!r$@j*R z%fzqiC#dAxOuO@inTVt-55auufzb!zxFIlW>rV$8fNTy2ox3(=n~eSW^Y<@5S-kY5 zxg#JVgJWyL?MXfpEh$P7ZJgax391PRKr`?AoPFyMnKsYYzH0kJ6^}n1lnNcuh_4Kn zz93ZPj+EZ|BDm~tiQ9ZNrSv5n#3sA%u3p5?vlVfP!&juQu8*krk#qXaadsQA>EYS$%+XYM#BFsrUe3DpLc~K; zw0`2D?>29C!EKNGs2Km}@*p_A1y#b!8Uwd^=0VHM*Zv;@!qj|z3&c6%Q~d)kp+@Vs zlG`omEJ)XUC*!e$;xUR*VCrQJQW$U?ufgKmS!BiwW3+(<3u0ca!w6)0^EmNMhR2-g z;Fla|+mEu63%n!Z*`EuQ_2w6C%_OX};t3$;fB6{_Q(>*t&JJ)P-+q>Slh%8s=2XA2 z+1#CSOK=d4&h5)8fA;E4nN$ZCNs@R0s5Xk*s!B*Vt8>m=_u3MOiH~~~_2Uh)B#EMZ zotr^>_O<_nNBhoek~${3l5#0i;~6WUuppEIr`VCPF)Mr`&%9eJuNSiS8j zb@6NJ#r&29%7;!)u5MoKIe-zq{wZjlN_+A%vao3IY?6t_4HBi`>c;`Ddpq6}#VYbl zet)kBfyr2TrYfD<)p}gEgJ7m2UU-wMF$VmXQ|_|cOvQX%-HIRY9;F;C38emFDu}a6 zqt-1ONSAQXrMlxE&QI=4if1ob?*E-0eL;g7v&hqNDPg~m!yM=kne&$BB`BT*hcnaj zRQc)IX~7(2$Mv)of87#Jz~A;m33|`DB*}S5C~sT0Fl?_uBEQpS9|dCX0GNL~Y~!C> zZLd7tR~C)`eBbx5?7_T^U~xCfV{z|+@yhNqE=>x@37s`$(Hf-c@xN&%s^~H-Pa68~ zDSR}|P~G(?BZi%+$o?S=L_{BN3=B{MJcotoBPbe7Nf9DUhEc?YIHuMY0q;wuQ=1hP zLaa-j8bM*4l^u%`GR~48BuuS@2j33&o{Y2i{H!N;$qvP#$@BSef#6iOyMsiefw~8j>BGPDD zylC2_8kXWyZLP_=seXGa#}A((gz)bTld?pe1Sy`Un-07n9OoZd!?HMMa-S<|%rv48 zl@vELy%CeWM3Pw#$~$TL{Bx^yK9_v`2Qd$r1w5*K8!;#7!nhaw1cByxFHXE^?$-yx zSB51?N3@8DZA$ia%U1rKD4F(tO77gKA>2O+13=))0YaY$97%zQT$+;L` zrM^bsBUGX^Ksb3SUV;}=NDeabTcjOVkzHybnX`x)%K5;Y5*(>7Wz4?9)V_b@;E_Z7 z*4hc%SN17Z_cer^R;`3wf~SAvD(BZ83AR`Ot$~MK05d^&lS6rFUeM+q`nZjxahxbg zeVQEyd52|L645w1Y)}_?(B^TIdpj^A&zS<_W7`t9lJWAr$y|IuF)I=xj1c5K(hest z@X42NvkLN#?}S^#CG?5mN8MPV=rT(_ERgn6%US-D@uNLAx12VbU721QIpf^)mxMWc zqER<;(Mh?qmvrt#auL5xvi2^UXPBxwDHRMATd$sk?F*SIp6`=l<{h-jhjFq!lvug; zzq0Y-5D91z-rQWu6hv*yebO|-H)+fm#iilqz65U&`Vdw&$Uma`<5Jvs%~0tf@6`wA zNzO$jjzy)kn@S27R8H$O(41tFT5-BBB)TRoWWg?%N`BVix8HxGMAdB$+)%QT!M?wg z-m8G~nn_=~-r7=FNRi!qV5BlG-K?l?X30Tf7R2K2AJonTEItJFQ>=~+cu?TUuXEft zI0!KB3{8>eo9vOPoe{rcl+qZy;!=yD38o1ljz1*inP8$17YH*fYQUunr3^DgT6epi z`baMrZxqpX+P$dDC@TU3kbV1|lQV+sJz0wn8!bqIC0R+%;LVQ;-iI}r zS9GmZm<%pRP!Lh{+R*3xxy~#@g|tw3goMWa!=zx=*yl8-E9p;4${*wG23?eM`QA*# ze@1c{GxiuhT6WYz79m`!gk=^>Tq?gjUzL_tja%23&_6 z(;rJ~%+1CiBs0$^nSQM*$Nec*ePSs`i%KBO8YO2K^G!Jz0IlzgNn8-&1=pj@Y;PhB z(Eneps-g|y>52$Nw5=(K{{NpCYy7z;=RI+QI^Ol7u7j-Rqd8vVSj*=)KoqwZ9R#Po zQAexzow|OvT~XPrzBR`-ijG*xex@)1E%F<&2}u<9QO`mw`7o0h3AR2Pgz`iEZtdr< zyIf;6Y`9CdwaA27eVAe2gbPa1qMp(-4)u6ch;8$pUT+C^3*Z`#r-Z%;emhdNYEa3X zoVj^*`F%O{gYGt_C*Gn!#G5dqE&;l*r2Ui86j4?z_V0x3-z`RzBrB)BVqf5qP-{>A zR^izDQ>6Y32HL6(vwlvFmE@_KAP~P36eN}RPn7xi-OFFUpF8~^`aji~k6<9I?zY1! zcYp_OdbW}eK1=8%Ok49aXq4#xBkirjqUzds;UOfXLqJ+2r4i|pMiEdXq(fA?L%JIQ z=~hytK~PErq(w?Pr9oO+;@so&#(B^8=eIAf0fCv>v)5j0-SG>(YcH@+mEh6e#Grm8Gn(jOj-(oQSQv8{iIRKJd9sOHs zQx*QJBzKRZO>q{qN$ZjdSjvO=Yhp%TIulfNWjm;`$3GCsXjv7?8)2gqme?GzsuZfz zqQ~F5CcuIdsL0%ZLYf#r+q`pd++x(oINQ*8<#7Rt!H!9!^1m5m=&09~8zh;jqh4n} zB&L`sVk5x0iGoO%iYEze|L$&=xFZ{iUK83*Ca!loQCCW%aVhat|8zKoI-x&4!jOU% z3oZc$PTS9{sCoC19Mu>!u}U}0zvMIm;h7}heJM)%wxGigK1KA(OE$9jW9@ELqy6`X^n{%<_a&FYm^wrL-=FgY zQN|$vvq0!(G`AeUz>g_|Np+0D(Vt8*3-^`MxMV-ShBR<7138M7>L(fb3+i*U-l}d%Y`cuC1JALx@;=cx1I#KL) z@&69V6hS--eT*?SDKm1Y^sNMRq_{`6)o3rS*K3BpQVQBbjxF%R5bghr=D&w=-t2Tu zoJcm*wT$L}&XiUMt`o;y38(+(=fP}0k^vjgGQVGsLE(Y=zaM=^|NH`B>05z3lP>Q# zv#NnXqurYS>xJj>-vBX@Z_zid;O~`2j&5fw2AuleHy8VXerQwpY~+k(pu(JyNKb?G z@9RhRtG>*J(DciJ=s$ml>qAE3k*8Yz|98l~iX+iHplkc5ZARc}r@gf+8iM}Lonc-_ zu#!jkMoXM_iVEDt2m*&6sI z?10bXTndsAx#N&nTAVvO+%iT&*clM{K^|0dJtnV)7C^e^F^P_a?m!1A3>ZKnTZ_$t z_sMMT?bHa%1fAQGevB^yxc{^7>1)YN;6c~Y(gMJgnR-`lQBeiR_hovudz&A1KU(X$G9Bg2C11}j$Ck(zYML9EsU$=7+sAlmsbOhq zuf91PT&d1y-da5V+u+YACVb9!_3I&Ui|{RunHuQY)>fmB6Uwgo3?0U)O1%74@=>M1 z=P1lpJ#=v<$SVMs{prEjjN~+WMsO|y0VZ;QAv=1{T_ZsD2ZPbQweH3}rJn={r1ju9 zn-5S53;~V=*czZNByv2N#~Y%jT|r{<;YZ_EE-MP{jaGoQfi}nnIk_cQP2aiQ(;xhX z(FQi5>X7$#rVgH3R)UucG@pfN@UC9H#ldlQybwKF`~-Oc0EGAS?06S?a&R6qJivxh zW*$O1Jpce7VAAjxD2N$=8Gt3nC&w-*N_v7~z0cV~4rqnMJBhy>KbYqE@zquxM%iFn_+L-Q=YQ3Bu^9-r2$vM=~zG zRe57nOql1L<+&YFHUPeqbXrwwe&t`+vuT=y8SNnwxTH!$JxQOLU!mxI(ox4QIZ`y~ z1ZS)w^GFDXoD0l9xyx3pDQ_>>dR`_s3p!vqENmot|X4JovpZa2)we-38DZ22R`ZiJM zSR$=wryToe092axZI+lkjF>ze9^Vy*yc@_3oQIMm02`Evpl+`D@_+@yR znwSVUhZ>~GmWcCJ#FuAP06YW;QYxUO($l{!IbaUEE7HYwJ(c@=Ah9Rj-k;@jP!Q1U z0)$@`T$jGi&ei297z41R!9YL-2mm2)0NwAgw})ZHV+(E>hKI>hhnhPzzM;wXRYcjGTR!g*IFRvL4}fXFHvmW>sUe-T#8L7x zgdgSJ%>4$ql1t9G{xy8+>Mm5P;Gf{1hlZGVAk(RMM&ig%9MZ7>K2$}z2e^jef*TI< zL;0A99XGLpv+{Btgm1%L(PL&jdr}h`gcPv?AmO__TanzJ z?ox$D(#9CelZKe)YFDg>j(-*p&U!iVkgAk9kj6JLT!9dN@g;6LE8a0KXs^SJ;5 zU|*BXBkWCpFLr&GK{r@C}zo=br~4nroR$yN|u}GpoWf zXf^rA))J_Qg0UPDjC67O3s8CKqIRXnzRADwQ$MLuh{V$Ir$U)>ecKw(S%>`x^z|(}j=@Yw88>AHN5F6x8awUAvdn+bpJXxKO>C!>iY1j5f-JCZ?<>t_MjZTzu zNcBQ}vd;Th!FQw&RG#0!BEEiI*n5AGNC4QKGBPrl9nS=DPL!0CE>Q12x!*(cjUvL` zCtk{|GX4cF{OHl6*W!-N{#U?6(;`#(qs3mfzHh7B+bb^^zXSRiCfy1^Ny4~s z2g$QN0HKJAP)0W3e1Ji531|{7t5TXcnnk?&U{+VhG@Pa;azeuX@i@Aqmu;CBQ(bUcF3RJ zx+cPaC+8%h4{l|>X@cNp2xJT$J-x^kaA$Rg(E_ME>{ztQ9*T^EKwt!Gy(ik*z(WIb zID*W}Plrxgk7Iu*9z?Z;*RfYl^lUn7JstF*6q5x0 zLbNTn9(drXb9~z}=1W0(S~j@xBIef5e~gUKN_dVL41%p3BO{}b(3m|HHTBd>^ecn5 z6O)s_z>IJm>2*l+5&Ar@+SEuNsH#F2{0#Up%*x4x6cnq8MZZrx8>BmwwNm9?rX6eQ zs#d&U;}-~Oud}^x22`~7>Jr`ggJYzWA&Qjks!WS0@?{#UAFhh$_gh{Y`G6$rmegx2 zP+s0ndoZmPn*9O@{s033qs!0Wa32{ON|8QkuOxXbFf1@npC&aktg?LVMir%Z&_I zPxaB0%f3b}kC%#xRttDF%@OALy;wJ;lqvjBu>+a;Kll~r=gt~xrS^YV`*`C)9CNNx zs=y7;2ql?uKcg1bz($iE0RaQIu1Y4|5n9)?3!%a47_p}}xrMfq42}+_9Lu)6oIZR% zNna6r4xnS*8lauD+$aQ(yCmvDHsL{ZRl*>9g-9G4UJ2Pd^4ZFDQoK`CHmmSVGZD@B2U$&M}{Y+Uh5J?@sl?gE@=A)Q@8m)l4+`nr` z)mvVwS*-oj$7yS-7D9(Mg$PMQM>84kLtl^4=FK?HOZW9Uxox*K7Ag^OW5C9P*R%I# z^{C6Y^R1s+>bH~Rk?DxX<5sSx9mg?Q;1V|gr7aOaeJD|;Yp!-Za48@v+kbtrt^b?p!W5MjmwXZ3iSX-I#F%u zHBpK38pbst;Q-u+sS#?hG=f7f(7W`(X!tM?i32|T7JE~HnT4gwLhkQ!k}J!1OnH9p zH*RHPVKMlEznIn@tjM9oo;3vf{qt|hys%#XL-@{Yz3$K$1f*2hlrRx*hSq>&{ZLH} zc)USC$Uel-^#~~0e<#{N*2>GO!K!dS_txjfOfk1k9$hnhKTpztLfpd2Vm&CLtP@f= zZag^gKu6&>=k0h#iqu`XnSd$|nLMvGlnX%%8k={s9X#^>^QM z%5uLkf1R(#f8`@rxXazv#{0D|y5;INNX_A;y3JS`al6fqL>5=q)ok^z0@4PIMZxtF z9+Ea9QhVTZNih|g9Y= zkT~0Eg^dyoVG#PdSUtQ&2u-~Bhx}lDYDxKI6L@Ps&+@AAV99zE#w5-}xuL}Zw?I#+ zLmN3cIlvq1K7DfA=w3Z(QobH9aoMK2z2Q;VX_g}Z0$*7!lWEB9-nL;^9xTuiM&B5& zeZv%xC4Rmyc!yjkBKpOBd_nUcz?lR1cNVZDJ}=P{ba&GQ%zcD3=!$cks8TRk8rS_!?Y|;mFLCFm4I-wm=02xFIMk z0LmZs`nlTcBm!DolCCM5+M&!663gX-wj(3+Li!ozzP|HcAX{Q%7yn*(<* z-_d4!y}2N8gK@$GAe0+D76V~gHtV1aeNF4na@VNdXTN7Q@7e6Iyx09Sh57Nm6<-Rj zfmCk`=5`B;U7<|Fms+9b-66h)ZhQQnOm^i#nL|PbAyW!r_0jlWde(09KNuuc8loNc zy)zZWmC8_P6YYM@ogJUCNXRX7)r7+Ej=(Ai+km~Lru|Mw0d;Hek%_08%MLkp!&0dC zbzL{uNWt^eXE(yIp#X%9t4ksXOVq>N&>p3!1dk6}O%I!tII1Ch1!q00)U8`Ox^k=! z3m(sFgrnT6T3mQ42qql#KstrN126iE3$U=4c_5rd^4v=0H4A>sJD-J~;BA-3S{XxV z2wI;rsQY0fzCeW12x}j5=LY;GAnNlJsxiIuU(r(Wndm8Qe2ob-|4Gz!q!H1umHPJ3 zL3cBuW}*jUlamBNFD7=D@;npCqN4={vv=cxM7X)p7YEc|yu+UKZXNxo2o5qFb`7|d zM~8=-HEA|W^qXKN|Iuc?3cguti;`#mK*roQXU>+b_r?eC2b5p&kIa ziSg*Ww{MFd=q`IzD!#{k$cW8E1f5jYFhchlyN{PTJovixy;CN)e_1V)6+P*gV3v`o zFMXY_xE&{~w*_<|@Z(0pF+C2gF2|qW#Z>ulc=6o&#S32%Ie6nIPf}n_e8CS}wo1&_ zIs>Fi5YEHnG?fg`ReCAK*H(}%!C`6nYo>hMYFZDHR5=9&1vxpeoUMF~uH5BGCq?)b zqQeHsj~D?c>!cbQ8WOSVBLYd)y;m-6PN}kt=5lixS>*7f#hQR;nRfJfX}x4V?nqZ;XCi%b;}@ zq6TTDB?srKpy_9o&kYIh2DJW!6qzgOSJi<)pn()y6}yOdICIG95cdE{3=lBkuIS*2 z9(ZwMg*WKM`}k3x6F%F?&fm-{S|YzVK^{T7@@W?YR_=PC&Ty*${u@QF?Uq)19}fAq znBB)y7nGQv%x-e0rhAC1*o>7ML%caYF5Wls;>C-x3e%21bPcvvg5lR1(tE>hK4&}S zZVRYwv;U?2JBpjisMuUbXJWZG?W5KBU{M5c@8Fh13OK-CSscS1HrHVmB5&4}eOm@% zw^(x-dkDq!T&}8A2MyU+a#z=GY+uACg7t!r-slfXvRoqz*1RQQ9;y>TAr-zcgXWn%wR6()4BMo z2M-_5$$V0IP9TzTWEqGmTQG^TDvM{bFI%?rkUdg9rw3;&YByx^dil#C}g?D!e)30lP;8)FeG?#oI-@yj+A6~QC_HHCSpk6=mlH9E`OWlfaW$g|6` z0sEo(3$81y%5KPd9f8F0+`Qb8iJW?F1NC(uZ&sgNdLff;jjF0IbDxYH;&rC8SAd;a zW7b0^gn7pfRD&RY-35Y%8tD}TmMURXdS0hCaY`+R&@atYUFun%VQtS(YlliSOW7z0 zm|$wjpOS!L-ME z7O?=e4A||y1A7O`Q)pr>!aIAiM{1;I8)QP`*GvS`#;=$Axf)?y0b#LN#ax39 zavB1B&`$z#&-$$;?aOH`hS=j=8-ICVT|vq*mvT3R4?iTQh&%Mswc{yY#VqCVA4&dZ z9_3pJEoHb{HHoEv7aMmAKQgFT{`=- z6~qDXw1!P|KQK^Jan47QIj9Gv4@hslAW{fCn(D0_RIZKU_!zq+K+eCP>!EFs_n1q+ z#tO;@5YgfHv(v#6?F?Kyn4-EIWZDcXb8ONo;W2Rxi_$lxSZtqk=?ZA2L^YT>PFKFx z>qU1mW=pzUY?dZEwHqo(U<}&GwjGDNd2!W$^2@}PPI6e)5xKrHL;Q$8Cln6ZlJW?1n_Z>fqki!KF& zs{iOV&||gedhLv73*sXi4_LBV`=(oAI8G7>qgzDp#412j3f6xg(0;i*G|Ef)^=#DH zQQ?@7t^-|&d0z}&2m?_*6xsV59&)NW*EMKQ4|TL5kAv|SsTyzS3&_mnPEIFRrlHCG zEL^C@iX@quw1vOns6&QT5W!nb)@%aSzrPQn=XPmRQ)2}Rf#$ZC4F2p4ZVafD=;W?C zY)J_YI!e(4O6&rHg5+GnGpa-Ge=&6nT4e@!uCTd3DMSSe?0t>g&)tK=*LYBWKR~p% ziD5?@yYo=9__7?3w?6y$Hr+~s2V=Q;jdwQcUY>ss=ChYl8N+XY$UAp7*6e~lV5x?F z{02lU0E73aklIS&U#*|5mcKV)((?J6KI34t47-S7BBK%U?R^jYRQrBq4^=IrAt#8@ z?%w=FdynxokeflT^1>iJ@OQRxT|E?G0r)#7f}O$-N(xGG$eaq?v0K;XHyZ2#u4q|c zy(z9tdtXs;=euVn=q-Hd=?e@Dgwhin6Fye)?UZ|c1lGS5tK_YNc%!_-3AK> zN4@azy*%Q>%Cjgx!${1(WA}H41mDb;@vs`@#$r*S64*n>`9`~~k9wQ2s_?aO&C^)S zw`y2;i@B2b@*E&7h2-AsJ~e*GFwS4u#0K#|+~!3k*nQ!cUaZSuec#a_!T-ROMCyBR zQcj@Ye;b{=vJ}!i33&H9K7Yi;VL?H7N^T3H`VNubuV4mVJDo_2~DZeA?S;|48eu;(!g$f8GP0$=hkS;UfCr z$ek##Q2WjO@V{7%|6D33RXRsi<@^cE3 zTh04DLnrGu4=-uTd9ZmFAL;G#cK|~%<0ZOD$WuIhN4Ed@9K2^wRM2x(chm>}Op<&h z^kY;?>wWfda?{1*-&@oh*6?a!TF}WtM`ZPpW1&mG<%!{Zl~X%;bmQM)N)|7ab~YO~ zPOY}{2#=%)Wd$byXs*c3{}0T)ffNUWodcBnR1jtssH45`_cf za>JnjtdQ0krvWd$2DArT{~9?8j2G29 zC7P2zexOIy)Cd%jlSSkW=_3U}!@inSbmC-9Ln{JFuqXueueZEoW!2F_8Qs3Wq5XBS2;Mk=$qd*3!l)y4__e^;> z-LmAcz1B;ytX-n1uA1EYY-A^xvFpvB);1n43?Yj4HU#-mrT7vx?i|Gz6nP^AX2zHh zMySCUqJyUaXYv>IQPr7s%Wn=0L@ElY(F=@zL_hAE?Mjqh9@5o)LYdpXdm>R@SC>pZ zGUpx1-&3Wba0T1*ere;$P1-a;U|Xm9RjAzcB;m#VJ<5?gJ@3_09=gm!GhtfqeOJhS zf+vwBJR;)$J>=mek^#5DuY+Q6FX>qVnbo&yeYBsmT(?M@BF#EA#W)CsjWklyw~+ zyB${`i@%DJAvdkQTG8uL6puadEcOu&Po_b9dv<$+CPg{F)LnL?v}bAkvmv_cl7sLD zDPDxn?B}0OBIh0n(~pgg?ko=#E$+GnoS*v3!@37)!BH6(f1vc+DKt@`^pupBS$L?U zKYqj~A&=Qw#}0+))yXSAJCq8B4%VMYSjc&Ldj9>)_aMHXoxUS%b*4bW#n zkQ0rFURv0V)i21w?kM1k^b`hd_ijANC&uy<2^o^xZ4|5%QY|5qHZ107ZEjbmYyKuB z`uYlQ)nw)5+P;2`q`Y%WSU6=Gf6)?~OQk@4^bUx|&J3bG81ni|M2^NlO-_`I)$tXb zx#bBOd`rVwBNg} z8D23M8Dvt-4?p_&Q+WN&_2aemG)W1a;A~5b%ID-x7Vtja2JLqeQW9A06j;U(k+*rf)v@!gAipHi-;oNJ{(QUiUhD_5>SVL?&< zhuUm^Z`Y136kWev#xO`jbtD!UR>na$$TLWK0Q339gUk1sYz9G1L;-kd=vJ&IhY}IV zcPMZn-h^V3MB`1E*tyn~$(-k|zd}Q?#%LbhN|=#{fwQn!C#36j6v+U=E(jlI?zE2> z>+0%WPK$7GL~bfoLM^tvtqt-$eEZ#b2w>HSK=q&VMQ*;oKU#dAB}mH!p+5J=kA`Mv5);Xeb=(goTUvQ|{lc)7 z{Y_0!$rrZtD~?L*3CR}>?xFkb#-!RAerZNvwgL==FkLX-#5d?!F{yRl^sV5w#yrk= z-8LgvP8yc}{@@DXX~l33>9>UIq*iI&^~OA3WfpnTdJlJ62`-PCE`yL2y;O0Jnm#3D zdmUNPgRcf13%dZe;Cuc(9TFS3 zhd;j@(=ZEPYS@vD6?;gX$@;qR*yM^K@@3rm!AZeMCE~lWjlFd1*kZrd6DJKaue>&! z4Xm=Onr1Ni$a9CO5rR*%XG0Xs61u2l^71vSm;s~a4&&9&LjEfPq+)+hghQktug&_y z@Ptkg=P9M7WpzS6#i0n2J1^h>Mu&X* znEI9PT-`_X5Y822AJ_65TQ3~*)Tb}Prw93$8h^Ql<|?u*RmSB8Fl@&6+^y6d z{K@4y&Ng^A7Jty}d9qZ;qu_^MIpiK*(Bwi}+9p$a$8JoNIM`u(9X~9rUi3b7W+?q9 ziVf74buzgHRX>bHOGqc#)@LT6_X%161@6cK^^eolD;unWx*p>4>K^V1o~oFy=PDA~ zUAV<-v@i$jZ!5W5L_6rPMSMqrwR~3a#*OCs+rqN#UqDH~83a$E23BKTiW}~R8IAsn*~2*wa(^H79GgUK(m0HL!0opUk(jT3h~Qzb;f)6p+WKSc#TI_ z?D)(K=%|DNFBFtBw#kShw7;w8--|CKK2fZA+?Mnn{ZXE7G`$0IYkcFtb2Dqx!ExH*XikJoj0R_LqyNQqHrq#qblSLczC@2IKwgFxFeYYvPqH$s#y+&ToVSEVA6{iqIg%8C zdl%GK|KQ-4Q%{_8sY2ibQfk{EI#*Iw7QzH+`V&xQ03kvW5|V8>8^IK106`#SuV7}w z7Qv)VK|=C-9#9uMEjW#oY=ainptS<(Xrj8W)6(iaPxb)A0Pi-`>K+W{IVL)OO1si!#kIgyEr;xmD-Vei)u_^NZL&vd{wp|izHO~q($xXyVki~ zCbZwTPSFZlH_B^-tuX3``;EsYs5JwoCzi{vBfpL;ms8d?;YTJ%K{TbR+PA|KliId9 z`Oz4&u)MsyvU2be-9Lg|oypijrAs9in zta?)qV2K4GQyF@~{Wx0$B0~l9*Cv1k^rIYl;=W#WEql$qyMJrkVncFP?AlT(zhhPX zj8g)cmj=rKmmRJr;7k71hXOgk%ai3+P{+KChGq!V0AY76XA}`}2O!*r>kVe+d@XVg zc6N0SA;)E0s_Z*nr~5#hvzlqB?J#%Uj>ARF39Aq^olSqNt6PB+U{Oi;?d3}AOigJ3q&ON z)SsMxCAz+R?Hk8R{~A4AR)R%Iit(z@PwHG*T+|XqeQTDYjmOmi#g%cxF7La&M;Fgv z62VEsaqu!ULat!w8my>rT)UP4{h8t6JN!IQKIVnt}y+S?}?R_m5Ir`HUS$O8_JCRyB-p#v3LS#2oiF9 zgrbtt%f(m)D8MkQW>U}f;G}0~Gi*y%K@Fk-l>AgBPM*^DolnkQ)Oh`I8I#Hqu%>hZ zrQBGsi2)TJkW`a!Sj4i5>4k+KRJfwW_7r_pp>#9#+3ODoTB9Hc+1;*3%bST&reu;| zcaof>+6=lh=QvX}UGh0VAZVlOoqw@ISp+Y4cU?c_J?~)gHJCKAa?yO5VjnZTz9q+(p!l9tM)y5&SA4lALXSpX`$4Z} z+=zu$P3i|t^{Teq>eSQ{c}|Y)eYk3D<+h1XVoFH311g?#bx(OHR9>nT4bOx;jrGSG zQ@EGFs?ON`I84l(^}Ei8kM(0)YJWS(Q+m(x^=N!b^|)Li*sQIsg*7UR*HpfIt7l{q ziVO`sS8nKO>+cXD`_cD8-;Ux#CsJ2`OQCjr##`%6H!N5ncM}&EH&monHuLD=Lnv#% zf~qLumMFX!UJC%(}9-n|=xE3~`N zfr5x2;{-kB0y0h={x2#3aN6DWx%7xueR+N`3ildI#5iRvN6K02r!K1b+}LxWtC+RU z#V30!)q|ee6Kb1X ztVmv-9F+Os-F%WS9>=qpa-59qm5xU$Bp-F&!g7aPqSo}sSR3&!yRM;e)4hzr**#e( zQ`_6~#xp|VdLH*;C+NnF@La;SPBpnlp$tltWC7@*bEtAx)pfnpgW% zy+a3>pHQ%-LE#MUSg2EfZ9CW%)zH>HMykw%)V!62xL+iRcOahzL&Qdh zV$?4A@0P)tWvqhGS%yGKCz-b#FSf1d$&OvM(4IwR$#g`->m0qaWop5AH-8*M8 zAYAl)-8RdDP2`QaZ8;hOCIRvdp||cz5>W@`yV<4-otGL1%C%3a36$ga^6!T@VPR)e zv0~`)3v;JZuWY{QHwuf6@~WL=)k^DiS?zzj{rS(iZ&ldwKkUA@Eci1N8~ayf6Ht#+KXp{H;sypKJp%U(?v2p zG&OYw;;9C8PUdjS!Mdmm-_iG7Wq?fqiNb~Z6f_bX012m}p*zy$N6IYX>;TAYL+=wa z!RIZfT2ZmfVrj|etwNdaR`e9^=AL@~O6arX_I6ibB?YM4@!JsEJ;{)gZyyvM#|eHl zD+#4lBtSvH%Gz-D(z=Vt`X~Vj+*8=2t(e?EV^(%UlMA=40c|9tjw-1CeJ;BFM=5jl zmPsNvZmdEcd-iSVoAN=ducfakq3)LVZ3t0U=d&*7?4uK zpDjnZD4ee#6!qUICsQW!aGrg>UUl%cWJJ?h)6iMP&^!Byq4gESyT`X!AMM}zb_^Pm zy?uSNpiSOp0#>AEAxyCPBvWRJ31(k1*WfbLxR4ir?7n?E!EOzbZ`utoj-Vp^Ekjh0 z^`je1OF#tQMtn)VW0~o(RXZ09ilETmIKTL_oh9x;hS)iT*9Vxy#R|H9CMzeW*H-Nc zz!tF)oqA*mwfwgnpO6}!B7zV?%1;Np9}J*881TkJ7(vO|w(%g*2S>eVxKyWl9pH^% zjI-qMlRx*~lp(c6_4d%@cS;(-b^QUvwb!zB;qW>LZcb;Ns}D`_souEW@pk@Lwld~6 zgKuso7P^Vri<{;YoEr0QRCFkG*7wN&jRm-=RAx}+zFBu*MKJK6o{f|7fbSx9uyMoD zwtl9a5H%784zm>Uxx=k#(a`I2nVx%H5afaC{tE~~JumccUMr4+zk=ec=Bwr1jbn|I#j}KZVq6!jRX}5!Y*d}!DpY9`SGsC zM_fGZYTF~>)Yn*s+c1%%cZi=~{LEs#wG2bpO2Y2*<@?>=ceD@)q!@KD9w4w-KJ5jn ze&qnfaEyGrs-|o~PY&PWz}jH{%!A+ZxD?DXjh`#D%O3usj7YQK5~kV0p7E3YmfE|l zW$<#z*5?luf?kB3(ecUC<;r)f#dCchC{ttb>sEc`8x7T+-h??J0R|wquGMs^LEq=K zmY4LGHX+rCWxPDt`OWNM7lF7G4#up+II2g2-Kq!Fa<&_Oe-~#KMat>@l!Jy}1?+V6 zFDFE2+<%n->Kr6A&-aHwm49O8h22sZ08J%8HwO$(QhIV|CsL^;&F1#paqqtWzK%1) z!7Vz2ix`7QrO*5D_I3_L?>m|T-0^flj8N+SWL%I!g3*CrvQ#sikd@U0nxV8=d_sF)Reer}n$DKE;glzUkC$IPYgF-8RMPSD_+$H z^)wmj3~P!k$$jRaS;GPOZx|L)yp zk}TEqV{~GlX}$C8Ekjj(0s;b%u(sNBNb0$=x!qBuwtC_9?TBo&Th|@o+xpXJ_8ijn zq=bZ$Xu?Ly(W5)4u#&%VRI|N&5_N-vqxi#z4;^bC{5naR`{u4Jupn>zyQ3g#16y^U zUk8&dKD7`6aolU`eH%++`hk!H&h$E7<9;NnrM2wUtLtP-}$Kc2wW8dtdy{ z0u4=$=Z^Dz5zF6>mFCfkpayaLNM-A{k|ek1i}m*7`T2PO+vN8p`Aa$>c_eVhA$*~k zd$E0a!Ht*$0Qb3T5|D{NME46bcz z`VY~lt>KY|q~U;HYn}rpUU~WF+G#Xi2*Oa~4yL%@1PA%EKWcz(F)?wPzTOi;-$Npj zu?2CP9@ob->TonAdHLW+g|IQCs*J`S6}@=_LTENnX9L7Tz$#Tw_%(L;^wDvIW4*K@ zjW|Mv8TuonNW^2QVSQ%&O|5*~u<~#AMe>AtQ%!uvrcSHhK89?YHwnZdFCP?~@|Cq~ z=WBCJ9eo($#iRz-j>PvmKNLoISDEw-BriTe&N}?0D~Nwedbd1eroNE}G@%agFmZ`m zZI#KRAixA*VptgNco0?miJg6I+YDP~yA_?pOR9M8d@jfS#!YmHPiLCW{2XC5fXfD^ z3J2CvU2o!&k}yT$f>{>E77_B>*K~aZ=W`m*;vgBdG?Ku3~$0IJPl zs#8tn5YbvGnz|8uGf!X-t{YR{>2_pBar3vIb_j%Ok?b{ry^~w5{_}}i7894H)p~rw z8MvpX&~=Mmozn!(S2<6W{!s+aw+6BKnG5Bc* zH$9TnGe%5tNYE343fwwIV!!$fuo>h?Y-y23NIlw8>r(F|B1HH?iWtc0xBaQ-E?%mt zYb(*57%$4%Et1kzUPMm@x?;-*I5oY-JW1$^l{J8j9Xh?k<9ObCKHouEN!EB0&BZ zjpf48Cc;-mEAT{J8VmbN7Nzp<2;I!j(}$z-zcn0LXmq}o&!vu^__a5DZa5eqo|dd8 z<9&6`T+>tbP3DHbm+9g*bI^C}pR+Zwm%6q(S~IEM`t`++FQpTH!g%;ta4Jl02y&D% zuc(<@Fc1%h55a4~7l$iMH`t9R^3otyi5ghPJ+QGVi zc<5sH5y-L?VZ zFJ@ykYO2)7=>yd>6{v{zXu8=oY^X&;Qd`-nF46Nez~$r1^FmSXPBUGe}_y0 z08v*L2a(KYc29EJpCJP!)uO+s#2YR0Ut-W{l~sAO8Wj;+-*uw}oL<&O@X|Jfh9 zO!QEUAyq*UJH8&>Z}pB!s=3pnD3!VaCG+&hIe+d7im+wT=Tk_;cCHl1;M|s*vcl#L zo?RlS-0i1*q|V*@=WYSJE4p-yD5n+4OMs}!+|5LmmgF9ODf)Lu-*onb z6G4AVCjX)9xB}kN+%E=M_}9yI`gia?*|=m_xTagUcGLwEx}}E3G0(T}h7lpJb>tig z>)U07XW@+K$X=q$FOJnKXGvY%{B2JF~ zGEjhF?_>Iw;-*|0YrE{gCaEdCIog_aKgMf63{ny=7=9AQDK`mtQ%0;%bO|VPudz8g zVg45NL0*oxlh=yRi zcgeYDZ$h&^c~0>gVV?!Of&^5=oEyuHa?w$P>eURMg#{y&QL?pt!pH1Z7^598$!RZF z0`FR}QLbCnXMOhm6K|VIWUG^GfmtqZgzafpR#p2<@^FRfIcyz}+Nw5{MarUahxyrJgk1 zyUXrM+VI52#zm3AZ8^~+>VwZ-7mMV{V%*wLz3xtS)5-pjDrWJM*J28>jS`2)RA%Io zjgGrRqW0s)DLvVcf3qnNgo!;kfAE{r@oyUSU^4+^fVI=@zW2#L+5c{{h)u>sZg_*d zGzJF+am}T&bN*Ik-4C*)iF*b-!raMQr>w#DJ1=O9G@T;fg*f_Wou2z;C*+DRyn7rF z5bQ3S6K8SLD=r|=Vdu$>vkC^A#AH#;Z+Lhvy+xk_!|yBCdYGHi^-3?~Ix&_w4(Mn} zDX~LK`9Sd+A8Oo8Bm2S}d?nf{i4WN2$$d_bL2|P#nRP<3P7d>?92Hx;?BJF|8K>Ai z%)|z{<~CEez}E_2e}@SnhyLp>d<_vD%R2HlcZ;ID^FBKp^{`IRdV+r^PUXP;97vV8 zG``=DvwlOJ+UIw#ATk+%3Kvw^7~=5uL+d`q0Yl}p^Dj-U=kobVKOe>a_DLbmxW?xz zQCEAsmf^*Ht=8)C$lkjaFUm`jL9(KIpGUU5O8ciaI>*x)HJv1c#(iKOI*Pu3BS4hG zY<+9Zvr?%&=iz!lY7=+khxzGmY^q0Qj61Y!qBEYyNlbWxlM3G-JfqY*U$$AbE_*bzhLkBhPlz(WkS#``;J(a5 z_mdmpU20{v93z%L>e@{n^fD#2LJiz5=;rMtStgbHiiWJDJ?BfU%8y-5yH!3o{ca3U_GD;(W_Nzn z_`b34TBci-n4|e)$N@1eR{y{X0_%P@TIwN&D8i_FXqT92ds1@dbCM-EUl`Upm;s?4 zX;THnqJTy7Z8zYQ1D_ZJp{Sw)samFiim&oZ5$}y|BEiEShJCj0pnBOj=(4EAgstAX zP!P*Ah`bZUhp+2;PE<XheE=rW{=_-s*>y zi-zBFvTk+u-6vUp-_RA3T~8H%U&U(<3-zydw#_yA>A#n;rP*&odeM{m8r1Pfc)7X{_juD!CaDJu1XDMhNizmYM* zu0N7SA{l>D!G`CNQwYZ0I_Hp?nTQbQP@5andd>foQ?Ka@Cu zyviP1!pY>;a~Vi59FTv!+02~1w?DjpQ=J^QUx)SqQ$BDA=R~zGW4=8=FuVwb4u>a2cgH)4{lA8qn%9%VC8M za6tdsP0;>~E28eu=KwUL?vT~^RF!Zz`|%lDXt#XWd0Q3!z|Op=y1De>N!YXFGbJ3oU+zBlF+2wds*7)dW zE`e@6&WP^Pb#dyg0r(T zM1gn6b^Ffph!HV7oMO(%D(-t_WD`SgfajMmM6-_2U-$;?i}yk~Me?(z@IjBW@aAVSJ=Uao(R!i?WsnacIlyK;k@dRDfzvmuu&2SbotP0-};_h&Yz5X<6QW9*_2{bCK}K*j!`5 z%G#rzZaZFOUk`qqZ#Q-4q+k{m%XAXXQ4pZ%;Vu~YMi}ei+Szm0=QO0rz<*@^^j>%1 zL42ZUmqUAv{ME~v+>a0bB;Dr1h{0Xhquyi1j|_OW@c~$gpN27AZ?p=b;+n7GVzUQM zbA~H5p!(z**h@7SUp!ppUb{JH+~{yQ?n z8_Ju#@XLkqui>Sbv?~{?l@1>k>uH9UW^1TD<-A~yc1cFwPa!8tS!S`p#NOmoe|fhj z&u2t{3^Mb|tjLUAD-%7Jv08$n`j;bh+|0}>1d<NeaQ$9I?6R67bJvndkGP4QGG2ZD$EAlZuCXfPh5 zBA|yBom!%pLibiVp_3t`n^ME;%isP7gzP6y965YX7qp@&nwXhCi5x=!K}}A61nLan zQQLaM5B@L4#KaW(ac2b(0AO(Q5DfYd^qid5P;Q3ez!}W#^S%aU5c{SbC38lCdAa-Z z4OUbiWI)S~+uvbeiACp@$}gL&!C^zeA9^oJYWur0NhsML|Al>+g_Bagt&n*^b6qB* zCF)K6#$7se=vo=0|myfdlNB851@y&}LW zwc@XTWKf23*GMYzFQh6t`HQSjJm=JDQ2AxRrL}U%sgbb^;zhgk09oqV`Of<^Or9Y| zKR-2m+HR2WxP_JaWW0sM$aWvsrV4GE#_M63pFMf&Csw2N-;+c4?YNs;p7%8K*x1&{ zTqpi?<;viFwjH(`jp#w8HE0JlPG4Mm9(}p_*ccTzZt7ZcfKsEC=4*-xTg9EVjxQ~8 z(5VA;31DrjJQ<&2r!na)eXJl{K z?f*m7d%$DazH#HXk&zuCD=T~NJwg#eWN#ALdzDD`%-%b)NA{jcB_VqyQMSzN|8dvv zdEfVceLNnyZ}+;+>pYM1_^vF~AId%#Z!Ec@J{VHCZ(gp!GUxu&eC8rK0)hCg*?mX4 zjH<;&zRE3p`lJ2p{@U05X|~)g@u*hip`YP8ed3X0J{b9!n7KUWe5+>-E^?ET{!Hb| zMh4vF=#1Bj24(QD*UM5_;ZGZ)1(Rt;*Vv%s$mn#*R{k)RLH>%5k%=aRHNGcdU|{KH zU`zA6JZ*bD16OZ@J$pIVCb*88w(hvj;blLTeRZ2rGpyhqZ|!TgS1Mt>IL~tsW`aTn zs`{nHQ|2T3fQ#poCV8GD5Gp)K;k#v+NKtrCL;lhGAJ|z!IGE~yhU+6@nIPv3u=B%wOI1ZQM40r4I^k$Cg*k(l$3e`+AB# zMJQ%bze*v_N*Fci((Jf6s6U(e3E%*^Ur2!5Tyb+31+ zB)|D0MET*Zf+~J|ipd)u)|u6# zk)w}iv)X-e7i?$SBjkV@Er3RayM#daqilN<$nku`mstF^wL^UAPl7-1jf;cX)0ida z+24&2dJsH>@zKj;_=r>Hghj2adjV@lh2;SixMYA52*M>I?xEE-Mwgm#OE`4QaKtozdSu& zO5bI4c1voZ$bV~z;xV#6HqKP1kuUTl94UPAJgfwfYyM5`LQd^Im0ur!9d+h}e%6xr zloxu%MVY|%ZQR@%mO5b5mMYl_+Q?dH6p<}P+`iJy)6SB>je9*bhlq(VXg4VLsYXcG z5&?EVp`!@3RY={S8UJK?Fn$k`oV7776f*b?{$DQ?GWO-%9X-$Kl@=>@6uxHR%i;$; z4~xw{kYxN?AjB|Z5e9rzi{JfvQVsDSEGq&EcZ^>5OIO|^x5!RD$^NqpIdecGb= z18uor?mW*5bAzKRL~q@ii=Q%m<3cTfqU*ihE)<8@qdiS8zTJHftD-;Z8<-NMhadRm z-8f|1+~gwHU3^v}p;SiB>M@ZLw_PmhDTpyAoD_Y*SjMYX`9Q~JSV!+{=;zK(D2dY9 z*Q#2Dl7iImfYjQ!goOQm>5JcC%vpaR_YO%dgquTzO1R? zs|!J=nVV|rTdQX1^*7W=OKkwV!7h6qo;eRTyP%qPy=XrZD zuQb9G{T(sUR!B;ZJ0-eW?_wBM#SBx1_6M6>L-_VOLwm#~7ykL^bq49kDp!FIl@iHE zZz9$>6q19*`7tGKPFHl z-bC)A7sSryWnOigS5t|({Q|@WD7jB;Z2pN{A8!p0gocJbEY;@Jt6^kj{z(I!{azKU zs8qt{UkAhPI(Q1Fg77tMz%=Rpc!;jCvBid-K1bHc+0(W6o=@a`43xN=;VD&Cy}v>d z=@_^Z}& zdlxSSd0_#)^SO0`PO3OudM~ITm75Vd%cPs5B%2djnWO8a94Bnb!+xIOc=f-3I>+@~ zCz2YQwCUkulM^x<46SW_|0Yt_xWAUbsj)kh$b#X{te|K{UdH^=rPMHrhi!V#6DQbM zF7)*A;xacr#lH)rVRW6&#T{(W#tC(SP9S7$^6F!2)Sdw!8Rx*{)QcQc~_ol^2&D@YegpL zr;3I^v8Tu48vor^-`F9Z^*L_FL!TAakdf(f`*zl?r?JdoQ-lQMzjX~{A=FT zBf%i``<6j)KY=Ccl37JQ}Y{J)9xai z1wHa&Dw&qLzc?j$L$1?56A=8l4nxN6!Q`0sEz65yZV7|+y zN|DMGr4nB+fDF54ShPy#<`Sf{Emk!C|$d1#>p`n~U@sH)`JF+GsVy zT0xphw;ofQv-mqtk2@;kU_`Mf=*9I(ExWI8R4M9)XE(44elv%C^BpS?;8UGm;H?nD z5J9M$N^;K=abUe*#*12gl1h-q)JBgry&)9Xv-W6mdPBQGwt1sq^p02y6#>BOgD;WtT{sh(&ucP}X{{-)oS$alhREgxUev?=>@Ei~bau)LWeP;`H< zn+5k_hitGMAzeJU46zU*z(DXcv@|fF6gT511VPzihdr#*V6_3kg&j>2IXk^n-q(uK zX{A0W_~I5Gvja@T(;rf?ujp-a*=`8`bo2%7bX=F>e(;HiJs7O!IA-|V4DAZ_JIpcED@E6v}?shCr&slIO) z^PX8f!i86=%g~NC&NEDRF^MktVB^#M{!YEgVs@dxk3}PfN%u=$dFotumiW!;nTRFV zq#i~Kj+jKW&}e~U!vQo)3JO;!>AQlJv4Xmtq`{%Hr)OJm=si!+x9{DAh9Vi7Gk*n_ zi+KGG4Ws+9DHiB0M}eCP1b=m=R8pT8mrSj>{rj&I84{rcA6`oWYm?hMD8>_}Lha79 zj>4TV#r5LU7b?JLmq>l?JmC?5o`VC(#V0 zNhS$mslSL8s@v1_N%q6}D+`tP2T9}~;d9$B=?CKflLQAd#iOO)=uuPJe*8}*?3J^+ zq&4Pq$7twH;QH17*Z(ziF?4HFhtlsXCBLtRuP* zS{HA27Oz~EaGmmpzBBgjrH57SE#>t_GxjZWI<_0qjhtocF4|}!Q=P_x#h}BV0 znkr)>W>Zs-VnSY$(;5c&OD)7?uf{)Sif0~;{j}#>QF)|JN%`vmX1^j2p+Aa8M}cTE&Ohj{aCw+T9n2FHDu2DgIi!hQXJHUB*?7_I=J*MKmi?bu*N;UafEMku)|pxb7wUh1uh?-L{uKGrX{(KtwbTE3v2a_Pc_ZhTk?7sxL`f;MtMHcmM`e;J{Ks){oLm3zyL$f& zEvjBV7Ud`U`2o!n3pAs(Z{*k*s63rp4SaO}y`lYEBv~@CvaC1yNr_Sll;Z~(;Ff5^ zf=-2!y`BqdsL-|;d~;ryUXjNPYK ztDp?<8`)xi7T7^|B#x$1rN*TJR2Ufb1h7u9_ zYVaLmcU6n#9HEu8u(QII5Wr{x38b9H2Q<7;Wic{}|705)p~G7#V1}Uk4rrnnJpVf6 zBmQCR(GA_*!!ckCL{r3R(0PhvA^73GAUASJ@nk-C&VJ?MuZi+>P_l=F>N$a%N zgGZAy1rzaKVdn|F?Kya+$*n37;{taEsm>qzoawl{f{J*_VI_eX^OJw+_!BMZVn4ll z?^RjD&x)gUcS_zV&VNaBuvYQLXufBQK=k59sF*bV>VNd{PBhhOZ?db47rm4e4FcTI zuH+BDH?Pj{-OXZUV?#}PAq(wTV6HVgExuU4>Q0b9jD=%iplA(5L!?unC8?`c_+m|06DOW>3A;tw%!YRk0pqNDCQgGBx^E*!&_1iwO6KiR0%($iS+((AK*# zWdlX{C+Q=5mG7g^j@UwN=UV?TSdUr=(a|jeS#xY65$(sQl#`aR#$V^TnYr}XSOnrt zbH&;OA0Ec7$7zt2Xl5qq#dXFOEZvKS&H?Trk@hQIt;!E`{OvnDd_IA3A$X9bW@bJ+ zex7JSgR`MWw=JFQ_vTG|E=fcmY>{ebkXl19k_O8DajE@`6ek-hqLJtii)94O8La(eT~9+SX8Atj>sN3r1b(p;A72f3_;AloFU^|jm-!QFP*;P%4R(8adU~Ms z*|J~t*#z$%(3U|z58l=IClGe*?Ce0}5At7wUcPLsaO140sR1t=Bf4A&1%_Rd62C2m zq8I&TgBYR{43Wx2&8tPszQ5d?BC~w0Kct?u9zIUb{q~(*!L;!wK)gT%zQrOqk!8n%|@Lhx744uvZ_y|#0PpB>!}RZhVj9C2MR7TYC)GkFO9RHpzG zb3fPNKkhqRUnD5xH&DiVvIm|XAkdqZG=*LWFE8)Nr?U_2;hn=Y=Z38)r4l;EZe=L@ z0vX_PX(yHaeCL}ii2t^>wl+rGXP|2d)}=o`XRa?VKT4OLFXneHH)=Tqla))~wcV&^ zW1|yyaZ_nsR;Do1<3?*Xo|DW=84@7C5lQZ@z0P0T4L>9W62h!Otn>6_ZEUn8X*Dsg zG18ibKdnDr2|z+7;CDVj0`!*S6)Ij7&_Q0fqwZDbL2tb;v#a zYwBrt10jJvWAaWDD0S9V8X@M*lgQ@eX`u(_Uh$RnYlq%1z22%XC;~=NkXPAbd~lG59VGn?|3A81AT~@GN#W;b6|QC|=&JWxqGUc67Q+d~*Q>0p5rC+s0Q8Q_k1d zDfv1KNl3+o&A*y&cfO*!@DFvRYa1?mQv!llCqG}!XUtb(xbKQ zria^SOpW}cD~%;@(B#j{SEMie*4HZ7Oj9Gv8NlUzU})&_=uiBfRAlPe1M;q?Igew6 zDBsAXJ5Zm#!9r9$J$8ZEM4F73AD6j|TmL}*@FnO~PTx^FinK(|9fGSo0s*toXi+5} zn3TaTABf&GGrk;-j;gb_I~uR}Ipds9ZV`M5)eoG`XsBf3e#_)&0-Si5;Q)*n2JtSn zG&=wkw{9)>5JjeNX{n?0naqEfp_u4`=U2VCN- z5xcuCDep(uc54fxKlgN*)uQg1ext|Ws$VeY8N8!`YZ)7#Ud+TlW?A^ds$}46T<|4{ z6~&!fX-d<-l_0}C5LB;dXysv`o`%!LlR8&AE9l@oqn^U;ePHBasq*t~1Lo+e8?+xm z08av~-ao(VrD>2P@_#+w{f3Tnyp^XaT~IBXxKn|!~~GtMC^S1 z?93IZzG3XXd3HH>uy-^#+pti7gkIH@k{jQv+(EHSy}A;2ZJhK?he?uwLC)4MbC4o` zHW8_;Z{}K5lcPBvNtG*TUjCExy{uye4bgtQl75|Cd;gDw??A=h5j1yFPMb2ZhZukV(I@1CJr{i}w_? zKuC!nQ@o8=8bKK~2tUm`jh@3A7jHFgcvgR#T^L-_9j0n-NlMQB)`Zr;IS82axRNoW zuSI)_&^iQUdnVYnj~c$Y&~`Pp8myCi_;4aYg=r7zgIikJKE3o7WI*;HfB_}XT$Nd% zEdvQ~MSnnt0cNun#|0^j{2_)bK5Z9B*c4KfE0B6bLJX(xq<@0cl3w2OZ9c6!JsC4n z7`5gQ_c=n@Q~EwuwqZ5*w*Q#Zh`LzL zpLzRaN-@exEHCf);U6VrI*J_+WUfo=pdpa#5eRK)%4gVHwRoa_GfI^Vu^k<~>f?oq zFcqKxV?#KOHfM^4t;>$h7Z$HwlB;#-hM7bbJ;#k-wk{_uSV)Pa&~x{*P;D=}Gp(Ds zpyfN-{1Gf;=U&i?+3guuC z*^YDdc)F7{?^wTSzqhuze^eETN=Qf`k0qy2Uz^XSc+aem#7f_$&~Az7fayaQZ2o&J z%*{a)0f!lCKjw)$KSdOemWGMR!X$6A>W|m#B3`z&FJhJqsVtWK11+d5r1w=a_ZJDUButqCB;>j*_kIW$@9) z1r3D~CMY;f$-4*3v)5wK1SC)pEF2t$9e&RsL>fWM$A^@sLGOKIz+j^ghTk*L-vk8( zA^c&C+RwL6brn#I6QF+9nskd3uV%E zPw}=Cu-{m3*>1;~sktMuAq_(_7X6e}I^(`Zu{ey*`0QB}gkmeB?L=AK1#6uxL0xs+ zXOkBlWTAZ?Zl7$z9SUUmqLrHW;a6sk)xIIDSw|xU&YT0Pf5_wfH8wd7mroX$gt}i-ISG|@9XIN63O4$&5 zGE?*GIE$*UPPlH$ZSUiznsw}N=vGu=E4jT680+uv9}2Fltc2WT2rIX<<4E>hc=z}b z-;GF3P!HD0fof~qZRiCBwV7#oHm^`{wHa3at)E#&Jk4G0o@8eNd`>b5e+zPaMD$$z zI~3aHc$)Vt2o?%;|3Yh-ZUI$IM6AH9LJ!*e&m&gN%WCgrhB2;ozRb$Q<5{pu&MLHb zvAe{ToiSEu>s|UiE55X{fqsfzd5Ljp#c0X8ApF?eu4kNU&5`@w3tA3m_(vsDB-!7If=@ZO1w z+lAfdQB9K!SMv63(;hsOh~HV`y)k`EoNW4lM5Rlw0Fw;=fg1vH3#vXfOll9G$z3T; zuD73~p`rOFynhFRZ+ZHkLn~kvfpm?66pyQjZ|mzVFkyqwSN*z($Q7kHkdeEBO*X8g z2n1>X;&bby#&uh6B{T%7`?%4}ZzCNt*2E7CcccFW>wt0g{!c!%PBDy?eQI?k0?Rm!wg$MSZgNw1k`u=w^UZQhZz- zMmNY2OG~qBfD54Q>2+KLDjF^>c(NfP15i-W(1UsH_BaW!aV<r=^~B3t>5I{k0J94J}7XNvdMc=u%=(Uwx!VI z_Po5NyU9qlr$9==jT<+RT_>OjnMRAeB$MoC_G-m4{zOB9w1Jy6WPdoO_;BQtzU{Mt z9=S%Cjp35)edobC`$c*YTO2HwL0<`D{l2qvwR@?xV}%_%dBHdAr=~kE2KP)|CBa%N zeZ6bN+sTWbv?wIm*@<`Up-=g2&O>syt8Lm2Us!SVui(m9KXm8eS^TyNZ3=pNbC9Py z;SkTA=dU+IzBkZWVE6$uqqX@Y%Yi~9KyPh^6R!#F$;(zF+wvNHM^71uuz(&9Gi2M_ zj4&28FoJcU?rRAUW6H_~%o$f=3ERHumaYgf>3b%xofn{k8_Y-%zF}()51Ro!BG>}L zh&;&+*ayiV%x_4s(Nl678Y!^Zfx@Q0uTOOO3}(PNV51^WZO><IK%)gt^OGP4mu#i!^3`lewm~-&CRg6 zg?wLoH9nwyLZ1WR|9!t#x=n`du!};P%7{31*2l0!4DG~_{prV%kUM^g9l>qZfOqn{KBp;an=HzZ{(jXN_v($Ii1471$FL1`fU$4DEbXMbUizC1HNSUBv&0@ zOG+oh0ljM){MqoAS<{!;kLub_f=f<<9~eEZk_;o}UV$7aD1-5--j|fO72&Vm@cF-`H3y>E>N~GGg@$a@t?IqI2lizA;dJ#f7a_tIQOC zos3LJTl=uApCfqDE(OFHb%>>>;W_6KjnC) z(1(96oHE)ls$z_k38&)~D5<64S;c1wP%6Hiw|g?z{WRbB(HFLTu^?G?Ve9KYnQ%FM zsm=f0tllDNg@%fHB5-ZE7&>7;mzSrsq8x1N#AeV^3|}ytyC|q+h4vBiHhoXJ_v*Dr zuQ?8Rl`;CdqKZ`4ukJUPX&85+rV8q`q6Dn+lfDk9>Njc3SUypDuj(P{!Tft~kJ5@k zRJ3Vf+K!!#4SF%wfh@0W-C>Y0V?%bF3GIZd;bowD}>bBmE-2b=$vs{Au zX>J8Z4>gObn#^G&+uqV-7JjwjE0qmCGvjOky<37ol8isjxW^P+W$*r z_|^6;rc9pR#-My;)on+(p}oERE7IJiwpOdsG0Y$e!k0jN2YqZ1w{F!+R9sc7EiEg< z^=samZ?nz(QCVM6-(Jza$1EPx5Ytc@Q|W$EOD7lHIrK?J_Z2k~FGw*Y&GN^II4s*r z$j-im)FFeP0&872Xvw)l!@|5l0_A%n(Rv*^f6(dJd_m4xR{e0`ofWQzaj9hV66CwsId6&$WPWH{=^&)2)o?1(4 z`*|a9lHQ3Ig~1PO&jXGUF{f@~{C%|;J%2Ni`=%mjVDaK$OPs@0 zz18$4Y8#EPm-InO`I|RY_I>3gB`@moE3#1@n{p@o^=t(^8oy^!O^u%0X3xF9+U-ZvJ&1y5x^_LIM=3Ed<^YG%@{4d&d%#9s~;XujbsgxV!@aeR`iW ze!3%1(9&^OI8H)LY(c5&UpEeYE)mWscRvcRbkuFdGF29sZKCU)d?dQ>L%j({3=Vew zwVq(S_0ij?pwb-&`3xWyS8qZTOZPO0lk1E`!9cd|<+uwzwHV3x>*F)2jc^HFE+#_< z>OwFcZH)t(dj0atnR*?U_px+47OTt4V-w3%w7MMHii)92RsxukR`$LFVv}iuR^b9@ zBeR~=cp5Q!1|Z6Z!`>pnBaxmR!d-O8AkM`nQLl2%k{q-paq{%=IR_um&&Y58oco0d z1Kg>Cu;po!;ry11=hdL@3Z~+Tb-Y0+TAU9U}I*Md%&)Pj3n3v9_RX72uv!vauyXD5fKp;wYk+>9f*H+ zX?q$Ujzsj(LM+c0SnJ}br}lIM)_)$?Mrh-z1m?ve{5$_16>gU9xw6i@3iKM_<+YtW zx-Krvnu9@m6%*~)Dr3;<^z`(ouoOo@!wM&kanQbY(9#VTU$8dt;5>5uD4;?oTm@pR z*((MtpUjxf5Imy6`b*EEkb3G>bQiqy>*reRiH}TIgGc$RN;PH%_{yEHawX%)T|2Lr zq4ZX@ZBOsyf58h#vGK_hpKHXYjhVqM?vaQ7Kes zp;Bky#<=o=_FLIp1=o8Sbn)T?xtn5tkx4_GAZ9|#&ake+z*Dhb{{Lae=9utnH3Q0`wa) zSAXRvy7|%4tee^ov*6-%`ioO=NB!@v@1LPuVIs6tr-%;xEkfAXOb(~=D4>n8ExABk zvh2yST%X0cTe9m(t|HukwTrkN_z!sY&UxrgllJ$e6%6Ih6C-c^-&dn9fv+0$dPN(k z{`(c&CQ-E&-$gB;nhxi2z!EM`>K^Be0m~X)jzYYa5Omqblezua0ZXt31hQSxs12tV zsUa5`6PySfoT>Vkpl1kjGsC6}?8>Rj9bJV7vB#zpDChV;Pm7>6ovCb-==p}FW-X8g zy*)TiPEI_CU7BX~D*Jx!J^OnRx3;y-sD{eZKW@`@1Xtx^t+eE1=EQquc`(PbtL^XZ zhKGegzp~z;t+n;!O)OLpU_uKA&#Ub0g+AtYaDW#6-Memih{3p-NLW%?+0YgT;p+Lr zZp&d1wOFsiCnlCQ^z+C@quk+v44w>08a46%PHpUIw+yaYETeZ1{Xbv_W{v3V7iQ#p z#<&=PQy5Bd_$S&j6j6n+*1(ZJ@qO?3w-9(mfEn@IO4$%LcJ@oyqd_-=O48@61DUL; z1WnfxkpxX(KZDoP7yTC2bynrUU!N$IkC%2pqf`Qn7kf@Hq0n=NAA8;T62M8>R! z;BP*|1^^6dOQhcvtbO&qqExM~y{a;vq9Ss}YDv{f1=kNqB1v?!mp3=hLCwYm!GIqU zSlhUB9~v9agZUoJ8LyI)A2&n{^p;P=4l#I1ycp=M(|>^W(T3FtcI0|`QvjSeJTt#{ z?*sHxz=!ZUIr-$$3kBqr_qnSi58-bhV5AU;z$MP9Qve174Q`cLA=q0mL4egJp2}Tn z#hH2hx=k@QLM&6|$jVA?mBy2{=9DSB@l`jUHsi_(&x9%|0dm|GE?T9LG%H;~^sZm1 zkf*Zf)M)6SQ~$>H=QHe!0MAD=!=AS%ZZ@B^iJ9y4xqe}tcYf~ur)sWTFu?2=)65kN zd9S?O+(qy}0laKR$b&Ruc{7&SR{#dliEoz_g*HXh3Ua&gIrFX5Bbc_w)%yT$In68X zsINZ+r>D=KKhJkO?-1md$hdivJMt!7WXw@$3|VlHXYpKntro+erqb(s#ag!pHSfJH zffA|XWDQ#PCr^-0*!t7(v=C+LtNtEY-~$>Dy|)PcIb7G)wyA38?nzx)yD}c3`-GJ2 z-}b~W-`k`OPB^mcIj}ey>dZIhwmQq{nV4J-z-a}kG1%AwfTE_Vnknh)L2wVt2s+xp z&KLLE$=SS=|CmV9{W6B2h`p%BSgu%$bx<=g=?RoPR~haiv+;nn3Ru*}`ufb=wRfhY zC?`_#P^Oqp_L!p}8hsb)l~_owPR3Lg6!db@3>Q_}_H*eIk&-VA834X3)TREQd-jDx90iJ+g;z}7= zJ2dyeHfIPRREUcJd!d=FtGKw}2p$?4S?l0CO1S9;9ke%pkE*IK&2^HrJUl$2qTX;- z{Efb~!j0%#8~i9JEd1Eg(rzwI!`K)c6(jWCjy@e7d1_U39NyO^_I4})O%Su@2|m@E z;^J{M#M${X;7IR**V@DZZ6EYWpcmwPapwK!i+*>m++4Q#@|>>l!$$3A@Bg<|Lxm1Kwc{K z-o6_g6ix1a!mXq;Y)TYxWnU~-Hm+c^lhn(=D%C?Sf2{sfiD?*Y(DDnIQ zG6-lg_^1b*0kGzK6ivvLY@11kesPzBbSC;bMQi*=Ttch2O>;AaT=i?Zl#eq;x-Ep# zjh;zTt@^C177>Uy^RCq{_ctqKKkfh4o!53LwuutQxabLqurw-w+qk*$sP(!XEjAuk z%{`mklCo=0QQ&cY5wxh1VQP*}%v^@8{ZTF>OoJ&+K%*dxK8nZ8+d7X`xv+5HO$Gm( zb=*OB4EIc9_U)=;0Tvc4zvAxgvBt(ml!H}1iYUNaU<_?KfB`Mp5Hv#iF*Z`Z5DBG2 zARQK_%}f@FkrE{?#uJk)iyFH@ zE#>J|<@*-o<3d)8Td|ucN3xFiLdho|>8h8`1Gk zP&gG97T%GU@AN`X{h(1_R)Fgf*a~yr0rh#QsZg6R! z?u3Kb*>KI>Bl|O$vD#yP@G!eg4%7B%tNh}m6vjYD$HIriQZc*-8#4{u3=B)}Uojy3 zzv@ZFNRmqMkF`V?5%_!?S9SOBm^*a*P+_8jipT*32wwHHDBrysjDe|8o*%&0uKlf2M}9}6Qq6dA7MUtjYHY`$Mv z!T)vPNyTT)u^+|OA1BCU)v)m}p4h(2jEoe2eG4?~0L}sM{{7n@kQl+XXXWS5w^M)C z1Q%_`Uxox9jNaU=FiJGI^aVH7JBUumj5=J3krrG3B#Ey^1!juKh3)Ljb!x66v8Qtvkd#Sqdr zo0GFV*^>vTyl~+GECc<`TC1zr65;)!-}!=_Q|YL>>tlIx8(5iV$J5@-=#O${9w&V^ zv|ubRf?P?xRxf9uh{56bVfETo9CEiRi{xVM(#$_2Nr$8}1}u?b4fV}D)xM*W_KVJ6 zN-XBCp`AJmK6S)VJ2fa#E|)o~-BdrS-P0Yx_dp?rh=wUw@?h^tf8L1tvg7;P;zC~3 z4d6@@Y2VJ{4_#A#I_wSre>VKHb{p@MlniKgsVf}dCDyEE$r8dfmLX)X5BPL zz&^P5-MhobsWZ@{aA~5#Ff22e34!h^4tu= z{<$Rd3JcXD*BW(%CJuuld^|28=}DNmwSIj)gG|E{xZ>!{ooO zu43m`jF->J6@{3WYD*QgxNv>@XwqoGR6i%K-c1beMi12UFeVTPB+=;c(0yg)bRM(Y zWD7rXpwb7L7#xswkBh*%desNON2CGV{cJ1?;$hww&(GPp?4xn~R0%GOPgmZZ*-nsv z?cy9*f6uJc00>&HUw0L;)1ca8VoXLyxbI2O`R-27c%=BYT{AY-9hS1{u#jYQhe>il z^Orc&=j|6I`^(7{>ux{5EqrISC|hhfLlR33lAe%oJ{ZIW6NEV5(0sWufa*aKL&xYd zMz3SfxF=9(aj%E>wXB(gjlJ$)8UTBL z@hdZSr_+x6%2|G#@WJVUO=G$3s+TA*+FLSiVsiwPB3#@zL6> zJcALEFQYtW0|m0S(=Js3sQTgBuM(zJN)_+T460)W%3=l4bbs~&7(x`^N0ibj69tj9ZxN`Wep?BXc1Fk($8TfCLmBEiK&!@KdZ0!A5@p zKEcjFk^mPhaIAhd-GgK7)b^u|VVrtt7^u}utAgcGxqCu(3v@D}=t!yKPJ|J)Xy zSPnkUdl?I|a`pJy@Pe<5yb6|PSi+`}_IDG2SAiAh+2dXu?&g~Gg_dm+kx=!?cT66e9*^WX zPMzlX*|~Id*5{w@4R&0fZ$}e+E{Z9ZeI_7egb*`q%{N%Yp6TqhL=ji&ymzgNq460X z|CZA0#e@$#Ertu$c{0L*Bv6|Uy?^ijOvT*mQ}Jz&IDf(May5t92)7k}ww@b_zY{(n z5GfFC4fYeH+{UwHup+}Q7`B*D4S5jH$pS((Cq_xokB)1I2*j6Hp3bcL!NYNkzV#V~ zQ}k99Av25@g$HbL+NrD`@d#h@jdG^$Fb&M6Z8?}|zf!c!j2`{~qz^!fBqW}8dg$7J zv=e#`$)o^ZivwQ^;lK2m{M_2~7vm5JJhR`(N;q{^gI7)8qrObdsO0A*0&fNp{=gfB zVECQmpV>d1ngw>=+6zEdJr$KiH@$#>fQX0}4?cRk<1e>nP>GL>jI5K^XX;%eCLaBJ zKwz5h9@|{F{EpM7h%jGXY;Gn8RYmgh)%gT;SqyGqD)MMVS(S9j3#G7pl&%?;7ffNb zP0ypI>Nl&UAE5_u1~y$r-hBZyS^jG12&A_%EHAn`^CRb+5HbUUPuOuDKzqHGbq7=g z*kY$ABrsm%Rm;p;>z+in6nL=rbJ_A{1=9HuV$jNtVSvI4BJOeD%d&X=F=r4_cnFQp zFJQD(&m4fJ78yJ8_VMYN$Cm?QyxGRw45;&F_d>)L-gE=mu4LKbnQrKM*` zX;KIy`p)-_ukICFT^Fav@ZA0}(NtaS+!uZwfDQ!0i@=p~&-zPBKWZNx!Iz9p)ko7G zFH2j`G!@P_tq`4nQvq1J2N!g7Gz6`HAxfKI8XDyfX0!q&ayVd?ap7YGEr z$SVky?5)k+NwBr(>t0$4nhIHmA!CcD274R8n=vt5FmKyIdZumCws^WjmY93ZW(8o| zkV{tNw(413+1HO#q?Qbf*fF3vff@Y5$G3Rfx0PiyHAjinB%FbIfQ7$w!{nkT>wIGT z#}K_Sj+@Dkf%9D1mVDXMm2}5&`8?R#dAgm!YHw!uotJx_U+T=%<8lzKL*|nzv2*=| zjdZd0_H>;x2Pq^=)I|&>4p@#~e_A%pb@%6x6|TF*akbNPQaY<|G`O>Y0XW#70t0&9 z^dhovQOf!RMp@WV#k(~fJWw0H7xy5=CMdOri9%KDA1%Blm|sxP!&TTTyzf;{aMx5_ zdGlCvdvXETvx`NX2|D@rGz=arzN(fX0;01>Nb6~ybMNZ7yf{VXi9&n;*t8(I(Lk_=wOj8W@cuAnXN^ zSEd2mordw7Ye`^g-Bwm+2UKeoA+yH=akrqix7~JR+|tnwVVanTPZ`Uj3w>+;A+gwq z$&Tlj5J80D3gO?l2y#K69DzqFiiDV$dmlbNGI@3|27CXn(L6F| zG3~Z(;#YhEmb=J-nJ{GsO`ReLaHi}sJ7fNo9~0~%LfZAHm68L?jQH}67VPlvu#nWx zM63b5_@?3bk0_a=$4?EvxTXi0l|DW0X^fosn2`PaqDh^_jpvL_TKW=3byZM*y?7~e z`s&9{rW#%o;jEhxJCBKYXciY*j`;|G$o6#c;@|({CIL*DywWJkx>qGgNkGjz`Ed%t zI$;4>8aHsvlzuK{<8D3O($hpkus;%s zOPY0dd-B(EK!-ht$mMfS6R*W9Ep?OYYip2cbZ@N+ghZz4yV~^hc$U1WwFbfU6Jq;6MbVOV!tJf77c+f zLmoo(&J0|$tUe5Ch0Pn5Fyl~cU$pOS!THOSVs{^#Cd(i%_|-#!lOcj+65h%GeUZLe zB5e9r4T9oYe1WftKnUYB^ENnTKS{WOK-7;_ot*d}I-@A(va&0;6OwIF5FYlA0LU4& z7C`uiU(Kv^d>(22>uVS>9s+^+)=YdDC=~IDiOb!SFUR<$p7NqyDyQzkwCyM?@f+OG zft-1jh-;(&g4$m7dQC?-;rVN=W7w({78T7zTcIMLOnQDWrqkka=C~?MOSwd*@c!-F7PPJ#`~+g}8tJb8lGrGd1AIg}`Lx zI9atAB|PtBhW$zk5}sjcN-8sMJ3(^jAdcr8xa>2{?A+Xr0Qw;O8v@b!B_`W4PAFt! zFX+s_S;qAkavfAXFf$ z?ZY!xpy+|J72zL!=X;mnf^(gqRiE6+UCyr#3hT%J)dINjVzF{4vb#7KxCAY^- zLP98gzf+27>`YgCEf2ua7W~kVAw(fxlb@5&OHi!4Gd_qV+N%DLEZvoP{S%YA9mu9o zg;68E@fG4sasO#~c7siX_|_SxehuB1%nu#SPsU1yhkF|AXT~QUc|i*m1(8Qrae154 zno;f;foSm#Mfyx35LEkTcel{kwaX9)q(fAFs*~6rbE}t{*WTY}V8D-l9U=Ps&>amS z0jnkS^zVz>XA=aITn&Ou?JlsL2f*v(qzR~ctFEdc6JgK8Vd>JMg2mTYn={)Vb z$z^Tj`-6_`Bdv=IUrt1)hlU~3+t;OQCMF+URjci^9Ewg6Y8 z%I)VO(|d1uVuG?}FES|5WMk(;fADGl*?AsGa(-=jpoBxO?#UQj03dI`B4cazHJ+a} z{WVHPVW=f|Qj@`ikZCDJMtBU{2{jDDrvg>VCYSEY^70kLpIq6~WWHs(rumJpDB4&r z_5*RAEMX?DHGD`!AfUOjw1kGhnmkLx z$$HGjmjjBDZ@-TuvX7##@chRGxa#S7mm2+()rs1WVY8dkyyF$ykIjgZx3i#4yedMD zs`a<$i<_cE7U2d(Py4gOwhGv@A`ol%+Cpp3cV0{#?!xlKjX=<&65Q4RX=m;CW6l8g ze(A0@%=}|3w1Z8CriT-lUsx2$*g9AiE_M_~D>_ie&>OS_%m4B7Cu=60yTsiSXCv_|MUBUXwi(cf1c&+XD zP4~{j@|CET4}A@mF=RsL;|0FsQ={(=yel20vM(V3&eA|wv>W4GJeW)4$sDWl%cusicsTYho;!EdX>T z&x=nEXIkz?e=RCJ5fDyyb+M$TKHhB(Rg>cKSsR!YwC&C|ChA@~u)W3qvutC=ar#gPszV>*I^jcm8d)_-*_&Z)GIts8tx1Uq$nc%v$M~C$}?Vj zsH0)EG3phpI9k7{V#%BI`&TrKnPG=XwyMLM@YcF-=J{2H*m4%=4#s{$Lto7A}Mkgj* zys#dEV&^UZ5J0{-gK`)M>!(e%p_&=e&uJ=VQ6@rFm6d&$qJ$yZX?Ck?G$=zD^-9Ob zgXHw6VejON^={`Qt zVKYcmjf;;@<}^Th+*PO%rDbgidu05X^SnC$e0%(HHmUT1+xt=W;7F>epAnA?TMGYp zoX%~xb7~fUq9@wxOAfxD`hCp7tNUYmo9)C)9pffv+R_ynZPgQr4nLd+Pfs`Jw(FAt zg~392t*Ft#GV-RjmLETQVRzdrH@5nb>oAUOr5?AHv`UiqFZIoaneDBq|Btn|j;g9_ z`$jh%(k)$r(nv{2hqNF_mvkfDAdP@@H`3iHozjhTN_PqpXYTucp7;IE`STlR4#pPQ zd#^RuT64~It?T-Q4hkfM`1o;@e?nK+3c`n}*rPx7C6@2)Vf>!*+czu};o$04a4BWA zy77opV?);b7F^$#E8;e}hDb$^dP>fF`y39hXSDykY9DlbDptlc!0oi^!{TO-8zKU< zD0?CRW#PmYi4s(`p@cG`8fQ|6(2OFYNqbFM;_V(ZdY--bx{{XmmWcr3+wzS6?Q4_r zu}Zv(?@tU1H(xIS<>z0j7InVI)%FZhVvZrY+h2?G*Oqar?hxVg1-6W&+y0iLNKJGM zj7Kp2q4{;F*cHH)0dU|gu-t)%?F2JSN*x{vGfturiyzbj=9t0b#Isc3a!m5eK9 zf9ogR=|epnN|9&}q5@AkAOU;`wITwcHptQOdR#Nzv-o=zKr(hTR@-D+%?8iFt<~wnBrxW#E(1yt$N`Ow^0Bn%745#|sV3 zaQNrDz5OoUH?unZkv#X#S=$Y8T!Dcg$MEVytF8_OyNgYtAieDqv-^6EFLUZMpZjdj zO~k$S2E0-E8(GDh)YRjl^*6Vepr+^MV*>VjH4o^&kg&k^X2J8BD{N})q0Zhznn1pb z1LMI88gUkFw)-sYr%(N=u)kVyA*Y{0H>TJy2_;{Cb=*#r!IQYV?iZ4}NSYAX)`Nv6 zWn@SCWJjR+z({oFn(lKurnqk8FU$g@)FHj6MjJSYqf$IF$df#H+X*B^vsb^ajFQ`& z($vu+c_-4Za(R5-}0g?v>nsOSr!~oR=7|(tl0MMhKMh*1C{(}jA^NWw1V2=Vo2>|H?n1q3r zn+w0v$B0)ZbOxLr0YDr&LO?*^Zs*Mor+BrNyC5f}YDztHxupZ=Hm4uN9@c_*USHmw zEYpLw^q<6y+!_wzdT-A%ik$Eo$mJ=lmA6odMm=Uxnd3_rNo&e0S0}K+&*)P#qZCib z#gJiUo||GDE+vWm3a7H74iF2de4b4z>4QOVV&J-AXio(!BzpN5pq6idJzJsGFsjd% zR@>65rgX-{HdHww(0>WoK9g5qoup_yO-@=|hPFqg;nKkh$jU5aW zz8(HOIre^9US8AB;L4aB%2HHPy0~?aXJa#X`0it5zE*Mok{kF}+rzEcod#RnBu~o4 z+IvG32G8A>ZNoVqWWOID+2cd>{YVZ(9fFqt~@=;K{Qtp$`gtX9`d`_!! z6~es{bLvr%)X_X^3A^Xm+I{vQa53Xl;&IB?(&SiroZSv3C2eUaLQ1S%;lS3FTM9Iw zfqy=LXX5&7#$Ia1Rj~wP-Yoynrc9lPJFQ=xdL4a-9Z;Fq1xy+q8)5IZG1&n#%Q43F z7-?eip>A?GWpVzhauNfw=1~nROfS|#V@(&-}~vaogEvR zDm*+qp!PuBBHH0KSanxT$wda;{D{R&^Y1Qd(gMhK3NHinsT=vWNaR+zdLUcE_@ zhp%}qr{=Sw!y6JdVgkslpus{M`E-D)fD*J}%YmT^G9akmv}ypjXN4PplJrGEzH(8Y zjmVjoRyqX;9i$%Ft1zf~d2F3K(|!Cen1P0x`tFlbP?ezsGG?~k8)0I?Pw9I=boY{*fZv$msVjV>;;AbddF1%Nu%IYS=Awy|I?d{> zZ~0lms!!-_m|?E;xRvU3yHju^QtVcOyyz#dINu}8EVGI=uQ88Qs2 zGIID+m_OE#;MrCO55XcHj3hoL zCt)Pzk|DzDsnHb!`#2+#qsyO(VV;{(Uh{)h^=>291!{`c&%**))pv!gQ`D5Jb6MS^ z9fy3n1Rj<11xm*$KJM;>@`k;2i-p%AGngikH`ymlKAw}*sR9`t*NHK7$Qob@3{VS} znR_3UTd4Wl6cIM8`r-A*1?iwK7|OswDCK)@IfrRB?9~cR;@QrO==e1{f-pxo8;dwp zi3c=kGz@brBA7!Dh(Ab~uB|6o^GDLdHLTD!;U@8azZ(5HBSZk=SrcOQRLA4_ls&|Y zx7Pg5%Q9eIZ4Y(zH&=Nk2e3Z7F~n|@qmC0p+j?$b2OFlhMc6(N$*BamlwTD&3CL_q zf8;c#>-g}u7}LF@i#(0Q{nLwbzZu|t2H0gVbKl$H2Gb{B$^Aol0lYF}`HW&dMdQhs zdZYjaJWSau4B2xu+4G+CiCJKA3fdJwa&-j*5-TIPRRH`lXw1yauU-{ZRM=WrG&>(@ zF77lpHED?9*P~RagHky_Yf3#UT)w$O zyn;qTLP9_=n;96q5TOBq#L}(iGRG{-G9%Cd`V9y8GJb?5G)(Bf$TUJ(9A0Hn}ft*#4MIz!_Fq}4vRGX6` zj!*$eiz3{J%4_d)aj~%=G*`E(0=cWN-w0?G6NXcMZ%!QWXOF*s%*^=xv^S3h5e5tX zJ7==eX1&(pMt4;hXXn2GOay|T0@%1ezA;IAXsdt237EC37ZR zdo?MSpr&aqdd*3-*lwHfy#1GzKn@#e+<|2TEkr*dcS78twDFmE)Z24i0M8vXBUT81B@@%-1XJVEu^2Cr!%RJv35+`3w<4 z#0&M>rLlGZuJL8rUyF+wTKJryo}iyUOPOw-xw0u309G+wfGAZk5Py85&5~#c*N-$# zAC>d66&+Y9^_OFx$9H9P!-*#`>xb@$`O1@}Z`ckdw))D~ELVIc#{Fy!m z@g_(L6gauMt%rD$mI;ZM<&@CridUwOUi2jq^>s3{LrG#CpdS22-ICJZ-;fE7jU5ou z=z`eUJ9?}q%WeVofG#N8Iou{2hHauY6xJk#d!oZ=KE--UWzlm%9F9Xim8pdZ8)-p! z!oB!+_7|H_!CIrq+H9$?i4I97!Ea|MOUK8f=>1kFXv@}jFja`RQU+N8^b1!{hs$2- zxN0kGk0-P+T_qw){u>rdnr>B=)0){cGU`?mN3)f`P!|R3)LrlwO5+ck+?3Ik4a95YXRB zB8;Xz$DyFW&ctGOqbhiZxINB5d=A2Y+<|f#k@nsltab5Z~R_y=&tu+ zRsX~4+LKF}sktj!pGaep1%guat;Y<`%p8McD>4^TVC!M)q*Ykjv}@ z!MD;}f0kzV`9~gyGh%f3ydo9mjvu9h*+SmaVfW>J<)ID`31AC-kBHr+Pef1{FIMqd z^m&kx>&ee$8O-ql8jmZZOoi0~9!~n}J?U>CPH&p^Gf_`An&hyPW3vg0T3~BmZ>)~0-!DF&{-B!L+*(x9T2<0oq%}W1vof96oKugE zl=NQYYgFi$VF)5els+O`5)7INq)wO(mT0&myP#P_nWu0LJ*F9^HA9ogefj|L-Evmj z$u$H&$S%qhF*ZaYe2q?0oIXfuEE4*A&!5{*Tue-dz-Sim*Z*4S0VntRq7S~WOg=x{ zIZf5^sd89ZS~?uy!ok6fe8*QgrGEJ`1YlOfwKZmm_Z<1u5&#Kd1P&|i3rvd_4jF)? z=BuM64yE%sG0aT{pm-nu;mt{cw^K*Mcm`Q2X?otQv-?vweTDC}9u09btIn^qu*a z;b40Bw^#&??m58^CYd3q8b1vv4yLDFFbA3u6`V2C4#{0>%2Ee+-dcUsMrIVG)z7!| zM_irUA1kTbnOV5l7qQ45b2WaEgjM@S;I#9!1A9Mfy%`JfEQ~UL6Xl~LZ4xZOw)M1F z-HYPZg+ud-Zz>1qT(o2RlJF4Kl$5~Ga>#-*U~@tJG&>{jYUj*PmX-62KsjZ9Joc z_qEm;yyP4{H-9sQ<$o$eJKcUamN|3dh%h3pAK=j>zS#rs@X2p;SSIs;dAR}4D_<8a zs10rIZ%c*c9UoG95RL!=&T}lmcrxypX}#`oOG6x1Ba5YwhUc!>PPge%QTnVRYHEbm zJFphnEiGDNruxv9RZ=pPxY|pO=x;#6Xizf`7BstYc3RpVOgMpMgu-V-KOx6`To7R? z(Mv-StP4)5%bU1y+vAZd+BF+#tOAEUw%*e#TiNeCE3?y-vDtKzPTuUwYb{nId zQS6Khk`QL|&wBiYMx7)Aj}@S)sO&?$>36$`$Pw~a6fQvmz6@Nn4iY#!Ftasc#j5Fo zAsur%537#oo{JoIx@E0K++~Q}Ns9IC4V`FRabgjlMJzQW`>Qp=x?w-O5xEIQ2#PFf z5lx1F?@-WG7f;^75GBIpk2&-8uS zEA}(Vq8bgM6s0fph7X|zs_Cz<^9|Gm^F%Sj z$6KnjMySY8ScX?8lr9I^Y>S+hryK1&)*;Kj8K{b8&4@(`Sneel*#5a1tSGHTwg`rW z!GQ)}3=R!CzaeK)V-FjIo4lx)`wBp*=2N6K?^Atn7-Ph^7^koN>!^#Rl^7?cDY4dOBwF?k@yFZpqkm%gd4Hvz$HgB^$k;Qr|ExWO68(Y z;_@;Is>%|;!@i9AeE2i-&M993%Nnj(p-eY##%26BM}UZ(dbl1uI~@|+HewCrj7jj%?nH(BVqh;rr~yV3PCvfSG$@eveML#R{8@QpEH*?09q*++=mO zS#2$`QWa*t25C>c@!~gHIY2!UnPWCC3=DpJ@!^>w7Q3URBCj%?!;qOY8~SVq`z2MsdB>0L@!-|*9=2x5br?uYK4JHJ z3Z!&ZAEX+iS=Q)Wa$nk`>t21$>!He{Z(yL1F&b|6;RaxdB4;aw{GNZ=!}?l?EOd!7 zKoXPNKLo=_GSX-axk4XXcw}(zB%D%@29gBN_NCJ5W!C_nC^A*Rcv3(sJ@_^ibQ4sZ zOjANpXcuH`0$;|HrgMUAeBIE9;Ba#&|9Lkcih~Husc8)nt9^QRdRze{^U>%nRh)cW zarkQ^seiW-;q}jFK9eR1BU5b=irmn_LTE@$t-=2SB;kYp*%xQIwPDjVCbNtR+a#|D zc%zR4v%h(@%)_(c`16Sg!oQsX``pA;=nejF@3IY=nl)TMf3Cgz>mIoZ(~bL|P6uF} zb3ljJKIvQ{#q>#|G{b#5_P-s*I>#m@#?4mRp{uQrHvJo_i)5a_9^&o$GEEhaD!?=E z>0ZN2QEfa8UeYz7{lX9IX*sa+242)q84qR(1^L{C8%uPT<$cZz+1lJGBQl8!?PaDz zq17#aCTJHovHm@rfA+OyO1`p$Ek^6gL-9ZTB5n{sQ#5+*?Dl{D?YrNxJUm454?H-A z;6CyHet5i6JQOwP76UCp^84IDYYa(1)L7bhY_OwI@J060f6jp!S_dGSMes3($Ulgm z82Rl`$|IquN$=ihWc>FNv<9xQ*-HJN^|&moPcgor~kYUBV*v_&;t8Qm)q+XeTEB^QP#gywn7v z8qc%8V?1v67~rb^_pPYRfBLY~*mj@g-7l}a4hG&o=O`d5a&4Fl^q(iYE<#{Y@sjUd zO}NMn`7URedY7=g6I}ImdlEA^6rV3#VTzJ%G|c)&SOl<_6nvdQ9jE z4FXWId0z3|>kwhnoh-!*BR$0mg7BssxeS5l5SvwUW?p#T(M34@WNFm8?du~Lp{-Di zs~Ux*wLs~;9ONVg;qlW|=!s~>Vb;ibkxq9Ycz$+-`L*^)%`@r+*sU2Uap!3d@FxfO zyT|K>b}xXsqq%uy2mo?k@DQNHhIdqgDRj>UnfCNvE^w=2g1anJzkdBmUA@n5Yx4#qACMW$m#+b$c|`JrPw40jY};R7 zgW54rTuZaudmL+Mcu1_EQn@s^J;$$SW!7oNgM1?+Vdk&N-$o4P_%gQhm!AggR}vn7 zzM#G`p>?un;7T1^#TZ*PWL^z5Bxh~CRF;zJhbj?*oF;G&>#K=u zc?O)Dq3Uu#3+V}2{= z^Xqi0K`(RnRB<%?(XI06sqGX_9r3+fp|Eh^pB=xs6t8p+Sst6q3WcZb$-E4`2Zo5= zE_OXIa*k_(-WE75wf6_^-CwwUc8u|E+j8X3M#_M22Y@OkRXlZ5JVGF#gWFHQciI`) zE5C9v%LTR|RasfSckyNAy50bd1x0pwO_`dewOlP;r2j3y$@pOT07{xs9_1Ljf~|Sp zZK`$Ex3g+@YoJ!yb}`C7Een=CL{_;^0UqA;e?!KW&Y*R0P>zD}efWC=wIBxo&R?rw zTmhzX^yAXRCa_p@TMr}z)=@@P6b8D(;R!AFy$ZM z$FiS_sq@2?9pX91q1m_m{?f8DiO~UojUbd1ToLrsH1rRGZLL#<_3=9!CF>uK1-lk5WyMc^nv0tS79^qr zOAzVDrHa4x9$EM9ClvU_QY?-YW#)a&76>sj8t^xh$DbIBm9VyeQF?F zn_2HNz-$0GhW0-+PYpF_gF?qZ?fEbf%m64<%A?7iun{;viHZ4x#G#p4VObf1FJKA* znjL6NdTmR4+bAd~z)a{Tz#E>8`2#O8?~5k^j*H}onvIRFbemm#X69v|WtUr9pNGBe z5pRMpfY1F~q(xhLKlXGVDU$~sNUwby>_--McepQROGB^aF<{n&d*j<5ug(d%csjhT z{9=-mZvj8DRImN9E}c0(elEk!WSfJ;R8kW0jxN*5iRY>L!BbX6ov%t-YPe@3wc=A5uApI?e z99OY7WME)`t?U*|D=ZR{zurIfbnkC=rn*Fx#nx8Vd^JRdc{@u(nbK09XDbzZ3C|Tz z)LGP`Tj(66dtYTa13sbk!JpgG!L(DMmhqUTGfotgba5eWYy5Ubp2Lv*l%5wq)JFUIF+x*c1uo|js0#;lA zr-br|^?V+#{+f9U+Y2avom?*O^M#aejU)dVbb{IRFlzwWl?t*Ek%9{xUq#h z070eF?&Sa+j23{UADJF}^Daoy#mC13ETo%<2Y($MK!pR=?0d;!j6EJ3odBMG~H!ozQu(V0*}BB2E**;L4KKM zfOi{aMd8DS1%Y4IP%`IFZWzAQ6di}9T^pd2LiIP&LR z02NQg;=GOJ`83sNtA=)ewN|gXzi`I3fJ@}%)p}F{ktU>Wc32l`8C-c=) zizgBU<{S?>Lmvy~xl(*g)c3vHt0ecE5UDpgNu zrwe*DS5{V5SI4;XJ)P~|wtxF3AG*;4-*VU6mtXEYa?5JN?LE8lrzkr=F*>>mx~!4A zcI?#yooT@PgyLMaS$N8T;`V(W5Pn`Zok_>Nl~q(614wRfHc)tY-3q84m%j`nK{Vl4 z?=|$QPSXpF0-#u}`@=O_v(fJHeg>!##z-Jk=V z{h!Z^zHoIJnso1CIDG^s*HOOeLUpvj#CZbX96;K(yh!x(_Xp@8(2C3?L`D6MtfIYn z05{_LP=kS7gxS$i9j&txz|%pE6TqbX!Pt$C?ruhW1Vc+kR@^2cqX3uVM4#kCg9qZB+g!1?)(f=&vo{SS zb|Gm1$9<;vbf@X@_pWeFhohD4pxAV<{N~X`iwAND#5@5t149|em`1A5L~64L`8<01 z__W`aaEQU0R)$D3_2oh&@xs_&`Lc)hhM?-dw}eG5Wv1Gy7kukx`QAsivo*B?NAUrYY)4L ziFE^&g2%H+pW3?TK{-ivjvK>y(GKvjCl515CTlu4W7FvST8c}+$#=Gw)mG#CXo|8Oh8A$eP-gJ*9sB8jPebW zBs7J9@m7*;WR8fyk1g#>Uz`oueBKlZd7V*%!fGCgd+zM&M2w0`*IAh|DaH=44GxWn z;Cj0P;$r~yy>ZWwViTB<1iXdCpTDBil~mXxGw z))@WlxxJv-?v)tMq^R;8OBUcAxN-ZWafU{H{yes7pYwI4aRZH zibv@OI0yi<=OdokFN5je2v(sBNobE9_hNJ_3gm|k*y(DBlwiVt!VLd}LEEI@_}ZI9 z+Oej;`ixU|nRI%UbQ-+6m|f)?Pl;xx8|aCOiJdgo zdAR~m2;IPg*o}27eKwYU8h*JJ?wjnhMAX>$KTLZzf20YAZ8 zR=7w=NYG^$Xoch#2Apu9^(FO!bt_Khx05WWch%HPmAO3ogoQ0f?e{Z8A7`B@$eb7Y z+jDASY&pq*vlpgkasf+*$Sh{Tei%A@)mkMdubD{Hh-t8GTHVyuf2nYQI+`Y+y=el~ zcDBFbjW+PzxGheRa2;X8ime%YAb!nYx3IDj0MC$XS>Fxv3MqR*_yHFVl#+bWE%J0L zC>--!T2fKP-h06uHUaat6OI(XTZrtxxX#F$n3;JI{Z8=Vatat0Xrhzip{1+bVcChd@fG=Sd2J&c)y)`7#G#mRCr;z|$LS2S5Y@~ z6wvCr`d7p%CkKD}yXKKmt`?n007oeDkR-W9r>8MdEV409HX3!@Pw+3%r11AaO6lwU z0o~U1s{ZNv?=(f*={7Y%={982Z=LSyK^)3yJCjZDV})o9&c5lMB}|o zSe-AiF_>UGDVqJ6&N?j;`I7COdSafXw&>N`{@2~7KTnqd=ktZXjBpx512XE;T;IntvCS|4*UE2+^_YxF(rSM*nWr( z`2}t276#xRPBT%u?*wW+Duo;1inx$|C<3eWcP9tASu8~CPNRUD5M(-<01Adylhn=x~k7mDZTDC6pa{oRf*RGEMk!@*t00LBC z?;(|EAIdUxT2I1410=UUTZ*YLS<%_B;UP&a|1xt>MDU*)H zNi#5{@lfxv_9xw2MTx|bW3dM*e>lPsG%un>8D)MuoQiFo{+@9-vp3?8(aP6qF#o5v|Bs{Z zrBS7q2Rc8U%kL049-AdIE2~_YOsF_hzN8by2$!?|%doE>k{+tM3lIb4(2NkVhJ_J1 zeE+F11|dj7{GY!|%GgJ|`cH*1>Q3Zg{DXyoa0LnNUmHwEy_1u+@@zls{Pr)pDNW@5 zVtzaHqXnqD|B;#z_o!*!1@<6-yfF2jJAUDMl{PjwyX5uA9EMlZ7I|7X$?lS6A8&xb zRF!@^DbS*jofiy$9*RNv?yFK%GgcdzWF((}p1L#R9rDKYx zfR6Rz52POOKh?7!*=XaYJvi3tghN47pKj(a$DChn9R#-ih_VsP7m@WaopEAh{E zyq9^e)*`(YQutF61jPbVyVnH(q#gvqf@a4K61KJ#u>=`FY-Fiv%GxC9f)j~71S|=I zl%h%EAylrN$m|mAN!y?do-Cf_CO`0ghRm@ILP3_mcu3lzZ^B~}N^78ZE&hBM?NMH4beKFCe&2pTh3cXQMv+4vU3%PvJNRZpw1( z&!Ea^C0xW23m8bOMKcy<33M1Cl0sm&R{kR-4D8DhP6x3J&;Wsk|Cl=s(2$g|;s?~w2&xAHp5}qNZELa0HbM>b z+M#|*F^$p8e_h1B{qZ|}y#JlUey4+%=7q-`&=~!X$1zB$y8WItvqeFNwsdaml!MeL zT66N^UFU)Nkup8S>jUWWmiD_7kSAITK8T;&d0 zGu#Nmj2GX_)!_&8gS7L;)qOa{bQvS)bK|Xy2U!Ksk251NEG0H4tsP7PapbRS^>zEE z@B3(b$smCm5CC6*&Os}!8OSV0uj&z*k5IsKjIIYnm~MSCH;oxCg_)P)m%mA(Fx2va zC}zDqUZH^FVJC&WETcK5<9cK~kEAiCl78H2!z+QpJWsqz&guK_%l3gAFBHYunv(5@-Zws7T(y73*kb=JpsEfJ{ zHaeh4O<1!p1uycROKFVzc5Pg2tjCA0Ysg4xQ>i2`=jl04{)f zR`h#xRDmHAtW0&y@c_ISIhe7>4j#4vomEAJnvTvS_kx!|sXd?YgoIvtvNwT3 zjo5*-fb@5(LGXO8;p6n)_xARKq@9hPoMkD&k30i@gfDL22z&w6hzMM0DF+4uq7Kjm zKsxPlGz1qM#=pi)$?Y6|yV&s9cz6NUC;!wh26V`S=szI~g_nhZum4-oXo&&-fpUE; zC);rU`~mu6s50If3h6+;@g$}A(e%H6@5Xt-K(x)z_9F%p6o}IQsd*udM)>}_v1Ug7 z6O;f+GBulweadwPln229^#AN70#w(ag(WlX)Z+z}k#);VWZkskuYDR4*r2F_z5Y+h zYY{P`{$LYD1YV9^&mra>>4=}{MMkaWW=-5W+lwz)Nfmp3BqO=gJg*EX*@ND3>P<;< z*NMcoD`&jOTej=w&O1`v8;aY!KCP7B`^6jcLk0&@*|RNJ_Oxq084TB>k$(vLW~U@> z3gy7kw}e(ZuSKYr=atswxO^K8@Sk(nUw?G(UEmW)|^c`HG=vJ;n50ifBio#$SEu_RYN;lx=U1|!|8e? zb2Fjn84J=)%BML-7hL-%2Q#Zf_wkeM!NIN2ty+D!JBRjlwYyta=82r#HRE#%&KO5| zZtcUX<*rFHPWRUIln^hZtJ~TGL!waZfZnSDIolo0XmhW!-I_RdalflNYPW9FSBW@^ znh40{QS!BOKij0|bZw>UWxW7{*|p(r-p+LwB|cGfgz`QGK4-dtGSoI*9}tt?d{`RXm8I>D--PxGfbCe zytWLT;JnZ$i!pd`UfKG*7f)quW@w9428ERB;tk8iia$)hdq$`i83d1hN7yRg-GFaS zMg|94%jl7l*S~u3aQ!68*MFGu ztm$Y%UJ1(bcE9G+*WPT1I}^7+8*Y#{{~O_%JURP@7j$c(CR|$1nMkd+709>REpi2% zC@<)u$lX{Dy1_s6N;m3#Sjc(cYm?p^>IP#3TvCsJ_Md;b2`=&PUvI}6!DLNO zU;jHNsf3_tgn#EJRlq&k|F?b)twcgMGcX3K|Nr|X2is2*bGjbZ0q6ho$L!BT5vM&M zF)hVJ^55-Y5w~ljDhhxU1ySn%7J^MdBC*MAf*J`0ym@(bMJD9E#Lo-e&-fo_AQb^3 z0q~HGh=6d*&kEhrc{JcEfah$Wd$TCJrtG=_2#5a`mZJ!P(bLGP26Wp|CqS8AfP=#U z(Cn{q8hNgJK!W8kLL1t9vwO`3n{=7H1)y*PAtT>*0WE{=+9`hz4i``b=-LDT z32=zX-R=mWGn@*IuhCz2Bi;T_$Fb~vJ56eyqoFDPr*I8Ui!B@5|D1vU%X9gEbpihW z{!$RIfH=5%97X(}vCzkQ`1b$rLmQx>cfmHPCsS&wjF7yuuQiVihm$$k5}#2%>sM}a zSs9|*R3rbutM;*!=WqI-UKX#V-t!r;#lx5gD~x^wd3jw<5hhR7eD?C_R+}?Sm7@ir zaf}+DtgqY`b4CPDQQSE@1^MNx^@e&*vcV5Gwj~2>i$?1)Ae-C_w)Y#@Ti}8iLss6 z*K2z6F{17nrO@yDT6OmYH-=sm>8pVqV^Z_jEdia^V)TqFs+2N9e$Y`*V7xyDg zD?8v4Q+!X!88@E1gjM@si=wctFWx8Xa;X7c;2{2pnd|&c(mV_Yf)|q*b3oF2MPWQo zTjQw7^p<&p)h=l}N5y{lv4mZNf8|y5{pZ9=;Vf5KeR@uREKzT+dj~QwBu+Y4aLrsB zwOM?hjkL4qYG-4A`jeiO#LyGnJk)NN!A7~?+ z=)`>_h4XbZNcrv`*uX-M1o?LKvaAyW4}iLQQ~wz$x?k2&IJKqC6;ge;uDE(39m_A@ zIXi#O6VRwSn4jL_4MIWMKuA5mR%I+*O-d=C%F;C@?h- zw5{o@MVZLD!rHdMCPQ{Vw1N?F!aRH~NED0qb}r7g!gYAJ`v(K>FwH zrZ0}WCWEdf8HGGn&*2u>rDa^74kfEk329;XN@wGq^Iq+S~kSwN^2(p)0qTj=K#++>MsqrF| zowHmQ@3!WHHmse`Wfu4%+1*l0KNUh=gz)|>8rzPis7WLR>>c}`Z9n80I=rkGpPsDEUjni4t-dYpX01lW5JaWAf#<~-`}S?eM0;tjpndSPO|QJ}FgZfsOVC1B5$Q<36<6OwTk)7aZ4;EfR2gi{CH5;SLXwxbaV;JBhJ?j2szzlCejDb z%wvte2Cq0))}2dydCu&Qe3cjp z*N<3*8!GKilBo&Vrhtw#tJ6ABrhncp`MqnU3>ctRjH zZE@Q>#jI?(1&^-^8wTE5?NIDGbPK(I7!UfW*XgJ|f3NGf{|wD-^seVnpWFLY7@WR{ z65NAFZ9*Q*`zMTi^k=(5KQ8Q8AQ0q1mc}SvPC>gjq>x{&9u45(r7tu3&0LdKNK~$- zuDI4m7>KZPxkP#}t#n;_4m#v8G3W%94DCzWrKf7ik)(4wGHXFj-RyfvG9hPeUEq4$ z-^-T3RAu#5GbT4{#;8lplEcTB9XeXmzD7I)`5$>pT!3l^{ADUo>rf!Ho$e z>=b_fl)%YvBrigdh()0>@GAF#!`8)XU3t@ugI%_^q57kymKHP1LW~WQc9RBv{*O;0&E!nU*0>MUWo7X|cg%t(0df@I7ba=*DPdRVb3F6|T1`AcF7ZW`+k}Hm&(H$j z47oDwlnpW$7Z%6QemurkQX2@OsMOaIiaY_jWz~J!U1}ucf#=lTbS&Oj@r#(K^=Br>SLrr;D zQ-dKo7!C%-6sG};LkvaVcX`Eu52)!nGd2A*twypn@ndFWVYpXRsZ+@x!!@7Bhn}?&m%!yN+V`qoX>;TFIE6 z|3#30pkR`5p$bZ`+Pi{vln=1{NaVzkAh5mf`@6$;^DH`5wMjb_RzW%8DQypx^QVAN zdT_)~!zl;7S0$VYKX_mDZON8(F`DFM&(o>mzsNxslPDbFI(?6gsV#zWB|&QpfQAy- z`|hq?7xomc$@%j3rD1k+UAX8cL@UFk&B`yl)r>5z*B0WM%J9WzIxd+dHY_ZJ!m4rP z|29>wP>UKc0doo#2+xZG@lS;<1Tr&XNW;zxUwwSKeMFKcNNK*Oj+{VV>XmUf1MOgN! zv`<0j9rHrTIePT764}=+%=Z>W**APQud1xYo33a!H6HOVjt+MWDrbHKF1_9;M+(Qg9~#iHWLQz2Xwl<+re3p5=;vgj{D05cES(oaK0_+-1++9Z%V(83oX&F;^k76T?^kbE$1j^ z7NJ&g&gGj*nO-0Ttp?g+BeNagv=ryMjk`07<%P9fjsCj&^@B3Mj2082Cgxno#lSRW za`D(R#;SsQwCdAxhkq1nBThy~6@3&B6P6JxB#UfCQC07O9CCUwyO8Oy?|A#|CJx-b zS;{@5$Ew`pQ*|3N8O&bfc)eaMvnJ}phJVNE$u(4ON^??Vt)c@z`YP!7SBtvFT?P(> zvZir~0h390mRwr!D0epqyT|=ye@E3mE1_a-b8F} z>FO8|lA0+)Nd}KSgcmuz2|}#&uXYf+-$h(8n~=Cw8*)u5el>zX?z=VAq12!w%_Efg#lqDTSPdS7NgWq_$!P2E?qdP>%tDU?daHejU7#0g6jq8K!Z@WTKR<9CCqTAHSP@dE;d0Zed^wc@_5GY_NKi_xccblq@pL0 z(xvl3O*IGJE}J&~WYyr0fglcHzQ)P${Y_#03%3buF?5Z#NSQHwFBS9F3z)_dRCJ-= zB1*T?!4%<9nIOn?L(q%rMap^(1`Pcx1Yd-C*o7ev2%8QOoDi$5uuzbGDWwM)#MeG@ zbH4rM5ktiLsDz%OVO?v}E!f2~ga~8akPO_(I9vm1Ue?m^ZK`vnHTy8 z)r0Jc_QwRH)t97@W`mCK_{uuilrZZPBgLe2Nrxg}?l4~2U1u>Rqv%|k-*zX@&=Hqt zNHPJ@FEHMjCkKXw=V^D!7mM`yA7;vaam$|z^G+E=t7rVMv_!M9G? zQ4mAUfB^AM@I9F%Y3j@Cp7drrs8#VKhS*Og&I_qCRk}IALm=;8HEX9 zz^Aq%P(6;1J8U3(os@XtX8EvF7jK(Xt~oEveDtcQ(JEGeyWm>X>u4uFutYANFhZsq zHy35&Q$v|THmj()49tfTVG&&d9YzFJ4xU7lIzw=!^RF=tRwSE~8@u7>S(`qfRsWh! zeYe7#I3LhEZ!U2ZQCF9l_?wnkNJf=)83_W>vgg>fd}GyVsDc}w`_<&fRO!|&XVZ2r zN|6F-UT|(g)A;mU93u?GKJe8yS!IcC_TnzO!I5149s=a*#6FbCfaeAlFCf0!np8*- z^{B*jNxg#YJmY8JIq1c=e0#A0+ocv)(E)OAO>T-U`cN=a?Aj>W3dk#YX9VT{IM z;e0~pSvm|yxr)=zCGxDz%;DOWx4lpm|JdYNZ&pg?S+p-vCd%do)5g!T7He)PLn_q( zTrrY$O{{JvG_Cbb!=Xd!xn_xy(@V{*^B}&WKT$oOb%WC#cqG8=6TKCT@}Z4yAE3BX z(r#WOHMu4@H<>8gK97N+g~SZWK*S~vetxr#2?gO$RDw|fl^?76Sl|>v-PeSvJO~7< z|JW>QcEUCv7RDDZE9ZALQR#~NbJi3a+`ODGnrjkGt}tVSMrZ4_q$?%mJ~Up1LXyF? z?!cYAX6ncDbl-2zYIGB8OtNEN&h4YZ4z_X-#GXgZy(hDtL{KXC*0&?NijPZACx#&5 zCQ}APRiN)^?`TlrBvwaY@^598y1Lg}DSoqaaF>kn6^b<6*&?F7@aE$e2##5{ju_3c>VjG%2zvs9jyrFj(3JIkIJ|5;Xs-^#_rajnw|pSv|)x}wV5 zEZ)kwgAu~~dOv+?GWKx#?9b5aMoO2ugO~pgS#JT9M;C014iem50tA9P1a}Ao4elD; z-GjTkd+^}yuE8O=y9Br3_9p+i=hmyb6h(b}HhZ>quU_4Y)7_a;`8_mO%SUe6%uYrs zkBSfG_&KICDgB$7tM{K9IcFsgXPoeSh8z(xmW4dotj7bg=og39Zi+B&RirmC5Wd33 z*9ZqaQRN1FAq4=Hh;|qN$d-yOe(QN4)2JKzJ2va?h*H0>RG}me3HU0m=RtRBbts}; zL4a5`lg5W3UEpd){(E_+UB%_qNMQtBP_Mj<)YWvB?wB-|H6YuI`GtsKn!wySN_0Ih zHrvUQyGX%V9?=B>4+st4hgb~zWSLl8Rz^O3l)%6cHI1VN)^A@OYH8#0Z{V)3dM_JM zSS;pgXbbt7!YX2_5OD&1EE}8CrM$Ew>w6ZaNfWC0U;5H7MtRC?iiivYlSMeZqhA9= z;H4R2V%+(D?R__DE=$?2hY*+xJ94m_ei*_eh}x1c;T`+#I3|79cRfhk;rJ?It+l~kLrqnB>)^TXXnmA*R%YLt^ zua<4jmxcckdrFslN(TdkejU+BNyw&>FS{lvd{$R6{@QS&E$x_<&?@Pnh1l?| zYjOJvZ-(Za<7Y9q>NT=eyjeSB&f^NH62AOrGT++5$By=+I{;`7JH(eWMO-MN(t-() zi9%p#<19RLxm}M!pit)sEU#0;|Ash!aJey-@Y($)jBJaiFSmeOK?g}_9QUjsc`Dzc z7-xU6B1e~ZM(7K4D&o2obZSLauHwwes-TT>c>w%{N zla#5|W-R?PEwTg7^_B4L>ARJ7`eDqAya)UYU8bbjR2`9Dl<*wk2u*O$IgZj!Aecd? zUlyOGV;Ajq8{FJt$j*PN`)sH7&h5Oy?ChM`Md15#nX60cTwk+&bRV6QS2i+{wS!J) z9Teb8M0_YLfG#pb)$UU1h|hZ{*Wh3twp5@d6hzDck2`@9Bl}_~CLOaxr!V=CGio_* zVgYsvHJy)#5{!?_MdWNd6azu^53a>SKH+@7xKVa8Hq9XCrO9uWg$Gte`&M<3!zU(e;@jo*+M1|;by$pp zUWrtqQ1iQPPdw(&6f)#8j+t6HMlstx(ZyU`78+ngoEN5$Yv^O6)MF>PV%^RT+k$En!qqRCzPt>OUwFa(?42}=C z;uYc`PR-}n{;Tw%sY0_I6@b>DiDvS1}sNR{oMM^ z{C$eyd8I6{X=kc{h>Xw*kp=;<5gdyku}$;0U`qkM1^$o8gQfBQ@Abod1ta=uTpjc8 z9M2$RuS&Ijb4|#W;>frGD>>KcfhqANs7V+46kItKTr8pgr@~|w^}9$gCy``x)_V?17Zh zqBII;6Kbi}UJ}J5e!O%xLz3b!`*b`ml9<_6Sn_@!MG#?-KbAmDUv3vHXo-Mj@4PTh zakWNn5wpF;i)F;_gIwCgniUmn&nc)DGu{2m4#?X zhX>FG{&J(yI0S&XcuWN58+sVwGalSL%%bk`AeR&yqG*-=mRiGcQA)aHimoh?^-enjON+k(>ho8fj@AyjIxA z&hu3l2FNoRDx;cw6O{}~V#PXpvQ-cfK#t;fQ1jYfCPVUfuy{>ZDgp6YLk$B%fCg0| zg%p5R$mGaGK37>_;rsk&crgx}lnue%t@646O<{OMZ#l7*)%~_?-m$O_?CHrAl~eXe zBnE2>2w-mbs@)XSnKn7UUuI>G7}xqR_VS~2l^LN(*&8&=(~V{`(W*P-|Dt)JFMs<) z3*kB;a-`XERiDL_XI|-Qt1Z)XaX+hFTzsr`QjbBitGEzytN4DY#@fr`=o#s_MRqZ> zIkEg%u2ibLsEtE+ZZu}jqmDx;YNZcR z->7Mqdg^nWH54Bk>sHgRsSmQ5*)fSM&2nk=S3M8@8z58O3OD=yI3j|roVd2*FZ!zVbF@!3S67osJiBcKl z+dRt|C_ISA+v&nz>)i6K)L8XA zwuAnV#(J`K{AhB9PUX<>s+f}eFSzTU8#)^4b>1oCCx2)k&N5eDqy{8Q5-HHe93Kc% z(|Kh%+8XF0Av(9War-dP20G8IieE|%MT)dnoM}V_Pkd?Tx^d%N z=(l2{U50l+>5F%qllLmJ!Fn@-q%fCw> zmVG{GH->v)L&I2xf?-kkCKRhvbdKbMxgm=)F!Y*QGhBr6E``E<@;sJCtAKr6lIARV zAMOEKG9Ii?&CFj>R#r#EWegJjfof199T!GJcqgEI%C3JAsbl6Mda1&xY((?a%8dU$ zR)mjjO5W=GnbtO~xphT-?Fn|BeQ65LylaMSe&w2MM>lr6x++CQ1^-+nrDVJnH7jxs zJ{?wv<+ZA|Uw3S`1M|e$qsUJ1YCrw&^$|^T>pfis)jTnDgY4^xzLvp-X8(hK&}qZ; za%nimZ?H^+N;_6xvv~5x!U}g&nT}QSL=%2PSsktGA71NZO5yn2AEb-;Vh=p+V#d@2T%Qf=1x|iixTERP7?n#E>|NjHeNvd*=`KU$9j|hXW z>ZP75CCtjN?V6C1h%Geq;}4-YLZxU+EeabN=CzsD zz;jneVOsYgaVc2&tBj%vz57?=pEG3Z2@W~rk~by2xnz7w%9JG)NvdHRtg|e^;dW|Y zs>p}u5tKUOQ#uFa<9;S3?h$@D2F(((mVtu>+*E-a!X{xOtvg`TL?sp1Xs~x-U%fWB z@+CpT*a6XcuG@xU>7)%wC47wT=W20(#ot4}CfcS@o^RyF55Y%Zoiat?jyZ_vSX6m^ z*cIOZcP>Z}QT=!e!VB#~SUEpvL{5UJB;wmOGI9(!lgGXeS9O?o1U)*&4ZxTWnuWSo z_w9R?E`MdgT8X#~&TDYea(EYnO(Bw=nSpDf>c;Jn!D~y92WXeEE*^6in+oy?yQo^T zbTnAM>y=&WO*nB^{d#HlE8fWTg%EN%hxsKdnD)`qBmTsOamB1FL(WQ8Bf^;UOvJ7C zZ_J0s%p%Bdf_(bKoB0n(SwU2pTZ4Ni)abQiOl0w22oIwPI+%RCyf3-Xf|TEBE<1jt~NH2t_JBfU0lZM)6AkdQomir=#L&% zmKR*f5q|A+2l8H!dLS8}MTKw}anT@#Ja-GmJ7$s33s$OBi*U1Q8Ib12JseNoV%sx2 z#*+mLjZK89TDs*%e(;QhS<<(4*x%>%F;vV^w4?>k zo7m)iC>GLYsv6hBkxVgnnWnorWwqb>A{?JP5Rj6X{Bc=pZh1+wktG_lftec<5Ni3g zjiYpauxkCjhX=i(e~s48*@R}pc(RWOv0L&{$$B)XG@e1EFL;%!XRvoAud2U zol7@zuQ+G`9FtS4655sQkIY~Ba5BgjscFk>&a7xg(+<4f#0_0nx6{8jot^Ghjk8DQ zWT%ma+Goy-fqtT&owRpaL9Whk?Z)3Pg!EjHxiwvrjcl&Pq)KxO5&*!lZ{+5RfB}+Z z7-J@#iLuD)rln<_IMhO}R&5Y^^0<#`jA*@-lPR?TQR) zI*qdS*qzlCU&C%6!+Uf4gU9M!bg4%rckS+nMF7BfSbvl(abR?LMGCt!*g$|ovbnIJ zv7^(70r%&V?!x648x+Nf&kqgC?0waQE%r3rLGk;^ePrIvTi%P!_>A5H8E*zj+fp6( z6}eJ~c015%En?U=r)s655!Q>~-HcRe7IU9!xBk2Dv8BgVocs#c!gWjO&>Q+Z)6h z7%Ege#6GM1S|l%poNwi%n%EySmDSX*$m{6Lqvlj%{=26X*kT)YTt#hxxSIa{J?REn z4OpbZOq;p!fS%goV)(#W$^pb5i>HfrCb=AP>4|*LV93}i^+Dp`Ac3JAga;>OB(tG7 zU*{eXK3r}Mvt-w@Rq$s`9#$TEw&UNe%kqjy1{t%~s zOM&0F8zNN(@l8cv79qMj2n-`9TP=X`@T8KAldsbDt(_PE0R4=PzoYK{fmb&+z12G< z(HjyAyd2tv4oMLKDH{w68$pzA)!n~pB4(*v+BOmOUufN(#SbMjR%q(O!v*(~p#ZyU zwFj%87Tz{~p@{M?4i8*e?PcP$-r^|TU_4+Y<2s)V9C=p86sm-IiBUbq`m}Q>A&Qj- z3OSU`o5RYlTnfJ#oD~myhm*B|swNUEfkF^{9;r=_FSA$Tz?8-cP@H(J6A6kE(|UqLY?8kz~K#^C+byR zFgpG~C2bl-r_&uze2|6Bh}fM9BcoN8oJgsu1Mu3+iE=(n%R7neVEYbn6Y=D z>ber5pL}PDW+%_@cyAZyZrwv+77-+G?DcB|BASS4-_a>{dqy2Ag9q&o>&A}oc~3OhnKfXy9FcK z=zns7peTIq`$!O={mmpn%{~+h{IkW6jND+>J(x1`hLbXS)3))#s2RT{wd@i@AvXUqkw#(J=SJK&6})Q^BXDSiOIhl=Jd> zyY|2&wtrzqaPq*5e~1;mJn# zO5AbrGcNAXQeRx%k*K({QpA54VWs@Ci~q}u8Eq8#Ho*t7``<1}-q4$O3IfU<0#fnH z-gmr5ppgU(F3H;0ki^W1(Ex8#5vgGAB)^V<4=noGb}ia?7JCqQwZ>pNJTUrEU zHn%sg-)S53GovkD{IpTb9?mJddX})u7`rEHy1otxL49$^2`5@HY0j^mONSV)5Zf^F ziD23_+riwGk1c$}F)6lih4l^LxP!mhJqo|LzV;c5)}3@E@A8P-ZS9v92rQNEhyIcJ zcPHSl5;ChC8Z^*t-jvC1;rg>clBUIWc@_P5!FY)QlR-tpfMs@6e@IT!U^KHSuVTpP zpnjNOHcy{li)g@IS7!covRg8O#%MT3hYPvbnp2ZiLbz+=;Y!8CimDhLwJPl@2dgNkR%kkd@*HM#A~GJ^fg>+p#u0$V zS}Fb(95m8l^^`I|-qgI6qu%$%ssb~TM~R%N)*@8yho7HJowDTq*wPQ&N&q;>xx{=AGtTnd@wWfk&tVrG>W}E&# zKv5G~lJxK;TnU(8<0F>SgaN1%E+A*rTLswWtw#3-w4`#K%gJDT#wPiS35?iD><;ZMG__4w|>$ z7XaIu`fqp4!`ko$%4{4QS8nYk3fFm{T4Ajb&A(ke)E@M#oaahacGSjr{lWS20p{b7 zA{L}?#bLnK3)L!zbu{%?g}jQo??KBI$*vX%iv@|}gUoA;-z(U!jKx|_U;**E346LY z64B*KI`!mat;)JaDBt|0pI+a;j?_XLnz{u<0Vy}eX=dg&!&+Z#=($dlT}K>J!i|`; zU`r2Pv$u46-ny<-v1^m{oQQPY~$ur*z2G9A$ z!~fR!$Gbx6zM(<0DE#YxNVL%YCE(k0H_NJF&S^z&UV>~n_3TQk2S2!)b`DQ%xPRh6 zM!OJzlhJASZmM!rM~^*RlK`^??6W2$_o6go2X5ac8NXZj6^#z$ zNFERBZox=x`5ltIp9@Lms@&*MG;>7xif`h@i%_8Fz+5Lf*BmAW?LsqB{cTnxs? zvwf+u|7NitIIDQ9!xYoWX{U6;eXSR6YpK6#d`gGi3Eg2UjW?nok+BYF7`-*I8Syf= z)cpD|v0-NxX&Oy;hcN8Pl)@Ufb%c_Mijhf*?$Vwop`H_f$P|3X3^FdY3h!U+EB?JfjK>B+ZRD`P~gCFrnF&@?ZGNCnlNL zqF!%lS1zg1r=bGRMx>Zyu8_W>jwgvG4Y4G!zDyEo^TfaD@PD+(IMO@VN-aAZPijDX zF=r-mlOoGR1U3=-k1ZKY!|}W83r7}HjC!0kwdRqSrC#DRjava=`@Xk)`q5C#jQH7% zD6?}jML#wXc|BJ}PX|`Wd$VI1LaDk_wJGPFZCM-coe3p}r6Y}vkM`~7usRY%1#WdR4+s06loxq|w}jV~LQ zqVI;C{7h?0*FWv%W+&tEG1;S57w+n>PWIM?Pt{K2S+XggwG*>dO>SXZQ`xuyM-EGd z;GZkc^}ILy(RxB@RTHd0wWKk7QHt;7BimTCy6moh-FV7PuJP-ZrswBY=A5eKAwz5& zHFuAi!89^PqwEi%LeMe}b`UAYjzG1m3$oQgod?e{y~(xGcJ<(}YITV|T|-HM-9UHo zs*&-qqX9G%A&ZC=RRblkL@@q+>qpi7QcyamuJoc|!&ZD4isF0Uf?<3mG>ko9EGH(B zt8eGMnv}|bzF#sAohcNTN*DKg`+^56(ybPuX&;5hZHrZv@5iJt+!7x*aQcu8amP}D z<8@peM-(mh-YQw5nIR>M_Ya|WuqXuJ9NpupemG}L2va(rLObvfT^M$j3(jbfUR*e} zpJkJ5hn&oI1j@WfiQT5Po8?M$BNg8gzn8Uq?@-_LvCd*rJ^RFopNRCn^_fh^<7cO) zrLt52CdT+f^5bdw)SJkdDS3tk!!48)%)Eoxuq49uhfw&Se}`h|3K8C|AQ0!q4%J;R z)@YJ3zj$O9Gg0O!w)~B-`vR-d5Xw-nAA=j1Wp5DOKRT>;e1)J3QI<(BC*% z26dzBGpkG&H#P5-9sCEFdM(#$H$;^$fOBa<4-MgS3>i%n!%rR#<<*1ZwuCalBjyg3L^|29oV3JbC1uBc1Js%wA(Ln!j>FE}${(l&G{BEh z?A~k)N)&M-;D5|$mVGC{m+aJ!l*a(4D9UQLCCvW=If~M4?=R$(vKbR@ z>!~y9Sbh!Oi?%q~S$2VPB%J!HI)_HEh|(yF5@I!le(VlDO(_~s7N zXcee%pMdW98+WTUNepP=ezua9qpyHfQAFPWBo`8)n0Yv>PJmYGc*UW!s{1FDC z>q96eT%Cp_EdZxhZFTFLa#1;BT$)Mne)SeKdCSiC$Wazlb(uu%^KVf6~`!niR%sO=rtv@zXNi@@<-0U20q&Dhhst#e5 zI}~7gwYEnTYt)+cA%R8t&rJc_QU}OGiqex0YAOP4VttTq;oQ43B8w)#n0)^2+4e%G)>lIV@ zBX2usr>)a$t(y%{QBsB#u!;#MRec+Qv)BFztL*=$7)%Au%_RjL=9)+$dVL6WtIulD zE}VbDHyl|B{}OTXrcb1fn~qtNxc=c}Hmjs@{{;g?wm(L;n0y3&>V3+XOf6fu-C$Q= z|3pr7@~1j|Jh`#*dZ14bme9F4sIaKA5_N3<(^sN3Ygi6uGe)Ho=BPiMI?stqp3O1{ z-;C20V@QzYZC-`2;bPEFJv4YZ_H= z#0J=8Ygi5#d1p7ST-(~eYm{i)}w)T*|B3QHeg3)p1OmjX{ zb5HJC370~blwoK7lkFJV1># zgtwmh1x>#eEPe`_VT+4+)?#Gcpl0@;23YjW`xZ!`d7t2lXWwBt%a=`y?61bwPT+_G zat5vBa{&L}Po()YOS7)H=(EvPi^U7&XHO8gp|ejn14x;+=Evg3rNlyIl_v~T)6!8- zb1(W4ka1UY1YdbFJuLsOv(DWtH)VbczwtKk7Z1OAnlqgLz}F5z1F`$W8o)*5LIn(L z$Evj!6buN-Cv@b4rNXC!KD(1Y_Q}a$KvlowT8iGmH{OlZ%ZBQ)9cBOJJw~M~)0+w1 zDtG@X$Qs~3THrB(K>b&o&0*vBFP0O;)^xs2r!<@038H>Ba(@1fnwra-1-+O?vfO{T z4bq^!sx36o;;ErN>GAwPHq7IE(^C|kp;N3I3)poQBJQhj(Y6YZ9BtoAW}pLsW6$zW z8%M~7XkVI9fb`xHKGqy`NZ$bgCD1xS&hTd8jD!i7#>AB-Wi@MZt*s_Qm z#gUmH|G;l?A9lDjO3GOY?T*ST;6B#&nJ;eXB{YDOvc$@^ftWqL|L4n=m3UcG{zI3FgiPb-O>QMuZbWN_=; zXqD3nh(+T`{1nzUgFooYjs)6^975%aNS#odRj~)G-G^~s%Rx8`@mm=PqWvKZ9#?{t z)wzbt;)Nq?{I45#gH`T=;Q@{k#}K-*-{$lqDU!Ij$lj;KzlNHyX@G7nnXr)GN`is5 zxhn?K5Z#pKFs{g+LVvtYi8ocYs!?S-*sw%?OR)gOH8f5-^4^+Xh&M?lHjCAj0t6zr zzC^9nZov3flrxMIXvfSA^H)5jHVn2GHHr=38ohr2otn(jiqf(D0kbkx<+=Et0ryDG<{a{K+VbaE$${6-mE4NrYMv zVAdwcU+G-AghlkN44|m}Wfbr#g>-@Zd(NgOmN?&2e-P9IL8I)$#zCfoxSgU_3~4~Q zJP3#QQN~*!CyJY5Y}xq@n!98b?J|#r*ad@qG-N9K!dvfg?@A%WRka+-VcE z(=aR)7k3v@RW+A1h2QwHrs8e#^Q`J`;F55@)T#ufoLbNE$P~jj&X(HfsD2`)VUXcO zK0J!V25I31x$G>;&z(Y?Rl`s;wa`ev1~0?OvjbViJa#Dh!`vY+a>G*ycl(UyT9rz= z_{;YW{khIks1Hk|B*O4$Aa2rocab)Rq|`2~?Kw#4Ev|2!l?~!nsgT!g99iK-t%8MxH0`hAS$cbA#^86fW#AQJ<-!s|0|@SsEU__~`k% zI$>h{JD8QQ+e+7pKJYoY0 ztZ}HQ1gK5VSx^rvD08Rz9=pfeQ2=Z@f9a&tb=8$8NLPG%6=uLaSqsH;3Tb>MjHvLv^#8(wHU z4rMIPtYH{DIy9%LSp89XJQ`MX+6nkc!^Bd5OXJ+9?^P^Hsem8caJpD}HJ|vtro{%r z_c*?WUR+@wseEdd51&i}4+PLy(8NbI2!S9j>qU}du3)dy+u81RvFI80mUvd_g6$Vu z{pLk!N*gTag6A#7eJJ0nKNk-Lx{1K%FBYLx;U5a0|Ary}vcXqpR<M!TcVO zaq1|W<--K4uG4Eq?x?^=Iox+igrRq0akbv}b-Jwv&@c9##ADgX7p(s1{RLF2W0;^q z8R!sEBJCvC_xlxu8n6}=1_Wo0IhlP`%Elxj4_LVds3gOkPQTT% z4LZ!bs$do3vI0j4iY6d#6tA!`l}J2KXZt6dT=H^N-70Mu9yImMyL_O_g%m2)Sbb-d z2ftHb4wMpje7P~#`Mt@AA14ag-Kdp3v~x+k6EAp8!E_||5^222cB-A=l5;nOA!a!u z%hOYxe*N9q2hC5uW#i|t!E=aoJl5fI(9zd?$um3-$On`S{i6QKzelN*Oe`V_9+%2A z0Nx5u$Mlxmd@6b!VTXx78awX?Mo;OF5L4wUVNZ!t5^*+_=9t(ES}4@OkAdIZcEwcE z(g_UgBa!`S|A1KPkqQetG8G-40w+6d7U4*Qlu>7}Y|;Guycrz^1gPf>{^V7>L+`8C ziY!Jq$u4f6Rmpu3A;zOMZ_LQ%Oc*gz5WNR; z1b1NlN($avJChEkGT+nfX{X7p4S!(4w>j3Yc-Kvt#3!MGV*6FY@P>EZFT`qQL`6ls zd_oJ@{5hk_?e{m2ofIJuH4|Pqo|(wkSQPy|RP+3vpnk714Yhv&-lv2=j{<$CE*oH; z=CTEvXuV`owR_MUjAO`}c89lHf`_2Up1wy&Nh5O%w!5uWV0?S8rc%fn>!yZ6b>lMV zD4%&1xN?bmxQFSL&#sI$Xzi`q^(*5DI#ft6OUFZNX9r;AoEOInV^IFuWzx|!EUrWn z+#uxMky}mA`b;@|lOCjw`6Lsc^oXkYx3e!$KE7feFTM5TCMZUt5L|yoHysA>uC;^F z;3f)#)(#?yzFm2<+gvKgyu*>9H>JuILnE<80f^K1)swyK>2=mONftJ^TL@xXg!_z7 zexVO|m^0bn*r}AM{=@r!tCB;4NawUAs)4!EbU3SOLSoE4W!t@0B(Kfm zp#uB@RiG9vnRN)c`j@`1!J(SG->MU{rN-xK&_Y1Y`W(R({S&**+V{D8Vr$0{0?dSO z)r4FeiC{NASn2}eU#x`d1`I2(l$e)71Iip? zTIdGhGgDIlXgE8z!(zZe1G*lk4VQ%eZDsgfuVL!HN+AFQf7R1BU(us1KQ z)yqvFafPoZ^J4&G%SXv)KRzP`-626UNuFK0@qJyVsX-B->MI(2FJr4z2)+4UN2}HL z(d*iMrG($%ynuiw__R9)1P&E7MhXV_xM6s9Mf2c}$yqlxujF;mES&=j#N&>6F=R6;nd?!~Y)s!wpr~RU zc^&u9tmxAxg2CuzoQW-rnJoCWPS_B-dm=Nt3npk{%pRiuiRD0}n~xN<`;n6uNg!Ad zWJ;(US9~A1AO(7F(w7^M(npQ%z*qw7k|FETbOERqqbh;0!I*qE=YUg03!a!G1u?@2 z>xAwVQO3R4Ee*$Qc=!$BdUgxHrQXC{tbr7tuN}-95zPGFZdTiBRDl{NX2C znkHWdF;$oJ%#!e95vMR@jCwlm`^bdizoyQ%ISf2@)kyWJ?E|Ne4MTyRU71@ zb*g(hRIG>F_gF#m2)U1DQ`+XxEb#v~s?5SKkyM=#j+(mMdI}3On~4y}#K^Xsjzr+x z2tAH7su3~3plnsa?#ez_dFlh0%&X-rGvczK9BV|_g#Ja&4V6|RsZY~W`LsZvT&HRm zml2XA@)nPeO8u9l{{3QQEV#o?Xufw`FpIP<;#4*d7`2cHms==RgUce(q&H=-)*k8x zkpz5?2!;F!sS}%O^$E?^gSw-UM~=1@ryw7sJbCw2#JR{OPIdY$&j%N;l>w+DIPuy@ zqE53Jzv#1QCD@3$B@x#ESKY_``p)~`=vYf=e45&}=N7EWxcfEFvJrdb1Jg*7jji{a zC5@A9D@X(83rNON8k#O&_3Gc{b6MGPtCw$1O#kv{Pr9)#wp4*A`Sd_1HBM4Lt5&-7 zu$EkhBmLRcAX(GUiV#b2#HZlj&fF1(y;iL`(bysy)PS>6k|PNf9}R~MyV6SE5ahmI zXci<|(=fj}Xa9kTUJFTBhLXBD-~$}lVD2W0fx(YW6cFL_Ze2N|OS|J zCXJSK@}jyHb!ztR+`f$xv|qVqWp^EawXu7!ajZF5A-3UVc{*rszTdp^5$G6bV>jG) z_0B_P^gLK$(%HwT;u2Jdy^*qeSmF0+eLY67;XTV1yL)yvw|mIY^PF|e4PME1-{=F2 zgA!@Ku2e>FYWnn8Pqyl@9x!pfOL5h!@m#yLUFFcbUp%3BL(}fOySiL|;~%h|d=+LC z+@tE*>v|-4;q>AAWj*;YN@cRfgv{uZzgJ;f_1%$=itlQ6>?s*0<*oK%PHXLS^_G7+ z%l-y%^T9@a%{C6QoZC!V2J$oMBS{~PR8;&dRquE*9f(*^iDf20|E$z-H z`!6>Uu4{G7&Fy9O2U)tdL2SbFWuNXzSu*pTK^jg867>sZ?6_O_ijua~&Em1wz_w)+8Ya6~$l4_q?$8cmBU z4wyy|7~kg3QN6FtKQ-}ldX270#z3c>Ka3M{3ckgz$ljO5r){`{9d+|Iz503;1K93A z*DvKgx9!6eg#U}bf`LZ$AwF~>8{H?8jnD6Ff+|Iv!A7mwx3!a;t)ftzJ!T252W{X#yPwl=pQ^9jqhIa%N(%!kJV~1W&nj`p~ zXD@krIe}ZFOHaF9t;=0*BKYyZ>e)kq&!wl}r-J8-<1B%P71UQRvxl?W*MaNt6w*Cg z-MWqT`TeA>hZ`NzJ;Arl+bBkx6~}{>*cTs`w=2UOK0yr}#nD<$_My-ii*5rA+>#h)h z`}O8?+xNCRDu*6Ddbdhqo$)gDy~=ZYJND|X+-==)hR_q83N1Jh_gQ^M8kYH^x3|PY zf0sKEaTS$kXg?-k`$hY08~$u~lBOH?FH(=sY=!vp>*9(+w5EJUB~!$|iqNj&NVKbU<9T|{5kse8nojGV$v!LK`b zVju>a$R!*OUhjpA|GLd*sSB@{Yf%IB*}3ca{D`e0 z`5lU^sw&P;Z`gmMCR2Ls0iH2Mq@O$Yuigo%XH7JZ+A zz|Ygb;=UI!S#L8ky3*fWxS+~RM0mL{>7ULa{|{CBUSf#7*py!s!n>3?Rd zuxJ`SDAY5E#H#o|%izDi=Gk1uKp=5Kd(YwYJfx0iFah6Nq_FOwGQZ%7cLh5YyAvYsCGIPeVF3iaZym;DoD zTg&Egk;i0drTu~K1|S<{rnvOaxTW<5kI(3fyG=`e1q#bod*b0e(CR0yD&>ak&~Uvb zTU@c@@+RH+GVC?UV)vK){DLp7(yN=(c>R{OY=iWG*gje(ZRqw3T7Tk}xRl3*Waj;?83TqUU%pw) z*>+f#SZ~7&8UBJ8t4df{ZyreVOQ&pv2Nrx@rYp|g_8Jf9K#qmP(I9%i(-&$JeDzG< z-xjXcXbA1&jgwV{n`!?eW)(Po)-C90CgxPSF^x-<_5v!x@d?6u`eqQ!6-STe(JM%s#xbpIS((Psf z*X8MNP4arTFTaV`1~7Re0xgThSSurpM9HbB&X*hLxVhcwvt*FK03^7L!HxG}V9P^o zrjFO>G42!ao_?)(A^W10;lbYr3pAf_AOgPF5zxT=g2%jEc`r1}l7yUG%uDzmmI!*HA=fAB*OU=75WM9{xn8&Y*ne7a0|3F;h*a{@{vV2% z^xDp~Uw1hD^e6wDXMs>@RFvsy*CeeO!<(;e_J$3C=%NJ|(nXtgUS`z8lG59kZjzuI zf_GBsm&bWBxb|*(fRC7ohM~CsBG>W{v5o)>@jot9)9)^lUrh6E*CM-w?KSudGGgoG z7?v>F#itHx2|(Ct@pP}_OUvJyW125kGSWMoyG>e zW*uADubsmGJi3m)Z1D&et;+MCj!otaFD*3hTYrHQHHND4a!GzlJBjTNA5fVf20rsk zn%441ES5q0#p?58@Tv9LdD$C_s8$wD?=Q8k3h(oyuADSR__2S73;a$#-C4px6>8e9 z0j=O9oH;{qr{{c9a|r2FN9NxKl)rzK zWGnl6^tea2A*vO3NkQo#2f{ocCy@~ZFCNK<$U{>H=BKnQj;6~^rp)Cfm0_stmcN;+ zIZ{HcFoln%LkL|dcf6_u#h%x`<;AWU5G{s`{w>v9`A4X6&tgqFf9)?A)>4|YlD9Ub zplm<`0+0CTT4%s4D9e$CmZnx^k1QY+`SA$3Z_wZ(MJnkh3osGk0^8p^PkFYbTf_Rf zI&cz_quZ@>ph1_>tjRu?$RLcm0Gs)qv`uy%tLp%|yS##fm_B(d#4T&hO5M6@9_wj%RCdoAwvnKIRS9 z10qbswBqascP{Vcgg^l~Du6NWi3Ujf`zFdg z&zS2WfP0>O-LB3W`v6!)U!P>wD2~rsHRbzvpjKdz?G{h)84YbD>@NA%X$_koIo#WR zIxnuT=Wb4m3*U0!Wnf@f_1w$Z|Jj*A42_ERvKyZM4>%tw+~SLG1X$TnkToAVocXmI zN6G*OM6fOv4Uc?ZX zIysyKTQcHjyF%kEQI}kn!GR`s@Jm6k_>3PshTm!0r0Lq(&%}xybQ=qp@Y?h@qH-HY zR3<>k2jS{o`e!uOR9UVo2OA|0%R`&zth`fFL`Uqf3uZ#yndBd_k3UD@fFik4Qfr*D zPF$k4%#G4`mJ8zbB-^oic)POr%#4p>G;#*ZJ94&5czy3nHysiIlDpW09D9RG!R}$ zvTXVwQ&KIPw&6#;GIwY05SxnP{dai4dE3nwoW9;(_{qr5US{kr;6%DBMO5$A+%Z)K zB@0!1xc9KfObE1XN>__zJ#0HxOLIhP!)m_2z>pTJq4k33Z2Ga)!unIN?BuQE1^6_W{%M1DcB(wAPlcGG%SN}!xwG~2#H>-^ z9l!uQJUT)~LV7vscwwZcr@umzkjUWPjj(o`639&Sg>S*~m;7Ms`p!rhv!C=L#4pX+ zUno#y?op+J&1I$%-O6#FGuejw#+U7<8y?k{l+IuNNU1U({t`fKIaBLKS%u6$>RE*f z5~r$b4h=76gUIxA*mpDYec0aBDK!d{V&jbMmB5sP;AgJ(og5U2P+Ch^a@2~tXOVns zxd-=Q|Cq@JA4DVl2&gVqSaih!O)r?_ki9d~o^nJ4`PC(g>Cr%S2#p^-Y`xpA1V6+1 zhujMD>LX!b>};{JfW56FON_6~UtgYqNsuq=LFhL zhlM9{T|LdGWMv{F)hl>9JLg|#2BoIT7Gc`^gthc)izvxa>5qP&K}^ro=E5}a=`W(GsG_Tt zo-B#fkT&;}Mg)cL3+j4Q$uL6y;g_hInkkqfWi9G7CTIJM-8wA)Bc8>~Hqaj%wc)mUMBrX8{0Rd( zkK!spvra()rPu$k(#F6};9WXX$>r=m`=pwtY*;eW_zmx@tRg4STCN}Oc4qzjOB~=qBP?OomBbCpIdedjhUE4BK8}iu`V5Ksb zm-G*`?~Jg>pdgz3H*TKRS3fX2>+tDd)6#+`^%6ORh;;D`w$%3M8h=F)m|4nm)jxJK}!Nl_a(EA|MG?)HYyI|1>+W$)V1P$>O>hmaJc0iYQ z&F{WYvVT*l^uDC?HWA>fa8afc@X)wfZ=SPOa41Br&|3!j573X~MGAC5QxCdoVyp+H z=$DE7#DTV$-DQd0JNGz8o>S$5iO0_Uv166&rPX<}M9VGpglX>awal0@kN4AitWm(p zMd%Q*CKs=Gi*1s!z)$_)T$9Rlmt^KgRNsdswe;38UZniI;-_2tSbdyzrfNebeRy7A}uU|BAF~4&*Z<3#a;h# z4Jd)P<2U{bZC)ZsEWNO_uQ z?Jo~UZ(G(VDLeZ$q4cl<&8F4sDx?!VBcW({ax09_*V(Fz5x39zkEP65v-(0`L6CXF zDc^OqBT+I9PhSRrhEx9)6BQ-kcB~07nCoWng#czDP&+h6RZo9k9PKGq#yNw7WXq-a z%Ot<&uGczpm2*)zVx+s{B!|PcrO=_A4g*wHtWhq-%H{4Yh}MP~x_Mhhd3f?rY4C7) z0N#DE1AkDehzGth@DnaQNjk*mc(LqHJ0BG`gPxM6n2j{tU@*C3VdBlIupB=X5;o}^ zODu9*3Z4q-&YBt?rjdimT-F4r1tc4jCIBL955Vcf)$^-B=#6vNku`8<0cnGX-DHP0 zU6}Nvi(OyW-J4=3fu=v*%dIZ^_cxUYvm{N>_XXu6*aFRNB!3gri6jSttia-P%Kv;N zvSR3{`F;eUPW>v?)Y=S~V3t$)f9&hFPcsFCkBbI;z^m?}TT5DlyvpG-tPwrsLhu;r zGvX=*99sv^TJ;4rv{LyD-9&_b9zs^y{4T#F38Z{wNa^kE1pzmy-{aDnBoN~%0Q)4I z>%|*>uqT9O?X8aKFF|~hx23Pm*6K1(&{NsoF8(VL#VvMRlEU429qxt zqQ9~zIG@-v{5Detz=H|~C11N@Qv<`sS3{jPK2#6y|guFPzL*9f@+T}5CkV=5t z8B2kh2suaHBFTFU;71$#;b?~Gh5^jI69esod-92kj6YdWh{j4$k zgLT~NoMM_Gw9licdtw7OnRIrGv`2b6xW~;(qE5=lL#lsj3)XUfn&1DZ&LowKbZ>1O z4>#HG%uOnCYw7p${BI%mVH6A8>;%mh2OfX4yuTC@TE>w-y;|P`NX5XZqNAhpdJlPB zFl3W!g(lT=GyrlW=zieYA6O;|NJd?P6AKFq^YiI^e?1vEMsTZ-WEZZB7G-Ik2^TXP z?%=tz^ijz9`mH&$$$+a7LoFE}aepK2{I4%-WlZ3o2>b3^24ud1 z2*6Zc)21Tx$C{Tm7EOj~g@%FK`xjt6f78XX!YtT$*>=fZZ#Ax%eQvEe&VZ_IVX7rc zIiyDHN0^=aKA1zZ3=mqi%ee&v9sz_FMZ$hBZdio^kDqS)#pvMW++0OA57MOsfOc&; zYi|Kq!}H^@_<$Z5QM5`zztKSdE_B#Wz0%iNVge7^tyc0vJ%ck33G@fsF8*WH(S`Hy z;jx0Gqv1z^loJhO!=0g&3H5Kw8?Am_)k<%+3lQjIW5%A^2nGmV9st|dzU_Jzh`E0K z`XzqGcAtSZ)eHb|7i!JVmYZtx^WXT~K=wMeUJ|4DfAO9311~>nmuJtqW+loaT%REh zOJmj(O|kGW$M9Fr9YxR>v55N1$ak;#+r1<2#9NvHhJ|1m(7=SF8WkndUIE4iF#S$5 zS1WWVmJu|TBlz_t+3#<|lJ_F6t(=;@g8V!RC@5M9BY=h{_5N4K17o>*i|tbV#f4LX z)D`;cKXfd<2OzUh-uvbY5cIa^jfa0 z$U%5bP@d|A3GrmgDxXcyw3~!H=I$_bm>;F3^pJt<(;nW-X%$6BeDi6mYgX|6u*ENN ziSS*9H(}g)VzTjYvAi5|s(EVOq|=7g)H#-E#&8(s{Hg|R4H50M?2Tlr(soKL7M_=d zQ5lo@a2}ug#*}k^T#7l`H!sb2aC+9V+X}yG&Cdb|{7^Jv<}#jncPJ{L{AL3E)_#6m zd-bCTd+DTjStO0?b2V~ecOAj9)3}S~O)FZ*a6S22dp{B|IRlKvDB$$qmzx=)R`KzD zPk!E@B)4=MBkY3{d4M(X9u^W}bbGooF&h$21)S{s9{-Sa`Ms+^+U(p!AFI^7xw4}%v=vl*#n?n!_rxId-!kaq0 zW8inclgg~KtV&5fDTjX<{F{c|$Eu8Sf5B`%?_)jxasxcKVpon59rp zN8|g_WxmqvI-wqbhZ`8iDv#vx(R}=a{l-hNfxg^-rP2S<;pleLpQi{f22&$io$V2=6rBu1WiDF|yg>nXv+OJM+a7@X4lrbY zEb+vw#UYz%D=ppzpivNQnZck`Qw{@O4ovbB@VDfhueN>Z|8BVH$NI31p62*Ifk0)cP8MY(tgMp4(OXeR z06My3_3E*bV)Y}2Z>sIt!8vu4ZalcLWX}`gYIJ{-OJYDD2Iu*Z>sSAn!RBL$>~-bY z=!9}GC@vVzGB9;ZGsrOG_w?xn0jX_DLv zxr407dilZuPR@!F7x@Ojngrs8q#GMKa*F0L#C&w=QZ*Ht4bKIkMA7kojl0dR7aDU} zeqSnC!#j1$RZxj>Mdkc}Wy<^(LaHbV!L1L{LIBY9qzC#OU|XP0W;3Mo$GFRz0w~`A z z!e7Zc5?G+6QM+J7>fGP+Pgkg!UCo!N%e4u363o;ew2++3%~jZXOjENC9&Y1}!=@x! zC$!Tef$HG~ouRyY&mL!`I6~YT;KKN7NAl0k>dN63Yuv*?*TH;?Xn{6gGE8sF7Z@D=?+4_uwPKo8MQkPrDPi>Jo#@I z55>hl{PX6nMJ!uANJ9qdA!mz1Z!tBCoKCI0wT)}+VoujdoldNUEcfta;%1Wi&pK4m zpAyw-5~X#0F&j2mtD12E?Q6h8QT;f466{1UV>>*nQ%GsPmuM>LC+hT`L^Mn{=YHKs zfp|ZCpVk!*`)v5>Wh1~Ycugnyqr%Xa+;28m;*#dDJ!r7mq-1|GF0IUu8tD~L#vSS- z^31qfUB{lWkJ}s^?bpFb?BPJ92Y9&Bj*lFY>DSlU8Dc;qp`r4pCXsQ&?O2q~ zZDJhY0%qi@oQzyMuFAnNLXqIi(d9I7lIY^To)(Krk0jD3Y!6q_~#y;=q}f z4is=9PNnP!>;^p*tfs&W2K`zcg~MojpnE;)kA-%#Ot*r_Mh)@A(Kvy83A5X2JGENA zriONQn`E;xQ>LOcM}GoxC5m}gyxNUW+uL!$wGr85{^(SJc0cH>aoM`Mj?0vySu%|B zuSqk$+|~+W>O7zioVawD=_^kqJz#fV#UlvhHwA zFDOUG2(-7mk<2z~bhV`2^G!>6 zl-Y75$qM3S$3#GCk=9a$F$`&T1Lq0KQW5E_#8$FCuL-n)ck9igF3kVI86VgpIiIG$ z{)S&4a=h>KT+e#U)lU_%XPF!D({K0+X)xr|K^r}HhV1UGp|5xGp*o?NDTMsF9xGr_ z_k|8{t!nzR?`9BNt6A-@dzRGoeU0pZY&?R-E=0i*u-p1uD zm_vMbGt5bR{&jXdFLlJ!OD3h$+vPgAYsa{HJud;tLO?HytA3_RKPwYVxet(jjXsw! z-#pBh19eK3 z7tIB@O1InHghbd2g|rgyL-qDPnR6ez=H|*>N(FZ#N#l`V_|`}Jm8l7qT764l3*Lic zhu$=U7VT=aIQkU)Y5Y22uz!UM`A=C!ZU1LNib)@!EAK^4rW6bMx3(|9FMNKywO^2Bw$*&|t6I6?Y&QhDb7y^jW9y>e@g2MHFomD+d_YHChu-UtMTa zNjMpy+)GA{Y-_S-2qCjGARyV8^-HtABB3gg+QcZFcQrkBB&c7}CiTP=VfOg%0466) zZ<`iXs=Z)(cdxrHK243W^5(TGPnxyKA#SW0+7D9)u@wVw7UiBJ=ucp)`4G?WnBH>x zlQ%h~053Qod?%gAywxjnn87p)BEUdwD9l^U$Mx+JbddcU!yjGTtS*+n@{RjquJhG^+?R3q5Bq76H)$L>PlY4S559A%C(3ljZ5)X8{|7~J z2n#GTGwu;rh2YOXBr40==JKC8Ihat5n|Uxxcsmm`s35wlSFraj$?S9r3(8Z>=a{r) zUVY3%xGJenB}gvoANhVuDK8llZN7)+r(sr`94^o-hvrY^)J1W{o>-L0sg~5?nh$`c zBy!Z21?5N+9b+7;ehoAJZY90Q-z6dYbJhoZ{2o(E3_2hu`1bZ+znQt|$%k2*LBCt= zvu3-x?uhr@F`E)oi-k6b1FFTesfgx(%Eu7Y%=vx&W_SQDq3`|uy)A%7-yO+-fM4#j z?AxW?oXb2W_FBFK(=O*#|7BqK>h|KL@hI{^K;wKlr)~1@#9~>Qnkp&1$k$(Rpm%xA zF8=4=q_HUVDg7j4dcWa58&Q!~B#0GxAXkeaP+E}u=~a)q=5wK@$3b?Ji_^PG4{*}^Zhko0W&+o{$g~Y_g-OpB0LEb>=slguN&8Xq)+VF4R9VW`( z`^{c*rw#}~8`jLAD*Yktn{BE?bb_5~ipOD&U}O%@ib4oTJgm4$z4QkIcjX#Js}d7G z9air-MUXBsI2!%2Ty_V;k%y~N0P*w4E&+XvwSrV0;|g6x{45do7Ye-Ub00@55~>0B z`@Ru0>*zwgi=Q&pmQH>b6+!*tUNnb1*bWzW2s&OBfAfut{){FCs9HY$EdSPW+eZWX zlZ;x&F_Roj;hFX1p{Xbq8G5qZ3hODWNJT-T4y~(2B4uUMzM77^^Tc-EH#whMV(ccD zs0rD7zP17H@zo4^yq@M1Klj{FV!;4MUgA>jhN*Uycv}jd;fAuhta|E1x@!S%f*5_+ z>XEx#-eWK81w`&PgH;3Ix9VO#U5=m4R~t_i%UjQu0UYxkr>4`N_v-yYjJBbFwTxHP z^2{%OD=J+l_|V#g49v3x(-I&T}KUB_r$61hDKtA@dm@J7cF!h)8DZ$CMIDslOY`_h;)irWH4Zc zBRj<};Gg@|n#7@iE*k3uhgXz~T#M1i`=peobqgF-#3labOUbC~!hh9tPYi_hjBZtSv+YkU2>1Ztr8x8bc<0N=NBBu zsENC-pSa_;h@*xdDGdLU@>|UO-Eg!R=r75~5c=@f&cPSI^KnC#FRaCkA3_S%Z|>RP zK>p5foC>)D!W5%N;tT6~q@Am90=TGIGru$%>gtZyI>6V5GeEVcw@j;cS((|pGx+bH zY84+L=+lp^aVdU}Zke?qdfLq0hIijE?Oj9ZBKnO((Cxl{-A+y)_40+}L{o(bmzGEs z>q9Kh?$_$o_#4c^MqxZt4+fEZogi)MtZNUfQ&APy`7v<3#OkMm&2!ULIJH`pYs=43 z-MB^8w$S|th^=S#n^@Vr> z0hDn|(HLN!q-WpV%@vk4&k)sz_aSJSS|@3tg+`p51j(LcrJcQGP$0Wm(#rMWhK+Fd zz>4;tiC~8``)58m<7sb~wQ1M7{{D4OVKQunXI$A=lSMxhl%^Q+lo=-y4%gRnxNCJJ z9sZseF_Xyfbd8J>n}O(4PmPI`Q_DoDdk%NK9*P>nz5H(+eouEIPaEja0pDwCoGy1q zmm2J8TR!V|n=2dhR~DQ?}xJksJ$khnqVfR7K-Gljp&*g=3Nxx}+HBCEchlW44 zt3rp3L@la{6mLG{Vwp*`6RvF;DSxH+0u!ci7p`ekRFX^F{J@|&Ry~xwT$N)sn*<>Q z*`4ZO_Zt=cl~>zK)!rU59z1)7SDSuZEs?wtw+^#tln-3)o(0*o4m~zf zNLQk0Vt&p$Cf*Du=vsO390l z2D7O1*&Kr>>pJMwbPpB-jE$(3wAAd|VQrOmGt#ufkk}in_#FQdNR!MFzYXCgs9 zAJ=6Mc^iewKy7b-|K8pn%-d*1zlSSJz*NM5sW7my>padkzkmPUsa8O*m_z(N<8G)p zI$9pzI^@{@pT>~RXq+2oJA_w&x4KGwIJ%&vF1KW3*e9V1Z*%fP%!%G@3;=ER=s3!J z5cMKbwU=Xj!i2#FPEwywN=iy@ZuL5+IPtOX$gdVlg;J&Rl;$T-O+U#7j|9y40cWrg zef1qY=>J{t2=#}A7gX%km;)-AEHJJ=w`T9_UCVKycHuH^#={RJ) z-HZ|wc5GMN%C~zr9nXRnD44S9#15PE^j(qz)SGKZZRVDo_ivC;W{Na7oh4l%uO%j!AGe?Pu_{B)x zMvBACPR;!eg=E&&FnZQ%)KatkS3ftj2|;cZJcFa7ZC=-h0Aqc&Tt`D$nPc5c)nG*p z?3%^}?g9xL?Ylf|_C_K2>~Cyi6woezTGG@(9l6TvtGR+jqm89qRV!E=4^qC5DDHcr z{)(@rZKm6tf3K`X6Lz32tR!y?_vcqcb~|H)!ZuYqlG4Nv$GNC~h|YUs)GdhH;jyuP z$JPB_CnnXB`*J2(L3#NFU-sYo*xw7{u|dEAH_`LBr@zy?Fpa?vqojGvtJE0S4Qb{3 zO;eq^UuX51;*S~pq;8`Ow#W7Tt>e6f;Jx`sGqvrZvww}(Mk0gWCrPI$`wLb)euVBR zB-+Nh6!2QnbN|l4RYal_V(<*u7?h>?SvD2#LG_5mN(~{vGBrSrUd~}PDn^XJa?5iCy z@K@Sy&YPeKz7W+2{V>ikxfcBv9Kw}xVM#KqKw#E#H;g-ohZ>4NB)W9?}n6~7cE zbfBFD0wi2gR&36x=;6`PKsQYPFl$FP1VTzckeQgs;V(0fOcM+rn$nD9w!R~GIN0Te@HAR$IO(q@(*5z8q)<_-Qd6fC?*vFml%oK z!1h_nk%L-v{>*=1eVvN8@<0fz`HK%-y0i1trj*vo&JJjCs2sHG{bvX}8Z`7X#@pn1 zWixinhHoGYAWBeF@BHSzq~xD*)cEoW^IoGWr9W<#=eoJYE<+6iaMeYJBB0)($W`&v zRsRi=Wt52)+;(MQGupaOQ42d_)i(P8RW%?D1+txE_=HIa6aC#2gsa0GYQHa|T!xGF zsorB(AO&dF9~*=B=dzy77V>rjamn4>dQ-&nFOYU)iJuk32?N#9N1#VAwD3ml^JiAn zU1V`K##{ti>O}Oc@;QtXix(vdKEzg!RN3FJ9`{oMF^>_DFnHZeZ1UjuP8|>b`K)@6 z?;^Yl(0s3&Li>#t*Sh5vi!mB0viDj3)DVehXeb0KqTh=Gd(k)os>^AiFMww8iezJB zBTn$*U0C-5P_}7U^OJRR4po1>_+$*4o-yPfrhs!sDk&h}Z^Nfi6fB@3?=n?c5soB<2gs`gf}SXIZ}#bCvwc zP5Vxg%N>|+&bM|Z5gK0O=8MPt?67CP9-Y2iR=o`0hMyo_YE0zHIO{gOEa$cukdoJ1 zvJ=vdVDSSk9Zk!x9oTC5xRkHvuMJxl4T|ND2ceerU97pUh9kz$DBLY-_mj^$qdhmt zJH37#?_70X9urMP*0_Az6}&ep3;Gltb@koTZJ+K*sz-GfK-gb4xPjo#^$Uc&{!5In zOQzC6;|1iJr~UPcr>7nQUdR5Fv2}6V@pw2Jp=#ufLV1DPicXyNdXV+7#C6ca%@r;l zcVs>*J~z|h^0LCY<=``KPip$S>UiH`Ks^0=hR}1<6Rtrk*?!vi1b+E?w^d}^b5nrK z6N9ekrrTbiZ`63BVU$I3i(vJ)Hg@>+v6kz$9r&&c;Je{Sz(}8M6s76T z3&!1aUs@D%!SkkG1(5np2yt3RiO}m=QYFTcF?2roeC?4aji-)5z={WBX?lHcKk89h z{BdhX@wMOurdr7F^o-;gj2}C^X1NG1aTKW*Vz;{Pfivpgc`aCd+1gPydi;8$vjut2 z%d+tgMLGwI8;w9Q8e*JJ>yfPnWqOwQnmzCiM$zPbPt3VO@S^9t=a#!{SIIyYC-qT@Y6 zgO1nW!F}`xx(dQC1N;TRh6GxqSv#Hp%$iOv_{B3oG(J9_&2E{uW{C?Hh?+t2=e{Kx zY>DU%qB2s7{D#soxs68Ixn+$g`D~jQ?Yi^`W9Lvr3(0QPw|PYucU$C~>>$y=*520W z=7aC#oSOpSJK_T*7yM}hdfJGn&KFZxg!lP^JbFmr&?T)Dw($5`( ztV+uu&y2_%gWLq$G^A8-iEf!={YvAlPv zs3|f<-OkIgv9Zm~{FMXf`xr?At$%tPHe#{nPTf~dXI)i;`(gnTbdN%kN}@WrIlC&w1+YQ#r7SK%N0Q>71w6*^XzCOx+=|Nc3E8o{qG zv=E=f?ChtztvF!ryr-e_Pv5`_s$Qo&1iisWPqGN&KW(S*rX*6cv;x&;2m~y4N4mTk z5Cq;&#_5{vDl0ASaULa^AI;I4&Q#gi*_Y~VVcyyR04E?}1|@Dsr+{`kvI}}R1e242 zPymWdH-3;%iXTaYAxC(=6BBE<;_klU^u^r(wT1+EbMH{q6Z0)XKGIW8-($OhhdT#_ zE#)iuR`B_)k3l#^xBR;TNu#61TG?y9w<`#wK&~EvbOe~MPyY7Vh@j0SYe?vhO7Lkc zj_KnX?eGsMYx@45go0RL>kb>*#x0%QzV3hG91Um#Kdk@&6!*}sV)Avb|4(MTlL57M#*U2Bpm3>_`o@exCpAg%hF}8FzB9cDi)8;JD8{cB(?pss{t(_i! zg~{LS%2Bju_gZ`!97f~U^5{yU$I5m2wt=sG;Lasj^@N0c9y>pz;VD(n$VEaO+ zjH!S)f%s+K-H5mzs@7=pw!#abLB5$kgtk3f;GudYhP3hVQX1PS z0H8gEFN>L!BzFqGO@~vh=PG*gNpvkg29FU4po~%`j0e+p z!3`0Vgqgty->Q6X%I-cnqlw6nGaYdn|Dp7@@;o_J{6;y3*>RC;7o)IXxN#cN-fcn7 zNng01ZyH#7NW-SqUTzvXo=@z7HuLlI^PUES-+Rr)yrim$1sYsy%Cxvm8NuB)5(cIk`8 z@0okvx0su4ws{SO28DF=Gg%7n&*r#+?jAx3?5=9>FfQJ|w*SiFYl6VtJ?tc%sRpt8 zK3pv?>*gtW0QCYs=UtnHnqJ3yAooSN(Y;8izcG};h|8)^gpDoXCO%RYNW7?=@FS91 zg|QY#n<_Va3o%JEqO)=!t7nrp4c$S{`|OZM1Y=2rfaFH!MJB7W=pn4`DK0c@_hO97 zpZM^iM~c>HDdSv0TyJ$|j9AXo9A#;63z0U+)KifcpJ^EVdT`y97QH=*m)nN6JB7QO zr`h}0ChhGb_L-6WZzsekBO(4o`Rbvp)wA%oAb@Vl;#-~?nV7E(v~*-Z;ctSv!6e$W z7P~MqK zdFvXbF_hu){<-)hL82h1aO>+Iwnz@EI!I;t;qBBjX@88HFCWMn0|P_&UV%$WQRKN! zjz(5i*88Zu0m#Q=VPQ#IhJHB@|0kiO6i?er<28|eSEZSymDFA}?kykByLF;5AC?kB zNI^Mhfe+jkoF2e#c~AK1(4tepi(3S{%otdU%O^v=OsPOjVS+I31QJfB$ zv)3c$>5HPY5`R$O@&F`E1Tkhnive&!W_>gt1Jq$*n{2=X%@h$Km$84fkv8jUeQCj#Ek$6c`9>JSp2)9?f)9PNxfc z>_|_=#QYWBNn!;UI3Vv}urZCkyKC)8$^n=$3s_nq1}}-pU!uHwU}f z+o}}EAf7RcNbVxI)#t}BmZwp;t$9r^it9aYZk7p&)`{F6T5;>g(39=+;H1u)R@!Wo z)Sa*glYh-g2y4Vt$1|7>ZD>yp-x^`0Mi3wQcezIdv7YFzf9xO$qipoG>6<_e;% z4{qV*qEMyY)+Iq{{l%e02)AJ}V0I!?X1Ua7^u6c1v1pOQ@nUQbU?P<%qdS1qlD22g_WQZ~zwhNpxF1&|*T%k%i> zGiSK5u`9bHFtyp;GR6Spc!z5BHqI@*Q%-)m^H#vqnsM|4SCQ-!@5fcVHy5?yXXV?6 zG%#{ThW=e-FSo{R`tcd=Yjg+t+FUb`LIT`zzh`S!IdhK+e;k@vrNdFH-3akz$8U;2 z6-IXa=x+m&%_iD~)srri#%}#=0+3LD%b`zQe)%dH-o5PbO%cBRspj*UekzZnMgHU` ziAlRu8@b<(c}GjFw>dIQIF&jLXrMPA8z|%J$JOA+yaUGldfv0$1}E)rARc&Q zP2Qq4=i{czIf(-=?H#$hwXN+l;3vYsz-UzHO1tU6M-4#jj%3!ry#H|giEk!!swZNV zpRMpy20~)y$bPO6dzN(}$7${)_GiqXR<|Z7O6jl8F?}n_nBKe#Ee3=BIlsnXEJt_K zbA=`{xDo|VAF1hAUwATd%arOTcnM|aBexGsc>26O&ICTV(B}R4`BS^Y&)3!Unlet% z$ZudqD?c?eGn3n9zRIvO5XhO%$uko(c-)e_Sr`NYA|yUOK2qN+j0Mzllg>_QGQJ|3 z7W@I~=GiD;_db)u>y_$=Q0UAbg>`?N^!+R9%vlD$+Zmv@y%qhC&Np&9p;3jBV9R!^ zG)Mi(;Boh5K65INh!*jVOc@jlHP05e3m9`BT^G*>$Bg~XJlNWp^2t}(Moif(5Q6b# zqPf$I$*j)4GX^y5aCK~s)S@I|l1ry4;vte`A|f5x+1Z|#yS^{?dw|p8uJN0gnAp|T zRbHO^MN1GhHrGKROwzNa`8;%#TA~{+Lof~JOcY2##k(5;sTBz(w--#~S+Sa>!0g`e zm_2cZG@7K@{g1{rX{FdDlGFRwJq4&smbw{rl9l05WV>kL-+P_=wdBwvK^9gspEzN@ z=lS=ICQa}2^n*mZ-L&78mrFwMVVV<@-=DV-)R#a5qPm;D`a*Er7fQ4yk&4H>e@J>lBHxRRzCs>1=z+TmW)ubmP{8=nI%~Elg?fI~gaY$XXkoeq;T{C~u=)Oo zo=+$T8TFcDeH^XsYefH4X{<&d=PXAE5|C%Jwe7hh!=GawZX}Dt#KaVH!wMw~rG1zw z6D|x<`y9y}I6&6StXiW3;F8~wUp)Uk|9h9-O-g9`f6nVw?K;5XSNuHhn{i!!e? zV*=JLDdqi?$=ol#9PhXip8$_>^es~<f&?JOIeGKmHZ`n8n4nO#xJ+h26z_(4Tl|O^*AZHM`|n3Nwvh(RNR6*)t2DxZ0LLX zpJOhVBEp`GG=M6In|sy^66A?5gvEI*H?O*v^|D6X!+TNr@YpM|HJ*F66*cJ%0jhte zAq-ww=11HQ#sNM*Y#!ZyWbJUHe(6m?95yZq`b_Zoqn>nR3^gB~bR@x4A-a7Da1HWc z_Uyh-|MLg7(raT6Es9aDnif6*dWK>*pYg*29vw!`f>XS`$W^G|Vw?9ZU>zWIt1}zT z)Ms?B`rj=Q2kyC(vyx-*MGV`r`Ife8{FkqkL;Y$jUsYjtiZ>Yg9zBBCAwB zm3yxVnT0pq(ccWz`8UZCH*`-xH25u08Cj{q?$m?6v1K9Vr<1e4sFtq+Bigr2KkFtF zB#8>MFqlS83#7aRV1?3FWAIxuIesR_#!k-30e(xE)h-uSRE(8hSJy;KfhM-5Y9 zyhm{NoI=5wW(M>>xx6aRrC7jN4M#H(7qO#WW3+GFIqG6_Y+O>aK6 zT$Up3D61BQzZmF;#k9|#C@Qyl?q5u)4lMSqCYqg8n{mXWF)|>~xvoR_(X+ z^Yqxv9`OO!U*>!(+Rx;inFsC>Ky1R$Lyl7k^L@^$ls)5t;Jc$` zjeZ<(H(MiFkk_fI@B!AGI{Yv}e-SL`h$nyim37KDA zZAH+IS(pKEH#fVN&%wql$X(gxUKd1fcQeONI(G#YN-91fXBpYnwF>{r@PN^imI-g{ z+Kh9kmB6N{s5`D{_ikNy0lFDaR|xU&8qLQ^L`2;6OQsI4dZTb#s=M;GK5j(hP3Z!I zQ=`JT=7zGYA94PKd`@V{=)x$2zZV9@Gb#+iGJ18rhySgtAVI5+%&T%Nn8P&P^nZ1I zby!r<-{&1t1!+XOyGuGmy1S*Mq+1%429XvJ2c!iNq&uVp=>`!Hq@}xg&*<;ncXyxN z`3HDrhPn65x!?N4)lh))p&y3>pK*v$py7Qr9U%xx>K(VMhUCU$MFcyiw^j>)@2Hs@ z%|0aoHHulZO7K#Hh9L4US=S|H#g>>wv`@#+2Fg@+&~f&=Jm9v@xw>kNtS1RSw26iw$>j1VWatC?9}4B{dXI2#E1F48 zn1fZ!m+9P%vj)Ks34vg#Y?S_|HAQu5BEsXlWlv8mp^?FML&wzSM}{}`t6%V^2n??7%UtQTUP0*4}XP(> z2;zF8uZ_T2y*PrD%bEzcqv?V*6NC;2xRw`Ok!jpByU51@RjAp-Sx7x+7c?6*QP1J< z%zfGdLQ8fu#iP03a6?1GNRKpx5V^%)2&`t3a~NZwI+2&@rur(I(PhgI_H(wPN2&Pa z>+Lzf(d1;1R(NTx0f;h5BIEl$#&{cavbT=%rppY5^v1G-n13`k)bj z0zta&ovxsGlGUjMu28x4`vd^Fml@dJ4bnbbWYtiMzW3=}HT1!;lmlmg(69h?f}k+8 za*9wV=rjF~^si9_%{IB|?~1Fppco0gm5T| z<`*9p2$W=|#S{_YkdOc*7hi&wAthH<>z>9}u@3UTqajxYbQelx^C+DKb5ZY*oGtPLDA^GD z)sxq5#saCQD|Ol9A50LcUz_;+El&(!J^PMD^hyA&dNUh9FJYHXh}LoT~tP=85@G$<^hI7J_w#lmb-*HwbP%h%`o4s+Ch+d z3s1y>Vi|hIpnVn2bHDy4dp~8A1@lqWADK+rsWHPK8gC71NHWflE;wKX`rh3~jy6_I z{(Q^yWP!v>UTsI2Aqn|ImACqsZ8VijPJ6#dkw%F$)aRVKdaUBJbth)d++ob_BdYY^ zpuDOg9Up%?b-rC{u)C@O3KY-Lr=^$|8_PIk|47;^t-vqOHHTaHWw?=?oZQo=bLz&U zmk@%JX!qj*f)w**pMgJ9cr!}!YREJ0#F+_FKl7u!DmeQ6bix1WV54UtIn>fpO4CTa z^Wyg9W1sTSE#10>*4<5ORV?~OLUH*bZKs(ERA}CBw;2`Vfgp!+|2MMfmrd)3bD)Yc zR6&*0{G#*f=8HU*=n61a~>Qqy98dHc6h7 zv9!jARdDWBj(oztJ#`!0WaKbaYNWeU`a$6*4?I1)TG>KRTO)E&JFaQ9W@3uRov0)_ z)zQJ45K!l91}7iC&7aTb-$sIp!-Z6Gva-_ z=11_IsZ~xkX^uePa^oiMw`^QI=JyW8+!6Ww(bCdV0L$dj4H+>-(uw0|ehCu>cObeN zUi96$#ByX!Azn73B)@ZvTOV4d9lklbL{k8`VkkNEb0blt69)l?P%vFsv9$b_PUBjS zcEj@t&6i8Q>C7=L?(1q5sOs8S>Mh(M)3k=uVxq-JPJc1^kE5o;KHMt5lE}AG?&(VN}E3#ph4IyI&&~1$29g`J-S=+yvlaAc21{ zGq6uZ>uZ}HFmUk8HCbq@lRvDBk(B>Mj}w;E6S`NlG}billjS4x@IFb4x_|i<-p$u| zp>SA7H;~UeIAv10Vemvdx7?RTtVtwc<$QJLEc8tk7)@u4HuOclm(jdr? zT?2V7b#YK&>dmhn6pJT5Hx)jGi%kvv4f@`63C&2wje3s&XS$Dt1|7|v@KYmmfvjCV zF|i7S(;I_@W=CMd4@r)6zjlKlZf9JOQQR6$VnTjc3ZxUb%cK`Y8lq{|_y}nTpgQ78 z-HD|FH~`S3o9eT@{q2!PixbZ1V1ZV)6uO|1y4#1dpJP47RA~@pj}iSTgAc3wJ?(NB zjS#bLy1^)(pjc`b?<>^KZuDq3_O_5W-PrVrTGx8tE#FmCC#UGyQ$nJ{x~(Z3&bs!MHkS7sX+w;wSqj?Iu&8L~I zt;kNaV0Ql$XIqbGGEl{3Be z7;(%Q)un30k9!WKy9GuZvg3YN>9JOb;(g0Z$Ys7HD`m&f0QYz|XV)tQnNG)OVm(yF@4iCE=v( ze7Efe?G>-BhG}QS^v;z8y$I7Ke;HIU!^QPCEdX@s;e*Yg6k(!(! z__UF_{`6t4`pmR}zP|cr*HUp7-uEd~Zv$M~X|!?g#>x;&?feYhnW3yE!PKVWe9(@| z!SBeU=US3lk#{I=tGkU;397MIMA23&se_+R@zN~_3$?w5u85I*KGTSazrPFhyUcGk z7YDIS4sJYU@ea#5&$6n}dGErY(Mr{Prg@B|+FD7H=AP5=3^@VF+LrNn^_zCe3HxQ- z(c#|h+W5Q8n@U|9O{P3BD!`I0tk5VQl%4ykfL>ZXMb^W2-9;5juiso-@U&W>MQ>4l zl11x^+EF8ULyZEji&w_QyU7+2W%SGcJ|-^#p%ZwZNq(r?IatQBZ>A-8D3Wv)WBs|e zrDzNmXP99CaB7ppd=MNiJ2>Du)dEL{~? zFnWETP8ZcHGy2%?qAMlH`WO=m$5J^Ywla|MId8!n5f6d$8Q$>;Gjpxha5&WvW<3mM zISlT~+-sKoZ4$F!;7l6pd>Tl?uUuEDSvOXpA635V$90|hswCtUn_dQ4Lg0Lw(^|RN z{#uPG76=5cG7^4U+bAC*%BFoR!d5UU3^0Rxad&@3z99Q@oJO2-sM@#>HxZ#7Bi(Lc z2@RK%mX>yOWQXry6sc`)ZaQpR1L=@~v`hAQK4vP=JT&U54z#G*HIDNtE{YyIIJ@{I^O}!wad$j?v00!+n})(V&Qig$;Eb7$3>e5m7RAmt-VFZ zFfNF}*t9~|v)cpRx}ufR?*;|>Ft!mx#K};p(h|Hq8}Gz9UR5-pX7hECph@a{b<(G)=|4SW#ZeimCk_lBhgh( zpXbgtt|(;loFPTxhm6tyBTY<0j|>MGPAH=+3Gh@s(YI(*j=n>NT|gE7lnPu9$gXF5 zi@>|`4XMBmuu90{LDl*&8#XoR^-%cT#KiX`MVT<;IV1>d*TG^BgQiG)A`0biezb_8 zCMTE5MuWzC!6cQ0?=I~OV3kgV-B*=mWznFm+1j`P>!dldxK*%9Z&rUCZVx`dij2m1 zAxQfKE-}d0#pj(Zc+GA?6N672&V3ggNY6JgazvKSFEF*Vd#wTv&8-l)+W;^<|I;S` z5YPLbucY=my(v5d_%N_Rb8>R>@;ae@LOb}75K@P*bZy(7XZ&k1x-iO`AMECI|a#o>Lq zT?@YE__VaNtSmJzqo&}Y09pek`L%N7i@S`Wr?qybT=sJBJ~TDhKDEGwxH^QCe53?a z{+t2Sv@GxtzZ`zE#Mhr4h9eb85Wwre&(H5T{l)z2D^%dK#KElMMoWX9He$PV8Sl#) z#t!lxos3`9a4g2T34-J>!5zYV5k{Z%Q;!v_r89j}uQ*Uk*SoK^oX-2vJ<2UDU&i^Gx$&I3@xXR$Nz|D8 zR8vy*2wL1e;y`Jw%k%}2-b(sp%F%#x)l=)?;W0ijG1HWHdjB8k)dpRoVMnf`gVhm@ z@&*|!i(95t_H;{tNSM?)u&t!<+bVxNsq5KLxv6Y(mGiP}VUf_EitF)*O5JVktIFm- z7v~kVf-bT(Z2qHlLR)$@4_G;0U^Y!geNDJ)F@w&?QT<}hNjHUD+4NC)y={`M!Sa@C zu1#gq)B_5wNA~x3{0PC`!XcqCw2}csC;4dtzk`Vo!8YaYv?HRz<~=ejxa6Je1l4>^ zu(B}d=BQ9Y=-uAwXao&MI@S#C$^icD#Af)5*o0$f>04|KeUR08 zUHHZq@Q{5&&XPKadH@JGH%ig7W1eA|VY=Q{zv+0=h={}Z4B!yW&%R+* z%r05^KN1~@RmA=s^@0g3akaqPUi}~K<5up44Ugu*F31>q+m~;TFXWCa8;5bnhOuBZ zm0wG!jDpF&$*U2Ai24a))eq?XZ)BXZSaqKEK5(TM+J5X53Xch&1Xws?Cw`+0^530r zwyf>CeEoXs%?rNR!s3;P=wvV@Cxk#%J4GoRMsOI%nIMp&g&FcPP@usIjFXmyXdqme^-e`0a171ehPZ&_y z!PUgim*FqwVZjnN`*AN{z681b@?UOe1c1l&F-cV{V_)RQ&Vf zp5{=aANP^jWU76M);UGHch z3}pa2!0!f0yT?cAEgRs5JKNea&DdL1_y#iws^hbyA_8cTR;iF)I)k5;i@`xTQ{Dbd9@iJNk# zIbn#2g~b=pLP5~fJBZv1%{Y!$>GS&HlW(ndYE;%9%^ zTrT@qyQlhF2PpflDI7Bdj{)qd@J!%IvrIzxHeac-VeZgt7yxyvxc+Z2HWEw~UQx+i zMq!NH$v45h4 zLR^u9)tk7(DS4rx4%nm4Z^|1N94*@QxpB zr0+m;(`+?jPkB%6C`xfpcbBmBZBNX@wzP;Ij}_Qy?XV1xb)ve)`Y!h*q&<9{cJZpJ zs%qhS=LKMY{cQHDZ5HvH-fj7o9(c+J0AsWeY91aLx$)cT`JUz-CpQr9Xow6=LjSMK|z?iB!IoL<2@StYYI-(0X?OzT`05sYNek%Z^m6pz7yN*!=1 zFHe@p#gj<1;1V2EieGBh$)rEe8%=tD{&SI!PnmZY4KHJl0C*AIheScV@s8c(El|lg z=_YlC+JG(JcFNblQ?b=R1X&WWt#s|}S)tCZE^~)=n|`oEsi|R*`0bTD2!@r`Jl%P% zg4e5UEsAaBrPkOIES;W+PU&l*Zzf$;ns^R>7z>!(XcB>m)K}T zU#?}xftzqa!{e5U*SY|CURnbb%A_A03*3LlO)RA(J0gGtuRe3aj?3_+L4W)0w|s;3 zK@ot{|HZJi6g{v`@s$imZL8wP6^k^>TG`f4#gf?pdjB8-8&|JxQ6`Q6dD}QGvt)Qm z9(EwuiJX&|fpz+9?G2QlpD!)k4=|^Q&``_~jis#pg)IKQ+6dKPHI*HJ1q>ME7u;%+ zbqv=c7y}2EdOwsY!v%?9)FKuZ^ldti^7_S1m<`IlXnYV6cx;LQ#TcN3Vo3+ZCuTOS z&*_5H7-Vc_ZBmT!>#cF%7~FHs)9RAHqdV@sQl@v@LfZ$-Ok}~s*>^rk|M_%w*EljNMgRCi*OTdblixtI=iG8`t*19-U)FpAtj7}P z$V`6$nja`I?}!E>^?vpoD}b zmBCb3nfwB!)qgUhdj+ZxTJ@$@_|eV^9l(s7+7CC zaFxYQKUCwN`?S8k{_)eNj(Una$7w(o2V+_5`a!l#S!rO3%*Qdnm-ASWP4w4p^6ps-FF3uLI5R za?CFd0NB#^kW-8mK#vK=Z50mv*3E(L3oO23eaF`Y1Zb-#X@O%Y(NyL>Ll)Q%J|)o< z)@37(YM4=|b-{ei@Z?uL61Q|omeU{VWgk#LY0;3czC=cMN#_+HvEq72A?`2l#f#Xb zp-r=Z;j8xpbE^8Z$LY=iF_6}096pZ61{q4fWFDy@@$^vA!jB34fNyOm%e2z^KI8xt z3xERG;XVvvYuQSWmS=;zH7KiOz1%)=qsk0;Z2CiHQfmX)$wa^^62l4O8IcRn+Q! zmDc*%sT?>^|4#2sruRG{KQZObE-TWM`=Tg%lpKE6?YDu;c7!v>W~ zCtytfCF-&MJr!EediOdZfj4AXs75fS_7Sn^OM1EcCjnCv#0#Vw{!?SXRlT14q(W$a z5g%|s4hm$j0RYE^j?r%>`4hmz{rZX*|FN-fAm}d_mV4$if}ADV7j$!&1@bk>_(Ue& zFuAq+%$RC_53~NO=Uj857CG<3TvL6XsI3}AYnlOgsUXFf15-wXUl71a;yU(F5a$$6 zdS&bFoj>r49oZJlB!xocBZPd9){)V%&MuQ`yksXf28(&&`!W9h*I04-?-~ISxTU3~ zx;kYm-1{x4b&-I>0CgjE51fRA1Zw*_?({eM`^n4qP;COLD)1CgT;l%RrRpgj25{no zA;V-5gm#>%x9f+vjv)5XlPu}?5ww76R}iaR>>k9cFkcj6%^?E_YkY zK7ORyF=+v<&q=g->t!17ahL-4B%YXnwSkz6u7MBQHG-4*-tRmlmA)xP8rH_#k-x#v zeRJXyHkqKhkx`>XSM=>g*==-G6tP2Wq&9HLt*JQyoNNUJ1)7A&X`lQ7NryB>TF}=4 zTWPRq@oY-;XmxkmP^`f>i?#|f4#<6=5^~VK{snb^6$&miw!ERdshy?yTAnpmk0tE8`Y`z zW>KM{q9RZrgKe~oI$8SE4_MI-4-bQ_6F>JB7o`KAf}M}=2pH%}W$zshNjTft0a~=J zdI9LE05iXJuN?!QQPsovzbo|cp|!(ve-OXBh)m6#sGJ2}?ED1R03$tp!W<4_;DUj( z!3C4#NL5Sc3ENtIP4zJIp;K|$&itmoq(kMJTH4k^5E0wchVht!651DE1LZO$>G0Cm zkfve@ndk&9o(8q$*li|tBdTqWajRzvxbQ4PgveI`CnYQJ0{AHKR+xEP)D?03 ztiKpg>+}N6_29pYwif7m6}|BN2T~qA0pnRIo7e&GsK;JL{+)kBU&GJYCMH-2XB;Iz za=Hum-zR<4bg}u#QlRh~usf<12J3Yj>1---p;0uhk-PVq5=0U(2ahg%01Hmo(xtK) zh;hqyfy7)^EgO~6sMUDGPNwtj($X*g0G_sa9ajj$QeY3CFh|B(jPc^rphq|7!RdCF zh$zhw9`N5MC5>S%r$sD~pBS$ETo(VQsX;*9e=qdxSs+r}H+N&BR#cn=vZ=SAs(dTJ2+E$N}xLWXvSab}veG7!wn zcz^8s-TS-l>I+=-uTTc?rT?3&uMortixcvBz-i*ZchTc=@%}woMFtcV74??(3Fz5{ zo##a%(3`9S@$+o0KRAD&d8-B)|1Q2i`@LT>S-byTuaj_5StgM*F{OcgTYd+s4N`x83= z_5kiRkfR444hTf7d9o$s+7tKZfe+m^m1DpBeit(h3G9qm<4n#LuoTx z-&?xEDRI5lNnO5R$_0^DRFmA`VXk ziS1Dgh@px4JR%~EdJ29C1=&m+{(4wy3K?C&ef-Jv#53`+ znA)Zx%#O9jXVTNz&RK@xw4xS~@?a5KTU!UXfS`mrG#r29h2Bqu-91Qo8fEYRQxdpp z?YYX-d1Am?KB6+wA2m5WZnzfgr@yRt@pADb@yW+5|9{W3>(z@}i9RMx9qUu1M9|tn3NotFrBY_E{tHaW^Opbs diff --git a/doc/design/img/hermes-buffering.graphml b/doc/design/img/hermes-buffering.graphml deleted file mode 100644 index 9e9ebe071..000000000 --- a/doc/design/img/hermes-buffering.graphml +++ /dev/null @@ -1,906 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Buffer Pool - - - - - - - - - - - RAM - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - #0 - - - - - - - - - - - #1 - - - - - - - - - - - #2 - - - - - - - - - - - #3 - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - FL - - - - - - - - - - - B - - - - - - - - - - - PFS - - - - - - - - - - vBucket - - - - - - - - - - Layer j - - - - - - - - - - - B - - - - - - - - - - - K - - - - - - - - - - - K - - - - - - - - - - - K - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - K - - - - - - - - - - - Block - - - - - - - - - - - B - - - - - - - - - - - Buffer - - - - - - - - - - - FL - - - - - - - - - - - Free List - - - - - - - - - - - # i - - - - - - - - - - - Hash Queue - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vBucket Pool - - - - - - - - - - - B - - - - - - - - - - - B - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Bucket - - - - - - - - - - - B - - - - - - - - - - - - NVMe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1 .. n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc/design/img/hermes-buffering.png b/doc/design/img/hermes-buffering.png deleted file mode 100644 index e172544d339fdd1e03f0b70776abb32d2e304616..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 72804 zcmc$`cRbd8`#)YnWM^h%6ImfE5wgh$m7PsSM)oG8;xw{lQ%PoIW$ToTP)Ig$maNF; zd%RV5*L8pH-`~Ic5BGIl56<`Ne7%m>aXgRb^Lf1QX3mz8ym$G3CrXxoJk0@W3(eX5181?X_Xd5}Qtjy)Ph$2}H5ci=Kx5TdG-=AKnwR#Ysf3n7(7uLz9F;$bjL4F`Nz^9hCc z2nh)#-4|%+B2hQhxt9zt`quxYJl&$BQu3i+4#iNu^zQ%rkx|dun3harx!)YsJQHTG6L3y12RyCtgzGv{EyAH)It`>T(J>xioHR>4UAg zi~{m!SJEX$>wT`BK)-!^f`p{tn6dwshmSaaUAE);bEE|JiUKvKH-?4t)P}C3K<)-6qCv~eFM+>hNm&s(`r{>oy zHcltrUnuVxVhT;`{`c?4`TpnY~-q(BSgmHmX^RVO1PrA_&-9Pi$yts7Q#8?n&S^ zKputu_oG_Af3EKD>51-6sdN}=#~m(l9F^eb=eHfz&Qs~==qGg{Ey&~R=;&BrF}`vO zp2e(+Y#9AnDJdxd0nN9GaQg5~x^RbIT6CP;=4(}X#68?VK*s#W+3joWqem~z+9LUk zY8o%x<(;0L7hw%0Ej!$iW9iZXuilW%k0N;e2 z5$L*_2piq~_+qW|bbDVl13mq2z`;g9I5C5noR{qw9K7u6O(a$#*4EZO78$6rDhasE zD2XWD+uhyW-`nZuAA%50OQRPdW>)Ouz7uqHW6&hv`*3-|+K0;;8a<1nwZ-}`xi4Hu zeW<5ch@>WnA;fa7WB zP^o!mB0qn6dsbGKkac&R@0K840h#(Ih?ocxm!c{9`l=CwmZyo4m&@;C-NDkJbv6wbxS?&@QZ{XUqqGB zZ;dP+PsSK~n2av>Q?iityie)p(b0yT#S8G)$KkK>2?*jIJ1qCgkr15{w15w?N4X2z zA|WD$gotpt=Ze|((HoT9I)QZ&M1AO{kmJX&uySiu;?I;iPbE*3At*vR&0gs7v)&H@pbPy7czFa?+r}C)pHpi{P{mp3-Cnu+;?ct}rZ4UPL zU^i0wMDBESc8*loUWNbQ=jYd9yRf(jzZiYXCoZcJ*_T{AWMhdI#_C7L$4%b)!v7#2 zHsgw4vbX0U)X4Mhe00`hd(9RW_Fe4E9c6WOh|&F=XesXWcSnyNwI}62{t|K*iXQoP z1KV>M0l~q+xi_=J!{5Apbat32d}}67BlnpMY&nBymWXWB6{Cq!VaLy}PEc}R;rixgyM}76#3v;w1sq`euC!6xqe9e^1cDp`zqm59uz9`V zdaCN^xR`SO)=*wuTE7)16J0Q=ti;s3>N!ZaD@b^^kVro-qw8yJ3_6Cfvf0e#54HJ~ z-3rlRq2b|*tf8p>yLa!dtmh#M7t1RssB#!dxZ)BqZm_Vhl$DhYi51lP?|JWSuVcd0Go;etZ*kyn zA4$OL*my|Ft@m?r z^xM*sMwnmY$jDBQ^npbPq1FEGmUf8DOS=!TadE>;q45*Q7r{?XO;zO&mF`%5_fE~* zOF;pb$IZi|fSNn|U!MwRla-w==FB1{mh=Fp1MSonNokK_j_KCwd!m&SA5Q^yU>Mn1 zX=!QCnA}`8@0(D>#Z{vjE-3IvM@NU>m64HIla&{?Ehl^RE{rMkT|_NGvl{`k4mJ&i zUZLK1nP!(B>kPkk+*3m(H(Tc0J?49Zw$_d?Z^4IFgb~p--*0Vi?<1JhaEVlqmw(?0 ziTMOUOG^vpZlq1073=L}A{o47G7sud$VKSKsiIC{VPR5Ax{Jw@UV?NM_Hh0^l0OLR z@q46pp1;AK#(_(_b*z1DpQlv=3oqjmMG{fD6d!DkS74 zuWsL*9MrrNCqft0X^$t1e(WLwnW*?&a`V{5HUcs#s&?P^s*uG05y!~QxpBOu_Vh*X zUh~(FFZw(?^#gO3o&D}sQdG{H%Mm*+okY(o~@_&&06pZ_(wyW0wjCvo|GN!PZ^zcx9tYRvoo71b1B+t`$px46l@ zE7Na6AisTIU0%Lk?|mMk)B&QDA-S|m?_bMkO%bw=ibn;~p0_is^Nb-9EP`?sH&TIT zVlw{fl^8@49UQuZpm}N$Jn`I1$eYGrrSUyHHZn6aBO)U15>VT@v}}DJsd^K|U{I)6 zbdp`A@mnIl$=JS&V)^HpSawwr`=Q&d5oB+o8Le{|rdQrDhI`L-r*~2@asEzDl&f&F zTV7tRc@ls2+LaW^F3a7YBTkPWKkgGr;JW%{{(TNy)sMGL9Pe1Pix|ub(wW&MR3GQ% z;Zd8tMkE8z#SF&xaT|0pbaivcO&zCJcgH#2=+=q%J1!1}UpcdG6*I3ut@}RSkU_-# zZdGDtY00G6xSm-qj9%<^;KRxi?{&K}y@D(0LuJQvxtSDeyw|x6c=TWURu2S1(kW}B zdcD7W%R4)mTld*viIYWRZOrN^TG=0F+I$6x833)?`mr%FWQ>d&r%8`c#KA0t=X$=uNfMH1$RBA&Spt}Q*Kd>EC7?0j$u3M1bvh0It3 z=NM$$pRZgPh)YBZI3t8e_K5%_X;xm|@AVsFK5Oq4Z?@ttT?n`}bjQ>7S41YISk|z6 zeSKPxd4`QGlvi^KGAk!T*}K^*rnllmoyOGo7#aOHnXl%Gt5rP+{FNu%dOG4bI}>$XHH>{|gpO90 zEY9XI+>pvwnMeBAl!yUUpcBh1@(oM}o@n)PgorqN65GniN4p^zOzg+E!D5XJAHCAT zmA+rq!OX7Gg=861kP&uX+l`cGh%GjE(Jk`8r5nCslRb7#+fTb^uTBfx=7y1gV%iEDls!w4LsG(lx3 z3F(q*Nbgnj8DMz$>szB3E}(CHO5rt!yROMS+IG(Mi<->tNov<6#iNfLo0ewi_!Xn* zV^j(>GKf`;eZHUHOy?iP5?C&}QT}DGo+&oq0z0=JQqw0UOg+hx!3;CG;?#i3BuU=kW7AJP#LsSSTai~peED3)>OG*>u;QWRI&$7D?$F;1E`&089 z8&=w%+Ny$%ld`|B??kaP3ELe_Hq8uHqgS_s41M%o7~T-8)T}vw=c~@2yo#Kh z+}-1YE{kpiY;0_(0Ns;|i;K{gS%+wrzJYQd6JUDb`_`wwVYo;}N=iyed1tPhL1OGZ z=410+f}37|FOz03LyFCG>xgBa(TH^&jsh1KmpzIwVUSrFGNneE zSknt8{aSb4ds)&2b|aO%JeaZi3zVKD>5fga$Nh`-ifCu z%qZnWU8zE=KUfZ1C9S^y`SQ$q(}Pn&N&?DN@%|2dE^$RH8mG&*36+PKXQ-wShKWau z!Y~k%aPGE;40IYg^fHOOM&a1bj*T-j79vWg4KP#|QgC6-a~PR^9Q{~$E!r3_@lw;L z@&_0OH4-N^@cr4k0+tF?qKGE#8xjYGc z757^)X(FzhZ)fO(r|mDid+z*BwPKFK!_10+gZ;jfE}emvH8^`B z;yt?g1~cZ_{(IZ!I*VLvYVzifot1CGfmC+PgF5$yQxvJ-U7!dc6Qml>95KW@*#5>Z zqJ&>K$gC7v+!MfGh0FM+E6&UaDn zuaED1`xx`X(6UC}@#B;0r7V8Ob9jl>tI&5__6JLI1Bn0bGt&5?Rv5%?Ph^FkX4!ki zBxU)*g^DZB-YEP^h7^%98HFEo!)jz(Q3-wl2YXc%J-G&%u5;bNNjUg-mzoE;N3oph zR&YljQ}OCwPyca@QWRoCKVR~n!B8aCa4F3p+LuSJNB;dY-LuyPt=`daFE?y$V1K~q zX=|U{V$4n>Z^^=Z1T^BHF0$E5uF<#($VchFk_~l&(30QAsE0ku#r=icOcu+B!fOQw zm@}i7%*G+B=jrnF^C~@2N&+eXI>o$M+I#~z{deW(Cr{dJDKl_8{u2S8@05}j6{U7L zZTpN6&Emc^#``EX+)PFR`)M=aMD43rr>CZ%_~yOPwXD%yHf`6Jsk`S4^h%Y}c=y}4 z5z+TC1P(+L%}iKW2IyAg;~%7qi#6I@Q1E6O(g3kC-hQ&P6ktkJzjTcP;mLrlg$ zi{;piY(Kv`w?)nui9i(gi#|Vj=r=QIWnVp*(EV(Hs7LKv@ zQF>w0u>Z9Xa;W}?&d#%9Vy|EuFSsxC+F8kV0B;5DseYBCk0S^sMJjoD{4Gs&~b)_kn{aUaA|~XNa=Ee z_rCRh=}J}3v(WbZqG2#mMs&_^dyP5ZYm#FS(3Z{Xslut7^EuH&Ogi?^w48z>2<&hE zToJP=K2Nx|I}T-qUHKw>4^mmH)BRzHURi!gzjF;k}0!1PB5z-1m5aTGX*s;T~d;`SpTOgumkBFI6PXnTbbY%4DL#D^l*iY*!#TP+Pr|#do7et*b zXnDebDFx`4cmN2G{@1D^067gl)4hM?h*ftQ$VlHUEG#O3X&jp@e!b027ivhnF`MAN zGSvpG1Tme+171=h1_=rk=CJ?uVZp%oorfxQdOMmS-8ADB*smM|jBD02PFWzaAG z(arz;LTdbH&z=<&j6J(aA4gDsA4gh~QO|E_N10f%;SU zuNzc7J3G6+IBGz<7y=9YGBZ=+jJ`QFJ>414#riz}IREpvK9n~ZS$o^kB1b!X90L4;`vW4`E203AF`0qA zy}dxamC2yrL1L)1u(Z7Mb>Lo_0I&@f_g`jurLIr?p<0pe@Xi{*9mqm`Nn<@dPSE4n zllCd==)}9(G@3_WOo)kT!e#ECdi`1dQCytdYyVRxPimK0D8t4AW+;_;W;Hz5Q%$X`Iox2(Ukp ze;^mEFE0n?e(TMWvwtE%TY1CeKg=>Pa%i>vy$utZWaVj3z`{-qdkI9c-ktB>5rUvW zOim-=c7`tW-Mt-`Tln24JfA{mgSle5diNcuRnSOe-G0vfe?Cz7g>gMVa|dtI1CX&` z!)Tz4(}_B&F+S(Inh#MEie%P7V;^-=>sR8Q%S`pY&7t@ubpY~#0LV@(%+gK0Gxx8} zWai~9^7`>*ZQ1W#(m8ul!PuFZnXexT`@F zjGNxPvHtO;%RUHiY-z?y=f87o%K8`3e?V&i24gOl`NI6?YFBr65xQ_f+oiGk{r&yE z(h!-SkkE#iEKbAOq(F`W<+z(KFE=+=*rqqo=vVt|R6UrfXmZ9j4Rl9~0xkLg@ z__!hqcECRQ37n_q_S>_?jZ4RH2?NLP(+hT|ijp+f)z{zjRZdWP>FMnau>9riO+QFM z4GkBtZ({oL>`R-T8`V;U4Qhu#miC)LlZoIZV?zq6beUCgkGTt2vh@BR^!<0I!hVkV zEGf6RX*T%p0drI?)6K#zJ5i9~Z+4%czyPiTM+{&w^XW5iZGw}a|4>q$*JnDFV|LJ= zqe#NlzKNl_UguewnwmQB1tP{)EJ>le^)Fuf%Qg$XK)Qe0%#c5PpbLuxnccvXnAyZP zfR$7VbqjF=qG-=Y;>V?lIS1F#`k7Nc&6B-*V!7O5q*72sq z3@%JgGC%Bwlt<3U*n=xhtsDx%zOSF(WFk;_LHL>dMZw4Wiy!eCh;VWi+@A1*%Tvi@ zrMGr9P?`5%oY#SYjBvodFCLB9Z6>eJD>A3nZb@)qqoWHbYi^|H=cbk@xSjauFSESQ;t~A##z3#k;ui*%iQl7?Z!0T!AtU z6-G~17R&Yk@}NpMnLy@qP-f$yGf6P!`+y++O`Nb?H!hp;*B5!lD{SL1eY`_6&`-tq z$x5ydT&;A`vQUFKZ8vbLm+uaWq&)lc`^YuY4E-_C63aA4-bnr*=Dh_Gk07Yj6^w$# z%v%w0j_IP(z229%22Xp>GjHytI=#+1mIPg`!w6Nv+Sd>FBWpCaU8wE(>kXe-{)c+^^KUtPFuO2Z{)k&QwtY%~V`uwY@i0@m(mi0h=x}Zg zJdx2*c1m{Syf< zRsK)u2|^4Yj-xdLVupbW!9;x|AUJPdmI_V>P7UtHEUHp0(+lRnrzM#;{}$qz{p*zpp2<22%r zAnabV`|#|}0nL4$jbnlwu*`>bU6`KY2@MYqkJSrlRm;{&N)bBGV*eXWT2O%M&TCMa zt1!`wFpf+{@9&XD+`C8Cx^=R=tPFj!(~SUSEnIbLFv|E9NQE+W8YfSZczelA>LXoj z!?i-aho2oiHkWU1l7~HG>k}LezQ#K4WdN#R1xoBcJQ$(iu?zP4TdT8-^k04C96FyW zqK+b%(n7ln%sHy6Xz@$_;Oq#Oo^*SpqKm*d(@>&&4<9}hshfTiUH;wu1+1|yhtW4) z(go$=S<57sYq0&AH%6}{r;uKM5d>{4 zuY~KD2NfuL`;EoXr;R&F`q1t0N^a7d0xh__Q@iw9HIk>l2$Y)USgWoSWmOx8PZdFZ z?5YVRI1=KwCys1IL*h2tp|GF4}La9FUe zd1Mm?D~C2JGWubw0$|nP5Qc!9{qjZc`SlnnIiTj1!g#OWdy{wZ&fziKOa7Dvd@_WSJ?bbZF{e(Qn(=C73olJ^ zq?Fv$c?PZLOb?0lF6lnFt12XPnXV#HD|zu^NJt1|dc019mv-#(`uPSHvJN{yXXSdb zf?3T@d}ACikv416Tc5`8Mnge5*{i`#I}JK28}^+sb#5UAB03QF*lHPO(ufsJI~r z6*19Y0FoU9zdgw;y;ywku`UipTMwNE1dT66&T2 z)T$W)VGADbz&9m~EjZ9ZFb7^=R&6xQqZ-=TKC zyQ0vzK#y9L$_CZfJGpDms42-mbJFA+C>^tlGCk)83Xb59q?u0s2JlgITw#d^gqkUH ztW3PJuLy{}xIn-TWtJ4b9^Lwv{sHX`G;OH%3YRadWk~I{-Xphv()8QJxY0uxlV$Zn zu^Y39&a(c`T4GTsrU=)s2(3CABNi_+7>-sR^=yx>Vp#5;A=Isxd6N$2+T|x>BsVvp& z%_)NIcTb@gaB&f&3r$msTa1zgoXW5m1pZS48IA)z@;%1qldm9fq}!^YJQkoN37l@9W8v9DjK~DJhF4sPqiY87x$k$Ib*d2oeEFH zxMPjW?1`(b6E<0T6bxJ!Fdv~yQyC)|9v-$&J|Fp2UWsdw{NJ}MvvUWeYK-+I-RIX= z#sdzJHL2D-4Z|ZKfd1bi1a;M>Z}bCssVbK|?wLc&l;*F@8f-EosDc=LMT?PI#d6CC z1YVx4^+6MYhOv6CM5xnQHW2=5WO7yn3^MOyxCKYS`({pHuc^Z?tKytt3hX8kMEQ&F zRpb7wgYwzH+k}%aT7{5utAM``EMdIqN)teBFm{(`69~C`Ahr;N5fmD&WTOH8M1xC{ zs#W_S4VSfkZI}aP=9qkm=nxixi~c-bWK?{*TOkl;!JC({D;sg;*I9BYHt>Jv{`qFu ze;~UR`H~xg_;$s!KImw?x=6x8p>$RPD;z6fsF9=l*bLTjqAs{j@qIgSiK3*S_ zxDKQSko$$>(?+1JdZikqYu!_G60;u?sB8?*rm|CdA5*4QB`72W&M;h~G^TC>mVun=E}W?RlVW~>1(`&*$>zH#6zO&f^V~TuJW3GCOQ9iHdz!a;PfdyI_jcocCKb$T z191iR6_RlX8thA}bI=e@^nm%d{`P<$o9K_VH4B2vtI!8G^VWfE21#g=lRRh3MPBjk zl^V+b$iwaobV~J*gx;g?I1HEFy|~<+J>Qyt>dferj}Iz(VC$p2KC>dT{?^u9?1V}` z=n!AMnJ-_yTvAKOY|N?{mFuJp4Av%C(ud1Q#R>oTgm6EjKu(s4liHj(R@o>EmLNZ0 zU$9o9Gk}W@rgRux&-~C8igVXMX_2=gjn??bEX2laS19=9?%lgrT3R||Uhz5ESdy_O|2r>-3A%G3I|bPs>K4(jOSA z>KKju%uFV4qoVw`X9Ri*4D<@#2Kvnd-HCoJ7dmkNAlO5<=D^kD{XKOd&6>82pN$$- z4=!zp2Fd`5<@Wv4`WWTv(IKJR-sL+4%pbNlmX?5aNVm0o8-Fi4+PX(+<_G}UL#Np( z2f2j+ArxC+7=kYVsAVB@4{NZ6Sym9m zNH2{;QDk-jk!Pj{d>u_0a$$sA^twth(c-VcqVQ zwbZrh(Qa;xKV1K2nVVnUC1d45#d6tSia>cI1iatkOTce1F+n8z>D=!XJRt(8xpK0J z-~Pr;Xfgr~3lY0~Y)Og05DJjIme%uufdOcvWv++d5=~7_!9{1@nEz0i$aDo&4x))W z4#BFy6LE-r!O^!}UFs?-vH5QU>#M*QWk-}s0mq`aaZdVgD#A+OGg2Av*M+SF*Lw-L z{1uFije$1I>nCwr8XN5x??0m_j9Gj<*ZiBRCVa_^BE*|gpSBOeaoIr*ry~O#LqIcw zzQapziHHeUSXlEORZL?F$bBi#`iOSQ`a;3*bk0Eon?Ir&p+JW3AbI$|j%m2as5asA z*Gr9gE8zBmq!@*rQxy?`hhd|L+`fXREe9o2>iTMFF-t1aImtQTDWBxwl&ca3N+~F1 zn1zpzc;6Sq*At32Jiv^L$c^Bn0vYIxL8U#ogAa#khM5}frd)N)1rY1+?+?ZhdMV#c zWHy7+{_5Wq1#A>C0~Dk&9n8VHqu4v}b76RTX^G$HG*F!2pnHeQJmU#OvG+`jV(wKk zwKTC92;1J=TAw~#mxZA`l7#Z#i=4Jsg0T@(Qzl^!=z4jDaQzIP494^o8qd(zL~m7V z2<$H!$bBy27ifH;i1Rfn(+dD&7gG5SU2jbd>2>otHhS zc2^eE>Dvq1Bb}BmHmT_yVsvE?OE$`oC7}oEt@_<@bZ6}(W$a*2FTFnBI=YQ@Y{6S6 za}h5^M_XGP2zO#jKs zTCh6|_0$GHrzy=JB#RdR72><2RbhY{|J-n)8UQ_DXyb_rSPZ!c&ez=VhW+hAxHx^u z_e5XWbt6l+10=U5yBWs=PUxv#kqdFoBB7ELx4>p=A9QnmM7BOE-gup5uGjBfk6-YL z$2jejqhHWb_lw+`9@y%ftgJZzX{RRUUL$HJ*w&>1A{P`map^uA0JU_OskJj}G(97O zvCb1b1@!{w&S{ZmG+gG_Y0YwQ=LoS^0(MfERY0J(3w;iwl$;p961K{3?f z?jAGb_ncmDg&E@ooR|`N4o+&Zimq-86jAc|TU5ewT?4yO`G5M4G`Yfa(xOPiTC@W= zPXGG#>!4cJMjaDl8D(K?^WBgec9UdK5+NOag`1qE3lGjnq!QtJN`DIbzD z=#a$l(T{D^JJ$ONc-0kreRn~z!FdlZfG4*Vo7_6l6ZA}+$5y}_0A?GA%v0^)w!jEg z0+Dh!>4TCb{=OJ>`Xcuz9xV;c^?bEct3Kc?gZ`~uq4NuJ&}1TP?Cfo-*bOC??WUo> z+d=VgJ#&A@!-Ty7#87O(WA77@va+&6ScvPl1S}G~nsy(bdtv(6!M@De(bHosRkid- z=ZL@U=CJ`VPS!=9@w}7M5@POkRuQY|x?+Av%H0)#Z(N2GcJqHdZdkcWYH#i8N-U?? z%_Ot0hp-J4=4lwtf@vk6ou3XO=y$JS$Oec>7urC`y_nSrrO^0b<8KIwcg1LV>OQ|9 z`8c!ufMJP+1nk0bMqdFXVQ2t;%=MI$oeb@CD#GG^YM!8k#|5!!Rt$c)FfHATX zWVhE`ST0`N0}oDc-?;ydy8d+w#2KN;kI<=iLFve$#zG5YYgt)wT&(~i;7J|~fk5_! zXayhvF4C{y9Kp~4ivYDw!hPXPgvq0@&XNjAKE4;g`Z6;!Lqf#E!?Wqn?}814RDTwS z$EYS=I83Y9_#&4A`3#;QqT`suT^yjrXz}M! zTj}?9uwA>{2;3GwpGp9+H01_y0?!SWh^u$r{~-DVt}QiyCNFcSxm6H@uEGNvnFOI4 z_wOo24(y{f(7D>!JYnj7%QPAeSDhra{dK|4=+qmwe12%y(gPWg2%&zhU<317wH^qRTwQ<0@44C!!h4(Ak#D zD4jk}&7%rFg%t3-!0ZH!FcL^-k+`*jhR)o-z$?6pz-8F&7IY;gl44>*tHhTC&3oYan7auWsu@*qz2A?F zJlWc63g24iRhiqZzFcJs=7BQM{sCzzrTttQC&6!r5PAA?*bR|VK}ZN@2h3d{UFp44 zzkyx)Cj8na=yVe}adC0X51$)W3qF;PW41=W!_T~e-j6d!-tQ9f17*=+LqSy1LqO0HJFg%mZvq;FX5(l$6qodD9>e#3crROlt6 zlAyJV;of-$^1*5}?t?%*4nDteOqt@j6ckk8L1m=$ppXNNx^ujd4=n&1F+CaC(PPxF z8j%svWo-c8!D{Q!a@#*rxrA!~e)F0a!1O@wOFmx51xP3N$EocTosWqq?-kdY1Dw88 zEKhtH&KTTEVm7^*kV6nkoM&O+X(2;gZ_F-LQQpTk>Hm*vNl!F3iAnkvX{z3zU1`TRsEGq{S;zW!xk^{^w-{WG9+Vd23y&2t4i)lH)~)h;zJ0}lLM zF4nrtMpl$ScU*ECyc9YU7r`;!Y{`FUgH75+H7JC}l&d76PD}h+a&9 zg(VyQY3Acw791R$4@CyltECVTz{Z`Z#QOOGFpS!py4}@I`!>9e!wR?7vYUf&YGPga zv`aScNAJw{9yPZ3RAFl$bcTaNf%KrZ=dY({p+x9~TwB`W94>_RTcE!gHI+JBn zUq8SF3U5gV&xhVU&6_hs+0>n)`D?ccuozhiA`s@^-x+u*111$UHHy|Z^yPoOQWqAY zziB$>|2iqp8Zh1PoFO72VhJ=+d~v zgR%FLvv^THz*5zNL-BU}*u9Wjw}2GW5s!I9B!neNBnE# z^UXlb&Z)=+Pnzr`rw19(GUS)vK^J5E9)2L+JE~TGKL$xpR8;Z>j~7f^T@RyN*XJ z19d$U?0jQtWTL-^FVIo*+3k0p4E61Oo`CcQxobAj1bcvZZ)zB7Hii zpmD18SV~aoh%09L4x7ICA2%*~ArE`+GcQIcF45TAcKo#;BHmRS^wR9~j0S(>tG-hL z?FLvw5}MJwk-?q;)8HVu`6}blBOKO!@M@@o$W`MNOLN|i;xbs!rT`8IN|Rny0bq|e zG`F^U6cWwd(Kg%_mmoN;^K0HA6gLnTVREMq)+jVbC(gPQzm<8MKmZdfN0!XQNO7hn z1O1T+uUtHDLih8=zlC)agM8FC&Ct%P2VCq(29D5$C zzmW^O+TicUL1@g_-ZbMOF&g~tp@n(^0(R=TQys~Q>wnVntgWoXaB#ot>0y-iuPZ91 zl(~iZfSAj_D5t>v&N()4)fM}pDY%Yj$d#J6%FXF^$t;xE>&KW}zSe_R4+w*K@{W&g zUJ+v>w7Lw$e^c8#fZPJt*^rX3kTs#7;EtdrO)nz50l3I2M<7bW%HuW zH>d!R6UVT4(Eb?zOlRYqnBTK?jDSG0V&vpn|0h`QCA$=n=H|U zpORYO8Unv*N!HgtHkJ!BPHvsYycrb~!u7et4z#gjzVZZAQQLU2C!3iug@7z(@(A|3Oe$X}opiP4qZSH}D zT-v7JTi5073^SeX6R(JevkxEmTgmu3Kh<_0=AxLURn0(zwMP+X9pIJ03mzs}(PH!K zxLP_+hK6ay#l>#V_jY&ZoeEj*W!Kx0{+>)k6Vnee8~i$@-+XF4v}3UbqUBgTUtWKjP&5+pZ9$u%C0a*r(tsFDJZb#j?w8XaE%r!nEsl> z3|?q-W+;>2a1A+}3$(q|MK&N0y@nUobBpDPCVSGdz-XrFoOvR%t&7^w5}|#;;R-K| zNJ>rZd|{fJ?R^i2fT~v)kmPwTu6sn>V`4v_G5i;JMxeJ7wusVshmq&3FrICpC>-?- zMO2ePcQ{Ifh=Q3vngU-!fwvG=UYma}_x=pkaBwtJxypOGR>9G1z}7L!Ewh?B#i4G` zV!;%N3`jqrnI0)j0Gs>-p>_U4%GL#MSTE9){C84{nvMlQ-3U({J<)r zw^#F8eNb1f;VPL$fCq8$!Y=AQ@ZmJh7O!Weob2yXA> zhPK+92fw~@oTDE&6LR+iIcKlw@UsdDl|%6zkRpd^uXbEYRlA?hLKh4SG~%PcfUGSh zJ&g4?(TTl~tDN_D{55HW=qgvr&Du_E-#XdU)RfQ@qVS^0=I|L@e`T2BJQ$G%vh{r} zjxmae+h1o-y4iaMU%@gV9DR zbmg6IdX+MQPi)J*+X}kuE`PiDB>?&?LCqRUDk{BsfwwvP4w%^XXvQ8s7dF98#y4AY zV1wJ@yy*0}q2T~LUBh7M7uzDUSUQ8r{GMw728)nvK%k{KB4?fra-)wYF$jWfPUK0F zR>QR*cyT}-8tVBl78%(7BW@BL+7*AfXPD<=Xlju5VZ?2YMNx&{*BY<#2G?tmLb8XL z1lsM#d`f5O=Q3PjHkpZu>5?5Mp&58pdT^O9zyC>Qfoj7q8;6tEv$Fb9YW{}lDS2x$ z9he4H?~b(i-OOzA$0H_As3f3W23@LDrtKbw&m*;a&S0UeyyOO6rZYugy7;|Ui-%O8 zjIW0x-G>M=*37$qThj3G^6#z!=MQ$&`Y8@SNujm|HMQKfuG!Z%_#?9VO-{MU;H! zx6JQq*iegipob_qJJjxX(-fHA@W-u;66`c?WioAH96ppaZk}XHZM+RjC95Z6UD9zDig~jpxMoN}%0EG%~gi zxi&z;{2pqexhaRfFn~@x$$H3Zq zo4rA0mhN52Lg9qb-ai+T6B62~sE7`RZ|H5SdG}qTmUMG%sK;f&YJ$U#K zv;|@cn3oXbfXs+;)sf`ndKlre4+07Lku8*ywF?McI-~GvO72I! zH%$1Nz$>Mo`&cITwy(TSkWzJnA9wgr=*6|@MEK}Kia5pl!}G$hUq6l470heyie1vX zl)o$Cw=Ha-M{~Kk1ee9cYnHO|`59E!9Za7*sr5|UmWFdVos{pZOSV9#In$A`$Z5fv zG^%9fM%T{&qH8o4%#ushtZm`T2)mA(T}`Xsw}$^4<@#pgKcKeF)uEPuq-KbKu7BfoiFe;nWVxcz)S%e#wem; z{Ew4j6kM4&*x!(@Y)nZ?qQL_tNaP2pZZeR1viXmqA3SjTK!|A z0;Vd5Le*_oSD+k4Uxukie$|};%sz9|EEb=fqeO=9t+USu=R-ldoOy=Nk99n|*y(@g zhbh;e3pZqTsda#sH?U&9aG@4lGNdr>d%aQ($e5p~H@7~*=%JL$rn8i>Hk5+vPtpRk3!0#*IyQmCz8=wcHM4zchj!?y^JU8E7JO3KL+& zp%7$+9>!mJU($Q+3Da1Nf!No_Fii~&0n(QPj)8`PeW?tpX{ocwAEQgb+~OkfwdG%L z+~D30yrNZ4DF4kYb#|~Y3yJ$G7$jMpZf1r8mufi`Kz zB+xswl|vQq8Y5Y$T2}DoSo&iHqH>V@Q5?|-`E{izx4FfU$WS!ro)y|cZo2g-+jAGF<`F>v6TS{r-t zJ`13s0=~Z-1UyVhO|3X)w%X78n#xWd{1SB`v;}9C#+LHyf)qvf9TH56 z!?NM6PM={=9sf8#3rjF>2{V8qFi88pNyWWKm1__z_EdO0v59aBGUn}dmh`(!{^gd3 z&M?{S$;C`vmn-!E33#-!L)UN#@$kf8(u8hCHm&^HOG`xH2E$Lt;Grx~37jv19u(H- zid^5VRhZh4?>4yuE}PZ;*bGO?+b=cGf^)!VN^MFPa z)KEconko8=-toLw%^(g1M1rptKp=wFU|)I)n9{qSA98ISPBn(QwT;RY&Qy3KZxdT> zpuYkAt~hdw9CLg~Y6L-ab5*JI~)mEctqezR{WMbPYGuEln34;(7r{=*7% zB6&gu+|DAU-|lbr@h0=SVaqE%LkmOvNAv#oKA~t~NJ50ZL3GSBgtvr9@ZmGv!e)Jt zzb!f`YMiHWy|C)x<$Un2hat0^GT+S}07l>(8Wsy(YezO|#yc;3hYX@gLsiv0qrb1foK@^Oj)=^0WXR}&z63u@sqN4ySU)29t1{_z|^zn2-1Jgke6R}qs z$i6nvdN1+U+AcoS&sY1J<Uey&rNFD1I2iGWQE)C4QoBg?S zAi&fsB+=%WzklrH+Lt}q!-PK(R3psH%#;()W6yma2C)B2L?m*ojM57LOp5JQ$IhYX zpYo;k7T$*;$?E~dv=p#@E}@z{Jw+m+imKrlK(B`x{V1kN8Ca}|(#L0S#DsCYZiqOT z1K9u8!Ho73rrvd|t!gwdBrNs(A&rE}gor3Z2gcC(R}-VN37sj@Yfhdn8$BGgw7N@q zZF4Vdew21asGu?bxNKl3I$XT+j@(vQxbL^COMtIafn$4pywoaBv2Af8dW6;m26SZb z6llyY7-^RqWVZk*mK!eQp33Z}t)yHx8Wd@s-8{{9&op*m?z36!uhC^LztqbF z?DU!tUoAqKO1A)%KxbCRK;9u?-N|8e2U=XjjFTbLcfoOGg)jj|IkzzHimwsZ!~b42#5a=GdZ0Pj&rEnnrN+`@R;l??`-%J zD9zf%tW+Loes@rNvg3+dNayYPbLYR^sdtlLAqoS2bao=F*|HGU9MP-;O=)USF!?yf z2pLvKPxyYtz*14~_nnD-OO$Xl8GhORuudCCw^k2)Phi;=O2nJ}`EEe~R<8;iR&{&w zxYz>~;h9Ly08h z+l6FEI+t+zQMbWe?jd18mHrqCHXPJcc5aPmkGu0!lTxt+?@G^zwz)k0I~Orlkt)l$ z*x0^?+slV}uv6y(l=fd>r*+s2`%v~ZCwCAX9ZSwxKP6J!vEBR2&Y%T^c3~}a-0%3V zmo^K{3bjPz{bU(v7#LXJ%PcRGGx^7H5$Oht5Q()<&2G3}6pH)+GTXZ~(fza3T{*6g zor($yy;1Knn!=3)k-g*|xt%iZ4NtVx7^rfTm88ud*dFB&FGAnm9Ca`L16r_9K3yR^ zed>ytgo9Rj@#KoJBhTdD%eH1g5oS98Nyz3|dt}-LeE`1b_r0k~uzqfne3wuOT2x}k z)@WD*hB2+n0m!nHLi~k4BIV6@ z>qd4esHz6X(0H2E(V?Z8`n^He8p-JBNJ@^pjzd>vJcgp@*(@~)RV9Tn3jm9a8t+K% zHfYOf;oQOh&*^U33kgQA+~+`KOJ zN_2EviW0rgdi#!}7b5M{5ZH zIMS+(B2Q)2`>&rd+J#h;?*|4hMI^uPS(#UZ{R$lO-86hGXoHvxb0lgepd#2iP5sSU z&eDAr4Clt1NJN0__!)Y!cZ2HL_ zhXKZ8ovqXV8M!!WbQ6LNvITku^CA zfc%+Y^TpgFFw8yYrqs?vxMn49gNpS`?6#lMqJNii;+Nv$hKGjqQ})tIByu5JIZ-vrGTU?=L)AH3~ z7d)N)0QLueqGBq%NDS(^kbwTrW7=9hgn*}|CHwR-iJ$-ck|jx`5Ul4Y?W^Z)# zQI?euVv+yy2`A0Wpo4D$I5Ke40{V`{PPim88>ZhP$~!YAcpfa3ce>VSwf}a}m?pW8 zOtwow%r3&QNNIC{E86-0BYl#X&eGRo`1#KV!(k{|EQs!A+nq4st!3jYf}06YZnR^d zhPrbq4t1*Ko5D>+Qan7yNU6`H~R z*S9ddxFNG<#fK&HBffou->kaJ?_KQ4Efn}!A!)O3bRq2QLRblsEM=v_t5-Nn23KkA zCIkGcM`q*n^+!Y`1BQ+P-P|rg!5*VgMxl*_r0w79P22&_g*)Ss7@VfRdLLhggR8f7 zNxEE5#`DV8fj-Nxg`QdTE(dRFWZiCOdfsU)WX|9)py~O7cYaHk{>yX=O`gpJnsJ+| z_H1YAD;(?3o_zqRb|#hr+LFyYJS5H@&_J*)B_4f2PzH~VA|CCAsbE;(X8OP2v$GFi z8)LgycJ}?MyWVvWg}!SA_KxoAhG;3dpH)MWvI^(7(Kk=nHhfte?Q-{EeL5>UenpV? z9jA70fOguEl%qv7d=;jeYf#bJMuof9;zolWd@hJAC zk!PVlqNb#?K9GF9@5UOL*FC1nZ7=MF-z2>0dSm}}YQc|PYP6%cT1T$laWTR z7>(>qD=ZWPDl(^i;K1X$`A1u1S{zdkbsT=9>#urW;B8i8t2M_E+fGcYtHa)a<*Ogl zr(TVY4vwfJB6MBn1Y~Q$YL4h!9c)N!2q7bp&RoVrm3H4k%j}j>J%V{yH#jgbk>{?q z_7)6)c0EuMQjLh*^~$X7&|J>?PzU<>V!#YIm;lfrKc5ogCxjdfE8U*p-@Y;TZlhYy!i!=?XJlNDD6(UdJDq)~6>scbJiR#sAVsY%KDLQ;d%^J#smCL!By za1g4?rtyMDX-K3OaHv4CdsK`*H}4zd*THA|E*GAORo?R5TN_s zjTdpPcu)-Zzk3U#&Pcf;TjrO0sP@IhhK?H<$9_AnyXCbRt4XS1?@l{LXX&DHIT))F?EXH;EQ z3{hLT+#|4nLoy(vAvLwa6CkN^8fgoQv+gdRMJ4C5ja22mYZW=ie|d^?&-pDk4t*ST z9)G!OY&WMM7-fDD+cyvgU;s~8O3TQcheE2HgOu{fzJ#%T0WGkl zo^Z|~HZx$+pt^(p2;CD|y=#YxS9oU^Wss32pO;k&;7K)nxS6^wb^jF?pEx+;H+;E^ zPb2IbMeK>r3mpFn#XVQP%AZ~ib95rw2ag>Sx)Cdi zA))|;2LuOl9f(QcZHX1 zKEumPMkAzBRd-G%rjBoS9B35E2AnJATJt|PE==cp4N%p7u2}SR`+&Ho^Lnlqjo+`2 zIVT*sseXM&ZdqB}YZ7U^1jad>L%I;rxo+F3>Xu8|WzU!eFG3a40ch7yi zdFs?Dtd(yT841x>uGn|lJ%fbE7Kebx^$Y7Cw2|3HeHG{7DH<~yV!Gf^H?{}Ki}t6p zLY*_?W>970n7DQ876Clq_pl-%sno1EjYIyfbysZ&8ToSO&0nNcPlqly13XvWu87x5 zbFp?9%)9@Yxgkcf?&V8ie*Slcn^*j6ebDw^Sf!bC^Lx+YLJa#(6R45C3<4LI_LvdH zPXQDGDxSHeXb+#T@ZeLDy#EL5S$$X6ds0vAz9w@yJpeqXtkqZg+RW+2{9W~=vupdr8aCZ1%F-6!4o+?6rK3vTg9SG^iz*9 zIHL4Q>=sA6q`etB*MCHK&EL$)IfGIPTBOI0IpS$tPl8qaoR~m*V4>X030E7q@#Y%;)569oCYU z_nEo5fT|bEz-tduP7{*K5eK^Fpqy{+buYyhcJGDmsUhS+>m4(#o7;(1dGBNB_Lri; z@Q_+*uXAE)T?^#wVxXb<^j%+?lDA8BIrdr_MLFW5QEw!SqZYno$fj*x7!Qyg7vmzt zc-T$aUO4AuPu4`@#skw_5{WiU^8>=28@_VBNJ|ckh`49|8tYOYcK6Jb6ovUHRd}4( z<)mEjWV-XkU|Hqij%IJ~#)pY%icC$pRXlsnmm#3K_}GIPKsahjlJ`TEGbzReKke?k zW4la)NTlZ*w{M^FS+=e><9vz*a@&bDGBD%7QY5}S^oI6E5h0;40~}sXkZm+O(JKFI zt(CiEw%b)2{?3~*6lWXxPpvynK@x_zFuoBiZ-!DVW<4_r>=vG z{h%9sBKc%PX2XrU^D!0kPx*+v_a5Inm3&9i(KMOAkKD2qYir=6uBNtq@kNszHYCJ6 zX_qj5KZ0UhM;DRxsl72Hgo5%xB z;>0`~Oe|24yp!k?v+0I$?$j05ewL?VXsI?`V(%Zviutv~CYr3N3hGoDv9z-3$HzCmxn9N}J1_z6mqRE(Z~>&zwGb~G*fDKoCf z_~i8m&jEOYH>t-Rb69yGOB*q>8khe=P;+yEu@o1KJgQsW+DR-40RvFCP$j9AKy1lobTEotMeRFBVW2`vtmYciw~Cd)4#KW6uoRk_6U9`vKL zr(Yele<4x58Hr%=5thT49Ve%|x1Yns`oa3}L$G0SZ8*AlL6j?+llS8tNBNOE@=F_rGpL*d9F$|r1eWNL;3bm)C;G2dA)_#_qRQn zyq2G_Y5PtIZ$DlnC%DGxm4cFzzg>`miAgfI^$;0}`D4qD3}J`9DNfQ#WG3NoS3OeG z(=*xnR`J+A)xZ5nl+)q6hu^8(Z?f@q5WI90*HZB2?{VN1J4sXQ_)3X=f;{F$!me-M zpX%iTZkgya_?4m2eMoTb2D6CtXCt}vqp&LpnjS{WndIGa`%6<&)(`KOmnRb$XL#^O z&Z=#pcgm18yh!lJW|b1==P=?{4Vaazm}L;LeFRbtifGjrGO%zguh-~fmM}r@Yx&Hz zzP=L^Vb99TciY*qlQy#b@oSLfgc{^^3m@_C{$HaQ^d9<#v?m^L8aBKqQXoAce-Nrx zaK2CKj;_i(pXtlwB?o)yVz9r52s5s}>nZ}y?1wX4oOQgDZhIC|&^XuTId<~oZ4wp` z=%ad=d^?~l{OVF7{_~@jrtk43r?ei*Azk0u=USc$`{w)@_w-T;C#71$wwc#vYz^Dp zJLGRGyeejVPr13xAvvhBNK$+7fGRTbs`F-+GYO+42|6n-$-n^d<#5eopSf85zrKm~ zGzi$_z-L*dAK(YM>KnT!B9Tk%uE#s{jIXZJyuL#Sw+{?2&VP_)+%(#yM3$oZ@Pqa0 z$2+u~NC-M>P;q8%M?YdzR8)CAxA<3WFt1DV^S70_vnlV(b)GI&QM4=f45=`#-5JSV zs=V9w4o(_mELiQsWoVhUh({Tf@wj2Zi@Lg-3oPvHQ{+93 zt^aG`Jt8haLKXwmg@~G`K4)fS0aIgV$j8+LQ)28FPU#tQdyLCN8$N11Kr__L$WxQ5 zjZCth=tlq2j~>C}C2fpKjJwAxMZ9L~;8PNt zz7QWDk86rL@sn)av>UnRT*P(T2`^hfW20R=GL$n>q(n#shUwH=~qf!k5iOYe6`(}`v?-VY^* z_9upOuJY$`o@=eapJ((vgOIB^0=_hvhS`HeO8jzVI&;)&W*F~^i` zCXph?>B)f4V%c|V1&ZF(yP(XD=4>w=e-c!B9BAH4mNB=Bh33vYI9A}if6WvPd1owBW$zPXLj!|2osfkmb?TbCK8c$ay|#jg#N#_n#q0~4 z%s~;)33?-@M<*B)PK~Iv-cNn`^5x+F^~tHS3lk9zDuOe$8G@CvPp z``V=sTv8hbXy%0*Dqx9~@!H};`PUAs|8v!)W@QQ3iQ3z8|NiY}z>RrHnTmlmb0-`V~9X{UoXpAuXZ zP;(V2ldvb6+_wB^$FeUg?wf@U03@Y(Hj3AYLXI=jjVrMx-aDMx`F-?Gk4rzpL-{lR zeGl(-97>C6;!T91IK~QfAjD_8EYG7ss$5mMa{yNw9lRwN3$ zy2}b^up26I$7LKYWTje5%l7P2#0`fEVak^V=}TH1e6m<%g`uAEET5=eGtw4pft9c& z!GZbgxxrYob-3e({n_cyCOHlDTps74&1s7`6hBmoZ@|FH!tJYPGxZcrQ1gS#{I{IP zKHBqL`o2|TX)6m%uou<|-w$PX=_`J1MBi$ZJOe6ZLQI0#Ou5ZGldCKrFxroAnP9yN zIC;wVr2Isj=OeUujFa0Y);?eH=>1z`cN|v-p5YFVJw{B)$hXn#t(-1y(X_E++@IYj z(@?MsY+9n*@U24?1TUH0K2t9f>Zp`?C*M%^z@=tn=>zB_MDAQQ&zr=|h2h-IKNR z7GM&-WKWOvB(jtE4obb@z2>pzOut9jriV3VD_;zGd^g1fP40-{2NM#1jG7Tdmq=d3 zqHI^NG*@$XT}=&*{(^9X>xHTzqn1BjqX6YYqt5XXYc7%TD(HpKL9WDgZI`pK$U&x` zYq2l>iNU(TUah`2ZEQB4RK_7o1Jz)>X0euUUsEm+vFXt_R|Mi+`hvhp_dlHu>QR)$ zey~e7Z7R0X6L>ILE9fdO{%f5X+8_q!YR#RATuLL)plONMs>jIoISw2xE^~MmAmwt6 z%|ZOBZ?jBuGajqQfg2OC`5 z(`EAcmV@Vq_s7?*sVcET_3i4DST#ps!YrCN3ULl2MN2?6ZCEsHEXNW+U0 zW9B{)xjv?tb_3ViWOw1RkL!miavfZKatdH%D1?QZ45jArwSh$PRGyQEOI_CNd$ASi z)VXsv4ICYdOK+ZTc~%jR8Z)^pCX^eEpV$rkHU@%|wdV{117EL^usnHa1k7Ca>CHDT ztSa8VuySN%Y=5OkA;pmoCHazuURj5+&-n$=HK;eg6CQ^hwg7S+ty)2!B%`>?f|4v7 z?siw%4NV^eXx^3(s*VkZm;c6vVM&-07_o)r5)n70q3taagOMwH)}#`NC;3~T1-PIR zItzbOqroUpD_KwhiO(#2Evc9g<2N}!6x|&lHPcAcICBqR@-Xt_zyM*g`9k7#`Ija& zT*pGVpj&La?J#s;qLtl=HeTh_hKdB0w@o9UfoHK>s{{d&^sO%gWH!;@wJ?Rs|0C%v z-X2H)2^WCBwe24Gwmv0={Vjs|<%I9{P4O%c->DfH2hY>JDX-TWA7b+5 z`1`CIPPudK9N`hUIzMo?z;MP7a({lim4{HtrX@OBJP24(C?TY*`3&hQpg~FN0J-mH z(VdbmbfU0T*wo7K%NsW$VLRVs=C_oxo{@3;l0;E~t169pd2jQ{B0B^my2j}A@%Fb4 zf&#=O^V-$rpe#sWKQaY-pMz^tos*8%xJ&?3+VFr7c^!N@w&)?Kt21ari+(woL*HG{ zIjq=O*?)13wW!g`l3JEAb^X{|*6L6K)3$c+Z+qkZz}s=@yqFP7&TkSj_^J~_M@Q!( zPdTwK4di1mH-A)ZZ7ojK^IZmx<$%VjrI=f_JSSH@)Sw)Zafr;d*!I$4ZPt&G4~mR3 zRV6`>ey-dPkpurYU)!sW&)Gbf)-~$QwQrD#DcgqDHzVWY|2ZwJT(9{tLJcL7Y8y+L zCc{EE#8vNFdPD}EyadEq9da;TowoHDgZ4!Lj*6`*dn}Am3LRQ}U9o^<5X(U^zi~6P zt7*4R?2eH6dL;8JnN>NdJY^}56zyU@7!;NNz1yK#<52UA=QJY3(6j8kvMlRd>9zYy za=tan-%Wp-*=*?Zgn>KAiLF6#4{kkZ#DX;dfz~5LG+tg!A8BY5O?mf5-PSGF;{QGr4L|R^R z2Q$Au+>dgCkiJ#(Xl&&= z0U_au;0UMtI*VImXK5dxp*4lOCuI*#zYGoB9wlW>N3nTIxifGyb;N@afa1Er}?H zje6IVxn)`z9QwbCHkZB-52J27KICx%w?om~DK2tn!>FG_w(>Kabc5WBGXXp4Ci3$< z`+~M)YN8-Tr*k*TBE)Bo9%dFs0ny_on^@S`E{30354{NJNEPJr*yU*TxfM|I<7 zlod`47>FRCRM%~#|GuGXa=~PUr~NZ7=L?o&bT#zDwJ0bp2|Zo@Sl4|?DAt3C?iJlH zlXW7^BSc!0c4SMutpZ<1#(}<--Ne0t?lM{MF4Z1@plr&wy-MaYBJ&{vH}s%(-b^VP zD6&fRxc2bQm86i65LX|YkHg7%7CQeIKJxAdu<#Z7N7&r^8NFsG90!r$4k22)#!8rQ zry3Gxw`3px5%yyaD@8KZI+yn&sz_G2Af+IqpbqrofQ>rxlt8tDynBP3bKlCRUYfF6 zhRbZa7@X6&L%Z|SxG%kMFV$@zE-^cX&`2pVp6I>XS@5|gw{1|qFOh&cZiW`2u<$ZK zqyG&JM6}(+upQ;=C~I>^xJKW*o65Ttl)thwv96}?qRT2)YBA(}iy?CS!@GcX^MA_N zYK7tt<5u6``SlKvx~36g%f*QV$(CNUw9(<1?)KB#Gn<-e3mPg92@G;j7$I~No}&M; zh@%0gS;?&M=02So=Q!bktk)=*6ka|2_)(E;)*-w5618^!9b>@-mJ=f@WY#se|1U&f z^-D(&g>E{i=xYpqnKvw+2ELPu9iP#kZ4+} zQ%FWw{C4-PHT~0t9fRn7ReG_@^t}oIW7xW!N|2OD9({RK-(zaLRqwb&1xKO_ zS27osr9Ap@0W0@os|s7`6=DAiz`E=+!&f=avHn5T0dU?g2h9ZaS$-yRDd~os+=IP> zNKyvwrwdOds=?bc4GLwXL#05M;86p-Jv%RN9USNn-@Ih(_L@5=e^DonW${uKOba=7 zyVwGr;`7On%vh=9H;*3S^fQf_2X%EfKa88fOULdA%K*+MC>XNi4djd>6pXjxN14rq$P@IS$~%a5=lhQ^!STE35&?NhS)2*JKi3}(E#Zg>?wVt*oautV|n zBy*R*d>(l2UW;*6IQqFgQ)feByogWQ(m}ALosE^%e|S%Pm(7>|i5@ZcoxADvW8`?b z+!eG{?~;&^K%-6Dv%M0Qzw6IcEkY%o%vlfkLfBmf5b|iBE~IR5QC^x2tpe7bi18_z zGyk6-A!@;uaw956Ne$R{TE~W8-3-WWVEU`bUG)+(pp->;5`yZSJn@Z7+QA%!HL};# zJt*+6v-_1Sp*D$>hH)*pKK#~}pGQH_xBrEWF~c}gp+<|*Ya_&|vJ+LSZs{R;lobgP z$|)efvWJ0X$Wqd0>(SE2ojHJm4TQTqVgC@e#{n$z0TH7oF}7w3BtI?Rvvzjd95?*Q z%`5%M&HJ{QJxM4eu=dM*AlcQ_tQwPizEI?Mj#sa-X)Ha{UXVO}y(Iu!q$$H^{jpD* zBXP;2zokgbaM%5BNXH(B$k?IoI(_OC$`@I|ty^2peJWiGNk(Pw732FLawlr!6K@`Q zcK_Oo&ctx2^7d51CiUedOitcKQeeeu@%I|vF~qq2-ba51ny60 z7&0UknD!TSv{x>Nc;E7A7YLps-p|W^a~qPRx78!{l$+8 zx^6xGaQ7j19XgRljiY)5!432M%P~sY`o9)6ePrX(EK?eeZ&~?5RIDrg7a#t-AGr~_ zE-_0>bzSqPCN+^XGPiEm4->vL0>9TP+5h#+x3u>t_V>cmw4bU>hnmk?br<^kjC5ma z|3Kn++kIt&%r9OV+VPvX6Tq34YG;dinIz{XZs?rO#clH-v-(bL#_!s3Zu|)R)^}+? zlpS>XgM!vQppOf9!u-%&I6>|;Ga%x%zw18SC0u=i@YC_sW$mAYyje@P^LhiH|0jE` zd*D`Ho;Aet3{+tyUSa*9SIu!5ljGP^_T%qQJMjSCJTnzsEX zGgc+CQ|yiWla-G;LtJFVNry`-ctyqohKGlNXqkmS zO(*FI*KKl?|6*UCpGIjc&JwrdtbvZsqq_NbNbq0O)~fE`uU~kQRwVmYGJ6>75gb~4 zGW`-Ho{3ktG`5u5748e8CWZcs-`ohl;C(mWf$Mlj_0SH0w+^G0^iV2D(t;G=D+BM} zbA%M2ExCY>6iErnMbln-V5sVY1Zn-hDMkfC5)gLr{^zm~_8HuS{B*ThynMp^CmXN2 zI!jA}A2a2Ga5IF^rEroWYd)r?AAGy>#n4qeF6Cb$hNugKYLb%yH!trH8NWiDlW-~k zgnlXUF={a>1lS|NXylm$25#m^;*ozfq`sxt&bRqFNUMBGrm%m1YZ3G=a0603jgFv( zy7~9vyFMrBT#t!yMl-1H7f8Q3v$rMVt8qubSq>g!PcQ-mAFwb$MH{GTGBs^Lo-jK4 zWm%vSvBD%0Rhdp^vDnX(>Ch%7y5xTivPEVd3!DaE0Y|Cm9Gv+V^w)m4k% zy%r`fC-CU33tP4Whg_DU3Hl z{lGkZ7la}Z@PSh#c|rK!E0CBj9Y2@oXrj($U1!@&U9m*FK@%>$i;u` zljSd!);eRAlFM&kK830UP}fl-qnCB_!vI3llF`yTg_e*>3=fScLAL=1$lQ)gXUi)k zm$=ge?`B2u;`b+mFQ;8G!9t_@^Yj%$0 z+;Q=#kO9)y@(B$Y?=G{FNK6<`VcZ1tPd@LDmmq++(9lq1>V$Ytj2j8==bj2LKD%|z zSiIKQ*rllj1(9=#kWI`%=EPO>4j>Y*=RCmkp=W?4E39+JfbS3#u-@_Wju0_H&2%F+ z_6?|JgdP-}Z_6#kn`92198n2!Z@Z3LL(t@N0_-Xdh~nv2@wrZ1slPUYn>Q3Oo3O7wA5M{ce17Tng!zSq(-)vypYWXvtl07=LpS1n=~m!7APygj zH>iBmzLdLx^@H<9r0~la*S&vBFd)l*tM-=GL(ehiE+xNyT*2~zY7Tjy=$S`Ynzue6 zus`^8_pQrnInm{u;(;&8=VI6?sq@}Z`e9RV!e@hvZArU)nc0ZV5Z3^LQN#8)TYm1% zshC&S%W)%jx`FHrj*DHEEqv1SxG22O|3$AV6H>ED{J5YsGJY<hXg#ZqKJ%iWUsuR!m^0X zOuHcr{Kmq{`U^cf1Ew@dgYatfMBnj-A1*JYKD6W>j(BT*KkTk$b2`IcB%H$tZpvVj zuk7nrJX<`%^IND#v3y9E=P23-#P<+s25>XTqi#5LSgdy-Cmq?B{A6x=O=J$%_-B40 zp%A+~ON5q~Xf5FRmBozfBhFL4Exio0)|dP%uvwhz;iCG1>c$Zmi*Hg(Hii3;TW=+q5q}=62BZ#s z_g^1!yqDU0qnwZLWB--Xe4*N+@1l4_F&%jV{QR`7MkeM1v^r>M+j;l~a>^My2<*O? z!YOCCt#9wDi@=_sAtbJhdL8memyye4MCIwdmGcP0KTs2mGn}~d=SvY4&_(edXI+4H zVDX8k5VxSoejnX#bZZY%!(og=jkp>aCE1PpO7DDqFRqm1yQZyzcOqoI*0{3Z<}Z@1 zPi^{7immTR8Ee*${{EhYuaZV0JM0-vl~|V@XK%2(?ULG;YN!z>4kw24>57J7PTs#S zog^1rvSuC_@7(QLUU};3W=DvLXcC~eYKjmtf>S?8d%$D3|9yiIb!2K)iTCb@L3etz z8%1Shd===Z*oX(D2BLkYtoTHVIy@^R zz{Bj<&`4w8cIhQE+CaVZnr6pKJX6^HI((Q_ghEC6_JZeHaTe+NOBcxL z^U26~a3C4RYn7~*y%u~ErH;pUj=h98`V;)cRH6C&;*f5O;i*P~P)lmURZ!!F<_=*@ zf8WjCME$J%92dN}`dHzEHl;JYyR0UxQ?B<6hlS?Oca(0~0qfiXmv&pDJXzIRExz(` z)yPp|RRw#Ux~ev$tFIs1p4$DEqpjujPadsFXnkIH`EM4@G4k)_g-?ust|U~V)Xsjk7rdimBuYFKsaI(A4%wAMc8&HdKMPe>V1sku&^dc4iF8Uugk zk2vbRcDG9bbRSrn*0Kf*5Hy2nnemi36f)?C1Y$~+4z5;0=kB3l)fc0G;HL3=Lv281AA;2r~vEhnAd({>JVZZdrZkB$8w;4 z46HN zt;FLxO`H2i9>{rKhV?2ION!t1EWZ1%;2knE_Xu{au0HlS=SbdPgPA#``ya}t6!g*5zeDFE z1)uBGwU8Or)2~&S3Ydv2mb>#98}3K~j8B5@(_aSV8pbAK-*=T0)szS|GnFhRoIBaw{mjeM3W{A1<(X5k|PgvUNCMK#LEz*EIu)zR;Td9!4D3Rj~p9ml){MQ4<5DQ0(oe0axP;bRY0SZj{l+RI|_Fsmga99hSX;3I0BLH59;_SaAjdbxu;w`R=g|Xi# zi9nBmdd-=SaFT&W#K08Y8TM!VGPZ=ZDSyW-iT zd6PTWi*FedAB!#DC+}b|8EIqp3{@EUy5|1q~qa)^&&$I z8u=61+e>f?LGnIdP*3ZR@(kKf?kUyc!IueZq`w-w3{7O!Sl@RV+2voXwkymtmFq|e z8)@GA@d_&L4NN-PDUF!JzGl`dHwF0kl3pSk{afwzj}^)k6tE7M^}X`Sbclhz4mE!W zpwj$g-@`6sP$kH<{rsFm)^;R-oc{mi2Pw%vtDCQ+;--^V-kH+4mn>M%moh_e^FB0C z{Y*m_l1d1r>7Sq30ZS_I{1^#F?mhK&qa3@Oa&DLMB3itYP_N$-^u^`y$&zFDrxPA` zw5~fYw_&$)oiysgXq7*uS_PMq5~p;zbYOa51V&E8xjNr0$Csh(8kJf^u}5Wx`Y z^B|FBoUmIy3ffFC_L>&+e+G_fh>Zi?8ICNzd`v$$qQ+S}Q*YwkIhVx7UdvwnD!H$v zKOkC;h;)$55fQg3Fcz0fixiv&eGehujET}JI>*uW+uiwse>~lLE!|$0@X*G+Cc(a} z`INM3dg`ZvGdvA*QiFssR@QthId4N+_A&M5ZlWcDSODGzfce6%f$bb8r+50?iCg^M zaz}4ZCa&;C%~UCOjxjTe@J zsQf-X;(TV))Q?ZwU}9J!sARflf9kXB_4hw@NxIOTGdm(ysaYK z{O`|BH6#V|iT~ipP~fFIT28%@=h!x>1OEo(Yra1_ba9)8`|oZu-w}Ax({s5qMJs8^ zOkcGiTg8bs-TAks*JA%U<=*S*G+2^6Lv@px{6#QeVR_eBmw zMA}7uIiwMAiV$lPE_w&{#Wk%Ri!*mZz&ihAbX;5q@J>*p^$EKOQqzX%dajez=xgV! zY?YEqbk(WeBq^EH?p)39p~KW4#gp6~_MFsRDgKG!7j90M2WGpL(!BFbcX@1#QadzI zI0Qod)igCbYstyB)c= zkN4clTi-=av8U_6#&usUleCq|!mzzfBsWvIQ2g3#=;#t`gw{{7e-LED0F@)^gXcv@ zX}iwV36)=@8oW}~<`PzKvb^t7Is~izgv_&~+VgxyS+E0rr94aF-ce@0qzTSQp3&-) zBpc(nuNQy8@>Ct5(ak$ykzakU|0%#Ib^54AsSoj{KR7pTpW1)Is=WHS$vV#&lUITv z&aQV`t;2rIX*#k(nB&KHQm2=BOOl)k-gd9?pKtrobFMkDsZk%Ngh*yFp z{ZW-Y{~D7nxUE}!1Rz-O8kff}Th3v;Rig$}@w4If6HCvoSv!b)mK#?W6Ylxb=-2?- z{o;_!I>S$G?fGMe^e$PGmHPNz3p`Ekzw=(cvY)V%shGFq-}mQd^rfeX^9TwOKKg^N z3GLie*#%GW7*zzGyVX|l_p^zc^N4=N9-bY4J=cfKd9y=w|0RatH2ytzXiOLLoSy*~F@E+$!C&*o0VS`Q)WSU(m*>{%)Z^5DZA`Y7PN%nL#odTj zw+P{5?~H$bvdza=z7|LeflWd^F2<@;Wj+H1QJF z-4%5nm)kk>FG+fbZL9S7w%$bSx3g9$9DnX`@wO!%Un zF*L1sc<`7~8C5SCo!$C2+LF$5C`$L3{yFXszg1E&UW?({C@b4jdsw**fzYsDLUY9( z*u76MJosH6{o`>gANN+Rf+p2v)A*|U+g4Yu`E~jc4Y%VD5VfzpUW=Xlw%Ce2KlSHV zUXoIWlobel*mo^-1TD2Q5k<^@K6ad%N}A4j{pU+)T_DnO)!sFKF1w`DkYa$(7G+xt z87+eDyQr%_U-xBKt737(3eS(p0W&-P9zK?rZcl#8*QwLE{pCTri!98po5q*j-}Xig zrWnh=GfoFPvm9lr7XL*1JJ^;%M?}c|)fgsmA?Spv=Fj!qvUW4B9mapdg{!b~WtHCk z6`+>8e2!km_c69XpDu`n1J5TTo{x6%L7n%jsy7%_9{$_l$V=i%EPM0MaO%i^$27Hn zy^Im-o3pR2&u8!~#n8!W$df!n{gRoel#M7n>OKnY5Rcj;ELbWqdDGPToyz#iK5}~K$1Qk^eze)*dw=OsxxH4^EK46?W zUL)vZjl}2y(HLV2FjiP2i8*si3->VVbXbf}^WQhQ6%sG>cZx-`11DkCvqcx1dwJHX zzb9TWJ*ad?V{59Y`Gvi75iA>BH#Ms2sprMaG>rl1aPiUY`~GQ0pm5pWo=cvEh=ZiH z`f&h{~IJb=p6UL!YU$vWT1)>BD4cxA6!xj^CDyskM-`7){`dg8gv zbWx>eFM53pem?Zx@>V`-$}u*+6~CvyaUg4Z!olnvgFvhrpmb z&Vt_Tk0)|KG%=>5l3Hm_xu5PD%MOgKyq{LqOdq0bgHNV=uS+{a~&4XhM z3p=$JUdWj7nYBFhebmr2!LDWp&<5_?G6 zonn`$$5DDd?u|ot=J&HFH>VYR9*88`%2|0(t)o3!jExB`*<J`IQU^cYu*Rjp-T=RUg4=+;+5ajZn}+i%d0AaZbnupIaH z)9PW5|0)2|64<1jcC8|fFQ?jXU!TgP>T^|(oN*}lxgErP2Vy<~Prg1gX|i?jmVV1x zaq~zEtG9a@QBbd;juATkFn;X3?IAc#%*1o7zX&jKlJi;MbvD>M-zk?1o3-EmD)yIo zM2EE2=P&zYEizt36j6+PT|qj!FM0A=>1w)HL(i?e13q6p1}?$F>Cn-m_4FsEc6LoJ zey}02@ph3PcZa+A!Gj0G%+Z^X54v((^;w{m60YTWMBjq@r2<7taW%DWsNS)h#4Vm5 zw|o+wll>&!;ZyEK>J_7lVi66ybD5mq|DX|VIojpzf=b1L`|MAh+9VS4!j1XVgVSsG zRFo~Rl!h(DambtZ!}TO=fWXSpqWn@Yb{?(eX_k-;l8b0NfszietY{i1G^x|K7q+cz zeUEzEKtx0&qf2R5m=o97T9S(|`DTBO4HpjVOiRK{Z6R=sOZUeTr#IHRy_lKz7~4x4 zd5i=$5fR-PAPWb*KS(inqfDwgV(ti1OJJAhmOR*d)b@Q$z}2AP4VJ@7_XC{{W$u`M zDltdm)M-!cwV*sem&Ib>y2&6&K#vtDnCb9wp1Ejk%hH#y6c2&~Qkt`fV(&Ijt~n3I!J=GX0c!2Kpq(379LxuZj>&t9nGSW&afHcDkV>heXM zaq(vd=yI{szY)MT1VulKPCI?Al$_V0T4mX!hlpeIA3^@?$Y;zs9zG3;aMbYPuwF$U zm1fd=d5>x(oUCHF@uq-FeBy#-2Deey_UFoLJ7~dOxu*nfFu3+_{a4uArQ{ zbcxCwiU`fwqSI#8ruNxyC+Rqv@W?Cl%Ux~*RRxr|6aE7EEKe|c2M-)Tmebdyw*UYT z*lSa{tNl&)%Ep(;$1aR9k?-l6WTZ=GQ3<$o4Xfv3?25IDyDm6| zs09g=GLz&~-?h->vYh5`M+{iv@ef&l+j2QG6NGbcOsOhyERP4I+& zL(_xSZ{~>Z>*5xdJ0r#&hy{8_Q~A)+F|AD zL$&m!xk=qk_jMY;e*4OmQF+?o#-@c*IvW2x!js}j91`WAHLLMg&d?C>v{!vHWMFY| zWw7lCb`g1^wS0D2LAGTrjWXRF zE~)q-B+^JHY3V2T$orcRT>;DKut>FcF}|Ytl;H$~i3FH(oCAM73ceJ`ki_N%P|0`% zqR{S~-m;OrnvOsbR8xg#~CR7 z;@Y++yL%Y9qz{$n%?z;(wcJ|u;2ktk^_UNywdta2ysy{#@bkP5YwU#qM7Y}fJ!06M zL2-6RJZeL}EVCm$WL7StD%+}GST%n(x*1)3z=F#!=Rhy4_2mY*Z3z+R*bG?C@sD%L zUn3XuKV?K4Pmm%(om4vA8HPa|{u(a`DrB*qKwfK4sSt@bxQx3jJJcR6C&1M12=p;Fbw6nTZ zgwvb&v4}dC9S0)kWB7#1y_KzwhULRGoS!h&boWTEEG{|wa&`tt2y@96x zVdfuo^d5>|3ZND0nt! zU$Vp_Dcu{{;)(biF*C}C#0~i5qO2E%3d@plfPw*TiF9)-+jjH*TtN)lRs=R}N`pcK zO{(SU%j_(~glGM|E4{V0>TvrYPuq&mb|;XaQ|?O=&1KAe!(!fhd6$HQH-JCDwYcdT z7!I2SqAg7|9L^7FHIiOyo^XHX)fVktwSwv-U7d1Zx~+!%bv<%^f~A?Aedu-Wn6o&Ti*3KlpJls&WtBD0_bX zkl?*7eq?fomOje&-f&iSD1ZDuS zK^Ct^ew~QnQy&6UJQvy-cvkKwN)B36h>Zr{n&KwB*g_to=MsxkvWtpg=zjQ5OyYf|@+qVD6AyW^hBP-H9YhLJSrU$taLRp#9(S8bMtV9SC)PVh zkjrWs&ciK1>@F*Q1_dn#mOOs*HDq{>zk?BN4gho)x!v8Ck1x=gcqQGqF#khN)ceAQ zrAhXN0+r*S#657Gt_+0got{S{v$k5+?k%pX#Vh?Sg%`y(Y)G1D5Xx*Ms5LL&` z(N!&i-8!VmO-nXUU7+7yBu}M;2u}`v<+zk-F zOsN_Zv62Fn(fq{CO+86Q6XE3ETbJy3Uc2l_(&F~N+X>VEj<`&QyVf?X;WP=h8rJu`2GPlE{i3jVEL|=90Y?5|^ zmE0L@1`^4USi#ZU#N8!>vo1BbT55Koo8YvE8_-IxM*Iq!wrEDAb@({Txa+1sLRv~n z(j;bk5|R?vEl?m{3)+RwHmp=Zi$Zj8f-e{yJrj1^`%zd24>sWyMt=voQRT=CZ!MEV zJzw*`eWKSJC=rmYhgv{!SB(UITYhRQ{E?Ko+yNlF_~?yutfygFK9joFC*kq^mBaj_ z2gg4|@ery#l3c=)%hLwvUOvI~dK(!o&)RBG6sXtqDcaCF`C%8B#PUkP8B0!vxpCSYl#FlXz^xXT_<; zIkiEs{^v*8388AkeTYi{P4-23^djs7;#NUFGvTa7K_b=h)f~)}+~*lbU#ffloX}0@zc&RQx7zVO5{;lqyR{_m#aDafntirOG>Xe#p~`15h6OAQ1<72= zC-7PN98XtM5`2+7)>`nv2qem5EcJ?J$_+5p--%^$@|A<}4KDM-Bz`|Va_nIr2Q@!* zQ^%S$8(QD^K6*<5=qLDK%CT)_EIOq#7FGyt?7KPUb_rJtQ#gjT<*tzX>rSc5uUeK*3D+j_q38^BaJ)tfnklsmnhpr3LcvXeM9)}y zdQ-kV?rtHZe{|-SNwklzZRKlf`tnGSj3l>Su55s-R*Sf3NIemL8?eDjdE(}Z#WDYnyFU-dvfaYR@du?s$QVK?L*|NvWGIS|WXep&26_q! zp(HY-$XLo)h!ADUkReerCW?eohKdZCea~CH``!EVd*9#ZkMDn9$3Bj|kM{H2&vjqJ zy4E_^xz4qjub=X?=})s?v?NBM0|i~SW8ZZ#ioUa1AR6>*tUs_z%E9sLmE(&2l?5~u z8=^JOJzhy#&6#2%{%CRJCcs|Qg%%kQq7PIDR!F(j*qchloaZ139gMTzNl99M9QqH^))ryz%9XvND#vMWhVA1lAxEP3yl=H57V zUr{=DO9z!i$C?;xhvG!_0OA=v*k?I^5JTooL~jB2)Z)rsN&RG>1_^PCvU%+bl+(63 zE4q8Vhuzwq(UD#!Io9lc*_5*P2m`Uh7kyn`T0#AJXX7PEReqG_+h>;8V%$Ezhllp# z8Cm6Xg+0aGu98%~{SIfg-3^hN9Oc9YWjIi9JO6ZVC0f56TLkCqZX{R9ezU|NTNg+` zRefN3YZc&EV5;53{k^~`AbG7F>GTFIajBXOw@#@@O*XBl#sTqV*w}sM6_L|G3Ad&< z;Jt3(D%mVrZLAqaYM$jM-@JL~R_sl#abuNqJ2^Q~9XMB0pA9xVWwS>CL?(7Gh(*1_O6PQ>erCtkgB z@7A#$hXU_yiVFdH`~dW;oF7m#fAYj)#2%E!K~@TqXTzqN#UpSvw}Qj!H|d@RXX05hU8dVxH-kIYN-e)d<0rs`^ui;eIYAXC#a2JT<4?c&EJ9?C5=DJB|(?jd3vt_aiL zFy1qfH-bYn((Py3Pl4m0{XY{x<@?@YC~?i#4pWw-t5@#{ny>M)>&UU%`;LQj-{(R_tqS%H`O) zua#L#_bJn~I#g1)(|;$>7k_M0Qc)`APcJU^fT#4lT5TDctBd>h<^GW1__ZTsLLn~2 z0e(V-3@DmAj?YYahyV2~by1nMfWkgWiv|}HQy8k@)#d!pDbt9n-4aFKqLVq7ei(+YFI0kcsVBsEL2~#f^{}_^x;NgmnS-7?rQR!}g@Db-S`1 zGm_Q>8&dW#7VxBgd`0V3wWP3OR58Wm2~oBHp0Bh(9*~w(J!SERk2e4Q38EeuNaI*N zSl!6VOL|N=G}rxyqAX@weE8Nsy;It0$BMm7$y=`q2#|G|<8mC{VeWW=*MD9Ad0^n( ztoOy}`JTO`rM<dPH+f~iq7z7+ZFA@Up$WK3c_ogzn zma+36-thM+l8K9P@Sx$|+sdY8j&l01pB{HmUe*QeOtl04QG|qsV5BdI7Q8xpF4i{ z{V8ac6dk$n@M`z6RWxqeqIyLC>-xA#2fqGDr2?E;T%K}DM9?q#5D|J5dan9z=R!nwnAaK!NCAcwoer>He@K zRF9wMKl*26s%qkKwtsldt;66r5($WOkKO-1zqDHVQda)g60e5b+hM-Oh51=$JAd52 zh-D~x&Kxf`nQd+PRmH@6o;U^mc3N^m_~v= zVZq`E3&!~VpQX0WgzRJJKXA__n0_BFk7-xF@(Dd-V~^(?_Lq;BKXCSTWu%T`%eTAq8e7cj^^9;H_)#$r+ltvP$?> zC>866fqnc|O1QLYL&7CFO-j z=DM7q9MTX!bXIG`Et%d|YDUTq@6I~zhcO}X;WoqaIyLEcX)#boR$ zBsBYWSSO;~&4qGYqdw`O&M`6jZC3q7tjtKk{;0pe3-;P#p2I~+#s!QOVKbf0C$E*zWjOT}MLsY3T<;`&%p zR?1gU3KI<-BG6UC8}J*7jZ%6X7UyT+k*-z!RPnuZ$AP`Ytlbx>R#gujIRKl0;)?7~ z*Z;bD_CH(}Kk`X^Z%|TK{axaF+n$2=(8)P_f7X<_WZ7-?x+Jiji0$L@ylVss#em7u zYoG`n#WXEf(Jw5R7x@g1j0=0Tx{W9EBAwoJ3|e_vZ_kL1JW)V!|H%=v;QVq(Xy4v- z!#o!0j&y4lzHT%z_$<@obz56LQ2&+Pgkl)BA4_YMmf5D!qz zmRQVtZ4Iu-O_MwL`D^CS`+EMMYq{>EdJe&Su<};)MT&TPTKmrulcSp9_%T^9aqY09 zTUOZnm-m{lDs+5oS@xLwt1Id}TGkc)D4TOvzKnyY+kiA&5HFO&l@vd?_lH7nsnMy} zwlrW&Hpc0O#ln!G-VEehrOgQr()x_jFzfjTH&hX5W)(boi&o=eayhmy*$k=)-t;E< z!FDtQ4Kd6VR7G^PR@Lj0JyO>Uk+1J6_PE#l-m*!IGV|MXO#7ubu>LWOpSBnOA*_or zqH53}+x*l!J^_<&0?hwpib$iZxkfV8<@LO~k1gaSgUg6&P&jM*=6C(AllH*dtyOGT zAw=TLcIzbX*z3f8BXlkRB*8amI2O~CX``2XUTe3=B=lwKet7>XLGK%<_fNjno|9Ee zF@emKltfzky0_o*4p#DYE^JSnasJgF!8dkNGcKX^cxt;wI{)!2Lw9O^@3|T=WHr1z z>EyT=cy_-Kkrn>qHBS2oC0W$YyY^0>e0)oDWU&t%*b<0m?mU;#D4qL0R4#_1!q7o? zM~}{q28nk5>=$^D>O)KQ9igy>|1%?-sT;~^mmCG2KqtP~C zw1$_m^{iXpZ4+d}S*{o0f(bgpQj1*-nR}n*o2^2O0&4QRwS(nUK@=nWd5LbEYhBXBAIt zC}q$bKl$vE**@kFwTGM)M%H0P+uEyx_&AdVgp$-Y7d`3Gvc9@f`nlV&FSa8C<~l6d zE1~jh{Y)asDA1CEmBQ!HRCS%Z+tG7R8b(IGECfh&%9hx>$;yA3$&t5D*+V{CI-ooE z`9N_=(bUZOx`m+)uSf|}k2Kv90v27DZZZ~3{7BrQR-FZuE7yL1-WzZtu$=xMt9e3$ zO*FN&>G^LH&BjDUU)9xJjRR$Ef2Bcdf7GZaoC^AKt;=F7ehdrf}vM3#tzlrw2224hq|gX+88_YoF-Fj7m%!A|r8>oIq-WmG4>998h*-`vM`%T!4Es`0|r zteJ;9oURf<#fAOaM!nq=g{tUFnr<5@tPfd@nNyt0N~b@=+}C>OCaSC~CS$Aiaos&3 zuKX!s(`CNdm9h>GQ!ILTgA_WEI-idg#X$5KVJ(zpG^_ilO1ER$=& z{KF|feD;lO%rh1?&QVfhmUq(jn_C10YgPRgtDIYz)~`R1OAt>-Y6*3TACNlGmnhgC z*h@ax!c!K$jw~W4_fGk3-w1*M>*QaN zjQK7_VBe0?;& zM-gth-$3xe%10b_S4|pK(HXBP;Tf8M0@^`AokpoBS`b0aC$kk@$-kSATr#}r_I2`p ziH@W^G$7|PL)KYvpFsD3k>~Jpcy_ywA~Iz8CR7eDyg9fDnlQ~&CZji=IRu*cT>quJ zMNXtGy#DCHhzAy(b(7UvBX6rqxUy2bUFq#eI-)1cZJ0DF`vNxcS|7Qb5CspxMp4&R z^Ss~j*=Q4v`q4t$X&%nXz(S%L@)G}jToR~=BD>2v5%tl|zbXk2Au+b7kL_x3@Ufot zZ3d5zzae(xJog>N`<7J(9xTbbWudb^MMZ61Mwu47 zTUF4}i>Z4$tj2LjysmWQ5Xzg(OiiDbmTnc*f`}Y$ zS+jYK4F!-5n}amP1qc=2ltY&+Gc&UjZE4L=YX9`frnx_Mhm*ov)6+A^JEGUn%2L0M zM;qwlYSLUDDmS7@Q5dImP|J3O&BcV*^U}pQ;5E(Tlr!xupN0(G&M1(X=cLEJtBIsr z!~@h~xhAk@>=B2QSj(puGL$b5XlR^i5cPk3>pnd&I>HPazKohx29+o!j#d z{cLmM*_O{kIJ{_IwjNE4(tB+jXSz_Y{ZP=!x?JExw*&v=gR_7Qel}11vKE;4Yvzsd z)dMem;!@uiJ&>p=8YRa;*fZ4e$e!v z@p8Uap?&;+Y+Ffxf9l=7JL6V33VUr6SBpdeF3)eJc@#7Tdzh$L`*#{$WpiW zH`bLs;do~9#4g%}T#NsmXuqwZQXeTaRG5=;9&#oQ>UwhTkc>u>v?=(h_kxGFpY`t% z^YpiaZQ+0QGJ)NT?N!PH*en_K9z$do_|Q?yKjoA z;0hfFpXPcSXx@9#&W!Yw-*ww6DS5reQq(njL#SlBjZ}!^h@(#(W5^W>hM#@fjHz`u zohj7RwY5R>Vk>`AkD`Pc0J{fl*vTi*C%Es}ewYSZ3!c5ma@&uL&jJ(ozIC|G>~6Ze z<~u3Q3i`MQRxQ^D3NGeKepEJ>GHXfMdS|br#%HdJhua~x?30wT`%B~9F94sn==yY_ zSN-(|4<1M!YJ+?-x>7lBDL^CccLfN<*r2cR{-f;>eZ-UCE8=S{IiE|#%$%~Mq;tQ{ zjcNv08oYkGRPImo5lUDg9A*Lyv^3MAN0RDjCyZRx&rlX$fjpvR?$js@VvQh`cx(nt>5!DiAb^Z(2+8fn1!^GD{tZvT93RolSoZT1BKF1NU^fs zh-L3H5u%IC7}+7-l6E*5k;2cOL~0!-&Lk!7a1SzT1 zmTFy7Gl)O?6ckB z@UBK*lSI-;dQ!;B!Le17v_(!X&Y5DTcDEIiuO9_N{l3-*?oU$dqMe~8!PlMVVX%bJ_P3JU)6=sAcZ^s7lH{s58^;XR zduDN$9Qg*gC`cU&k=0occb%J=S+nZGp+k+3#Tvx}NEKcVcyGfPFGs$6gE4r0%bqsI z0%$0+F){IN_+E!%pBXiL&eBP$H|UE++%+;8+hm(`bynk^IZdxj0^*xtbEE#!PDS#4 z)08iA$KO$Rzyu@Rb}P&TR8S_~&+y_*jEr8$JlZj2F3V}fG(v+3;ngaUnhE-19T!JD zf2#gTe>?OJy1Tpm`#q6sf0UOuqc*j_dos4TY71!<70*dZtum>Vw5{8;8l;qocOMB~ zTbqTJTm}yew00GGy1U<&a}9D{k&*mea!tiE-&H2cMs28JPTQyps>M-%{|DqkuquXc92jB+RwJ= zjihga()9PM&4@!4waz*=xz3)Y<(M>ElU&;nhu8?eExdMR;S*29%|iR_06jzEDa?;K2z(> zhzTd0A#;8U3yaTBHSz`X74r@Ao#%Iwu2)9&u{|_#-st+)9@c@hfzCb0OiC-*{#0D` z4BIHB=Y2vBqi3VYJKhaGBu)Dr zSMY3q`SRc{DZ?ydX$V3(uP9FQ{OoX9ec1?bN?mQ$6W1J;s%hQVDDy~>@VAaP{qeUb zg>I|#8=yTX;23|uGwHgiP_=y}bBQUfWY3XvF)Kb=R`|t1ERVTZvZ!@m82^uu>5;(r zVf}EF`3K)Z&13Ip&x3Z48++)y%r%aE9G-_3*5N#&GLfr-UTB*Wghx^Lsy@$SQtYy| zXZ1+rpHrfDMZjfjUc0qaLE@HGSL|Wx=S)<`P~Y`!WINT}=$7@qa9r^54 zVO6`$V7FSAE`@DPR$H5%v$JzABbq;d2K(!`F^X#bqtTUhp#WI!sVG51(XnMAT&?V!6O_+5x*Yn8i2=! z^!(KLc(N8GYUt_d&Cx^swS*?e2VFs`b0%M+*Nl&kqxh;7gz2GZk8C7naHdcM2!rJ~ z4F1eznZWg>N2?hq>+jzzgcw*CN+-uGbw^}x)CZg$22EtfA&I?lPLEn=TZN3SMDn4) z?aLEtd|T$xh*PPDCm=LOTM+!%p}7w}TV?0o4DD&5debjaL zule8$BIZNX39tZvj_a_&_p?N^(kna9dw6K0Ar}NiL1F@git+)w=nKq(YE>pfRv(>C z)Mi2aw@D0M!NgtUrw|J3{($EScz2;MSmLrBRJv(|_?!qw4yRxwelMgRxi#kDtHu0% zD-wf}8l+uGVx`h!8VzlGaWnX6?yUKJq)o{U<{VUX_SO8ATegj!SMXU^fZoDrIVn&U z-&JCI-`l4uFASmrA?@3Um>izbqukQ?zxFO7F8oSnl9%*$AiItlA&dy^c znoBOjk;@y^KK@VvU8faW0a`rNI=e4=RR^wvTS>d?#4g8O$5&IGgs6D9@sR!UN?`s` zEZ7o?e#|IPc?B7QhZg1H#kF@C3-pv;6Ps`zZPr#FiSwo82JZ2V~O9J58Hy-(kj|BqS&~_~8Ja zd7g!{Pg@mChtfd8Wg$&ntS6E6l~gHvRLqQGovLGFsoQ^1@7!^n(omp{qoA^?%JB%Y znm}UcoYU`KEXms8v<@cQAjp&UWR`S7>Ho+$HZX=8mVPgo+5TX(8 z$?u;KroLS18?-{0by`jM(y$YmuE$D&9Q>2+R0iDNMdqYC2OXE6F8Bv3yETT6A|p{a ztg;)OI(MD#GcY#hoGTU{$B)|E*o-19*;T-UMy~6Kow)ilMI?=$BsL`9f8ef{zwMZ$ zhavwquYk$wEKIs?sz!4=G+@Wxo7)kauj>?5zro?dHwp@59upl*O8j|Zmjg=+r`1pP z0c(SU>g(&vUaK2GC_O@|9>?KgvEenfm=z_LC>W-$Xiy5%UFN$;LoLiTt#;h1Yrop> z>AT#_Tdi(AO+QRvOkXl4zP5WJdw2Nf`mZ-vsQKUM%s zLgTGTC>~0^K*u%+?RS^^_#7V?84(q|H3s&!tV7vKx3NA|X2n3NA|rVxv^POJ2_JjE zTd`=K(X(1J`TMNqAJ^0^kcNsJuVm|I9Ni%K{jw0%uxGm_L)B7815bU+Dn6>aZ^sR5 z#5l#E70fOnDWs&!8M^LxnyO7*_WB)p7?6kJ!bB^|b_m)QoESsHx0oZ$o{X&QQHLsmuZV2t zgo$MlE&1huk+jpd52YqgJ8X4ybiBRy5b+Rcge!8D-j>xXsaP5GR7OMfp0%*&ZI9m? zy_(AROM8e3I;c;Ljjf}PC;CadedApI&c~Uv!c1v;jP=yQo6@d-Nn~*41 z-gdJ*fJ9@TS~!WseMN;*Z^F#G6WQpXy?pB1*ZI zzkt*qZNR}*qV{VrEb4?y%BG4svd;=~$OiH|>UTYlH=`z5w+EWQy<_7Qc^N4pvsTm* z^loI#?)xBOCXt?8si!8hDyy!vulOLZ^ zO_kaZpGZC|`};@s*m?)1dAo^iNSV zz$lhK$*0qUE$G_aVQkbeT^sw>ILBhZ6TuY`s8S=+D z{5=lhyFzV%oA1@zgZVH|k6M%8KSZM%TdTYWMST6|Rul6{#6LPhYz0U#+&^k~;K1rW zHs7O|1K({R3AeC@xrO&&nr~J1MNJ+HGhLueW58X9IS^^3=K;NAF2B0AHoJ|9on3J8 zMAAk90XxgOJ-C9J4AbGYtSzmN5}Ru=RQtFR;mu-=urhjXPQP97M=zhz@|`z`MwjJ3 z*>R?;!osnep?Dp3|H2#`^}>tMv9Wbw+N2!vnHrv=*_ArHxaEn+$)nN+fr!pIK=kIO zF+n143F7gC_lPTlhb_NtsnYMjG5wNpmQ({+zn}UvxUJPyCGc7L#uP!78Y|hnbEm=l z2QcIVM(+@~jL_HAFJHclJ0CzqU1a^AcN|=^MPqcdcdx-+vr-Rzp>9I`WT*&P>cc{l z;1*sM&H*PsQ!dl2Z@~cBW~5Xy16JQr>vl%Nf|e%54P5 zT=E;Y@bj-x{tg#!UU@~uofoX0pN80#E?_h2jEJ07QJY0am@45Eyg}Y0M-b;Tug{{e z&dBvmvFnM%Ar*1mSlmZz%1u9yPfUEeOX}b>G&G!gKk@4qJKqNGQyo@YaSd{3*%|XhypL^ z@VeGVwg(M5r)Ot>>N5oHl9yl7ZwtPB`SNPk!~|3ZjHt44?RJ-(V?BL2)s&z-f0|=g4hq}mD`?iD zi|j`=OT-7$&uBO+*+mg1yfQB(Xc)9y>()IppMtzy7E9?%`k${|d`sGR!!&pQLvM<K&%=3g zFlV5rzb9J;5wiJD&hp5A_+JW1+9=Y7a#22()0sA#@Ri3p<5xFdpC0kb99x^>0vDH^18KixNzFFd>eVQ$K(9 z3EK0M$z&^k6(uFo16BwTQyDN^fb3=F0j8*j`0u+%#rXHMa`UdD@IAlAB2@eBd}Pp3 zwDFQ>Y+iSjJC2W!uj5$;h~*=r`Xz3s+?3DMu(Takp?amrka2iIhQ<$5Dfhd zU+f7dir(e9G#)l48=cND%;ssBMNKNtK`k(jW2gFVU&Jt4_v}|w)9nYKuIM#=-N*XL zq28BWR59$-Ps72`548(|kJfte9Ozmis)A36Uc%QJA-tVZIjJaE?s)} zD#qlRl{;wzjr5ueFZAj=zxF`L|BxG_@q|UDX?fYI=A_a2)e`TwPr9(~9T}pT%A@Ksb+e4^C*bv8aq!a_T2^tJqt_3rY%9Gym85a2CF6#@8NUrS|hB=OF;**i?_b+PYt z;{ejJDvY4hIR??Tf8DjDHY_Ko88fBuT5{G&UDVxjP+i?BnhS+gpdGfsP(QZ9Ux?Elx|8SpzQvub=YYfY zJmu->v14v83GMYCjpUg}gzjOv`Z3tAzE!(Ve^i3<%fkG8ng$Y4E~ihAURp_uq`t6I zE4qIdZ`KcilcT2uo>4sKTLL_y9*IrrM<-YVYwJF+b8ADn-U9rIW4Ik3KU3h(g51Q< zU%&3Br=lr14u#e5(Dzo;dbXZ7csI}I23=a(+8@yFr{U=l=@a`PE$|Q-db^n?fuPSR zDJkK&2n}Qz<@WvVq%p7U%ch>Y{}9n-h4vJvVf^KZD~Rpfrcj{Lh4lO)SjRIBoMOiL zj6Ojx`wVPsQl|NSp#F@7(>MNzPFJz#FDIO8+(Y|A*00;J;pBu2uh0=(1cHd95(ebL zAzIxDFA*d;FgB{rEk_-e!w2~V1Yp6o@7%eQk1q+X6U$xA`Ah0GL9C8t_v9k&mSD+H zV(^puy#j_OpN|eu=e}>yH=vIbar*J@@xeAz-ruXh#ZwRi_e}QdY>TCOLBulXba+3k z7_@WI-M9p&92XA~2pR>g^t56}yZA6A9}q)a*>?Ih1=V zrxf5R_Z5}?C%_FrVQks?aeC#yxc5Gwz|bckTpgQQ_g7a7c^H@eNOA`B=uY24#iI-y zt^NCJv0%9-MqThMnVl#Y^Nx3c+}4gH9$~pTIfVyrt%1$=hNZgfd=cGmtZWq({&wDh z-VZ_$zyb2X78fR2SzGm*XQ!t}ndcwe#Ar~4r8n`Tt7x3a@9?@Q>5eor8>ijm+-6;k z(>^_SQ%Se{UPSSisB0!f^TzkE-G%RCKYp})SB&DAsm;MA5Y_Q4Vy(C<%2R-bvHg90 z)O720z|3KHzHNw<;Td43aPXjNUs+vUjXcpf1C2-eGE|*!23k^oPhFhiq<*^Y=Y?NA z7SNNJ+K@die!HW?=({-}*zqro_xG+)*sidrWa=DjBA%w9-@UZZvqb)Oiko_O8f56s zg@=cKjgiEe2ONOo672JSSReeez`#JLyLB8owQ=i|gAadpPR=NUeu5 zQh1H~ehJ;cRVnD&D8^mT5xt(9n>$|mw$ujcj{XSz|9Q_vozRK&$LCtobcAXj+nc>` z6*p|R3524Dq|m{>Z{N5#Y}ieRyHB^gr>1>z$(u;24o7y@A!i1bZus}U1l9tyA2mTj zyX?klt)RXsVP6V{j9*`yH^(htx-dE1Oib3(f|e8%6fV)1FRwV#zRGaV@)lnWSE5?H zSxrK^T#Nnp<~EUYnk4wO*LW)F_%k)9e;y;v*WD%$<9@k0V)FQSs?UNFGK=s7zJC1* zW{?N%SQsT~=Ybd6xYU@OUcG#YlpkY>N~+(0--)p`)p9?QYC%#1lZ9J~H%==?vKOMq zkf-!Q_<&=9BC@@uIf{FKo}(g}M8w7xVdurf#9*6pJ#u1uJlIpdgt9C2o|vh9bOE=? zohB7ZDk{el@ZVc4{ATODZ5*RCc!GK&us+HJZ7=k57H> za$8+f^HYK2ZTWK#aZk=?ku3Yst~m$s1ggzJNATsA(oiV^Pv@|n<0ymG%PloL3yFkG zLH=XhHS^SFTP9D)rF)taoHl}fnPlM8fgpy9PT+YK(jvPY7Bw7RoRe5rprjbl*N)AA zlWx@VcpY5fNt~M2U1c7V!aj#cI_wLbgD{~dPr$TX=9=vBA4ecJwJIL%funniX*N1t^j3Q=CVT)r$njsul`J|vQNyvP7Xy9b=7!5F1;0L)^P0~2%EKQH z4=m-oY>5yXe-dT_-C|L&qx-q8Y{j#f^#w1Wj?}pq9i{(Iz>rDH1Zh?#V?GS>d36?y znO?TJ+4`oCl9D;F53g;Itt_B^P*pT(d0{p$VvrpCe8&k>IN@=AGYurm+O&6fcGzpf z%61-BdDYr_7B0&?_sJl50`{`5s~W6Th?vgjE)(GiU~^R-LjJMHj-?J!%!@^558{Rb z>&8pn%G@hMvXAYnpsdp=@SD-`n3U2HxV zuZLf-u>rzB-$QScx(pe$xD7UHqcGUsq=vSH_T1Jw%-p`Pv3a}ztARqQW0+k##}2y@ zpo-nmJ;0l;58>P*CCJRHnYQ6R{Bm5EMe4zkJle#RV&ZJ$v?>J_(f4U3y7@VN6fB zxIP6b{CRRa-mqqsL<)$f5V=D*6rUci4rzJ|YK0Ym)D>7YXuQYeg{ez9Ujg%Z$_hZtGFuVTlmfR_3Gz} zJb#Iw*nqnm^%4kLdrHpX&b+>;rJ;cw+j*gFWZMS&=Hw;abW&%8)T~AV-`Riy5gqc`_82SOl{`>ar!-D$Db0#58 z1b>AGF%h>{dQ|b*FZ4@Iq=YTukC-9gZA27TNiLa zQ&SVXY?$4eK=q!RNvg8~cU2i*MykXh0ppOu{cZ z23cv`5cBiD;Kf}q)K-@aW`G!t1{mHld?KY!A! zUY$^-z&&y7*fB$>!^Fg3v1?!-e{xd+Q>Q?rauzf{Y-kDeq(j$i@Lf}8iqU}+!B-zB z&o{+PcJA8cj%=#b9QIvTsb}<$7O5x^V-krD`TA5bLBTIrO2B!6K|w?9SzI-OGBPrW z=aM5qaz{J{+ns#g8b1-tE`ea}AP5RbzbPwIlrukBv&`~Y=g@aRORF233Js_1aUKDS z0kXV&<;v{b99Mk$dVBjXaB$Ckd*_NHEHy3wTpTv8`_?V#U^bF(ICv+6H)CVfTYX2x zMu8IW^72|mYknCXJ~&RHC5A+QLd^b%wOX>}s1wrAp!uAOW~JT%Eb5@AZT_9q*ntK* zW$t5qTFGEI%t4yUfrl3)2O_hJTJ>bLwTM@|b)|x$qYZ)g3|hgPlJ{Ah2L^zRtQ@sQ zZz^de>0?ntjaWQE4Fy&R4@}G`=ObDePJc}f!=5yN8)jwLkOs@PxVU(;HZ2;&zLsD~ z=MTGsY}TwFIpD~O4Wd>_qk`1 zf0$G9NGCw|>CVlYMAcy2KB(9qWUfc97AnR{M>G-7I2PZ?Z@;Ve;ll?2V?cnpcAd7= z)O>{)A9iU5LKTBW?;_817+`qEQwIuD_oMz(%MwvgP*~XgI=?%#T3d*Y8xZr*Pt{=U zl{JpLe_ycSCc#2Eng_@{p8*Sz9Dg*&l0xf&RO;cK2eq{HUj*18Xywy*?Q%fz!mJec z2W%$wU_?sY#c?ug@Xh^Hl zk~*a1Lrta%Zuol2(*jmx+ou8k+4MogS???AHWpaJE~YA~fFrj(^Hu z?GaQq-@U=*-n8NT{4_c4HTeC{6-B@=WDeqUNhyWN+7n6uj>_&RpL0>Vv)#5{DHMXO zVf1e?Q^|V^0IT|F7SKTDai|nQ3A`5D6Z^!l&+x+ejrmKcqL^nsvTkmxF-fV8u^^;+ zQANG=4cdyT96586ACq`lP@_=HDf%9Ar{@8Tk5((4GwXT-(&O=jVV4J=}y)dreGMc%maOyJDU;MRS+Ai5OC0??WN zc3I|a%X5j)oM;;GlSWiyos99!h*l;fg9{U_v*Bxp#!YoHJ9 zzgZJxW%+@5;4>pi`}S{fbC|{d?0hx^dbeeO#1y{*vayxCBR8vYf4MyZzaYQy!t4Y` z+RTBV3$cA^c7v0+qqQi%l2oC8f2Vxmxe(bsYrMnv*@#zWEf+%{}Bvprh%pZ zC17ZyCOMmI>u*}kS=`s=o^nyKxSjjFDZ*B@1p)BU&V49=6WCV>w}nPJsx$k_8j@Xf zWF-Gn`_5;fFyx`5JWJq8baLJ0{vLyPH)rn(lZ4xejFre2>_`8{!L2J+t)g9Xys!G| zL?+^8gX7{dj}DC+W)Rda3BDK-wxJM3UsP7o(b$0jELzi+QoLED#A}X(@KpT31nn}I zJ&dqXpkb`tbg60l^x0=E ziOnu5a!n2Is%;~>Pptb^?^_%T^9k$`1@6K8VgE_3-O84h38UX17(W(XTGZnPy=uw^ zSthMR<%Zej6LBc3w|f^HHai$;BQb~!TkBZO$)j3(ah_>>ky7;-%7c5|IQdEOgaCG3x6)OTo4&TF%sH6+8M}&Jf zGBXP0lG>)zXOJ)4c=N@(@Q8?_N@KyBzS)FwYx!|w?3=13%KV&jvD#=={n6(aCvHKu z=WFJM*DQ#5ZPq4Gcdm;e2*4GMl&j%|l*|#7s4T4!&sn1{p&{$`yO7#*DgIsY(BV7#GpqztD`>JLv4NQzCt1esviwLx1=bA z(wgd`G&l>D^3hqUulK2{<`&4#z>IZzTFm!~y*yCep1ru@j51b2!?$o{M)=*k5-W-( zq8nfLAW6&k#h}dS>(PySr2XM$92R0!M@IMZyBp8gMW0>Cd7vD4H9%$tAxKTX&;0aQ z-?>`&*400_dlgjST1Y(Y#_c^Vv&A>_UZMj9X{AyZrDWKvSDJ^L=Jsl61Qkl!b`%S2 zuo^sms;}`^ekpXQOuI!LP%>m_6^+R^bhq#`>U?B7!#->~625y8}gJ$q(EF+Ti+6fpA z!68V*e!1qYOb|y{83mp_w5*_7ua%|x60RH*Gc(efdm6mHHhhgFx@nT}SjkaeSDTef4lz<8cVG-kNyo!q`rp~$F`kSHuh~eI=OFW@G z5emB|RW6KmQ~BmH62yEZ|2DmRoyD=y(FDO3OeInGkQdvuxwT*3;h{rV^-4?eWg zO2$@W;Gmsfj0p*e)I-@xwfD_8NZlp(n)s9Kev=u5Q{s|Ev#Cw(!r`#C>{LZfs+?7L)!#ITw!`^EMaPulaz%$# z+vdrGrQ=)>TPW$v)((~U03s&9z&DIMYRYv6&-wOG` zx(|kjc3>*Gw+K2W}QnR8;i#TM1mMlc2)9&H@2O=RZHMKt6$6eB_sr zj!uR14w>H%TIN;!*Cf`=<_6??muH8Vfcn|>4Q`W*C*06;o*o`@bEO1H@4d_DN9Cb3 z1Slv=z|{c7{7x7E=MPV}N&y@oh>)lUC@zIJjnIwR8GzWIJd3HB8Q^O`wNEXdT>_(& zW-XzP<#y#rwz)!cm2ceta%g;;vGzuaQA~Zwd$P5m7dvM{lGEIlZzxVu~mugfQ{VVyf zYAaa;s*i8pL^?qZpylPM^L>S?6pcjrah{MhnVehL3Evh(ILB6CVbui@NN9%6o^f$u z@%hF`etBmTL0s~sNl$jj^&YXx?8&}UsE^nk(CcoGiNKQJUh;|FO8*H$(e&=_^FPm+ z++i$$6`PF%r{8&EUbg!{!=+1?rq^;ld-g2f6|P;SSfI%i9hsos>mA@$4W2y4aBRW* zF&Z2i8Mz~jhEwxL3;Gx$617ct7A{2ey;C*Cx-!CZsr~%6#t5=k0hM1t%MY?MwihocUzSp zT{xJDyf8L%Jxhy*4X;1GAdxmOtM60R>k8q?fiG~SeBX%WBAQA6ME$`=r>!ik>Xipi zU5HxzoH5r)K^RV(#A`|Z3beyN%9rg80aG8poVtAjH#Y$Qf{bifmei4lgNhEI%khs3MxuHSpsg4y)2u^&>!F55+^uatIXAqX5i%9ia_(XzEBw=vz zbTaqkEe4;A_s(h!-iez+Uhr)NYPU)LSf$E4{*%9YW*3H*Eh$o9A%TIr&JNJ|tO^|& z8G)_B7HFucN%flf3GWWM;db*_o$J@HkHR@BAA8P)f(KS%JqM&9PM#bA=;BW;*Uv_1 zhIVWJdiV5v-t-iFdEKA~CI#Z(GD=EH%FBI>j5^@!OPCgJbeb|f+x!W|J|J;D>hdqN zRb(t+(Re6VC%0gHpZye2V>j)G7X<huMV-nST@NP@1xv)6UfqXbQD z6Krvgz16}(Gy%RUW8#a3whT|4h&Ru#dn3^yTSR>a-xs9{j}oc;Hs#@;Wd=$Ly5(*1sB$8hqyIYGDDa?K7gteJ|Kt)4@ksGbKj%$FI&c|l z8$Pt<8*MJF6I260?0;um<~4&TW$e~tq&ysSJ-5AOzWGkV_vis`cbrQ*Oyg%5LZb4O z1suldQ*3()cSd6JHT<2c#O&+F>*g_ zU6&ihS$m}-t$I*XlTN|^uOd-VUysDHwEaUw3ae$4)7G0Qa_aVEH&|-G0Zre~66raZ z6g9Udz}NZb$2mEPmyQh{^_m^LAGKI`mFKR9RRAZRNB5uSmFz;f%&`EwjK(w#=EoOS zsxNSA9hVsJxTHI`Qj#{EiUGL?QFw?O6urAZ%T^xei2hure<17?N<}d??;WJ|D`C&J zyiqqLqkUWv;P+-RR52ByRQKg63uWDMwTO4YMRyX~yzz!Z6;-~gBxxgA#cuj^+%-D@ z&;AOa(BBSkr{58Y7se#JKkGmyyTLqwqtX1seAAS@h!;W9qz-MYf_CoE- zjelBH%0E$sih4#wwY|qpakIEZi#$6mb&;4gNMA~&m&OdQ(fGz*A(}$J8}#8krQbo; z^3EQUvjcnu`N=a)^NyxNa(0dUQ~#H*kjZM$P;V5!g8OD~`h!@|CH{Yp!9_re1aJBG zE>I&D`WGy_5@ho1WqAM!KC2GkpdZ^LG7v<+qkoWrUw2|EjL0rQFn2QJyH{_7%lF_cu|R zv_FUy^`dAhKv2|jmEFo(#gsp4(A?b z^wto@fruCA2-(Od5>xsc2W%4%P>S#ezcE>m7~1{=i~9ueZhb*)RJdqcTU!ka28M@C zc$W3G8t#H85>!Lg)jDqgIyePxY*s4u z>QsQqhr6?Xo)iB<9^(T@#g_m%F2iW2o}R0B_5*3G0n%SnPDzps3u-EWx3TM%MWhdv zP1C>x1^Hep5&U_3!hB)%grZPt&6?Izk?2p*^`Q~ZaM}Mt7NXRNUZ>H;3EA1iIC?tp z!z+5!Yqe7Nun0{IIb92i;I~VvkQ=bS6J|>AdUl}wnY!OCnGaQ5T~2PU#lss?WPgh{ zBITWXylR^s_u3PI2OcTcudfZ&9uQ|qZ(xh3T)MRPm*FCpj8qVcdyI^Ja;)WIF=wt4 z9*&JS>WV)z%YlxPVkqR07WWncqnVxFDNk5G^ISsxfCeGBLjivN2Kok{(@a&ek1D$DHQT+c0B6nVv|fY&a|z~o+k*E+oY`ayv4m~8h>204 zLGJ@w*;6=Ew`_6n3kdM)$YWwxAe65`@2paaWLU$3Xb1Y2NAZOdg)#Wd@rRW|{r&Yl zh^|^GE{?F_#H_3=u?mFu+_uVZ2bu-jckJ_XG8iLTRcMhN4aFmN6?i6fO!)OTdiob? z@Zp35YT;@={T169lz?VXf##qAgp!G+Wuv20o8ID3{XG;aXt6;wg=b|p zK6>Pf%wxj!z+?`sH9MNqv_{-%W_I@13*=pMo`6-*`%W4!`SDqi|6O($I>bkxQHrIM z)ZX&%(r!(y#mk@qhBNKXBr`g(2hiLHAl)OiinHCa?1DK^URwro9AI7$ z!WeE-_CbYl8wFaBDKuj694M$QQaL-tPGtdR&^XQ#5*D_oJ#|vOa=8^)7c4-cjCQJE z4Pt?eXRHyEKoUFW6$vFFb`OD$43B^d9pA0b=k>I3r&C=7HW49dGUzn*8tG59xw)>M zu5zAObNBU=*x#c)_tR=4ZC)U`IWQ#Gz!qhYts+@m6ApR1IM2sm=^E~*4aDQ(4-xBWo%Q^5sz%Zc&g(dot*LR#A zh;-KD;+%#?Rs$1UQd65kUDB|J-rloUe1Qe9Y)?Nszv{DYxWEmcJ;w&I{5Gg9eD%9| z$Y#tx{%BTwG_7*^uFxO?av`ih^kvYxpUSCvI~1)|T-gZ$+^46=NSM~9k)l(e2SRm? z<{V<1nU$4B{agGW4;Jdfd}(J1@&W$8dO z-!KY_A%fQOFB*Yc$w&?VN+VElHiCl-6SLj-uYgB`H8uWnlvTpUl;2DGv`yPTcr0C| z;9`3?kJljf@DrQ0pP!eFEXE(8lvnn(G#0=o-qmmY(g(`<*d9h}&u#OpGa)Vset^Dm z>PIrC@`%8vsSwhV;*!eT4lKc=E1G5A&(%+i5}Cd5fLp+c&kCOXt?=(617`yA5c>rh zB6I=9Nh6TU&2VvwZEX<`hJ8ymnFXB+@p=}=6LtrTCFG;nTJ1QPKJao|3RE1MR8)%5 zrwa=U6ADWPB;eV}x_S$Cq2$q8v@FU&oG^oUFG_1o!?hapMkxj7MmJ;L@*kq?a}?U= zoQr%=NFShUc(mzPTG(D5L+~t~;s(`Q`|&74Bg)PsFGOzM+&jtwhu!}{tss)2pm;Vx zP_2ZAhv)IeJ(fqY5o8LuVAl>RJg+&a{rV9$wTHuCs04;ExDgi>N*oiDXJzgyD6)9M z=kmEtj`b6=5ON+gS86?b5C3sM!Pj^6#QR}jaG2Sv0)Yw68bHSx;CIUuyX%7vx*XWI zV3V#c7jnyN3LZGoR~{-8Mb)S`W*9`tI_EEoJ4mPwdm_w6W@Apk&iX$O+NhoE&i6Wq1>*Bd|w_-*?15-oe2E(MyKgNV{%n`fv<< zrPAxrzZ>mYo38W9!%*~>X-Fcv?0rjqW!c5d%ONIG^S)L?WJU7+C5C7uv=GB%_>qqF zvmg^-2*TRP40oq}LNcb zik~eo)?eM!-F>+NSM+^CeeiqQzSmE8oDe?~+IHpLM>L^naffBPH}u)<3Y#Oh zmOv_ir6wMW7)*RVON#R#Hj3Xs`7)YG>XFP)a70X>oBkE}s|l z!v<5Mm*gJ0mNBR%GT7*>F7_A>FhpzRq)yfOWDP*-PrA0pemkyU@#1)SIfvSZN7g7i-z{qRh!@3 zjtKrXk-%CBQ}C}3TUwk%H8UDLWpmgPlsMS8M9ds*%v3sl9~^9W7>}yD>@E_Et!eFZ(v9BK_gDE8&2=4w_(_`FN7pbi!JS+=Ocpd=>~$;s!6cnEa<0 z6CGwFIG*w6x;JTZCk$EFnJLKL6f}7G0)eJ~mT*?K!89UsRIk>?`7w z4h0Xk2&duFYQ>k^rwB{2{)=VjCY$vdd{ePA`9?QXY`V5i zMaBI4kJbf;FH)(u6QJ!4E5$11-Rwg`-U$lSO}mQP(JyjxvkxUd$U$9Y)+|w0@A^+S z_;kSRr{Hw%9E`ywUQQ^2tfYxKW!6^M`BoYTfJn2p00Vc-I;vB=rBw6yTal882$H|t z!=OL9DDu`kkxQ0~c-gfA^!Y|PToYPZArn_`#M>NkE)c!J-i(}#@uKupB15f~q%?9g zPt-oy>^TxSruw(~IVG;yIeB`eq@;)lgQg#Sm4AgCWWR1(tkC;%+z37?ED4$~<|&>L zi3Z)H61|p_UTQ60G}V9%L?ALZXVw=UGY%Q;Wm?mq*mkuRxL>h}Jv#bGK`O-Cpx6^M z6m{kMxI?$C$Xvb%B~j`HrtT5`&Lh1)%7j4K$q+PxS;IXhZ*JzL-iQ+zJ8P{Fj9z-X zbUUdr<*&_mdKuW*E#bQBu5fDb=(Sm&1e0f3X{=KHqluAj%X5BE)UCX&3<+Y5{34QF zY#>nFGK025`1qzgp|kEV#b4 zAiSilOh4RTeL|fMLVt^>s8UFU_$8vs=l3 zercL!ggy>`rsM7p!GS!BI;Q!wf6V(W4iF1Xt7<5?TuJDnLWUZOGJ! ph$6UP#US*@PFX~d#{clvcv5^)aH^d{E|>V-_U$$^Dl~LD^A8(pak>Bi diff --git a/doc/design/img/hermes-node.png b/doc/design/img/hermes-node.png deleted file mode 100644 index 96adcd82050a7b35349d0e4e6f861e2b50cce8d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 124515 zcmafb1z1#HyYHrv1`!m5K@{nd771xVxAalx|Htj?k+{TTN-JG&LM}phyQoZ z_nmX^^W61pn3+9m)~x;Rwchx>zjuNZe$>3iUCNg;DgMX4T z`0&71O4A+y@H!BGNZmI1h5$eXyc2t?;*z|(;Q1liXjTXwu>pKq4^CR%?ax)Dj-c*&B8f=;Df#S@BV4D#j zskt8q_HyPm5}8#@OiV2K`_wlxg+3G%6lyi#C=l!;8loUoyTCxxPyItEEp)tAg1UP@xQ11 zUmcQAg{cTM&L~UNEL7dQ>dG#}R=>?e__ug_JxMzPk8UfE^NBH2i@V%nDh_(LnNK_! z3UtI+3nT%|^6zJqbIt1`!!tEHR0)5#z`X%{W4}Z$nsB<1oSlOHF zaozK=6zhH^X^GE5J3In+U2)-FR6&7(Zs0{_iD@TMG*i8nEd7-{QqvG1m)w#)+A#4M zx^-Sd#GXD7qSI^{)u}7dlxJn58g`=wt``4@DUtf#p7pTIt~q0tI$qJ zUuLnYlC#~g6q#>Se`I`d#GM^oFNV*nlhNt$*AANeugNOY?urcVPJ99b^GQi+II&dF zdV`FBH&HOJ@~Qq_OLAJX<1Rya0Gd3|e5EKm{8K86@(2mz8F(XkJ2GP>bW4~-@c?$# z(vyTK=PEP72wJgvnNL#^4Z~~2kb-kemFPJ5?cqA zz(nO^#BH*EWxpEd=3Naw?dowLSFk3+^0T(XKmrneig*_CRP)4aoUTp5tj{~>PyNEo zWZ2^0VxqhmS_qrlk*RvvWOLEP!TYRfjXZ?wl&j{GytKUg-;}rdK}m*vKkC)P0t35f zKv}FrOD?IR2zz|{?Lk|T+F$tVA;f0b?e#?G$5{M59<-p4`7Z_>Sm6wg9g34-!I7{i z>J}35Rvnw#<*N*C{sAFgJBy&>H^6dpB^8I;QLyE@yw>^#+nv!nR}9CD3E3Dl`u>}a zMfPznk4qP)cyEodYokW~*@>{=F9TbhA=LI{61mqW?&4jQ;vo>(?cS1UYpuFoGeHaP zoXhscx0q8Dn+K1c^ph@}m`c2b;d(Qyr-p8c_ebLK+|4^5mVZl5Jy)1yz-)@ZlOpN(JP!Du5Q^NjeB4YrPG{0Zhjm@sjF(&O}Z(>-%6pl#wK5r4$ zN=GH*B*9KZ#Kzckz}UX}dWma8`oZ$2-r1W%k_Cz{0zYG16)R?Hp1yo5CMMbmF4aUW zPhw#CFZ9!y`&>W#s`5Rq%S%Y%JZ0N&akd8lYMq<771L}I6h$}zp19lGOH#qD7fWIn zP#Q>5Uo3Tf!T{u#sWw`^QHXxwI2!h)ev)w`X8Jjhp5FD|#B^&zJ~)!5F$^pDO&0e7 zV<;(#`OL`yy?-56(cN-RsFv{47(K1}TKAB4PSWkhUlGn?K^d`~t@=t%+Aj}XYfT7# z8I;jqJ^A)=!Ooi1^SL5kJFV4HYU(P=PfN4o#Rwkv0$Ft8Q&hdkTJ_jR>)Y8oemt0^ z0~*7M6QJr;0q^Oz8t&DB(M=){Z+6HtU$++ODpw%e1_oZk(rhhL#>X?ak57i*hL2@&UJ9 zX2q%`HI_91CzOGqWjVMe$j0 z)!ktErrDbD7MSSd+AX4?&U}_&xPxjMU!YPb;C}wNG=Z}>*F^J;eX8|e(xRnVxF-jI z9h2{3jjHfPDTa)#7{de;Lq}@ovhlGqA2N_TNPr^TQdmQg`t|X1O9}U)LP`LteB(z? zGV8fiidH!W@hyBNewjlVp;@RmAzq{2Cm{dZXU-vZf76V(*3vXqvVW*LI_@8R8%w* zS6NwkCCf_Fd9yw;Ki_OIUD~>s)9-!0I};&r9zTN)8gS6ij^CBd+~g>k2@Cto)a^LW ze^D(`Yp9QhSQ(zjlQ%l;Y3cD>U#+7@-ROMypjM`9b-wrL(IYIsUxzn|m9|~Z`||}# zXB^eHN8OM#KKtC6xU1Kz{+DgXeT>F)k5i9YkLzr^9Qha3hhBdbTy4Bz{s5IQnYVr+$#Z zB4-*`(wHD_pF*Mi3b%i9Uo>9kV%-KWHfl>m-(*^sHL0bWTjM%9Jv}{1@T}LJu5`{= zD|`%pP1HOie~;%RM2R-V+^7T_O+!mgdLO+2XLu4H)@48f1ULcxm)_}qo@ z8S(D@1qMJyr0nh5=CA>2i=or&6W1apcDmQ}RUdt#s?3?s8gLYqt7ho&CP;@thtp=S zEtA63Ey}^sSmm!~7!8`D@pbq$SNmH`5)J$0nym$wz8CC3;CO3!y?r9C;8&^8z(A=8 zftv8H&d$y)y%v8V@6+qDx*y9lLB}GV0>{z}<02i*lV1uyYreI5Vs4TALNUXB*`bIL zNH1-8BxrmkTQjH!Zp!-0AIPHPrCitU^;)pirZ|lW?})rBkzV227fuPZ=o6;qYADak zX?@yF7PQTbU=H{q7r$MK%N@I2V~ecYKQCxLij8+t%pZu)S$N?Z;lOHh<$` zNww?TvBUK&-q`%s4qeO;@8>KmEVsIBB-{euQ>DO@=b_|a$MT(_i)H`2gq8`}xzn>q zzm&1fgYOH~BadpdN*q&okQond^lwH(5BB%1Ppka&J(7rP4n0iwf}`ud(p?OfL}28c zT`E+hMK^u!2R`=Bt`i{vzD??&t|01E-&3r5k40@2wjv4;xZ{4REcc?94lnw&@fbsm zOJxd$fE9MK%k!o2a>v)IuVeEW6~XDa%PoVh z;CqCAH~W^rVUjpF&(@0e?7oq#XzP|O9U;96Hh5-Ffqj|e)sWD|(|~yzpjWVd1*vV<@-PmBXWdLNA5YN)~wRcs3j|IbweS|R&a40<#d{DY;?AP^%9$n!uP5V*T!5BlGv0)L? zMRAL_m{$Xf%rHml;@KCb=Pfus!&l!^WrKAb$Gu`y$Z;tqr%0OJZ9i%gUHHEULYl~H zRs85C)Ft1CJXY3>5jIGlz7?}$#!RU3b%%pU9*v5E^O7N9Oi zHL|zf$&J5N-2CQ8+e6YIB7>L7F6W!`0l5LB0EMTG{v)uk$MT zd8_wik9T|0>xuG{Sgzj?zoJHA1w)RT<#?@t%(O4Rf1xtgu6(}RXk(mDbEPg=^U#zN z(F`}ku5j4Xt26O~!8^U3CeG!nzr+NgZDL1$!HNuBEPvK!`*nCE@8A{VOZv-!ja}Y5 z-G_=nG!}K6FUiBx38YeI;nmc~6gkH!_3n#79Ht4i&JK0bgrVKd)6IYc|t?qFC8iw%pAlzI|EB6218?Cefbj6 znFf0kxBF&qUsYny-mzVJ@yok9nYUcvvKO-ePWil|$wAy>D*U;LJjJV0QzYOE|EJo! z)!iQyX6!`HIyR|)M9HeGuQNFE@NyF002UYZEHeC`&lRQ^3Ys4K{Lvwbs)dDk6Px&v zk;biz>^?UUT723j!lR#1x*p{1k-Tdkl2Fc4bllbxew2`PEhHZ074A2ak+jaNw>t<8 zLTT@$r8BnL}^o^r#J-!6rG`?x_<6n?@9i=S7{b=Qd@=NO)Ei$+Ve2$4$>h;CAY zG+#F#+-8cm&a8b0<7!PS1cpGur15dm-BOSs{^;6`o{CDCr%cz?wE+&7?6^?Qlbd7) zJ@z8**WuNDnVX$4fSDv$?}m3V?e$AyEJtjiyPK-*BD|a1_KVQri%=V<7L<9qa}Z`= zjggvB{{O`6{~gqyOc0%MGZrd5G}?I)7#Jwg8~TJJTZta9v9ZzKeD#9K9E_F&yI8@O z9SP05q#?6*(n6}VgoI%jOJ3J|&P$#q4TEdFQ6<4Mqoew&?(U(iKctF--*NxWj13Iz zIdXM#n|+Pe6rGru82RbO#fB_NW8!u6WRdy~<%bJ#~XTg*nywK%@KMxYf;v6K5Owc+$NTS0&#8lugVQ3N^U92|u|=9t#1dAGk{yqJ_X+D&7CNo}T?933dbyb5zvQ(vp#Z zm4nzx!w2gf2lR*ze8f0C`xW^#h=H2AGp90M(9#C@ zBN`OFn*fAL!X7?V-cf+=>_kxTqc0z?_ty^j1qMEq(SdPLXZ_yqQ3F?fEEm`Vorj*(8!x-PO3cjA}_? zWA#|!Emgp;dx7|O6OVq_R(3sTsU`rR$Ln046NL<}qey@kHUujO7>U)^a?ME_Au1pi zRyC_B9Dd2nl{4v+*0z-XnHH23LZ<#v-JYY*fRWm^{p1;R@O>=cB!}{ZR630ai7v92 z8K0^B*WuyyM>|vznX|k}AH54tCi8Ck`^j?_8^3l3fZ5Wdy#lM~gA(**!i`N|dhA>E zU_gbWPhJ}jdUNeA3aA{!Bta%^wRAPmq<-D*KNi_A9KOe>M#~%jWESxYSf_rvTA3ts zg_{ACf8mrs?uGyFL5z`j@A@>~n#|5;&j7hNwIoJwatmp4q-{^HA<_4#yHhU!t&*ui z7$ySQ>(NKi%1`lBW!?M(aXne`KMLrbjDqZaNAZ!YFAGReFFLP7`YskQ zFkNzVh<(z&-Bs%>^GN=}Z+^^qIiU@|0smN@T*iXTV3Sx$+B1|Rz5?!9`Tvk8K6oKH*jTE^TeO$~0Uye&|dZzlY= zE?-)CDMYu<`7i1eQo{dI0tzY)6UXgZKRB6)gjc;)$s>e$b#&aw>RI2kuQ1gb#)9tO zy^6hpP8M0E^nXA6iwzEKn8J+TpzVSFwa4acM|1c(0Z-7hh=Cc|=ew=c;h-QrN2)xva4gnfY4|W@pVzf3`L zE&>Np&_W&7cY~9;%#S)h11@YIW_L98hP+_!e39l^oI=u0-vVJdT>hRlVUq!5zEayO z+oCdQyW}BS0hLDmu^`fUeEj)dFNuhW;q`C*#jd$KGcGQ}6b^xoS?yzQo^!FBS2vdv z0hxDYX^NNIPl=yAz*%;x*!J>$`J4bse3|Eqk>f?uwK~Sz{ucw!uZ9WSw5{7G9X!pe zZ*GGE>Lz+QD4QMDvveLT8gB@U;1U4}@#Wd&5em19EXJ>q|E?VgSZ!b#k=gvY*RP^Z zC%?YcE#i>Zhr@XjD>DKibFc5)_PbAAD^8(lV7YZP^*>Ayw zTb$CWL591a{J7DiC9K6gBjGO|=TJF9i)H?$#?pj+!UTljT_l&CsgsB;oqj!9?|EF$ zqlE=nHFO3ul5oHB%s3?k!h#bsp(s`aG*YBhxfIPC8yo6+o@bNldJ7Hqc_sEwi2m8B z9>#t)F0$&0v8`8f&Q`M;y7>TLcG^N>uGLQS;`lAl?n~X!C(^X-1?xi2UyT1Thd%n% zYd4P~cw@tOp@Z}vM0sKPX-q5)l25HC>!zB{o~u7I)q|s*%^Gd4Rh;Ec zeV@!MVVR~mLKa%c|Lk&G`*07h4iUg~?ng?IO3#Trim;(dpkJQf_WPrlQX(YbRgBLv z98+?dbe=>2l@`sJk*o}`S>VWqgoK3Uhb-*s>Iw$zCF9Sisk>KFMkb=!(qm5yge-$6 z@AOp1uaImE^hIL+a+91_^wN9v%F6bN%0xcJcE7if|F#S~4}65F%Ao=XrEwlAWPZfY z&+mv0qWM?GPSHZE9D;~rj!iHjk<%;^&Nh2}xyR@wDH&j$QB^fw9@er})}8vljMcx5 z4nmcJ)@&28t*FV`nwy*Z`uc!@c=nE|2Al^7BbRBXKUZr79#xJcoEyA+eD7WmX`8U) z@ZNyHvuUx*e#7R=Q>(%loF$7LfhzUw?d^`wH(@t$SeuQNRn6HDAKAT;nbB2NR(5s0 zTx@Z7yt|%mgHP68j$qETxWmBvuBxMRyH~d~l*rXTwarURQiCVX5;yl64V+SbY%F4< z1_uX&|L@=jcibMmq?YuHSVpj1S0HF(9|qYpkrWUY=d9zv?kty z^Jw$By)u0%k!3KNDGsi&n35{n_y2rA4HQ#puqqvKO-+sQ+(0rP2Nl(FQCaKNMv{#L zp8$1tJHpnOiaizb`k>eE72K3-)5yf`;qktk^10#vE*?Q2KQpE(Xy%?jHHptRw^is zO?l&bOlKlx_*PU@^z-LefV}+h?yjYF9*4{QT#Kh>XA4;khtq^?Y6ridVLPZ6i%~bM z$7)OPiDn`-dR%@sGNJ*hz8dcK{15wbzdFDL!Y}Nxu;jHc@&$AQ6FmEEyszil+l3d{ zG)+uQidImzc6T|os4%jGs=$RMLF!Xd>U}GZ@sL)I)6&2>P467^u^Jm2&(ECzU0vOC zII0PCX2o)xp4aHoZjK7WU~R+g`J7{ip{R(+{KA4HpD6X1zT%F;RGD6x0v!+?7l(n0 z`sZ~DyZ#;gHY_ZRgzQ6;(_o=b)JsiRfxbuAbP}Jv@uyFjsz<$Qg@uKE{rwS?0)+~6 z15Sx@N!+qR)xtb>NW}*Q)S3FA8~F5tib^z?Yijfr-_uZ2SI>jIEE+X^PyRlh?X8fV zZ(G~lmWh-Y38<60>x>5zP0~E4rZh)fVCQo-HgmOuQ<~o^Y}#(s^m=rgT}CaFIzvcz zXj$T48TEnU7el%CGrQvMF4OKtNh>#pQaI*}e||v&Z#NtcCkk2`40-p$_BNA zR&@)xX}%VV&8}I=&j@j!-;bnFSXAU#V`*fRGqX)^-Fj*4`K-xwG;?(Ld!Pg|VvI_6 zTwL5n2DJ=@4-KC?Ey;t#h=_@aA3vu1A%$41WN6?*=?aCc(R*101Eb1K;erP|Salj{ zH%ZqW+Ju!2=GyU?0QB<$WcG_Agj=A>7PZ!CeLiE}d>VJu`Mmh?%;1kze)sd@4=mG@ zl}nzmlo{?266P`a4^#x0yb(-4=xuBcK!XXK!~eVlXKWzeKblD$D3)3KFGqxT>gx*m zQj2$|w$+Z%?8hCko>ouTg7Z}n_E4&TMeX3t<;huuc>URwHe!J^aTs*rbh~_f7^0QQ zo|$uHfJV^5a5W!WA}%iZP=s=Kv*4&>m_MU{UjN%Cy_QAp#H8nvV~dNrii#PGhC4eu zgS80)?zN@CF;XuYgRlcln-L4HxuwMs`y7t_%f$iVwawSruD^t}v<@V<_ zJ+*d&(_UihcLA@P`WbHOA5wQ7ta2ZO#U5Iv0JwR~+O=D7oUJ15%7^Mj_?tY)&tP!f zwG~ooB!uxRC97-HXzW^8@YFFQFHN3LA%tvz(9%AHB(0*Uaj#LoDXIZ{x!*u*cQ$X* zK#z~s1h#-T$;UjGd?&Es)BJu%XJ_ZTxn5nyvbvNv(Md!nK5d9TcPQ-i4QWpT%!(Kn zezb1OzB_$DTvt3t9)!uZKk zG&jj?K?&5yvKU%oZ6_xucd{}unQY36E~d8ukQYqNGvzZeP}nNWP`$2!>GZ_8bwWj_ z+EG?f|01c9r^5u7x57JZ!1HQ`naj$@w6rXZoF%1%epX4j+T=0`(n7djEOToUA=G^K z0x+==5;9rNi-o>+)la#RkdDgbzrJR+ED$XCQdO6B!pXZd-@T#r=*e(5=@gQ!?$*Jk zQ;CUv>&}u_9&AxZsP1l^S-TPc%SleId1ot5r^#_O%vpRGa=17n0s>g7dwaJhMtty0 z@sM2Er1N2KxU!Jl+YxKm`2XSpxVtya#3~i2@Sw!0Fw`(0QIiS1L@GbGR1ZEk)2U+$2SxIkdJ zuOV|FK6-_Dxb?3cua3C_>cO8#W-RZ$sOtQNaikA6%9fto@P#qbR9O7=D1gEJL9zL0 zH#U)b^tkxTL{CHGh`QVceETnRC#&HF%YggHrMc{WmA3pRFU!Si^6bo1SWO=X(KPXt z6@BrChC@L@u`yi!?8XEVYQ5Q*kLS zKrieICg%{kH#;LgdzNG|e0G>~SOrc+>>Rr!gAa8v^amc`;8Z(e83bxDrE;1nSy&X! zs2?=$u-yJeLqH29oL9&W%?Fzx3IQc8)oZgp#;bI{QUVHWnsUA;7eH-Z6>P`zT) z?og73VoU;>`GaYlFj*M3t28Pd7-@1|qiYqT>R)Sp*}vXLbPljd2$PN2FzIywUP&)wB@8)&9HL1_nYMP zMu`YDLI3|7Xb)$m=tI+|^}H9R=uah8nafbcin3d;3e55@79g8P`?+2CYm*VK@dFy< z6rOqqL4?-l*g+(Ff3!;SErJ|I+tvt4jZf7!eb%4lM<4Kx*8gm~E6~@55>NWjz5YFH zTHSv02Oqx^N^Jg3fcdAB-OQNnE>E>0P9~ak8{zDlrR9DLeRt z7ss;O5@a@RgEHdp`)HSY^$(=Zm*eImsFJl|9%Vj0J|MU}%f?7uX~AUvfVJf`vPAju zTZ6WItD9m4B$qeO_Ee3BVAyF9UE>WoF%@30^U}o|fsI6!(6$6mQmVK*UF1{olCw6?S*HE=Nq^eF|7}1Y$QA%mWyouC) zfXDGLfp1tBlarfUuMzol_iBDF)JfQ3?gJx_*~9Q#J96GzjFX+Rw)4Y!_NQI5X{kv+4(*fPTsa6|j)Cq6?F81j9rP}tjOFWZJ zdxeEhgHr0j$a?cpa@02pz3n&H~2U4oxEdOu@r6V{t(3B2zCgWJA<}#DLv3(+!+dUc`{8wcLA3Zg*j`(jT8|Ndi#FsM42J4)yw|>nDLNY?TCQ{sw2f;f zSz6ZtxY{<8eILgnPk>}~5X<1j_CYU`ceIK0d};Fd7q>7JqQVitr&U;CnDG0_Yl?kyAKco3zbGsDFU>+>67G3mXYcsf`*Xr6RH=l_})7UPB6nS7wt?~(dzI{$z zy(cBj+IXZ;rgoW|Oks7X0ij6|9G?+I6=q=-a?T>S&8Xl!@r#x|c}A}4IaJTp$#cy2 zz}BrFyNt|oUwSn>JDb9afZ5xK1`;JHCg!<%h(fJx7JZ-w1j5_AJmtK(z(D;$#Qd&A zr?u!ENT3I+}j#*c%mx^YqTWRMaSCJ7=fl|NpK!_9=F307>(q}3zbvu=Up zp}6TSeo=u;9+Cs;@aDY$D6}x@F#N;aIc>mFV6w%{?#%X>qidRFxQLX^RR&TFuzKN#~f%W)}-}%DN@4 z>xB3|d^|<{2!^!=h)W?n_*R(v4W85>wr?K1tK0gg>iFf4r6^5^ zV8Z0EyG^1XkEN*YAOBWey^HtC;~XtY=C5S8UylAN|K_Lvz|-pe8hOeFd5UrKtc|F$ z@&PWtOw>ts@2$!Rpj-;Hx3;!Mc&fMs2}kqUufBXkS0z3%Co|K>E}3E9%wv|i@!BQ=~0GWml7?O|XtrwGo=%L7AKgjkXAgbK0R3?j(;N8V(2jg3C7QC;kKv5Ft3c+G^P1kxPi1zwgtqb8$ghym>-*eqcC3$;T=h@2;;MT|JM z-%sAmZ9Bua$#Xr?6wuqzp9vihC%TF2EWckLpSD!M7AvefwEOa5l1-h|cki8Vst1-U z)TSAD9{8OE$E9uap_?4>uN_+EB7T( zB0)N*ZvfE@K`ZU6b*}W`9h7ho9dgT%h6|F&qJOt=HhZd=Yle<6JB8lzS;q)Kv?HBV zqMF6p`e-lTd;R`dzZp={rdC479Q5Z%ZK62!LUR0jgCox5yPBDw351s>SnjV-jl~~P z%8D8v9-d7zV1JUKcKW@=^6IFs?pQsV7y_~IOwT&kd|W(hnZd@}F{MdBhzqh*$lleU z*&UJ%4iNFVwS0=U`st$Np+2SGR5K{ODno7Rf4!GuvL^hSaRo z!4Ixz_Z4ZiwQi>E7N=N;h0!WMMw{K1pUdCKg{})>lGr@GhV1^&v2e5&w{{*$J~GA0 zM7ryy`LyQLYi>SMi<4d2q&Ny?`a#KFe1?lbJO78PhDjp6rR_RrAG@P`J*-tX%`oim zTZIC=@HIi)Av#YIoB|{EUt70*`$av+rUA2-`U(=i!>df3Lv#8A0(;1!0Mi<+qQ&O~ zn8oJ97g;1iVA?N30ia`L#mB^a{NYAF+q$$PczsN>Yo}A@)O+^q%(N2U!CTXBB#s|r zb%o^VsA(fPBhQ1c9JpFT4+|T$gR`(W6@he>&>K12_{K{Mv2QY+;AaK>JLj9pTb%o3 ze|>z`=u9AN{NX6PDiK)+FscRRnx2!S;Fy*3S1p1xuH8MQm%WZK21+$H6`#9pze;r) zt(+M1Y2j>{1jREYj!-+xAQa>fWWmtVzcYPD9HU_E*oWbmVVGb@I<cs&m(4VRJ)tQr*lR1&Ea`q!PbFF!FvB-%1r+-bE0Jv7DKZW;)J*vv?SEpJvg^eJ8 zjf)N*SQJ=7oJlX8Nte=&EN)V@bS;?EXwoj#dw#09req7wZ&2t3I+wD9b;RZ`IP~f zUr5$eXhqX{*R_dnpeZMU9r(iOjC;%cl|qu|>OKjF!26b~BAo1ncpUKC8J0tXbV%dx zYSbIHyO{Z3wv7NWrhdkfI;&TliwKF_uanVhJD^V%Bs)Jp=8*^*J&E(u z9ad38=rJj60}mno^d|DI^c-s}jVk8ua7{& ze-7&VdvHH}$6;Eqc;_+yb3e9E)}S^MEnUbCeO#`lFZ1gwH|vjMsx3cxuukI!2EU?j zmC@WjpNUTT!r{PdB3>dRE}>9nVX(Wv$lWT0&8b_>D18k4e)EJC9a{H@(USWbFHADZ zG6mpk)Kv5%D`4u9rQ6l4qP{DZxWmwRs+w-M@B)VfijF3+mNxo$*lf1L&{F|<2KBjBRVN5IyNnPM6+~*AK#cHjqMp~~I_NMo z-leeVe-Wkvy$Uuh`!lHOEoXpd3FvX?=;%Cej1UujwoAX~4Zk$$I0MNOZ`v<1GBQBg zCg62*B-wOLT;$}x&>(>{T+CXy!kV}Aq*gQR`(7U2Z(BS((Vgiskgc86NCLIrlmkjjON-RYD9R?uI;qlTqkvnF6mK|!Jr{a%D`c8q}$tNa^mOcRKtwcY}2*Z+diX} zYQiNw?kt}xl0f=?qd-qfD`_%ujpw%OzjVb zy884fqB(Q~Pf>zZA-H)U#X*wNb?uiQcCB1?Tzh*v7!<=nP=zvOP~fl~Wz!=3Zm-q- zChGf{UI56a5H-eJvuW{aI-G7TAm>>-XuX49dwMk-n++wg)bTR+xiG@db}ue2__f+@ z*HWxbjPJU3Sd&tvWaVXLTXJew##1gAvEOXmUQTwW-kl`bw7QP}y*-XNzr0{dsAA!I zZ8yjdCIk*57KOC5JmlL66?YrEAEwUS?(HC*UFI)@Ul}M0fo)NT7x@S=ZyJvqEhY88r4Fe!I41X|dWB2J(e1 z_UhpSd3q{VL(-QWx6e3GbF{0{ZZrSRJnw@}EAw=AUg$P`{^&l$(cR`%y7F5{_g$a- z0^8lnrNF_>UP2-VtCv?Zyaw0LrrFC~XAubUAmvr{}#gw~LZv#a~OgsQeMsNw!y~+h7_gbchfX%O8y*Q>5ye zni8ERFeT$KuJfXH=+_?*4w2ZSekPe+!l#vvPC(Q)uUyw9L$;14+@ksNuC>fL=3nIq zM+GuMV6B1-v+CuM4B)8Q2YD#-KCgX_n6yQ5{lo!IIv9s2fmkUJ-Ol(Vd?{zRyWzGN zC9zxJNOvEG_%?1sXPeV>Z(3L5vRCssr<*v1?kYE`SntkP6OH$3)>zy2x1g`jRwp<6 zES99buF~SXrAq`*gfBbfL5X;*M%by6-YxCWwxQXocs>c}S7t4F?eVtdENv6%skxcx zJI%4SM4_v5y@QknU^(j!j)x)ru+?nf`l?M9#M?6oVJ#nOHF2pN{8W|rwA+-!_l4Ft5QMAwu(2n6|Q_Hm2I?5c-xZIo<^ zguPTgbYo!F#7Qq+s}~UBN%2PkGed-&j^{Ss>zp07gJlPY@w_?e(;b7Q_F;=$M!84t z<=PUDAQBcYv2yX0!$HR?o8h_~6;Kl(p52-27jL8{-nw>T4~%0yd!@7U+4&o>kCyJ8 z>;Wm|gR_>&?3#Afkdb~UsY*O;^Q9}2@WQ@WujMB&gxKKbN2JpM6|KYZCVv&nYR z7@v~pK8wQ+OcZIk#}(L{BwDb<+l4#^qGboC)@8$uiFWlD1-l@F7h+;rU0c|Fi)}&y zlzKb@Y1VIrB+nkJ>#>Xr;PbHd@s&kzLkC}7l9_$86Dn%D?A8o18A1U*+$dtt=SMv^ zPV#;|I55*H{D7=fw}!=TZX|7`H;VS;3bJ_d5Bufk7(;N~B!0Ti+n@U5>}9fp#x_=} zdT&qw+m;&jVVhpG6Xh}8W(5@B?I%GdH(N3?G7%6l6%_?3X$V%1?R{8JmVJI4X~_i~ zX=zTT9dJ({<(7DL2)&XFtDIlCQnhf$&#;~k<QLR|p&_~EiXvMpBRJJPW>ABsrbH_uyP30@Ftz}3B_bu|JA$RO4kxo8_q5t?YgA7urP zR+tPlx%}{jrf3rhPz~yf--o;jWEzpRM-MpI`=?MPt<~}!91mf#GzmWX=SGT!5g;&9&`O zj_~It<;Dh+x<}+g8as_ldp$Fvnnl_BpMX!!@7~WYvceUUAHAOYKulYfcMP}t)IA$W z!ve5%)2{Rm1`H+BdCZ0=s}ubLb$q4zo}Dt8ja%O!f20koha(59Z~SK5D`OXNuYtbL4hTxrp^^qbxOXa^yu+F&bf0hcB_W8!Hoq!$}Skn8?6$;NPu|EgzCHfKg4V;}Uxer!jv z&-!XNSp4!UWmJarJQx0itk)i(+umu=YIvXcLveb z_ks}9Sw|~GqIgdxdM~Ij86mo=xN8jBwde^blqcr1yhQZ@q{^aZc6OKTt~?ti6TOKf zypT^Pg?plxYNCQ}-bOF=VT~mAyRyPec9DT<4Ok*s|FLd4yJN#bWb&t&cUe!g0M-Ke zM3|9luReUZM`c=t<^eneGPw01M~QME=;Ta362_wDZO$N7Kh8|YVh|w`M?N7N{MGdx z{$c|iL*C+hTo26(*BOBau11?k`AK-wrhfSIFj{sWw}f9GREv~$Sy2cYm7uTEze*Wk zA~3ehqpbxK%0bh_xMrAGKR#KQgndCP+u}o?R|dD0MiGX|4UBsu)AznaKrCymco34K zk&1U*m?`h`o_{C^D?39Tp;G3Ie;Vz-(wqfQXCiBk&qD4A zbE`kJ!fIT_&HnDoMFGNWwA6c@j{#E!1``vC?=&(d*uclgsA=f5o*oi_#^op=Zs0zn zrx#n`OJnA4q^X!HhYa{EtCi)4iMuO%0!CeKtEoEqlhE&bkj<{^0ciMc)jq3cDv(~R z+Sw0pb&&+jqFewUT=OqF32+OkmPWzm7rS9I*1`` z)nx%mJS7OowvvksrbQwM6j(?BT%@z!AWii5g*_~0jhjT#I8)dv{z;)y30Cof$m?q4 z=4?Kd?_aDuv{ofRSnQXmZcAdY4k-Yzx;3`EfJ2*dnL0cogcYEhSlWT>>D02y<9?h>n||k3hl(mQl}!_b z6y2h3sgf_?^wHx%vyYtJP?!Ahcrvbgqv{wfdAuRwl?MJVHYR_CnGifpQwK>b`!+R}B62^1e$ zSnPXQL$ERx?xZ8fG^fti*_Vi>gmxBZ$#2LCZAENMP2|Lt=)e)-OL2}FKLpHeuWq32mJrvhA5;SY|D)W4^!bdVx+^lRmpK}cM`&GeFG&!KIX`Q?>* zbMJ@100-^DcBnbwR!A6cux_M3g3B@nOO3EuvoxUJMmtS8k!P^1R!4mo~7} z{KG|Xy#OOUJaXU zG%~daHu8E!H>Hd$qSdG>l2cGvMeUvMIJaFdY*}t4n4~^Q!PV>^9JF%at_F)GM`ym* zOciSy><~8|Q9lbkiAQl#NLKF}w<+PxKM0=jzs^?!b%cTW_0Y*nj)Wrw@j zX$T?8*-Zb8 z`ug>2+wHkF7%%sm#%jvR$$?SZp9+Jl+D>1MW|t$7vm|L{Z*YiNUtd3vDln4vLUQ&` zvsBl_GwN$LT4VXibye^8^!$2}mBlYQwbvg>t9z*4MGFib0FKD!$I-FrdEWy)AO+_D4hA5%uj(2}c z62|0CB)O*%|DTxJs zFN@jp^K*ZHbO4lGYO0e*+0*;X5m{PwUyCc2a5C|$RLFBRJv|5+8P-QIZS}cferg-% zpOO_a`Pmqw7$NNB*`)K$mWnD$Oxchh;j4*LcK-xGM}~dFF%kajr#ETCU2VS)N!SD0 z!O|4)H$bjIxCT%w(K0&OWCh)}F!3)>ScshMj2^xbfw(=}VP#yfN+H0-N8}vAWK&}= zYaEMCA0oBY^R(jz{`|DCMR;5q(wB*_f_9kdV;nDoZAvNfURC@4@OY?L$_ z#ajrU&CA3jZxosD@omE1*4E?a|A(!&4(n=b|AaSUX1ARUTGcSuX8G)Q-M2}pNI zDAFa}AxM{Wg9u1>NzeNFd!BjUxn|a1=Q`&AJJ!DA6Ze{V0Hz=p^#3YS*#zlQKp6Rd z7;*kLN%>z^t>6sCsu8!F4W&{g$-ue8n939?pc?td>;d9ZLsfN_H^LBnGvMEdn_T|4 zA_TPlKcB#03R~BGXg91>)d~SrsUZvON7tVGw|f2GmE(WsnMD6bO`V>R;kG}I)Y07y zV_{)5C>{RC&%OU|hVp@X#XBJ@3k&ulVYbe1y@!L} zdN+aO;6Gd;%ufYVi@QAU@9ziDBUnHu`p+s9_WtZhk|R`EU*B(UmIG?q1f?y^R`h?B zt^XhM_}||K(}Vy2sLB66L|t87Wo2b&^#9=!Ve?#~^no+CsHh0WWZVCD z@|Nq7zj{gSjsSc>2T;8OMLE`-|?LN&CJY{L|->J zGp_@oW#-|i$6S$REVfiu;jGz9;uGIwnWDBA_FcV3F? z>1_p;=lu4wxn3z}jTU)EU8Sw3x8yV`fYLEIX!WldF+T!Z3G+n_{qO3 znd6*QlNC3q1se@DVAeP|A{7*+Sik5P@wIo!ZN+BSIjk=l zTo7uAs7`@FuXAR|6VIL*r@0SF#7$AIuU0yCYu9X7Knz9&N#=*whVWY4csK#RvgSKUn{oJC`XN1A|{J9;eos#JF>C z(M4qK$|@>0nSo+kowG!$1ExCqm?oK%latfa(>^yBw9lTw<{};CJ>0E4B=wt&@}CX? zT`^FzuLvIhcmi;HpxCUFNeb%`yqJvE2NMIp%HKr-8UcqI?J~7}GMamk_0z!zm4n`d z3n2WaO>H>cx~Z!BRdk?}Xv(fe@$p58J`N>+!1nvwMR9+G0+p4O0V=>$0=Bvvoi@)f zVPE#aBKSuIh@6}p?m@`QoP6fTIpR?`JfkJtA)g3Ri@$x}TcqEQaFt3w+8}j%p5ZXo zjZbIS3D?dStMPk=*+{(IWs!MZ7Wh^bk@m+ERcU@4CaFUD!W&;d>#Dh6`$vXi?ko3# zHzefMr*tuDp>rOmc+Q8*ts)}^?1Xp7D7A!gG*re6_*y^1aXL9wI7cSxoUbqTNr3bm z2M2<#uCBJx$OXNnw{>Cmhx1)vhhTzUzkc1&*cg{F#Ou1(m%2iTA<5k#!h$^Uybptl zNH1>nX>wYEL$>za_i@;AT&nUk+G~+*-6l{j!DOJXmkq;xu%%gT78MtVJ0<$PyIX?b)-KkE zk=kw|e>j8Nj@Sfqf;z|dhoq#WB{rloy7}|7n2NG8)G;}kSUy30@0%0!0z9N#;dOlP zlc%rKI7)Te?q`0*uJqkpJ;*Nemq*auG3oUB;?ZVrOa#$yI_}!FCk-HWBB5T#RKu2& zrY4~XPJv>!pxc42r|0!twK+owkbNd5;_c|aFAfQeio!HjFlKLx3J(u&ZswbK_Y&@P zVYdM>61+qp4izGE_xpWLjr9q}oiBM|G!e80U14HbG(M^Poi2}u<>tGNB zRCqblm_Za}WyOL3T;8>I+{YO5JqYE+A8zEJG2+*SQ%u%2HgCRzJJV9s#zno9W(5$y5n_#WsU2jfYzd73V$U%W z*;6Hhg0wO6-%^SzrOkNiKhlZ&zQ%*>g$DZJdHLha@+e-Xq-T0D)DuF`(CZO|*9cOcazwdr+ZBj5Ejeo0owvXq4jswlBY+$o z9sBMdiGDr;yDm{d9k!-Ng&r^w9x)*>h>(tnt6_?y^RCkw=z?d$zv#fz9u> zgcO1XXf3+8tPppjSn~0Ze@H$Y z>kZFKkL@~&-n0Ch+x)*j+#VwMucfy&`dekCT=jUXm^^njcas&@301NychrZ{^Mp*t z1mU4XS|IeCz8NopH)$OlEcw2(bFdZi;RC#|q~r%BI1#UK(P*G! zAD^8~wfy{RYpd_m3?W87*1wy64Bvmr#1P$3tfMa98GmfW{gpNFS1tw>bYcA&L>SFE}9 z;g`9D_(Hcu4|j8HJypLL{B24ZxZ z?utTP_zb9mv+@sDa>CwwDPn3}g*2O{xNezh{kgytx@TogK>3dKnY@SD-DN|3kEKz{ zSI8!PW4?k+G)$s}abULHRgB83Ds;D5Y!BIuQpg2iy6!{ryS%(SRXi_{R(}6Bf+z_L zHwKbiSOxQw3`|U3`riIMUh72=mXJX5_|XNny_1s@LuPz@yjUm}8Lu-cbbWcKn0aDg zYdZomms~C`S_sHX(62v#lGD=%YWVE1u`D@Kdjeux#+czf^hA8tkw-!3rw=Jn?^%rme{1ZPT#0&?YXL}mQ8wtrDMr7$~@ zV4zY>j*aZMIy6^gCi)c&21-N}q}LFP21O`kSY%|REa;osym^?IN^)}iNx9kCbD{%jk{sHVBPHeJCVZIg}DlIMT+nS!5sx=?Y8sdrmUSs(`K_al!E$Q&V2u}1o zmPGgVjpQ&YTE%^?d%Mt>XTu{A%7OgvKjSVSBnYD2EMH6&oJ6CVlG}gH=Y-(#weT$T z!e&t9Wb!$>(q)zy9G=Mj*RkmjAI(|{6|vDF52Xk^dC!MtCt8VIvfvKq)MoUrx)3DY z827OBe$!3PODk2h7z6s{eMWjvx$4WhrdJl~6#C7Jgk;JMldEqs+=z&X$fPYwu2-2N z&b5I9^8ftV@csKO5C=*APIT0;f|Pi=8cGpw$_Q#Ms@#Q|ni>@BLj4g+HrN7|J{!ou z%1Rat7IbqvTx@&+%RFV+q?0>OjVc?ye1V6ot*yb6OLZ-9p*&QMkS5xp=YCyCyY?$Z zQMSkkOzoSnel?x*?(et1kE(VG)sqJgOsVZWB`&1`7W~Og9;f`F4|l0354#{zvrX76 z92`t!5%$x1^(Q4=uvfkxVSSPa^AwY~UTEj5x5|93l|yfNHyHxtPu_K_ODDhH9CI-C zi1=HUu+1p-2fp9R5^n0DxW()D z@1Whqp_ki>VOhAM5}~NLsFeeQA9prYwyPQ8T@2tN>YN7fYN>FPGeed?h%DP{-0P)B z^hin~1{5}{UT;1Q6Ydb**x)%oDfw0vFB}%)8KZqIl{Yrto?ttu`t-h)-0jwL@wi=5 zL8tEC$7=$)yttD~R(9~yCr^*bN@FRRWa(!auWYW@6BK8yU0Ve)^IeQ>V2kYWV|WOZ z1*-lxCmXUdn;;Bsj%Gh>7i42#U=+eOY!Xie2cb_s<^+}jI}3?~2+Ngr`*PLEz4n=i z-r#J5l|ZZV{O6uhPq!)dlmP)o!9pLI`PM0OPR>F*i{j46&sVc-*b=>DnEG`;oNDB; zxyq_ay9`NK`8^_PksyDtnH~=P9!e$%fB~=VASMMo&r$__u9I1Gwsd~7 z6Uwc9Py1KpAgbo$;|N%3M#vQs3I|L2O#O-zMir4#sV;}|Ac&pHWM%%Xo>jjtH9U@h zN(Ab&(aGW=iR{1}Q%g0+h5@-U84-466U$~-3dlZ#^mgy&33{pJNI~)6pK&j&G0eAS z=O{iG7sR01={3iuk!H|Q3;n7@=b=)_ZJIV0cq%xddjlMvh(FO&N+q~svtCb|zj0%Z*X%r;&Evgk_+>Aq@LO@>yo zWmaID&DYyeJblUqxMFSJM7DP=gq$W}^9aIzzF-W_H-qo+*KWNr6n+o)K1~XM`URe- zIl4%3X8erQ)JfH@9Ewq)I(>?p{e}8V;A1>JMeqkbPfg9sD#2AyVJ42`O2u3puC)6- z_(H$zUjthwHih7`b_tl<{#0G`0 zNZTbHPC#Fc>7B`xW)(77QeO(GofY4*SFAU)&7j83D4rngITOYDp61_;k5e}Z3L9?* zOA=j$ZRlznw^lTcMW!U0>=?OfKOWm2w)G~&Z0z3)@?Uv!Y>RF9aVy@vUTeV@6cmu4Z z_I3~0zhwJ?D8|sx@N6C->d3t%HI?u>4A@JXTz0o;Kn_wv4+#;aDmQ$;;k-ZZ0%EXE zy)95=?tpCrK&|sJDT*SOdVtG=yaQAx#shCK^7DfJ-Fg6ogHwa4T1Z}{eFgWlqj;;t zM4W-F4RbJ(R=GXS?C$z_@H8(8r61GX0`X@&=Rh843JPJ=)kNq2g?b{LKB=V~ z{#Hx{MZSnU`No~@Vyb`2ylhJ9c{PD0M_2cgIJL*=pQ!nIux-RwXp=KCnq79MfBiC$ zADR46QDJxfW=hg*rZJEi2@39`)l0&5q`pIc)iBlwW*e?@zX#&j< zEK+697n~^)SKxADN?e7MqTSB_GFrEJMMbT9UXMH+TcJFG|BNMu;G2Co*PhJ9MRe6Y zKHA%=a(J#fFlAGwQ(x{WzEGjx4g^!~0ILCui0Ai_Zy$@^Lb z^b+`-DzY}(#l5xK`RdjA#mSB>8EfNq*m7-&j%T{ABV~(xPa%U>?`QoDtz(YYB-GMK1Gh%i_8rtD32&#X+eR0PI%5zGG)9?ko3@O zr(vU)WEP8F*hC?_<7CUv#$6do`18&K1A^%8$ZoL6c6N^2nCe60%EdOpFUERVY zVp38nGyut1JH49Xn$qU${qdvEqzWMT?{7bIhwYr#T56ewi^nG?OJ>VHd(5I$Wdfvd zVjnOpS`bq40|0~r78omy1|W^NxVL(}KYxA`d8Oa#IYsGX-A=TpVmgsZhw`rkS9->l zH2kpZK=yLYXb58fkvf)6e$a#C_;l;E(7Qe;q;*B(!gq>Hxz;StNZetT)zm!L?p9)B z!HKg%XJ!xKj;-e}lRUL(_i0@|)`1h*%QI;)1;Ks(#uP?=a-%J>kGklb`( z3z?XiDSvkZm9n6*M9lHonaMnooK-;YnnmG?Wx%f-{2+^Gs8m0wcX_p#B?jS`Ur?4L z<1lT{R<41M8y+5hXn45ZcB%Mc31}vFTq>WPr7Ba$0Z9AXoVBNw&i?le$Npn6 z;fwS0JgG{cV!Q$c7%V|V0qp|;+dC!s0d^JwbB*4g6`~MMUq@2ihbnJk-9g1c71goueQ z-8kuINirEMnUXF-c`=1gUx6r|BcJi=%yOxr;WbuhP}d?p3cS7NOaK^wq+jdQ{T>}w z%NvYEmYm_5g9xi;6*!(81AP#IT5P{_Z;`R45w(c<2@y7roumdRK9saKtW+=Y^*X-i9YNP z23=@7oq&u|JMio0&!0y}_MS?+hldoTq_@+%{$p5S-bi23JtVws=9whDekH#Hp(NG~ zQTXq`t=jRS1elewaL3+lRGFEncP^mACE{16|Mu-F_<67gTve^J)hN~E0OoT~aV9*p%uk=Rb)&02*FttSZq|t`&IY1DzGF>^z>aa}i{)2| zNSs>xTg;}BPr-@_q6(;!Aa)K8i?R0f-~aB3ZHMXZzgmF(c@oO*71a}(rf)-ua(u-s zR8*gjMQoImVx^xI`-R)MOpXldHP}{_+o3TXsNGuV=$pb|0z^If<^ z@kvpTkzWOcvXG14hpuw~CxW4tqDiLMv&T_HFOVUAe*P^1I2}^hEiWTmGCdX*rVI?h zV@v%9_nzB@0M132vatHxIulIr4;TL;N&Gn>K7PDhkMCOA)021ZLq(p7Kr^6@%IiE! z)XUQOJve=SOiVa|lD6SEP!dN(siG`NkuO=NZRji^Z52eXI5+8wrETP;d{P*$;s9=zgklws`0|s0;Jh}g=0f`M;aB&vT7mhTB)pM4$W9w{B&)B)?=p@V9HNk*3Kw=~^sH6{% z(xFgMqE?lm!3(2S{^tvR(#@WVoRFG2nau}=17H!O&+@vjdV)llgzMAV=z4TMG zOxX0b7LFJ!G@l?~+jgD*AY+( zPbmmcI@PCsh)++KVKXi)Ec{kimw#CCG@=_IaXFXfvHd1dw=Tb@%X9!Py}G&zIFd54 zr&NKIsN$63IAK3|h~OaSeKTbHI7Vcw7w70&k*Qi9QaULHb+eAy|ERBRzlXfNs> zNM99WCz?Jy#&0^Gmra-F#CldBJIFY`ml6DmjFL>d#M052oJwy0mmDCe*dIT6a^>9? z9^M~8z!ofOn0XDTAIvJnbM#XB**0R(n^e`*AVPxKcGlLPVq=ACMW)C()2t`*<-rxs zXN3)JjIY!Uns7Qfoq|+cJJG_9+5O{($<4*SB@GB$1?Jvhu2_LWW_m(`*c~eIygLdN zqS9L(aPz$-+V3F^&IGO`y7cLR@6X782^F%iumFaa!>?}=`q*|)FB-eKDyHdgyak1* zs0_i~VW$6P2@4OOYj9Wt>3MS5lN*7A+xBZ{Bgm&^pF^!)2)}xlBb2Sn>IMj!v#IiB zcai6KfCl0QKG9rx#<3f4dtmO-UOkw%3`OfY-2h9=eX`{pSC_M8ATVWWsgOw1T3oi6l+i;D{r4ed*)7DzXM z)S^>--tm%!6ice?2M?f`V0|H)Nvc&&S^2X%!TIXdE0CVS+MO;lo==es zfYD%ZVB#m(o$ufG!R|@q{+bF0HQkWaQwkn=SJ%ec8lOcwIH7hJA+&4t*!?UH02BHx z?s0)k@K+7|%1B>QN<#TM^6#V7sk|4P>T~(a{1+Z1yHafJFDiP-Mr({v0}G1sxTb3pmWr0Laefc?%@aTRlQY zH~x_}9X@Y|hk`{)>Bh9&=A)B)57=4g7Z7n7_eN)e;|7ediKzXBPwv!s>6uPGBkmaP zA3U-@aIQ-%ce=YBfj>I4-&XAYmVQvy-xg;`5BUcUq)aya&dQ?U=Z71~(~loNY;{l& zB$HP$pDu1%{DDpS0lBO3DJ)vg1@BlpKU}tv+!blcgaJBCz@-d-XjpN`UB}RoIgv@x0%G{zQO#rz7JWBWR-uVAGx3I%i+FND5e&t45Y0nf;Pg;ZqS}mqJCY}{eUAH%-~iE(C*i9 zSN6)_CwyQ(A@_SBjp|EcRWvG@R>=-5_R5vL1F3!uHX7_8T3^r-AwZZN&>gl*O&t5E z`7lKig76D^tDwL@=K?Yjdwce~umI4Sfp0XT+Xz|DGP7`Ml9=#$f(-bEfnd#PbL9N^ zn5raWnPXerRVY@&Wakq`PT@6|lS?+DJ&&61vzel%PsvKiYoGR`idvX|9w5!Pa?PP^ z=pn*0K7NKn>Dcs1)gewd`;%=;bt}hHOoY>2m5kC_DU5%N`5pLr`UVDoAkt_aSFY0H7TsW^z;R*U7__B6MydS zZh#MavDzFP0^MKF&`@br)nlqyH=$U1-neLx!M7|7I2Ze|AFsuaERyndp^0pq%Q|o# zRo8P-#r}pKaSqB5D~Z<*_!#e8Wb?DYLEm0^g$kdsS6IToO0_Za#XEi6&ysOOoj-Q7yOr?K4NfebZaQv9RxM4Vp* z3pNaQV!+)tHeLauus;m{WotMV8d^G+75+JGqY8O|<4nsz---8guH}BmbqxQOY90++ zvjfD`E;7r)x(SLY_ln{>dacl0w*iPbUy;zPkPxN2*{jjlJRP+7~7)AKYPQ8diguIEJJc7vFp|A*__ zt$HUELh9hH)A(wJ=^wLm4#b~c%PE*IsW+;tDyMvsL7A2bii8PVx(#-I0onq*UC^#^ zOhg+jlY{xEfV82frR5->1OirIA2#!O|NJ#*;h5jGXzrKlL8$2HDqfs%jt_#+MDsaU zA0m(Fu7Lr29|unKkH9%+9V-Nn8TcHT_x-0RiO9vq<11sG7fW%*{A8hL?1+KFXC6f- z2$u2~D13;WtD5W?wpPZ^g`6?Q>b|WWpp-pp06VRlNiki4HQP^3XUMYx#HMRBsFbF7U!fvXzwOqnx zm&XEl6blN>m%>W3^g||y1nw9jGnwohx0&Izf8#rVY5+J_1zg+Qi*bg8#u(ad_ z!WXjb0AWK*%VG*PI3SU+7uU%I8-tIpd8cB8nVGrHc4=pGli=}V<%!mhcPD>~?KD_~ zoXdz|9dvD6VxozON$`u>Yq;#u(f)ph=g$r1?Iu74=|0Ovhp}+ z=Ta{uUhpor*WKgc;{(+y=RVRYU|6($^8uAmv7M?EjPv=qFa#m`o3_Sr%p;hCM@Wnt zE)GE1P7Cc;!(SZ%K;Ye^9w1B)ns%YbpTJOMu#YBFj4Is$nxI^iD3X#M1#D)o!ogV; z66K*hD2O*}njE-vhOam6Plz66@-|5PdcIi8g4`_fC&&mM!v z4UV7U*q43+h97I{>RKBaiAI5Wc8r38St2Plh%vpgva(izSZ%0yYN_BI%;dZOOee$& zeFCJCxBKjp9=xom1Tf^tIC{J2g@r|g2nGFoJ?AG($`TzYLT-@(91oHlH zP>tP6o5g>BGv%b}H^r%-}QU8%9R;XM6qYOB4z}MFjjiuYqx}EN(t*s3hPYW&H zWu82M;hrEw^`(C%RIc!FwlZjF;)g zdnb);A|XDrV~PY?)4wYz`L!@IkZ>3j{$Ml>OwkAGz3}6U3*YNA1}*lbK*}yb9|2~i zXOGDQJZorud9gxYm<}Y;2?-*=#h#~_n5*2yK4M-=*Jc4ax>jl~E?{^8=zEL3o16K3 zO0qn_Z-GhD+uJ+U6>;Ls-P1EYHTBuEXW%_`;e?C7ef!pP;#3xsJcd3_xoEmI zyVs>HvR<(eNN3KY8OXsnl%nrK-WX_QuhaRevcH5Ss{r96ARXWYzg6d7^(-nH7!iR!(bhr> z6zL$|4)ynstu3;&=J~5K>eX6K`NJUq<0g^wBS=nOUV=KVF6Hh`clD9tOVTe3KVMRm z1YFMwR$p*LAp!y@k{3SU7qB)rTnNW_Y-YNFnO~0|KUOc-?XO|83(wBZmZO_k3sc8@ zl~FdmtDmkyQ(w{WiP#B;MqM~fFa*_wZY%yJ%6w;p#t}YmN zjEE52wztg!Ci-@dU!-zqCD#+K$BKw(k{WljDGV1Fc3xQOn~ zs{6jCP&qmp9Vwbj=Zbq+XZV(ig=K#%SIT6VRx)PQ{x$HP0EK{TGPB`)3Y|YcrnEkZ z?-XLjaK0Z8=(NB-wW0}FLhN0BMf&Uzk#_XRP^-Ws`BpW&J}%=y%TnOquN0xgAtWMN z4V(u)<9auDcXv>E0CqC%BCirm7}#}t5qz!m?w!H|pr_XV{832b++dNr0|&NnrbUij zehUlkj;9~}R;t4JQnKeg=Hj?)1(^C|^iM9fmE(#?n|}{JJPHa5K<9!0^PY^8Lh<66 z?KZ$0UY*`Jko^F9+O`lTR#wpODpJR_gfIl-RJT8)gy4b1_HVJD268r2D~uA1j0cUu z?XCBxd>dM@nK8)u8@kC@)agMO0-9X_KgjWQvV#B(IEXV%AuxC$E7KOR50ySQt~MU( z6n&DlCT;4nGkg2VK6}hwPAoUUq|BvSDvr(~cIBoNe1ppiY*tg*>Q1I7jpJ^x2>$## zN_g!nuuq;mejK~piUaE6f9S|N_L%vRMj$(n*J&wH0lfemS5CfkFi3< z>{4+kC=L$~6q0gVNPsO) zukcAa`f_hh1;}S!lV00a|GCVj?3 zZK*{NHS^E782YZRp1KmOgM`@6HzVdrmh$CEnxa`BuBK2&U@s0J?|b|ED44`Q077LJ zGP)_`e)Q{Ky#B%x7#J80hVpZKu^9${H+ZQn3Q$)!BXGa`A zxBpP^-VK=BP4N4~pL@)Lk|>mFzV$L>Wq5N~_tLHNsmbBi-k;fnck?T&-^nMYA00Am z0v4&4`9?zMNB)Z4Cit&gixrHg9f$hzk8Yj)K8tSVy?o1@IM#&x@10fqoIY7=#o1N3`$Fj)g^!zb>deUW z@Yr1G+0nHfonOP6=%ek{lj;?|O50?yBkW3xM)uCR;ju=&F`>%cKRiSFF2CE7XFlbi z)BcOnP8IA&bA!s2dGdv2cT;4~k7pT>t&MvxaRxFs>TGnUp++8&LPP-wO+$ z*DAa-s!T1~F)=+0>RVIG&{YpEwkEK%dz}#LDDwN1JR{$UMb;j3aeD4DpyZ<$&Yb1> zABxrNZe6729Q`i@4d(`MPz!5ogUva&6}C^^BDF<gJ_(U^JXMHXSZbqf*vo0Gciiom7L*-R&E`96>AKepG@_FaufD-%5xYc7(gzgFWm zR>NZqZ;J)u0{%FPZS@atp*~D)KGDcWl+SvBs0pbNaQjqmcw5w8Cg0(1^Bs{v!CC9* zzNxilve73mz^symn{%m9N5db1nO36wO;7|9kc}F5ihX&wf8RQ_oxQU5-fs9Qrz9T? zxJQ6M+k^^v3uaqLu@B3Gb{}L^u+vgqzovynL}c)}u?iLeY7F!tV@#NRBA)r`)N!x* z`0SRNYUf&MF7e;qY}7s*4AO>oA==)Cvu8M7L}-fM%5LTUZalvA4o%HNg7vM#LEk_L zL~d0#L@_1*d1ZR&c*0cQ6XB~JpW^{-0FEdv?Kbp_eY}#!v^sa~<~xNOhZ~D0B^7G0 zl2$+9^s4I*|r2@ z39zf6Tp|UmbZ4h%twA}L5{Ucw_|rM>v%oWxk^&_1c+^XsP{iQaFVzg6$+QG9p)r@Q zy|4Wv9?e}Q@oI1y4;u6Yziq{tx{YY5SN_zB4Z*iaU8+@E@H&no`d8(q(4wLSM@D{t z6ZSr<*hCv(e9DVV)6U~+Vy8+;Kt&lFH--cK_1e5R>wI{}xK~qs^V?{zZZ_?e4K6(V zIoX5Wp53EIFUlrwxjA>3FFy*SCh0aiBj;G<^hz5ijat3*{Jm%R5eX^y-5=8fWdF;1(&UH&A^=ouHT2z2Xp z2~I>#%hU(#uiMR>81%eRT~iceXMp$z_VMzAg}3%z8puS@8*vfU&N&zTOg$7mmi;C7 zHL08V81nIj!m`fL&ug2CLRDu!uk`<9Js>ruHaA6Q?R2`i5{<7&SC6VGW~mB#*vo6D z`tdNGcM=Ww1s|*k!a;Qio@zSW)q?8KOn^BJ{f5C(b0L=`u+|IN$ftVAi~r zQ!=Y$%{3yd!{2c$RZsn3fdQTqn||AmT9mS6fV=YX@X-D{Nse=0)`=!b1~|Z73X+m8 zK-VxjIJgU>^~RBi$jHS>vod+go*CcrMRCOk+D#bHMKhH0m1|LRa@Stly-xi0VA8h2 zj~-ksZvI5(yH|~h)YrnVNp^S$&lwsqAj;NcwY6>Q6K|80a6_;7PuRDO53`@Guus@} zH}s<;khexIuC2Sdt66vvzI9j+DK20~@ip=BaIEz;~F@a zBq#YDW)~i~dr;PryfDK6z!724=%$pUq@eF@15kbf$gUK$Mf0<>Fyh@pswPnT$K2o9 zB_Ng~pNyw}434EK%-sHoM;VxN;Rq+n%+h!AAO*G<4L1l~;pYF;q z%%WV`MQU>Jaj0?p<0_Fl%F1Qy^Sq7VAd|v;dmSp8Kw)NRvft=TN+|k%D^x?`Dy7@} zMM5;%i<7+mIP@&%*=h?D%cg>1X5& z5k$ND^^q#Vm=G?$RaVo@Y3gX6?@qZ9HjnbvKM<4NmbCm$l~6;~(5a_?xGVXagDS!& z#^m2}aC-46+cfV_wK5&f_;~unX6H+R27nL;eEkZTLpqLqqz`ZU_FoL5UVoKBr4nvM!2q&ZtMT~j(W&X2OC0)9B&xTMP-5=3kRhUiiloAA!pjPK*tr=<;eRA(zicTB$r{BOqed+TrUc-*FI#-dq+yxd-i`@d7BfWic= zxuL!!!O}lnU4er&=X-n|$PoD^0so|HzhjhRR~EaR80f6>YiaP%0G=HjG_M{F%%0U!X{mp3#rdQ3Ayqud0yVBOg%ST^mQuv6V_r$yY9MN_ZFI9d=A;f*~7cZ<{1_PQJUgCoy6DJE0q$3uB zZSU;glAFPO2d5|GTJ`G7c0&64gPfGYgCk)_x0<0&{Au=cXGV&zwh>j`xuNFf(!4buHRIz;81^%->lxU9{v4UzZxF$&@Z&+=Mlc!>AK!O z)#t_6c4HXJ)zSFWY8vh91B0Mf-q}$86yFK?N||#V*_gVQmBNZGhumrax&lIQ7udfj(I|xXk!mjWdu1xjfuoYL{(K4=nN^@*?-Q=q%cr4=$31% zJOjWvHa50nR~PUeUK}jR-v4$6Cs%>i0H{voz=>433{~X`{K@!mux3CG9XK+1mglC8 z$w{wIT?JE-qKGY}qf`umeUurW5+Nc)zk=b`b1rUeRj{|-q0uLDbpv5cjR874iMrki zq5}S*Nb7=!Um30Jf*U1s3|1(wwO!%VP2cuJQTvx0<1?RA1mhU~;=l5P^3s&;GFdTE0y3N};PedYd~O=D=AJx{-bnVS_l?G^Q#M`vkuM{DL| z3-J|C1?a-wF$*JKM6})fm07QAz&z~TC*vsAxGCtKYwXs`6rjJ5eKSlr)l3gurOe5z zN0&6Oo`y0j2%dG{ie-G@y|6-2jHq^sfXf-JIi&uBkyz34YMNBj-(IOi0yM4`Q|s&N zK#XMt9Kt|hVweT`DWD$b0t&e)@gZgRn{7ccveLOZq~)PR_vP!6a0H5mcNq_)ZE<^CMA3Y@w zMEPXqDHC>M_VtZ)PoDML3l@LdJF?}B7dGW=qhDxZU9ibY|5(SE9SXKwY7R^dIligC z2SVOJGf#yYE+*dCMiw4!77%T5gZw9NcT;fv9$MWV=1z&I4Y1EK(e(%?PZt9dRl8Za zPoA7#=(ccY611}jZsi7U^iSmLNTBU6#nRdkP^dbJCv|#$8eX`qBbL-3AQCX!#<(2 z!-*E>_)SEV^Uc?Vfkxg7w@=q)U0#-gg|pi~7lzII-`jBs;Zi4FXWn>vH_hH9jBfLy zeqMf|Za!qdAj5zIp;yZ7nfq7@?LJAz=9U?S^Z)!-=jjTDmv;MQ=CJA%*VuJ>wf!Z+ zexQ{lM;UQ|aL3wlliT4)4Gm$Y|L7*c)2EpNUda|x&VhjNqZ0c7CJw+zv`WoI<&XY{ zOU)^%sfGAs@wdR&^1^J$9e}1jvBjNVmA`71%ePh8P2UXSKP0LZzhX^#2j@=}*N0x$9o+Kz6ZgVH;CVs2R*{$B`MiB{wsal!7@4Q!GxIB*A| zq@@kHE+_;R`Js!o-@hS)oyj81bWyikV5nL21)3zFbP}cnlRbWZ@Ol4sccRVb255BN zkG4q?TRs3PB|ks^JUc|>VRtVzBPFGG?f5k*7WBN@v!+$@ul7Q1Lf4QB`L-*yxz&nhAVT4}h^T;X<05$HSo> z&54&1(=1mz*se91F-Qsa5na7HO&Ns~9$n2!=(!n9-7!HPP{10xnIpy4wL$`&wz`a# z)*P@O!DJ|4n=A$LYGa_(T7u>fj2At<@;0lFzPh;qV>3Wm!@kv}V_;|Zus8RDQ#KTb zIs|l;V14L$lyJkzt%dpuaFi`*#t8Jizt^xeNyy1Xic=f55CsDMq0uSF)M33Zjv5IY z;DBz2?;IRB&PNC!O=cL?tKaUs1RNT`d|5gF4OEIiy$qpiYEF|7Zt z-~d54%p01LqT%lT7vNN$7kdC-ZbAy}vUoqNoA3l2D_M$Q*9){04Y~(F3Z%uH+=GmY z`WZZ6KQMb3ybe&`3i1R(aE-On_dqWz`VRs{pCATAojf3b6ef&bimCz5CQM-ss2+gf z%Pdv;Io_*L@p-zJFV8@CMGTbd;D5fm-mUmp0SW=pFk!^_2kb#lYVOg~Y2NRkz~YKe zP5lcDd)|NGC27C}4j}h?U06m2n{}+S@bC8lOJm)Scmys04*^?&)wL~{0RlK&5ub1f zhw|A9(}!=R8k~|=fG?A?dInfX;38Tt@+XvY`RjcTm<44uHB%FlA2n>e1HcTBZu2EJ z7N^I`BO{nrHrf0BYICah?(VLH`<5CYg!gsCaePDHN(ugHYQV(>T4A%-c5R*l7e~^k zPeU~{EFXX@nENwVtyt$7%@-tpBy{|<-5Et2`+-D;o6AED4KMVe&THkQL010Uj>5>qn1tg_Gln|8ekWMM- z?hZ+j?(U9_(hXA52olo0Y3|4Q&z-qL&VyBVCShtloO``&MT0ututH&T~bRxA6v=*an?eZf@?b6ivCk!%GkH$>}FN zsc9)G=AS-I&`pDyKLAC{T7CJd{)xOu*$r5#pE9e)Ty98(69W6tKDcloggpX$)%+v2 zM`7P75jpF6Gqu0p%qIc`G9W5@_+u6S^j`Ge*5nxA0u|x%1zDW6DN` zfa5|>Z#loZhYu)pK}F*f9i3AHq`rWe=$V8>C+WTf8eRx>e9J~kobB^c9j|h7l+mTT z+k3>I(=E}7m%CQ$Pa3aQq+82BufYpN0u`LPO&1eW*PAT!9;g*)n3&pChTUKQeOA9pC@N}^ujfFPT4Pg5;0`QT6pOaf=HrXwz5@dZ9YW#N%Y z;(=*e2r7hnV0UYI)Emz`4co~>x_n16`)P;*6Tv42X2F0<#~*BO>Q$&z7SKuBX7OMILl+t97=O&2b*5 zxWdEbk+<1_Emy&yUyJcDp@xGA74(=um>8&))dMt-2%4$I4{e@D#qL|0$&!5v8PQeFfdm12`GAS{DR<5yL5ctaq`dnSV&BlH> zRcg!E_YDX2i<;3U(Qq}D=sj*oYjl1Erx)}3W?q5K&rA|BBIH&}@$Gm$e5=cSv!A~f zH-ZTjwopTm-!8or^^`A;6oJ*cuIZUF4PA(pw13{2A@M;zT$V66jmp5;kG{$Ac05$1 zeKVuvNx>nH1?o|?H0T%@fC^)`hmfDXlGFX|(L=KD+esAA>?)!4|BwM>P8Pt)TYEE%wE%=23f{>k!6>ziAH2Nb-!`inmCU$dBZ4ev18dfBd zM&hGFZ*y0=&!JVJUR&k3PfX0&wMD||-X`!^^vc6K0W0O@_hIAjg%_v269 zF)J7@^}dQQ5u+nPW?LFdV41Lu5w4Hk+X=Y^k`2fQ$mMkkkT0Y6cxyZXPJx%VAwE7{ z6Pt9*_};1O4zMcnKAP+5Qodp&3a9j`5{dVtW+?I?rJG%g@bKq@BB_0x!)*{nHEyyF9W18R6%nD&Z;ovlw}&o@c`he@xk%FBxpw6T2TO;SBmaD; zSAd|0FTnl+^yd%H1YKVy4>%f~-|i-B>etL#@w8-T`?Y%(2fnF=^bHODaN}h1H1E;S zo>jNJpy8EzNe$oPMO&d^0ZW6^*bHB8u3Th$I3%A`9m zd`Dpk)m|%aYOihB?p-Op$vNSP2%sVf%PS8eB0G5Jg|NMNCXLqhNl+F z8){t8f&Kj;UXtcu#iXLj2nYxWn)Wi-0^Bl!3KUb|QdhWHH&_pW;sfl;#KjROqH3*x&q6F+ATcFHW3We%!U)tZ$H&Kqg_sco z(4=}HKmMl0K69@`C{84re8erE9Oexq6T6~LK%!pk!Gdu})Pio80}gegAMMUtR#p^vZtY}xPTlYuuYFY?wt*@j0UGRt zz(*X?TG`)UDp6x6pX7-L-pr|jag|#fN=r&0ftwp-18{}sOg?zox{0)J)_;8RgE?>q z{-o8rsWmvLyEan)J+r>Ec7N^WYrSk~TJOQ$hgx}KrrXi7SLEIT-!Vm}7G!!&Ulup- z`q5*1GtG~)yFdah>s(ue3d85Sli>YT!9~&PQt@@RjuvuUwX}*hMmP@|f_R%7KZ58t z4*u&V?SD?)>sx8_Y%x8e!Uy`nMNgJ}tCh59l*$vCZoF?_+uo=D=DvU%C|3OWc6nz2=3ftPTulZotDnt@6FP`Ip*q6)EYec zfk$6L!6F5^CIa~^gpWVNSCLG@K$MwajF%8x@ye|k?D%{$}zChI9Y zSlqyE_5mB}$RG3b*8nvoCo46Zt4OAP8?<+E@BW0+b{H;x5U-M_bv{rt?86y>}%uEfwZ}vaG zH$7NDGL%_Hy_2K&H=~)~GJGx>YQtEU|GXhSvUZ1Xyt#`_waS23j|{e3U8-HJZiyA^ z{wz?NRGg_K{G)}r$b)G(*q4 zFcML@=Wm&%mF;pf{LPqH`^6~o6o8ntcA&0nBf z;8|~kv$q2-Vq|o*w3zi1AX5Rg5hG45r^JCBOMdeAUs282%w9GE$ZjH`TD;J?4k4yeA9Pkbt0*!l&lxOgUrEC5eiXz zaT|$!Nr3qzVQTft3G+HyW69P^B8}>0X%?1d8@Ub+$R?CkEEXsN@x?H)~WsMe2(gAhFEo*EW6EbP@7+<7s>I8}+d21Ss)NstFdgopPMY_g{F-L;UTL8ZgR ziigY_nCHN=6+D1DRsc9Ij-q30QL> zAUlcno9m&rC0XW4{7>A1C?#?WGV;LF@6umE>c4I*+ymu6$P}-33%9joQMgF~fd}jF zky5GV!`=Hl_QnvcNwi>MabaO~Z4Kn6ORLOhvwi0K4mPgcJv{>-82Gd4N0;MhfeDGV z*d+DuhcDyo%NF2qwUnkPp7lZxH3AE?D|N(W|&#iWLnFO>xVx9Gn#h(dMoH&+$cBWB_93`MBcJthsuh7^k?RnsQsgvMgM^8VeQ&FCG$5WvA-vH{x&ws5wF z^QJeFY^E!7+e+kDh3eJy*^VjT`w$UHNBtFux!s##$3+7Ln48<%r@2(5bJ4a7W3|$p zXD2SN?FG((o}BZn;nBLDH!Eu`(|Ca&;H!3FW!}LHzU@ehye16iWE%ZOz@dV{R#>;K^OS``and5ESnUli}m$0i+SoJ!@>ZL#iE`pxbr11FT;UD zxG1$S-ouxo4#uJ@s6YcylnR;O82M`r;C+3ve}9>A2dZ9y_tqhUTOCOcc6gvF)ER`ORQC)m^5?|F3BYF-*NeaV zF62Jl(IG0I=vY@(6|1410Ho6-C3w$pL%#T_iiny$2a6DpXtjbJ7%1V23Fb3M+@$)OGI{7}b_CfN%b zL{$9tk03w@oi9$w+(rR&5a~l4ijff=l>46!p=gdg0Hyl#pP!&YDLtjiNKP6#eK%~o zE~}!F@b&8iuoi+hhnsZK;M)6+ZmuEkun?9=pw<=MKv}*24WHl5Er}P<$9gC zSlnLmGz64AK|3E1p`)Xti>>iKsHlL)g4g}rD*GYg1K;xzq8K3Qr4FkW8>2lE9~KoW zdxM9M@5k_>!D=?U_}zd-;Mc*B*bG5KpASBMQCo~jIVPxeK$+RQ_QGs_zTWQc<`R7C zgyBWDhY`h z-E26~c#ui^`u;dxA6z%aNA%so&CDds>emN|C1x8&{ zk3OigODZ(!Q3+>=*#GSV1;0|A7FE0}dNbG^82I%vNZD%u1x#y)RJeUN2@uT4Krd^b zfepXLdKf>LD8xYqF4Ih$#1*3-ni~RQnLPKVW9f>J#hxWYG!??12@dne{UtIa^Xm z@i;JP0-6E0eK;qr=3_8kfsbh}9mRJH{PW>sE*v)qoIyr(gHrp{F>5@>K?Vc8Q&UbL zEv}eU2azqw{cuIBc7+HrD>m~ODN;PJpZFE>XsH2lPz4V<0Ud2qT6OWkKSE$za|G?{ z++4bZu4-*I;Niyac74D$wPQv8SzGgG4=({TCR)IaSc+!AbALQkI$8~{>Z`T2H7nm>{}Zo%!+6Bk@n7;IEqahh~tiu|xfDZ7LSvSd0z>XeBi@{9K-x;fh13uNo5XU-ZnIa!~6uR2U9x<9O z(IRVO zWLct3(%^ETKY>?zz$R>+t8h81qMF}{YCCEJS8x2@He=M7@UOUWb8ph$fH<+rGa%vL z;g5XUQj-kpAumk-0)|8!9ts0v&IOBW|1a|uf2yxy($eI;ifX&{X07Uh(bm4_BPS^9 z9Bg?{CItDUdaS?SIpm3m}tPVEJLZu^@TF-vF6hcaYbu0m!xrI7K0h|2dmGC5TU<|tWO&`1a-$Bi#v#c zN2v1$huqB;u|iFqF5w4PgImfOgL~d_0fw*ehCrpZ9h#bm;B8* zfuLW@sz_rH6u>WNXt<9Esk5G>ZHp_<^GstaGO#p%b2vD!Y5i#dWojgAcq}ca7|ZR< zO>X@})fgAqBt(J~gIUN9c4*2YtTW|D9doM^21T#&ge2G6{-$Z#GenQs!$;~{oS`8P z=bE5XtIadfLZZSO(Ht;dKoF96R;GtP0r3k`^D<0G|91;fyq<2|(MP*ao^$1TV&`18Dg|$LI)^((#cQ=BZHlOQpF`9nxQrRd3=HNedAp8x{>vU(O$Fu zWpu%QyV;VLHi-$pRjHup@%kjqKjBN~(btHHsDq?WuwUFJrG1CYW_F~4-?f0gAHNCUCvsquNy(%=+KQ}Z!H#F#e)#JHY zJ!nb4#}4xp@K|!><&w6ndGYWD*Bi=~`=q**uV?3|SdQ&%&DFFXOX}>FTnsw~6Psl> zJ2H1WDCZ(At*sSb&v=uSV4e&Uc$VS3PLO_@D^L?LUmesDtk&NW$0cLjrgC*LsTXaW)RuTM zVmru$0znf+b0i0i=GjH1!d*?>&R)9B ztG_)Vj(TIP13k5)i;>GT=Mm`^mL)stj{4M#h3#Dt6V`OUi}Oyt03Hy5atGDC>Rc zpXUk(#%T*=j>k7c62an&E#XeWvg+-ho#g06E4AZ#ID+UkCE2K0CV z+8Bu ziH^n><+SE+zsTU(@RMQ$YOWnI1MftsPKSYvo?8zFX$Z9Cf1IDaqLX2;bf98bLWUsH z3HRa8?m=v#I!-1d{liV&(3BebacjihzoDYua#mLZMkYiHH0?e z;BANWx)GkNg?)nmFq(^oRh;b4Q1~D!s(fa2>7FiePYULIFI9>40#ZL4&Y0P4#R+N` zp;SLuRG@rOdEKU3Cp0+d4#)D1cXb=BpT79+G1-(Yy{5Y&@qMYu_3cU)#^AzH`jm#CpXq&~gVo99i+wLuVhCC0y_F7y z(&Hy;>GL-UTi;ghysDV_O*R87dQs;P^{HFyoG-yBYwN8LAzigBLM84X(;aHg6S+Ol zyyMD)l$T}&Oik@w*(>B@EloLF*5=Uhoy)HN{SI2uLHgA0JXOX{{I;E3LL4y?9rq9SmE@gPUOQF^Zto0hf+;$uEI6>)Elq&=j~6qhMt$jv57ST)zZ@F{e? zT6xJfe*30z9wY?SI{nKj`#xd$$Uh~qGh#S`{{4%03kfO_reCC+FHnfT+7AB;ZgLs+ zO~{B8k^l_?AWyXFz23Nc3heb^F5>RZhH8^ig*e-%k+{B!&W>q`w&>)TF{dXzpGHY{ zPVe3?u(A1^We@*Zx^)ZDa>xAM!fa(@TNuJme2)u3F7JA?nX(#yk>zZ$dH%wBX=_nY z&&mF@oW-XKZzrheu;f+(_#ioh|Vm+fZYF=XgMkJX}eOI#fxe>z#CQ)|Wsz4lP1DUNYZbb|FH z@*iI3=}qZ9@!99m8`gOQCAdl@xHMddEbL{q`*)?pQ_3wLRyMVfVI~pP9H{fF zdp4N+Q`HD&S&v;Lh_F`s4M=$F+L-B97-O`_AHq5)w!=M?l?hx6*pPp$9X(J-TfspJ zoH&r{@WE~wn6tQb5DJpr*)86AEG%wMXI#~D>(FUna#Pol@l}|(yKxOiI^}u2VGax3 z_S>{A5s6vGNAH4#o_LAC3{1F>?qP3ch#G_vh~0#>W5z7R6e{_CF{rLQ-2IiVpV!7tVv z<>~VmO>~~g93rpKq4w{b`OyK^ZQ4EV2+H3*4s%~zgN*X1Lw%%$aG27^WgyI~vOi4Q zT__P#!0hB0^4hpt8=q-t{P%8aa%%7W{&N|fbJ88Fas7pnMfXz_pKoxgLx*dZyK1g& z6w?v?$AW@R`)^3$-8qwVK1+I06jMdM>^i~n+I%ray8RM12pysokEfflZl~&;Dd)+2 z+z*gxLoaG2;RS12(KTr7G)rU%#aM4 z2C;IrAcVkQu3Zab2m2QXa*r_tc=)~q(c_Nh9#t{PfNmL|2C3{0w*e+HV;_653lb9Z zg*Q^iZv@Aqrd2yZc-1k~$TEF-qd^IY4p7M^QpBC^dp?HZMcj_bS^h=N>UmRON=L$x zZRCC~(|h$iCq{|KF;^Q=_JJ4vStQkFe(VnUH`p$>vU(pU-(QR@H^eEGPwQ!|USYoc z$7+}6{*4c@Yq0f>YN^%hzBza0oxz8>GP3@6xeCR}Ne~p}DVJ--!EKC45aQPEts9RC zB~;xvuxJRB06&VlGC4|6muc^=HDmcRvw0!v6E8&$Oca1D1D%);<49SpRv4FmX*5a9 zl+01iz0SKA{$fHtruVPSul{(kzTIES!{Y8()6A;G&NNnO;?8f-YdBqYkVQaOSSN}i zwi(x{9t?@8cunD`wT&t+@vuSu`hy`1Z=##gI&SAlRrVL1c`0ThU7cEyPcuD8*dI<% zr-S*cuV;29ccuW8ev0f_&|^zxJB*k8c1}kbvag5Vx@x4LK!+^MP2+ySCLVO<5~Bt= z`Qk%GTxt%mWG-US%`N87=iM&~ad6;kWW_uM4d zRJ1x=MI*gbt2k!bK0AA9fEYichq_8vn#CUEnH<0)V=VO&5?%iqKfo}o^JEqMtacDJ__f2_z%R zm!Hiz^M+B_FKtfgvuNCQ3oI*xl1%dXF7}v40Yisup(nd=5^>T^09mKNjwa$?IQHvU z;odkSt!4~Lt-xE?eJ>aE;gwc8Oy3~C!*6mkAwghGr> z%3~Z`llLkoBgGDO=ZU$zr2eP%P7^8B_=?$;LS`g*_;Ozc`LrtSVx52tgPH8plN#TR z)BQ`Bx3H;J=lpiL7;8{k(ug(VbhU3FS{VH|9 zTK%xVcy)CJd@ewuXeZ%8L-6#eXBggI>Wu3Vzhth=PE>G)@;8?+YG-+^QCKoE^iBZYOw5U1VKjCtgw)ozUN3Y_Y>8y1 zWM6Z5H{Oc{Vmg97K|gx&8Qved0KSwK7L`i^b~(f>_+0XSzAoe64(vEiXp{)H6n~ zdRSfY^+ag8;|(xk^zOF$9fqnEO7kp>;=)g z#!w-FJ1uax-e3M3tFOu@crE=7p<|`)w+5#FRTW)D^j|0UH2bX<=BFG+=0j1)=V=HW z=u+QDZ$(dk`Yf4ZcGT6H(i2aYWUoEqKQkSM5GNgOurPquwRDr1ls^a>qDPirPLOgi*5>Bd%zC{K-|~+M^5= zOWfNLBCDYzh$h$E8;;zem~KVZ)G^R#vf34O`n$tM0a9X(ecz(&0JpB(J^XDH&rE?p5MEycR zl_p59<>UtYn$A!~Gi&5GgJ#~*!946?pe8em{T$1Qkbyz>>)4CyEAGE`KORGnskFOJ zsHOcG-#_iEm=9Jy$6RW*ZK%hM2M%$gAs7}29K>6EvxqC}N3!rhNva#zFJ|W9f{~-> z#w+c7J$o3{-yu*qU%xA$2bI14{?O6QK{ZxC%;;XFp8`*}k%#*{+w+PV-;0j^7|;rs^>1e{UcAKuBxQPh$ z(-X?a3)6o^Nib*p@c(tuO?bPx_R4F?#yZ-A0nVD zjeLzMMAWF^Tn_)>*a1min)p~IH7P8Gjh2;+2*UP)sXm76`0n)fjmYZq+PP;4e}bS! zwYVi*&n1z2_j~6ZoTTtG1byN~%ex5O!{fs+`Ksupkki(U-ttANf;9IFQSLr5>+a!iq@-TB(N2TP=q-ru!bHir zEDxz)-QTAqAjq{~jz^jro9pu`9&FX?sbs0`A1-~RCzm$29FGlUnwl7ifLYL%?fiMQ z<0;R{()JD&$Jl^)tv%K7&;Q;dw6++`!#^8tvGv+IlM?DE$t6w~6;E}-5sKxh!9)0j z=v~P}4^}^T20o8R0Me55sN-Kzq*{-%9^U~RZuG#!RVT$Rg&$R4`XaZ->oThBr3MzP zo3O6OJb7-u!91oyCsI8XsOm|kf;H7H`2zJGj}5y1u>DisCh6vg>Uj;zxWC-!{k_Pw z7nLH&<&Ke2VEh_2$bW6++8k%6_ZNl0=H}uk5o@SOmB)Fg^>ma;+Mm6X+K5d^pYzqt zrYm7tHSC&^*E=NAO&zUmGvf0YJ-p%ogTry)<7gok(GwbvCsFee+Y*6r`UZR1%!0LQ z2sd?QUF_ec@_JMs1s9#(SDw{{EEKxxK1QOr@|O+%dq&$YT0i<%6n<9 z+Jhzd4__fs4BK%%)tl}9KBWdQmz~P$c6E1y5;Y*6@oxjg%5z|lcii6D8B`My{{}?) z8-h;ca-O%-m-TnGL#IrPMniI8|E4wx)(O-LIJUsrpahPSzWS2oBsKL3O@+lfLW_jL+m)3tPr5iHx zilpU8yz83=?XbJZXuFfh-V4lcPiwheSm+%_e8_=Tw#%+<-_rIq3uDa*V(FY)4y7Xt zpL;T4>=!JqCcy1=tJmPnYHGvj+`sTLIqwc$k2f_-z(V=FxXtE>gE-c_=E{=V+ONJW z2ZkWiN)DjirtRLqb*~&O54-SCY=0ew@4mL{Pw#vweH;8G;k2X;4!uh}z-k6gMIe^E z+>QV(+=z(yiuaNe|4YLHz2ir#O2PIZ;K!gdsBW7);b1TPl=fDB2lV(F!R>-}Me08# zKe>|4d2DoGj}Zcm_Wh33jU-xlfgJ2*(|EYQC}?(jtD&mevw5N=Oje2o_XZ`_D#m1* z(}v>vf+7(P4v_ef$9fr`cLu7>4o%uYQ>G3xk=9dVAyGerQpOvp+tPwf7VeqGy?3Tr z6HCYj9-3QrFX>`yB&xp+BBMU{XfYc(XwZFIE8-3UJ_r*rk*4Nnh1&t`3MYCaJ+2Cfko#6;;@G$|kz05$(;lZ|J%1K=2lHjIKJe)#rL|2F#sYGj{0%xQJ>34D1n zbbhCJ0)Eq+im|e>Caiw;Mf_SFWK95>z+6!PX5<@iis$@BRxHdx90A~ahPD`w5VN?) zBwq$Rrwx0{(Lp37C0!@7?Md1AMcnWrFXZViZ3Ldb8{*{amIN(C2(vk#y9yu&|)B;ZRw;Z*I;`m;_NseV&u|o3e=iYs5TvGn4xP79~ltbQ+D}b;^iw zjCO~N=nsqpavoJz{?ho_Q4itdPiF+^~+`UIs2MC4p=q@HIR{ zYmWar1=fI~pKOoBk;GorkxH4EY=CV(a2*`=?^#dqts*|c?0=gWzA%Kv5D40#fDrHg z+!k=ef&v2GZdujv1jJ>HoBcX_*4^C=P^&Y)erWYi6cu4jQm;KZqUy zVP0vfz7RS4)f?a1CB}!QVG}=S=l^YyD;1>{)l+&E1llhEA4N=j0W`s&HK?klb`K~p z#XRV~W4MZ!2;yr$9IS}rVp)iqGyuV#1RsBOdwXsJ9pQBVJs~$3HP-W|)RQp)WZB|; zkeHBw#|j7^51FYTiBZ|?DPk7MLq~O7Kf72H-TX=bTmfa%Z0gATd`p#vHY{RRHe9-!B7b#wdDfgHpfXZrkkzeli**^*WU|A<9b2xVESz$6N~ z18?adA_UF5_Lj;&6(Sdp6kA6{iy2bnT$B+^Rco5NDC@9kA<}|ZAVfS|> z&!o`O9L=Yvl{QrFoA*Gh*h*d|xt#kG+ZmP3QrV85zXe4kbxR#;Ysg^djYpB&B2i;AtyNe!l6u{GqSNjKPp2Y&Gx zz`-SrY$U+9Cy}xKzgmF*+K&FoJnY|z2njd#_ut!y;g$8|A6St88C)s^O(_f>moJ20 zZFrEwovbCW{I215=&deH&D0f@AkeQk{77K&ur!-j<_O^AB_)5)&3)OPd;uu>dw?tT zS@ZTi%9)dL80{Oz%N!I(JvikH_S{Qb&(>A$qU*0sO* z-OqOc?-3xlL%+AlfR0lg$Jt*&2xfM8tAn>?Tz0fPZ!RLTrf?)eEY=t%A!1H!dVn;URV;CY+d+KM=A z$YFK@373j#&UajNX3E#iE~_b9^qpgs43MbxeolD5i==E_@=;SmQmigRp{GWo{mPeL zp3i(ZHpghM4eB@2pvGBKj~{4C9#k4Ui+`u9T+ z7a(%M(rwwS0Ma2+Q^}4vX=#z5-94$N#dURc;DrAWTsI$w=j5nKboMS_4uH>`?CmYG zVblz6dp|W=4iFbT*T>T=Z-W@GNeIBl{)QQ3KCL3KU;j_dPuQ*L>r7kH~<@h z=9(Ro{!33j$LEK>*5F8*)6x0B zancQgg0Ek}C8LjlT0nU2U2A;RtiXiozaY$Cd_R?*6L7u+cK^F7XZz>#z4TF)|z2lKA~a zgf-lwyuLnakHd1+?KtCg8H@M3`m%d!6CBS4yFxD=ofvpBUqkfcw&>7UaS=Vkg**pU zfO1$SxD+v?2L&ErPv6sMA{-wFFf~NqOn@rH7b|zL*R-B)T-wkdECv)mlySL5=Gq$d1{%GKK#&Ga__X zqK261{MX(QJ8Oof{E-J4*M_1 z+Bz$4@dPxT^j;|{mVC%Wo~Q5V>euX$bQkN_Y5#Jxp}@?$)=AM;nKZtPX$u>cpFf3B z7=60g@IKocs5o*EnKqQ&EPnypXJ%ho?&~qcQYeokWoW%WOw3#o;Zp@ z58K&)$>ebjD(hK;-d-Ox#r8k%dC6_ErPSb=~BOJ!`)IVB})XdA1n&qKT?1T6U9LaY+Jp z-QB_vvg7Kq92&k^{_67$JVX0;RHJn?rLZpH*vjd=>tpm=54$EUxXqw9n&ancYjqXo zRG?u2NQ;gNSD(Vibm;2AjVDqpio zpw1CBH9X3t`lnWX7xWKTF00r1z75Ie3|6*3JLqF`OftT|%HuJUB~JihBB3L5KyDdJ zT`ih(@jBn-T?Cg3zLsNaoP3w!YK5IrkK{q8WPNl?6h_u~u4Yf{*a8u2b+3#{em2}6 z>R$Mk>NV7`9qo6q_wq;7($Aapa6Dgnbu%RTv);x{r>`ElOy}nOOsm?uC;v@X%Sh=T z{Q9$eGW>&&jPZREsePLkmX=>x;W*zn)&--iwC;W7Ru_H!ukq^l!dLzpZ7^S*&_|WN z9Jy$Gbdkp5>FsqsYYI|;!2yAnlN0d;_sZsu()8%s5r}|2olp?LV+GqVpx9%0J`23! zPo8`Q!5hb-zz9S+gAu=mLVo4JaV~ z^0I5{c1^drIVe-&DFgEI^t+La;-V+;spO&7#%NN0 zzKkxbQ59awS37`{?EpBW$bR%~Nn`volyT z)Per$MoCe5nJw>LC2vI~&&S4UvbPHjBD3Nu^rh79k>r_(jNIzUco&6CysyTv`2I_j z1gT&{?S<6Yj>Y47$_)Z;gZc1Z$3%JSk(Xy#<>y{={yihe9h5`_v7>bk~~2esW;bzXq<()N-|F zVC&-O?5rRo^Kli0&9y~3TKAhWI+Wud9ppBlXRHmwR#c*`A^dK0U{X{;P(wgGxpDu8 z-+;eiD5)m@vOSF6b~Jf>S(iP{?bKaG%TG}tyE2$2PHf-4VhQC7ZP1qEFNBUaSa;%~ z<^3in2RmRcK|h9L^E#b1+!rW&8`+)r{LyW5XWHlLzMt4x!uslT4JDVoZ zC0vmhK%)UPUoru$-ES8_M*CAc@m{EZ_wj*3nJN;4@g8D)XKNVlLN4cRVIbau6Hr>L z^NVS7EH^k54`jq%i8?LpDJ_zgdd2vuJs&G&4c!&+(PdUg zW_@v-RzBZenV6!T&MmN6Gd?<6%_c^UmjBBx*n;K!g#w$P@2z{Lg)IUU5quvDi3i^%5dja9*J*naWXUTyu(3~#*uUOcPBwbxQ+aq%XWdJ` zOjR{Dpj~@vh2!o$kGfMGhLBlFZ6YrgA*qMq4##d!SCgeCYUW0Wd}Fy;kqJ(a;9LU% z{ju#VImEoYGn3n(Y{sHwIZL^wiS&&zXw0gMgzpzDEFPh&#xI1`4^5*v&nZ2I^UZ2D zdc+O>eL}8_xA6rdYwZtdbJRQ`mBQ#zS>2nj_ePV%1^L&qpG-}I&y^Hef&Q*{Emj2i z^?3I~i36PD_U6gt3|c)dtGl+-C4EY$flM91W1b2GxaIH~rC+BuaKiP*YV>&KCbGPS zQ7JGimt&m&`=k1_`s`2a8+kTho&Li`0Qx^wU+lk*l`cqYy{oIQH&NUF-?`7_18b}~ zlGGp#;9VEz!`*I zrW~7LYA8{UwTunEXaGiZo#*Cz1GHrYg;6hDX)0k(X~9C8F%;9W@W2;-rE1T>nGvZ|`n;C$0BRwju~2>tx|Gyc<3LFBWYWU7)& z8+@}J4LFtQqPMo})fe8%V@jL#MI);J3bk6+fhgn0Q#kPfl}z_NRi5r&cH4$PVl_wJ zaM{NnZp1%tkT6QG8ZXL(KGWx+L6D3?dGD@1bl$vhkO$S1amy3u$u^p7| ze$@Td5vg-8QWGh82dh)ZfJAlWI8y(S-1Igndvxyg7H!_4of}p27Q8bW4|GK(I^E$i z`H_e;<8Vm2m>tiII(^l2YQQSL(*N-``N_%#$rnx-CPZ0a?{a`eKt2xYW!4K|74C_Y zXWzK_=ksT9a`k1L;oruIP#cNoYKbtqU_ni5aV@QuSz*a8xx~<9e91F}u;Zl9F=%6t zk^0!Rc9d(<@mXo8-UZ_Bot9k5zk0UgbTB7hQR0{}bc(7=YbwnCqO+@nn))vg%Y&bU z;#w5~B>KQ0q^zP6+w#sMPXo-!pmNpET8Q}Q=*8ERlzx*MK+jUK4yk0hURYXJD#hG7 z7yjYS4y*az$hX?1v&2lFXAax}W#hGL;P|&zjs!t^BMv|vV)rePNNCyfhp#t;G88T6 zdc2PmWbnJ?_*&XudLoP4Akcq?orJZpZzWA#_)lu*YsjR-c7eebRP-vHEB2);$mN57 zH6BpB=3Uj?;u_T_{$dLLX*r=VtzR7S?lvdIZLOgORVjHDAUX!VuVT(*T15-x2FcHN z(WiWLw7T1JMGf;27P#2DJ$BB1#uZiWSkSq+_{7vkt|+ngcp<3rNksQ|_dIaT?rPs- zt41WoWi`Qo3k18`;NYChM9h4Bjc@s#f&i+cJ#7=xr6vEJkqe|=!|?GKW|1HJ=LvHE z`81I}u-L2gVY46_r_}4D{@>j#>+s$i%pWaTG4(2>sq%?_KC7H1jtQ8^IPTs zQl9@G&_T;hMKpRnf;R}|{zCkhENAkXDbjIhrZfmeswC`04fAE;BUVTGd}sXqKFPmS zv3?av^Q$w3M)k4pCZHu`kC~M37h9@F) ze5dkTr1L)dAYG1m?~4sRhV{PfO$Md$MY zlO1d@M)41 z96r8dkiGMT*4Q(tAnN=Z4iw|c zQ1OEGk(HLlwYH=Ap-fCtI!JZz?iN#xkfHkXiWHI=U5I5gOEHSh4=f_3A^ahU7fNIF zT91)=!_u0jX0|Y@G^|wVr@7Q6^LdpuyHFMvq@n>aZ+3Hqi=1Qc_B~JEZdgr5i*+>F(|Z=}u{o4r!zt>4vl4 z@0`OAexSgxXJ+p^*1ayOn2dLFH=IsemW=P!brB&MENdiuw-0__8;cIGoX`y+zq6Kn z7g!ojv3#oM zuc%_>y~E|daJVx)*I|~OwC4HV-N;d$j`)p2=3o8Xr@G4PTAJO3UUEZQjE>HMUi;33 zvZ^QdfshRw*NlXgNXwP`_I^+vc>b{4+ZxurfI2jlV+3}D_usLibb)~oKy3FjyG6$> z0@tB8%6rzp)27O7;W_HJcb#;4GWXvYh)Tu<65OU=YYpzr=8djumtawEFgl(~DtGRo z$d5^|Foq8JCUImbZ8|=C4nz9I2FZPvVqIaMtsW&hFCM)b8qSpw{{kDDm)Z6TeAB4bs|C9Bi7~ClBbLRNMQ<+%@@Bri>{i8CS;z7;vOo^6eB9)0muRc@OipSlt z7HTTpm214#*~rd?Q_Dnv#(di%-hSATZp03=B+I4iP4qW&AW)RGZe>QxMYk}^=hoG- zm%+NmJIJv_rJS)HrcRnOOR_R|II3emM$JMadumHzt`_7Jlxk`{pF?-ZfT7HcKA3wj zmVYIVNlleAz-9!L4loV`EIg>MKr5?z*%naQag@~5#E&}6t7(EU!G+HQ5Ggx`>W3oi z-xD?i$0v$HEgp$Djop;?#}RM`aLB`d;cT!B2P@Rt_`oJ%6}SaKpW3B876c&3X#6Vx z#(oIH2VtKGD;2P2H|0{vC1=7@KS~~gy=WEESOKyX`tw|YqG}8XU{1%xxMsrDY)u}9s(7Rc~RIkk792WqLb5U zHfUcXLz&oQmUG=WY65*lP{Hb}EBn8%43l#o#E#;qAV`v%)@fYet=m!E5rSP%zfV`& z$OP6~H^n!(n6C!{->@9;IrBN4Zy5aMRmf9GnP3ihRnZ>baHOQEnXr?eIhr7Xd0{hs zEG|M#gx{1vB)M!OV}8$29q=kbMfxuI3oonmGG4-BVP0N<@T(3a=Aad5 z?M~cQXX&-ye%Jjfd(X{ul6atS+2eA(YX2Vo7Sp)qLT7r#3f3&FyR4lpJ@V_92^d}5 zNUH$>sM!1WQsH=l)Y+4}fV>N4E{S@8pmKb?GKga8-}cpuS;d2r?_+NN>tRCbOBvze z;kcuYrh64-WiSu%;-IwAU_l|qOVraB;YW+D0xrkNL!PrD*Q88S+8Vx%PhRx(_Z_BU zBAL&jPd?$R8XsEdU{s62+}s1Ef4W|Q>Pz<93KREWdHu9<4uYu4jQRoVqt*UmOG{879Ro{3u)Wh z3lPVHLt7>_V4@rqGoPP_mF2#xE@_;^T+&|GnK|%IXPSV*)GvYNo!9N2k=O?Au}0wD zp~x3eX7$n6Q06})g9lcl2Q)@*MIQM6`GcDq9?wa)E*+$jz(3xe7ea#Cg>Wxm(|Vxxt;N~>RvOP^POC>8mN^Jov`M=X=JO|`M0ck zy!>onJX|A07OTdTS(353BbM?)#E?>dzQexfT9*8Iks?3J|lZF{X@Z-;m+B~--{)L7v&R%Cc>=l^ldt?T1f{A57vsNZ`#qAlw9jU0% zoc9Y>g_&(jd=H%%4og1|gCvdW-AhOzqRA@*pV`eJd)^4XPP^1qWDBB7-_=)7Su_jy ziFXe^4{0OaNGKs#UfT|<&4D_L8jj)RE|knq0{!Ox&rRdZV3)b}$ZXFPSushMtz8q( z!k4@S98Zg7TN2-Vc+_1ik>3vf?4yn>|T48t0CsxiXuT#+Nkt~7( zo}};IrGj9qWE&sjSf#sBN~MgSpCd5Qu{NSrOQ(%ss<1guDkZT!wg}r>m2n zW@CcBjH5jAexm;C;q%fjie}-LNt59}44wh+z#==N#TQ+xn0>69@o=$%Obh{kVTt0B zwfY1p$3uc`cilnVA&ZL~5csJP_g-DYH*hhqDPvVBydiF^8HdK(-wQF{*Z#a7c|hjS z^H@|gx{pK%uUer5AJ~kR_>%3Mf21~Sd zRK+~+(vp&9XJ-6dUe5!Snbt+YKJImHkX<52JGRVI@qp1YFm!JUcE|1vU`2QQ_pfMr z=;q$ue-wBjSr1kOHBLVfu0DazcFKV+lZEr7oE-+ShM2Z(U$jqH9+uw?p+HCUsk8D@yy7 z&r0(^vV2S)*vCaMUysI^@S=(GZ@@k%$&QQY% z&6cU8!-4>9m=SK75iT)w75;xMz|ovTf0AIT;PTw>UlOwD)VD8g2i|j&Y`oRrl;zpI7Iez9yFWM%*=AT3Fa8 zk)5>M6@2H;uDIM8_&0FZH9IH$`9;;sakBtL?Jq0IyHW7QskzSMAO7FOo59UwLq-f| zzvqv54ZJuFyw7C#{|(%VF-VH{V|vT-)a9u$#>wDs7%Kh<;q$nb6H}mvmIm*JZa7;@ zW6k0ZGV5w;@J5FK7=wNY7g6jvPW1*HwD*3D)@+xTKERVTz|+33S9qycpxK>wK>(BD z7W=2^h`~8c+`RTJTCI5IhnCn88I|rE9fw%Sa`rg4N0#TqA27sQrp0`9X?XtTh*Lky z6(sH#^5-q2q0|w>o>Q%gQ0ZiJ`E(Vuh_UNmj{J^(72Eme)Wn4KaKbN7#dhzrm9^t- zoWt)|se8gdj5xc3cQ`%A6V_Yw_uJdJbOM2E0%hzbb%=xxwgomPvEM%*ALA+9RV+6aLD4nnc~dsH#e2J8gQ)UnzdF&63)N2_BK*cUi?md8NKvj z3SVD#^bHb(FcV>-_1@VT`qtPu{#Uz)e0a`zw1sTJz%FEliuk?O#SaqCoyEVr3WLJX z$D{j;QemE9V!@GRy~Y)KV#HeD3-aG)0nG3M&Imrs2IgX6)XIP z*@*CqJlB;>pXBjhZvVrmd>zK3LzQj~1JlaNQ+yO1)#rB=2dx-*&NNd9yMvE2UH@w- zB0|6tzDxI7hr zJuzs|FP*Z3C|1YpMI$Z3uGH;9W=2C;i2G=vevzN(!x0=aZ=2_WT-;@Kfmt6`s-Vm2 z!N-&@cCV@Q`}jzR_UcX1R4K*3_O6S0?K%hj3s0?%4L2mI1CAaiA;Euh3Pp&hWVD3g zi&ASJ_2&fX#eGAs3vMqKxbLLD)m>;rcR2wKb6cX+Rfg3J2GlP-taBFGDJBfip?3l3>Kljh_^ zM>u=gwr^gum=`X8rzyO8;&u)1Z)a!6jBf3|9gp%otvS{Zd2EHg!(QbhoRsVKN-1n{yOj%)Ws60{+@!(7IQ=% zWfwd5CTjLuAY6tMeOb-xA{?AMyP5f`W8EvW%->wFl$r?g@KUqDcHFjp+oxk#$kxq> z&W_FC4npD^;@X!&f6R#t=B5Yj`Q8*ukV|V}az~OjdKDI}s>ce^`z9M|xYcJD>2IR! zO`5+eAiKoF;+Imwpu%fx14F*Pvqi^_Pk(X%vw0A-RyB(jbD&r`r7jubNv^x&TQ z^1S@eg!S&tkQP*6d-2XdU{`2Og7X7Q@!6GufybTsku1uJ#vesn~rh}W8>?C;*2 zWA)zgyAQKN#b(Y4aPpciT8sG?jXMS`oQ$gLxWwTGw@hhgOb?||ch9(o;LGl4TvG`w zdWlK-`}lUPp3vC@30N;v4%h=>1$)eePQc3xxJ-Bh%kj|AP=a(5Al|fF4#w}(X^yFl z-;lhN_LaR5c0b(!@gOt?+^hitbU?pdZc6D19DM(l#FG$)n|7UHFmBMoV|nTnW`3n0 zCDmM4XLr#{BZz5LLI{gJBIk@kt1@?eKAtlCuRK6oPEUKqrx%a>AwsMZB{yaSim~&YRs?96P!Bs-?%~(9+z(SKNNRt54uSbR7($$ zvFbV*kN=GlZctF+WN@7?<36`??kx}boYWxq)@jG)ZCAfBIbOM(LcEYR#g6uMg!bRF zrhmtMH?tY$c+oymXoJyMcv`kqVyZnG;g6YUaKehH*L?69l)ZDSjpi(g5vjsY0>Uew zSu_HRi{6iAP3{6Wcr+}mHsH10=Arrd^XJKJ(8$c8NnypQaglmu&FG3Id5Z=x%0L_o zeFPr7zy;{&)+{^l7GOvk^;jS#tdn|D?!-4qxlkUB!$nHH>Js5wfw6z1u%+HGFV@na zMW-?I>(zo5++86ew4j6y0BpC z-0Je^w9&+F^U&e7;k6=Cp}pR6ARkA(G_yVY>eVZso6Q7}^n(mmyV|_gg&P|_D;?Mj z=~D3o2gKdnZ*pSy*suItzhgHH#kIGjYUUQnmRQs0AS0n9y_QL35!Ac8hZQY;`zn8T ztsFdaE1kJiG9_eLoYS4moykYKTM@A}Cus6;|?9hMx`XP4wOl5&A5Xapv$0 zhc5q<*BD2EpiC7^=Hb)_Shr1%%Wml@^8r3a#^_4J$#s>n8uv6m={6@gsz2 z^bsnq!)N2PKjv=9Sort21pA{7Ck@SQl{m0OOo-&DNNH#c1{bzDGVPuz3%2V-W;h{i zl^Bzt;?qeh$Xv>#B^z_;#90l>kN*tF+8^_g@*Px4PMavzuD0VYoz^9Y$%slmM(`(B z3r*IJWNh`hqhf96oDG6w&DcC>q43P7in^ceu+>)osNK{=K-xqNH(qu?Wlo#J|B@40 zZpGgc_5mVPSfKN*_{wasw;S+~pyDO1$jTZnsd0L8h&!x}A;C_Qx%F7hy+J^> zl!ym~AKt{yk6tH$hVFGD0m?$)+_bjd1Mf@!djQb9@SrS3iKr$C@tza@2Q+YTm^R|K z(`FfaJ8FWgkQwaBEe4Sn!J(X>AytA2U7RSpCx{mRsSyC zNZ?l(0H9qm4`lRRr-Z8uTORV-8Planu7l>Z+uw4=kbASU~3uc`f%kgXpP}P@}mlyoBD$z_3^!x z(;rtJ2cMilPFo`_wNHxZS8$>@5Y;|;#Y<9jRxQ1PdKuDjCis<|30rgx90v3$f_ zoLVEgKPj0WXBh*af65CpZ6<=rJodCCG45;IWNTMtb4}fQljYV8*r=zXkO$#i-z0_ zY!3JT%ztARX53k{!rpnI=z7nqAO80geRBbnW8l*XwAN{gd&ab^%}bPNkphBoi+(-niI)^mf2qRn+7&}hissk@O|NIHLtJ?* z+3>-?P@dUk7-yGx<;Ih9y_J$FRQnvhF}1O%J=YyKqC66nHxk7!kJKJdj+(uipxdW9 z8kX|dD7SrjJ(h^{ZiW2_QCZ&id-$P6pgj38{haT|zREVcktGi&SckU=4&wM*p?|I~ zLVALORos^dKRili{4H_3MWM4x+W+NUHQ7z5AV_p_`ll|;$mr`bn_WJV^Bs-^CSsb7 zczaZnVy>(tQDLk1;dWP!uo;uSu;JDW?puV3SE`Y{;6-9_uTdzdi&rt6D6w&Ali7Zg zg6Aw*r4Y$q>(K6E<6s@P^;)gMk|M3J%d35f2;TYkW=6tO?y>ND97Gxy-H5x)E&DhA zES!`QX!A(K(UtWHPw{uuqy}idO!+MXI!9I zuAa>|U@rTMi@)Z-zHi~l=gP&##`cq#*W_Ycpw|2idbJBn^6rVY)CBkT+1x%h_`gcr zDk_63t)_KqfnP2}B3}L#s1D%T7N)`!A;y_n+z+wHPT@JyXz`S~aM{q~%DV@km?}-{ zf>Nnns0Sy1_$=nPw&PjevW~>d+F`NJGga&|bZ>Z?iMRfA+|=1kT3dJ={OHK4O4sy# zQYAwN&$)bRcB&u_o5dQ`qXUE8BKd4;Hr*8)ERnt&WTYR5fmEL z3VQZI`Sjh&OMwvjx(f4rz29;{3dwPG0?Q%E?^PUZ>yEEuh2IEXIaVEQo6^{aCRbez z9tBN{?-Mns22y?tG!)IkoFJl}N;P>FD-EMDvxnbfoc=Kkulx|?l@~2_HlJ7_hwGe0 zByCd3G3VfnU09ui71N4~$6ymK>3c1wTMuInA$b7_xZT+=Uq5`WxhP(NLlljq4MqI+ z(P~_Im;Hsg)Egw7HNRw*(-!BMewueUJlekI7cbG_S+9v>(q@U z-hmY(xc*2yry^}X?=OIk)-YjZK#Q;gxuJ&#U z&$;<{#7y^sx*9w&j!_!qt9o)(Mk3n`^##ov2ud9sjFB)qi|)#HfIc3CGg@fgTzu5P zH2*AiX`U;`h4MyTA7?b7DSZt*-WhOyKI14?tgpMwiK63|s3Od1_|Z47QhZ#j*ZsZ4 z{NUK76n%Kc=V}JICn-KH{hiurgWAQv-zzD-y{;%J7=<7@j|XEc*<{QvHkb7D29u&Z z=2?9ez0eP+VQ6CLPp$3Y7!nzww{QC`P%3XSCRa} zD1<4MqHUNd0eZ~^juFq-zYpFTw%%Jz4sJ})naT7~q5ALO-Q3*hvG52B8-~mRv?2+4 zXxc{ef#o=-%oQV?u#1?|yW!*=vvP^I99(Ur8S7nWM-KFaOG^lW093i;t1=zq$-qJg zrLMQ3)x|3Q_=w{3;t&bSLo3oYh9I16(|n*O!c3X2_aLasf_)OtU96+&p^J|G1`$M+ zP$KtlGu7`Ba?N5x&u|qC%Um+`lo+;~ITmN%HKfHQy<9J^COM)AR(khR%n=4fhWw>> zKE-RDeu`Dr7Tk>WFh1NZn6@a3x*24l*#11@njie#I*EL`B5!1@v&a0LDs~h=2V#=MR?{&mBeA%bhWujhLGSt#ovQ#~V@Ub- zc=i&l3Ki*mFWi|2MdYV1t+DPoTdl3UkWzBIE>OOu!H|K5vhy|w!G9+I^f+E7ww(Gs z@e9P3(Lgod1+!HP#+a@Julw=4efp3B4ULE;Zj7GEBF(6b>ijJOSh7s*v`untY9W_g zNx*20MTMd-f5OL=lJBKY(Ar>Z(VA4}yM9&zjmW<37vPZRYc-rRRr9pHEK+6tRmIu|plN*=Ag8eEmk* zTy-b;3;cHo5Y>=oF);kdAqB(xEUnTjAER+Dwb1R<-$W)9;0Mw)8A`2xdbC zg`*FQC3MvNwDLIL^0ye%Xm)MJ#9HuP*R?w%unj^c5gQs0EKRSwKL*^XJc@)zwuT5|WbJAcDnJUKXfhu{z0-WS6^< zYfS7SOWo)rJ?;m=7&#(u!ZW0Yy0Q<_e|$i9lf0iVHrDYC{VOJ0I`P9Lj#+JOcO-Gm zDCT9#1!Q8onAzgf(OCXk@+JQ-xAF5n8GH=R>P3}*_(_E@N9&(W6vdUeiEwW)yzYlZ z>gt=q6v`-{71(TZf<$=b^J}tNSROADxJQB_)VTUBZqIk5O_W%}x{JIEZpUVon2r44 z2q3YaT8`@jDi&2GNA1+ZeAT$T)!Z_O^*5TnTZa5jUKV!jem^arS`2!kN%+vy_UXy8 zF#XTtbD#5$eM})PRwwSE{u;XhO>D=we-Zh77u(CB9xT4~ZXR(?hKi{lK9P`e|O z(F`bHMUr^l59%!#)TZYbKQM4PJf+lmPo%*^MTC;Q3tXqSt4BG8Ecl_xT&cuGWusGTebuc98EP9>@Mm7fjmw!>jpa7^X>8n@PWQH-` zR8&+|NHEoWK-mMagO3!wbG(Pk#MSm53 z9CeH(sl65T@exWAKv!|qn8}yqp&xO@ye(<2Q*w+y^vqC1aEz@j0!h@V=}o7MDzG7v zn$u&TPvPj_#(aS&Dku??Q4w8-XR?0P?!#ss0`kT;_j4=a<7fp0w;ll(VFstNozWNc zKF$8N#iWeOgfE<*K}=i7*icaa2vHOk&rmbXdHcDXU{|@`n4akUG+_=?QHwQG&90G!e!9O7`~#Q4gzVc_?ZIJ3D3Fweqfms9hm?444vl|l*!nx#!{_i?a`$#w5jEG)SCTfz zIwX;0Yt1Y7yR&Xw2pTZF|Mp>7^iF0I=87Jycna_5U%UK3NApj%*mx=0LZV>i7Aw^Q z@0-Nt1Ko`i1v5&;DY9S7w24&3$eZE`WeR#`aOSn=LJ)jc@9gs>DJN9_!ohE`Lfaa? zx-RsbAudh|>Vti>FfN{zh%o%CoBS^k^L6n{?()8EGV_B!LPJwI%z_Dy;Xvy3Al(hr zMWq5MD!Y=o1b*YLCzVgFugO}{eT^P07cQo;b&#LjI6lL_CGGfQ@qPv?=y-S|H8-d= z`VGn-)f@zg@%;!*feb+t-y`7{_XKGKuk_cvoq`tgP=6L&N_;*35=pe+iuQmH$P(??A+1a7e`tw#?*u!vD@ANc7FNl~t`$D$ z)H)vzyyt^wtADvB=%dhI8bw<`C&3tM&O4&m{GvlRGB(+R`>SsW9>nC)gza>ZbZ+*^ zPxV4~6zh-jIra6}NZJ5h^~fe30`&6^$?XOO8R>?X!+9r=f`J^pQaQn(s8;`iz>gd? zWF+4t|BhIRA-$^E^|26U<6t$_{v;jj>nkp3y;u&jh!3#6nL6enYw^^i5U~CT)nP5A zysgktxjY}J=J<=gHy*CfU&F~s8`0){cDW3L6Ycm-;yREMs=?em_tTUg>FUwz;?!_& zy6o_eB(rX$*Kav_cE<+PxK~nmeCX~_Fr6t-*=)yKq!byGPKA-gKHsdU z!7{X5ZvQl3x}Q%E`N|$W(F-Jhesh-~i$r@UoZk59UD3Z~Smy%IaiM0t;~onm&WYT0 z_W6M0-uSlSki1;di?&mer%mbqJ(vti;6p{lUxC@5+#~o~I|bV<{3mm!OXKA^f{fL6 zbH9I{!l2_hGDrCMzH&oHBNrXBF`4*myh-kzm>hM=vyg>hgsV0zH4lQA!?$!kQ}F8c z-;iAGC|&LJF3_h;Q^%8D)C9h$kqnr24`78kM|Dc6;{C8!;Y3-!WW2 zj0Z;|7Al+^N5l{72~#+j`Z`LuhZ|1?^2kw_I=`d z7?`=&;Ss+7_f?B2{}PNk|I}i_SCqm*R3@D+AuEh%)^ttaXIU>`KJ0@~_k-zO}#Er5oZ2P>SG?pC?%HG=(~5we~f92rQSxBmz^_OWnQTef?tq7wPl3hE9tstgY=xc;>*3l+Vo(Nwj1`R#a~sYaHv!clDs1 z#pxFGp_Y6{Ve84g9PYBgZ>+Zs{5X)kLA%uen+KJNrKlGx8uN=;w7e z!RfAhZyD7zHjX&e*iWvu^ur5bmZY=a8`xEj`%Y5|eK%76C`mGR@}8`JZut%t-8S+S zhM%LZZFC-N$bTQWj2kvwkzVGuxm zOXvxqeHpC3b?3kU{r^1mQ~K7Y1&JZQrP0~_Kca0Sw=?FH|DJH7?Sn%cxF}Tc*;GPy zgb{I&CrGoL%9ffi*TF*k2wZS?zgc}Eck3kQgfD!!%;>dUn)=mdeUyDBVBVvq{^_XM zcDgHTxY#j4O2|1)<5l%OpWf8nVM_YKw_VjsccLdQ`l9epIHK;?pbLtPB`TGj77e_3GYCfOMN|j;kMU zb?)L=QVRA>9sQjUE%*k%sl@x6?>x*q5C~h$X4^?xN#e`qkH=>^RyQ}zl)nrW*o7Uz zC|&iT{^iRnK7FT zmOt6kg)Z=$M83oUy{0V=Z|%;5#KjfCKy_w*bQdx${LO$B#lyz{PUnTOh%e-j&}&}% zy=nI5>b?uCp9$62P)47e2?7KMLkYX{Iy>;!HDf`L-=FcK*`L)BqIbCX!%B#5o7UuN zp0wY;O*30A;oF+|7%3?wbv$2N($GNU>kwto>H$)z%G{mZqqrm|8&R&1urT?huWGAn zJs|+GGp`@4!|;dM?a*kbvXCGWHZ$TQLbm)~w^BqMgJMY_yVhJZi0-%Zkc!#|E^mr5M53-yiyyVq+e0$ImZ zN5jP_$RB4~Sy9S`2wu3z@!fS;X`11p?oP+qTwhJ^&%lVP9*4GzQ*{umHZ@-YPV(Jx z`yiTO7!lt%LdWXYqafOaOgNx)1{LkG90};-;e0KyYAD|=%rbg-oGu4}EyzkLNbmHU zoL&O#O{?5AJg6s!hECaycld-D5-@8tymU^Gn`YaYqP)cx_4@*sB>>~8XnWssH>lTz zU1c>hQdQ0l>GLNJTp_@5JcZ6b$qrYcu}>cvkZI;|({ImsWF#Qk?+9L3Tb(_?t%7(U zXN_(G?sVBmfO9=^C)2Ku$)DWvDb%_(WAFQKa}*>bWlw-(0+u~2o|pby^c(T`Dn%8O zN-O;PmKg?jm|w&9Q26(R_j2DLDw9;9X2U68>}ZY79}e?II+eM_GM9Kf4GoPCMVE?Q zHR3?NLd;a9zfHH{;jqljt3((#AEEObZrb5>lU;Zzz@-&2enB&cN7Z(>7i{%eR5^(2L>&NPFAJp^_Su&pD_Qs<-=k(aC1buZ`H1_Hx=P*{k(O{n=LA{-kbX7ym&0Pif(k;e#OP|JhUsfav)(S_d(G& zEW`0xzVD$0%E!F}0R*hyAKLVgMR3IDD(D%}@4OaPlHXlD=u`Sxa(+6B6%wZHws@4E zZ>(qhOK!o@r*YDhiBp(2uUvlCyP4t}Vxa!?34-FgeazOU!mJj}2V0cu?e5dtu3A{U z3~FnpPg+ZPz;M3S6FR6#8HrG5bu`b4i5lpHCb+Ly`pm8QIQ4g(BZy~2x$AHIaz?nF zhZ9W)nT8zmyp>qQMeMQ9p105GAEXZxK@sw|EZybSM>DEzvU{-{uC`jy=VNaKBGYu` zBZ?gRMJbIIPd_k%(IZYiD|goXF2UTzm0};Ize3xl+fhl#4jIY{8f{lZJn$bgr7A$b zfb{-vZfmu(>q7at54ch*3L&BNd`2qN?0vw$W=Ti^*6ypq-UGL2z`Sn{P}Jv;_t;F? ze#2TQFAHK95bbtc=>l^e)J?$K=7CA^P*FVpCat2}fJFo+xypY3M70hDM`Vdc#3goD z=#UOO@^x8o9WF8qnPq`D@*rxC-5G1Z#m8Q1ZYf45Cf#iNL^b{%e<;0$*YRAXW9o0& zLLEH3LAy^(Oq`V3;N3USQ{hYEY3Xf|xuVXOp!1f$+g6`sWbS{^jmyZ$ z02a!CMThhP?84P^z&%%!{r!hCwp5gxgppPcp1}3O3mcPLxa?829?HtH3m4@_vI1<6x9aJ&>X|+?&bbq$$qy)QfZ=@Bo~IffPGGUN7qORG+M-y>AH97^Y$`nL z`>!%%WZ`1ZZhK0faEo$w1__LFZOHf)wMAoqa+Ha5cK}LuJ0G$ka*>5ekcYoQnZfhd zuYGnWXrQs95y~cvm-*W+lO#TUjS9fuxX(>tVW*5tO zE5A8v*_8QxP8XxY!0Z^-oN2P*Wq;8-icUxKYYYq~&<= z)n|N?x2Ah~tVrVgl|T8;v%REb5_^F*T)#}^uL^ZR#XR3m*^!aitd9CILwx=;Z(euL zW$XuTUBSi#L4*+#6I1yP-k*Hl-{{@&l73cwRaHrjl(0gF8!LfbObGd2VBs=%N8Q$f zS(gK-71Ty#Wjo1@E@#|^Z>PTNsCPWtnp#mH+)|J6Ar)mIF#^uPVB#JquW~}F%?!(nw#>~Y{)tDU_tsMhE7R<+GZh5Z z%{_E4cdA^6PC(IXudBrr*3>@9psb?e@V5);e|I0#vfM|B;(1W;S=Cvv1my=LL0r@S zg+JYQgkm;__U&tVhe8Dp`+L%!Q&UDUTj8b77=G5=biZFJ-_D-3e&};HGEm!XQUstH zy~=?QLe>cyWKoFs#}N`y&loYs7X zDvixZoQbs(Vwhs;gZr!|}% zCAE}flC$0&^a4zK|K~RcfW$j9paEd;kHMn$s-sX3eE|)99r?WKaKO!>^}0UgdpHbx z{ne1*C%&EvPR#88Zf}wR67~_E5F}Pd^CjLO18vA&ud(c5pF4OK-qK!S*?m$ zOHt!M1W5$y(Sw=klo;VTkL2RwA}AV;=Sbj)>u%}-b6z_$me_UfO;nIrCIU)=TTdR|u*8?j`7;Wq)EX?#Wvpr)|e306>0 z0Cqs&GjUY~hVJ$pj@Qv(vC+}9ES=9Ib2UEb5mv%-=((!8D^122k`EKqV{J1BJT@BY zBVGOd(%hxbcQtb!ic|}%r`lctV<5aBaT&6QovnxQZx3MHs@T|frOkVHcNd+2i9F+X zk)<#>c}GvrX6IKF@I{6_it)6&oS{qLdI`wVfM>VAXrRk;;hXu;-Q9h2wvEt{d()P_ zjyy9&?%!{85>(VfmyZz04=*wob4n#JV#s$FQeS@ske3!CsoHnIg7^`PW`ia9`b-#$ z?D|YyS()O{r~##CAvASo#bsgsr2+S09+d%T(7$M_-i)}Tx-D{`seSVKZ`TI6H*q0v z%faV_O=HF!s;b@d^ZdX!g__zFFcBTMhO>0T+n(d&Z*FaA6iAgMT;JZN^E(%_)^Be_ zj)pDN=jmc)YbH8#C0!h9ZHsa@uPY7%>>6+ntrXsfa&Y!;XA^>$&V*a#PAuc6&G#16 zHGlNO=|ly9eVU@a1Si|dI(URKd3A`jeRyO93lr1X)wRAdi`hb!Re8AL;`|)!X0``K zznnRMi=S$N0<)IzwADxKrdo0jS>dbE+eY8`_*5W3U|?na&zfMeZ08rdt6sHTnwUN2 zu0b6E@hla7W*IugkGAKeq^&lyS|`s zcV^p)pEy0Q{@&*E6p}6~V~7&>7b@p~_%pdd3LPrZC?f{SEiI;*u3A8)1>9AI3?Wj` zlL{Dms|~)o)a+VhG!KRx552l|-oL?P(S3w{A35A&!p_x2pHNFAi|a$6 z^R>rV`DvOuq?Kp6WdYQ_?%JQmD@*8?#3J*h@50H*ggk!O;ZOIEj&1|C3)X)VxOU8B zi2SvvD%c9Cyx`vc-agwUn3#IWX+8{8iPVc&!=5nyo=)d!Gy;^I0ea-C?x0#ulFv0RM z>Hjh=&`y~yg6aA0#w8>mi1d}4JE4mvQ>JN|1GQCw1`~l{lhcRMcn5Bb;i73q6&02K zmiqsgn8jY0UwAYmfeg<|FL|uWO!jre_WnL##jXLR#3T&@926zJaeREdy`2ypEmltu z=cc1EpXDe(C5he?KrSCO5>F~prT`Na{Q~Y1xH~lsEuvoAV(Cg*vtTcpTMch*sGD~$ z{R-ZxsOaebPHKpV!xZ(_;79(=-|u8Q_^a#d)gL?kF-f=q!rs%vV}dcC-o(~6H9mgk zhtKwp9^d-?O#6j`pwZ)$_BD-ZlB0$&_HY^aaWkY8cr1g2@VQs({VhEi!uzRQ-8^m| z^6S%7>Jg!%hx^0O(Bf(T!$4+%UQncqvQ~YR`_S1NM^bWZe^});F#A4b=Mk1qKns!M zG2DK`{08zt*Kl`tsSDRgQR_9uHmx@c*?Z2v_OpsGi8o^ zUWA42d*3=Z$f)V)CaDVsV$weIE?9GGB2|npTWbmM-uru;hRd8jGreHd zS{FXV%G|JWy?o!vDAmV3EwCcgmR8P-r%W=>DykZ=UF!V)&s4ZnLOiwE_3+ZF9sMEw z2$tf+ng-vsJ>)-y**ZEGHUIa-4X7cwxVYHaqwJXbJ^NeKztI|O{kul+3{whCW*D3P z>{akf_jQatN4emu$vzovo!<3@0y5c!G~_0y;!d5TLrQxozWVFsUyQ|rlEnBAz2CoG zWTr z-q`qf$$Em^;IXl>^KJ0er z&0p|uAWB-Ow!mHzOD0%*c-i-iaqqPS?5)yPZrk<6r94eRte$U%=PGY+dLA$QJjVhY z^n+EX^syvMFH*LtF-_}jb65X&KQPct1P^`fES%-T4azlG9|z~BriMmz*ni#Bz_*kS z{AL-W+X8%i^crinOt)_>d=LIqf*0^@*|XxwM_{K6t%9&%L)v*4FpgVGO7?+qF)-it zv~GV~dg`_Gz5e$XIGzD79B_W~=X#6XmZ+?_dwC6wdUJo)dRz8ToE@|DQ{hvDlc1qN z!{b;*f*x|$`ogy9$?R(uELs^`tt+xEhogT!ex9yKzF6kYQI`pVOby5oWS7Cu?W^aY zI8a3W^yq#ivsa;6zS8tW2PHafUlQ$bx_$JGnQ@k{iZZseEWAF1{kan6;c*3*1HuXC z_>|It$AxMCaeb!zv=NMzop@9z?O;R8p;whQ2r9rl8wO?22Lqj_MVrWKb&_hnxc;*tB zoGq~=_&~f8k^h%g44aJX2>39NkyZ3+@dfW-hDbQej%&YXIZG@&f)N(3*?4Foa7OF)2QH1LYmc>W*7O(LwLg{G!=2A9 zERX#`hSt^x|6Z<=vsiO&Gdq+UN{39pNjlyZX0qXtu$;NPSAt1?5<2#5{60Lq9=x?& zS(j(xE zdf?GZa2c;HoVCYeQX`BAe!e%c@DU;L@)$rBBm-5GV-K@ z{@Um$3k%E9$$GcQ(+%~CRuGx6|oD1*&-~XgVM!~w{(%Z71V?=TveIr zb-K(j{wbbpUDXXlhVM;hRkgFWR#{4bL&xN3we4>|69POiv7|E}nGJs?4bI%`nazSlGdAUhb278;@^0Z@*D){}Vp$wUyEk2Cfsp!!D6#V=i{n3=151QuY z)=O(EYIh1bVsLyBFR+F`F{~E_XnEl*`)(SNIuaQcmMY}w3M#DJ+;~?UqdkB{x)>&G zoaVKwhIBc>qWl@Thw1e#uo0vYcsr_nU1?JHqc)dXkRAfzfT8Rktl08Cf|}|K(k0FB zf@qN5L(&jlw!+3?nVJDTPwpj5h!&?E%1OE?LmmNd ziqWa0Np{fvNX%jB8NQaE8aBM=eY*eXX%FV;Wa{dGnbKSf>s-M%mzkieKOTcK=f+bl zj6b{#mX*827z4d%?DHll21C+~=O%t;C4``cUBA}J7sq*Ti3JM_4MlrCqX10JGc)x! zwbICli2;a5N<@?b`hqs%ZHd1$6sMCA^Ac|)+4^<^BDPOa)1Xs-Cpb4j*G{5@X*V{)ll0D?g4jw?vr2h%QhtH#c{5Xnmc^r*_jpxsJ~X;xky*kKE*s+9Kr01H4x#WCQ!ZY7&OI@Mz`?1dW(>O5tm z{B?R($zakFY$9a4@?~fI@K##Ti(G$)_LR^xPL4+kX#8W*U#mY@zKmt4{F{Dqw2ryj z8?nox<8`AUcYQ~_vI8J15bcx!U<0&pWl+DtLmun;TP>XE5;e^LHXM0UBcsjUzZH}W zpv)G4;z#OK;{RSse?V-l$+SJu{r&rQAU^i=^c3;Dk{kNvjAyy9sAXZbTfkOPQ0|a0 zSrMm8{%^_lU~#PeNQAj*a2;>$k6lWeV~N$)r@8(=)vZx$^o-KG$Kek?0~?vLtFxAQ z7UF6mH}48vfp{Gxaw-eO^;HD$iV`lzKg&!2*nugL31WP-sJ@|*k%-XH9I$I}18n5Q zkdiqnIwL~?k8>I1Lck{;4w9#@tyZvc=HXqawXnEoz`xk~>A2X$O2;JG+1331Dcwwc zLCs(|QsxWX+XFIA+x5RhFayJSX*BzMW&9|toQD1X5EgaM4fs5smKn9Lw%aB*xKzQS z%zHzH_ea6I?Ekd@VVw_WveiwqG&qQUAiex1p8+z_ow3YLFsNl^Odxqq`FwY=XDsLg z=H7oE0hqrlJD>h-Z(m;QeQcT8gJmXZ!lA7z_e|4_@)w@IuD6XS4Of^wHfm?dOqprY z@Ai}<5${~Q+Wd`_dvI7*@`|Ip?7#~ip)asfk-#)4q@e|&+htwhXuyB8<2;e(aHxN9 za1?wt^#8~{cZv^nRyHZgRf2}56mi1<8>=?8VSe@Ml}?j=c3vLrvC82AXjDr@mDAR? zJm2JizX~F8(AgIhI&qm8QbnY5z}D;+PDD&wojch=~voy=1>rzDSRzudH4j@4(QD8^LRN& z%xyjX-5Xc*o~Xvo&=3j&y0r%e2FACD0S3m$=LsDCQKSNl88Gmupd8yJlFka>-2DqS{70nT<7~5g9yyVB zDVS6-h~5hY!-ENowY9a>R8&nmwqU*dP5f5$1K=nF0Tw2o)YO38gBAF0P^ERW++2HA zPbr=La6TpiVqSSrq4woSr(NN-6@F7o%j_o%2^{I)8$6-u$JCXmS`S(vUY3Ugjz1O_ zgrubMjhNpRHa0eZ>j)+mmS`iU#KD2BV>7TQG;M2abH+xIcr*P!rI{x` z0lNrFN=l7bHA%qBUe>f(fquJ5%-VQ3INj^^z2Ssl%a*WV`!Ng}qP^5y${{b{AqTv) zNI2c-0X`fIjEtZ_Z4bC!nM;nj+Pc013G6y4uLufMN&VktRTdM0 zdneFP*4eFP*zyH?0C`Bgh^II3l>zPKe}6s&zqXtD7o+wcT2;h!5+M&~u#Z8;!7Bz* zbZ%+{LX$t|w>23jC-x9t@B;!RW>J?InSRS=Cv_S4718g>wl?XIlhbHVS&)Eq09IMd z!-IbXn^RgxN2k;05twuOJe*2_9Yre#ykzj9QCIA<31{>S`GAO+4hsuwY-&m->~%Aq zEpEJ!!>_EviHDD02xUPYSZ#B1(R}Wuu+8j3rgUx21ya+T-MqX!AUIIYoTCT^(KN6I zNo3OewQgUakS^0m0?IU~dV71{0??DsX$SNnMF&V2Ft`Gfhs$aS#x@&Ub}ZdrLb z==RixWFQ=at{it@F4K$H$A~&ejf2yaBd6Ao>;ZxX9856jHyZ zoSdBLNQz`5a(?_-lr1Bg^qWQEes~Bbxj;wxtFMtH%UA};xg#Ez{PC)ShB;SnF<(SJ zlwhmYVIiH7xcFMIIS~t@?0SnS_&7{V3A8qb){I}BdTP5(eT21Cg)Q$DTq4q1%HEa1 z83w0dEiWMW=ki&frSk@L6917E`SKA0(Wd(%%~(M5DGbbyAOQY}h=>3lD&iYxlLOEK zfE*pQ9(MrX@hhj6UIPBXTH|lw%Q7%7Ubsq z?g;G*S2@B2n{{ePv@4$?G=yTH-5%n%IkQ9p$;!Fe%J%+q91fIMu-OqH<0(k1#lpv* zEB;6)2ex*!k3N)?<$ktU{Rl>*NXc-gqEAk_I2}kSq}s?C{%N+5=*Edz%N@$F{_3MT zCQEi$NA21>iWupCTx39w|@lr$vjGonRqAfM1b# zi`$id4x4$F8xXS1;3rX0? zxmv?`nfEvV;sj;-fvAL^4Mm59X(KWK=seEsZwfZXw^p-dGo|WI&C8KON{jn+`m6s{Ev@vGTczy5OIfSRbg;asSvrBOwWgW-%FBQ0S&Ba8Wckd8fV zm)!j%K>9V*dt#&a2?7>JxHy%B5e)u&2*DYcAQwnaFHdBCil&tXfcEv#3Of?ig!&Wn zR%Pm|#+W487ofy-f7X5VC?L+2tb;bppRfj6D!e+Wqt)f;bpb_yZ_|>D)a3!GWJLV>w22383B+)r#J+Xt3ZwKu{+jJfq-w_ z-@o_P!7m`sKx*+8U)9VxQ{>lD3z<5rKwLE!F$8iDB#*+$ml&yu7(o_NMPK=Mz)EHe zfb>w1BQotXw7P?yDkwj-gD39=tl1CxdVm9MhcWK;p2uv(x)| zhHfXP)6K&JIIjKf>jSS4I0U}D!s(vPbHQS%rBTbRNl2$KEptkD=XJ%gexmETA_SOO zI8|(NTb#^zoYdVmcC5(JQZ6qxZsHA{uSZ@asrKI8kCDxVSaSp^EDOClDwOCwcr9>X zMxM_5v+S-X0PbEr<_)rL-*mtthZ#Yylwt9wFJ|dMiOX6b zoq7*iPyBIgwy>}O{GGFvMOkpq28bqsuiT84OT165?FzqRYBku$iHSB?KS;1PUEaSb zrsFeS&trcqO$(o$W%-z^hh&=02n8%s@t28)V`<5%dLxpf6D08BaWfYhx59_jqW)mw z^HtWX(T5;(t*&whW~uD4N+b;k)YGCAyv|6om32|ik6HY5j2SFKYA$}&fEnsPG9AkF z;2ptf%hqw@52AwBFAjTwf*{8O)C;%(n*fqxOfDtI{X|au+0}{^a%kp*auO&R2t-e> z_0lcRn{F>gVO9C5k*;6;k7EcCd!DQii>~X_varQQ*YBSd%tTpa6MU~JzW-s}y+f^7 zl~toNcfp0E1=-(O=;HC$OH&7qn4aHywWxA>B+DH>Agv_ZySTiZYM{>>Xc}^)5X9-j z@Z^4_v6!ezrzAFzoAeA_m)L&1!QnD-7%&|Rx9or;DF4sF#IS39Wb1uX`ukDB<#yF} zV`}~|n|G@)Lm6Bo2e?$PpqZa53@$;^k4XdkHp;*$3;<={=SLpE6>q_~Ggr{u#)N86 zH4c<(k_%YoL*otQ2}@oMIk_5dwLu^Ta1kd?KY3GD4XpQ#-{@bw3#9xukY|#l17f@g zOwiD7Ys=VLZpeqmBS3?xm(7M593g0Ag1mx)UiVjrU<{)%vL*N7pR$s64|sjk&pKY4 ztQiKKAbEdsA{!*r6F67#*tqU`GuUhV0vYR_V8K4h(j=T_GBqV=lb8;_^Ry+YTzUFh!oHUfqW(W<|XTOOz@d#tglmC71=2iD zyR+;+7z2>Ey{-7zY)%h%vAMC7F9c9tZ}TR1AMaL-SQ$R zm3Sa91|E?o#J?d`XWG-xgV>I#%SpRDX->;RVmlEj)g(~;i~6i@snkp%5MCc|>oi#b zn_;xucgb=EKGyT!6__V^gCirk+1Zhhkg~NACtrbAhu%yIlQdLcQ0u4>e|v@vAmIvZVel?NZz>MtoqImfh9`f?X%+^ss-W95i**lkcj8DyHQqTkN z5kOBX3>V*l*D3%g2IA;!AgH*ujSqp!O8=qD zgJ)H=qSQ!cYSXVg{Ts&q$J_84PYvy)AFD7Gq_0asJFY48>m*uAHcvTubnbiMd$FT$ zeXNicXBUUPLF|qpY)pN&gVxE1dpZheH6d4A|KC@M1hzu)X?y$ARI?cK9LD|fWFyaI6-SB zE+!_q!z(N3L#tmM#l?Fg0s)u&%Dx7!jz7i+@*6&r(ZYOI!zkLz3fMd_imXi_vozao zpB~i4MdR9DfgT4s-1-{(VYi!^~;kChtX)5b;A` zPpdL62;2M9ysN_D9lJ^xg*>`&CM$NxU$u!FM0E7g;bEOB1OK2P1hokB>Ex7@kc@UO znx{szT9=hvEQ#_58sLJSn3(8{a*O%9kSD-6T&@|XE4$?f>c`!XR0YDZYP3~srM6z( zdtx?&53CYjutFq2t+R9KxZ(w?sHDJ&o1l}^`3rg-0NC!^N8huma9c|0kKZ3QxEu?w z6uJ^8Pstt=n#+4(+NDtDcD5{ZabD614uHn$u&6WPO)fl|q%Oh*G2@7<7aGXS?*Wd9e9 zJK8LHFC!xZIP$+gfR!l7Q*(229G#qAz7s&K6#I&;_}fBRVA^j*xB1-9eKW(MlgrcY zN7(2`a^9-cOmdW^{P%huzn*F6^BqK+-#8mQD~mO^C(zq~-%Jj^8-#lHp6eKZSk=_h z`iw2M8SrAGL*{iW_f-5SGF1*|p_Mk@&DwGQVh`(#QVgFt#shjZu(2t5+fnziV&`ZU z*Xoj__kEY_qkH}P<@0@vz}$q-f?Q56MO<8r`Gr!KN5BiK_r7<15LoZN{hJ#FxB?In zf~v{?{~--D`#%%I)i-=!{sQ+14_V=$`48a14-VuS6EO&^4Ann-0X< zB{JQb4)hvlA0GT>>^wwM%D(WYed+7LEI|ddhXfTPbC@3s9Oqz|{J$fZqLAXSWaFWLh?Zn(%fL%xG`-eu3V_qTx9}ye*%~Ep#W| zM7Qn>&ZmpC#@H`w@N`f=PcLwQ7^B$9%jj2hSNyN6Rt2V&IjLzG;8g^AC_*TJ;lkxG zLHcL#ZlL`kxgo(vMg|5yH%|l6i1EOYN=z)b-c48&1)b|rJzSXQ89s^pifPnfvA0C)v{0JU( zmH^2S0=e#FrkNgpH<-2E-QAlT8%;VDQBfFxNeE;EIE<&~Jr=d_v+MhfdDft)WRCB) zlEF_W7nS%34aPd1ZmmJ`QPe?E(e+yw-+#d>bke9>2_EQmw*Q(2qKse#3ha>aq#1f3 zCj%s*B49NKcF)0GfCKUVY?F{`?)UC>otBGpfQX~pD2jOhsnImUIUhY_YDsZ1{cm~?c7b#r3EC3p_AI8}aI$3JnBq4#UwM0Y7vG#_$3#j<2w> zC#B>3u<}6U19VbhAtCFI&7h-Owl@0Yi$W!+d!fapRC*9Jbkt=<0bWU$M9)`;+I5v? zSAg0xqK&h#wjMU-0H?}7lg`th5U0QWoY~ref-kQ~)Y+MH1sgMOoe4O_Bz`xrl+hO= zmLLTlq$vMgkZ9?DNrW0}eU;|=orn9CAD6zJCm8+NZmaJgNx?xc`9{+YKku z(jADF%5D_6cvy+iv>7-a0R40RN@ViEpf#`p#kYsQMg!Q}}@D#x^P0h{i zK94QJW(3;k=ciC7NJX=<9~%D}R?ym9LZ@qeCj*iS@R&d3R5T{S!2cu_Ab>?P+{39>s^LImJJ?t*$A9v+~f8aTpXMgRWk zSjHnmI9moqY zB_arx99MATN=u%+?fx$xfeM<^OXDc0d*xS-tJ$RVBG#q`kp+O*z6S3c#lD2yD$)d zbZH_$gX~%gY%HFxa&qBdQfdBpd7wFZdU~M!f(5Oz=Ky+IV*lP7KeSjhj{)o}=4tWq zuY@zB$jV&-1`eoN5s^#~B;W!_vFyZUvlk73prE)LPKQ2_(~!-+h^s$~cZDJP8l?QE zZi3|GnZdz`%ddz;G27os;fKDIRsoh{cfC6Z!sdQvyzHAj@HL^$8O_e_7@P~eJw5Jj zZhy$#KpzXfL(;i&C?`mf7QLxIU%j;WJb1Kx*(Au%nVAH=LT&(v0heT9VY0AiYDCBj zrM<+i)9P9-8HVRPBnrw$5wH#QdI!qxsX*eD%#V6TWZZqQgNv=^vMdO}52f@Fb#P!0 zM6D0``+SgYq4QB{?x&*Gw1o1XV(x59`B#ixaAL}+4cKC;i44%l$f~&-aNhzD5s??; z;a@@dBY#!nocWpqCt*@8{Jn@dY#YmgQbSp+l*Zi##X%fAn{6r2fa}8_b3Jb3Sy(t)WVPUjWU^Xy?N?tr zwui$jJmZabv$5v;=&XN!3Zdbpf0P`X^vDxEh|p-!8S{yKj(Mt#;w4AOG_{`<-BSoV6hx z2xNi~00gKeO?|7Vkc-N6E5ucI-yjNb);dN1`zNH-*(Ji+%KqbA=Vlk$)cVc%DbDey z^9-TW#HCh_CYP*EHFzG0=L!s4qRZo(kS`_fcJPg4f)3lB0;J3Kq>*-y{7nn<^Ug|K z^8Gi3VuL%?x6K2qLvkS=@Vt;TsF;mmLC50cw<4?u^oopTK{@XaiQ!uA!h!Y)T$5wV z%(4iDBS(oXmg>&MWZF6seKN3GFff*TkySFd(<}X+L_|%jU8-eZ+BMdv{ zDQv_X>IIr8tk#d>p7^9BkkSA`@hR>?!^>}v)(D{z*RnJGt&i$-2<=Y~!Pd@5foAzLXsRBs>1cGVW z6!)!gTU~0h3IZ`v<|Wce^l8RKq1d#cTR=A2SM)S*aKD#&etx>{BxkLAZ)T}{7Yi0> z&c_~@VZgD0gZDLdNCU3u9{lO)X+%T>D2G~9g9X7+J_4pig~A4Jpp%=zrM}F-n5Da? zj!>?Db1w+l8${kkXuXQhmNzlk_J={JD~FT0W@Bd$2@4Z=k!6bq)s>?F<0hHLuGYhm zRk5vOO-Tp2BG5^%feZbuYsum0{?7gR=R`}hAPmdtoQp)Twt#HVTsv%i2!zGJ6Fy8K z8iNlQJ+^q=I{NxPr=$?s?Em*hbc0>1y$nTbGN(|X4KA%MIJHyM$77Z~zv|$b{Gpm^ zfD`h9%5MZ-xN!AFbtsEZ;_){(H$eJye}AvvWH0)^>)4sNjmxN#3*vVj;B|a-e{jWD zq#QP>Jvca6R}PIl7;Ow9GcX*0yakvB6JF^#@yiQ_N8kwA%J=xZ-^*Io(Se7mGhHMp zN>~Ccc`qn7vPVHwjEsWPXtVUba0(EH_2uXTPRI5l*fdU&`r8_?)99n>l0z=xA?)C2h0T$Y% zfWeMepj@neIG9!eEn~Ut`yQQI#t6Tq6R5vNHG&Rp;>kwE$87?P{$GpchKx83t;-7!*^E z`SuCi%eab)P6&gT70f6nN5{oxCo^N?JUp(`%gdS1dtgh}Ei5kn;9z@5@wH9(umvqK zFCxdqz&?{Je2`r5Z2#XsY#f~2+}!&QKAu@;sz3&w>HE}NS$PZ=-Mx4RvNy{!;8h_f zXWO!`ccZ5|pSOVjbn<}l5jgcWqeQ@K*>k!jnxM}dG4%!Qejn*NVU+$`diY)I_Wb<( zD-1bFcVD#9#5#n3%;(qKwRw-@^eD9qf=ov-jaOM+z0gK7PylqK2X&hu!hdGO1{~8pWUa6&J!4=tc{)R);t{~SyTVW(D^-00NaLfLGtxXp}0!Lb#Xr} z7Ars1zTwdG2CAa_X4#+B{i)cypvlBOV*kj`^y2A&(QkIzg@=O^@_o`-YC*-4fL{Ia zT0f|Ghq%VFzUX}}g;8sJF&y=Jr5@a3Df4#z(N$f}{QLN;Poudcf%)G34jqu{a7pNl zj%-SZ;P6@AJUrQEaltiDPF~d(Hs>9MjPF~o@Wn_>z0F+`bWIb(L0;zR2A1@3HvysF zqwzH?qLb~sl%$jlp)NGUi?*E=4!?^>Bf&oGm5CcTi2{oILhd**aDX=7%F0Ta?)#C7 z=P=iaF{>G`r@{afPA^nPQ|0dvH$&lSp<37;lLf+HGSln;v=MMkKBC`2a$<*_V*)Rr zvOa%#hfTEnFp^uWtdx>K1S%=OCq2cfyagj7j?9m!elYm$Y<1IheYvDm{Oge`xu#_^ zBU;GMZf9B{w$g%?{aV2d8}R61{PXj-lk|Pufe;x4%of^7 zU46D4pG2KruG%_o9_NmS&$EOX+;mp05weTNBR0|8IwA*@QK#bZY}Kmz5?VHhWKlKUWc@Wp7}qJzcYB zXWN5vm4`Z*Y`Q-xTa+{wKYSLI(KG8%qNmrQqNUvd8q395XcIp~BqXqxU0q-MKHOA; zX?b7}f^N7Db>>@Rh5NC-T+a+Ee{f+)E1jkCP3)_u^il65&=Xzbnz4PbGH$k)T4i%6qQ$+_*T4WO<4O? zyg8u<3CXHDs%s$FA6L@v84ETdK*%@52~E)9*$C=MdG`ti^?Xl8@&new|0 zrlVrIIQYIJA=S>hA#23ygl$EY-8%7IE``7^1a~vj39pU8t;(c4ext&-0l{t3l!QcU zEh3CHISI&GZR|NVqBQy-3Mozew!Sc1@^l$1z^9yg_(lSZ0JYZq{lS)N11hb|mu>@E zPt$?tk+Zb$E(5kwh0>MT*E{6KeA-}CNESR)`}ZujktxKP;lFix=~A`EZ1F8&e(RFU zw+%NZ{FWjyXcLS|LzMDPCB3!}3>Q5)H&J4_5Gr42#|(eu16j1+E$uJO31s-d^SdL?X|5k4&(AO zAHt2;U9T*axO;KQnoOL1=XwH&rHo_#xCsKhGO`m!Ck~9Vn)}a%`TkY9$e~VZcqLxB zDYQacY23Q@#;ZI!_Xex^>R2s^-(swhVJXwjeV&?m^c{Ytjcgr;^guH$h+hmf)ntMN z0@|1wa=;uij$#h@N9Y=^PNm=e3MsTEw!=QHg+2$i-fC$)`3rB=Z{e8hO{{{q339-Y zdv1;{nmNgx?m4XjSJ`RUacwvLlC!Cspxu_q*R1Y>Ez`ysqLC+-KHB!v0b_~$9_II@?`(xVxe%hNR zYIw+cecE@ISA{|*M0fk;60DwU7fV!-B4$-oXWz45!rY&1#f|CNbz_u@mjhi0Q9llr z*!R%GGv}7X4XPnsDi6Cf-1c@8U5yOTu`P~TW@VvLnvP|(0|e^g;sRu|mSDpQFFSqj z@#jbb9RILo4u3~((;hBRdp<#}y6u~)QxhJuy{(6b&}WNqQyihA38Ldd-o+oSr!ICJ zt6{jS4J~aFyCMNHnU4(5n^~XKuw`x|7#j`$H{4RXt9i2a$bhsuYUz7)L)(i{U4jEr zk}WEn`2iv{6Ui9ORfQ!KPV~k4KJLjz<9dbp)!@quY4y}jT*s*0q7{bWn_nF!YI=kd zx9@@u-Ot=(QaDEyG7uqZWW^q#Orq&Q{Km+)&yJAJ zcpEEYD3%kAo`NB?N|#7^Vj=Z4LfT1@WiJ1%n zH)62CG)16n3H#VFf>)CC^@}14{)ha@9nWUDS{8Fjw9SJ;D)J#$tGl~*(Wh~Z1D!)B zkp~Y%#;n@jhItP`p?_iDRwLfNk3UuD>+j@2LMPE{uWN2@)?^SY%^opHYVOiRt{;*( zBVuq2jg8~5fz9mv_B&1;%Ptk_*+=^ zthe+GUKfaHe7#pr=Ojj^ZVnFmjiV>qn3#u+LpvzDN2k;CdELMr&3&( zvLBgjtfc8G&qPj@iCn9RytdMvfXw4#xOl{No648@Bkmp9v&W%ej^o#v1Bon*{7u{$>5ulY-?k4 z&rd&vnBXDyhSeme;CJV=;)}Cf$dzw2e%(TDMI1K1 zH`KqIKnH0M@Io=I)>u?Ct`gTJ|5F`2y&s{7nwUx2;qrtOmOyNyN&F#@mdw9+fd(3D zoO(aTMt4p#eiaWCgy6H1tDpNL$|-flX&^YLm&`5JA5+SaGlN~!z!LjBX(0) zR%^cP^LZtAS-hKr*)~7g`(_Fm+sQkJ=B~OBuv;+scTm4&mbRUJpC)Q8D>}e_`w%$v zG_6Do9gExfJss1cgsS1JMyKTC>r{3rj8}ZQt`u|UZn#vCpZ^BGrK)<_Sh1~4@bC*A zzqi?ZU8!@6-tikM6o;LY;-eSEot#?YhQzvDfM#iRKFXTw%#%}7nPNTtj(tsigeznQc z`NalG&#&ge8V}fkQNM0A3Mj1+XzWw*HTUW(=|iJ25@l?PIOAXSBMgNc6-LW@4#NMn za=A^BV*e^cXQBV>Rf&rJ8hZesc9)kEP%g=zjg4QCllubE3@Fz!*G>|ZA7IyC8lWW} zyBHmFA(XeBAIKh`Yk084LsICqb*|=M|7}xjmT+!ZzJWqD z6qiTRmySoHI98;F^4i?2*pk;Zl7-7!-_Ps&81ibVMc@3&ey9dHnd`YrLZv}Dp8KGP zsroeZPw_67o<@Jh`iRbuJ?MmEO@L%$W^78TWlZE8#uOH+&q4bM&-gD;71Od>v=&mE zlt7J+|E8vD|77K`0nd1MfkyI=P*J-v_~Ie)QH08eWN0D$AkJh!PiksVQ@%(U1!BvG$y#bM4Ee`Ds-|%IaKJj8oPQ?k|8> z$gvqoYDMwMNCE)!NTc@4*;U`Rd0DIbQ8sqnF-!=qB`TLpFV=w&rEkzCGr;#&<*E$*U*s6l$} zkgXg(a&Q2Nd3m*UbO;k6iQ;H@L-$zehbKL(wYBN<|4@%}wYR}jw&|zlJ76vI)k*N= z{AuN*O|Fi`I!srbn@~rVRjOFC``bFMQaI!)-kC{%(QFNUcsbCtnD*;#@s=}{!ht3h zY+<^!z?PB8r)kldAr0N0SK}Xx2YG4^ihtQtLGU2XTFZRIsN#Xl*aRcA_*x`dR~86cw~x|60MfR^rx5O5k*!JH%IV?gig7?7X=WZ@5?RYpyaD&? zbL&~{yt7?#!D&%5-;|LG>!<0xn&Ot&>D3JQZEow_Wpr7H-@Q&RG&zf75$x)`?MHh@ z_)aQn4f(h$DRyNW2laXhQNq|Z00ROz3Xr3~ra}AnJE-kbQ5e-u#vjc*vM3%DzJd#e z-or7m2qTL7r7|2wY7FmNiF?|$kgcN7dml0bWD;UN>8Y(9tdaKOyy6ep9u)3*^>O$R zcRS;nqe~-e9z_a5k~97Khm;w@1r_L7e)`DsTvd%X5Rez!B-{K#sef0oPNx{R`2HjUIa)qnUi^hD%&BcpNKn2 zXS590hZHGm$U=S5Z6aj?#1CV%(sZf{!hiSrEy7Mlq52pFi{5c08RcA#i=V9k=od+>TH`-Rf53(c#+kKCj&^1HY1xKq{P_vW?mZ_*@8-Ak zvMP=rsM6#WUE*~FkTX|n#9Is=VPe1Q2u~D)hL4G^z|JZ8FyL{zM zYTgzzShtqU0u7mJH_7*LdJX3TyW0(cxCX&4ScQQ37`<1@XY{?g4Rrkac%c2w#_R7N z)kB&m6De$$X(;$HEK3$m zK2rICn+a)Yu_pRE*K=}+7j_?eTa+4AAEB(RBjH&$)Xw#YKcOi)>g}L{{}{YluRIdu zU0mK7XINLfwuh^B+H||)KCv-`!V})W_=w`#cx0)GmgzVf74nXDaLIz^+HfUps(wvA zHqdPRQFscOF5Je5v3A`fvb{*OdpW#gLOwTu=0^+c$lPSDgqw zsVT7Dh3n@bk1;Q>r)lbK$OH-p`=)_NqHm}TLusl4m^?}juLNq|1+^ziJW{qsH5|QnN*eqqGz_f3DjbA?=V6Z5aJyy$p@X%wdvQGqz=e z{?=uHN*^qVawBiuO9C0sEjOY*RH{lHX@3(Vg!Hl4`B;&04g{$1-`+Yee`{NuZkH>JT-Mf_JGfzbOM-6Z7dA`qwvp~HAIH_8 z{M9VCHK7cmFbV5h2&AmSYj}%Pegp>!!kO6XVPl0Pw3KDfJuVnp-!Mar0Gbw*m(yT} z0FuU}Dyy&%pOjQrMdj>rPX9}WC&Aak>dw>JPpFoOWXeQhcO$#@4X)xjAI0F8h%g`~ z6Qf&5!9OIU5x%DNjRsd)5gM2c8(r)yBe(=xDIVH{RP#=R)N6aMXZ*n;TU;WAb9!j9 zK+Oq6f%Py}8=mZlYW=GFD!J~x)5jRK!kxo8FE4lp<|Oi^#YMIr?_-SBf6QWNyKJgc z^iXtf+ou!Rp&)iEcd*qvU1zSCW;YtxsL@h32P7_Nkdvb|hsV1e+2@gW^uy4FtPjh( ze=ZBgzkaEeV?<#9z>qK({aYt;no4PA;b}jBnL_;Xt73T_?{Uw#;R;Y_tr^d-d$_Wpe~W%q@lteEab6;I$bfGk$3Woz(E2jTpmnc(?Oq zODRJeq|l7?2ppAVT}0zlbKzPwQx}#*cq!~Ki!OS^ktIyP=g8K)OD1m|2oXjhbp0@$ zFf5@bxsGo6Z2g(8>3d&;QJB%2vgO4!lqgvDfrKvNso6Ukd_=gQ>C6cJW-u<#mX5l_VGswGT0na|fQs@3vTHtjpLEVqDO05UE zl&ONIgQWx=p3lc64}t!0f|#estww{W3?~7h+i7}+L=?k00;BI+UM05VDpheZTKK9Qsp@?|gZRre zUK`sQwG2-l4`=%IOK%VTx+WWZQ-lZOkP$~+t>~Mi=pfTU=`7OLn2+}1T$e;J*--dm zcHO5<{PPEPcOJSo0*VySznBo_>h*@lbdCFlM;cWjCm%;uIbQ9mVUlhl+m2&j^*KZ( zAwk(4{2kV4bwB6$Am-*tl$F(HAt5D&kBbXvv4lidt2f@};wv-DZL(a|PQGshM+E$L z(kY%XP6w!qLMG{Tkc2i(d>n=zHu%9l;)*V`LE@1YXAOxHRYV= zLA0?&R~;)6&$5%xYfR6Zqaar_l9(ijA;k|9<8*<(8fi7KLWX3SNpl8ZXJfu&lwzs- z`i7M?lh4Vt@rckmauRUMPpNdBfRYR=WJXnZ6rd0iTA7+U2Lhni*u^4Sr6&c0&!>Il zpam?T_dtnHZrZBnEiVZRka%!@`ldQgcP0Rz@IvZ+!9sMC9Iyd78bO^6Y76&f6s z{NwavUloJNqSVn@HFV}#{OtRDzZQkBg%S2+F(rjNrb#M- zXr_Egzr$;x{GsTf>y0m{3kIwO9V$Ew1e7qB+DIzFzX7%T0Gt4EF(;ADK&+25r5KQ)g!fGTF;`fe^mP!)4?M(DzM80a9Nn#Avn($!e}1~R1I%TP z{K4Vj`Po@rxg?K9hZi5w1ti~yikcdW4i;4EgW@>L7Zx>sU0ib-mWUXVmC_nCmOyvL zGx-tXXVCrOd}4IrvinyNoL}RbPwTlHJ^eR$;{rUmd?P?9;WKN!5U|nF`yk?eDC!dKPjxlfP#ZQse=Ff$t?G z#?3j+%*<>`0fWdP#|DbdG3(X$*E+#XU)WOjk4PveRt{o+H+IDSyGQm174LNYyILh^ zNEXjZZ;Gwx>|BOL2vI$FG!!6y1$8JdoGs&54r3r!*8?gxC8crj@TE4ofLshji50tI zsHE>`vSIKi^AL=yl@<`Gn&`#{G&mO62L)XZ2wEgj^R<&xQ>7#%yq+JgLB-4BELaF8 zZuW1x_NM6l*{NB+=~II4b5t=^IlPP$VLG_g^ORSCuJK|(h5)_?NCmKv1O8HR$b4C7 zup!>~ZeV7{&VSzLJMw_$j*-rw2i$KgT?97x0>#(!${M=5D-IqWcOEt+ z+_uZSAPg63@8EB!(RaJ}tL)-(325TmtpS{^(+?Ta08g2s1BJ%kXJ^}`;BJw`pd}u! zP`iVNbbc@3+I=3}1j$4|!6z2=d-GO!((FlQP+Q9YZVH~y^GR-%`LcH}0`v|97xCwn znyBbkd4d1K)>}ta)kSUJ8&CuhkXBGY5D+Pm?h-+|ySuv^k#6ZmxfG#u>wdr}kcZofUIl*KbDC1QuSk&DVBCOpIr;CrRTaMByqw%{h%XIP-Qa z6MPg^v^_N$%{U7I6jPw61lkASDFqQ62LM%+mzRh4Ub#4!pQd_nJl&+(LD`wNY$@h& z`15LURM*Zr(;nP~!biJ8Uvs#?TT?@vo)>qQFjuvE-kZDGV7%chy3}uJJg&$4JE!18 zpRwSx7!H{|_x71o`+fxwg=ig3ps_mBhP8e<$6?zR48;}@-4$x zJQfwlGb8o=|GrQt9DEO^_FbW3X)2HT-RWPV;R4M@Cvx|Pv#wJ^x>PRrckb_=LL|mP z`S}TKUCDse(5FXAD(+v6jtzo}&0}#T|GyT%c1uEDeiGDP9)LL>d==bS>+yv5T5Ss@ zEDjdfWi{A0@(Iio3Y&0!*zZb{gyj{MI)g{ShUIv}8sIEfXDjeZHJahQW^D};l01zJ zOqgvq#jvoj4jwRN4`O3#vc`rQHY{P6SJw!jCAQ2&W z{o1%m^#Z-U|Bco1ky(+2;I0f;b?pyxIt(LHIT|2T0Wy_FWT4&Z>Fg|1Dgv52OG``8 zS^a>~5Ks)L6ls_1WY3RUU_kVQo)aJMQFcClPM$*~mcX3oW)BaK*{bqqmqY;h#5mGu}&(%O^1{8l}{?51p5QkAn`0 zF(iIXk+ipayuIOYFe6gTd<*oE(NR&E3p9p*62GQ$Eq3<>bGqK#%DUvX>(;CTo$Y@ z9mLOz6L(daTJF@4p&SF-;AjsXIUX&*&My$!1~s47n0yU+^}EyJcVW~7ysQzzlLmwU+(3mdGrqt4-Yn2)!aNDU$KbTtZ@W2jqVAhN| zX)tHBC?z@8)yb*x_OQWSMgeH&b^hHeTsEXWC>4bI#tgDk5d=SQtoj=6Xybq$pQ43lcC^iUwp<@Hue(gFNKK`Z)^5w*X}H-|)2W5# zL$izlal?jJrsQ(#uc7lw`@)SPN9!_*Jp?G=%36~wD+ha_Mh_-79yHKp#bAQ`~tdZIrnUL@rkLdDLk_Tqk_&_1b1KwO#{(GCMs^NPZ$4wX&x3$`F z-%&eH#9TJuBV=U(I?f#Edt>EmWHn(@hq1Ce@#Ry78$#8#xk+LN0vjBHx;0|L0hdSi zH50f>!6TKns01v&&divEy`d^E<8tM+7S}u^{xMGv?f(Hy#?>^HE4Ux{@rvz-LE+RY zria8EXkeHQjmsKSKm8prVU5;)aBm0x(cqvUV3`I2*G(Xy`sCZ}anBAIp7h79M4l7F z_q}GGQLH~eR-FjQ6ysByv!0`iv`g4wc+o)il z!UNZ+(|jRAhLW1t8nrhafoi}i3szy|QFVY>c^Fjha&F?r@1|fdl|||I(7;`D zf2Of`#+Sx*67m)d~C&Jm8o}j-=ZfpNodMGMdvk*6? z@;5UNla)_#;5#r%YJKt}KMTd65d9)s@P$MK`XczPc_0EaN5wp#x4t+>Fe=_NVk0DvJ$SQ%BKPJs(4&C>Q=TdQ zym0=za#H58ja|v?dXQa`Eq(8mb)yy?{?<2O+j0xbzNFsVHb(HT)hn?teHB6P(Xj+x zbX!$tTJl#?fo=JVwS1m-g5R4UuEM66SBvK#{Lxf3kEA591q;pQ3+#fy|X$2MLu)CRU!cz$H30&cg^NibhqFhB6xxT}W?n0voV<(2bJn!p`F#4|xV;{*G6ml50hiz40p#-E?+#gl}TtSO7( zTl+dv^OWhehYW7zdwe0@WTDK^nCB?AG9?>X$$zmmpJ$VhaTf-hTOo1MS-N4pH9p9m zD9f1W%%*qVfx}iGv;kWag0-IT@W4P|YXhWjSKy! znJ-Yr& z0J{?4(~vmguh-Yr?U34y4Mok*7xE8?D<%%MAefr`W7a%TyLCvqs9ifa0k5@lk#r0p zKjo=O*}e_dw0iT;)cACgV#u=7g;w+d4!E835rS^CZSOv2=^ zkF0p>16Hi>yw8|PM-eH9oJO+pTh4#{^Y9 zZd{kRpq zmpr8Uj?yf5LR30s)fKhtubkIhR!Tljx)jq`io0^irEtkhnK#u!5jZQ&3Jx@ zD{5^KAp5@MprTKtWkdc$-p9M&Wmg*Rt7DXB*&0yBglLy}h5WxUDyUIm-X!%)K4+_v zy&BT#_X=N`wvD?x6t^~6AgDpl)~1c@t+do#a`10GC23D24eF-z7?Fypj`_$=wEeO~ z+oV=f32Z!p$rjf8^>uTR(r+E$Hv?w)?Ck7&MG*~+rJxupLN2#6eYI9C=C?2Hdcq7o zEy8!Wm#8(b?w8+zAj{jW@vdG zyXQbszi4g+8~1?6h-MCVla4W5Y>AICu~B~@*8TRvKKb=dEa}%K9OqAZNcIC6B+bEv zwz-QoR=lW2QkfSn56ZOjzDQx#+312ISEwO^C?M_ow5;-6`jYo0Ug|ph(w6M`z3$J( z4sSAuwr8C6qPR^KiTPLltvYnl9@c;GPC68+wf|U`EIf_!aDS0;mD6X}10*}nHwT`SWyT8K~JytZ$+I(PWGVPWc!R$Qk( zo%D{K>EWc`Q8f`>pdVb4l#0NP%W< zp+zW(H&Uw!M$AO{i*$jLnZaKK(x``hRLCrK`yD!;P=};`qPuh-Vo00GzU(1Ou3!F| zGvJ_EZ(};+`UDCHJDyXj%66t_fu=Y{mPWH;Lp*Izy9NGORNOr5g>PZ z*lcA23kaaQ`O@P+v|L?8XoR)17SKlrB_$gtd48V ze(kGP<4H`Nd9yEDs3AkU=+G0zq-qLxEz}oC5OEX}fHPBFKZULahbpU44wdF7smCS0 z=-Q*g5$|c4nHU!rb*WGG7XH(){;TcG%c1SeN9L8ETan%WoqK9*2nSuNwNx&w=L{4n^q8mKnB|_GS+fvSYSzM>2y3O?c=fgtW%x-VQu-~;y2!$uMV3j zhv7hRQVr{yuxk4q)GvvJuFCi=G6eeQqM5kV2X|ugPQ(ktJTXsm4jEAq=Sj7nLcD}M zgHsOH^DR~MWDs#ZtrHRi1T^$^@X>E(gc+Dy2Q!LKM0;!1XpAd#iiK2b;{5!}eYpFb z@#obiW0{zlrVhK`CmZT8FPUYE>QvMGwoV}GNCi(!Z)?&M1L1w zK=C}Gc@CuBtQAi=bwzSRr2hw#8}!m0PY0IY)UPRw z2=R8SYx~dX0C^Awq@&9DQm=8rhr|nt6HFYA$KfeLgyJXp)_P5jj-m7iqx4-L4Kspx zG$=wNR3bm)1wDYDxVgK7YvRPIu`I%EzdKQvY}=rtq{gpQaT27Z0XcK9xfCYgyVX0Dh7|2}z*RQ9eN6sT&ot18a< z*4xs_`6w0^`hs{@*#9XXUtzxdcc<~}f#_TPZn{N1IfBlu;YT#+ZGO;?Mpj9A2Rb+-g;knGj$yajNpBNxVzQMG9`=6*+Gw;GNS2R~s%Ulg%5Xv}xS*W1U zE{X%E?e8N$tjLBh$PYoI=I74_*A@@s*g%XC?h)n-wn`w>1RMIG^L zgi#5(=l4y%kau-<=4AoT)L#HMPcwILPMe7Gk$Hv;9^QjpB!B_RkF{KHCM|oqUd?X) zGATu9z0C?#b+iZw4Rt@;j_l7~Utb4x8a#MYAF+Dof-XT>>>LaTYKXfZwyIyqX5wy+ zC?vtZX;vbEgWkdH;NnYij6(qMUk37Oty@|kOdzD$ipARg<5P#zjyQ_?kT2j7;`{~D z=3DrvQad(80n#h)8q8DY8jTbS7f;-Ut0=fA?Uu7nKaYtRY&Vl8G`8$(d#BQQwY`3!e5I{v34$9M@VgPrR6$82xFQFq4Kd4dGoi*#y{$Q1s;^o{3M5@HCA*~u}sW-`xn2iE7YxikB~f|oI?Ki zHTS_TYKqHf66YHPh1tc7=nOk0ELTe}+t3gaZ%qhlt7@-)Kf@lx*+IKPKiG#l{mZY1 z=duP$aFX{s^>8=LE2>h=aa=Ra%bNrH&Zb}y-HLJO#6$l&Ba07|mDR2IKiO)}A$-}| zP+!ds@>=D~txUr+jLU^T9 zaGyK-){bUYW=U(F?C%7-wk*j?0N~{d=*7R$=KCiOj2FNT)K4j#R|~~l`(ClRnKEEs?w!Lq(6*e3tC}Gz(;_VW|aIj-Dx^4(KXp5 z`+j#)L*LPTUqDg9v=Ee#a9!-L)JJ$i!pW&S|4L42j=lOVFI-exb6y!NvGeKQkd>k( zCY1I{c@JvN0IXLY3$anW?;UH4Q1^#&yph*um6i^E@}?mzr4{Fg@??{z!V!O>3=lm8 zk{Ql_#1dMcGOlPgO{22*y&SD@k@iC3lpOW{e*dDm1L`!^(#Su-9u|B%Qwg~^gt*5- zPq_(Zp3;zse8l=b>kQG(b5TNCo3E?=pbRC+@PMhc)kJ*J)kgRs-r|UTt;~Au;<$8N zntop3VjW(2&^GpXr z&n+b7*g`-R?i1_Rg0I=+lAR3Hz)vi{)ZCp~BS5e4@f{Me1bwv~GvYCGB$ABYiFxRg z&6xkfm>>49LK5dtH{FX(T-W|tnIuB-cg2{dv=SE*-eVY73ZM}@2@(5K@NaP5Ty#df zvXX2S`ciW}spI9mL^;2I20;OZ?Q1p`WzX9WkpG&W{xQt4a2$E&^^Mm1??lle*>5yN zV(5fq3%2|~6+T1b(nE_ycd42tg#5Tz=-SpP~xVPiPf`{WCDDQ{EbW}100N8pW_DClqNkAR{GSv%|6 zZcIrK$E%<7&B`Z2z0Xi92bo?OQC)@Y>_uH)_mtlEunfw7j2Amx4*0r*1#wlq<4;e@ zO@Ot8Hs!h{!D1t|-n?bkDVibD6wI~T!oI%x{Lt>yLi}Vvm53Eve3$b!rUUn%sZWt+ zx;l9X=QU377?Af+Uzqi)Wou-RUQGl6+ux5NY;)j_-`t!ckOtv|XI?@rfxWglj)~5n zPWMu7p-Cz7L@P4jg7DHgER369Hh*GK&SP4aXysc z!+ML`JvziYPL)t3*nGS?yxqFxJ%ImAH~X{DE(A4l=|pp8ijgy@B|0@3JayFU&u@RT z1lEr#^-iU8*sJEDmNZK&HtE&b{RL8bY)In9=I=q-5`62b#LE7P|>l=PO!tKdDh4+b^IrR9wC0dF~+f zl4EXWO{pt6``fLqO!)D&Bhq>{oN=T!?fIKSYbYhs<{)y8u}LQHx&RcpOq-^YPee6N z_ta>St3;o=8GtQau_j=-vNDuLpnsJsb}MN8Osm6g3TH~n zXObIu$4{WE1jcT1%N81mZkLlpw7v(P`1SBt^zhKv(4UvT-sQp~Vqstg(`a^c2D2K- z{PY?p<8QDqF;ywjl6d}!r-%MoZA%6&-hrJ9NHZ5MSoCh)I{FIC<>A1NYmY6FE8RN- z2BjTgkyqh8SC@WP2N6+jcED16!g#*_{)5-tMV@}Y@}v&RxUFkjx`Q(qeOtBv-gH=khyj- zT`HLb?Fw-rB=#V+Q-SO;D4T=YCwojwyr3aBpD6;(yO}AnnMs;5|BD?mR5laF5r(!j zMl&-eF3V`V{x1Jas^PLAaEfA<0qY>4UtY%C^DWo7 zep}xFyKFNapcPC`PUbuL{7QfE0{B$`76VWWKy4V|AmR7GgKt77<@wAl&dz1^_J^P- z-bGay!k3{w0HU`O;8NFIhFI`#-9>V`;K!vizoIqjxVK;We@;q#7HtBV|$9K*T#)b}6^fW~Md7tFw} zKm@Ug$5FyKU%t>yoBI{|!PF=4KxaP7CdN)#UXUKD9g^XK6}d(W#Y>r+*KFilgcRg| zM1&Y@tHxhS)wcnP&*q=eSG;IXvzGz%ZV1fAAd8TYnD`t5>4Yq)4?K7O1VA##QSkLJ zbI|Rs70)*7%e&*G?OTBaqS*o>Qe<{^b}+0ISWdg8{bc(WbdDV7N!?_%*^*<+L=;=0 z$9B-30MmH*ZRb_BOO?@q_Hf*ArT*Q3rD{iS?9$(F*GHg!!^~FKoNA0si3{~ZN&;1y za>qw^LP#ppA%LHFu=|Aem~}a8x_XGA-$e4pA_N!iU1gI`RecbKuSWk`m$gi=%4K0& zT4R8?wljpLS!-8Ifs%~{_HiBU?SL84|IfEe#~mbLLAv{0VWQ*zasfa-27u^|%*>v6 z0cnAQV%YqR{uth8+_eFZk>8%qADtbnmK{Iu4A_eeHg|z_pF)xS-GkI-*PDmdf~jW< zb=H?g?cXY#83s3vnN1Fo&4CYt^Jef{+n_5x!wZ9-m+p%3w^8#q*_rlCV&m{$;M23? zN<9hUm|KI+Q)4aa^y)%^f4SfV!KDuiTL0F!Z)li)qc?PT;tH(ANiBIBp$&x5Le*VNE`_`nO#3D>G>*<=qgkU-ktC>ju& zGN}ohj;OqR6@09}-uAY36oG|koB&*+&O`3A>9v-QjQfy~zyp0t=&Vv$HlK5a@iW&YPCOg3mVlcNXvO6o#uP9^Q7_24hjhQ+Ll*AI0G$8 ztF^09;`7|P^fUOUm39NLLYOUq>#5X!Y^IeDPllR)f9|{8+-FVa3IlTaqJP5wc&EXD zEXSX^eqyBX+*b&p(X6a0&Z$wf(eK!<^xvP4eA?T0+)n=A3df(+eAabS6aGwigc+{* z{67Way-y}>-nZcIpY4oIOis?v^*<*(>Z(Ln!Ft3M;NA(7GaGb@*A$Pvnc%ep9M?}X!Sl1$<6{6!k4jYBKrKD{SwHbALzFW$oYZ;BHF>0MRoeD}><+GeAk zf{uTXOmjhdRU4% z6&KK_`CjJ$i(re}b$L_aBhtqkh@Tk-4ctz-DDH(X`bdFhXM>cv^U4u6jv{{hvuCx% zgEjTG;~3>f;1QyqD@VyymTJvBO>V-UgjA3Bx4d6Wq#uTJh*nwR zbY}bRVd6qJtMC|{Q`>W3w)-|`V9`uB{5QSB>nWEJ93W*#AqIf!Bd4giw}*ab8up!} zXK3gUpykELPeG_YorQ1zph2S+$4JWIQVONgAt@;d5MAclI4u;3hZ6`N#HBoNOY_oei5v2W z8@8MJoT!rOKw=YYh~0mA#N~ZxP*;=FP&*qSY6-T^{0*J~JqA6$DKyit>4!+Rau!h^ zZPQ(g<=R+tsAz?nQJ7NA+A#tn>~eR~w3F~7<;}+IINsGIIjiejgVUVk?)lt}NVm+# zgN~v~w=c~gB`TLx7hum}vmU@L-7I^!Tr5xka@`0JgMqkej~%$2oHlZ$*gyeL$Ai83 zASImm@Bl;rcr!yu^tLKQJ40p!ce7ggna=#N(ys1DyQEiEZ1D5zGOU0-jd zsbXR@RA06qsBeQ~ufIhXrPHKqI%#=Kw*9^GMHkDFF{c&=k@kBn5l<(U$Jy&a{qKE) zJ(=J}?-KEV@8;oN%FWd38#a?CK0Do7^^5fy>Mqxw*>NyDJP%f~&03&=ptjY`7}d*s z{bkWw_ojgL2hhcBwUxF^KHR9DA0)13*c=mu-}bF0e{nksmZCdKucC1{&9;p@J`x8y zh-Q|7rYm`J<#}<)Q_^SS{^p8Fa31?9!@O_^ zs?_UrR#I)JAhxhB^iT-vduerhCNVZfJkN?iFfkb!8d(r|J!6D-dKSD%$j%OQ>XiJU zJij-`*ew5^2MqkVkKta8J1kIyEhMeT2v5~ zLL(fgL6cq-o^iR>=JFy~<;P1G43Gi61x01TLqb5og*Qwl6MzeN%Zk7kp5U7g?ne|y z>XE(u5~(yC*UM?LjEU@n*Yf#AA|FeC_F7UN z<<-o8GI${6)Joq%{F7+PWLh%G(~j_<>Y}mOkdfI^sF|T6k*N}|@+V88CPzGXTqE^! zeoY$zQCX!{6%jaU-y^XzkC?7da+?dol1?DeU#{t*l3;~(CeJ3bVWjD;-m!=IA&ScBQ!@h5HYCw1`kccVRy`0m~>TVA0WgSMCfwxvoo5r#i0NyOt)v^3p$Ho0Rr2V#}GJP3$& zb9?*c)ExE-i&{#Wv5#UIs}){A)f5o$@owXlbJ1(CWHC3XnQUFbNxWC1AouzFr4T_^ zI(WnjXqZuuk=ufBxzac}IRRC3E6p=CC4~kv5b=pko%c`WwoeXW6}aGx2FzUS%Qzd?U5yR;(}}0-d z`%Kv0j{7(^fULsf`rO85ciA$E_y?kdzJ4~SZl|F^xV?~3jiBjv+y&_M4Ho06OEfAg zkt9rBgc>ahHa`TON~y2qkMWEZxS#Hey*7g4s z<-r1QiU&)mf7VCeoq|N3g=aD#=<5pzbb!^vLcP7{z5Vcz3MSHz5&}En1|(n~xTH-9 z8DKzOFc5l&`B$v*Hfc0FcFFp+w^_FuyKkP1aK8l6R+o+92$enn$*H!Ba{E(wPu-7f7__1R$C1#~fB~GFDn*-CXz;vqk5n=`g zf0GR;$3N0JV6S`*uG@ zQDY}OkMZX7%e!0|kdibkHOSf@h_fFN(}<50(K!6{eoshCI`}0&XeW63gg>4Sr)gl3 zy&JBDeW{O8#@1@6TcT8wK^Gic`*Nk6kzTVP4=_8t{MfIr8-I{d524b+L~)m-U;K)E zY(`K8cx8|fgCFS^VIq&G$Fk2$yzUK8I%`;cy`khX%YfaubK$7A|FW6ceRM{d3Xs-(-~^H))ZPqviSoe(WfqQMaFd7X+_- zY@&;@y#LyYzk?7Lz)^oM`~O$>RR+>zo9MGEpiWK4(46zxJjc7Bn)S1@wm*+v>^7bQ3~_z^g(mXj z7mc*GPoj_XzMtz}xnq6O6B-sq_&t;4@j`R{&sF2^NQFD|P^TyP?a0E@an(Wc(9t8w z82&ZfIbWzpiZW=tW`BS+kuKt+w^@fx^3m(ccL9^5-v`d|PGvlW2RX^3Kf-;mt5)S* z>F4zn3bH6UQrcpx0C>$(YyIH!Owt-J_rmT#q zgRxN{YqCMViC=(Fm#58{J?p%e3oKCWmLL7eh zQ{eRXDU_t{HA`LxSU`}JVr0^ZyNHW7Zw3@ZGv6_|3=8J1)<}a&-rjEIq$K#(k+4C~ zZ2#X7^ZsJE1r|VrrQ=|5Bw~l*4UL!zs zO<6l*Wdkt5faQsaiCOE(W7HR4Cvx3D3BtK8^NBnRriILV1#S@eL1Nf(1g@0H!<4Ch z7-<_Aa~kfg4_Ae%1UgnPB{f&W^N7?Lo$xyYk4D92s4y>)vViAwLr{BZ$cp77ms6?S zC5e0>bHnWHt@Y*eNgBR)82f=}TA$UJV@;}}^{I1JU4?RuKdJ!Xg+cBY7gbjaY}D4G zneIfa(ZdS2jLYbLI9^JeL-noA*1wvj%jM0)v1>7Mc@_#z#gRr3^s!G8?o63`)b+`( z9S+X-%0VhMu}xpXSmT4jkN4eyyZEUE`9|OqxcWpjHa5$G=%y_k*}VnrQI?fI$isW_ z+pv;F4_XDX&9$wo4JvN9oN=)|_USHfK^hmtpoPi9NPnESP4BmgZBYBoRk`^R2-XQm z_l?+|`fB+&o*|1jsW`II>!iU$uS~jyTS!-sidZe1_n?*h5kukQsD!eJ&v#1 zc~e=;Yt!po)xH@?{W(6r zZJB2nZ>I=^pGc-=(rlQ zu_3QNs3_&1;h|497pNYRZ)GWCO-0e^WR3I>)<`-3QYQ*OhiT6?!JcC&27D|=LVx#Wf_E>xd zGJpxLlYy6;Js1r(!wN?ViWb>48znh^`XPW9C=JtFj}eD#S9qiW1Z zo1Fi~-@ivDgV?cgn|OIdf@B_h64Jx}oPFCCZ+ut)9qS?*baJn2PbP4Fp`j;Lr*jt; z(^UG`ef;O74e>uhMGrzyAt+?)NsbxHgyrKD@Jk{jukXP~N?7F5nD8mJ9m#hE9HZ^+ zUGzSXD}6?_@r}@-|KsnR9QW|-5~InGrs`NHdu&yC3N}TO2gz@JEl;jzv&+L0&#)|W zl~qm$QH9kR&+V!i#ymL9y7CdBRlJA6?URaI+ z?RFY#e*Fr2zipLq2&IpUHCSei>i5mZkSjlQ_eR94IS(gBoq4X+3wUbYZabtwnU*%ucar*2}*dLcntBPS;Ax^Yp+dd3g7ibmYyxg&VDPOiVX;;o_& z$47aMTDApK`qLdMMP5UHW^EMSv~An*#Ax|M^%#i9NTl=}pXvX8_$uF7@!fOM;_H*o zUYP0k*K!kb7o4L1{c8PeO{6VHw`rE8MR=q$7!ay{6_ss7+0mJPgd+0o#JQB-e{9K| zEdMy@Uq@mi2_|27h&2kMINzwrvCrhA=OX^)Z(Nm&t|Bwt8d!a+GY!ek|v4PYy7BJZ{A%UQ_Ck4Or=;8F7|H1# zl&E6BNNr~*s~To2;83l3(j;F9@8$QPx;JH0a{J~y^A2YqFfB>G)YL<;|) z1Q$->3BqQLx0xt^Eh`WY~C6 zqB!RiP>5Y~PjopN?P;ZA4@7RG`JQ;Od0VHAK~Os5(&7)WD9K?JBt%Ti-h8<+?V`Rs z`WAuXPpRfBx}~4D=C=f3I;QzV#l_DRX1Y&!YPR@D=MfQDk1NtJrnf3e9Z zKX>{1fHh!@3N31~Sh8km*9XE|`%?9}M<8Xw~BI4FhhZ7^X<1Dv|6`(-{b9P(m2`81M5U9HYT z%)TXF(>rns(2pk=t}MyNi0KB+)Sg4U?=c%p3%_PjBQ2B@f0CHIUw}vnhO0O@r}^=Y zLle?0xNde77DF)%lSt*}j<<@<&D7>Q&bBqA8yYhlj4d<2FtG^Pk5v1Vp)C^j%}jBa z)Hn+LwX>Ny#Qm*qm}x4bt1EL`9rgD6Pjp1Gy<{R-k%4EEL0F0%-2V>w>7dZ{?M^(s zu1nP|$Sk`~60Tk9x;^&=wKT>HuO8Qd0Y3z3z5`(*DBg=^kGri5ijo@S$?}4yQc(~` zg2R)T#Qy;4$EuHrTaRwQ{hc<+(Ii&rf~Ydm17(_5Oh)UAlF&f|6^ zwP-W50VC-IrFQI1J^j(LqLjjCu6SR+L!E9lIj+_N#f_RU{GzE%-d>_HRLDo z@0#uMP9K}$DbU?W~MOSO6# z_fh5WMd{3u05Pi|0`!Fg6lDL74l~#*Icaa&`>y_4Zg5}SZ@%EilGsvl6i(I;TKmrw z!0z~+ZJgpv$eaH%>?|V`j9JlED)P;t9qB)X<@^+1Y0iy(b^grchTQS^Q~#5~9XK3Q zkWQTn{%EG0|DWH4tnmsi2S5iV>2iTjDQ1d`!1;| z!axtO0;jLuaMVv1=%r$B(^AKx3_QevL47hlN{p>#}h2U{ytNv@A%Y~`~kltEmh+;$Fk zf%uE+`mYCeT;4{fu3PiyuoaoGU-)KX$q}4lQZ3RJW#6}yVbR+v~t+aYO%DJnH!=B0Ih)$VCZ%+Qu z_ut+~e(aIW>@ZE_O1s?2kOYR3frmF4Avfbhqw)hze zDHL&px~2EwuTOCdhro7sqJq64Cc;w4pJlwr;{?xWE-73~N)wc1}1Fftyu9hHU>8EIslMN)LC4=5R9pH~+sLGttc zz&%OpRu5y##&~~NaTRHG)zCmh2@UlYI{_fqz(ILIwaxVWWV4R9qwDD`V92-sSvfpI zrDLZ+9&vz-I+HuHmtOwfo553EyIaL0;ApN+MQFB{?A8(wmXx_O%*5T3(82RM)d6IP ze$!V9C^u#s{a#{L?zW-ed(GPUPqwLVSA2uVrm;FuYiZ6$Sxxc0nmilmv#_FeaSSNf zI)9tYbFr-P%SFBqTWlMBxho00Jn+!l~zOnXoi)vL3>}Hv& zw3+jZ^LaRr9!s&ooRK9xBcof#L+JD4>|Y}JEej8Oo{koi&BxsYa|8$b`&wVrVy6o& z#o}4DcPH`%QhNXSkXGew3BM5 z1r?qdh*S*twn`4dOb0q>RkTsbaflzmC^7u+rOr|_gVhw1r!dnnGDK@g|2t18%b zej=`z$|83{c(ZmQu}e+)_g?>lLZ#8mM&RvbuAU;*65hY_Bx9~-E^WRkxcCY}Mo_Js zbyZY~-?NRhnuiFOv9@-8&As?73tbQ$C9IL@z4Ccn{|OcJ$@RkQAXM-3Y`4LCwb+va zg4oy_%Qw5o2qPiM=kSrj`^}NKAw1NpJTk*Fn^e62s|6S}bD40yi|YTV+!Z3}_- zif2J{)OA!(TI(ft*2xc;xXP}@i%(Nv%c8?GFV-L%)bIHgA>D0vPg`sv%yQ+d&Cnr` zpkMVf)A7UKJjVv?fG@h4-`JH+h6c*JhGgxBQjVjiA>Lp0^M9R3GTBr9{q1)nzRfvp zu`!|)lZV}K;XvONU4|xH75^-qpZYI^qz(7i4xPpJuFo2DFE%d} zfFK0XnyIOY$?kq_@p0UJ%Ld5JB;By<|6Q$dweQ>#T6)lhUEiz8$oyRT2KJgs<}9~y z6K`&>9q5`E9rDKZGBq96^YDr4>*{uzVnEG`{QUfu{ocSUcB4F-z06xwbdP}fjkPrf zpl@$lqEW5pWM|(Uymnx=H9(VhagVNn%_RIhxro#EufLK$7%98DACJ2N35#80-Ha41r2n3~Kt(boyg;wNT}_pE&8`k{0K&TH$9%WdJj5q=CDF>Oq5YszT( zXkYChnirwHnwH%B}`d!Xe^)*O+J+9AhYM2$9{TD%Zk!{;%gRD6QA zP$cJRGQWvm_X=A1l03{2c+;>D1?R<2Ee+qK|6)2)WXjTa1PTlY=;e-xIXfTHR+)Hk zRzDsHOYS0meTNM{KO_$H+txbgX&BfjB#A!Bif4NreaNcsSg~PJt z-T?p&)YOvi@7RWpSRHC_Z7mM^uWsgC50+Vt>ZmLKbF7j7U zr?!SnEe=L0#(X;<+1GoH%Ij7V=Xqa6oHTY`{XsIv;>OzF;+b*f!@XJ_Uvyb=hvH9n z&&}eLT-iJ-Y zJnqH)QI~*L93-~y(qMPFl#}b~l2a=N(~3V&6`goaQc{*5b_+|{-yj2%G-fLG(y72^ zVRYZw?D#XDj^>qYJ#|h;%Fmx=?z36AisLpUzwN+~&|lr1-DI4X-`(uZ{2J1)ZD*^> z>nRv|J_P4=xe$vXGy4D`axk9f1?|4AyqG^o@O`Q;BY41tTzb3wzC1+o6>VDfdZE!$ zo3yIK7B=#b!Zb6p<~Q_x-Jg!wpCKjTUv~drZEqb_RrG(0ZlnI+W?i7SucX zT`DFzdN8e(VDPH&=tp^j;Sz2LBjw>vAHGpi(k;*;J54gaeEAX-0FPal+$dtAn|OFD zYpY`)9Zi+VeLr>+a2>Ilk~7aOYq{^}(!WX3 zIY!TOuO=~(pkqMm0p)_rXF7l9T*&63REW3j!T3t7&u6B=a`#WE34BUY(i%AviY_V6 zO~*wjv9EXi^4&YB27kabP{4UZft_1t6fhWW=c@W`sq}HT9eSL1P9H{ILm)M zQNpa;g>zT^YFmub-P%*pI{t--fPMNS^fk>#+4uTm;b}?(O_lcBXi44~%x^zdR#uec z<;Bg2p6xd6ciFl=JolAVbBj||?(j{zRrzBQfAZ<)Cqq6)(#c|tD#wk%vAn@RAA-1) zg}+wU8$a88o_V*vHQBgmE7ohfH(gjLWzejtFzm|Rn`to_NKRXPIKDbg^u43mc1d_X zaw%BjJN0eChgdd)eh~!N)ZvySuUC4pxzuaN+qSKxx^Ai!o86!Qd;~)!t=LH}=dq&* zGi0nG>9iAFl*?fY`K9`I%}fp{&g0P7arcnQ<>U5g+29FQogzWq13hi?wMALn3#RTL zHv0U`D-It&riEWE!C1wsB`cCh$Nz!KkF-R9zc+DKPQyg;_S2)#-)kSuah;Q`&lGhD zJ{R3lR}?7AL4N)oq~RBsn;P-+@@cY5Ucc3%H9%K9G#F3i)xo!_MI(40YW$6`q~MdJ z3o9F?XmP%o5sUu05IxEnMobS*TnP4S>R7bjMcz{f*QT9&ZTDQ`5;YCqo2$G}7QSL~ z-OAasfKXMgjU`~=x!9{lE#-iER^|(>ii@Cd+$y%^^@{vw|5ixqILbR_3rNRPhWrZB|Gs+SUO{>H zL=(v-q$DD|hbd~RI;Cycaji$Hff8c1!s+xttBe>5F~ z&B{PM6D5-ZMXznDti!vpw$y2i*18AV}oug6J;_=1(@2+{t1ALOGE8nZQ+Lw#gGabB4%DOMK zc$(~e*X^1~yQka&E8KsQ7+ewY^1`NUb;uoJ9NsAZuGU#&0dj6`vjM}4K|D!EFk z)t`SGp0=?cVSv9QeOZ}toLz;;)3HqFDJr&A&077KSN?v#teN!IKd|BbbI6&DmT{PC zF2mU5&xDV4PZNKXm1KXfe_#(L^{yRU?${ zD=I1~C@ARc%o!m{9DM1)>SMbSBc8LlxtX7z&)*3%2?VwDRh3t5>!hj$%Iw7hNso8U z$MTYZE=JV5L`~xSH~Fyukg3GzTxC~Z_OJC?P_2Z3VKe`N)ICVa6VST1NK1Rrsh-66~cIP*{fvHQWa__2|t#+$hP4#74sK7&IImd<0 zcAZfVwy3MGs=(|S6}Wv9a)eeT&oNj2hc(YiD4{c7AhK*2)NeXYx0(c^1koKT@p)7Qa~V40r`i7scK` zoVLvyo3DbST-gIH=KQU%HE@v%BcEvP`fP9pI?hEO&mC)1!2GgM*xwdoeGF0hy}-2p z#y`y=1O?h9h^Jbs@`!uOLN#WZ{wT?tQg90h27Ll`b}@6fNA>wcRmo-pI4Cu=KK*?oLsG^fQf$4`5uy0CQ~I z*BKdEoL~W$QGrRC$wV)@(6G91ha$Zv*UCFeFycyCSA;#fSt9VCo{suI=v+m>`RbW5L!9_QF31`Qm^Ryf_4= zAZPpRCbFh2E$x#o@xaY8A_Akt-xKjkCUAWNzBd;;Wo@PT`Sl=gS(%>CxApcm@bOQ> zRORXBN=oKT^aPSv-T!8~-W{m)WSk@VX7 z(Bz1SfZ!5j)L&h_jOzu5udJ*LOCi^8xD;L}pwgfnx7KXT-kPb8F9mzat>I;Q;N&-R zvwK+Z7mMVuq)e7&uSw}pKmRLMrHQUN1-1A0GXm)rH#Znb7U$Q?+ke{~f9~wBs~3+E zadA+i6VF~^(pZi+x^?=zfl*H#%*F7(R@@|P-6O-ZebA<{&aSfRhO8Ia`=r<9KXbSE z2s!;Fb0XY+fo1V7d&G=m1_TR_%N+n~4G;khj`kKfn7RUil;?%b-TZZs=f(EUHty8H zT$ASocoPDl_5r9Mvp$u3#B8~5jE=*Q%8-gGJLDN{a#9k^3JYf9kSY^ z6wx0M4m`ZTuDZLsJ3C?98?E#`(7g=rHUeJvllQ-G!GjaD?*taT`^ATQ*Ngr6>n7)I z*F%?xWpJ<<7yEo(muA2l!UE`p-6l?JLY_s?$fjN%_p!jd*nmZNZB-RjOh(wyR}Tk` zg%&N}x^tU>npyBgbDenbomBm|_QyLk>z$&V;FVPiBgkmBHZwzMd2k~=dx}@H87O=v zm%!@fBPlU7FvNuY2y#w0{c1bh7{|zVFHONg*BMEE0J>h#`L@_+ZmePuu_fIO@*Xsw z4D#x5dD@&DBsbdlf6Q>?xqQ>isKxhi)wXmgG%6l_$0s?~5ZMEP3MKBZr%bYI*oMG|Ho?Jwg;IrR@yi;FbE4s9_1Q)vKNYLiH zGCT8Rflnp~@gK!dV^E>4)BI+Y>J`BQ;c{13SH>IX(4h@X`P=kgmqNt8zlk}=Q7K?# z1N1#Fj{zqX>#Lv^Lx$j%Ewy`12imny(W<7~9p zxxlpMb!^8o6oj8#--Y6zzTdo3Qd9&4A}Ig&U<10W@*okW?V#fN4_R<70^Zj+ zP;zpzT{&Ka6=+~KSP}V8`nYgYSFfVqCH>Gyn5c6{fl(NJAlh|*t|8keoE>X881OOM zgzS1Y;SA8KcZNpk$@`XK3Z~;{_?FMk!nL1So`+uEeGiR~?0^vm2zh3ZpBXJ6IuHLY zTFtBI*1PJIGx!%Kwe;BmiNRP;Q|M^;z&Czr#{uD`Em{5|rQ@RQ(-h8^fwMnbb_)Qh zVbq7VkVtC{jfetLR+77fxQv_nl+T&=zl6qVZhqkgNmoKaZ=F4XGImJsA=-+RmcX4F zn$oXZ_1YI_A8&su29lm-mr`=wKcjU#3-Ms#Si5*rAN3c|sPQJ7;l>Oiw>fqKri~`L zRZM*>j5UWAE$b60UIN%5xR0TBXBxo9WZa?o$*zyTekpm;RiZLO!qK_CZ)9WhD}B{8`>%ts@p$S*a`1Ft@YI*9$HXmltQ*M5F@y5`RNYd*jDF#8Q@lyO-iZy&9PoOw1=O3N_OeGK+B9-qG4cc0gJ z&(xLiOoc2KNZI-VI~?pYb#d4B&@U>_q`6CgB!)Gsv7Cevg7Qz)cLS}G4QNdz)5)*n zLqJT4W<_Z2*8yGDf?Do~oxXBM8LTatNSo+GRUS642E>bk?Tais)A|&Dt!?*`s?&%V zR9+5hEm7@u6BeT}$?8&Dt_h7 z#yN7WT&XeN0x)Y;CnuHBLPNjc{+U(dVIX0>pk z;NU#y;9C?)g@t}eP@YoV*Q5#iB4O@(IOmNmYtR7f>6UVY(>E;;pp6(FDMgW`^7F6F z74OyUax#~~wRO9lslyZ#0!}m^pnS>4I>|KfCdxiC1A$rKVB&noNp$4;f1mcLE2|D*S;zKf(bL!o;J2Z!V(mYRgyy^%cHuQ4>v z>_wF=`&Zuqk~Vs*o1k656k_~|(znC4VNbOC>3gAr_K)y@cJ&|z#>$^x5C$N@fuHLC z5HBP04dOWt<>j-L&jW1cCcFRezxaRw(GBWu-NtNLQYys2PuPrI$Ek{1YenpS`t=hM zd>Oy6>!ij_%lv)pCApZGrJ4g0#{tXxKTx?GrJ8L+a#Cszf&&3d(wGN26|t;^CV>WR z>RY~-3UIN(E#@Cl1BtIuYANyM>VA;_kkwvWsEKF~99A4XnoyTh8&I zTsWDEy`~dfdusiu^+#*ludk@WyB<^dvl$w$yQiI}l1KBrF44)pwppr#D9W_imwmLN zFrmlO{k0b}i48Q&pWd4;ewDC$iWHD80IXQS18Z{D4UB_835V++vTuK%glV4k=eNE@ z{25vq%;PZ|l7l+GXR+RN7c8%cXEJ3oB{5Gn`6^3XUZZMuR^662DyKz6gP9S5qU`v? zmJf%V(-rHSY+hO>J^YcT9=@gYvV{u%U%n@qVS-e?2U8!pyU z1RH`Rw#X3uT>Xe;>Tm}($Ul0tFwo)jxw;W;)xuLs_)59{?l?k&&Mka%7f4OfHONK( zz{MU26!1~(?`&s!BK(ptg5*D$@FPccLw*^{%a31JTUI@t*0P`v`y$4AG#P-xXIiS; zLXU%%@zy=7E=`bP2_1n$@o$?*zFz+JwsOsJ6?l5sQF66q0zP9(pg1lgMv?_Bu#IeI zqI6hZZAzXhN}N6n@rw4iMxZM}ZyDY%8?k1dAEyvQ(2~bhCK;q!jF|1sB*||<*B4=S zS;?Dk+g`FDLv(XE6f%>T!*s~dcXgmTqu?om{2Dlv?nFDs%y2f1I;w)MuWvXNx2St0 z3=9+ubU$0a`TkV`4*F9ybAg_}F70=GT$t=pZdqWog){%uib82~Ms5Q^*f({#idJBT zPk@N-6p?JdZw?->o;}|!Ml>K@4E&^|Hn!kd4hqVFQv@;(`F|S&PG}JXjrUj!dGy8< zq$s(plf;|F8FM5XV}1e;R9-IX1AcWk1xuSe#JHIl;q4a<0yH+$DXN+iwGkht-JSFO zrMDzJAQ(f#)tC*~h;9*)V=~0OB~}hvD9yYEhy$#9B(~lv%;F#WS-qo%Kb^9$5LKds zpt>Cuh>r})bvxkKYT%|m0@Va;NjcmyimCA^f)!_x3o+Oq$^>k_U=eAi;eyD_DLr3O z9tWe5ED`@C%=i8fKtm1HI(|Apheqn=NE@pfB$HxZ)WeD70SK)DFmXGYRPf+SCjb`MC zC7&%gE~g_^VuU^0x8O3(#f_89kf7#Tz0z>-;8C9~xYk00Aa)0qJPv}fmMj-{kv)sA zArC6_+^Oj&_o(sYX5r!_o93N!d&oqKqHmW#4 znJc$6c^6!N&e>qSFAshYzthRc6-(<#h$CS}fa*GIwXi}U`GmGJKoBI(#avl#nD33| zc#NY!hpR`w;)RL~x%fb)CL~N*v45y48-z`=A}K5U%6Yx{i#IyN>d`M{t+BbL*UHD; zh~9nQyv(6j=)x5j348`PB0!=H#9q=2=O8k5{{nl!>E?NZ2NBdRI6vssNGrHdz>uI! z-}0kVYIL!ta`NXGbZE&#+LwYSTPG(#+nM@CAxq~!9rN;sZWrYMv&blRS zn$(kTG}dPQ8}dF@GPW3%wGB3Z?0l77Q;FnPft0*qy&<;2K`i$@1}?TOEl#w5gkKJb ztEeX^A$@5o;%6a;(LSqq$y9GGeBlTahwXXYM0ZWFKH z2dyuHuJ2S;G<>dkN;Yz(r(EG;XEE`%?#&8>1GjXT7;_DFf8P^84r7mvJj(KWmp78t zo0=2oEFM9;pVy51n)?kMTJ~{Fa%vCKVu-bDX^MA+Jus2JIsf6Y?}mP+c||R-L?FGp zzW4xzVpjRv{&ddu0>z)~e3xozH?RHR;%sL7s==J;+ee_u-VQ31Rc)jl1-f^9ze+}G z5Bom9TtF&86dOwwweQ88l@in{9X?!dMQ^OK|k$wAfIilzcMqe3P^ z==k`Fq*l>H?)7o8ovipbS)o3^U``zQ7f8jjG1ZLOnrcyajf|QXeR>>ia=*GWrJtZL z>sW1xx5VOi{yopkMPzl+Bke-y)+5AfY&+x5EbNUykHlkm5%KuX6EH|M^8W3NNQeH{ zF0v*yjmFr3(X>Riq;N}T{|AYL*1`7g8zFerbyv5`Dz($FEmdLDCl@+?4$QRSQ!aRJ zAHmp)0+JAZr;J%~eJuRoCyXc`{~h7Yd5%93yEVy+ytU?`pOi)Ew5g@RQBYm6S1*5AD4)? z(Jq+HC6I&YnkF?f6$=)iuU$v_4NPc*`Hy-5{x9a|7(%8bT|CXqd{D2R@>TlJ2zDKl zJvk&8QhYKq`9t7G0{U0t!Nm8bOb(7(zq8~rxqY0zR;Lr2AVbka1Fga?(l&FCLO(2` z?MST49@N*f4$uy0+B<1xKD>txZFGYDHh->lRd!4Il=B7d@2=Mdi{!C>id5jaXM_%S z_3sc!g>Ujtiv@{db1${zlJ{4Szl1#=OXZlS{f=%%zLE2}E<8xnE_#$l8H>7EP9=jm zh@eSySI=QK&l#ifQ6w@l!!{!Rlcp-E<^WEU>W(QexUS^k)( zHipBtjy=t&<1a%`=^}Mm^8Dz&6HL*HwJg~&g{!>J7?1uNsA)an=z+W zb#bG6Js)Em5>ql2cLh4eXgXc{m3zk6j9IeU@i@fH{-?F!&e7g&d<5$TBCYh;ct?Z1 z%KF{m^!fcE7)F}(FFGFm2qbz|SARO1ygmQ){_!*vn)>0crg|pv>R1?2fi658Z#fjUlR2~R4Wi`TBDysP2i_|A94hf@Pd)}R$*)vE?FrDNT(ukYD`EZc$fFyq zHjeZq9E^J&&aLf=-ofRt*U{<~ODW77?mJ$!3vE{nRk!)%Yf*-MtCqV+ls=yDWDlZ^ zeNH0UjyP2@NQ8uhaBy(IVugcfPh%PL;>7>x)qQ5y!PPeMkm)@@Xk3XO{s(KO8c%Yw z%nj?qzq)=#SV%7g=?`iruaCLff8y)29%+1k=idHb`1y`aOF5}7^lxo9(^>I?OWdk9 zA>^i^bhAP?^fA}t$9;9L9O3DH3aozV4i#4Tq`~&b_|ZE>ZB2Wf+0NNdmvJLk(0+z{ zUBC19=>dGyuxPG5yh*Ma4$DG(n`AMeZl7aF3v8G(Y z!UDpDyYrrYZ=m0;x@$8bJm^#31##&KQpHwy3J3Xq@fGFp90kSbHK^@;Ntx0}P<5nZ zt>4`sQkJD|*K9lc=P8)Poeup@E!-O56@+Bi0$Nan4FY{eJbMTBdNT1VxINN=T zhH!A-7hwDKFPoGYMkGoU;H9P(KOG|k=k*H}ug2_mmUkj^(ppR{6W@k#=@J6;Z@7aX z_L*_`ZlNuSrEG4{^5Vh07x)xgK(9&+J-u8E>eIz=!s?6>JvhV+i*ifk@wWl zbWn!|E>7*tm_<(bDD1)s0e+bynsC-b%ROo076rreA5aIbl)(@rdTKq!%HP>_Iiz*l6-f@?{_8X51k-07 z*gb*Sbcuw-cUO0R)eo!WBSnp?MVPr)B z?OsQzbM#%*OH&V5?td81ao=bgIWtT1FU`zB?27=fjx~dxuMxx0Ft~e#goAcODlTBgQx!r zS6YG8_$(Z;ZwwLY?NHg~Dab;A0a93;v3ZwTDOe~pRd3&4NN8$KNj&K+(}%r-@F|4T zQ{?uYgpOtY7%G6WR{Ig5U}KnewkOiJYHh;9e4$gH%aa~tB`FZv6k$6s%(w&+6Y?W` z0!2uI95J9P^*XPMn3+^r%PT%Nn2v^hGky&}o_^QvQHf$A(farFeks@Uu(`50EiFcg zeJ%!V6LXKHV~J*2BB+-UAXuvj)CuG|8^%(UQJcjosHEgiU_U54CKz$^#^{dTmu9(n zCYqYPQink!ra%55K$-7SFUjU-c;A%NyNts0wK6KHczf0|zAOrJK$?pr%7~9=Nrnrx z|Ab*Z4e}A_ps-^jre&AFr{eA3Jqhq}pEUJc-&vB#g*dil>yeqhv{tu?<|AmBWEzI# ze51kDI1g5Z8Zz~?lfeq zS*jHmf+w+)Wkw>$k39*nrPnIdf1Z`Z1gUZiMjLGz;q@iaC1BG=-K>$oMCJKYK$p$u zsFJ9L05S41>|H%_zYR;JlrSDS%8F*-)WRo*A7As`B7;5KYKtmsa~0*4otkqRYuWCU za+mFLJc|VpAxD3-FE}_>>kx$S0>Bk?+?73X=yrK_cb+?FaYlMJUM{V|0B?d%3Ne0*+?A7Px{nEG=@Zbgb>v)CMfwEos< zl*m^JMPql1;7^Ly43Q^o3+_YsdPk7vQu@7P?^e97+Y7P7*VNcr26pUOquVo&x89gn zi>xTQh`*o(PA?q*K5qKRa{+6$+jsyL;VtyI>M;@+T6zyv`hHCb91Kg>L#T?VNzaf9 zTPXG8QD~Tbv{f=2v30*l-8lf)vHttCP&QI2{fCUor+E*bZCdNVs;G#A_#AE|I_#KH z_b9r2EnMF;_Y;5YLgf*efYX`J2EqG|TN{wTT;P*|I z5>ESN+g!D;*;)H!8z)A?c~sBb;hOS*p}*`2tU_d15VWYRQ2SB6To>%X{YK^k`u66U z=7YD+0{XP|6bP)NrZJ<2V2LfOoT`L)yvd z;v0nH)XETym;%?%+a8hIG~ey`S=jAGjA3?M1*Vki+c@!*K_n>7` z2GQEgGgF=7{50x7(D=W-3-Fi9=QU!O^olRa%gCsM*Ld)-&)FWYrk}xh3R><5a(1lV3D4MNbz z>XQQ}=g#f`3uNg17W-Vt1uhx}v{bwsbkA@+Z{uWIx5Vf!Cq$SsybkiAM_2g32A-e3 z1>Ni!ZRaNl2{t*(e0P8=(y0ha5~w+vs%$-{E4=m~FhfE;Lw;{-bP4JY7ElMQ)v7G8 zHY)w$D7ab#%M#-s`{MS>O=+Gj&uXLB3cw3iaH>l&OL>7Q$pbRKVG(uAk32O zNBOEv<5%X#`29H+D(1V%H_$<7yBLB`ALJNO=sys*FJ3o)yB58Rb4vwOVM(Ug)i zU0EWNsX#+l^2iRIC;2jpP=U*E{t3U=Edoptosaqu>WP+dv!Q4Ms@P(T7~+`_lX0SB z6pf{bj3}Q%xN zeA>F|Dk=(!i^mt(O!^ZiyMX=4;o;$?lvmAPPaoF%;P6O29HF~`VS~LWhpF4~aqA>R z+CW7JfF@P+bXGNag9F`f-aCcUH!h_&@ljLnfeTk&!oJ*OgPF=M?C4Jj_|m^oOx#&S?L(ZdI{5IndQRZ z_;~6ggD4z7tF?YqUDDhHr0oU})c4}W3p3ll4hu$>sU<9c{RgvFX{_r~hX8K_8H{0k zB7K3C8F1pOU6iE&_oPr8@DbqeR3mN$A4is7g5<7QEnD6d;0Q462eNr=n~#t_3J%Zm zgH8dccb`k3ev7gOA%%>1mcY%!V-`&66BsqvUc7wHF$@|o3I>v|@5B1KNlOGMFTY#z z#|(Nw9w7P_XtaPK%BFR7ZEZS8Ui`Z1|F6G%&Oz{oBr5HdVJF~-kp&0?bP-7Jhrvl= z_|TwWJb^Kq{;ZzApePJ;#SyAI!UZV33YXlG_ z7AWY>p;^f%g?)Ob&s?54Yd}E4+=w3+{F79Jyg?^|m*xuKK8@r|jI+d&?3ucdVpHf}(4O=#@Fx$WKU znhHosO0wOI{#`(6?gSALe*U|QZHrbe(l5pz6&K{>ytqrS0V+UXsNHtE#*#Ke>Bkw6 zi3gDwJ>1`2R}_H^jL3jkYS3@BVS^M~q z!L~z%9Rln(%r*lmw?apSO&ir zaP|20*;1j6a6Ix_;i11jyshn_Kq+rB-)HBW&$zAk9pLHJ6|Ae1GtYI0_%tH>*X!Wh zBY}ZMQdJ6cxI&rGmo|^!hwFCGa*>slrFx1t>#s?<+;Wq_rAe%&1P|S!@O{O+3#}WQ zE&&A(XX5vJ8JN2R<$oh=gmW@@MkPT_fiUdciTN=h=qVEro$W<%EwDb8uQ}SlW|aq{ zItbDx`=kl`Z|9C;0hk`BeeHQZBJ7O>*aZN~{~s$fJRJeKdZ7y|`}?xO#9WjVNcwLN zy4Ud}66lup$bX=spnFDUV0T^2E|0*83BIodEYIZQg zig)ETz8;V>PuLV{o&1b+GOv5QkG>Hwt$2aPpfGuzqpGD;c6i#3kE&Z;8=)8PQNGf* zUYh>7@csuXCFKXlY@X*M6`tq&%QmJZJz~tWIh=1Q;Plskg{wJUrp3w-H3ci8U2s8# zkA|93&86KX^5-2DB2%s|lkFezz1|~#Z|IXvNm`%H7^>^Wp2}KiYy@NrknWm^YNU{n zlIbZ=|LG8}7UuXieDB?0Z@#=>-&Nb4MEe^0g9@%Wa?1Ww@`Ct=^}rMNF508zpjLC! zI-%)UU2WdYFBwo}TO%zDxaDkFZ2Oi($t&7cj`ZrMf0HL`RQl77`&+HyqxE=2Q&PfBLP`o&!b(rpSzu%P6`LB( zg-bp$*%^sHwLqAnQ*51VE@J5|AxXa|_Ap|a+(awg$6)FG2|irRUgf)mR`k5ZsCd!& zPdr9UJ>7MD711s`E3e-AcGs2p=*q|^)Pcyzo35+bp+SMX_svvVuIT=j$mXgqUxwhD zVd_iNXj?Z@na>#RfCHrxmwc)wxw`G{xjPURk{%RlwYPDNaI$jJOXSg|_a}QD_thI5 zxQX7iKFgBQBO;-TosWiH*-lTHo;Fl8yX?}}%fHaqRV_*{2oa|$^Z9IUw4n1i??~U9 zy9)m^b&%1{bY9EA+A>FO8XW(~C-2XdVZ<_W;!Wv)ERW6*prUVictW}6v%R!yd>i+!4RCTn5hRk`zwa?0?okg)8+a!y5@t6L~Q?GCByBJdC<}9kB z*%e+w*PbfBqNw6b%#)Q2@2tUJbWr_I24})|@%%d37CjI3Sz%Fr1-L(`BFirI9ylW0BrZ2VEf-P^?^;^-AQ`K|@guEm?7z6QHi3J)E*FBRLJ zRc~vz>TI>Shh9oVqRp9{UY`eq#h~UMEVcL!SJhf12a6x8$c-@nb%~k6tt9T3i~7Si zVns;_)DIh^*cXK|XtDxZDL$jL39r+y#*P@&B;CpS_DkYHP#RQDxmkc{2Mz4T7ViB{ z(!6`NaIYJ5vpzQ3tQ;ei8)YBDT~hx`@z_jl`dzkE?t*gHMi(Zb9^2VCmz>`I?O@LF zWL7=SSny=Mm!6@EmqJQSNKx%x;KVxTc2^5V*KdrVy!Y_V&;3Ld9*|FhCxaVL!qwSK zj=s-CcJN&z`&4|VBnuOtzwSbOS`ziH=U~UfdS7jfZU+<9l9Bx6rMq=+)Y)5TkSt=c zl3n!os)+Q3mydEg>U!5koJx&2a@Y8xmJcS^zW!I6svXOQX>!f0J(YFB5#-xFVd3N- zogct11D3rMYPMVr>}78}9<@t??zx<6{rvk#TiTp!^l`-Ui~Wi}DC@a9R;Kf`T+V97 zdJ67M7G_QkZl?#8eTZ-OYJ5YANV!g&nk?L$9MnhR`;60-FM4o)%ksEN9a*`ZPPf;5 z@K#-R8fy=JzUk&ur#bgS`*QkaUMc3SH~5B|vh$8r{C>81A(cRpeLyfBhbef^Uenhy zF|^DQnLuZ5r=ho{e4Dkkf&F$$3 z+D>Je@0^W(ohb>9LT`?({+)(S`f{GPs>U61PjEyWu{i@8mjJ3i(}Iet*9zYT)X_WD(Yp+|G=i3_d>kw`OX95cUn!;MFeZFO3&rQ3(H?HyEz0X!JnN17=e<#M)6T{HF@ zUXJMG2%)j0OQFwnf~Si6ahO~JnzM1Tjpnus+Dk@(IIQZ3;BNnf!zQ=vNN!zSnq{~8 zrw!I2;{5vc`jK)wO$1_^J2Q!9f_8r$j7{Dx4UJ<+?LwYO707LZJqd?hEYJ`ud`1BXZTqN4b<+7uy= z+gUnp3_SP8d7>da-OFvFXE@1(dz(M0z}LUARAqEE@h?7rsBTJ3Cet!7&W88P+i zy%x%p^0|~PIpDoROu0Jyx9Lkj93Q}w8xivj(|G78?FDO>ZW zgLiaE*4Ol5DU_l5WQ>a+tCL&GKk8W}t+IbpxPIO@EAYWqRAcPyllY#GrAe2{XwdA% zDK!~WbokD|<5`1y5=#bNI{u2)xwA0Jd4mz@h_kwdO9w$cBM+A-;x#E^_u=nD_u(j< z5(377M+>I(57!?MN<%^9uV#-@{Q?nhA5jKh*UU!J zc_TX>e{nGX!IFZhP#P)oBq?cE< zwwp40c3CP#!#yL5@McJy?&_Zt!iK*%{Iu`NNY!tHkt>O9Y z>~{Y^q%nAs;cAtsV|cUuEKG&)XE!9IjQ`=42lepI|yLF_a zoywf_e{x??n2`_dq8sgr7svMCT$H8ah9MZNr{!+9UROIvDJ>+|bryg@YB83F1>7V~ zn3lY=3nQlD;dhq`zuFZa#H;qOn}zPdoSH**cbaE3t2}eE0tAshZ8)7coxf z3H#WL-X^?fzmfdn-petyYZ^W-8pJ2qkDzA=+s1$|YhL^kY*5P^>yp^lqnTQckbC1? zxE_{bGG=FiGk{G~e%K#^sFn0baI5&fljZK2;pvCkkkUD}tTHVLs7I>u6U-$*n zEvbb_Tg^|DIaNqV?2Wpqx+#C*|6;`;DO+YvoI1?fhR4AInn|b-vexOEt7AB?h6ZnT z(rc&15*vL7%n#UQ|1@g}ou6fkkugTMT24m>bE#Y+scC+JeH^-wLB(&-*=5i~} zrB?O_*4Sr94L?f}CHA~CRkmKv7As1^>s+M~(+o8y&rk1vX5O0O{mp(^7t8vH?exsq z+y3uVJ%n)BSPj;OU&+~~`!t<X;NeaXs-(8&=kZ&;u#9mqCcH+2d(w{s*J)aQA zkQwdgx40VP^j;&;3ric=S^JH2S>d@7!Nu5_foHz>=@cFk(+@OGrQ4S&94%?f#8TrB z-j2q4yZo?YRVkg4!OC199<7|%d{WlCu6ZoUzvyF-pwo4SbwwRN(GwGCHl>$n#ve7~ z_2C`iK|qyCNKmjrlW#efM9q=5Yv9rV8s#y43#VP`N_7i`s$G3|kt$cxrud4zPxg03 zB_rdUO=6-B>IUZ0ZT^_wJcb3axM^PT_M>teKBRit6j?9b_kLhn%IL-kY{iZ%nteQ# z9CS!q8Pw0{L^e)kj3O)8)pEf=ak*~rf4a_TzcAnRO{~-VB;@6zvDoQ@wTV}siJaQr ztWLL;)+smS9V%!2bzS#(3#PFhUwxn@0!ACvZxYx)H<W_ei=F<($zUR)(Zd4g`jXR9mgINum<7P33x zhDTeqW%2#HK$5viUGIllMlWi*YxtNST3K`_66mi_&}??_d8)c&^mH!--mf@1^Q8lU zrXoNiNQ@oL);B~YQ0A8_Ev0F8Ok_=zHrn@&rJqLjX@`6Na{Cniv&z+o53nxQgqF6r ziG)feylsyu!ZQAM2JQ zZrDhat>+3~dzx$E($Y7~EKJA#h(I^)?CVR}S8vFI3-i2{yxEJ{`YFG+a1%srlHSBN z?4pQ8S{>8yurPcbF%Twj)tT1(z0p~L5qpzJdUcQdXFvFUE5(X%@M0xyJt2Ohf_5S-*iv6|QxroV5PIG(!0U?fA zlZ$NZ;9X?Aa{g=E-mNTiT z_1~ftz?TFGh6br0I7zt-&=LRU@tI!kV@adjp+M3(Yx_zb z=&}NIqp%P5_f5=^7Mi1nMAU!bLE$hE$ zlr!~D;eak3g8nS_C9n)UQntQrKtRZaz4QP5o^1_7=)~<&md*?e;sR4fkV0}f)pq}V zzLqb4{@7N_NUvvg@qStssM5hgLAh+l-53ArioRuzyYY0nq7yQG^rm`ZFy0n2Y_+9mRh*G%7m-(L zdIpdFpGQP~)@E|1g>H1_j$%R_kp2(XQ@6znGaG2MJp(YsodCw;HX-^ODo)sDBt&F| JOW*5%`d`DJurdGu diff --git a/doc/design/img/hermes-stack.png b/doc/design/img/hermes-stack.png deleted file mode 100644 index 79bca3d8321dcd58c8a386b3a5dd24acdc8969be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 297523 zcmZ6y1z1$=+BUrCQb0kG29Zwb21i0ty1To(5s(h)?(Xi8mhMjJ?xBYG$H!;ycYptJ z42zky=B_KxD<)V*TJ$9<9x4C;FU7tH$pHYoBmlteAj3lM*nUnsg#JUZ`J!qM0O;L+ z{$Tp83k?8(1P~MYq~MZzr0FauuZ$zWh=PI%ns>*5krmQtyir8TU0Bn zt8*%}6i^PVH#;nJJZ-bF7E};gaIau`9o-6fw2etffPqKwbsHlNB)&` zVS)sMFiAoF$gkcD{dx2JW`ng3?!O=ZyqS*bf%BtA1XkI6cy%w212K>8Twgww8eCe`P&~$n)AOGm!aQmr&~I>1aDsb zKTE+kywI4EM$cB%CXyL={m-KZGd+JyVcC&>;(r${$FKQvb7+PQ=w-kEdrSLcY1F`+ z$Fr)E5ng1!yY)`NC(A_+;#@%^KOKK~qa9kDxUU#z4<(}23ZIDP=}CE=X)+1}13K0d9f z8fn6&V^+3Cb`8Upq4Ck9vAvnux4e0`&Vsd+Bz0D#49Cjc)}vqT9Ve^pdJMIRcN5GU z92L;9f5{Xc_!5ch=i)#XCW#~S-&>49aj3l~gdLjUya7|hy{q=%Gcbn96oJQ7XAmOM zm=Hw_tWPkCwy`Fs^Um4qhyW}Q*iFdybUbU>mFzdh&3S(k?kGSn|7#mtIG`4TSYW^H ze^y(_ZHxH~CtQ^C{xARWgA(H7ms`DjldNlMCiSwd?&{3`EcKh!{KvR9PVOYl%tq&? zZBy$`lUb~C?B)nnNcV5sJ7hMr3gmy}$J)XQN?SnfaT|tC{2$WV_^p=?*KYMt9^GAE zgVAy*TmRWRYj`3P66uQLCksns>iYQDxN%k~+eX&#k&n*s2 zsN;0CyrfI`QcN=>{GPTdPBW$y&-m-n!o{JpaDeM*xz^fa>(s)GglN8^48)Y!LoG2w z-^l3Xcn&Wsze4dZS+rp*EGd!tKVu7%3jIA4oK8~d#mPu0_a|v-B{z#UbVOQ?t{OuW3JlyyKPF|0z^1J^b*2(~kAW-ZQ^jCy&`X+8F z-n`k`Fuh(`7;>^uB`Yh2_vx>4&Xp??Br&1z(v$Qv`XN}zRgM;Pw~mkS{zsfqMG@|; z$lv2*ZQwcC0hqA<0a4rm|IXNrk!=Mvr8cguzjF%s!kQc}4@`kyVUWv2}8*R$|}tF7!nV~1jQ@WOS&m43d?@7GBqV# z6Vf?c*R!-VmcgltD1l+FWWdJ3`DG|)pA``BtN6NFVo+Q>1T7@wqDRmC$FIfN)gp?! z=;&|N8(q=SAa^93s_?d^r14Ct07fLH6eTsy55{+I>)usY zZ-j){$;-1yRJ{!BE}~w8-pKxz8gju*G}hBKIk?&xFvd|=a{WvN-#jhL~zFsECP;t){ZF*4Y6r#EjoF?hlz7^Yp*h zdQplF@K(Y7$22mA#9$GVQ}XxKltyY-2K;I^?vTB)(vDUhDlVq6(iT|CvWF6{Ao3)r z8K1ho8lU?>$~t2f0bzmXStBkZ0le2qUsT&QIZ5%M6!J4GhR6SAb0fti-pa0PGfiC? z=gpQRM8~wHk|<7bTlCsOT-+lzEq+BvUuW{hC+`CN3?m+IsH$R!=`D%cKk6{_~>qly$V!2K24yhKO#WX+C+B}nr@NPNvDotG}F8*y(^Ia>)uDQ?@2JvzTuRF>8 zQ(S7P0%HaVrSaGqk)Qi@C}v{v;T#G{>=~WY>9u3helYdG*I3km`N7$lmAT%L7W)QY zK8Oe)%;u{7k4{?c&)Y=#2|o6q!+4ecwE$;m8~qrkM#lM26fx6w^99e>TPJaTu%3hw z4LDh8_g`b0m}~Re-U=V0lpOJoS8~cD7Z&j5;h-f8PO1t;wP#G@!*o3v|M@#+*Sj6v zr+ao}RINw;nzPYi|Y;->C3Sfc6d(_Ny4)~QxY{o%V z4U;6z@%K~O5YFh)XV#ZM%xbFj;d;6JYHV&}-I_PS%^5lPt)m9*DJ=O&z(aBZ2AJ^)K6Oz$J^1bb$%$QCH;s8 z0BSK?A}ttA_sA`~CX}!GUx`DdJ^wmCZGXB3p`l4QHKq3nrwP+Uw@f1_QYW^tuD2y{14!GhD*$X z8Y0ZR;D1qwmllEN4JwEgIJ^5+$?#~3$JKJj$=S3?Ldio$&BC{U;tijaSCVU*RErAa zD!vK4d1`m=UmQe5*?R@LO48?X<$WqeC}|>Kh+zyDG$AdgVw7^htGPeK@%HdO z7uD}jcb;GERLoKTgG(?o&62Q1Id?#>KtUownyWeP9M)<#T=s{NDBkyo`beVA$%u5% zr`H){Y0D!K0V;Rdi4bLur^dadXTq&>$;wtqio7^fKLp)YgpkiuBr)Q){$!YmqyTsq zNx=)so;!!pz2ZSTKat{+JczR;)vuxSn>)AmpRv^Lt^v8tYN)_=baVxON^(-VNhKGQ zpjir3yPoa78!}-h+o7WMmTJ_-y>}Iik{q9 z;uuZkt$h;+168|}Sq9b0BjxsB>RH3bw^Vx-Xr*5(WLdkE75N1kowsl5GNx*2x@>Li zh?yMhmah2-()dD9?c}0~`URSKj5aJ<<^8E_ZeBnQEl#lIqVoaPVpU|IvY?{j-S#C? z#K~IN=~D(@aJ&1a#xBw4N{?Nu&rf`Da#1OFb34`yX$ztDcV;8$^;s)W9rI`}N*afS zbVsCMUaKB^>&lMa4CS-Dk`_Ef_Y;vk2V?x8Ew zP}@?whxPcgbh1>9wn_wlpR&{6gtz<3>dU<9zcm(XPbb>G_6&{{#AENm{hW~q zOw<}GaTDE<&6JQ^qp)4CoV6He6+aY!O9lh7Jv~Hvwgag zcYpZCtwV2WC@hs_r!Z>-W`e5mRSixp)G?j*d84c?C(C&b1x zHPw*8^Ht14c$2RRd2mjbIsK;m7QcT@Igbl%$%tT>^BL;eP@$_z2PeCv| zyX7oGr3~}Q_vT&*+nzDDlfpxV6VXh|nNl<^Jucp1V_YefXu15bk%Pa!-d6AMyVC*3 z3AhQ?7Z<(i`uLU@Li1TS4f55?xGKDi1#++_MywWMSt+>OtuE;%+~mKoa&ckay`5ND z$ElH?6N3{tWPZ^u{wHjEB3?ft5p|~ww7S=(`A2S_taXy}iTexri7ZV`r>YJyjUwu> zmmmnW-Vf`Dz^@X+NPTPP6Mp})l3cb4|DvM1!a1XqWc5i_y&;>%hu@ntzD#_r} z9DOa5^3N4fNgco&x)h#ATTSfO<*YBql&$O0t=`5f8Ri85GD>QN)`7?HprCW#AG-Z6 ztRTq+{p@m@H+HG~E?Y}Vvk~{rkKd)CPWWd|;$fqBo}wa!hdg0Q8rTZXvD+hSg&p45q|{wir=C z8)LfojD+(eu~>1OB8Pyt8xbpr_|tiusr-HvdLNrK;#1FL$aCrfbJZDg3Afg-u3`(4 zOpC|3m$(`c101{~++t|AqN`tvsha9}gyq&EtCtkh6}++x!?3_Ter+;oD})4i_^KW{ zt-Ls}<7}-zcGs;6xtQ!Js0s@+o7u9V6PiVI)Z}}Z(V`wA&*Rjjs(!7`^Jf*xjt5Rg zTC#xreTvfzs7E;)`WdvI5VG(oVpbYmicF^WbFt6AamH>pM+HGeyx95C!4yqw{wIbQ znpfc_i^I!ZeWL+G+rOm}_jqsy#Y5@Kj^1frOGo(CP->H~ate_7KfraT^R|jjIf%(> zIT=-33%&ydEjU#=J2*RCR+w7doLl$*wmH`=J{mT_g8|(xnMwxbv@ldEW9BYC7eHrB zWaMdaN0cLAkmjf^$3}Gpc}#~7iORhu!rn4|vu#X^Ju_UK8#P6ME;@w7v<|mKQ8P-~ z8si6$#-^lowc_iauJ`SxY&v*)6KHx=z7v)W9wWL%aLlC&8Hms3{8E>BaDWQGqasqI z5F3xZz}>icRo?b~gLtLS((`dL%bi2|2A|z<86T}jZ^>x#U$jCtuM_2GY$=E=*cS9R zu6tu-xOPt&QyE+E~iYV>hPepgMeA>iWF>ASfGKV2ux;gG6THB$XVuA67RZ-b3PF1&{3Q=(0`WVW1}V0yz_rxDp?l8>)Mu4oj-oWWZbm$_k+g zc&j9y_qyN^B4Vom9?0TkPEr7$yrhNCdBt2fBul&Drf)zM!s0f`%h(Z>mXctc&)3V$ ztUA+9XkMPWc5|Joc+7oeN%0u}PY`IR@AoOfj1UIC*ycZK(o-dy5LfqPmxs>hWmA^R zscjLUy*l0;Sy4=MYYjN zDLa=VmE|Y4Y`JlaLyVa)r0~eTNlke_r!^?bgn}4ohVx!+WO5%ARA*l$D6-^OWgq|y zV+KFZO#R-kzENM0#V*j0dtkb^Gu!yA$iNEZ9mAD4Rl;#!J8rW213!gyMt|Gc>Iu72 z!_E}3d#-I=nAO>SsIBa}URaX+;(4L|Rk@~pvYA<-B?9gNmM24?o$X>+fJ;@3lZ}IC zd$*#%v(~`8CBif1bH`ttbEhI)nnp15#Dh?*81^@Zu-hq~_b|InmPtXWtc`Ix>*0%A z`K3q(8)ut0w8gLEChi?^TwFX!b#1?s$*i=rsTfSS_m~_bqdy5Fr4&N~j1N>oKh)=# z{@hn7&*c=LWa$MN=#P(WbZP7y_PwW#E5C*kr-s4A(qz0?Uj40}y1VFD*feKyehmwj zX|f!CQB}E>5K%4$o!aekOz3ZXEX*E|)-xRusMR$O{rspX!18QfP( zb)k?CP;W#Mqhtlyo;vkzv(VF9^tk_vWFnZe?L7c6r8o3AXVWT2<^ct?l= z0PA~4d|*Fozt8US1gHf7Y$nuPTrxAH-X6}kJ92^ty49aLf4{3aiOeElY$Fkv>Khdn zV8o-Af32^G+jJY$bIsbdPqUIb+|!z>km1AR=av6~Pp>|u)KasOK`%p0-|2Ry`CtzP zn7+BkwzC7{6nVd=U^~xJP&2i%Qk8h5+M1bTIO<*CY`F*H@&;tPm}R`;zsnniHBs#8 zyZE#TOCodkv%Z2pvs&jpQ%!Ake{cXolA_`^EDM_7abiqTj|mSaL0W>m`%ykRrwx4-xk3lK9NzT1L{jnj(hAHgj+vj0_Pwv(kN;x;WJH?a(~1<(Or8P zjKRcQT}H>gr4$J5(|a$0KSRhx;txz8VB|15pKz&ZD;LSEHkaPp;{TYR9i5PGi^)-a z3(X0PHfOepeLLNx_ALqsPs5j{qxmK7bxlH*>rM|Qqn>I!-g#qS=(kB5{^BwK(LX90 zI_?k-6f`-3AcVUr`zbciIYm7I1I9;EE$NDtqP8Zc-FlK{ASSR_n`6UHDs=^}(z8s% zG+e5_T*)SH^;2;X(V`*C9}Vgfr;xfu0T{YW=!1EG)E?xjM@)GAeHA{wBRJ? zHs6TbRg)L&Tvv)Z{+I0(->i8WcEM?c8Upf#>Y6tjAJl) z8$1}o{141R6e<~dv)q0YE&)0F-JtJ2=hkpQf|)p@psX18M)IA$)gLH7S$a=BI7Tx( z)}#C;#4G^xNmj#>=ch|nuIZxAN;TZ;Tyg6q-Jex$<|Q=b0H&*vvCbC^{xuEJ8{6T! z+*?R-JsLU(R!Ehn+F>^wUJBx+@D~Y@A{iSOUQbs_4)1WO@cN2c z&?TJgZ~k<(u{QTicG!@&C^<>}z7D;>v`{R)dmAC}y3Olc8=y+SrsNB~wwigGd( zRaLv_Zu9nLxm$^|SlLrAh&WY`M6e)Sc7N#>lhnPi$ZXmx;O$Yhe(1Tg6CY;fbW5J44?a|X%(6S5h7QxBw{+0sZjc96h zT6b+${A{AYG|ulo|eP2 z6j4-!Jngl$n~MN{xHz}y*y%2h41;hjt!3GDzt-f23}m0Cay1)U_Vj#hi()}knRvL?b zr$fY|qSv?&?7iD0@IJK^hiLuJP#rm1)-SCqvlp{H@#8Vhwj!*EFGBAH?>#q%x8bt{ zl3A|Q=FY!CLqsQf%z|pjV92DgZG)BOs&-;RL`6L+I)=rP-^80lwZ!D$GelXD!OguT zW1j0E9S;$@1{3&U>v`m1A-MNO8ze}8l{Q~&^lhd@g>BiQgX0bA=!p2>7gvYFizjFl z@fr#vv~^y+o4mZtRaswR+7mW9K zY7e;HB78Z7=qIAuO*uU`^(^+O(v%g_G3s8ML8W~ywJqM7MsMu-)3Mu1$TAWzS$tuM zzoXUB2x+@ZK`;W1L&EV!7#-ylY)m>npFl@9h5P z4yb^)U!F4(pyTXr6OIQnGIc9oR>yEWv&vf>>lkjb+TM#RN0R6e(|<>z7VSHEp2M>DHGI z!js}CfP$hpk+7x*wxeu;X(MmU+^QA~cEk0*;%~F^2ciRKhiS;u7k}~T98yC37N7j7 z5g=%?o8>rK;^8|!)u8#%Z4St2DJ7<29v>0)M9AI3*8HsJW0~4Oy___hL`|v%vG}DR zH(%ii7!Bfh$iD*NrTmn$ezP2I+~rN+DOMDPvz7O8AG_Dq=K9WNfPq$Bxt;-miAm4N z4CEkf1p80GoXt3}J>wxy3i^pt*xp4KqeQc1Cl%%ga4x^X!dYBwA->y%gk0zWnWoLv zt+~~$LXI3`+qNzqsR;$vDyO8@+8vc8YDxi^nu!@&i5b$evGaYV7myMt<;Mu`z!&?L zfDdNcJ8w}XMRKNwpROnY?4E&b&ey+0)QaGJ%v24%q{X2xwaL%MHzcTnIusjruHtzh zhJ?X4c0%SeP2M&)MKxaht%ohC!^Qwf!jO1Ktye*vc}6f9g{PSjL;fqhgPv)PunL;E z5VI!9eu_R1UoLqrR#>{twPtmlmQ{??G=$FvmWPw}Yc|DU-v)Ljgtw6p5C7bEkSWS#-%x7&y-Z(d293 z`*>nxC`?_ZV8=cfy>*?7|i(1>WOrPvm znp$dtWQ%eZUu`+rR9#$dh9lPa%ZsTHd`#`VE!Ws!3nXaC4w=De1KF;0ml;Jx{p4}%6)jGJp=?wf6Lgg0+`J=&ATC;NtntE<$NXv(PQ z2ib8{@#$Vcvt3XrWbE3Nc7pNQ9!l`O+fF!2Bj~wbF@YX#*(4%^7*o=wTQ~O!MG_DM z-YP{L+~NA`xeFr01nl7*kDF=!fgCLMJZq9Y9CbkEUb5qY3Y8P4+9Ezhmg=R>%{WY@FwMiLw_#>~E9bG9QeJR1& z52TqVN&Wf?l(pf}ft&3u;Xe-iVNmz&ep?Kjj}J6$S%D!6kG1?$Wp@ZB$Y^9tWd0MY z&mt@cJoQfi_6Jyp{Ao9MpkpQZPq98m=USZ&nKW5M!E?u01P8!OTKe1C;l;+3QtNDI zA|#6qa@o!J8?WVevX-&QPazlH504#%lQ1{@Z4`M3(Sd?8pb*1%6GQcH*TypIJ5v8? z-w+z{qOpfCOgX+KSV7kJ_B?+l98T?;UO~+s=CMnwwl7axNY-ry6|2HAu ze<$=Y^4fbNi~W}of0B8$AtxtFaL*O7dHyajHMJ3T0`K6hicd~%=7s;`~f$m0k7k8q_ zDGv|G*!s}i37Nmow}@kPv;ZTi6O2Ey>%qxG`D_ZplUdg>$HLxPf#=73-}Cjn z2+<#ac0&MQvyEVDujDj&0Pu8oIPE0!8B zZ|ITEY-ooYNvfHw`P&~k&VFqE*j&5C22#_GqKOD8?D0C`8#FY?BkT_H>6wy>bah9!VPp+fBL&W%AhecZEeZWtZHCh*** zw}8aBf&}ts75bL?R{BeDh9)bGfYZjoEp@2vcWs9Bc9-9^+;tZhp(}9jE5*l7vUVfN zjmvj--nt=%_a(d?_VnDoNMhV0ZQ;hu#?PIW=y>u3KRfB}!q<2amdc7^C>kg`+_2C`6;-Z_;=NP2)4olBnYou9dq^FKwp z-MoOziO_zN<$9s~>ebe#SPxqE91ylK9g}sY;v^PkR z_sD2t5Sxn|?Xlt-M1yLxDtqtJ*LWM=my>U=XT+@O`5ag$Gg+{Oo0_<9AV)Cl;Y6E_ zD>gvoX6OXNILyAb!8NG~>wOP}v^P;SZ+~{Gy?91B0*=-)pX_1He}?1-`@9@UnY_lE<-$q8NA#A)L2P8t?-WvH-Tvq+~monL6c6NsF36}F7_!`XYyZg0dn0a8!cu+btAA|Y-*QldpZ z1MY}7>M|f@HRsnqHr!}$TSThAfCq|7-Fk1-weImmgQC8FiJGbw1DoJd!%Tz+sY+18 zE^=y?;Y(@a&ekEG#U#yPnl)CNuPnCbx_RNgOE1tka;Fx;kD2R$jOp4UxgVCNsk7|Q z%v8_iGpIfvt|oWkvlDXJ8mxFn1eQ=kzzFR`OvREkfH2TQ7 zgRZk&xQe|Vm`Ori@s7oHe+D1pojPp0*lBteUrXKPx!p$=tPd21t4%o02g_y5V&rcd z?VrWQI%}kf?po6RXVZ5j z%hy~FzOqp&$jXg9Vl#Ip=6r2R%{)^~3jj=S&IImEUN3#o@)EN%yH0brCC}TI)Z3cb zBZ4zZo$pv&8Oe%wv$3o7yea!FLmTk@c1BV8zCoz2D47fc^8+yNZ~-HI$m zdYdK{z{=Xu?g%T`_!!hyb+`ZGX$QS~7ETRMty-Xdg?dpK7$PwlwAs=^{)?(7ck z$aR$DXLHK0B1$IehE4?t2EIgw?+yI&zVEd@LN2hbxc0VC**mc4pFH;0afqwEpJMb{ zYCV4URzh+#^>((p#F*pVic@L0=!45x5VID4+o(>)G@fP9AAY?o=sh<5SvBj^*xGu~ zla$h^*Dri!p|VzMLq5?-JhRmTJ6p2z9v8IY;M<{7ep#~oku7=EAUtu^=VhzgtU7K<(M}Rc|Gjki`r7F zT&U{x`TQX6UtKoq+MQudXy#m?Q|0VUbZuwHoo)3EON3U<>qJxIJ)KxaT)k^sT5{Uc z0wZJMJ!`;Q3+gJRjY7;O)87Ic8qCI6)DH24J12$Z!_UJzP4|;;GDTjVvOjmvr5AYN zy&Ow_mbWuw2mGDUtzHwuKCHCzrpis)0Xl10q1KW|&#&h=|2WTxk~#;_LBWuNzTS$v z#-_spP51$23f>@vTkundf4NHcmowG&db% z3qPieiIi=}?XKJ4zBin?lvqEo!(E8Duk)2TuCo=O@a6i*Y$W82s&FBp{4l_~U^zE? zGxibgvGPiGHY%ODJ`WF7axStIj(>hcL+?bjut!Ji6nhC+v8d*UnQlk=z6cgV7zwC z$hUlSu+%RYqaU7b&spj4`_L$#LPFW1_d=k2jm1{ubQUI^O~pK@sh0opdq_3cPCs$;)Chc!gEQd-?;Q~Dq-?`S5lm#aJ>7iqh4L>`^3yIcNd?SYJav@H{VY!r|P>CdCS)s5)1Wka<& zgiiN7lVMq;-KN=kvEGx!!__2g7%fiBxcgJ-M3Wod|NCvFVK&hR@xc;I*pP2$5uZHx z$1+EzYg?zUcs2CP;CPQL=p7Bv3EI3KXE#fax@d}q$NJf*ezuwK&VDLeesP=scs9Am z|8o}Pv!}LoLm-o;SW{oUH&ez2Vnvw#fi+iLqG%kWk!sm5?9jbmwcKD6B0OW|G~}dZF(((71_xnDp=|~!&)l}#ePjaGyKf}c^oIKRq&hq~ z_@7koV~qFpBy>|qD#kI_p4vXYRAZno3UZB=l+~fHH^Dy&pCaa!T&&)dG;;b~{dLU* zYfB8qanmVPBTEn&3Ntu`*VJ5vXLc+pGsq1Mcd=1Mq~O$>km~K^BRRX-l(ZQ5+|Qr< z0tY6lf?;9SI<<~=k1uAoU>d_5XPpe=j!Z#_gg)R zxWS4wH;+qc4Kp<@C#xUxa(hPxShJ&ydMb@gN$X9XP7avr>cJrr9{@}+$#1Uv>q}b_ zp`bsp`tRI(7lD~ysk5U$*g+EJWqJ@*1r>IKqt_m~Pe&iuJkBZ9m5T%~<1H4eAH(E| zs%@^jJONDg$B z$)!ZN1$e>brHP+p4?ZYmM&E_o9GnesoJLRl$OS@(Csa+AAJUHQY(j}*$+)`0u-TT^mWESr00A1%jxhnO=4ASu~6^tO%P`uZJ1V2)+@43$V- zj+Z6L44LBRXbd8XJmX@1R9^OM9t0mBL?J?Mue6Db$5mva1i!E$&rzgTl2J7`>)RCq z-Z?OBhL=sDUTyf`7tp9rQ?J~zw1~3w`FPw|+cM9zj3-z7iL>5oChQ|w%f7nSRWI?6 zIXxI1NdDJ8(W_rWj6~d4s>@RjfC)7Fs5hWMr%F+za^5Y_VZ~K$$@Ll#d2#a4e;?Ts z4gd_o04OZUs15G+8g&FMK|?YPI;x+Wh3$p%SC$^)|W@J$YQH%@u=>x2=^ zx9oj;-#W&ji_6~BG=DtbS}73-5V|HwR=KUTgFA=~+kN`S8w5_C3`Xa_?nIQ-)aMX? zU?P8say@CCQ^q4*q1J){u$L!+M`J&EKiXSee+%cHc{!L0 z50K0OpZp19?WZPd$ZI}VcHF%rRaN231S@_IjT(H{w*WvDa&NLzSb_6(JISUpSAm9@ z$*f9xWLT}zw=<>>yx45Z4=lRN)a)*lsN?gf|M=lKQ@Q)Aa+lkqkLR=ZVr55eEi7Pn z83Gm4=`x8ZQgd)BahMuH77JvcZ*3KsAm_svl@mLl$~&aatK zik!^LE0NxEbLHe!E}8Ko=CgG_G49yjGUI3TI-7R{plEuv%2fA;p$vCG)186t$qm0H znut%#P)QwVba+6z4$6p6`T^4elq(m9gR;)Fe73n;dRTc+5FuChN7G=j`QJuHR_!6b`zCS!6+PU~s;LHdbX6+lz1kcj1&X{alH7-zyK-3S#aRI$^YxT(et(LP;0j2W0xpht1q+Gkh(VD)ilEQZ* zj3W&w7lnDN(egD3o{*biSNs=Bkfj>0C-)n!&#qNln0yS`<`7Knhqd4uXY197Dg0Bl z3+|__ows1ZJ6xVFT}7JvHO4@azvP3iOGVk>GM-3o?#IHzBH#sw~Bm7{!w1|~5IK((GIBH$UT?7XOY459JrY3_$zSMkL`Z7O0 zH*+wZ=cu5<%#HNgMC2y2V=SG!z>ScL^VjBM1ERYtXRh&Ei$$mV&K&GiqV?1Jfz*#y z4E4Il9y84}Hyi0jOir4q7EpXjBp{|QcTL>He++(lbPs%|>~x$)uzAo_UF`O2-p~}c zxa>tpwnnN26h=(4pd~8O?C4&E5I?Dc!K}dED*1S)R%(JaDnL29toaCL^-M{q=v7i0 zU?O7SC77Pjx6<18k(EG29XjhEK;J$wGq$Fb78}xk&8Qi^(s;Mm&q8L#kRaGS#~BOye>|-!s=7wDnPoE4gkvdZp3@tjUsRZc!qWOSg^EzL*O%l7IR zzjqKLiuLWCIGv#m!OHr+$7~|1>{Q@7$%^Xr{7QigF{S_a>>#5sQqtP`JOLBsbjX6$Q&702P5K}l2Pv7T^j;+W;W3Ta>aPkVV zifAPaB4Zty0>iCMB{-kG`P?@p3Vw^zu{U?$P_TcNxsQ!%%2|S>Gq}eGI#FR{Ze57R zkzx2h6)+?>oNrD2}Rq8n~GoxKy_I?}Njl6E{^x{@pYVFh+=h`hj^c%el zZ5}F7SBb}1oKjn9|H#Q#BbX(#BDoeRM!@WZ+xPy*g1a^Py^E25TA86=X-B{GO48GB z?wx^vqMtobB3XBZddHK-4ZuKs1rBWk2L5#C*b)lh=Je98vLP2>f3k%glYmO7wbV%< zZ3L{1W0g_gVwfwwbb8y6f=^zaD+DuRoPE_Wx;VCWnU;eHY5}%vFwwtejCossMe6gi zQ;UenQPf{6iSi@>aYJ**@lprc2L0CPGU4CZLOVkgpUYLir=hpOwOY=fd}28@i-d+c zlvfHJPu9I3*7891@<-i;S%mJ9D#@|YXuv)$0vVDscCaK2RcVQRi0=GPbXxul0_^n& zAkPqcSnUkbOIvk2c`qdjxiQmMSa5WP3oG8O=vaG`5f$wC9UU2Rp3=a_cT_ce_Y29| zx1-?j+c*ZC0?{4PmUtZn0hY@eS?}C}&^qi=s zALg4q+;sP>*>V&Wr7dlF-V49kdHQxL-=W&J8;-(vY|%TPuar_~HkQc$vt2VEm)K#8 zxHpgTLlF^q1kq|hH&ln9nwRJfEAJZ|H(#ptINQ7M9mTHN9u@Xc(=1_b(M71QKF!Hn z(QL@%i-E3_&91Q~4+1O*TX^72NYGqIXzM*D5G(FBQEkaF>S1kjbR=L$2yhb6RK}@) zsZ5A0(4QNHndfj<{}C|GT3dGWUGcQ;!rstf?#b!s@(oYreev^g25&i!u&v7Gm(BS} zYbU@5s@x{(r%sk_BT{$lN+3l@F`=N({5A~@O-jv|H@{UvhC8%)?~bV-U3iH|6zG*D zQ|eC6f|mf2lK3(8hyK-$x4%8kCwL?!-iB%Odh;R_uf;*Fr1}lyBlM6TA4BeP-vKHi z@Q90FG`R_WCnnl(l)8Bjnzz|%vA#SVT{%1TEdd0lOir@bAmA75ESoj=KuZqotLzr1 zE8=;MtD{~MASgnL+h*%N@rS0$#iBYqT}H?KQ%1qTCq6x!e&YC!{p0XZ>#Cjep<>d6!%_yJG2&4y$ieR*wD4o z)xF*>z*=*+Ct33k1QdtjPDd6hPk+4xqK4v#z3cbqPx6q*Z)G)YiI?gaqfzYmwVBYD z3{n}nEdJ9Aa8Hz8a(>pyYJMMen$Xd^Efagdu~FF7RXyKWyL%L4Mpw;bs3@KHzVV%rWt*jiH>(&{`nc&tI7J{7aAakt-ASaJrwCj-8%()7OXxbeDmMM6{|f7 zG$jU0&uua>))nC=*Qc`xzGY`$rKX6i>I0p<@b*qaD$?8D#iS-3c}mk&jP0WvB8wCH$?esZui;a>w8cuK(Thu) zn(pSY=xTX6d$5PnD`8pMg!c41m&>Y^P-TX+R)bCzPuw&+PjGifp5Yr##}$mJwG4Kj z{$A(2TA#6%y1UKs`qkTm;F_BYH1V^%z({%9>?G>vn#g{cSfdB-`OtP%=Kaus&2UqB zltce86$7I$BDD3(HHMwB!ENtw(UBgTtaf@nt|+#>uP^{N`>{o8D{eGMXz$u6+z3jv zFF=20r*ct>-d!D&nA&z8iFFHt(%t(h^>kae|CTHB>yo2DYbdnu^?pS|j&;DurbDNgT4Zpv1DS`Tp7_Wadw zE8fyX)Z-JJfhgZxHLVZ|Sc%^mZEThY_j;U$i)_dv-4=WyUw?A@K(BBmG z$AWqMR~_>`zO|3j+-#wVS3`5e8y|lBDz#6HjKKJeRBLZ5w{j$+iq{(TreFs+81#wyw=~V``8M7BUelGiu7>VYr@-x_ z#|N;t->5P@TL~}M$#80vr+1AVC6MES3g^prqXN=3GZn5V=~{l5`zqE|)aFA(s;h9> z_ivu(NbFTEa&xein09Fz&y3awo;ZO89US~jn%M}=$&um0HYy9P1KtIHH%~^yg~=~b zJ$I-3^+d?L$vcC8Vum;jlrnAyz_D)3dOmb+8O}Vnw@XvKcbA${Un6_ym2?=Jd2reO zAi>Eh{3ef6z>XBF`%h?zmgi72s=jp(uT{X*mIOAOD<2Z5+b|mZ4nOJl*58U0)EyMi z@HLohx1&qEVDAQifwYC>^2r`4 z!SYb>yZzm-sK(mqPZICg+Nyipiu^(^C)9noO|m&l=Tt0Wv3aHmif_eSZ@LkeaxrOi^ch9>SNOxKbkq=uP_~`Vp`Z3Jmi4yOOSY%+3S$Cl2g)4nPB*DLJ zEcN7T7_a4N^E87xr?Fe(LPOKM;y!09w{62G*Ync-CBtEJE0n-f4IL2ZZC;+~>s4Ac zQxc!8vrC7*0Y+5s^=La3u9%U?td_)1?r-$SZgm=uo#ZUv)zy2ZO9~D(cc`l$%A~Jm z&8Kcm=+TN;4TQfo5D^C>AI-a%{CM*W>WMxE5=9E7m@+=s+d*0#2&)ffDi{rg%?jG8 z+N**)@>xTNY3Nm4{b~_sHZ@{yu1U#f_nv8)oEHlTZB|q$D8hL7Luu*BJ5=m3}p{PKfC|Ss} zT#Q(az6jSxV720egF@BI#QV-;WUlWJz4z(m@RL^!pD;ix_w|tdF7oZ|XC?7y zCAPMkuPwhhUrfPeNngy@UP=uzfV}JTi099hzz?H6nw$(wOvZt#>qcI%?-_3k8?ei5 zbX&Lo_|(LWu4?aZ>I1k&tA3zGa)u?vcWYLjWpb?iX%%E7O5f6@VADdmyd}pC37UZv zwcm(=*08vSp4@q%;7kG-3AntRPOucZ5<)a@Fgc193M=tQ11PUcu>wB7#Kdfy4R=pa zrhB?R2lqO8-FkrCc$o_F)e+Nky9Q*hE}uqu$U`%1Pxp}TYNsHQqsK7ytI?o^p(!?k zaL$i;+~@=$vjTi75|=Q-E@0#Ocea*-lPQL1)XB1w1Hcx;GMMmNjTt5JEy7s0x+kd( zx&?&?v?EcZ-vcsJ2Z&Fct?hQ~R|8X1FC;L{)AA(w_Ywv8)j5a>2xMOl?Api2IHVy$ zamXINIYvvWzNg?=6Y$}ViOlBTiA>i}*VScw{Q~EkQGj!oSh$jOMiUVkdo_*G&4ugK zdF^l2(pFup+4JUKw-!iC7USp6Vr`03pKXk* z60;7Y~!l%F5_j@G#D%zpexTq0t6=rB{u8DDAq{QLMBWJTe}q^t}rcNmTD z%PWr(0?A}M@orl7{-#SkA?Rf*;7C^TQ~+MC{*vP7j>qdeL3!9LSz@h+>lj-eaQ`ncW8_}J@wie{=+7-Yz2D0}@v&hiS$UQ%HAkruSM z`35L+l3@xpac19;aM1KO5#{64G`B41n;Y9dcg5u%KIsW(oHjz((g>7k^Y{Pe=1CZM z?F5BSX;aIYTOOUiO;0c7BQP@pw{Izh62!Dma=RH{2;LmWF7FZFh*%4T4+9!Y zTU*-%Y^(_9dHxz9%K7PvtikosuK17Y=;Xx4^~;T!FZaJ+0WjR#`8M#zlK9C@a@8j# zb4MZ7RKj&%bn}1uG0jXAJ#ppMg0{3J@>%A=^tjMl*%|X;-IjgRop>S^wMI7!C)?Wb zv!8PQ9#u7Cf?V%eqKURd+>sR1ZcF4^^QAgcHyK$A>fO$z)84D8>N0ZiA_eb{b+63J zsB0@Bf$~NqC11YkZRCzjpo7sl4QK`~izr$QgzXGJPH+m&+_!m2{dH;KluED`Pr zEou4jZvm^yn6}D3MDvE9X#dF3#qLgb-CqPoDr~Fpguf3GWsd6!rD?Gvj_fxU8jSEv zEzL;4ijyuZw_YLt*@@;)E>h%{0o_?H61dVmT4BB(6gTRecO_ZL9U|%9dc)C~UL?D7 zu;smyz+=|it~yb3hB^2WVgF6!OwIg#vVg~eNYwQpnx&Tr-^NkvrIVqKYw$DQ7a;>l zB>n5r!9~0qe^{eq)n52{e765>EmB>>lgBWOCJz{JX@}x4PLY{Oh~;)c@g0;N&9-HG z{I-mYds_6>*?e5?h7ZA5k4wImdI@f}(NVXIjH36bDMiF$4P4X78UsY>3m`Z*$0-l# zoml6J#2}XoTIBuA$}^^o1@`YP(erb`3>&6>xAtl95U#Hgh3;Q)sI#fgpZ?^RJxRg5JX5v4Jws3hXC8umnVuNAth4ufNy9V3OssSpFY`4^|L$c4IJ+D- z1IJnFj@_&YW1gUDA5x*Du52rlcjQ7`p`A!+w{^AS&8WaGr0=m(R2a?C%LDq`uB)6^ zy1djDW&9)VTTxkI>Bm`(X~W)$VgE7M0OMreu8WcncE5Q8a*xLX5>6Cq^H)@4 za-p2#M}{Y7Z}!gA8=`~AmXhaLP3KohF+ILjW$A7A`7?oUvhwZ|BV&y=7pd_Iex-6q z8z0$+mxPJGcRDiWD$8KOvTs6mNdl% zSYwuiqKD4waB+I!o%^B7MIcxwjTCLrZ8!&U5w;}WE~Qf&TpBt($}@m=)Dc|s!=VnY zTi+bWgKu+(uqZK~Mq`AT=C#sHwoiXrz$*H_JJ*Y5Y3s+e|L|EO$z-1(2HXpgsjfPE>?SFD>!kpB>*yO)Bx;~|^N_fC zNtF(=x&5_os^rl^_Uk^4h=OH^8)6R?hHUOoMC5a_eSRdF3%`!|Ig>w&aTj5MwetoXrsKci(d8}j}zkIZfo?y$4XcCGZPjv~) z?o?sED4*cy!s`vP4 zWN74A2R(6P=OzEm;&BRZ#?|)l@yJm8IU~NVUN*t#fh6Tyn@;E;2e~?W+U?E;KYM#^ zZ($bl(4EDFFMEUsm*sG+&VGPd-4Xvx>x|@i8`(sWB_|KsOLr|)zRW-w85Y!AnYzcj}^6ufo!|TA@+u#(~~(CjBFs@Ttn= zI8tqYn}V7|Ny=ces;zK6kl6&a|MqV>t@=4%k{N6@8(Z7Cw6v_#S%GSVkN@c!Pgi=M zYN5F#A>oacimtA??9d{6k6z~P(UzOS!FmwA$vQLNm2-3+`?0Nj8`Iw&B_dfjwp`gc za)SMd8?l?TM3JEt=$=d=bw+F|Vk$x_M@Mn(ahOChdOlJqiZz4joL(MJb?dXwqkg9UICL$^ zM;9TX-_uEx`@XWv+#3+MeB;39^imO+?UZq5h#FWYyVCQ|YLIYA?1R81?GNLfm3|bS zD#?Svf0L*bKHNB(ei-YaHRaGO6lfUP51M}PuNqCut~37WIlOHYG&@{MgI3pRTzok! z&`DQ3*x2}SF)3Z&D(GxI?xL`leD9qS9S^_Md`hCWpEyF`FE2!~u4t*VarU%x!b|v6 zY~&%m%WJFqd^0w?#Q2p<;r_`{mB?Fapz0%l{R>YzZ*$`Bv^=%N!Q^jqHZ~^W?5B#& zeYhRXnG8<~n7oEbv;Q<##q{@{$$87Fae_w`^uBt&c63biH`66)rISgZcVFDGM_+FwcG8b+4>=E8HwrE;=SdO zf!4l|r?5G^eNd*`ha6%+vQt%u!Nld~+eQ zCv^Avu_&VHV+v8R4U8sDw}0E;wWE^EYyV+9?&U+J>e;O=Rmiqr)BVgTmDpK+L3Z{1 zg6l$Yyo#!wsM`_VXNlV5L%-KIK0GW!1?Kzoi58OA?8T`1@t*rZPPXi4JF~oxp8U*r z@v6&nW1C=Cp-D0~>9V6pbV?`qc;E08N1m=CA9y=iz~VV`{JN{6Vt)kYqhdCi(DH!v z?ZP}760(B>>^YUAM*D~RG2SSD2qfY6M2SM0&$LHB!jTRS3MQmnyxaD7qHO}{m zr(f5vuC#iox#eqKH}oFrkIHH!&oO0y)3vdSH4;xZC-9f4=f3D(7;f}=L8>>&?TynhOz$16n*%J<+J zwK%*RzSNw&ld2xHM9m3gRB&UonzKOY)4u_psLvQieQ=Zbf1Jo<6)C3yN&piqVFD7I6Yr%$=oq;DY_BI(_Meh zO*oI(Lo2Myen>#`oE&3IWYs82w@?4LC!h{OiSrnm?Suv`81Hd&yJAWGL%SjG-ff|U zYfn#{!=MObP}CZfA9s+!60b{Pc`vVzp{_&e3|t}Kay7cesepr>i#;{dJx(r*u`kzS`r>f zl3qBlVwS8uoUvQ9T*nEXo-cH(w{KDx(Q^igqJx*6Oa|p|GBtEtDYwwH_VL8I98FaA zE3?ne1`1=xm$k}{by^gmGx2!=gIQ_ohzpNfYZ2q++sTGlSnHyhHvG0EFTR*RbMh9E z_xaej1Pqpp&&-sX;&E-JtSz)iZ~t1a zqdF3sbm<21I-E#V2v2iIxS71MtgtSKWz%!N=!DHMSC+}|YKpL4hu6%dZWP=SP?6N6 z31O|1J+x7wFY z5}vNbf#Aza&tlsz@AV40D!xmf6TnNOGoIt1T|{Th>Nc!hKhn(5E|y8+m)xI-l?W|v z<-MJT2vwcJCu*k+T{}&zp^7Sopexi0v&husqSSsjNATfcK1s~lpJjN<_-94VjfM9U zZOc}r#rgw4h>_h8bI6-D17s~{Gki~aEBKGKC@b5;MvI3PQ~6Zs%mI_F#Z(^qG-^&a zbpa1H^3)+W^`RLnvW z2<{u;nufJb?gj>mVt+g=WzzCNMQ(4WFco!OTFx(Vm-sF=m^=hdg|F3f;aq{psU)$j!|89|%_*rsXQ`31w2DWh*(Vro} ztwwYmYOH`6n>8|*nvAY)H0&Q$9h|I4wCD#E=G=Lx#7~uiKVf_uV~{7akddX`f_J?}^h_hW#@vk^ialV5}@|FASfe?I9dOU#j^+pNrh-85BKlY-t( zW0LXw&e7*!bXM!%k!^7Uuc>lBTJWbtZtt2=*WrI;rK?vhuS+S~7%F)1Z!7;cf4e^! zx5JhLb7^aSY4s~)I)XBKHo}? z+L6d9TE`A*%!_C%4A_ReJlC_}&5KJvl!V`)J%9dAc7{}|^7eWpE+)3}PrGj32omV^ z>v$kcLqhy|fdF}v0e7Ryt`oz!*IQRE%u~R?p;tteT*#SSC;I8IXxNfBI7&=_9&$IN}0_4 zN~<@DWd`=DBVMICMP_^aCzErNQEaR-U*i*O8=Xp53cxStbVUnu_-0D7mia4f7v>}o zpL338@TxN09RY#Y7+A(sy5%hb?!;gau^!yH_D-dIh6V6w`fB?T_4af{ zScwu~{mhR>H)C-m3J+Bl(Y2n;%-uo(tQ!?+X<8mMj90t;*=cvIgj&KiFG0999y2VY zJBJMAM*dr9El#l~6Kt$sha>e|@9(f-7kkSnG40jF6rILjb+VPfg-n7T>$hP)6#VSh zhNCwpe{o`%r#RDYZWcgTot&L@`JNieaUhoLCYZcPpvUF7lT*SJ;U|NI2_Fix8U9@9 z2hBfJ!jcNK;6>Cz@T<>bi;00XJ%k-+YI3f`P)Os_v~mOAm{6fv~j!u%#!m|BE1w} z!pHiGQ4PZHcq+54mhsYJie-FmPBFScMcM7bK_=nS5$Vt142L`Rldj=0`~bRfEg3x& z#2NIEoJCe~nl_JPaUCFv`(UgDfbVa{|^4#T%-<@7Vl+L&OR$+#}u#E*6s*+&; zpTPVr!F=Z`;uY(dOvBq5Eg-6zUl6CtqH|9rMLvZ31=a;i*$3I0;yI7og$(CQmu!P2 zezCx}uGES!kuObA93gG)g$JvA(u_bMkr=f}@QHJ)u(bh|@FVotq{;;$A+IWr3%c;g z{yFXyqjb(ZyxzQW3A+QgW%(2sw0wiG&2&xPZz|s`Vd$op(xBE!RE&SFsGuLo6w0k_VLrAn@=U#lkP%2`4(0LyoNgOF1 z0jr*Iw_b#jKk?wc>GW*mc5%F?sq&hoI(pcOKn{7Tbg+;N_?41_5ig$>9ZyJzpC4O7 zScIiFVgR~&7fl)yvCu&Yaa!C6mf9WuN@x>FU}D0wM(a9S=~BH0~<5uOn_`AOhZ?j?;;tfpmhf z&c`&%H%$Jn>p_6g=0$(Qo!0k~a#{9kS<9>vK_Ex=OKYUs7Sn{=f$-{~4p`gDSZy!i z>5)j<^>*dfK{!xya&OYld}7NevawB=t;L>;A^$<7H**ua5Ea1df06SlxYP4rDtP5c@@Y7#kZ~ zG~R}F$W*!TQ-R@YmILT$Ad}V=dal{d_{;r;mMFuSX&sEW#5F>(b+2R95*5;9TO7Da zDAqhKOknMv$rcwI{R}4SnGYG{i}UJ_K8#E06mz__=%dEJ{RbS8eOz2^D50^Xq5$fUlHHT2_^#b-4=7V-cCteX_C_AEDt z@SCxoOKyvu_b@X<5})Z^`3~VFxp3X#C0KmBX%D$c=yimBAy-J=NZ!BK073!b8HB8+ zmElH-Wz_+Mp`Gzwn{!+gkUA`IR*}ogGOX<&dTs!$@l1~l+{6|Mxx%v?QJEgJ*mikM zY>4{yq!QVU_x3PNcfg)TNe0$pF@qG ze46P*X81n-?Npj3eRF(%PWZXGyexJy1T%6a)GErrVDV>WrA~t$I_MT*^}~mp^OK1r z{!se#&T6i-g5a_;B!8j;JhUy+`@rYyn~)dtp5I&>QA@M^$et!sFX4MrdI!D;fjNDX z2$HcNqNqHZQGEY4fc;v1lG;`JXZy`YU8BgAvOo#iWtn@0V#_LFuS32bulgK01)&+E zto-kT>n|`?jDP^Qhn9<*oeb-M0F?B3u87o+jV-ir{Z9v8Boj@R$b|tij7*D=)6}(2 zG@iPOFmksKbPS@i1R8@e>{51B+otPsU*Z4tN~V8sB1bae zc4Xw)bgODKfNY%i7(iMs`fh|EEcYg6`ravuO{@Le&;a*^v8X_M6=F)QyE4r$gYCf; zBl1|M6=Ft`)k2MkxG=9!u#!kuKC4>;jGq^2bWua0+M3{x*rQh&eYzKS$j(5Ore($0EhM16k zqB28bBza!44DRk$|HI^sQm@yz+P;${8ve+i%1Xlx9@j|DdWG-wDMu}siS+5^w4<2e zf=k#1vI(qD_vhH4*Se2HRJ7>7s+HU$Y@h~vH<(Fy;o%n%FVAZe!an@+T7JIe6ydHswp~myubeObPfyh)n#^S zN^#uv+QrUQ{N1NM*i4J92SW_3wUZa;6Z)T3%Cc*IG1VJ-x*htwH?zAL_55NrznZHT zU^+w1PBgsCbsyWxyLt!T|Ah(B_EpJ+ND-P!3Z-q!oc^u&f`NAHK3PU$MH66@_dVwO zi^}t0Oi0}AO+DImV}QR)jXC++?(Xk61z5A|iTZe0c5~&~Kv33#yL-di1K`A{(Scj5 zqlV@lfN9<>g8LK#6ljsXQ;H@KosfnGJ~i&1(xpBD501S7L~2oxtFiH$iZLYM1Mo60 zQ*cq-tD|^%<<=_4)bMfvvq@O8G-v{Z|f`{S`_=q*?KtD6ev^uYG z#BhLvJvD2=n7HqohnqNW56QkSwCM~=#a!GIF{8eCAtMH&rw^!opY`%O^l?RaT}CKgM2pW=NJQJB@Lhn8Qi4d^UCqh2jP>!esgx1A9D@3f z0)S7*Q%o}cLHLWpQ_}gKU)=!hVyk0yzu1Ru`{Pd%Py@zKeRFda;1CAn5mUVJGWsoJ zV?x437Q~qy#>F{0#)ZdkmDrf9AVxh6^GDOCn(WwL#f#K?RstTPA@fmCR1duGFCMiW zsmp$6|1PgyHA$+<`mFs`U;C?Q*7c~#D@(qQ$u>7M^f8c?(jWV_PEN&SYlLqPjonw~ z+)o;P0m%TveSkH6fG?yiTxp+P*fcahy>fQG`S|Udy4&R^f8S2ao31Z6T3g;Yhaw)I zBR+S;6{jvzn`p#H2cn;cp(85hr(Xy=vbN;!);f57@2zE5;UCF1UFNr=z8MHhwUl-> zC+0Y5aiYElx zYK;UXN9?0ahrAkRPKj2vx&C5jXb07BeFs9hbr(`g4BKpKic&kQfvhjjKGue z2=>ABseIFhw3+;1Q^4J6<;n#Txg3gYg`Gm&g}A+*qPmW;XSDd8>xE7VgqNGo$43bt zLEfDyx}g!Tlxo`^$9i1bC{VrUaiMw6UmtlJZYEP+ctm!_ygMbtUfjV50(qCCWyAG= zWQ5yw0ZW`ukFD13GohH62ni{xZ@lGVOMe+R_*2_r9}Y4kp<1q$#l?6a`3!}p%N@@$ zh20+__fM8CQ$3tAM3-DFtqkICpDe95{yFMJp?Zz__vq(IMf(FR-F zD5;o+##W`{8KGjH;ELGXBgA&&{~HG&3`)vKf;Y!T_rMUDhN(d>50@BNy6~wuStrV^ z{}yAr0lb3vjO@kQq=Nwh4qe^B9`1K*PkNo^#C~M+Uy#u0Vn$o+NvCv4?X55$509)D zF12yRtBrSZ0QAKu>tPF%;NBenM{P6TFzZMAuWKqKFeK5o<5$(a;P5xib{iVLzpc}} zN5EN+US0OCIj$SwA)pTilgKVjFg>^I6BA<4WCSMZ%@@u=_o>Hs*4%bf09(-kY_$@D zjxH*CR(3}~gMt}Pp1|X9q$~+5y;grR3HyXirhw!<8Jgl9nRfDoy&jnsYF&BQ7;3G2sr~~b>KYN_X|tW!!ROiyOihmT;qOz*`B=-} z@#4{QnuYwbpkRbGl(9v z`VXj=h9i9*7pFtNVw1>JxB31kiWvLOxI|HyMel|1zhCRFHxm&x5m)%*$>y|x=$A_j&6* zx<`@f>je(0hmJo#^d6o0PP*{puPJwg75YMP&_PewXA!yw9;@my z(&pkh8l+Ft{xib!^EsXuJ|!IoQ>Q@OQlo>A^TuV-Ow1OqOz-riyR8HB&B%7L$%uXW z`Tg;>H;!xUhEUp$;7$V~l|KFC-z1!q*-hKHoKK%<;1easm5vzbR3HY{0`<0|vs-XN zjq06C!-Y%s2%A@WIogQe4j&Be23=JT9$#tGa~T?(m?Kv|P2CZ!JO97_ZdX!~x#j+# zDy@|)*l$x5nG*<}c1^TJwd(Z=2q*x*@fB1Q(EpfE2Z2UVEJmHUidP*>!5JX(u72T%v-?m|zh>5A-`_Xyd%)A5P-maZG4 z?GUi`_CvNcC-j_c75eK*q}9e#)IhznUzI^d0Fe`o!r^$!E%0)#z5+7b%MnZbShm;= zJ?0(Y0q}G~ppze9k#S4@_if()ZOaph2l7kM8Q7B%l)@cxneqCE}7`pNr5|pPq586lXFD&WJ+D|L9e*G zOZ`$sdga;KQ1N1Msvw&}MB@C~h`juzJ{io_n!drV=`Ce>GYlUE3Fi~gUxm)v{Vy#w z2(3;-dTM$UJ~6IloxJiAzv8iV>E&Nww21N%N;DiwTqIEC{d%I2K&Myfd?f%ckJ3We zVEKWXifdF+Yk$ZVYS2#*6^c6*F&HB6b!~TN=elCKsx+Xq{2xc#6dz`BecUeLQ{CNj z{RBehdvic1&DMj*8+rIPhGIxi_L|NGF399V6q-N{cyHCLV=$f1Yil^7HJaAl`QTzb zkv`{WWkjpe{7%>0Js}*t>~czMg!Sf4oI;q_m8xQe@@6NmuGvIpg-|tNFX;r)rY@?K zaz;O(T0U;qo1F5-BSVh*$|+1*j8uf*(Hu1NQ%JAf?=`X2nvs^p?cGgo1`o(k#$@>z zm5lz)KN$r+_56z1wdRFtGgUFUCO@{Gc~@1P@B3B3XM4J`t4|>wbW_VW?W&MALBKix z8})snE&U1Bn)TW~rk`u~RB}1_!fI(Y|1E?qoX^ClxvT&O>63|AW073zKL$9$q^HXR z5>xSBe)rCRg<{ja+FKY_-zi1zU4A&co7iEqa>X?!%i&~Vk4LIKvgE>(MGWao*;T=NW*oV5;Bq!|t_Oo2on z<-@D7k$OpISeD&)>m5Xa@6FjYPc0s>K8wiC&btAm`}LfIVUlU%J;4I9&2I%dMRZkp za@OPJrM0D<^dmsg9*}oYh`r1I z6|T#KaFp9Sc0}=fwdEmmhPu$z<~AWfl&h~DTNx&82fO;?`|KefEXJ$3Q=B!7pM%h; zLNMydGm&K$IzJ=pWn5p->peaLnN)q;TS`hv@W|SUIc)AAUSW+@M*sD}^p+$z{Xc7$ zTfyIli}aD^OmcaFzlcrqZSykSR_0M6@*h^ z3f%Wc8O~R#ksAL8%Wm3gIW9h(nBZ+Ty~q=+=6c@~ASGvd+~)rJVxtB!iWF$gMDf&b zkavK2QiI2LbWxmHizkR;zKmIyP*|o=-m9ICFnfMTGpQv&LFne*yJ$4` zMcV+P_yQ4Mc%=;P3K?yS#;ezew&7ATa?K5JFMIEcp0eXWHK}t`yN|Wbp=n&Y8P}8e zElmsw-+-jzJFJY`lMuZ!`W)$JDbD2D6$ZV3)3G<)0F){$!p5Un$h4W)R=+GjCzb4- zkhdt#>UENAnf%@Mby7(5YO6Tr&Hj4pQ*d`tXwmsf}mL_SmD zgx-l5)-&4R*st=?wMbhyK|-tyH5OK`S_Xx%!3CurM8S^}l$~qpc{|)8qG}xnrOPo9 zWZy19KJap~!8yz(db(EPzW*>X+W#FkI67`Q~56!n%yAD4lU<#L> z%DArJ`ML(zn|g8&U;MY5ZC*zapYMd%Hz~h4azkF0}2VCv(8*V6m{b6P(k+Xs)66 zRQ=^3MjAe>%JtR#NP|M*Dj7l(&d1Y{`SvW60bwX=@*jRojmG++c`=ilnQ54Nf7^Do zIw%Z^jE@?gj!3IGISQvUBc(XMLBqc)eO&IIlaB6Vu5WNasA^;DiX*JP(Bf-uA1Sn{ zNqn>}WhEg1axliFC=_^Q6x5~uUS+i09?_Sf$KRaE&P4C9Y8r?f0Tgj5V_y>iN$Z)$ zJOT3I-YNRklBy)s2SeEPoR3$Ha4WUKct=d!WG@*^SV}A1L`p^)Kl}jrDtcwCF&M3P z4t5?^*|ZRz5cu#lMgm`qjmp}Y)Ls0qzpFk*UK9#FPGNlYLp$H;nReRdL>`lp;(P{K zG|sP1ITNy5!q$z_T{D}6s&RWsNetZHNpAANj;Qib--Es>Yd7l8q2}H@pR0ZAh>s)H zj3&}3-!D<4H@xRWeqDgS!-~&Q%v>!ncxwNu1~>pcADlk=cFFd*-BSDhEs>eKg?`tt>>2RUD!Sy-!@-M)mt)0PPBNxqNP%ou^e^qBYR6z#(DO!+blBgbR4Y~30YzI}E ziShmSh9{R}$o>+L9`0bFBr^YTTSq5#IgRUVFJX%pQhMjECt)IQ7THD{^Ghh_E$J>= zX2XC}R9%t5mGmITTKi*6IvLkN?>mr9Q}cSsp8by3X)f(|C8xU<;`7*!6%RoSoM+08 zSfuv%Y=$+MG(TL?DACdXvl)q{CjSu*D9Gl+RBb&`)N5MOUti!LaY-}yP5Fn*Z^k!W#2Xr+jb6g{FEJOuM9gnTobXnxEh z(^N09yzgt(8_`l#)_wt$bjXpNu}(R0@Nzek4Cm{)BYk{fjv}QzBH^8u^l^Ju_X7`; zs8Uzu=ElJn2E+b>q;WEiB)%K^kwUgmsOyku_Hsh87Nm?7-1+HQ4lf$$?|@uWZ^R*U zKgzWm<2)Yg$~OsZl?qw7u(^uKKz+Mc@p61}6f&rY7f#IE?A#Rp3iod@#p6&vb8K)p z(P)i$r~0xNUn_Pw$FM&rk^$S}2QDB6?5Xio6q`uDt-p3{9Wa6-)03K<4$>c&1lG^= z9;?4bg@$0Y+0ncK&P@Pw4KVUzp|aSKU%t?xDPYe4J8p5R2dLgdsG+)ga(LzWS&kx-Y~_h+6g9!_j$b4?ddY)( zMrDO%UVN>j`_)Lva?)vt2ZKlyVD9X~H-bcW&-ANq25Y4jWCyK9xc933^qSY;J34Pd zs-3MkY28K6M0aZCoGAE_K?$Wf_l*t{;hyx&Oj{E>O>9`$kr|g2Aw0lehuO1ii#uXI z16E!vHN61p`&(|J(X&yw-sAuZ`Ut%|Tf@h|<_byJ+;6p)h_6TTywV}T$%;^eCwvOPo%=;g2Dz(B{T6gQ1*bB;k}rytRc zQYIHp0`QZfw-7)CP+8XMijY2KdV~6%YXMi)DFI*7LUA%geeV&8aViWW;aEi2eyy>R zorh!c(hM6L;rtAfhF(Fo^jl_bJ+JnKgGX1Tfm{8Xo8S+Dl95VDG*y8I9xgkc8`-XJrRtPF&i zB#Y)3IKoLX=9>WZre^g0ud(J|DM}hKppaG<#2@+ z=8KbxOg(|7TpA4lL9>oNVLh1ME0ByM=wu2FmV&Y>l(2IpS30CXLks(=1+^QpeYMUt zC_msk5~?fCR#J3pg{WH##oS#Ben1p6>Q?BljN1vA9;lQ#|?5(a3XJOHq! zP-$>uKU>;(M_jNB%Vyn5`H(7p5qWGqjZw=;zq^x=?wMEuPvS>VkCE<{wL`2i$QnyZpgvNRrwqqLRh_zSsUR?Ji8Z4NK+N7O z^omSbb_AxAlUu)h_}i?IJlsCqz4wsd-}(_DDZTguPuR^E1zttE;;bIz!sj+6Kg{0C z?K8n~9{G;L2iYU%O%N-Xgf}oGHg|}iY<{d7>R?!SpS)M`nZn-D356Ebh?#Fs!`1ss zS|bR=B73!e89jbhXZpT5IUgF}uvp&9_3(_16~e~KD3D~+`Ii+fra&je#Ze%Weo}d% zYBC_I!{_0H^Z_*FrOHZcOnj17P9>?m2dq4~55C!V_5Bwm=-KK&Lka!P>OALw(LEv+ z9|Z^Z^&-o_0HdJeT2IEoq*^{7|Hq*lGg3;F%sF4bmVR2wG3^|;G}7`jbSlLzKqdH^ zqKWV~L-O{I%f)7j9$CK>USTq}qZc=&oyM4dk+S=oEz++6#=iE0_so8?f z;A>UL?t*66M;+|FJcjSKag~nAB1gXcGrM0fV{d{)ZM=E^dDVki6L1mP`H59h~hIkGEKv7_kUi ztgf^ar3*5r=K2*+Q-rD+d!Kav1207)PuiIP;^f}x9cz~MaGpFTQbchznJW#uC^d4u zq?=VpgUAjKea_VQmVRINXL-|M{WXJLK@>;nNI1U41<^wn4W`=hz0;V#m6ZxTjX>L7 zJ~p8jaN;oiO6sBOFj;W$cjwxd4#c~It5sC_hg%qM9#h}IH{;P{GgJ{qG#P-zau6Ax zL29r4`Qt118qM;yx-%>3r&4SW{vEcxYds(r7USi?1moYC&ruj>SMg|k8aLxfe$&_> z5b@j%vTh>QZGnOIW_5zUcVl{46y0H6Hp`AlyY zVFj{s+uY|AyZe=H^$uOC_t?idoOl4bmC`ukna>9Rp*?11r!DcCwXc|gr(6p<+&9d9lbywo)msWJ#dWC#3P1@Bh%COSVq zym3X@r0nfbO`)M}J?k%vO-kZ>&bd=^zuHaZG|CFtyWj>lTfpXdxp|HcsFauTgWu5j z#LjTZ0i4s|{^S@w^r$mu$$`NZc4)w-I(Vjr#|a>Wu}L~Oc)N1DbyPQQ4GONcutcgQ zQZkhF2C_=7Wj!JY-6*Y&t1rXjK>FqdDPlvA=lYchr$5T||7|07?<89`VgSH*aM31m zWAFGa?%UBEoh;s-OGjt>=c4ff5zmpn*?2pfgx~*=mywWHFON7v(B*BMd+4gV2ZzLT zUK=bm)|sEA)F%RPcDw3an~we!UTZX)WiDwi@5x4E8?>h)3nnN+WRZzulEN$Z zT_3g=3q3FG-xw7_8o%dv+@T?6a||X!tCF5Ai%8yxub8<=2W;%PVny4{*7^zkKd5gB zZTG*RcpWZb6cF>NxRs)|^-=|1Xob}Q+-?8>pGl;HM;5>UJh!jMdt=911ZoS}4A>Ov z@<%}w5N8#omL2txl&P{7N4U8(0QTj{Z4i+hyk{=zs}VHW*xa0@*Wd?-;O_1Om*DR1?(PuW-QC?KSb*T} z?(f|Dz3;8{{>-nLHPhWyHC4NI@4kJwi8pn=N}{?Q*iAeUs7Zw7;_<%uGqx##zLoy+ z1E8v!ssFJ((A(oXOr#wKn*U7l2P62K|IR{oZVaL$c-QqWnV;DNBS;?iclv*%y!3H5 zqatg^SqNGHkLVo5MH#SPmnu zJGZA^vx4oQtp|4VB%UvaK?^m0NI70E&UW}8>Uw$(*ZY)WHjB@apR!1!t^sQxpgi^M zdjRNz774OqD|CJo`P^~sqDgnprYE>vt*YYgTdz4e_7zrJSGjHUs7M#*Xb*St{8-&m z0r%#`n$4c;`qG(@XqeTLK3(O}O9fjd0O)=a`!;sfN#;s;+m4aX8-g{zxoWmKuC&vA z2h<%Mhai3toPF3wW!e#f@^?>L+|?)$aODqx0f2#nkaPBs#l|H&gl}(N(qTqohwaov zpO>xErmb{5Es=I8Dp8ReBmyq+{!q{$RY*!3z}H+XV~w>J>0g+L3s1&<8Q)~vidWh#Ta&vOeePVem&^x z5?g~<2#SljU;qo4#P(ATayx9+@#t9lbi^49EL*vZmi3;t|W;+5PMH&k0nlnlfqBv#X1(0->dwy9fYDV?sQ9UhcxQ6}WpPw*CE0Wpd?jceKiizn6M`hJMr%i6@)w87yO3 z^$%mvdv6{cop@)wu9qWgliwppMu{!uPxjOIG6yZm_@O5LXCe0!w5u$K+)1MC9 zk{PxgO;o>PF6{IKEEQYT+oqNxK3dLysA#Jr0k*%|Kgvgm5#VpCZ~j{zIUaC>4lT_&ed-d2S&Tem@%+Q@ilqBbh2y08MR~_uHf?ya2uxv<=90UvbKV2rSI- z#>q+kbF)69iQJhYYn{tbs9blx;Fl8k1K5r2gH*A zK#eGCH=LzLDDZCu?G>J8T`XmbtgI8Z8lX)CGt*%bv+vKtz8@WIpl0sSfr752@6lNb zH4_!xvK;>Q%E}_*6*(kl)GwyL4epztrLVkPRll<83SgHaCJiqn5@QpIMTBH7w zxEKbGvSHPK(vf-8I=k+IVol$(4s_sq7YbTs5jC5j&yAq88GpE$a41emdgT=Qj{~*Z zUULCnxiM#te7&m~IgWFFaw+e2^Rs{?jS;^2cBd>)+Zp*pSs|s9RG*`s7vr}FjECv@ z+L%)!)ZM{1mQmN>3{W*nv3VAUf+qN%4%X@=v+so9*lkM&?J+F;?WfIaW8s#LqN*c4 zz!oc|=e?KG&ZoV=q8lcO1fJ^GzSJNTWj9;bWD=kD_w9mIXKCWu@rlld;el#>#@9~0 zi+_*P#(M>Ug~A_=x3+{o^4iXhDSyTVjkCQy^mF~XrCWko@!QcJxxz=1`r+O6cMEqB zBEGD7a&4Z?MtJ+Egdve)JN;v&=B0KVg2T%k)ZzML@GdPA{{$}lAI}L~tSfA~w``_8 zad1ZA-PA2YmoCl_?+=0zbI*6U-)ZCo zkO29v-ip6u@wSR9_;=Fx{9x(mxM7liA)tT*{_87?jmtvv@p;g$Ma^mo{mf>HBr|(PIXL?^vz)(4R`eCX1QiS#0aKvnzD7 z_go-Lvo*@g@k(O;I~jWvrWI|b37GObnk82gnoRqRwsYF3Wn@zp^N=Zs5aF#rZD#X z=5-sI@m#>??6dv)~R*|~NVzbVWPbUo8gY`l5n-|?7TP3z_yMi_i z6O%@Zo0Y~kh2}|?1L~)K5C64ig?DPVJ2~sv$|liz)DiE>{zM-U=?UhwWJVo`pn%th z+#DN)be9&`pcVx4!u1oAukf|yVu?k40p;{6LbQ>L)#F*j3>D13HVwaLQ~bjix`~dS z76L#Gt>W@3lV98EC?8QvXl=a*5kr2EtP0PO$K4;)P5$^pza z>rLeS4_=V3{g(;h25Zbj6yG~l zlAHX0pKQ~{sT1W-B9D{=1Pk)e{a?Pujqb1{{|$mPU^x(1IWeM?CNT z&i{Q&XJkD#vLqe%t?U23;y<(zBg&(H8KM9%z<)S4Al`_Fubtyba1c_i)IjV1&RsJw z6-Bu3KUA3i@?rjaasDbDPXdif7Vz)<{O6*{|HrbS&aKEwOiTpyeFGdE9I(rU{{K9i z@{V$$ASi!YUuzfOx>dbF9X9qt>t-MF zd1#Z6XhX8KTj(~*9Dfm^A)jF@aMIiUYtLbr=X1#<_Qudb-GNrX6wBw!sF889Yx5gF8W7K(DoqwCL2Y z^of%!*I645BlpXd`kZK*{;?^JSv{byAzMQesy&c;`OfH+h$h+!HO~iR;#nUxp5Xq!_1ClJqmNslYf3`_!gG% zM-yk+JI>>U|l^JV`ok|S?QvFhGNX7~RNhGuA1;%BUDZZ;O z>G&q5Q3nFtA|T3^zq>GVYg($n?C9VwR|qh}E=#L1^+Q)?P*tP{DH+&OdXp(G**n4! z4+O0LQOYP!ZdQZCYc-rv`BDrCd_)_xKK1F@(sBZ2_|f z^tV7`OJ*Z8Atud=j}@hV4nNgGQsz1frA}fkj9~D(X098^vv&Opw}91kb6hgmQ6He| zC%JWLzcbuzvQ(dgTcMSCGr&5QAwz|H!+hBux!&*fbxl;l0?jZ(2}DC~TvRr> z$@wgMh^Tl&#d$-)u%1752IUbvbrzMF=`BfeMaE3(jiH$x5KeINLL^OzBfAoqW^ds2 zL5Fdy&DX=hs%grj*zh{j{Ey;@q}5g#ys)97WoSL7L&d{<2$qooSIPK|o6RvXV*aO` z9K?!NBf{Z-t0pDX)O`4dn3}1CnzN%>WU>vEB!^2&@D~Wek@8QVtIUQ5;x5n^YE``r zw!Y;vn6#x@M9`YiK?CNH9Fr4p$T;f{2T9jPl7N+o4hlCRHD}+_m zl3kVg^zg8S+fW87)u;R0$oL$a!t7PXq@{vT$5pDs_9H&XEBfIU)GmEcU46Ls~yDQUvdo;OZml z_cEx=#xh98FL6|(EnXH!;eB+Dk;)~0ZZnP`{{aRI1(=hII+e8Jtk=Mr7qq`h}-3>Va*M`xv-y{eFs55FK+ zZsF+~W$Z$d-C>(^o;!AYAKn~An3~zg$|HFCrWp2Z5$iO0peDtpwg=@*)1XhMz2kSd zVhT_}AoC7E38!=2AT+VZkU}n%36a36PX)cXgY99Q*$>zbN9YI9Xw7HCBju3m0i!gZ`BhNSDlRc09h zENoEf3rQDI2&CgX2@qN!s8aSGz$om?HO@CfW;TkhuX3dV%MvU&!ol$_Gi)LyUa$*= zaEkG(ivh0>!)xDTpL1er!*Z=w+cn$A?!t_y>3_@u)90a*!pA|UXH&oe<~*_=BGAvc zXVvjQ8Ow(pDk#jV^<%l(oO2m&h;_!V4Y+&gdIgka#Y*#rf8XA7vMV~NfYbeoe>8YN zO-13Z*67RxOV_5|;R^w}REObmGXqsoS)&(#ipYB{<~Bv|`45&5DK4haj>hW8RP=TMX*LvFs;Se$_KMgIR>aJh}nP@CzY#= zo^+<*t@TQzaS`?wKvJZxFE$#0Cx=uHCM ze#-W@uv~G4x?C@LPTw^wW(z`IJmY*YvLWd(nt-Kw)cTN+)Km(AWgTQbZZ0sido@3i zV1i6p8<(3E+L|88GYQ(#q1nvQ`q!Q%np5SUYuafb^XT)s^uy>xG?4EAeL!xB1{?pi z;<0X1@)n3QvB5Ix|B`1ieGfg1fnS6_+Yi`dQlk4|a222Rnsp2D8MIb`fEr_G;frA9 z%wc)YoQ8kTy~u8LN{y}yq~nkzZK17Z;HZ(-s*kevIxQ<~P9c;*faCf_%WLnIeiMXA z4hwuQEEbd}-t}D+-5~UX^{<1S7j>x*ML0fC6ssB^FQz3}{A*9F>eL)oKM=%rxJXqO zFE8-x-ng2%Ct%6EImwXzL`xwixP)H4FFdCOTDg+=-dDW#H(iTrwN=XHT9`GdHyz*~ zv{;uXls^8-AmU%Z$UE-3g_!Kj)H1aw28Z^a-?3EIzRnt*gOxCWAx11GG!@k_wmjHB zqVWssd(j zVFy7x(&~I={{vCXwE-ciC6chr(B%vjQYtn<)|!GN(z1d5q4eb+>@J65Y>oI!iW$Q~ zbJ1G|K0~Xy9agBs6b++uJZd|GZrMPfrrGqUKW5%Pfd_9g=I3J+_I>izA!u{?;Lhj| zW8Ndf#cUM4Ky%Isg+z=Q*%b{Ad-Auy2AMtWYM8Y@MORCn^>~_24ccc8NIW}&5Be-~ zQZpSrl-kMD#8DW{;4*cVHinSIgJ0lkk;AePxKxnckT5OXnQNSXT_XcyiJg6f*(uGa zY}*(K<#d$zDXQk#Hx;WbOZx^a8;#O=;r=1o{rGeI`*dL^XjB&Sf|3wMGj7nwih&yM zo1Mx64kgs8PbQox4~i-XMm`M*p=@zM{PQ_NAZy+VQy0DT)P5=?Zl3U2|2z~)46^}S zY@#rv0|^iJ{!f~n&!;hSe+F|dx%nYeH_noTxB=F~lywE!8$Z3^8AkceLv z9O1w6M17zEW=ClC*IX~~kb1+;|1IXQa*tL90{?FQ87|CZ5apiUi)~ZkB`R+K=1NIg zZJB2&*-Why>o=^GOG7uzh<2ETIh-h=Gd0B!F6{%?aaJ5A$xrK;pu{pEB#yu2q2Z?n zNx)a5SmJuV(|2&=HapygL}7ZQQ(r9TXc((0>FIivsfcDj=7@8!h6QmbAzDrpvPOTnSesRP#58kO#i{(1;}Y>*^sO$b;ZD|vw*p?q%X};( zta#9DNMhN1naPTbHP|&Q?sJn!*EMDOVH%(6Q=8}b2LH6JTSo0dPf+#UG|p;;S2%YS&pQHd>EWL zuA*)2`_gY92)5jgbX5ylU)k(Iap`N8<0&Zpvmb@w~W?RuOCW(eVYM zoY@hC$!V~i>6ILo8$d2VQc%q>2DA(r6!$7hJRaFDgx@qt93H3PYZ)pTwwE_+u8mi@d?n@Nij?S+i5H!8veRbjd(v6O~j^I<}2!;&W4&IhEK9 zy^|=W?})_Qm7C4AB*bn=5mwm-EgEXewq_lM%VIAROfvhKLl7;>L3YWj{PP~eGXBPV zz&%3Gpqo^45iEb9g_CtU%vYf;53}QZY+PWP=ZamZol5TffstJ-wUSsf&p33U#Yf~oI`ZSw~&k}#k()e$8_ zR=GpzR(_&~an6=>EGl#X9|Kkd|tygdu0M)jA7U2GW?S z2M_FyGDIhQ1gK@t!7La4r8z=)u+ZtLX?Mtd?{EF`wbty)=0U6$-v39~8h-evQzS>?aEgH?(PHD`6mQZKzM^@ru*(YUX_?sye2`=0476 zw?xyynlopB60#MlA`g)-Eo!cdv>aONUj3&i`xA_LFw5t#Z`r)O9@8Q!@kdim+?tj1 zgC@3t9xsZ3xdh1tA5D8LT65<8OA3t(|40 z<>E3m_>zQZn3CCb={hAkZgFn}^;F4=CO@`VW*CwSvt6rO#UKcV(M%s8w%nN+nP z{FKIZwAJDK2cx5TF$xl`=~cE(-ll%3QJV zzR0R8Ac;A80+mFC=A zL=0&0>r?Kgl3dDD-QV%iFSX3UQscyL!Jee19r}*$Fm`T*0ZnH4z!lFply1eP2rO4B zBr?0_VqS+-SdvEi4C8ASA~`SNcO0wG0)U%x8fRCO4f-`u!$xxfq9AO7*^%(PaGhX> z?t{&6W((wE{@|(*u3JFF5WAF;vc0jnO(!mZYGn-t_Ha^S8?8zwf%_sbap_+n!2}(9 zmf?@!rVt}J&R}mXT70&Gj)P<7fk7o=-K9yqOV(}g|~`qRA`fNmv@Ix_WWlW0EA}iDY)X? zG294h+R>G)sa-nhH5v+u%yEX~V%h3onS9YKBeQon6h)*(8_5(vIf!{^1Ubqkp->=5 zsw$5o`ju>pu@7th-AJ{%F@BP<0>3|>+qJ>E2w~3c+kt?LTot(G1J8RsF@}^b51xoQ z=}%H}TbNw>FQ9dkrlHM&sE&9TzEUR=XDTCYQ^pM2ZI=AM#Ba`-$!ZZo8jIJ4qRI^;U<&)vu{^4& zpSy3@{Y{Yd5>eU~Mr>tBNaHu$@dDqGR%dA)@-fO(;P%z4`?0WAagyBv?*i?Aer;#yfJ`kfsSbf7<85TKn`lbl)31&m?O-w znBbdqPdW1SsTgxT@2!w@JCdM99tFZ`yppOH(sd(Rb)ed-6NP)6$y5axrSOV0k9rz7 z?#ws*1vU4XrYMxT^|8xh1g{gkrp<(T*;KwZVjAOP&5~Da=8z25rMB&B{_%;3%sM~` zkQLh*i1W^*3eo7SbtH1Y!@-gjx19V8jD->EVu;2|ixwcGcg|c^j>U8x`c2@5`wu71 zJ79G|VATiYh!+m^u@P&M1=5H;$q-8%bP|ekd(MB6?iOAYxlJSCUiy zGTzfvCmhc2$2xxM!k%4CN?ZXY&UR`+X{4hjjH2@P9n(kEPt^&Fkc`U#wHUB=T;)Hs z9vVwO5E3lygOtot0I+QXLi2K?^`iI#`~k4HOy1bwC}l#K6cK=LCg0%#JA&dNL2x<= zjDNgnvOIS*?AaE6WjwCWLo@UYqck5%!1*LxAM0Oo|STA_Atk zDN0CU{wqDZ`esZi1I{SqjGKl*Hs|A-7``$>dQ*nlFZeMY(F%EH}4 zLuNDXv!vdYF;1JlqO3-)Xk9ce_M{b*I9l~GlZdschf-^kr)~(ZRWnjeg@12Wt5k=b z{nA?Nxc?DEK+u?C6{azW!z*!4#4_zmTG?F$F z_jtKDUZs(cfM!8M0bm?%ce!$-+QCauk5WzJmr3BC5R_spyy~xug~HGPgB%8(HMIjy z=Zkbubu+|ZwtqYA7exYNRVDcb8{nAZYE1IX@*Y+_X*OU@n#)*L&1YF%Kg}%2+~X+%c0fEFcQ|J z{N%*gmD05rSWV`2S;ksmzra@(LdU7-4-vnprlxXXS>q^&5Dp|?baxdTNuhqy3s-ro zQO++>4q4U-XgxxAYwA)o!IPr(hc@6&Z!7DkbL{p+xoFc5&Bs;cq7r0($c*czRu^ry zVW?l#zf!L!1=}1j;g0_1aQdA%u&2oAIGTc(adq#)WxJa(sA!nV9`sy1RgbJ;+w=IC zmwne;OYfZ$?s_H~rLG&8lAWq02cKZHJ~sr~y-^m^>9wMXF{4D`(IoKnGcl(Yf@ZQY^Jbf$^v7 zatB*l^mY0ki-m#Rj(vTOaxG(hWY!DguYP5820P^VX9s1>bX^~=&f}w}KA0h3dO{we zCAHc+s~c$^zM`n<9MaT_kXr|G>|*yf$90{aVwk7tRAbi{`gJfG+~+P3A;IE-M)hw^ zi5wYJ7*76hQAf@N43u6gwD=TKd0ht1fUpQzcOv?cWS}y0Y*>*=!mnOY!)H5Lg1Z}M z9B5Vq6GOr89tiv&>I>N;ipUT?yfM1aEEtz_6H>v)BHe>FNfoffmc+JPztuLJa9k!) z@W6(l35;jTQD~r5%#fHwDiCr_GNE`gxKn6@>f+kh6?I1O!6maP_8F)#O;N0@jdtKz zuf6+1>l-yP=0HM@7*^R}Ag7NxKC-d+1; z{&L$|sbkmu@`q{WZBxF}hSRsOe%kzBF0;|acEy9dn^+7SZ>GBYK2J58t=x~+`rUqt z2~{=Ow-MzkR!urm%|9DD%V3+;92BP;`Nn!omCJSLEW*x5c42g>++Xb2J|LR9gIiATC0r5A9YpbMTF+V(v!& zK>Pp)A&+$XE;|rev>6y4!=s&mtW3ipyJYI|6sa>ZUJXcOw;v8g8&d7D zbgCQ2K5-+5TCk?e!;ht8gMxIj_!r=Ukv2ldU-eBFBaiGohyRZ|7Nh$Cr$Ddjs&{V$ z@iAw<9dlhf+|a)to`>VO4#DUar-KU7+veGI9-#y8oZr!%^c!))*F?+G>3Da*o6;*> zLk<9!zVr6OXrl<Xp zHOS)XM(ZKAIe_E!`MyseC6eV9AdgQi4xlo70~IJbBVBooHR z2V3iAS!1X>r-Io9YmVzJCyIZe$hGIL!OzzayO(Ot)V{-mO(oRqN8t+U6#Z$)L<1}D zrJ1-1p14F?A2^Bt286=9)eG(;@kHVXP1kCU_CVU5w@S$*Yc{sXP$aRIRjZ^-35xh|KRsckv%$QOBP_BX*$s5AA+$EOY%=ZZG+b(l5hvY<*dgoYD|Cgw5 ztg*v)91;Agc(jF*yd>n5M=g-#EGY5;<`+!a=7B#>j0O%dIJ5}b4wzv@4MuEPtg?SP zkeFcFOXC;#iOor-a!&`Eux2%W8)P_Oik;~y z^DV4YeGg=9GH$BQc8IBjwPkmQFQg+yiqoNa{MyA9ZX`u0xRl=@UHuh=IBMH=;yzjDCR+i7gI& zk8>!%<^#(5dpI5*TzI~T&U}SsJR@aI*HSq3rlHaXDX+5q4|h%D$-(NLLZeRl`E2d-&UkH}78zT%we zOcnQy=-RrnmmrdUN+|E8a~7O3a-5+*ONZPDK(z$X2q{=5H*Y8e0y5)CA_f0S|A5V! z7AmSnDF~_ODXK9&e(YfS4PVCa9Vs32Tle(w3?&3!MA`up1dRn~l!>Xp`h>K5M>FZ% z-LRK#)GQ#xz!J*}4`eMv2P*7+$Son;GVJ{oEJ?M!hH z;{=Tve=cqwoWaP%QNj$8^zb&+&Z`H0xPlVLnZ5@k&XJ92Bd4MvR55KCylF*j|zQ3u&2Vx1MHwMeCJ{)W!Db?@L1^Ux=s*F$*7 z((ri;vsM9Wh`3of%(7pLQIL3HJ?M?59Z^k}YN=Vk$ zn&NhV)|-Rws*?nT6O@(vnHG7%_-)qE&*ie|bO=-g|Qx{JO{ve6E(tlXfCl*px{~QuwOi@2oZvD+LXe(H%1qf?oBKcg9wfxxK>+riQ8Y(go%mW^hWM? zM5pYFa`I!deYeODoiQxI(M86Vv7q=Fe~q1@-(@7x#Qj!iCk*NC(n4q`3Bm<8%i-Zj zbnKV5l&F${VsSm&ZTlJ@%^u46EbNo zk3X-UxUk$|T{MtL`MqlA$ewI~s#A<~!j{F>i7P36)r!m%<4_;J5x$CHD%D2od;N#) z9&(RA%Zwtt?oi!qgwu}lQd9vBaB}eOI`Pa%7cZMWw^d3hzG{dXB103#ih^A_btg=P3doc7)j^_{ zc5k^X7Ikv@QM$DmQThTTk8+0wHQpa!QO_D$VZdzW^rDMDe~*85taUn z$3Jbi^#l$Zk7|9(=w9G*g99Shi zg5QJ~mj3ZTHR-HR;HG~B!0gg~IO%Y`>5>mBI!Cww7TG>PRcIwRfI3qHNwL? zoG%jJi$2P3Hv($m;;YtJOhn6zVJc@-;YTS%z!^Kigp6aTjWa?rEub3J0ISO5}USv3P1d{zz_{g7f|?Rlu8)&Vkaodh)gHVkceMS_@N$iTsA z>01CIEJc&fyfi0k7$$0%~>4(^K;W^$!y-+ZJ-Cd*V75N`*Hpae$1w4 z=X51!K11| zjZ6np=-!91f8xn}^trqdyWWdRY$fQlc&{ym<$1eL7&rc!_VM@}@P&9q#N{v_j9J-s zJ*tL6BIsjiA6R5n;JxeXj_Y}TuWDzSzT(WkI(`t3#{J=VM%aSG=5A27oWY*6*KEt1 z`>~rXLFE0OQJSm6Y`3$kfmPdh$ulU9u;v_v*t%}p$p7)m=5s*xzA%vzr;x+HcYA_Sicba zJ{+Ocrxd=2#pbrnbbNfr$hx*v$-ms}R09#byoZ;U%oOEEG?6xLvtk zGR-7c63ldeJZ`jZKhKsU3jqDLChv~u(*>RmG_V=;+{XG0$wLwFfhK>0fzX8Y`guLw zBK0I3h5Bcnqtk8zKR!I*CJ1llgI*ucR4_fSK9a|gH~nVGoy?|n-%qW!-WGe_LcDDK z{w<5uj^K@;x}5%fc~&4?uB&ibv#aqwt2QNgnL`ai=5nr`nE9P76lG9*%GTw=P$num zVyWiBsS39*=}Rs-EL8{ED^d{KY>xR^$1|Fsa$Ley@c!haT-j98lZW2nFULn_yRkp| z9;uSG>aO6wOk2O-vcYO$(YpFP_J^aH>0*_Z%whoj!6lX#<6kiKqg0=5743M_^Uy@R z@1k~HR$YYk+Qn9a<#Z)Vix*)GUxA`{{JLH4=Rp|6?g7ccLIygN2-X&peJC7H z^|Cv2Lk3kTLykhxF1te&k{S}thKF!?vmw%<;0gW>7tDzXY<27cJ2?1p){Y#&XLS6Z zZ;v;UH8|g~@C|U-g@Np8O>eAJ2zanu;xM8!P<|$ZS0R|@rkqfNqrUm7!z@x(*`adx zh!;#Q1`?P^I|oQSnuSA&FTHu>uX$Y&S#hWTBAmsg73}q zpn#|Sk$eTm9((W8e*(bl=i5r0z-tj~gVWKB*Lmlp12u8`4%MEA-0(pEtgpqETAH5Q zVsp~rw3Y!2X+c2HpEaA>oX1tSw5N+E)6$t7M1K39Av&%Lz5SE4<}CR)a_*$?|&VeV*Nu-)iW2UNHH2kDtkGg|~tnK=gM_SxpzO-~9I1dt4<`ui;LhnDAw` z*4^P$rUv2HLzLylq_;0PH+?2u+(x?4Ifz=%a({Rhlp`zrcxxgS z=f`xpSa7*K_elFVq$PC-(`{WdzeAiFIePi^>vOkDM)Xy|d#4aX+omy-S;wA4tFkOr zm15*{p%hAAm8Pn+-;$$hUs(+jrD?3hlF!IiHwhZ1eqP$V8jovBej3fbk+>GcrDdKK zb#;yxbQs6)wXrd=O#4HewqyHUMBAZwRD20I+Oo#&sd6c64W|dY^gCFK zvjB>gXcUL|W6(&u~Xoq3**Y}JrVpV*m9W%Rl{!R~r0 z^*OAHV(OBOwueGEM-IoS)~w(C^df3MYw=R}I(M4_Jzo9ugz358Zies7o6OVb^j?AR z4$~=EMAWY5=1*Mj)1->PO^koSMmxV=yV3A}Y&?(qQoxZs8*aIp^Y6=eT(;-kgWGK) zRvb1XNTzYr_W-%~(`gVUEF6nb)A<2&#qVX`ZpOo+XMB`rhC^05?X>}Sf?E=2z;(j+ z-i!?BinIGBb7my5hcCi>nwpf;+R-t`!xi828En}OxtaY zhnVLzh-ak;pBi10zP@Mg&@XamrX2!lQ$LQE$f<{e3hQRih<5MsVbGdlX~N;SfStQnVnh*`Lk8GTl<| zNTKLPc4&qrU-YhACMoO?!kGP3cTsG0`FPk0<5e9S6E?Tqmemz{UhxxE+vlY=%kUU1JS%3x^F94v(N8t+n#sTW(k;^_7BzDZzBrye-b}>iE$KkwXn9)nUPKV63rxB2$E?0j!})vGk;vv2=nh$op+2;>;swSG5V zX}aE~G7h5mj~jM*9;Z)soeqvZL|k1ruUTKdySVU&4Ax<=s5Xs*SmYAO3P*nt`0uhP zb)VRlAg5&;+Oel}p4;~nyj(?3!aax#`f=9eT&HaTDV>6p6ZS+z>>K0?>) zzj!{jdY|NNP6b7oX53FkzaC^rH=0Izi98%X|EBppU%+eB{QPeFaI&$%&FMV!&&G9C z_;1HW=y*W+(nW066>3E5rp2U77=kFhs}*=&X! zm9)!=O;z1}#b(9}tGqDxdO7yo)%6D-V>MczL-z$~yB=s`cS!fj^tBUu=ywF8dk^JG z0W?go&Kw$@rNLz>a@v>xa(;hen(SQ&wgen+%5AxT6Dl;}9nUG}u1%^^Z@mn?T+R8XxQIQCP-3fxc`H2Zd5@&$&m6x`E9<~jw% z6pK{xw7-%b4<)tt9Q?zOPjoh*O;`;}eC+w-DBVD1%W6|Iy|uOzSB&ePJ%F0C*}!hJ zD04%SUQOO)qBL6_o)xPi^?80x8vZDIF2>R1Y}gRbBiO`Il4{GbKo;I(&ZM)9LT23eOMlMO`3w^uSVGq#px*BhV=hmXg3K_2>XLe?4 zznnkRTBmaOdUKKmemoTv+cCtT5*A)eOxARtsju-7au|($hMm96D-g7A_j@5#yp~7z zc&#HJO=V8{IG-W9WynbRygjU$FW!I(SdXCfBJ!%H;1X-C8PCk)r+vQuS>x zT#~GLekUm*wPsGxnfw=PZ&U(fN?Az%i4LE$83LT#2so<(%gi&V^P>V@ReCfy{FY@F znq(b^Oq`p(cbl4(g7uZ=>&A<-uM_Pc)%ZmMP=wVV7kDoF7^-e>JQ8@QE!CQ@y5P7f zegS3p@)Z2nbKU7rqGwV?R;s#ETxl-fnH~yo;A*5WDw)GY(5th`-iO4PzXwONGQ)F4 z%dpA|!&G4Qoi3D_n1cDQif7o*OF8%d43|@HCJ43Q#9|_dZLERo{B0M66ClA6S`vUp z1OpyHf|0VUDOSH5S!7NmEsFycDQVSmiwDtb-uK%%fqKedF0~P;_=b~D0OmSxBl90S zO012ia!RvYH=T5)E5VQF_*u;wk2kH9{0|L+|7VliMCiSP8+~9BjuF%<`=oesO%M*w zqzSYQNgoPHQ)ef}97y}8rFnxrryog`GN~WK7d2pj@t;99tYQXR2z9cNxjAqDCwCL# zqtXCZlXJaegrNv4(fN2-GI^LBbK=W|?mT@lsrpdLsx5;n+8J(4IwzzXqwl%JbW1Lw zQHt0ulM&RntlUbtjJe_a;Cztlc7X~9rN44pLc7}?++TmAQVlii?pG7Oe0EyJV~MaZ zUpAR)pO+PUXBXdJOM3;b>$-VdA{A0_{PWz;%L)8J`H&3N)MAa-rTtg}wDnDo+_v|} zG#yjt<CcJ!y;1M;{lb9J=l;2d4Vl#Q z@%sK;a3#NL(sj1~E-hd;^G;2a#eY&V-n#AV*6Zs7O7t|;CvWg-$yRagFzLm6Zn%Q2 zD@4?TULUUBVw`%q`bYLWAI@*)K2-d^di^Rmd>mf)#QZi5S8UX(;o0t9(j3X4suqd0 zMJ0?!-j?vNPe;VwAA(i3jUn@nj@B4> z`e+9Vdb#jUj0h_Tr)r1^)~WeJ0ES8=qIXPG<~<>V)(FB_7459I<7fSqtRbPX9&2!< zM;%lCXp{N~ScNyJgwtRt@Bj-qh6j9M+DAKyLGw{0zQcssFoXaY_ZXU;ntKK=SsWn}+@_nh_OoQr<;2a#;fXIurSsf5ay4m>eRtVtvkiPO z_plK|hK?Ar=l;9CvS`V3(`R(*>>~fryl}tkOMqT#?UZI&ib)yx2_ld z@!ZklMpZ0#+iR!ociHA^pZ@j(_dddtJ#fz>&pb6_{6^zqS;@t4MO$vW>AnZ;F=3PO z<2D%c+Tz#ayTU@_;XL_MzWi?=KK{_Hw%d#^=~N2_`yKRwNACaEHGjU5^Q^YjIq`O~ zJd6>ehaLKn12)`b{P>N>`IRddzxrBb&n{g$Pdt3zD}Hx1Pkh_;cb)R36GSU~`%UtJ zoP8$l)~$P2UT4(5toE>LtxzAwU#@cYt!t4#HodIif@PO*#I_cCb}^<7={?WHu6iR6+zXd7I~`F1&O@d1wlHLN z!GiwnV8DyH-K8}?KqmJ{5?;=zn|3)Sij~#i@Uw-_(pMK+m zAKGWa#_MnJu(fr zcp{?y1Nv^W!xs9wd6rKlNY_HX{mi0d8NT?zlz2etr6{({J>8BVvL>KD@ukGI#d;ryhTr zGhMjgmB$~Ii9Wfyiap~?6N)MK4iiAN$uJ~G@8o|P1hVVb$& zK01kc&U?_HxtC(wBp0tTxonVg7(_tGplJ-&78us_H**rfAV-bXDUPhzpt!o-w97MQ zH+G=b%wQ5Z4o<^m=O`V-n-NU{1CtP|g26Ei1StXrE>lhcu-hhvIOf(EK&)fH6iSvC zt&!l!B+M#m$E{_mG%y#?z)3O|1PC@Qf_S8t)*K!qYi3I_!-KO@@s`uPAPHJwb+HkX z%dDEUTM`Z|cnM%Qc}B#^qJp{12?n`g>Vr>+1RHL$ zel!Bp*R8+ddLres)1OaeGx`pxbLY-y|M-jzHyLk)MF3=~xa*!f^z7XeZADwWZ?i64 z#Iooev;HWN^0}v< zkM3Jf-Cg?DvQHj;>bL*n+{Yh%N*J^Gn1>f0Tf|!yXegjjkqwN=L`J5h0}tIhu@4f* zZ#Z^~Z8sGu=g(b`!ke<#zxn+yZ@1G{Oi?t24nY<&Oj|d2$hv~_Yl~kq)zWnANk<-Z z#D1Nsl{LfuG-|yOTWz6H zc5k7b-b)JpJRY!51`m3a;c0E3CPUPh`JpIx=`>>#W%rggNtZLw@)%fAG%!$yFeEFf z%o>BLjxu(TMF8tBtCog#(UX4bwwF1=^>_N)IQ1|RnI<&8T?}Mls1ITHWClYeC2SRR z$|7PQ*@z*H`yE4}jBY&30pnX2ashiE_sdll4rO~hOX#}gFD zymemTVK>q?p$_bD<#Q6wz#LAj-OplDBIS^GlTS0-s3kUF%1o6#VP=yB44%UYav&$z zFrf4haDmxsVy3)pQ#rEn+kqbNMd_%^nj|#hi_gyz_hϖL-Mq@ZfI?%nv)JK?Rq zFaafK--Gs8e}WtxNg$R-Hr{MPO2_gH9=h)AS#x>HZFk%<6cMN=2dTbq{Svkyc=oJ0 z`n7C#-u%x8?|GE5sg$QVFUhwIzg(&Bi5%^9;O@Wr$?xBKYpFq79FK{8 z%WXH^d5`Uhc`lTWv_C#UC-UAubpO=ZGv~~o`|`Zm3+B$AziipE>YjLJhKN>rt&D-9 zYu7Fx|J*SRwPgk2;3M~Yb+_ckK9+Q!U1W0t!G_Q(-Imt zRhwDOgVMz}RoJZ6Gj0W-O*ODxMXs6Wa}YvkYu z2ZfTHu(=h&84PthZewWf#R&|pRA+5L>Kjn90P=)Ejw2!`pFJR(k{x_JC+ykR%+kJ9 z#?cY`PHmDB3xA8}PpbZ84ub?i3PV63F}Mz306TDvwH-LmfS#d{c5uWL!@~rE>0N`3 z>|s|cGm5X|k=oVD(J*aX-XgA|7TRC+$~WK{!p}gb^f3yE ze?d8vGBW7!V<+|-(ATm3zVWw8;Jv^2g3FAYj(GZo3*?7r{(1Y9f8H*i(lKe#%dZAA zdqZm?A`)+AODSq^I!E^NCjHmx7vYBb!I7h0y?g4EyxxfR)G4Fj3iHAR#_Pf^|E>J~ zV~!TR{0fE089HpxSI_$VcfR&NX381Qyzs;C{QQ?c`t6a&9rCf0kL=O2XQ-{Bf3wHL zBEixorD2U;@JKwPjFmYfkwM&cZt#$S(O>LY4mY6oRHFISmx5d>p_(Raynd>Om@4Us z)^NOh>DxE{?Y2w)_fId)oYSO1gKDZ2VF-=^TNxOl-&0GCbzlcDptj7_wpeY4m|MLW$y&+r{47@q(rT>a)Q2#1 z4|l|zM7@$j6nOX*WvXuZ$wShe?T4;Z1v`t#EVkH-Nz#)6RFk|L#Gd3rp*Ot6~@dcM& zcjZmrIq&q%x89i0zi{kRXH6Cot&A!j;flq&Eix94diUwoxpQYCFGZ2>rw2K67jSf@ z^!3(HfuWri>&cnV&-wfbr?vk9q7Kn|>yNT6i-5Xz>vG^>d;j_Of9FXi{bO=9$hqUj zyT#)0b~|qsK99qiNvKH3Y^vjLrozBpL?$nV7DUwis&-06SNSxgWIYp`3{_IW&h?b| z^Eqj1CR<@6bPrZh_VKB>GiHAB@YUML`XHz<(xS4!2e)0D{-1u4dobdt0iQ;ZwRkU4q33NQ#l4?Stlp>p*j+@)%JRfkESR@A*mckWiS z36w5Def#&7g+619Zr!?As%H$x%KFl`jBliLlz7N|@(aiR?6>Fe?z^UWa~6Ez#M2*p zXd3ZY7023}`8be;uZcmHO{x*dPPX)|rCe7o>P_`Uqm?Ut#fp~;LEU?HOX^0U=x^(F>bogM?;D#YIh92EmH3DnP%h3@G#%6X*{WCB^p$LN0^P> zI~c{Lo0OlE$j+D4DIA}N0B}sY7{Br%Uy`1mduSI!x&|21HODf*Y_wQ8X?OmE@piDH z746V!YcsQsTuoxOQmaYkSYKonV|`(h6f?{uxRX`SeR8#$#{GFbjE7ICa`rLjAuIuS zH%e?VH&X*}9eAb%1*cGA2<10q)SQjxkSTbG7K4U(%o~a^--l)E1?#V>tp z**V|*@g-OPyi?~+5Fx%nZi74R)UiN>LTpS&6dJQXz8LZ?DpiLJb-+@WaO+`o0~v8Z>m^fPn*?qemZj{By^jW@`jBSOy2v zMQ5&#My@wvi)}V}?4k6Qj=8hvPoMVO@R36w_$RzUyGPF+yY023r8@w^7GFU#YUWgs zu%@*YvMVa1f@~=_Tu_)%$56A`HT)_(pW`)7v|z#&+T4Ul*k3NvSlG<3{22ynwI_u{ zRtP12TlTCleOq|Eg(u&OsGVoH0{Gi|WB{kwlY&i`MZr@{$2RtQ*gY$zyxE(z3%YtO zt0&m=mBS!hZSOHrvXkG?c3_8gzL|qu#nbzssXfSXl-YG!U8-9a-rB=(+Y`C8Bi0d- z8fV9xjzS4G>RK0nU8?T_IL|QP^Bxq3G` zjvY!1D{OI>t#VxhHi*60oUQJ2I4<~HD;x`693ltjq&c*ZTX8?Pomm z!etj_ddM+F55ExuJJf$BpEZT46q6n?PD-;{Mha#yt-&f?>@cK zvk)N&^h(Rn5rbLD&4>*p$f|kj-@DbbVIxNm|Mj2G>(!@ssOGDFg9Vtmx^Zc*MuA_= z!Ww|c5gmNg{(O)_L=WEc=&%t(R<4BaknVTT9^HC$lYznjSv6Q_D`GlPGuK{8b)Bs7 z(5|sB6f_AAPQd}SqPU?$fO`sRIS`vlfv6#Jt51#3=cL|Cs=bqalCXr0>Db(A1J_Ke z9m6Z3qKL|Py4Z1`M#Z1(wmQ8UWJ;nJk!k~$84p*rS0nllDNZ1+zyuJNbwGQ@vyGli z#9-Pa*P|8~f~`aumob^ckZjGY!MNLghZ2p+S&!Ya126m04l!0km7ol4d?$y9C=8jx zk!Kgf4GQo*&%~~JBM;mQmppQnI1&?n5$7J_?RYAH3JgM6!XPB)Fqrcn3Io&Cvwy(T zP}jI8F(gHB=RQJE-{)a~IBx+h`3yvXR8W|>{+^$Cai*-E#X}O0*LZ|zkD%EmXQ7Q5 zhv}+us=`B($l4w@%hCdk$u6khWAH4Tbk#g9G zAtH(`i7jfL1@m5h_Nf^n(-*(~sos5hiwv_~uwF>e5d(^6MR@jT*}V>UfA2oS#q4mfNtV4-9;lqL*wpDI!G4?8z&3{f+w>QZG%X|k-3@UjvtyNSpHZTy)UPn#l zAjhStgEYh?C&zeV8p65?o9b{E9f=gqQ99rwTIW*Q#2BQ>8A^q$7VIefi|HLXF>`qPMXCyHY;{=zTLegjx4lG3pYR~h6;lL&s9S|}hjlx5K zCs>wD__>{|`JhP?6vAu}#37Wk1%uf^Vxx#H8AbTcnVN?0|VjW)zUiJ6>Y6z|CB zV9v8*-8qflrQ50j4wUH3m^Po8n+^Dj-CI-Oq`K62=$TW%C`BBQ9Ql*C-b60sH` zSxeT^Eh~7t>R}fhDfS|c%-Sys6h)KOcnlcA4A3)n=1irlWi^-TJ{g$}*qG_`scAz6 zfmfn=^k_G2P{dRnQ%uHiTGFf$~0BT z??CqX001BWNklGWBl*f^kW#wJ9m8HbmV2?8)xy;om6rGlK<7^v@qNi!i z5QxPh-4ak%d-bNlnY_4~g}sTVH?C!}h+F#~vR9{0P-nijuE`>(z7CM7lRL6OTQ0!&SF7b8Lp9$}`l1JjFpr z?Z;ETIAhi`PtM?14nAtXRAuY>X4O}9$bw_kVtFnJlcH7#OVC>NA`EnvQ0Xf(OoZLlcB0ljnY;y~DkmGJ2j(X)lSWTKaS2jh{3Q6Geph51%}pd zPp^K_hTP2LJricWQ9dxO6RFt+58E~!PwvF-WInPFi5%|K1leqSk1*SLlX?WWrtmoX z?eZufvH?HVd7XSdMEZ;SCIQT0bHrKtP&%5Q!x7I@0BgPkcY9;Zi?S^3 zX=66Ofs?as7Jm>Uo)E|o3m_k#nF+-)m&FAORKLSV4c%kkU2prxT|CvTH{A8iQ!_sG zrGMLg=dFA6>QQA_{OXdKFU+1i>At^Re#4@Luk5k!E=L}B2npTuNaK!q%O*xfUdGW@ zSs|=?%guICh|>C%76%=%|Ns8vk|m2@<0-y-`Va5A_1@z?apcCEP3YXEvm;u$lIG7{ z@cgqc-f;CTx7={&@@31v{=F}6%d$K?<*^&DnKWeBp#2ZoYtMb(zu{&Z#97xBD^}e9 z&xdcg`WAsBE+R3Hz;SJb5Cf)0-~NwtTfY9`^QWD>(WV<@7;v+&fU;=iij`me^qEJW zc=-Ag#`f#q=b`%^yZ-81!~a3Fnf0RDPMdZhU1~OdVL&(_MNm=dMc6mTbap6T%rVrAfpQplNN5ZqY@zqUFgW7nx&wB|+1!Zp;3@2lNHPaG zcH>H+li>Ne@jePqED_`7$+JFGyrWl9g72yyPvLD9-jV94)Y2sg_6v%6emPoIe;bDy zF-ZH=jMD4eTVP;pLMJS~J|`k%=$M01q_%^+j3Ce(@&z0o-@b+LgCQE;`vEe7P2Tx8 zs&xpDc-;%*%GeObIZ}+kb)6W<)~;V2S4- zJcT!yg*ud=S76lYi4A-`&s@aTbq|9W?^X>Jue4Zd`IaX`Uj~g|YQo8@%QQ4?tbr3Y z)}!!7{mQZ!wS-Ui0yu_P>k?tMc+*;@afscZ1l)dyZD>qPJj#oyU87{fj`p~ewagxu zEEufSQ$4Qc9**ioFvcllcm$f!t1+bD$n~lBThk-!>sa#+5^b{1Z*gQI0oV=KrAhqn!xW70>Pbg*(G#W|P zSDD1ABH3-Poj!f)iDt^>%a%X!*i$oKcu9|Nn#H~G`kRkDF!iA+k4>NUtRCSI(aB#t zzDt)bIdg!q@-fgpHQSnLAZp!2BgYKy)Y*Q252qb=-FBl*$3tjjokm7buvm6PhaLSv zooVGt|Kf~U)2BTfM>vCq4E*4S_G#qa0KzVS;g+aI+`b3x-lL~cQi$k-AKF{j@v+1! zzgo3ore_lWVIpSZq$U{N3NV5?E7S*MV>SwICRj;e#+re(mkv%fP9>S-MOH@&t3ZVs z*b8jhhB4qHCw^7c%wRWYKb5b+?aZ`!JUk{+Xb6;O!I-w8HRtxQKuAlO4Jij?`-_Kb z!iyI&Ep-k9mW7=91_mCf+zzXU0gD51e~FmQMg}Fwl%s{HF>?1J zWHp@ZiLF;hp2P9xU&~ImC<#qT#S_LuV#aF_Pl;u7v1s%<E@JbnzrL%j8AdZnF7= z-(2}q{id~MG7EGTX?sNha(a23FB5IDte7y>y(glNo^<5@TzFQWzP;OLl4N#3a!m&x zx&H~DJX&&qLg&rn)u6`wTy!Skw|{Wjw}0@p=5fv@TWt89^Gt#dCaLMvFmsK$70Y}O>4DS-zQvbz?OvMv59j9M@Tj7W@#lYo2;L$ z3ERS>>V8w2fS;+X5go&rd(&DC%RpfEF3c0Vw*v_o1_`#q7|aqam9&ZmVi4j8GjI%s z%JwNF7ef)|P}PkM_yj|9C5O~d76x8od=CRtmR=3k;+YtfA;~$0h$D?yK{!fx=Bc=Y z7*I0%!P`xvW?_NW7}9+U14Gk1myk!K5GcXwRJdJ8z#Du3oec+!7HJ}!SFlMfL>Va5 z6oA0v5RavSvX62drh>Q@P~nc4*2}V#^B)#`X*XxVK+WZhhVU@M37LVtRr26gw155h2f)y=CVfGV%tp*KW3r~6I&#d zSj&%Dthf-DwH0US^d)sg%yw5-cKN^#SKN5fai2Jw(5=((fAF=p8caUPW#Rm!4T?~8>=A(+N9R`wVm<)dh3tgc#8>p?!Vj5FFAL_ z=wTKEchtuYIsbS6-KTFaQLD5D2I@oHckSBc!ax4t{d?}1pi1^)ZXF>g%+hsM^SMfR za_|&W0S-&IXxtKH)!P=J4eN?R-}+#?aq{Q9zHz+lce)` z1s5OP1z`pt#<$YDR-Ri+ujIH429g+}9fZO;49O1Wj!eW5cd4d_lQ_E|Hur#O6lDkq zTQZYL3|nbN;WA8lu#tJR9E-OcFfnD3@I;bW6o%Z44}MJ9d?|1&LHD68k3aI{jAve4xM0!C3l_co_6j23 zt9P$qBZiFIV9YLi>@al1U}J99)5kx1Y$ZEh9zJqt#+)fzXV>ZDpZ%z~I%L>jHNi&Y z{fCa8xM<-k+&-IZu_3oal(O62I}aNS3cigncpPu^g z6LVjhw_yIlS6^A;SFUu<4H(#eD{Mq?~ON~u-5^*8PeN1_B3$tIwyVhqba(WUiz5- zju@4OCEdDpJ?V2FO?Qbr(ZE3in4LWljTtwJdt{4kHmUr%#WtH`?J`hYdGbzpk5jn;S}HL^5Dtzwe#@ z%}M{b>(K`vU$pSmR~Ek7uYcb`Lk5l=KW2ybZ!>iGkT}`;$$vloq5B@Io zU-tpiTIxwh6;v#7{7r zi9y7GH!&#Xp@-wM7*e|vF@R}<9WZXWC~j-Jws&{`ri~aB z91BBGs4WZ}bR!1fjmbHNaJL19+Ox_I1UPwK1A~Z%ar0#(wrvU04z2u{wNKRoa9f$7*+`DP>%gUE5#!|lm%kjR z)}=LZTd}gKZQJ;pyvF^Q5hj0`Mmm;a80vt-K+MM(*h-A#QnBESu}qovWOl34Pr3V{ zQ&0XDPcmlQs7tOnKXzA|^ff@I8+y3Lq{U!?up!pY*9 zVMCAb_pOD5nQ~#+lXueFv>~Rk?FJErA+kDn^5rY#K$fd!A1vc09^o&?Mp^=ixSNRw z(&ph*6-};t#CWkb6 z&+i1A2<@bA#E0u|ic%Jc2vop*HYqN;e>j4-JREqO!R*6bLXrqcMsu~}YuvW8+U zrP`$tV$@Py#J0pJVR4)32sWJx`6X;aum&NGCefpN_1$KE@5HgTZ%Sn&Z05@X4-DpQ z1tswoL^(?Fker~wks64?A#y4+T4h-|Eg4?Kq%Z&^5qv_CHNnB*j{!;(n2?H zr8ZD=1(F9;iS?FLbGh4mJ_lqOlr+?CRB$r&zF=hX2lQ4#14Oc?834#i+0e>kH#Al^ zkwNfMiT3cM^W>{s5Tgw<_VMf=;DoXwq#B%rFiW-uA<83G`;Fx;-xCutE^;vTHU=Ti zz#zm~y)UM?S$xetV=_DVBAr@2KufGH?IcJ%lOu0e6PoN!ZnLQ47tP(Gglyuk!~Eg~ z0b`LHnL1kqwJS9II(C?t8sIDjk&y$j;Y0$KEQHX^X%>T21~E8(kwFuiW?j69Lc@Ix zhb43GiY=*U|@ENOGupxHH&)_zh$5XUHDm6FA&9oM{)#2mcHPPJP2(~ z;YSQ-4wM^#1IYmJWPOrM>hQ|Io6m^y5KU_3ai%(%7qdO9wC}>%2ESky;rEiPYQG-U<`wjm z0b9hu8*D7i#qmFFjNR?QH5(HrNCr5IL5ukVH3-dM5Y|+yNim&0?_rLc0}}(os?A|Y zUmL>27#;9QNjAv|iX@d;t08A*KEqNV$IDz<>d@SVWkY@nW-`|-xGwru)5<_*B}8OU z=)8R+U9?{tXD$v2V69g(npU6e%#sZ&cP(^LXO><~YpEGUWCBc<4f$1Bz436&qL_!5 zHnhwpvA44_)8e)~_G{?4Em%#uT31>#SN7C~QXsrIT$8Ih?baCi@}wh7g}5m8%6JMq zJG<>4cYW*2|1D%5dECMM*Xd{KuFNbdS}C?Fss@_Y>#>??ld%j(8zCbwr%b8>h&b&< z2qDE>acW~BTRSqla$;gj0HP!>ZZlps905T|GZ(l>;$Z@^c#utI6-=(b2-|plL7<6Q z+h0Wy9a1IQhtEZr9nZ7=9U;%QD#8>L;r!c~oZX<^DR6s{Dz%JXs(=iOBF+}^I zGngIf@2oQn^#GZFhu9JLBJIMgeZi18z~I7O%;1XJCY5#-lIUN@Y*H}|=J5zEJs2Tu z!X`;FHbI}daFXyi7m(!egxHJz4c2qyoY<|)Cp37Rr{_a*{;DAceGr25&zN4De4>?_U%!}A25gSS` z=?kf&_KT&jOGycI$G8GJ)JUkwCOCN*s)1;Eu$4H5Xkm3btL>vbxos(P{D1kI!{D*C zVqOCslkS#%lHH({p$5w*zf#?5SNHPI?ZlcPc-1@Q)izJ|B+pL*)?X9%fB&mJE;`KZ`85W2_O zKJVGHGjqDPX7 znbx`44$K$W+Qt&0dpKr11Pq}R!~&B{6@A3ZT~v}04swX9s=5LhA7KiXt#7*+lqpmE z^}*VKLCQ%L=a7ORd##vH0DVBi1O%~lDxVlrKwLLnTv!ql-i0E7BL=hu!@xRc9D@Mr zlb13Wl6Dpb@5wQU5<{JYT+EO-_+;mZ#L^3u_lUvAzHYi({di!MWE;K)B!IJWf(Vnm zO_6%pd&uK*+r>06xy0ix`55F2-(j|?P*avGJ1cH^pv@+feM zm=Uqp5W~I=%o=iNmSo&Y2}zK;EOZezv?)x=S)ycQ>C2`|A(3ZsBC3jvA+W-zLO)ZM zMSUf7=W#1S(mqbbCRihsFL_MyoX11!W9mS9&c79&;O129S$Q(`H7BMEhPeqznkrrwdZzN|in=2jC<|@$(&&|B_qQ6SUft+*SIOj{f`}Rtc z+fsL_Z*9)ta%>Qw{t%4P)daJh8Us(*v{Wa(fyvA4r&o`40=heX+) zj~A?^G?s~#2y_Mpv}^*YZf!9~(j|dIJjyt%^_MlIP=(=94JLR!n$@Fc2A+qI8cKm@hm_!ppK37;+

$}oNkkOpS;F0rD5Z*t>YE{FJWZF$kz-tiMOk)-7n2+& zJb~dXNlRdN5Q7k+#A%CIsy7&fE3Gh~Oa>PnU}LcD0NNOq2E7zUC67;2rfVrOlo|9y z6O#k}Vm5)Xd%HGSvv>&caQl%2$vLG!L1ZkiQ83tKNTQBn(^(M=6eP=T2@b~z7aMT0 z3PH$G){_3UF=uX}GoVmsZp2^#v8FB%R;)-d`#8=p31Uf=VDu6MA_(C&6D8RYVAurh z&7F!4MwBA3Qh3?N{I5PHajb>e$J9m|i+?LT&AK2Li38wU*+?9x>`)WQz?MGaiNX0y zEaFq~=M2OU_VE_P4N7A%s~SWS=xiGbVUkYy@(H`|vopV9)KsXi&=D>+GUBKWmb%bb z))dXEAqtJf(h1C|*f9YZ5)3EWnTr)23en;_$L?!FO&I5hctz|5wdExd?9%HXiF_b~`? zysna@R05{$YY_zLMBzcv98bDKZQuco4LnAun|NYIYf93B@1-^@pKvhgYg$c=9#jkF z8Z~ayZUQz39&qY)AFYrGPg%qa)aFZ#lY|xR%9em8k4_EI#Rgz?x$>33dRpE>9WGf+AD*VD# znK~(3!3aQ*PBe6F;7no)k#_kg?wwa8?ntFtA>tYMT+A6*zWr%wWjc0fH4G3086xBsFH> z;)DteQnngsIPA-e zkDaZe{7E@7807ev-Q*+6RmI@&hA|kA7;+OBaS)xukaEcpu{Bnfb0+}I65SwJ%^5g? zSdt*Z9oP~o0xYIcXQc2*2JPTsVQNfrf_;qA1Ae0*LF7=%2|F`vOlK|%GVpK%*1ViX zNLpH*rvYdB<*s&uX}~2V*ulg(nzq&WrFmNELK;f5UU};T4YWt|`m##Taw6IxT7qe& zq88%JvX!)({1=%sG!a4Mjdpe%(f{GG!()My<8R*v@gA#b9m;Um$+D=dY4SsJ* zY{n%ABUlPmpgNB78-mw>4mOvzMO!iI+`0392Y=vqe>?vRr=OhH-3WE?gwj#yB`gwu zGe(yzF=r}>co8k6F@s+ka5|A$=pXDMaiTL=P%9YbMJBl|8=`qq476i6T@yC@7x7RI zk88vuiwi>p^{)+xU6#-yxSUNSx@Mb*%j{+{KpsQN#n6}m&2&Qyq%c@^Xn~=jS0+sb z!J1TP_C*3CqQ*)tN<72mgh7ZAe;ijWiqgGD zpmI|p!)c9U*3H$N94hxgVlSRBM-%mh*J9F7c*V7PYh$jKtxD$FDxig}rfm4!z= zha<6>QgNffT0T6^d?to`Eba`Yr8+Sa*@-9phQ#4`DZrqIk$@PDz001BWNklIOeuRJEws9 ze|4`B(@U<6YaECkFR z!hoO_JF$8iY7)F>VGv8QSB8eOS|6|*D=i7F zFskbd(qf zF&O$Hr^a4~@uV~t=ae&~K9ble(N2i67~nd=-*^v6u|st+lr}`$5#7PC{uIt zDxM(@^*e%WoXs|L;{7(-8MS}y_Y@@qL)iq|HsU8)p9n*QqsBL<)um2mh%E7G=7#xJqUVHNAM7To9aW5-!z+G0A{M3G@@n3emz26rfJTG$ zBEUV^$U{+KwNtt+-|S@hJ$hD@GS(Q{n9Uj~K+-)QCodZ~DaNOKBb@LM^u!#KDzO=F zHe)SpsvAX5K-Gk-W+qIYVk7fLFK8K|cSI6TL(e4ok(UAH7QkcLM}g*rlSI!X7M>6r z7<*ZpxP*b*B*=;$3;_pSka(EEAbLjvLv0}+YpM)}#5~4H#Aft}3_CEp8+MRYMd&56 zxDxeGLX}z^hd!z^V{Zq`b&5h+9&3W{ z=a)RoLgK#{Y4dR43fMDy%#F#8dG)b|j=j&wQf|fL+xH*WZh1bq(M-pXo!Cw#y=>;rCbBDo9A$KD&YOZjM|1Lha-wLc6yU>?GbxNAyM~Ra;H>+^0Lv+-p zdkHfK$vE0vxuqmj;Us+vb-^fWvOk2C!_3ltvf{9q2J2Y5JIKg$H~x!uvfy(gs(_$? z#5I7<*g1C`18^0NiCXh-g-z?FjKypMUV)IvXHSj_J~CZJB4Y#O;4ok+_>?uSUHW79pxP{UJsiKh;MwTv_GS(J;%$cgc2F$jol%WBqC}If_#vnhB zF$R4rjV(n?D{N*wCG@ZCrtdM>Vrg!lOa3i!FOf45u^$;YFF^o$M*) zFbH=wVaV8_8G}oIVvU3KB1dNz1dQTlUtkCuH5v1_Mn{kwEWqprEO=5PG02GnIyv;S zhg0zv#-qlt5hg$&6Bn0G?MVP}6$Xqih?7W(Z0|wv%zaEP#&~nbq$fEd($pZ;b;z?J%2@SOs=qx$r9EO}n z$z&zSs^MdaB4nx}Bb^%2C$9lcyr@9rtpF)Z%(haBG=6)&R!#=yAc@PE!@yw`Ml1UG z+?r0+uwsMBq#xT_rQz@~2qep(3YQH68l`0@m#3ZMju0=5UR^d;fwUbghS|D=XT#{i zpoEN8HKC+o`LV=8)a78}h6EihA}f}Lswf!AU z%PT^Za-`tG5AA&_f{kc!n)bHX*sy0}ZVYHfpSZAylA;73Gm!`~3y(-Ht0TZ93!VhZ zbJeNrl_eD?u-Amuxu})Bgdrdrm~~OhV}R8RrYZ&f)97!GW%GSHB*Enjw_FcO=dfy-zbQ;xi z#RB1wfBpnDk1rqeeh$vmD~aR8sSd6Dq!wsGTVS=)9#0^h8A-|JwM|zR=lVPN?<{S0 z&;7mcU)^77-)f$u5&TqOc7BX+Cm!y~2{_z5B%mtjuxexcpEXXqh0^{U<3(Dt-_nQg z`%K^HBKheQq0(~ek$H8;%^;V2t7n~2dAX9Fkd*kNgMEnhf996!bzbZ{0Z5tOW@o^c zu8U$T4hH#|CXhFl=3$8d29&k?ycjQ8aP~L9ca0GTO18q48i<*ZmXiH1kN%ED*yD6M zl*z48u+iGtHx9zP)ef9|HX=$$7esslh`Vj?OMp)Sh!`w8{D+=XelfqEL%+|Ps-~&~c+Rb>-g*er$>%^K;Bc6(ruPqyMB}g;?IV{xTr7=F`c5$G%=mnLL(l1- zmBTR)u9{D-lkT`86y5RL{R%w(u{~Oj*ytr3g+o3M2?ji6?4$okf7UDVD$Co4FZXi@ zR0f0bV#98i=0>Z-aX8SU31c!#EbcQYyteC3!+`7KM8~do1qW37AC2DjIdqsr5hu4I zALTyb=l))Oxa-P(izWX!`pFS1gLslm>G%(A-J)Mmv%i4fE_(UR;QJYnW@C2qomAK1 zeKuYD*~Zw@{-aLU!j7?sT)SD^Tr^T;Bebjbzgz%^E3pzFVqeZL(h;j#-4dC2FbUiNo5 zr3K7JmHOPEitX2P@T@Jb2OCPSYk`kAfm=3*+bbyzgS#l!lb;W|E^k$N-aoSNp9$)x zZgS_rOyFB_;dGLJs)9Z;)LRtmw-HQSSqh*KZl zOH9u6EuRgwDn{e}pIW!>v+)U4GxdM9B8rM(cL)(;(Y4fABm!)O_HK_l$XL<;S34;yJD z7LV}-;D3yPN7>H#Y5A*rhqNI5QMeEST*6Q|^f&H_g?}BT_V2S{?1kegj{=@;fDhs6 ze2b3D&n*UPo6&W>ws*6m3G}7g(?7H-@o(WtNkn4ZBph=NrmHZ2?uota)}=E z8vSwPxRdg_lfOZvCG~}73gJ2rD=rMdfDu75H{0_%aTu0&V$I9%xwz`j$B-UIHy@?` z`|ciP9mMKC{{Eqs?3D^3!tOYH1=Li-v2*p})~iq;+IM#kZ{O{gviyH^vu5h6Jnz7n z0-qv&9Cq7yXHe*U*T!YzpUIOvGLq{LpQO5--+gf$^`0Ls8jQcExR-9T?C5;2&MId3 zUhCa;9)?H-T5gYdYo);di5;_DA*)?E{M>hkdN__>cQ4x>Y0BEk*oR=^YVuD)N|+@> z=Lm`aQWSsHad7RR0ptf&*3)T!3 zKiictFFC|ba$Pz2hA^Ch$Duv9xGas(1xL#*U_|vd*feu2bYBEy_k2Le`@Astx*bKL z>vg(%dA+ibcpiRkZ0q;X^`z#DEOsgUDT@6A5R&k&bV{qOf^Pq-cRFV|n&TDAR#b4A z**I+A)Q?ghI)Am-;vU_VE)gW8VF| zlWK3|<>yx*{rm&RdkpeKO319xlkG8_}`yZ8Q_fAq0g zeGWBl3-y~1tK($oXwG|HNAk3iH|ImQo1VsBD^Rsiql#w9KCyGI6(@a90b(eoxb9vR z=o5g5+<`aC+88>8`bG!PYWxds8F{Cw^`lm<4t3S`#My1)@O(!|_FrE}NhpDThKQRI z+#i8ilpeWC1lfy=(?dCdfG;DlH&_@;{x>jM<8Sew0o=b{(m`?*91S z+va7yyZ2Wx_Jd6LyJu<1K5RE-*Ki!r;yU{CVx%?xPgrWny)x;VZrynd>cA`&TaWGL ze*4mXpEARLC#5Khkfz$|as{X0tsG0sYsUo?C}|G|?5X?}aMf8)*D5IY5APkP4g9C8 z`4->j^@F(2`wG~WkgJkv`ycWw zew30&VG2CHBm>%2%BD9i>J@&`eHqQhm+LE!?S6hv9saxDTO>x&fmnqKp$FXX8n-K> zdWEpzQkJMbwmBG*k9u0W{6%YHtV4e1Q|Uuj?)mDg<-yg4d~MR7kIlhj-vSk^J&iV^Pr} z5*iC8jA~>>YMKV|2a#$ala+9iRYf6BJLxdU$>-6l>1>X!-@DLQZnHj3O{^#H@)>LV zPEXq#Ro8>(D|Io7KSpi}(MqVqs3&HMM87E=C~VDXGdb886MquV9H#O6%oNw}j5A+h zIwmf>v^ne>j79jdjQUu!V*zD7M!G6RDkm4B9i3=aUTc~>cOx)dmV3fyE?LjYwz{3V z_oezAajFRc`LcTY7!rUa*H-=mM1!qxxwY z*HbMv2jf2oeNFfe@PBYK$M}o^IxXU=_*gYVISg0W#9~_KwY!i(jO1MT-%)=j*m&&i zK=}(q7Cqc6L3hap;@xgpW@O;$cu#5n{{a6P+DKjU>#&|N^ocxqoVYhOXr2ecrwurN zfh0ra-*x{*!R%M0kP<|sz^0vtxCH10dULdFEU)%M$--`r6aDz!i%Qh=|APdYnM{<~ zbyNrDc=b2(x_mQJ$4z82heVDCOT5(a0FtEtXMvJvnBRxJlZWx{V&;xCyQqJDEGQ9- z`uJsa(XeulQ7tmMuwg_j63hX0-aS1jK&Z_+BY?oy`5>yF4UZBUv7 zkH0(4rj@_4zU5fwKS9>xKUuYuu%Q39R9;I`N2bn`^16+BM=a;gO=~rFyKB#!^(t)*e2C2SEm`+HExrGX7O-OZi$TP!sXTB3yC98{2>wNXaqhr{V182m!$H9}CVW7Y@N7?pwd90n;_(RpB>$~!Cfg||zViw( zJk7xkxFC#gw4CZQe=(+)@P`t=AR&4L{R#36qnzCX7B%v@ZbMY4Ecbi?V2%}7kzcM{ zC?s|m{_DDLa4X1>n~&OCEdbkFCJ!>Om!Vb(dNu7ddrdx_3_j}^k(#Gi-~a;b5n}+4 zO;B?NA!5NO(_pXQGZ5g_aUocj*q{?CbW)h5m&F*I0e)ebiD~I%#L5h{2ECJi6`yzN ziNHmE&kztWmaBrX^iAZlzd+J;I(6EUuNS~siOYC3N9cwqgOdaaa))=&c_18wS4=5L z5f#HYYBCO_-TVWj8Pjj`7OKd%AS?)~-kWRL7Lc14HbBtIRhf>3*pTR@e!7>)@9shU zGjT1b=LrnQlS$gIT+Qr6`FCvOg=+uXpl;{Isd_THBoo!<66 zTiDK@!|Olp``6PiKLp@9K;S}w&w#UF`IA}ertb*laI>1UkopGSf^Q6$8X?c)eUn>^ zSnFSZD=~B%Zwmez7$qqlZ~ecgu9phK>UVwGj8IH z756mzzOX;Oj5uuAx30Btvi-K=mDy_HE!GQ5t5JLCBOoN)m4*yqi*LR|KY)}?Dx^bn z#cd-*8U4%|2=bM5-HdQjNa8;Qb#@AGSS#MJ3qQnZG0nQd@$>N_1!ryU$t#o$A&H4uKZkLqXDGk@B5rcn%Kh!o_=- zBK@bB+6=KM+giqXlT@7?`uBo?@f#5~1+FVWCxU%h+6e3( zvPy0?gfsfcL8Ru&PdV1d!Jod%(H$Djy+#vxO_zpyADjBWKae^W!gG6Lr61rtsGJ#Ot)a8*VgjtEI{ zB#_RozS(K_*;p&+o7q+eApHUaw}rQ$c2zZ8uu}z^4KDV|nz6^{H~b%a{!j$Zi;RRn zqv0%}*dvX(Exg~oB%{fi7(SwYGkt4FMN~JkRA0KWxS>l~rb!P1N(}MG*37Yh<^qpF z3zhR0j3Mro9E46RkEI1~>Orf34O?OaGuz8VKqRM!a`e|(6lxD*DBYD1H{@t!sJG0S z2hgV0g_C5T4n!W&GM-}OgpL>&dt&}dAu?NpMM?E&XGw0)5=6<=&BiupUW~ZXlh7+% zla<$EUOX$9bQvCft!P{>9z#Ks+dOlTFTjQo_UxgO_80b6Bjn*2(UHBX9=O&4c2dbr z-P`jWOAeLq-XIzZAh$Vw=+JmS3nlc^M9%T$zTK!|JngmJ@?6g%HmtFcAEQnT2ym>g zPS~+kW@2h7eqqg{yYPo#s#7iPaAq2Ut9iGGR6LpXtu)Q7&(}?!Fz=l7tt68xfF5*? zceGx_c}Z%;jNZtZ8TD3ZvQp8eptK#pW!ts7Kofi*P4fVfwX9NWR(n#>ee;Zu(r*#T@o?ksYL!~xBQM(DJbqO#0y(o2l>OgkB` zIZ-j6Bb{#)UdhPMc@>NMX;f3ATr$!9ysc9t4iALIb9!4Q5t}d}lm#E$VUKNY*s9kT zO(MTw)Bhg)gY;HwfZBYQ`9XD9B&?$H$Vqx<6UFrjq@bcYT?3iZh=7G7BeY~VV?M}` zVnM!dD3zJG|K$oL0DGCOZeAATe53^VYFDh^uNdd{&_a99-y41Ai=oM27N*B4EA-CjgN0iZ-ZBGE4ET(hbFc%OnKN?7(ybR)t@8r{9c#fKft>eB zLWc-L-_v$;>w_?oB%9e}#QA#rqh&>3W2^&va?X3PuyFLl3du(XHFkQ*zym$4n#83U_z%^u+)X9Yq=TYuT52cmu@q3^QJaij}*0-7@C@hFK0PU$FZz} z86_MOP3Q-(&`#k~sZRJ$Qm@7j3HOWU(6EdMxseY)qnYB6unf_dsP>|8$DXNt?|?dq z7eRQvXqpjt*$wJ+IS-sUsXo|bd(E&5X61%~kjM2X|F)D9$0f&=gsPaH&h+44OQVnN zK63N~8uj^=R!ZEmXCdWAApMcSZ(3O)M`G?u5k|wwmEklQzk5@YGEO|jmn1?m(;i@3 z@%8wSucWDYcfW0K0oU!m4wT) z+Ai2?tD?YA`u7u=q&Y}*k(lsNGF@bH&gw5D$1w#anOOOu4^eo>!Q+y-gR84=i(*_k zylxQwPnR2ATqh0vGjRA1?lcg^4ADZE%9LP&$Kdxmc3z^%Sh0>yTrYUS=4SZ`965&0 zWmKq$>;!2j4J4szl!6=_mXOlhSTm5}RFZa~jb;-rJVRV-Qu>o16oD86j4d+cOvMuq zaYi959Mbv|Wcwhblg(0GRO0BJg`S|GqG76eg*}eLm9U+9v835KZrj*E_SvW*g1&V6 zeU|tpB2ZkiVkzD@`7Pt;Oy%Dp!MzLx2RGYRRRe(3I2{JBQrdIVr6X6M(I@nwVkJOT z#V0l7y@;d8&Qia+!y3s@q_x6*uV@H}jZsm?or}-ku19od;lADNDxTmG40ngn%PdvM zaS?mQ)eUC#d^p8-T`lmqKmM}Mdxz-|(B*2abA0Z6)C+{wkxw-G)X+ykaXB0j%=-H| zvk9Yv31O@;XW-32H?iqscL#-mvd|zSJTSZyJmR$gtAdNXtDn`Xu6y3Wz^OTJt=Ysw zNRup$UC`&cC+8os2U(|hQ!GFk4%6o*^iGmeBN22bF$CzRb5TJ4##h z2Hc=Je6Q8aQ(NL+QtoaUADsZwbfkz*p#Q1-PK9xfI*^5ao+dR3k3xM%2_vy)op2I; zDKis;>wK>WRIXl=8D(FHGR)m#wFKf{Ch@rwFBDf|toYCen+2a9f8%G0CIe-v$+Iiq ztwJkD8L4Tye>f~6HWhJONo=^}!p#z!uG6p;lXOSlS6-CBl2Uap$hO z+VZlPB+{}(K9$E6bMa10Qlt+lA+|;#rZ<2(ZS@jVRq5_0!oZ0mFVj`T(7(hBp2g!5 z@T5$AD=(EI6&tR&4uYs7jpgUdC>CQXU)q=}V783DWvAKTjSht8FodiWjH(y8pB)O1 z3(}{7+S?6y&swS0N9_}2FC?2tqM5@GIU`%iy$TK&HkD?xaBGZ)vS#i##nju`|CD&g z+wywsqI|?0l?os~LN-+{triHoH&tCNV!fqGpfF3tw6b5R5MByHrh#^$DqU!d<$z3_{W`jpvevl`KEfAPa~L zS2ef$)*Q4XF@4%#@|bh-HT9LUP|mFzqnI@%ZF#Cu2%M}Nditn5B*nUK7HQp{<4Cg` zOh)&iC}}P?+WbBqSBkh!G7?New;!?YyYk(btd+XU5hmHf#w2@(%@-+jG(+A#Nw5Wn zm4S-)`&ad}3H-Fys2k=%wk>K-6b%3j%j!PX29F0hPN#bLfRh^r z``!RK7jYCv&2qlVpx15gBwJPz8_Gw#!FqsUUrQPbuxuCWy`UROP|uGKC&yo(O{213 zBmAfv1!V-a4@h?tJ9FDD4H)r6X7z18G$R8MezAB4x z-}Eh8`82d*o6(n+dcm=gm2S`iuJCbc%i)9ej9E;a1{RAQAl9WeOMmW3MWtx!(tr;OJC@)&MMt4&k-Q zU_a98e?-O0ynVn{Dpj2hqUyBG2!hh9uDc(FRxR({_P+t!HO^{NP4T+!ktCMe>ikBES9+2jjv+4@~#qJr7!hHTZ@%wrxK6l1xAm9Bo->dcWnP9~K zCuoEibN)F}`zumH!sa{Q>;zb$p>ip>=R{$Ztmjum)gM1(lB`fjZ|tH2yv09vIZ5$8 z^Dc|Sk1E{mD4RNJx(#1^ZFg)Brlt?E41|no8lQI=>VK&I&`@g~FeW|dj=h|zyX9gPiHc_ph}v^h77B|<(q8&`?a+pfq@qTs{= ztLNYreG0YFkl^VHY00rn+!7Fot(On4`omZRu1NhMnoO1IuBexeTG0a-mI_`RuxeL~ z23)`UI07I#-|=NX!hLfAVeEO?FP2KO|Mm5w?|mz_*=YPa!D-+Jxo`plYn3}anGTe& zMI8=ZwWd;Cu&s-oI=^_5;k0St2|HNqW&R!q8>@lNkvH^?Y@JkyKGR^ml|04F2f3&Ir#s^u*u)g3V%aR&#n-1}ip z!fn%GbbmO?;h?In=k05YHJ#2p*y7D>boab7t;+1~1HlY5=fzzKH8vwy0VSa6}_dG zox(g_N|-%3EbKZbj%8&~-kckJlVcpyn2#8^wEK>hKnn=&F(&_YCHKe zSMQ?ek6F2Bu%I-e zGvNopF^9sSZVDb-LcSc!`Zoi`B zbkHp*+1R*AJx!8^ESI710kBEBpdpzDYLr@pXlq-@XIc+p=eT;K$?_bp)mn7G0am42 z)wm$P^ect2?YI7=cIyx6oEd1@Y!Ql)n7(*-|4PVG(PFQTiPzWt(#}G{X1G%~&0INY zEZFPC6W*Q=#kCYlm>3LwdZ%&u6&Q&KIm2SqhvajY0StD5@f?jNO8(E}sNV-tW~kV5;F@u)rJH$;>qkP&WJu@t~2-UdL7buB=Kg(phBVX?dp&`IDtwoivz zD?|nduYoR`Im;w(1bDWb96X(_A|cE`sgNrWM2Q1rdVClwtdU$bCsz;H3ru}<0n6UM zD&2#Ea0b?PGA8PePuH?Hm=kzic9}uP7!7$J8^1 zvHJS@C0uz9Ide#w4CXI0?Rwt`KuY94wB6vlpcNO*6YPQE=GOB~!llT9!$&j0$cX_T zd8<@PNRK*6?FIegjve0)UC=$!Sg8|PI_Y(qg-p@-C+aSA#t%k#LpjTVQt?u=!Q z4WnbQ>7IBx^UC%gngWU*d7@7$+AML*kEh*z1YcUltH)3VTR+RIO)Suo5+=1w#5SP53yXmZ64+m$auU)x=e_eMHY~0^D-3IY;d-kAY z4aS-EFDIl*H{jd}7XPpnQl+A`=`RK)h)O4`YGxj=n&h!)xMeW2A;Gs6r^TWGxT!(A zo1jVsAmnoj`elso-=KPQ)884Tx|t5z@2^b_-L+{3$ZZ3o``~nf#J3DK?72saxyg1) z)E&&_R=4{cq{IM~@H}8R|BBb?-xL?G9=}N2l)x3z}oH@VwzE6_?AUeJ7Dne>|kwySiRqo%yIv_TSW_=pQ*IYDMOy zY@~E6nnlIzF?lnoR_DH4T|n@gC}~Vmi#ikam>_TTqJ}F_&!p@Yb+T9+wr*VE%2U;} zk)<_+t#o4UyB`kesXf-bk0F{H1PP^P|ENxdv2pnEmPpx;4Cc})x z4E&-e^el-UWrMg|djlYm#ft(3LjSs}!GLKMD3{1OR`&FdLpDdhc@S(!a4 z2?vQ%W}*^ZrHI{=q2PHHc$bXsIAn8lntB5IFsg@fNS2jt+y?pMtld{Y)clrj-Pe(n z>j%zez)0H7t~WHP_k@1$qY!1}4nHf{s6$!y&&6sCZyjdyuzq557=vQ;Nlb^x8EAvn zg%r550LC+!^bTJvxRv~qtMX9OvUFzdnRExt#sl$6=OuQy?@3CIhD_xNu}VxtB39x; zNo4tuR+gZ_KW$-YdiD`BHBaHSjk{&}rfCctZ_1Hos|(HVtz2+WR?Dx;Ohj5`1|$5T z4GP0!(xi2vv1A#I<{K;v_j^E z9um6oAn=>4k^@>4s|%quabF^xUkG^u8z}dR6T(PC zH&%e0R)RU?TIon^Rdkygo|p0>5X1P*ULcSWyn;y?=k9`*NjzXuP1_z$pf(koj7Fux z#5y_zCIZB&RG)?jv1y5)k3b9cvGF~nbwwC$>It2y-q3pnXDQgk3K^>yXbsDTfhAHq zxV9sTSt3$?u|_>kuv~B@4Uoh`phVSf59oK1c)LYIGX23pYU905%OIUC!0uJV!w-_e z!|YhW%@+p(3MGi&nZ2j|-8`KQ&$|*DHH55qFsYN=MF!9l^E64596Le_UM(*<5m8fN ziaVtHpn)W}Eux@9g)oLk*jDjLKck!$&EzR1e(!~#DVZK)kY%^8PRV#UGX79E{mr40 zarZVv<8=$f@9=5j#wo$d&kkHJ(07B#8hH%j+(24so?Q!L|xD9uGqXm)rIO_YZ}H% ze>;ow=8yEStIex(*&wM*$3cKsbaffLWz(FR|!+-528gFpQQ4`rV*D&XVdo{5qIXQiDydjj_D^o`rfiZ>gTl?^%avV@^3Hk- zmc$P$Iz|^d5x^>mrv3hwCbFu^>mLUzE%I+$OlBOE=+?$0ni})DnQ4hMcbdnL<+%Et zuf?L#&DlExQU-#T%d(i*Q$X+4lKb)=kFaq>yc3V* zSi_ujnC7TSM;Hlr+QC@4fyfA!I3#uH(K;&pyuGEM1E6Mg)edN9z~qn@%HR=l0bb}2 zvI$~w;X*bHr^qcVo7c8C&Jo4M$;v|ya601DdNw)TX(Hh?-u@T<$sklkLkeqMwLpr*eiB5^u z@QQN4tXHylayitT(A56SpN2cwgVV_ob1YJWKRqMkaX37PsphTC?*^1H;^$jI4>KMW zre)uK|AtPQf`4n?7?#_EWg@S>3uopyAIj3BPx;2C_R1m?miOk2vxwm<@1PNnkQ8U9 z8<>ITU%k36p)i2~{(1n)8KfX4%Xpx@iOvdiH0f68MF7Bnfc|}hkq71Q*bU<;-an_9 zSHzahiJAn6kIXe;6j}}ZYQCN(IC3KgR}*F138u|ds&T=9`t2e47sGqNCp5Y(p=+gG z6*U$Scu-HnaqYbvwZLlrH^EL4e$s{dOZhNVXd(t8rJLxI`6~DD3T}e^0pmY#T(+Yt z_?)P?R;H~oU)7Eyw!v7lw8F8&W}Y@w-$6Jw2+4nUJb;OE24=QXJ~GX57lgabiu`HL z6~z>!Qi1T~S8zy$DJmf-+dD221Oz7)I$_hY4=XJ21;indv8RGjbRBA`Ok%~=<64=+ z6r>6JYbm9<-!}3g&)qP{M%VE)!HOx#@SDTah!z`)KH4zgEti1Cuk1e zs4OYkEK6BeW!~fnEBv$+L1q;zW|=BK?V)(#aj9}bYDw}SqsTQ(%Nf7jqxF(WwDr&5 z>vJ8Rs|v8OBSTx!CL+e;HM1c1#@UGCL>pF&lf4rIDFRLo?+iI0F-AUsd9(kxF_K2K-j_OnNTOfJY?LsWr-pBMohs!r2QoT2q ze~3;UqX51llVo8TQ=md~1&%lE&`tj?Nyjf#nJ*y51gT$6VFN9_Q|!$a!g+^`qcyPn zN2b4M9nAnW3OrAozzV=wwrAAOxo!#jSu`yKFf#s$I}l7xZ`>LdNpHrlAayxY(&tju z3?|j6CLaD(b4i_GM@zO5h!{ruLld&wslEd%kw{F38b2Jg%87AEZAYlBG3pH)tB#ua z`E>{;s&D*$EuE)>n<_E{8QDa2jhzoeL(E(%r*Lt+*rZ|r3Fob`v<_vDgvn5d3?d-f z&_uy!2rzJuqzjMPq5F1y*MgGuApI?yK)F|CMP-hz`c%*3P1q_Ch3XE9MfM<|U@zhi zsyBx?XUXn48Y3Gq7%2y4TwHWQ;yAORJ6qbxZw7m&qbD2taIt-DWFfnUN)hwfJ~4(CnxwV1ApQn-9a6#(q%mPW8}2QMGPb%p`3iF^ye1~A`?Wk z@Qu8-Ik!a8f0O)H!6ZfyD~-)MM#!`>TgFSI6~kcRjyL-mjpDQ@Jznk)M=om56O@dh zqnIbW77>@A5(zmKZK~eL?%$5cpJKYIjEV&QC=MkkDWaQ*I>C?VWCRq@kFCwFA4;Iu z#nGQ$!#<~i7Ffbr%%ATEE$(+mCgYZQVUeo1hK!|mCWu|HvV{^03u@x8>JlP-Ny{U# zK+>n76qFS}EK=zWe{xPZw@5yW`N_*oG+^<=AYiy<*O;zzgXY;#X_j$YQ8uPb#2Zxr zi#PI)+5k--wWnlE_ZA5Aeyth~1zBcffrgZsPl2!IB9BP^J$Wr}VtT+LBAPckkqXu< z9&7wYAB4Ebt~IKVf*1rBzB$P-D>NXNWe4S0{ZLxGGACWQ#JiG%o>wB7y-a#sKYwkQ z+S&qbF3^~GeVw8{cfNlTL?`&n^FW!paMv1-b>{(eG5t7A8AFdrOv178Xkyj$knN%0 z9q1%kzfJnfG|KGO{;YGQP?4upz)#CL?I6A6>-<^^*M6)<5VBM0tXd70DDhVi!iR^i zg$t-T$R`Y2MAF8Fa$yHA?OEy*Pg<+|uJm#aLhT+;C3IjJi*tud;h(BCFyEy9)fchp zUNrE5glTphX)Fiqu`d()$DXGYe8pTM!^R7zQ#%(l3;-JqEJd%VWL5X8<}bgku@Q7J zSo4Jgr$moLLp#G+9z+@lF*}v_xC&>eT=U*5kKl6X@B1|fAglPG!s+7D;RV6L`qT1} z1%dek;_96UqQlo?5riJBujU{xiQyQ`Ea`BOdG{l?ri?KT=ni}|aF|Jo&WMOB`|bj% zm)kF>%lxFVWjwcj31)hXqT@iQDhXRACi?znh*W8`(u|IO#PO?v&}~%4uv`JS7x@}lYD%_odWe5jl)rhl$6d;#njkgp=ObD};N#PK%|CZO}7 zAw^H6=k-ZS!NzzJS6{9?Y!F}o4f!rCc1C*sf_WnK5LJcV*vT#$<2kaV)-`*$kjKQ-E1*Ghpw>dYM7@f4PDCJyu6GvUdN zk0%ceJ5D2Lyu^vUpg)rl%>}`Aa^RK>4KG zhzvr>7OO!}7zA_Zz_V}z%ocds{Xk+Smn~8K1=Yb}2Z~-=wljax`ko?xmv`tfDWw^W z%oOKO9yN3#{m!qDU~;Bp+;rh51UnR8+ykzK53B-r#8B(WnAMmVw%B}3yR8%yC2iQ+ zXwJ_*xV6zJQ*pd>j}PGDc@7pg#)8YrD-C4bwd_Mn?!>9@uvOzLnJ$D1&BdEJ3`V1Q zD6Qe!=X64{0yT~LpIiV4ZW%TTIM5cES{vmtWT)?r)zty}QrReN3FhB`A%|S5+z?G0 zUQkdTn9TsUl_GuX;RUaHEV~Jk&>KWa2f0AM1d>GXDT2cwxJjg+V`@kCJ#kcyzfxr{ zSsRtai!|tCcf5Q0HJ8C&aFj&sAV~yH;R=K%?}i$~`H+)tjM&oL9t%CN@BBb5a6i8ft$q2 zDtWr!7WpMj#Rs*Bze}$~R6?+2bYIP9M{s-am4VTiXcmRWRC0&YH^&S+yhWT`jvbS` z>BNsE2!Ot<`Tit^0_oL}&JCP&|{Isj%eYV$q+7gcI76x^cSd$Qzxw*R;mt3479ORNDZKJfl_u4YyO zurS4E7&v$+6E;NyaDVD%4N#oIGs;7o1Hk-Hej@ka71^sK&O(i-Jp&m=>?KwbFC0t)oymuF$V4R< zBTm9w@Z2{Hq4$rR*c>9+xF<^}KJx zU75WNGP9`!LIf}k!1*WMVu|t9lzz*u48M4Qaoiq1WE({^S^G|@sZeT;pjKzrC7)z$k>;d`UZFXI7c-#z zMiG^USRfJ#%b<@M#0pAYIncn(OITfjzJXdkE8R4e6V+ytGQ&3Gv%?z!NYC9l1l$H) zHvrnbR*}4^SbLC@Ch<(32nAdnyXh5Gd6NJd!U0CKg~ZIZ18P}C=upHJ&P(Q! zA}{n97bRzcAWx}PR!Eex#2tq$>|qi@NQgfwn+5bVu_@z-E11Fxn=iz(ClXv(pK$+N zhA3$e26{S)LuB2;0{8UrY_5Iv-i&Y7EGJ=2xhMbb5xH6-AMBh@{Ck19_J#adk|x>@ zoMetrb72(Q+_>GI(Lmb+8W^yMm4jb3P)sNAq_sDL}GeP9=2 zS0hj^#9E!5xHE|vc_w~EK=2puALS4j5QK%hr3XyBLS?m$^y`!&Gb&PPgRkEz-p4Z> z8!#WqH7|f(N0{ldQHD4`EzRCl%oktK9%fH~U z*}yf>G}IekE)H1E_oKuO$sS6NNBjr~ZEz`8N2XyLpcQw=OdJUqdtfL;E;aUaV>ap{5X&ImX~j)z;Ce>vVZaO8^p8EhnlyF5@+n;=B z&JRzs{Pd$jA!xrqHRqco*^#9pa=NeyT=+ve?ZEU{N-X1#nbDOr`sqW>4>mKRaC_KW zdLOu&`=vJ8)zaEd{(@UG=4L^qkp^BhS!!dqQ5zYPQ&rxtKma9ZlNV%{$IKgSdg8!$ zlByl7xro?(AuBXt-Mm{6k{#CJM|7Rf&GuDwb`hPf^AGDd67aAINT(@_g}d_Gf_Z5& zoSa2WGVN%ozlP6zp-lh4W%+vW5B!OOGa_^Y=+iipje!gJ(Xg<4wptLW#akMZ^LCj} z=&IdcA(23W^vZ*owp8?-WD@F>`9@p$jx}>=sII>3BF0Os)O8}I$J}B`rG*#N1>r?e zCELS?JONKHnG-_IauBzOoQh=1~#PdU#^I^4-3bP^_RgCXUy=sJ%Xl#iWd z;Q6L0=qhTYp4RbYScKkx*WFrt#`-~=Hatc1b?(aTvFydaet_rRv5*UaZvDG8`np5z z9@5MfG8bZ4Pq6ijV?-C(RlL?2n`}Kdu8tA?MhcI7q$nD=-u?GH5Ec+FehjlAy{Lp> zfAuv85JYrLER{a1QWHB62X{nlaZXqxG|j*YmI-^$h-TG2JC4Cu%6~ZI0j~7`g)*hL z$yGybVehgrK1F2A3y!sM4c89|QF`UVy+~JYIkvug_J$Z^7-l!7WipX|3T4tB3SvUk z7R}uuXefhX+Gof{X95Q|dBf-sFeI)V%Ekpc>ClS^&;gx4pQlf_1e#su_@*gh_i1C%EdQS`pJ^8ppP{YM7^ zY8K97QjJo?1;oL#Ol8*a*qAIvb2yd6p{%}mz2#jZ_%x9!G+x;98pEx9Z%+Zw)1E#5 zviPy8nF)N2S4$_h1V9*rhH=SNRpS8y$q9J4=p9)SQ|um2q;KJ880nuqH$gs$MC_RF z(wSx4=eYi}vISM@^N6`AH13C^pY3!6UR(b%soDwy^0l#>C$i z4ELdez2>2_&F6C`SRNH@r1K_zUie;;G@f|Vh|qF?z3@aS_DmB7PvaYufpx|fd}riH zLsftn>iZD#Zpwu?Q;9eDHmLAfX;*wd{Phkd`+YsNQSGf@dv~8EE4tx!4g@E5L|d={ z6Gx^4THqVDfdZ?K31$6vLvYfP{&^a;2B@?O!(`wcX+yl9D6>q(T4DOwwU^k2oyw%i zohV(bpei{Qja9liY#6&8N+w#tDs}q&`6XX{Dyxr@5GfsCjlLn-)bR2w8k;OwM*=!G%q^wHm=B*&44)IYgDMpC&b|Ik&?o71Eni%8YP8swZi`d+=G`k${C5l|!q z1f;tgq*DPA>F(}^r5go7xTEP98-uSDcb)L-rk^G*pR%B$WMIA_-VUENywos{@pG|+fGla4n`dYE(I`wY6XYK`7D znZ32)&cBcJch*m=eH`bkSedR8WYY5~lMs=eD*I5fMihO!4tC4|C%UA3`9_E8MFg{H zOzqRwXKgz1(V;gc1lSZBb zxUP%Fhk9xQ?L3R_+;!vW&F$aDsgjg)MWXAqz zBJ;#I{#Y@E#s9Y~xi&6WK8f|q{Clm%;0GEogCPnf`F|gB#Ft^h5z$}d0rkJ8$*#Mo zKV(!M$ajZ@h#};K@mO<3W4C=?H5tQAERjY-FB9oL;r+Rnp+t?;Gg264kt$nz7HbCX zwqTDT_;sRUYyzgGuL=JA_x@#Z^e4%n@g=LD!=_=Nk*gdn@kEkiI*{|WRIk$u@lRiUT z%Yn|P%iIyM*`iUx8K#=)2K60U$u9ywir!PDKruNp@P`Xu@n9cw5LiWNRFnTe?E3t6 z)DRj@I(`!N_XeI_@=F59i^b9;l+dz@R|0^q_i^*mf~XVOqUZvlg)0!02LN z09l56MOjf1SX1J@OqQtU+tzX5xD`#@P}XzMPT3b;g4ZZ*&U4F|oHGEk3imXr&nUp6&dGCVEV>vgs&DRwX%j9zD%XXJOSm=eH`7vZ6NOdhJzVI<%SR^JnFWI)C$jQkJc}yR zq~clh5+RcKoQKrN^DzES}w}A!Y)0<2U0M zK+Dn1-h>g*`-@E<2f2Q+@^xO*O+M-uk#?yn@0Bf#|MLymoyM=o_aGfP(9iK5{jiHR z2-Fo8rS%MMulqc=W9Z+lhFB(GvuXc^YDy2ck?J^WA@i) zSnx&~QNFLYFhjrPXbVjlHW!Rc>xTVM?UUW4lBr=hDEvhQE+8~OQSO^0pAIT0tRTzD zm2PjWD*f8@zI^xhg?siPRS^3`IC;5N+}Y7i{p)pgbzQ_r!-VuMxwIxpij0G0N^QEP zhsIxmpG3bJesLo{@Q4M1>3RoYdIb&wQ71?m6aCkr*k_b5t*q<4hRx9#r0_jxfrU57 zZ8MzP2lfap*OFMhO&9WhP>>+vbUo1q@mv)=m*d%6@EZB?sY#=If-YGFI9frrJc#s1 zzgr{mSuCbmSQlE4e?B1jkT<*=>#gG8$4ES<{SC*;;1nWPa9M!mN1v2|ct;*3y$i$1%P1|*>G9)4?@T12xAK!1Lk~d|nFuixdMXBg z_gqhK3Q76Yy!?t2TJIrw$#TfKhnw3;!vX_pIK2L#8v8fnhBr-xW30^&%wK4{+?MBE z>Q$LX_W>sp;h64yv*0);I492YNL-P>{Zr#RriMTLz_SVbdsfx!=RbH?D?!#MP69L` zJ7FQ}j>{$1!R+q8LTUNV^R3A@9l|)jlv%M+R8zWSDye+qN8o$Sf;0xi_Xp|chWr35l<6_2dAWp?SfTNqOcB#=6jIk5Py++evu033Ou6?#p?8D z%#{50(VjCZS~*Y|MVlrr=0!P=dctyge*a`$Xq5MUD^@3!!~5ZiB2r*doM#T|f)JOT znb0o%SE+Rdnn!mr>pW@~xC*LVg**1u4H>&(We|w6qE}JODzkL}qpTJ51bZ&a3 zFr6`e&Xoms;1+fma=x#hY`es1B-rn?#Y&P3+~`PU9o!j0*z|haTmVeW`r!~G-6+_H z{bC4o#hjqSkNCnlpoXwDG5K-A&YG6>43Pln&3D)D$VPl~nVW+BlOeM2+4+;7O465j z4oX|U@8{n zilznOr^WTLv=BOu5vL=QK0|E`uozL{%|DK(l1VF7y$595XpVaF5{t0%-cB!Vb&0N1 zQn0GdjC9SD{zmOG9xY#UDv7f0CcB>{I*1T4UBl%F9{Q5Bu~O<6ru{7 zhb+zEpKp8pCb&i4GUXoEgul>zz;yUbA-f00uOq!7xD`JCl~bmUrIj)Jp>JGUX*ov0 zXwiDoRptJ>_z%UWWc61Z{$WTlCyOYrR23^4-+9p{{gNce=qiW_36g9->$jl8&N@Jv z|3>d0_@nVP0lgjm^sgBu_~z*{_uDhLYy4?AKi5@vnh?jVH)iXdzd-hD29TzgI!GJ? zizp%aE2TCAuX%y(bbPSyjNG$ZXSfU{JK+d=-YbW45kq`gbghV_*XQSEAu^4=7^Yis z%AD6(Tt&t{-}xPQ_DDdi^AjEvqv8bk3UXJn0Nrc^?$mJ)uG#9gOW!EU$R~m6H z&9zON>pm*QDsXD}KqS9$Ff z=lRH)Qrth`2sTPpWIh!RUvz{}i21q{>TIS*KOasPBY;j@E=P4ekDc4$ki;-X%h8sr zp=qr$jo@TQEQ|&RS}lh@vbwtgc3u=S4HIzGpD^?kxFVT)7itM#X)4Qa16yua%G=*@ zsnwIrzyQtnDT`H*=lA!0c3#F0b2m{x&pkYj8t%QGZQ}>$Uf*UowNba@)*7`Knf z+`u}$fHqdzS2=Ge^XD_7yeh#HstAkv`g1*bQ8dG%wrvvG2y5$Zf@eFq(dT?~jpNu# zhEB`Q^|StgP{JPG44;R>Q2U!@J8o)zlSFl?iG0?yuJ-!;iI*&{KQ=X#zEL76Kj0n% zf5KUX)3Np7{H~2I>3HvI^j!=GTcoiMOXc@{=&cR=mFUgt&-veFV%eA65v)n;P8;_s zNI&xU#LSoV%q$QJ#+mxjzN7_WihpE7v3<8yySmlQ%TdL)FL7jAg|=_kYU|+!Irm$?VXj_@eHG3>*lJ7Vn2e*q-kyFdH0j1^b> z{3fKcl0E8$kZV)wmgz~aH*HcSK$-oUh(ru^8_=8;UG}MI=MlHW*d!{7F|2r;JxYni zv$$?VClyPahsby4N0KIXoRdO5L3bY2lGopW#y3S%MFhOOtgMIjf^Re}Psdw~yyCp8 zM5=3O7ociq0T^efY^D4wOkL!4o7RsmwqtP-_%LiFn`jA>6sk4!Vc&owY1r}#d%#4e#wdJMfnsZu$DKPAC%>JF^`&&^|FCF`vfgmpI3=%s(o=M< zu-g^vsDS>FW5y)uf>ft7mRn!GCkfWpPb1zCf5S)~;L9&ydu8P&(A6D%brmxqJWwv3 zLU^q$el-`J?IE;&%pu+x7n zDK?ORL7|{V(3`Q>{YX}DlhxAcf^rulfdjMbh~9R@bXjGwPaqfUPvW1l6l>TS%V3vHY{b9uVFc4Bx6x{39o zh~*>7t5WuAyL!RfSu!Y}b!$YLV{6P_ew4xYd&T<04)Z|%T>wf{A0}Tn zC6U{jjbs>I;N%9`20#5a^I;=$RuX(j)HjQPkA#A<@AwnDC2`iK{#|rX7vG8kk zMS${3UJ9I(?CR~~)icYW*nc`I_fQ6o{1Mrk%MQN|61w*!9A|bPA8$v_4~B{ZYnrx~ zT4AGzhZ1KceoF0e{?73D6z>_?zkkb;*NOkYlT$IRxPK%of$*1R^{aV-0S=YjBLSF@ukwA_m)SU`t6Z~0ia@+5Fon*u@=KU;;}plzelt%g_M2}p7@%I3T7LNc}f zr#1tbzt|3Nh1?zKL?f>)5OD5quxL(as7WPNuB&0YFmf2}Bh;1=lGPH^!y*4Hms#!h z|BJ|q%F-;eiuBgh%^0~)^ZB(y&#A7TCq;l#cCH@8r;_rGwgSt6cxwU7@Q{1O35)U< z)zAV_kSVww`d(hapKuecWW~*Lpz%${=$#VA*8ik-zH2|o^1y-YiwLuBGiAJ9{~f@C z^6NUBOSF;txGsN$A$rxYS357|_$c;VU-?c76=Ig&zvZ|s#MbU(k=c~3Nf*`o^D4{Y ziQtm7LSeX_2QPn3J6WLyPFays!OK{Zz}KoKxK%-9%bf_^PZmVv^2Zm}#)H$hp-$-A z(f9KFSltZQR=&=U0SU%ys-vc2h!w5t1)G5XPhV%d5-<#`QtjCVGQFUrZ+Vm{$^^5sDwDEp+IkQLIhh-A@F2=zxBVtoZN$BzyJ| z%VF28i(MI~ zS+AL$?CdRm(pi)tP}wMChvgWRQK=BYOtB%M`2`_*tf;z9n@UUxLeEK^6T1DGz*`ci zzeVpVOiXwsX0avDbhNzxrezEcG5YrmY`mKIyMLPrbh*#JSnkIR+i@H@bGB5!d111= z=tA|={mtR`@}Jo#eECJ)Gmy{o6x#RWNXCh1n?yh*7Eqy2YBk7U4Nh+l&&2&^+UtAq zI+i=jcA78BBQNG7(K6bg!sC-%Nxs?4Gnx3mi-o>PAzL4= zOaayrYUHrI(^s&8JhVkwy5<`5J#hf;&}U}AGAWhP$VkEUVaMS=c_%965~+f%rxk*f znAXpqN~@a8%Y>$lsM>L2Z%Qo#(Av9Ec-S@CE*cEjjoWCsZ>xUTZ_2fLu2TSZX_M*oM;vj4lMF}A-@ zX>?L+2j0iOo|=fecpF@k(K~#@qedl(+oft{T%MN}-c!zUT(S^1IU$oKCO2iaOy@4=S zTCX7T>aI6xIrj-9Ei$s@H@tOM4?Sy(^ObDu|3Am{b$x*V!V>>U#3_6SE2}pZvKM1* zSBcfvPD&|>-(PvXj2Saf`nvVpI=fstlpIYjvW=V3mcm(pSxbDtb==|Mr7M%iD!?B6 zq^_uP$?ItTrq&d}ANCPz_<(f;9;9x?ZL_#_aDu2e9qN((aCg25Yyun>55*DB2PU2V zu$v4b;A&S0>Ki$W^yncIp$;S0wAy3e*c-(!M$$ za2IMpEQ;$qiKn)g!P|p&-frRxeA~SYa_KxTBdKXtM~ZfLdv-pG0(9F)Sjpwu=kTB- z&vv#sbT3P-j`=z&4Z~+Tkh%>K!;d4>rj3M4ksIpNT!s1Ff*|C% zu+0J1Jv`ztz5byTqg)D|`y!cv{Akk=a#h~*kMC<6tkFFSdK8G|c(>!~S3RU2%+vUs z@wsi6Yb>U4m^1;kidKisUM-TPJed@ao1=YlLbb(Ot9py6yyMTFnM4n7o^Je#cx>$? zhHL|F@_~B5Gt>J=b;R%`pP}R#1_;b{@)j!d%8w=xW|JWc*#|%5dLB9M)%9LuC7@Cr z?vAB#`JQ|NeY~CAEN@{gnL~~IWVK+RwMqY7!KVYkn~+jtb|t%p4G-c8FO=Qo``ELK z8dm!!^ezlle^Tf%y5vbR@HH%5j#LUui3^9Z0m|_Ea3JE3;+SK;y%T9BiQO_A%Ey$u zYcdfqUO8OCx@69{P(8(U*v*GV=NMN)Md4z9CQNL=3D%u9Ss9p2WC3Qtl+0@I3WS2k zzO`soe!Ng^wxy$my2Mc40$4^|L8Zs_K`Zbzp4oM^KV>ytP&sm)porwRhdpxqT228f zbm-8L(f+zUi4a2F_slq6reKgnSg)#;viz1j;*I}}uA4sDJX+{V&3-Jp-s4sFTyvVM z&Rn&pofVH(;c)7WbNMCO{ZtRUtiW-WO*o*#28ge@u{k3NmS>h%I=I5?Grj!6`9iB% zUrH$2Pj%Rmo;x728`R(#P(V|j8|;DKq5OcS6Pqnw)h!mLmhvqG&do%=l?sN?LkH^4MardZ?s1`2xd=42 z?Wec!#@+|;-(W4!ZEWAwUYfveGwUZa9k0NEmU$DpPiJ~j3DYvtv{1Ihgn8P5L|TOG z+=cY5O9A$-Y+u{HH9J#B%^u`c%;I%)xbb|U&&koD05eoZ&u>cdwaQe>=*ReEm zFBd~ByTeJFWj+@JG{BFV56hbC@QXC6O?gj%VfBjII+T6L^{f_933*-*A_1fzJ)h3| z5=*IdtRW*zT7>mu*|dRa>fbP*j8WBAvF}KRw`$C%&cjaT`fr@A=2~DGyCiM52Q$S9 z5=^HNk8`7(2UmBO?Nx??FRh74&>4oJ`s1_WQo^lI7PAR&+cJ%xi;V(>1vgHIe6?SJ zQn+kdb`#zqqI#C!d5yZL_qGZgBuUss9v~6I%J38b(c3txLry$;xso2^n9M!K!pWm>DtnPsx^XX4YO-iAPA8rq zcF|gdEN7qfLdK5Oa}0HBbDvK5vpkGx{m5ghS4b+!Ja3LPfLg7W z^JI=>{W5rH$qqu~{E^dg+9E$~w$X97)E~90Sh-MfVdv}1U7v87PBRWjmS;WSUt!6b z(;`$_)?s*k#>-vYY+%CHox1NeJFk#s$SAh@lEy+NIKIo*b7wUzqTzfn*31j0GJQAc zC>p_2w*<2UXMgGW-j@Mh7fOq4x zWvr*Mo_pc8I0~}_J;dL5njRqIvzm+#ja07NC7yrU`iF$3fG9&Q_Z~>(E2M(SZyQlff4b!CEREo^2|7DSfGJF zordn*Y|YG18M(nf4yNr_`t~`3z}Pf+Fn2sLW9i&OR3j}Quz35SL^i!dR(aWb#TKbK z9|(dD!RcMb89<~v0Th}tye{5?7}YEEfv8YxHMjaO9|G9dQwHx>1eQzE)sK#crTpT) zUT?jr`?xipWQ0Imy0c8dj(1^8t3w!lE}n+K&dY@1NLi@wAp&Q=2E_JwSwm@O_`$ZL z1@9XFlhtgMXcu+q40?_9%SJXT!S|if^bO5CZ*B|G3qd6QW9?>(P@RP+k_6^I9FBvj z8qiX``Z~!_os`-m`E1jbDh#=}Jf@}Q2Uk5`#iGjDMXJ!9(WL8@Ma$W%get$C~G zi(J@n()~&a9+Xjo$MgBYxz>`qVCk<$GumSkczb$O3TG_k;yPa{aagO_RHdu8lz`i6 z>92#7%DSGc7j#huG@iK$rw|1A8n5SjBu6uug$4(sDV%1PJvY$f({nE>71ZZ>;E8uC zfhkE3q7VvTo}d!RZfMQ^kmM$%_$(Z;%Q>H)M@F2?* z#d3uyE``mK{n&iqMFVjF6~Eqc)v_Ltm90j7Wp$j01%>w$zkJUB2D!i3{QUtmT{Knk zIXhvWl)IU9*qMOSW{8d3a-n3#_L`s$XmG4iw}K>Jbv!jrCBP~gEJAN=_@4&7S4}s2 zQK4S<=W4O5Q13DSEZiAtUtrg`G}?>o<-QU`yU`F2^vQNHub26nyb9_q=(AHZP^`ve zH|aIEQ|eNzQS6f^1~^Ri`%oCO7Q64vAju)O^=iXHS$J{{eHRNZU9+Lh6+B2IV=K+Xr#7QhboY2WjWo>K_y1wd{8dIAa*D88Dlwzy-_$ZA*UcK|U$tHyi+ zFl^J2WR(JWxZ8yagDya+1O;@53kEc~;MMG=3lw6hWhh6_H~YqtIjuT=BJiIy7%kP= zEY(;j0rGRR|E|YjI_$T_H25v#7U~PdfSP4^yUZIP6&9-$tIDMEINTg9yu)QK0U}&i zAnNV1YX0>Co^nL!Pp-vZ!{aD2|enjrvE3O}87{Q~}5 zx255;KVqp?y)E!B+k^2YFyN;?u;*Kle7SCG0+ZI)W8km6fb7F`G}XNq¬-KH_) z!*x|rEFMyhc`>b~<#|&VpZkR zI-?z0tK&zQ`^$MIw*;i#e`p;dSp7&-j|iO%VEPKX~**@Qe%vjqmvcyf5tX2ye@b+jXV&QpUAa-U3PZ zEGYNggRal3Y~Nj(-KK~0{2Ruu(_a1U5w&zT!0$yh>;Y~G46{ibPUT86IC1TBj4xJJ zqAQ?ZY`(N>N^@P=_~l64mOmCF?g+OwHis7RGga%wBk;lKF9mUf?FJ1D&hL_$!0pRi zdO3s#JlzHFqy5!q_oP9InFn?O2{OKSSjfsJX?=zok?25dnQY73jKZbLeg0mO_3bM3 z%jtBVw6P7E**(#4hfx(0eiQP9cO$xyIKH*{)=PbS$;I3n9d-ecjg3i?6xx2Z;W=!W zD#`VhhIR5o&ctx1eqqkbP8_wxQzOy+62BO7#tdDLnKR1hf%?pbf&;hn4Xe&{eJmgI zognip#6T2AB?<*X9Z>Q|0Kx9Ze1YTu0VDyaY5_+Kw6>FC=L3kea9huVKoTV0PQbw+ zN&|2rI{v4jk-1Klz%r=Uv=XDOK z9EGIcIs7YDBcXhhFHP243PL5|a@vz$-p!5_>g@iPkJEl|xT29w<VgpZ0*;Xn6ij5%H1F$@~}cOXG(__v6J{eou`ngU z=f~8WeTnkoMCvEkH?&~!$)zu@g6ck@j#@+e6lb2i9{US99T4dy_UH4mD2smgh4_Vp9Jg-4O&^~qTIhB3x ziZ{D%(9PxY9lfo;e(svK6EZqJoGT@I?iK{(m5$GMmu#C3rpJW6wWwC}cQMpfK7pan zkqK%+%g%u8sX!LH_w_FWJ!!Yy^7^MK6YD>IByE=)M9(kXued;eo_ZvnJ)Aq(x;pPU zRB*H!pBk(uBu>LxTcN!4J8izam9P~f<5Ujt8S0p@LwOF@dbY3J-5#`|#~Ya$J*F>k z`EFVG^q(e@t@*%%GwyYpmJkyQ%-~#)h1YbI&<7XEFP_2Yqua9vBBC8mtwrGnqVYogoP;ixxyNyqREZKAZh$&FZj1;;H3OfyO3sz*Wq8-<1LA+P zivYhXktwST`1tKrwiH3no5K9`5DtzqM(u_Qz_tG~>3B6=yz^u}=c1#8pO+m-Bu|gf z>(ie05DP>QJQAAMUS^Q8v(0>kL1(~Qg~gy>Sn!A_KyDC6 zc+pE>m+Jd+FB*jAoZ$fWV5XznUv1?drE64FR0Iz()wi7Y5}*s5i2#|d2`TWeQl#oV}L$z&@7m1j(3yxq@Xn$Pa4#+iQ^lDLix_K3h$O%lu%Fe6Hp7SD7VdP1d1f^~4gr$MU|@rmRy@6NaObeW8r; z53Lk&iu9_>9j$B#pc&U94TwRgdv$)9CZ1hZiF%vUFb8_NvyUFv{~Ym^0A}c0Q0*aC zkwWzeU{BdB)wUC+a;_HFMc8w?4_dhEf56+8a~A5=s?P|Qrec1`s$go6U!sJtcC#!6?)pa~K=vA6neU{w5zdZQ&`kTGggAPrOhGAr7-0SJzdilQeV)6G?-aY>GgEZ)Aa1&-eJ0 zx_RfFTfM}%ZY|YV$lYGBLA?`XYj)2`R8n$F;N9+s(|uGO(M_>>p`x8Y)os zX=^Qpxswe8>B+;L=(EO`eIQP12TZkGpY)|qXUn$szS|l3cscCvXx4LT!(~2w!QGdk z15p?zWEnh-bFf4WhlQfA{MhB+04I^1l~C}> zVzRr#^Pn)_K0parHvxdBGN4-28-_CvuRD8@T!5tn^5T1ds@NGxsW<2fTya?n7p(i*BhCXny3In> z_SQhG&db3FH`Hq@T4Gcf=9Rar+v))r2>?5RYia?g^=P3wTL@6<>+pkvw}m{rSSJ>+ z0}ldVQ9=oxe@j0UbCG>>m=Bg@1nGgnhPk&Z6BlM+?PFi=Sfj8)mHbT4RL!dX)9z2U zvuL2371$ngzTTp3z3jx0IN-Qpo)BS~mnTusynL0P#`R$$#OD-RG zp!cIn00y&Avzsi@8C4k3JONnCnyHuh=BdwpGljU2&Bwuaktjsd{NTRIF;w2|5{CIm zDIwksF5Goxfbi`zfkzhrK&IgmkevXM>l3rbH!jZ^#O_*GU$HDfF@Bqw^Pf-6I2W&-)q{hz{#> zp6GfwWD#0CZfT2VY(93D-x7d%ZQ%g#YpWgM&IjE^@Q~h^{gQsk6Oj4w5qYJ0&~>(K z$~LFEE=|6XNip{dj2 z{@=~=rn3s#rULoONjftf1^TCnIO`SLQ(+zOkgX^28xQ~-2zc)TeIvv5?w%lKFC9Nq zE5`&sc^@Pyv;yyyx2?~#H{Fw3tWd*sxtUn>y%Gf?vClA`<%xq8_JdT)KyZ&mgqz-A zM&=0s7D?m!DoUh!>203>=H!`Kzjuv>l3oW?XEIiOi2HqNf!ubn$EMZsZASX(d$MTA zz=*x$AAP)kvqKULR%Nbyg`skq2l&lc4UOmmGWhQR8Ba7n_LJu+tCyf zO}+4FJZk{dvjIE+a9Q$bJ@4xo=fhdv@Jfk!Ac!1z11%SUm0bw(!3at6TFV@*K1AfME)g~i%n1grg5=tozJ|)`>{S@mjTQ8DlqjoTT zI~Fa4JK}f^q~{Rty_7lV%@y$@;en8l0ubNQTTj5=^GiJUL#IBz=i~Vhp5sV0PV?o# z((h^Lye}0-qhSX?QFw&9ECInWAM(em7N3$#I~_bK&W-MUaJebh_+Ho^RyXAC>G~j} ztXJ)^y~QF#*|Ia@<}KZFc3k6F`UAbi1xT3AC0q}eU~ull@vDmxw;YZBg8}<%~MN9?19Pk!;`+Z=gTJCi@LnHMrPdXGf$b8ib`+6^mtL_h^rfabE9(?ID63`v<-8Z_-QDRa9 zv`i27eljHOTTBzWH*7S{5wFFFED!^vLlQoDH~x6TbL#hL6aPJ05wl`0e>CX5891a$ z5O+5<6Kzh%HBCzF6Oh^f)&o9zxIUEQ+s^?g5?25)0=>O>Joft21^|P>tZcqmvyUYw zFagX>a6c7fM;uJi_!Rm8B%fW&`Paeprxd3W2Ml3tmNq z=B2xB1i#;Ye-2R7D?M)k0_5gwy=$q~YIl{VNV#w_m?nK52p#dJfbpjTbM0nN}<$I;-j(F|eOcA{P{yiS2C5ZZw-O5}65exi7}GeY8XXMl?VFe2Z#)fE*d zfir>*a5F%>J(CQtE13Ym__%0pO!i^t*bcW=R2IntPXl_Q^r=m&ge z##n=63g}OxF~yHAdFd%O?+^wd%%C$Vx?8P#E{mbRW5v7 zaouuJHW*)Ag(i_^FZ+-1xGscB%YVMPyR<2(1jkzb{XD38Cn*4 zyhG)+TuRV=CXOayVkAk5L@l&)|tf|5!( zMDfdSvd1>m7`!SL$xp-YtHL|Jg&B8}KC<_Z&s(xnA?JGQ={`Oz-p+ZZK}l{`9fg(} zp{G^~mzRQ9-n;3Zqh_#FN8w$^L?c>2g2UB_r0ixSr43PFmfzTiF*E})^uB71h_dzJ@pmt0)Z2Kdb~v*X;iZ0j?MEU zmjis-G8LutubJpvnFlxVH#0*@ljoRp>r2RctaTJ0JB=00egrUw(J2lx7stN0t=}26 z)vl=8TG-gw0BRl^+$lPMPd;I6<7+x;I{asm0Kvymx3BE*|9b)60OWP>yC`Chz0447 zhAOmw+PMY>Ps~$5E!RX&OG9JfAK*#@T)qWRQJ}QqDO0brPD0k^oB;j+mGGmm=5m2T zHb5z;<&udVE6E9}{9^#u0ZIZ)1c-$1EYip%e{#RrDtIm`F0-61!6R@3dU;d9gLDc( z0D;)-!UzLgf2l+yF~B-V(y*(!xK|p^oH5Oglv_8|I#H2gF*oWkEtrNt6o||=o+S@G z@A(m`ULvkf)VOT54-c>zE*}3*D^|}6O;C_nsh5j**y>Ym%UHb5OB!y#S#kkgE zyDAHB(@cgWNH0S_tno2=t{x^n6EIyHFpVK997Ls8a7)G*9N?Eyz?_p-kRtWjs4qHXJc%>hJ{vD znVJp<%)aPnAd6BFkEVhk`dxb}dzScK|d32Y;KBZvhTCtLs$tZGz&;gt1kj7aP6G68Uv)0aOTqk*ch|C>$2uqo>Z`em3%T zZi`7ulYE+y6=R&WuIlli79Aw!&rtQ;HOqogz!xkP4TL zHpFfHC^vNH!^p>7{JGMV1X8n(WsB3>cWpo?bL4lmm8&Q(T*n)el|>Z*sc5v=-{d&F z+zY2pXezYKuWZlk%yV$~{_NGfok;IIsD+;PVX-SXl=J=JayP!2-XQY6z90f27vBp3 z679cF22&!+{6kHp?HwqX8RS>(cBUr|y3{NS6*wZ$SB$>#w=Cb@c{1PDiiU(*Br>OO z;?GgD96Gw5R4SHw9MkqYFjuGL{{3sX-HOazq6ZJ^`Q|Z2151@xRPv@F05NJ|2E~Qq z3@=vr$lU0;FTSOEi@3*a&zM{%DObz?YZv~H2o06z=6Ltq8ZmZ7z6{T~`CltCi*S)d zkqb!%%)E39_JRSWV&TRB<4l2i3iIm953ex)#(YSdK6y)nXjFA&^tX&n4JvccaCJB* zXA}7yC_4jfB#qd)jOyOSZhSqO`~aWx4=bYH?1eH|&Gj}{Do{%tcNqp2RP*y&%`#%& z0lBWW_tV`uO{)FRtrcp7dam=<(u#3 zK9`!2Q6a^avBw3D;|2iJAj^|7IrLS03hFd%_0>jIe1X1FeQbwiq^|cT7G-6#WTA>c zZ3YL`#Ye z=(viCR#Y~g?P;}W0xm%9jz;rH^gz26NtM%Q5$6oEGKz-j2K848b%m5Y5a<#>zIS`2xib!fT zSF-nbmDUyHV_NNrKCpox)W1-#yF;4avawPPjz7x&8RGd6#CLH)U*JujDfD8pv7Iy6 z!=$00GGWBHQnl?%-rteYw~+j8;65}trfQa1gNMIpuBf_8$o6oxbT_Wn@P6Y+SOGiU za^~=e)?-kI@j>P$buZ5PnK3Q(O$+Y`zEzAJ*h(*rC=Gmph0Aa*HIx2n zkDZ_V10T>b;YiHqvc;k_1A_)>Rx#t`k0M47ND&>AQejC=484DzZfx2GuC*G@Z zQ2?hVW@NBj|6z;W)GbOdS+FW^B_t0H?V;i6AINJ^)2wz$`LMk4GZnzkW=UiQ(?y?? zAFa(bG?HS?+;cw&-o4i77gmF!9+n}jq^v!to^!lktUB^JQ~hatr9 zfdOyR9Kg9PVU1b&bf2T*b22b6wL*$7p530pl-!4Uh)(aGWzcOd&(aj=mHQueeG^Pu z_D|+C&;(nK)iT-;|4|6(?CVG_U90F&H2ccFn`JVMGVq2);)**=D(UgKb7Km$c0x=I zZ9iWifE}R@GIg_lOaI}QFPGxR^T`wbBqyKYi4nCSPO^rJ^6(1^=fXRVfx#cFdys`U zbU%XE-{H%X_Z{E#?5<31`zizmN%CVz;EWxRVp^yoS%vF={vdUV2Rp5BCOOO-Z+dPn{6C7!Ea69NGfx4Y{6zE!vwVwDika)W*s~$QP zTpiDhi1P#yO`M>kw;t%iS7fgi)MXDUp}T-FMuYQpTpa6lWyLu1BNT#;nxQ|++E4@` z81yq%4_dE*R=>LAnv?O3g2kp<>zCAhB*J;8xyfqJgiYC|wOZVh$hO&sWj$N>f;pNK z)YtEJWIgZgTqUl7vuNkAW#>YS$7ywZSjLKT#6|sq5$8F}jV4(x+pSMn)o9;1w#U`Z zjpIuc=u!mFEfre^|eC)?Z%%G-H@y?o0pIGCJCCJD%{lDdunmEU0R!PQtB|s-L zUpJweN{OWwpGI6C4_^p;fIEV)J?i3$0u^(=*^VupJJu+I+_-%mJYs~s_p-i z2{4)p?El6Km!bxuSxrcGae>7fA?ZZPK$RmR-nXocU{)tIX25}8Ybj}fjgsrt?~Goy zD~1*3SW$@0jj*DrG%zO8O|r87_L#-t`rpBjypN1*nK;-}x?8hh0zw2HAMMM|$qQ=o z)!zMtt!MpYjU)w4WTdd4quU|YX~U_wvwu*0(;g1zZTw7}Vs@UdGPTDyZ&%v5B8x^+ z2_^(j<)(`VBwe=y7mzwok0G(WlX>LeC-!h#dTSJj`m$kjE4!OAo}cUTm91Bqsie{L z8VvfwZ*?KtQwtxGX{Qhhe&-<6v28!DL@wkRyIswV7SVAAXJ#k=AUZT(VP?JY6b7XU z12U~SlO`{}2?(8bpk|gOQ z!e@0BAkBb{At!8F*XCy#Lm_S(z?`%mF`Mt zbC6a9&%svQoBWojdI+l*78V((U7;^69&=q^@wU|UZ*k;DKGMxtfqMi`LbtPZ1@DgK z{c1C5TBIKSQh-kGoq;@$1vN*-thA(hjGoE~Tu&CX*^UeXN9%G69s;|SVA&|2HHNJ= z+As%EJ}g$5T*^UxlYkBJxS>8P0#f(`6f9*Bu%gSQVAi%a(T@s^33xs1oN%192}}LA zZ{3djJya>c7;gR)>MTtmxuPAdZ_*4tBdv%kB z-xPmw;o@1lOnScB0Jh1i{PFj~c{OEuqcmr_1>P{zIFBLqyQ43G{%0IX%ODR6?7I)p zpRKj&p%S(8#n#wp`X-MU^#t4(c+%Go0y=^AKbR{G#p5*=d$=bUv!Q5@uBk!#%hy{S z2EXu3D$k!xPur945oh=};)DgC_8wDLpOE6dD;(T*-pd>|^J544UM7-lU?PD(a~%kC z3dy44%|z$0S3>OTzR=Zw459fZ6cZ&?Pu`Mj@aK4uDM?wN7DZQZU;;-!hTDzsm^ zYk+iYEFPQjGHvV{MAw9It%2|3E)f+$g*u!q$~@=0`IuD|V9r_TxgR zc}ZsW9r87;zV~@~#lw_@`RBWX*9WGFCmq5^f|u^WfPe_*?k%Cot&xnc{o7Kn^F;P* zW9tmjfd$EXzaeHRY|?o-J&Witw*ty zZ@!&0^U!N7CnE@@Z*zi+!cT+m-tX&QYN>omlLhJ3Kb;jJts1TCuj*PGmwqs*zGQra zVYRnNXZP)60_r66Tk^D{KdpMc6mGlS>v_Ds>YaIP4tJt?zK!QD<$%{%#0S_;C!Egx zbXeWB4^;VDP4de?x9$3Gshm2~2Da{YHg~WOd~4n0jBGepzaUGzN1;PZnD!?UkYhh( zDW6J*pbBt@MA=h+W>0+QgrwvXLjmnhqU{`Ynr3TD7o(huZoyp8`cRzw~s} zD>OeDjj`UnACSPczg`tK>bY_H?v%)07rx&dWZHsXDV=IuFISo+Rdt_7b|Xnz?-5_- z4hs&q7d2mqkAN(MiN*1(aNn#S-kpQ@QK-Xv^!h$`qh-SQS;hFW7GZb49IoNq5z&V1 z>t!p;MH~dl-Ru1GTb;2aM;JOjm%WFp^Z9x}^WRo|mzR23VYZ^g0ThG8KDFmF_l;np zmL?!Cvv7FZA-Ev-z0r12z;60W)lElf)|j1F^WL2Fs@wHUxbT|C>E#uHDDY0mk58{e zU_Uj5o8l7S(pJrRRR?0@?k3;Xt9_T(1NTQ-s|1N*xk-*w(50%rl+nR@(ip1 zQ3O1d8ddtc#isxU@JnCtjrAt#Bgdz2C%k4O*mD*DI2Q0*3yFoj25qMl5!J=E}i8S8s}1IVN2E6vw{?s^H>KTPwv z>;b|h93a1F?B-rKyaUPV*hn073)=)){K=}OM24MysI`_ZkirSx_S?mB>b3*d2{&D~ zGDQb5*Vb{=sy2zsl3;8*6-gA|;SFkCNi+<|<}`4Xxv5$_wdI+F^KG9j5Vj4&2f^uY z$-{1VRlB2sm6hx$H%%+r*u29+U-#o#=9|uMTYHz2^yfMLYV!)8V3#R6}E|~%M<@wv6&~0fyBoHe2avG*Rx9PV2`V$eb4uOtJ7{k-OGoYH2> zu>4N$dtPaBJgo``m?+Xx4P4HDU_ccTR00#md9tGPT&HRZxT+s@_iw2!dEhzYWO0UG1W>i#ZYPCK zTCMwQDPpV*EY_dA&b|D-oV2)iYrObPPp%)X4Y*hB|2oUI#jpE$xP+#hPPSYnx%e8C ze_K6;ZbjKWiL`&&8+Bimdk;Z<+K=shTLn%kftkOx+pQn7w6OMZ6vC0OcuMuuJ2xvm z5`%KNQA9%AJvPyrn#|=guz0(K%f&A@v!Fe=B?vG&H&BrKaziewY9JP~^CP}Qwv1+) zHls+@GFdsr3xAxF6*3j4rGfh8&7m{0waYhL)X!*Yc1Jf+uA~ z8#{oVbm)A&lP30{JgT!@tOZgevsDJ2z(E|?uL9i75m3J}U#W)#+Ryd#eZD)rTma?? zu%8kR@CI5X{*w`bQ$!^|+XE#rSJ&4Nc|ZyUNV(w)db&ct0BXd^dZ!;?`jg3NRSRVE z0HPC^c7Q&}4Qzu=(UQayij{Xuo+N>N^Tem)Uo0JA_c}m2*k{dSpUCx(C{PcoDJwe+ zyu^GgO<6?+5)>Y_MD>3}Nu@B9+(id0`x z8W~foSM${9emUJq^0*$j#y$;=FXHM6Z}fIbv%u*z7|V9zp1GGImd|=_1y{AY^Lfoi zP=TsVH&1fV^sdLs*PBT#)?ZNtj+!`D&)#Bth2DV-G04#ouJ)+yGs$x?ZgR2H}S<6*emV! zJGlJZs>)o}rZF2c|IP1kv6YBz-TO9&!0qs>yGEJzYlG0eWazu+`(yHG>dIr+=#){R zxVEd`%PP&;vtE6seT5#U-(JBZ2iqC|{Irl-#6~0&AHVh+cpH7i9i}wud-AsM{?=)n$Q0~ve_O`6RZUw~N98Zj1E%RV$m@76 zKk)5id*pcw^OvmW+aYy1qp3_QUqw-e!I4$9dFG(|J!T&4i_F#n?rq=1 znmAn%g0OYIL`Epk1B%t>4w``7^@5M>>|=5F`TY``U%O~KU)nA3ywsf$MF+rwSYgf* z9QlM6V3A^UKMYhNFeUA&Nw!cm?4`~;dOmI1j)Pt#;#zWS%HzK^q1Cv;tww^|Wbxw9 zLO?(OV7asc=ys~51NTUP%5J|i1W4I2rX4k|hk#$;&mc=xE8guW4sQFE2f(h%ezBIy zIv4O!TWNk-EtN}QH2U_hXwmOp@8vBY&{zPq1OWt~D69Zp8F~yzdIxhA%YETkM2o`j zFTk4ce$oX@I=-|7ZMw!JApnV-C-yqItn%ysqWTZi9sxGmB^Njz0HC8evHYu5*GFSk zG{tm!z5sItCR<9rV$b>p7r(gbo`*gMWpLP&Q;pg()?&o*AO$Wq2!5fwDZ7%`R z@7x=C)Y8>=K&s^Rw!ho$UibhwDaHt+Yn}TCIyGj#T|{{oCM|~xUoCL7tXu~?J9?=_ zvEdqc>}R93d7n5;ILyBpqcsbCZ{rw&fFoXdjBRHVe5T_0neDN8+u3=T-zI8AwCocRhT(`Uk&}-w?M6GH3vGC5xyMOQ+xcWc+tOZ@(n$CNP`}oUW8R zMgrXF%Yno5OT*|aF5$}d#5Ei$cpKL%SjwnD8%_B7km$aBp=nV$U+5N8onuI$^qlOgnBSf0)d`q6m{)eyu16g6^NjJ_7GWnu)MCNzjaXNFCA98s#l?eErv zy76|;-n8paL}ov+zz&{#yT&FoAq}+I%dhhj1g(>R7J0gS4=@!R<1dsw7;U#`5+=90?J$^a55zVI14FLt3nk3#_+jc#qD)1CEr*)1|rQ@H%!Yg zvk*xtTRSz4kMG~E#bbFNzFRo-W@?3WJQF%zQ7y6zaS6ND#^0Qu)P3NkuIGA>@6<*fbNBIGhi!+g5BO?gzb< z%b1j<b6#Lb5OtXbo4Z;a6G$@=D`e=WP+^icZWfw6QhtKpu- zgZX)+~3HA$44i z__x)<503*x@M=c#wl6We#oW#!$6#u5GLCw=mS%-I5QysqFJA=dpUg}a6FlTTjE5g7 zWHit%h-)c-7|(5PTHm(~QSqa2LH#dm(9aKZeV^PIv9V94iXNxv7dc8|*6N)EL264bRe_oHh)z5f zb46B(z8M@g(>)#>^ry`lvppHJHfJuT%YS`0LxXia@{XIa29WuG+8t}GpKss29S+(b z2wHS3{&@op%Ni)L_lz$LR|$WNZJ;E;c;b4%N)aL`oPC^k^L~PzO?2y9B9VSsIh@Xf z*Z{(w@MaJeA=bL@^XZ~RE()k|&7;@OuSv8EyEB2fcCGjt&)@1#4!OeRhM1w`D@FKY zO%Q3OcPzYxuT%`4&*^S9#D;_4v(8PjR6cMLw!h?e*&90uVFvL6HR8bjqK>2$poIZ4 zUfSdz^#DX|Qx5B^one4b%I30}>j_5sSNW%@`R^f!@odJirQj;XZ$BR0{jQCx%oGhV={n{|BrS8=%m}@d1+l>age+N@W<_g zsX{=|t~VV{*xE9-u>%5KRDpBp&Y7KUN%` z8~_3f@W0{$fgKK9{I3KqtGFk1Y^D^DKSUiqWb%VH@&UpMIB9Hmqr>rKUlz#WGPP(I zDA5F544neepl&oEOLsU~_c1s!($RW$@f?_LKl}{;DpSDvUiU8xqr5w9phN#Z3-I%o zE7F+OVOAS*IC30WL=zj=KKy&Uh3;lkOVw6b; z8sblw(i;*v=bB6gK9zx}R;{HX_U6o5tA3S6M}@W~7AQvO?N?!}u+8|-Enu&8ds|x7 zOgQzB8;VMJ@sy6M+sHt-#)R1Nr~ivnwfW7j7_T_!g!DNjUgK1TrDWs3HEaXqtSQ0{ zy*V|aUn>4E_z`z+U^yX`Nniwq%zS}N5h35YsQU!F z^273DQF#tnhz7w;p8SMXMZ#JUu|ph+Wxv_8GN*k1sI#X9=TctnLLK8F4|*u-s6 z8>jQr2YbhN-MGLi`^T5?%vD09v!A@a;qvUEI&e(d7=Ky}8`9Sj`e4sw=;X}5y$120 zLu{;aAhN4{VM&=o@RkkSrNA<>w8?g;u3=^G_>N)%9sPHQ^tQrA#Nb70lEy)Ss@wXE z=RWg!{Pm6Aufb|sq1ZR3!51#-*H341bI}-O4!#gGXL6OHXH)s#k(!k|pDSNtacxfX z{03tS8u--x0)MmZ1&lfnTx_lvrUtfZ1ik-^WTXC!;@^Eu)9N;!@qnwlM!HxXafRS& z1hR=9UBk<@V3qRH!5?u>6FKXNV^mP#nJY1~5(uU|r#w!lgho|FPTUsbRQmM&bD#zI zh2c2N6nWTY>Q~IW8xm_WXgcPWWdNzoO+j|2k3W@rv48A!&)221dEMaJK@Gx@hpFjn zFtph43ersnY;_781>;b+J1SJ(kKt1tKI??FDa@bV75o~`R2pvvDGfI2zi9OdXEmtL zm6haVWnYvhT5i#n4oy$r`di9ZY-@R(MncS~3X(x9ECh!AjrR3XC}C ztK5)uYNPsXs3L+_jtqf1(Tb=S1{Y82L6QO0+U~)FWz>+Y{|1JEDBUmha{G2EK#7O+ z=h62*`14_K?q^-@$zAvHmzMhqqHhwd21$LZbwgEU59Tj~sLiW*2?A2yQw~g{Yt{O` zw4Q{0`?R1JRfgg34I;JV>?YeZV2Nf?R;S;&GZ6E2GP~eEbEIRus?fUZuu~^RK^!ze z$EJpiHd%+C&(ugUdnfm+L*00XL0d>NX$=z39peRP?b|Ry|4n4w*LhuwbVBz`2z@Fo zYPfnc>Q?!fXk2)Lb4{%oevf6e>`50Y4d>l=C#$)uCfXsb-==F78oz}K#)m_H^Hxp> zJ)Z;2R4vu@{&>#Qb~VJms@PBGfIQR;=L{++<9I8giTS_?TJ^29-1=t6eVAd21x+YZ zVol(?6f`#X;ZVs3;DpHR2qLvhxHTpw64^J`B*XO1QXbcrb#{Z(JS>E~v5mu--2 zA~8`&7^eiX6i)hawm|T1q}OHFS#@Z}|2>e_yLoN)_AB`qgUW(8%0!A(mALdQQ}?)z z?}MicWGWytXJo|2wc~pU!qlxlD!H)qdk02zdA+04$@vOQphO41((i2E>axl5<^%@d`#l$NWT%e|Hb&^eJJXh@xtRzzXSjy@BbHrc zzn8Xs75$ocP*I{&b2jCZ=X-;0T;jhMIRWIsw!hz-kA`Z^G7f>mNg*sc9UkZU^eX+Q z$i(KQMylrfq{iYhzC!4R(X~-qdG;>MJp6!qPtO`sCnNPCJGhLS+KCHGPA$TZ!3fA^ zZtS~N2_=o|zWlWQ|5ha+gxiUxXYHWEllX*Xez0|(mSl4(6UlGEqg`x*^iJEcK^BAt zu~x(i+e%j_|GT2*tM_V7OIMH(C-Zysefa_mmmlw7DS}cDDJ%9pPe=Hv|E9r9RJ8cG ziX4LH{g;4zQmEUREi##`kj)bL@Xg8)Et46%XjXdaXn`Wam`s^3+mHSs&!rc+|NEcY zFh^(1Xt4A72V%0n7g!1X<=_)a#3Fi9sHZf60#xg+951bBidcSIvrBxFgPatW%cp=P zNfxN1jiT;_gAkZePkfOta8xM%_IHM?uc9EM4`4kn#Mb~)MGKIyM=@iMjr?9!5!~g# zy}AUsQLf1Rfrx@=o9Lt20KqycIaMx-zEmejtnrVt514X;MJtqgs!bo zYe8GkL$c?WWOL^_%Z)QX;|!}ch&H7*GMX9F30p41#Egg)gv=Ek=sra?9~dh1ToN8n zvD}>Zg%&p(qlTaN0@D=}YYprp+~ewQ>wN{|hM`S_v%k3<#9nXn1*SG88L0C%8y;da zTu9qF%pFV>#YO|*Khz!JL+8~~-aK&U%p&~g=7^8as7@=uQh!W|IS!zKFFvU|U)$dE z7IyACaBX|DcdO4ER@`tfb8Vc4_@snKpTXC-pX=qYR^Mz~9O1sYUc4y^XW4EG>xrFN zfGUcRkYxEwBB7YYfmFh&gH`Q`R5QX-kOiR`Uo_49?Zsa#AY@*--;x5k)X0K}Palc& zM?_Il*OY0%4U8d3ozWoKAkLtn`m>DY&oPq@!qT3uHpwP7NLgN(J-#%@Ue1?Dqlhir|t7@90;KA)YU<=oK*9>w@h9?iFvHVaJ9 zSq%~U@e}DFsOc-g4S&LSc|_1z26c;{XXn%NOhqRT3T#s35_I19UQdsnyu{cGtLfOt zZaJFGD)>Fb&7|#&9Uet85u)A;6Zux1UJIj3%%_W20%#RulZ*eEd>7M z{|(30Wr%w(k3UpNsPF8=a-syi347tLf$cSM7LX%^oi=6Oq_BmXNSc=p{(@rL!-@Rn zd4Tk#6SKYCdi<#2i}2EO^1T<2rJ8VZ2;*+vl1K})#DWQga3N-|g0Dgkb}`DzUyAro z2|2dDiuE{X5lSA<5O+85;Ha381wW9miFi(N&B3b)0tZ|3MlLsm+PohJ4kC^A9;U86 zqJI!ENUlu5!+uZ>Pq=)$m`FUfm?)CW@1bixX`*F(18Z8` z`zF&4rQS{}h{*+ItQ?QF$vuTk2*c}=@kxQT{or}82HVvY2$Ak6DYRp(4#zR(4N+Xd z9cOyMjmuPC<^#yj#@<#qaZvH8&KSd%CO>K6&aoXH0%G+(4@?p(X~94-O-65jl56{R zJcqXabW)Kj{IlMo|IF`jsaE0ov_~!@{j0@XL$y%=VDw5=F9Itq?gX6A8QrXY@u)o;&@)mDG^(i8VN9aedJ&83$J`!6vSeP*i?hOKwi>mLAI zZ1qOYQQ+^Wh4ag)H8)#=#L~kbhHm%%afI6kkwc(iWP5@yn^dH9pP$gteZT6e!OP&^ z^IOl2f6YlC)4dp}A-k&~e&P_LuP5N5rW&JcnBn1V{oMpU&2VFg&=9(R7^Mh}l{C(C zMt?!<-Hj3XK~@VjQsqz$7U-6RHwHHg@sS6^?*)-z&DegF4drGC)ulQR9w5i6*wC96 zeN;_Donn0`&v$n}Q&v|`$<1BQ69vq%^xF#lIyhttx>QO*HHb2jnnLORS%F=Em?FPb z;fXt>;<$$R*<1jhjZgTs=u@P+E3VQlSqe)3f?B|Az$R-cB^XAz=e6sYLqNpwr#i>5 zxZZEb59M9(8PFL%YT_pT2Sq=0&eN2!>|a)i2<9B1{pTulpt?eC2^^kQiA?-*@>F6) zWSq$A>aWo2WZsHA(WPzZP>hXTLgmX-3QSZ(NGhw7%*|$YY3oGCLAVe`@gK<(V_?;? zs}M3FZjjICD$LH?mfnL)pDMH(z8w~9J+}x5-Iazf2dCByFCboxMyCMVm74H7x4?~; z5iB+L-OepDBw{u(56G#+2#p4pfrv8gEfL2@9I$w!E>*& zU-eM{An(H-L*El?6eEEm-kgS}kS9jk176_Zv$siDb(H@SxyoD16?Q+SsZpQ)0n+M^ z;ro17=Nj?D*=Vkmw2K83xMYw4Fin;IgF+pEk*8d7fu2wEAd4-#p)v;r#z77tc5Xwq z#;i1+gO?P6)Iwg5b$m{fD3&^m{lQJFzzsemqsU;jcMaxf6S;g6j|n53Nas8M(kRek z-GB7*^0GIc0o66#-#;mIH$Ajzg2|@D2k%RzzXunboW!*YcN&%Vt&x z=~H+#SRtwjYtL|&Y((ePylOG5?uV*pincM^>p7&>hfH@W5mJQVf1QuZIWSdAU^xdGBS!b1* zm+AA7(mi)Zcwh|lO+!7zbC(Nl%j1w`zEqkJDg$YTuJGv$M$*AX8ey29RL&OF{nGkH>F1~?dc zVy7y>>F1bLb0`T!1TgjR&ePl49Z2U(?#{aW8}09HxA$+#azL&SWH>j|Jb42)^SGq{ z42kpx>JRZok{TF-_am+RDmFZpo<&47KP{69G~eLHo-Ys=f$|_9GP$scn_lo4C(9_l=#eg<)4h7;ClQi|9 zTpXcN+Zg*u=CDyF!B!RuA?BAZRuQwmTy^>grqpz1`=NZqbzVhq0T6CRn(3Wj7$2bX zmb-flMHU*MrABc~fc%943k0Bfz|FFb#bg$Rm7xrgZq!)Gt;G!%L>OaCfVPMkJC?Xx zV%N>*EiRcE-!!O___258feJi|<3((tlnDszLjUn_$<^Kj97qwpa2?Mwm35PHJR@=H z$c6kZ(HzQtWnMjR_ODh@NWdd+VheYCx;tGiQ>6hN%#^7D`PgVSfRc^5=0oJNg#}vV zSO-%xVjQW9Y0E=A*=oGWD>|n@kncQ8wc{4YpzM_giji)o)E*?$6V<$Y#A$bI)A=m7 z?)g{<3<4c>ZFL~`1Y}|k&RWkamGXO@+z8G(U&EaWsZ$85Sn(>8IDG+Dxn`4*oiFR_ zL{+)6#bS&+YTd`t%O?tJwB?*(j8Vn}_G0OMNq+gs9ZitGN8N{8Ptq7iAT~qZ`Iu$Z)rAQ`qeBjNR-}8HGm$5O$Nn=~zTU&h znOvk$bvE0YT?#AkEtlT{{&t}Wx!WWy3)z&u4U}u_bktVZ zn#~Fe$A=TfLt;~Ot_-sN=mlzYSf{Yd*e4^-7)O8@IDlR&Q!NEdEvTI)vMK!Vn38re zWSS2-2pb`P4aV^tvXh;$NR>y`F=O-31<_=E%Bn*Yp*$8XJx>ETH-qXH=vBxUo4qY z^_nlp_b10#aE$_334G(rN&-r0)Od;+Ff#So@ZAzLE&}wCwr~``9~fc5IOAHaBQRH)kvoHk&KFW^TvUXpOA&d2jeN$vH zONHytVV`IMCB0xg-)`_din1Ct@OxtNdpT};s6j_E;zfRd{-phix! z!TKxchi1(d5=$HUAfF4apzpiOwqFkArFO*pPjIpXc0B5DmE?_;EA;A!Y7Cz?;@ z(*H(4qhyR4?{Irl{tj6rOM%D0FzO*m=J3LjA)x&exO%{6PNWtOSwQ7x9{|SHs|t<8 zHi;vO$!T$JrltN7-jvqhxcnglzFb~g(O+g&Eg&T^TnCSDYhS1Gs6>ia`Ea48Fw*6d zfse+`5|5jUe!9SeJ&&`!Ff63A^F~%=u?H5h;*yFvlV}QePG(#+gva;I?u%z6R*B!; zld5Dl1tMCSKcIJ4dM?F4Rr51|wxoY_J0R_V_V3&WjG=!jmCFR*AYNi_t{+e{yaH%T zKvo)XNPItrntZ=6KR;-p7?v5dVbezoYfgp<=AhJvM0F9roc=DKCEB(^O9%0i0mGq) znL|trh31LEbV53e4jX+)rV)rO5O*dJFQ%1GUo4bS%~fSN^%t!TTauS1+jY9@I~2sm zC`)J5R@Nm&sgT=xBtxWz4h&3zbVIfwECLESDY zQ+7`tm=e(h)bV(Kk(%R4V|}(*w}0BWPCSdoyW#&YOt?#tc@!v{WLEYH~KjW=oXGxZi;OBVHtf>Hjxcwi842 zW4<0bNq}+hC1iiOEt^u?wK^)`Px;qY=dZ@HUzQ${*!nNnjQI51&v$onT*STu@w=QHTIu>wbxCP!+;&gT>?iXaBvQt z?ppc#w~IR?iK1T|85i|VZ>v+yU`*+crJGHJ>%Lbs;_Qrkkvqauu^!o$puqz!M~;G^ zE~UIY;&T2#Qs6_m$k!~XTQsAZF&ib5E*C3`yw0&E*F6M2u&uu_>X4PsOsq*|=hj_9 z0u#uv;uWYD?|6H%nR`C2>U;zKK<+?Kx9BWBSSkF@sp#Zt7Wy?lxd?(oWOoO)Cfm#~ zGz}GP%!J%dhCU>^kb>1gGf($-{;ZNM5kt&G+boDWNgtaQ3CYb;>4*6Zds~vIs{f|_u5vW+ZHN=pU;Xm2)SJpVtF~NpKJk%p%Re(pbKj~iE)as+$7nx67I~VVqA&^5 z7PN=+*g7)-nDqkcE#r|2P@N8#q0vUTP{(FZU3a8L;8vllQFx}oar_0j4#CEYV{N3k z776s5Nw?`S#SQ0+(jpaLNPGyOc13!TXb1H>Enn>B1HmAV9?01&Str>Nd0g zQK>QRznuu*e5qd7+*zo#eyp@6hw zh+nx=L765u4Y8l=DEX8mho;LOb7{* zh@5eEV50fNK^MfELIxH&eC*t*X@Srb|De60!yzG71)o z{R*u6Iw6nhqMR9$ij4gc$3Rul67RyPL>`r$WPCk+RHf}ZP8>ex2j0$l*pObH8g4QC z$t)FTL*|i9UGQe``^&yCL^8lYwLR`-p4G&co_Sea2ACyT7b)&_t4)g)%I9rNa@b?= zGNVVZZFBH(b`|NQY0FFf7Ld5f@W1J9>~MN}-N^M4UShHS=QU($22h27z1Iz39Ss6e zbQ0W;iC1abNS>TQfyIq2r(-l-GIPT+@zBFpr=(EU@ktlJC-yyQKl(gQCN9i}SP#5l zV@yYQ4<#=l2W*ngw9L3`dyf$eLof!OSGM1Q_+&Lx3;2ow2$$R8G>oMOU=ilfv>9@6 zP50lpDfPU$G^hPGTib1;c4#{80NdtN7%>lhLGI9dRj59BYA}dIuOEfVHpJ~7-wH3T zrVH_1BTH=Y^s3FkceY<-6u|4XMuEEC3!_{7S?0DE!Y{w#gr|}2r78V(f)p*)h|?7N zeE;xqwS&6*tXz~-O#7v2T1+;|KCr#^)Gz8vL=lVQxCxRKb;eOy_mi_~5rPYeIQS3t;H1NL%&q6|g9?)UBkgeR#O!jD8LNwyxv_xxB= zUmz^T$#hV*6<%T*Cfs1@p7u7>}ftIhgPTzOJ*3sJ5RF_t>6Gppf&VwN{Z@3(0`2U^D#>Dw-gAA~<3Y_EGBSESvjD)RjsdS{M(e$wyH zx$9Y8T3%5Y+b|^J1cTAf+PDRjnWQ02vlPrB3Y~p?oqrDO6A^u6o9NT+#Pg9a~DZ2MaS@!{^o3I&EyJQxs<9Td(s`|c|^pmBn}O+4+D;1Pi% zFY*2I%JY}!A-H8?PwU}5O!%a4Gz~s)4W;M;<_jsa=mLW%+!wE=tLzuZH~$U?Q<^9=Vb%8}o9XMSEM8<@># zO{7z@aoM-a%R}6&x6$QXdrSxDH0+DrT?y2}s2bs5p5N)4GwHfMHsk7lvo}@7&Aw|b z_*l35@PiQS7#Q2>n>fUmZiH6}jl-kbl5FEDXa1LDJ28#=NLD~jk#a2i&q*q+kaDo- z_*m;>LSQLJC{cnI1{3EG;adFO%a(GA-y=4k8~iWW6$n!*g{R`OM0>5bO<>SEhzIjS*6n5mw5E5icN;Wv6=cE%FVjFgt5w~yD zydUgaFZgEiCpZvdTZ5ZZ2S2K(us(?b#9A`Z(ST|-=lE2#*yI8D#r&d(0p6xsYIf;) zCJgL`cbOw9rsO@ocS%La@yY);P&nj_DbD|qKl~#l;|-Cz4UtqVPdB!*Wdc|7h9thD z>%dm;6FV`BaSvMb<)6KS(REX;d`!cNo#mE8aCFE#!dg>$=*IPL>dS&D5XkTNet(u> z9&*7fQp2_Lv<7EXBRM5S${@(pbbhUMPY-QvdZ$_(l;?RXW|X_ZNzRlo)V(Z_AZ~me zWq>%~H{qUnfgdsN$FsWKuT++8*m(g?K5wE4Ifs)J!mcjP^K36j5}v zFN+0}bhYstNVQoo4=D(ujM;N_%DiBl(rMGciQy|v*E?{VXk9vpQGCW}$1@cQ;GQ&Q z&u8bq)Ns_ak1}_#%X#agz%XUxO$-Z~?QYETRJV3sX;F6Nz?sT(9BJ!Kr`XKMlQY*a zN$eNZu6lPA3>kjqkhBT^8aOZcrvTkbG*ZP}x^{#WX@5&WL!@i;z&qf8rm!s4N-vDJ zxRfTPsCmb>Beerj{Ju}r zs@b0&Pd@j<4)lbVHuyaChXQUT?H;Pb{%D;!0=}L={Z)*`KX!Z!FELdE@WS|F>vm)0 zrM9$vp&mL1+}^v5{yJB3IzQUSTf2qz_TeTZnDA7LZgoKirjT*8AgX}0iMx<3wnnR8J97)gFP->1-`lM&-uL+(wBYRklV}GuILqaWSX;V|$Er9=ggl`1~ZPFN2 zABzAMwnT`0OynxlsoI%5YM)(9%}&rRr54h_%Wi*bwwb|F45g!H^GS;fbLX8vr|OWg zd&Y1$tGx|!dm#tk1}(RY;-)-mPb9CxLDYSUF7oMxv3gj?iws(hr5HC~RF9&_I4Kon z>84p=b%oK6HFaq~nNp!T7l@W&S-V}m0T@kG{yyO)C$C56UNe^!^gFvBFOZcoC)o?P zz@p9z!0u#Os_!)rY!{$m>O2GrO?V0(Y%OHxn|tRVz3XvGJ)PX)O|KK&tM9o>BXbh- zu~EtIHk-!Wkr~;HN*SrVSHRe$tQPkQ3gRA>utZ?rdvTrXcOk8n6$`#}t3)m^nB*^! z&J?I+GG*)uu3#{fG*vI6A^&2;M`fHL&-%$MPcF(c(Lq8m*au_sb1zI53mbn+E@K;G z1-QNLV+go~<~w^lCR|+kTjK*J>LeyqLETP8 zlG5{$kP0?)-nvBCP?DlzN~E~W$r!#o#sGv37SaIP!mXby-?uNsq@MR(b+uh zR)a1|2Oa1#gjf=`0~-aQHmBQW$x`*b6xFKy3|DW;yMvN{Rf$arh-g9F(qqS=JH@9H zwrF(Ur7v{EgiGUr2*stnaF|n~eX+2GLhig2D+Ql2^BNI8()AC{xndz~-X4>!9v65y z*f(}r39Wf^pelc+rXf!560KqEMZ$BSZ2XbCkWB?PLFvnq>XpA)!urV)j*HfyXll>C z$HuuPSBPiQCKU1wuN!Yk@y>xzzi=!1-T{5h)Zt6_G3B=L{f~B6IjfBLG(@ix<+#+; zee5h0{g!v2uVp{M*8HM(fy9LuB__+fZ4pV&9N4MGYAg z_|&4oANq$J$`K_^a|w%>vWEOzVHKkT&wmH>sI|v5#QZ+Y?eZd4{0yT%4-fh|mXlDs zMS}&cNlrzbyW=^1OvOfMI&050b6q^~cXj5R@xu^(lv$b<%R=UjJZiedkC%3NytTAR zS3sqhCbEg;k&WP0p7LfP$BG@mYBPu@GYf{yH0W~p*0qBN1>=Qn8i>o18-6zZP@Iw; z<&4W{9}f*v^%u#oQ|8xb(5>7Hm6ORQtN{)>($|v5fG(urpF!TH{8hfIFS4gf?b2N! zZpaK#l`7xwdas`*x|iC2stf)4YJxr$V9hwL!)5N`)nmUj%QBmq2KUl(E;8tNGO@mwa@o+=h$}k7o+9$oi>((IT7|<=F6Glp_x*>DYg1&YZ#L z*tXWf*N&(Uo9rsN{k0ACGj-cREU8%UDD5>bo*{L;Kh&q84?N`VhDpk}Mlt6Yp=;&? z?eP#W(*!ehn*lPU6D6NuiGr!|QDgb!0=nNro}W1~t{I+GpkPqvuFTC3lC&l2@WvIm zl&Z@LH-{w#+sI6&BuZV$VTWbw1Nb5J)W3V;f*)Em-#$uPgt1?dNjQ+grn?g^m?_tV6a8kIl`13b=Nbb zEeQ4?lkFH5%+N0?UC13ci0A%;lVG@-8RDxqG7Hg2lC&E>exD>>HP%#cKB?X#s7*3z zl2uGx*Gf+Xcr5?>oY3UHNp5p-+$h!d$hqMb$SPBPoORKvzJz&U zyNRT5AELMNs&s_F3OdTW0)=& zP=6_qqJ*8{KY4{u5RKX#*C=MzLK_zA`7%M{hi(UkZ`6Dn#a}>Rkv6|nlZue0KvQAL zvNo8ipbTYzpW1(oP|gQu$p{qTY#}1k@#TiYj*$ zaclB~khHz!uHQjx*xBaP1kbqr=#X*AL~*_m$iY*8mB-|8lJIeK-^K{RG<)YcDo1T1 zt)F*}qGF?xu6vS#^4JfNB$W_@GL=GtyE)cfl={9lrXj9ixG?1leoLEDOTz$5igoI7oTT0o5mW2QP}H^az|Wep>d2z1vsCK-9z9xLP| zd&`p*c~tytvXo-@dO%l$2LhT^HBCSuh*Q41sEvh_-v+Uaf+i%cbS$q#Usj0 z_A7Y;DYBW0xY$4K89z4L42>nGzbIKt;%I$Jfjy&zEB-)FwI};23R#P2)-V|hI}|c0 z05=Dx`l3kTBKV7^G4>$l%W+Ol@nA8oSh&FUWT}~R8ZQ5dXe`CJt+3%_&mA_UCxV6|v~!YK?&gNw?Lm2o2MJs_); zDwd(9P&AniUEp_O-sS0*spA}L2`@I0VntS&6jQ~{@9RI5t+xJ@gwGkI#7qYl$-^Ey zAypTJ?i;pi1mCEiNgv$vl0{~yB6ee%AjB3K7am$cON+JI5EkvIU^CQ4!VgE`8Q-#X zjXoe<{iN^ugBjM9_R?6LHB;o`3I_&~66mn4w1_ZF9&Pf`7!;?MPx{z|-f0?Ou1a$@ zN`z2jzCb4Wi>WV+&r_sji)8vkpWJwm%+xVuUgfW4Sh&6%(lqzw~Y{g*$jdQ#tDHiF^ytmQv;@GD7u2u4tz=Eayse`_u2#fktu}8e-I+{EPJ?N*8v}vN}&f-~Gw%5d?Tqpiq9i ze*eujB4%Iv6=UK4^SW@a;DSCDTJz!^Y>zaPxUI~HPryx&dn+OGSCRbC#y*kHXpmjI zO8RZUw8PlX$ z1c&dG1pFw0EtyWXb_=tey_%(xwizu!f2zF~V)|a!{xMlT9mVL`wSG%I11ls+`uWL)CvUqj^F+zT$$c;v|w&A$v zmoe^GXz|>vMNgcLn5itL#VD2g8A=)J=*w&TnY(etC6T5C8oH<0^Gdo^FDg!+Di#HK zz*ZkL2ZWPR{3f2YdjF7tK-Q3CG6g=xH+`rSisDeyD7fV!w!P5QuwB$OJ3e78k59I3OIWp3J(MA4;=(y@PPu~_A|v1 zY+Hmm(a8EJG`Z7PI7O3+%QP_CZo)oO3LCaC1CqIR6_(*n}qM5huagSrp>3 z=E7I7Xc^tl73K6Sbl~2b*8%swN*rZZN#G8nBejGVwN&(>mNep~l#)*7vgyxD*|bRm zGqQ#frP!GflM#Z;6O}9V{{T`!t-c^y$3~Z>t~B_(N#Ij~2ecW)lTD5;#ka7Bc}kfR z`$lyaoLfFwc?oT7QpwW$(hJ+!rA8T`0rh8061uX<W<^mEU|fy{OgpwBCOm9k3}^;*iW8cbdFh2UuAt?XA2iGq zEh8Hulc+`yLY1iyikJ~9G$8+wG>BQBP_4+pdl@k`pbQ7S4+ti{JmD_H;}rRq53fnb zYXW%b>Of@3l;;$z;uWh#7_U=uHjc>nWzL|8jmlXy63Uit8d4KvRyrNlCqo3K4sF}a>A1ZXpF~v(c%~%vo8pI^V%P=L> zk?Gqr8S?SKl@gK;D&c2xSjAd2*I;+p_0UbkXk zYGDi@7OjStA9|4zyqbzU9!jz0C$hsE*wZC=@jzky8SrBL`$~5ExZa#oA=%!`94!vje@LwUhQg9u>s5w`}^Bm^N+5*lLMKf`&A zRIFX@hE!D9>IjdeCeAtGv;gYHRGaA_BMB6Bn$>Y_Yp3-=Y>rIx0(72PASXhZ0kSIT zo~RgBnipu`brtdrY!BTX6T_hXtdYPg$p~g9#N#;G5-;I_J>{Ux8E81dYXf-ca3tIS zUcv;>F$^ygL#85zF#=NL&&~)2J>yn6AuWIqkeay)UyRKCu((-4!Qy2ZHw!c&tbzce^p?XJIbLM+5S`bcgGOmu zye?GC28p5qt$>| z5uFoxhJ^P)@G7z>(SPi)+w>+i_(g|1*ics+SCJiq?WD?JOMQS8g z&3W{}XpDA6noZ$GOHGj}oAd~~svnOnn2Zxwq!J)N7H?M*`xaTo%{)q%wG3}pD@P_r z1c1J2TjFMxlOTO`tTJv62i<6l7B>^=61*&vL(a&Jg!0DCC=%Z8Ob!j}eOKRKY*k<~ zQ2S2How+EgDdtN2+7YJG!ypC;;GM)Nf-0p!fWfkp zyjW$Rb|zE7Ja8^_69jRJnLdRtnH+XS>e)!YGE52$a`46i-#+K@I@?1+c?cGk`;d~> zf_GlhjSWaIF2-dWIeahGBjP&(db7fq!x)I04Q0(JuqbPEZw76Msqrhq8xVEIxEbB` z3#?asLe;6b>z6VKl<3Gh2;)8_!8WpyQ86pyh$#7ZYlF%rtr)~PZsOLhnb-OGV88JHvjypwh!Tx^eo_W)i9;DrEQ2;lWnB=$o@mMvbOc)3y(V{$Sk$_V3y zOvT~VIe2&c}1u12V#IDJt88BPG95(_Kc4z)>6ogS#43P%N0QQQ$*z6ZnG zC`e}l#*gp>b8#Yv;!;2!M^RwQBs?Jel(B4zx@-^vf>)#%FX`oMPoI243L_juC7;l0 z0gSaaf?%VrGH#AA#qpIvL*rj7V8)Prn~0l@fdsD)*wI0-pxpyNRTIM`>{`ZrLYuV- z!!d^d#wMIbJffi%tg)U@!j&;z0&%G6nKxjaOe2PEtRx)xn&@W$7zu@c{68Ewml{L;9BiJ9j+5d}A2ThtV^{|$QZr)r1;WF&MBd)#tw$UTU9e?66 zzq{`8w(Z(jqwfgC&_b zypM~?lwyylMN0z0a_^lFE||B_0<2xVHXUvM4?g>G_$UAqS$o27Uy z1{5qVgM{P{ZuO7x(xgSA7fV)uY(NIh_SP6=t2UYBm{U>xdKUdsheBrA1r{ z`F1+L(JNtO1Le9BM!rA@r$G(?#GwH=3L~JgoB(L}7#pMOZQ>tNm4v^5b4UgpN* zHHmhOhnB#JFuE>D59=XQ>IhwsEaiskI%XrXHMS#z1!24pllJIb(j3P|#uB_<=+ru6 zg9b#s7qexkGFb}Imy^6nU?^SjQ7jHbkGB(dpsRmpCX61&<2FWE-$a}+{@{}NA5tcc;ouFF7d172z; zd=s1q!x^QwXRJB7^Pv`fhLPL-%1jtn@haf(x=Ry7ZPgfZ6(RIXCGY9M_Lkt~l0(X{ zewuUJ;4xk{Kk%y~7njrM#m@t%cYEe74ArwB#rI?sQrYm88MT+@}i}FlPLa0stSb!voO%7evCJjD(q?;H(k1!4#zhcC+ zlOT}w!juvm5~dX>NHb;FGbYHexRF&^cdHQP`6r_G!nBqga*7aFRvspq983?BG!h7- z3!y@M5}+PBW7pylOzV6k!wVEIk`oah-;trHGW(;VPyVzJ>B^40bR@}^!KD;yc(v}^ zhL@Tc@jLSj&FIp-kV5pxsrM^eGuj<~8kC213OfQeAK zL{Kenq0ET-F~_!P+N`l_JN)p&ue|cg7A;y}$Sqp5xbn&?+qG-wTADU%eCk;zV^gWO z?OOl(p|>x-o)9MW$wABpLWp!O zp)fFBw(K$CvZ0a~xB*;njSK_0jk+q@w)7}&svZ%ht0Tw1Tfi?z`~?rK*Z5QEI~m+e4eu550T6X zUV8U)=%!PuQ=oWdcYbdzlDa}NWn5!`p1>ff2{@gy%^Nf#wGc9iNk~F5Y$sh}V?tEQ zW&+tXqIeUK+!xA&mpL6Xg768%couj=M5N*cenK^2{Vse>w1i&+2JXD~*7GhtGa8qd zm)E^}_al!y5@@;d$}7)5|9r>rlf554`LuQ%JCcP#==dS$of$f^&ae++CfRP?$9 zeyjg0)e5T`!*B^Es9i8=J2ogy)C2+b^gWRBxj@9)KnHty68Q?j>sGxEmfFcih zWqjtc-(A$Ld*=vXet!PVH{X2v>8CT{pK-<+H{5VTetvvs$DQ48z5MD+guw=^5ebm% zDagFaJ1Bnz?*g;K0R#sDjq&1@3*+YGKv4K7J+2=ZqUS?(DPA zUcGuXLvi!XH^2S%+W^qCS>rc9{zqY970xSXQ?VVb8*^VZtK)hQRY|6u2`W>8x|z!5 zi{^SFGHQ{kSL&+87Qw8>lXLZr9iup>w6wHChYkY=4x~_E;J|@bUU_9{X(<5Q*}e1C zH(ViBrBGfmWQ|07+6u{Y7Ao(=^c10sL+XWrS&j_0+8+ng$rS07_Sf`+vJ^qQjL!gG zV0anb;%bfbW;_uu*4$hx+n+)5c$s(Uib&uvL(pHUCN7nL9v~vTCM`b9m_cGLN)^wD zAX3oNh_V(~hZq5houp-uG&@0U)P`BMjx}d4%uqS^tP4h=A>!A&npq*8GA1$jsL0#Z zfQ4^-1HrYkc+vl|@$T2Rt9ueaqB#OFLt$|wLZ*=-X=Cv#C4|MxD2~LN5b`iBe&y8h zN+5m}0V~@Ip~0{3AtWsUc>*+nXR$ZHcyc0Q+YkC6rgMf@q9F7biaDk5{RRnALLu^F zgj3eM^mz|OOCBWMy4A(T>7ghzBhZ<|Lw^ap%24b&{;zO(-LBZybp>M{_{-f-l znsd%MXX3<(ZruCtzu&oYX8@>>SK;Xw{(R#m(8jzn;2pwD*lHF3bh_!vdmiov03{_Q9XfOvGGvH@ zF=WV)4jno~5zaji-+A?oR}dno%koxjW}qx1wyA)q>9b{+{uNFr#GshI1FEgN!cmDL zJWQuuRs7s3GUF9RC4`xm5W8oC<0Vuwo)j<6T!h|ZybLkH%Zvkgj%1B7a>(^5lRS097m@N@ffEAI32m`rkc4T3uLdM3fVq-h z64G_Tup)S*B*m<_q>OTy6z!QD8b`8B4vLj__MM0nU!aH}FKG*sB5Bj^L?mgHOqm=W zEVCG$?imRF#pXfiPDCE~XO+pJ@YzUJt3dfhY)JBvNEIOl_6CI6Fr6ciG{UEWywF#s z!t6xf8%P^NY)skW#X3bKDq8Z4r)5+WmMlNm8JV0Ng;yDhOK(+N(UEMs4ql!==n>Td^to z)`~p_=#g5t01~j~Jc!Pe>`iO{nf)#)g@skre2$NBnm2yy<9`$uR>c%*)-^G!*_PL>f(?&G4V5GxRJ7~ib25;$mAe4RYV*wB~s0(gl8CHJq|B|p=e&3 zWg1?_gU}MjtM#JvNdqEOY9eLJXU4LVGdY$5MnY$XU@2meQTadit~=0+;`r}U6cp(o zpn$ZXqN1SK6&sis3-%gIG-~X|n#9;+uTg($j5WsI2(fp?j#v;85Cuh~38FONz5V@h zx3iy_oxSh92S$kBQNHiW?oQdcotd58JMnnNLigCO{yw$`7FslM!-KBA%9lC%`2(Zvx6k@EEnRO_f z8_?%aarVXXJ{8(tW`(gPAfW=$mI6+I5J(|jK5v?uGp>L)J#VexnnM@Z&~DTe zp^#U@Lz^aDe+rmxQU&aNMQCbAQA>bUXo!4@7(E3VlGIWIWJM~N|LlKA9_K7vxNx`K zcDwf4Yj@jiw}lHA8e`5l_oV#}-XoGp@(Jb0Dm(HoMN2)J@^%}mUpin#$nbhtz!Rdi3QE-|EE?7}L_S+GE{5G36Q z)%6TQViGBx%_zbPO(>55$BSa+%@LHICpOOf<)YK}{mmZ6m>Dx>TyVh!H8nL28#TP* zh6}bGycH7%e*lH9rk2=XHMIgS7iWwVL z*m&Zlu$(c?S6St5Shp5(I1k;jO7m411rg4;dxXGZ>gl4!Msj!( zPw~pxbY+TH*5G)>a2zNIr>r+jbn1+e{i^jJ-Qy${WRRG8N|DxKFHm{l5>X#beN?ZM z$&u6viSTREP~soxc*M)@Nj%}2gOW@`G{jEjhB3wz3!PBTZm1~TK~6vMfCtwnXO}N#EpWOqM$fpDa=_Yy{dsID5q_B zRUV^@nm{&d1B%O@QwXcj*C`M)8m!aUR5sMmT2&OEL>d?px)RE2lNixlqtFn;6p`cQ zGzldS)vbm%o*3k(b;Aoy#E(EBUOuhKFawB3k3Vqc%=3by*052-Yi_=1+re80X>k6; zQ^H#p7vX7Sr*Ta<72U{qWf<7aNMXdhtDFkcuto$ZjUqQF7_NmND|J>6`w|rwM=#D- z_OE&K=CnV@Kn^Er2iuq3WBDyMiV9g6Xi@DEC?PacpA+L%`r#TLGZ;# z3VkNqKoilELPz@_vd3R8I-Lst<)YK}J9rPEhKZrhPm2C{C~PAiyqSu4Z8$A{LX%Ux z;(3dt?9aplq&`cGB4{SECJMh1Ve+X_Sr6@$(4>^;%YtOhn3y$<7*3;bYIt)RR?%c+ zDB+KiykSHr3aNEHS8SRiGsy6MJnEBq z^A@1o45&P)Ot@h!eqS7_`60_0W9ci!V$djS1D%b~7lET6FDb=K1kzKMI{VPLbTPDH$m@lMHy`l?FW!D3 z-f-dtnKxY7QX|LUp5~}LPU}C>YrtEhz5mIYzUbqEbBBO`*HFZev z5=DPy1mt!h_7Y10Xb3%u!Z(0Nl<`2rE4D^G;&RzWYNdLK+yf_IL5`gD$S>xDyOrP2 zrLVB(C;4w~>J%wpjXxt}g=DfjDwI0;hRH z*|%)t@2&DLyN5}_Fv@z5DAOmK`r%d1{BnbLE`=U-|oEm z+B+&pcNv3gF1hiiCZP&t;mF?{|5KCj^M%`P{O4^q{`04%;zxnE#y*P?64@_rQ(zP| zezdfPS!D1iav#Ba6u~ko9)kBtgcZvU6uiz z#%X{udvfyMlB*t}l7r%SBNAL*$g#E^M^XS7E8{6p5sEMtUEB^{uhXhW$JzB66<W32-qzY+Ns4E`e?pl`!m41}Wf; zkO_OuQAoon4WrhU=0*S_!8#UIT!#n)a}Z=t^Cb_QCed}$O~lj^Qo(gEa#k6d0`HXj zBbt=U?HI2Fk!EwW>H`t!62C0H;;4-x?=+^mawf;HAS+yi3n4~_OU6MsCBlA5MF?2| z73CQ!f|#d)Oiq3cVr8nJQ4(es!a2m~B32ING0)&RpjrQnnLO1NWpWH=R018=8HIU> zl>u$Q3rq+hCNepq5(+|(l}WA=RS2Ig6pSW(P&}1mbYwhg2GdVKQxbqW3z7lltC-1w z`5s0K%o`yN-q#*4&!|xTP-Il7uf@C(6C$?bDGbj-&`u?jlhk1auZQ!m8CFTiRj5K0 zs_;|7sP|sI@3xDLF=A{4p9E%nWk!X~`h#T;?VuHaiD~X3V0l4FVJ6403=*}VWpcQg zht`vsoXE~s2H^>yN5P8WV$Mhcigytic%@BY=YqGR77}g*cqVo9PNj^3gOGBpbdLhM z4*I(8-YE!mDSM~E>EPW)I$1K#@wG~Or&6|>iit{0r4pH(gcDeKPYwivc(KB2OJ67V z?RlHdacZ1j^yv4-coe4Oo*atwob$*$TEDiBueQvOJj0NWCg!YfE#8x3HBpZ^9Tf`dJ0JOk8qg|ZZKp8Y-X+8_7Hk<%d9LozK z0SVT4&44UaVzL#wpk`DeqB)HTAiVy7skybO4g;bMBih1gx+ZA5Rqz_Ad;D<|&w@As znj&pTN9(3Ztbmm>WEm2AX>;iLswsD=@tbuWo*VdGb`F z6@ymDdI}pO90TFTm6M>>q-w>YLLe;5tr!FGSfic!eQTORfZ$)q~mR&M(#424;3QvJG zBDCb*7{*csfEKt)FhrumA}ZOFAWl`G;0-zPpi#~{fISH!y&`l(Du1|{=tJ!fPu-M$ z0;^DkDpX+^gJNr?T2JAEs0}5C=g>O0R>H*6Dd4B}q@RNAfDMQ|C1Jb_CjbwHwWogr#{k)iZwP-v_C(sIG7G8r3!IyRCcUf+&84C?%!3k#QuQKY!~Mig zb#|(Cu<8|uVnx~(QBnv}`3-BP*ee!t)3q5<0iOYdAUdj6<~uq0FPqbiPI$nE;jQ6! zqO(vaninZK-=M^-;zv%!urr`QP?(67qw*zUR;$#Af?r@sXOxRs@2`jl#2+=3V$2E_ zRl;Npt>It9&E&*(0vS|SDKL>T@)!Z!Qi>E$oXJrLOxaW+1%+qrl8V7t!EpW?`2x_i z=yTSKh)BA1I%3(^pg63937K$67|&YjCgl}Z0Lu@x?z6IV#0tNh@`@ugTIZL{ag0;&NQ7 zR7NE-Pl^$N7Mc0s^s|ZzUL!c=<}wsn8kw9j))R@2jEEqByAHx*sF2B#{w0?Sm#ruz z%_VO!If;7_U;?+|abwzm;^BzoWSD&15N~`^g(_5` z3dV7>;N<^VR!tfa-AZz+0d|EA)fyRm=uhKiW?EQDhPS)~2`{&YikS|C z#U@j@4v}qSHg*Kl3f|AKug4KtWV^%HLlZaFE6G>23AC`?- z86k4YTh2lN?LSt`ibOF2O2n)@TP4~QMR^;_e1MY`x!^=)z#Gc*&WKlhqDn*}Nn}(2 zi86&z2ta@oCR`PzmsUmvx+{`M>T%PF1Ca|w7p=#%$;mJk6~kJEDpa8g%PH`2ULXC` zaaN9ea_Im!4iUx@86?BVknRDm%A03x=tfgmN!4hU8{D<^7eIZq2}hi2caSqBnW5gG ziqWbBp2-3JS?37X0qWClxL%w=ip(@L7@cU=7?@N@iD~0~?chJjC+lD}WyuLBxxT3a zhzt^BJvAqVD}&HuuFWZl$P0;}B7l;~DV|zR#jJEo9P()cG3hP?a5+C=qM%F#8M87f zWOym)!GkCUw31JDX!#8ujx#}^h>BuXojX7w2(WQMK@?S2iptPhbT1=R@}pK}1jH{X z;dD_OoKQK3rOpRv6)UQWP(oQjbb{rA5R(cPYsiK|>-@e=5Sn`23WyW>CsaOe+Y_K^ ze_H#>J$+SJ=3&om+YZ^f<&dpgUU}Z(KQ%2?sKS2>gwaz+KXsgq90A4nsUs&hUQuOh zqPB!7(d=}LqCzG|p_B+%qD&5_8V;CY6b^-EkX^XjICUt4Z|OmEWnZ z^k}(E&Qg23S{x%rL6?e{1*v5~%j9S=D_{}?$k`2;kVvn3gxKFG*OcJ2UNLJl3n7_g z%!Iah_h~a$BD5CW%LwJj83D=p00(E-N%EA@oqD^P!o5_`3LuqGs=4eiCJ4u>3X(NE zvcIrNNtp6Sog#9HQzloAg7K0!_u496^QjiuFFK0O~3ej#kQ+cTDuyReI+z zmz#W8OJ#Dj)rV~9pW!FrR9$^&lp7o&AcjIRN-h8`lf#e7y%D^4PYxbxY!t_=V!dM& zv$9zaK>$^d^8p?&BS1{msH}zlELhno>haiL%GRP7m=0K8D$FzJX_4ZUow^co$tM6f z;TSllTr(_o?!@S;ScO`CB$&?)Tqj z>5Cy-w;cB1jb*A_cJ?7z`Qs1XDNTuDwRIy*fD~%>JnwnS_ri5RP-2*ELU?>#ptecX0 zW^$~Rt@uAk0OLm1z=k!)P`v>J(5c2aby3Q5L%tQGDM@223dG2%u-dVEAdNGjz#ANc z@Z5wBL@GfH1y730mU|FP=mHb#(~2M-i!yt1M3Lw*lG=>S(IJUScwsTs`ktI9++yjB zC@xoGM;@~(OD_?jjO;}4UKvE}YcSB0F)I&idDIyNg>hu=#39t?F>Blw;E(4Rw04!a zoi&#tfD2KH<55I`07e3J1(R)hPfoxT6ToXi3yLnuFh2d3Gs8))3}S-9$s0~7t!8qf z?qijt>V*n{08I&XHUt8cLh+oC=yry;@Np9gvE0JU>0iNY@5X$u<~IRIj~=TveOb+DPjVB63n79}DR*GAh*!l3}M&r^c+M z2w<%QsQ;cEtS4hfBV$J~9}p_?11nF7FG`%N>1@>itYe`%! z7Eus{&`9xUuu5uwu~U6AEv@Z|8I5!%N2QH=FY1Z-aa;cLINYh7Ib+HjuM8_LKlJ|V zC=i-A_q&A)=KqMqRiR!mV_FDYCr=tT@~vl!%MZQ(2C|j2W=<__xU29#0df$gQ8QhH z-{jVg23IRbT~S|aMU1*~nwq-@F#()BZZZQ3-GpgIv*h3oJ_QqA${q@V8D2f&phPAo zVQZOtAmt(mh?o(VMA<|pN3^B89!jqYn$pUqw^i<~ZV^^=53F`dOVn^m`*_5E z4KHTrj4_YhSJbE9&zbeYllRc6b52yR!v8pkalT9@hucnW?Si3`qb_ewS`zLaXoGNJ zCPy?c2w*656Q&x?a)Z0{+=LMqUP7Js}4Ji_CX#khwM1_(q)8f)Z2fgo8s8VBgigUrG8a0D9aBk7a@pn>xtbXGBRe4Ub?7gHoI-IS9^;}u&3JOo zc%AezNA0yiwz+Nq2PK{_nyNwB5?FC%3iA80$B>?6?lKn#U7(W zyvFy5^$bK7%^BUIMIK}9EkK_->zG#qKC0?5TISV|Y`wQY#@l-}hB&3pF7y_PJjTE& z>ptpK?lCfNJZN3;+_=Y>@D?h1j8+Xa6g5G-@O*I}6+K4P8+S=>+!7+k1e$Z2>i|A8 z97HC^!5XCyzMC zjKd%&Jme&#@*;tN;Yp|kAA)~@2#APc|iBMGFpe8{vb>)+f8+(kM#JZe9ya(g4`>s<I_sd%| zr&VYDmpO=2F^~M~YPDod&Eo$&e6zSXquQrGKafUM46|57wOrDWg^H0e)R$f8dvaJ2 zK@oQz(ix=4DbP))sH_=14U`fUGDyNjj80A9UYO3b$H=X&bRwes4)nE>

n>Q|(As z0>ODKvxr7{6UD zm@!5V<6AM-j46L6w1yQj0)z)AJO2pA2HEA}@~91ca!5=J2=R-V2*z*})dvC?2{ft2WD8NU!^((B5+Hg2saAAJCe`H3 z3u9P>^c6r%7&v*uDW!$@RRxto2}k@?%%H?`dm0;2Kf_~G_Y$x(=Kk^tddNmJ$U5Nu#7lS3#(CMU^y<+rObdKllr zScmIOP3n@#Nvt zUQoSp1Z7VStO6|BlVjk-DuL3mILJ8jlXDeqctaqfX zAC>mx*sxQ@L~SNlSXz_X8zW_l$RZxYby{H=Y5}MDHW6cOyn59bYoh&8Y1yI>Jxiuw zg=K!-3)F&yO;}+`tA0I|LFQDI?{_NJ7E4T9zoMEjdUZ2F0-ysReviuL{2P*OkU78M z;CTrd&&1vsqSQeS87JZ#*TjHlaJoYj@RwC)u@rLWSyesd7JM@L9b?SwS<~p^sP|r_uFVFcU!A#_qF?PGHB&yEy@sv;GFx$cS_GS`+Yrm!uQ|J zHpV>u@XdQ2bV{SfD^SS~-+g)Vq_Np?zm0bYZ;?*->8zPwe>n1$$&=ZJtJdwibnnx<&xXDG3@8>tZEekSkKa9h%)8U4e$l=|mma-0 z*kHhR*+!%9=6v(&^ACUU&Wr2x+j6Jfj}#TBPyOPlp|^cA{j1rtrnhd>VfC)PyY^Un zg8|zYaM!X`JKB~uX4IQ6JbBNY*)tCQ{rMd_bt@)vS%Kir&}a|)F~S6JFegQp3-;s~ z?$rdZK(VC8sQX6HkMg7iOX__#Fec%;V!14g=fr(-FHmm3%SbO5P>DS`hOP_CV;01RIJ=MgX90Pq}=(;z(+*2jlraVS{=dkauATh54nwL^(7dMb*0>(nc<7Y5mdX#K!=qeL;F`li9;3^he7G#w63AHv zXfkJ#9uW|OWjy3SbU) z?`0EbRGim6&OJ#6+9{H4=r7VPkQ87zdiDWh%w_|3J?_l^OVu*Q zO#6D$b(b9b;kz%3>$X9Iy?=AsJ_ny>8?bEud8h7?g%$*N-E&3Ceo~n4O8?YYypCEH;8AX)vR56`lp-)*?JMX0^{azPOHP8r0!AYs?dxOmD>ZyWk zrKjq=0P63lB8AL=d20M02%*BM<#(_kPA&VJ{3qHZVDOYm^3(EQ={Hog7H$y%0&yO^ zmcSSI0irECHSt-fBPLsjSj$gAvO-i5a(^=joeNP=BCm%^LI&32DbM5}m5MSbB3ER~ zstJ)ua(%0o7t)(ZKuWgaS(XGS4TKX#D3({%L0iC}Qtp&@?Y?&34Yz68svX@@Tb?lX z{XZVNBSts`3+B(g@Qi(i-hTsKnf7(r_fUU3Yx*$<^n2uh_6#fyGW z>wfyldw)D`M;76Dcf5h9dF+83#SxA%#?{u|cl*URTz&#xT=c^N z5#bcT#eY8F>I;v=2!~+$)GyCGX7Je2Z9Eajz3xnma0n(&7=7~LTYWR*>l8_|zM1yHh?h=2 zG+kq0oYB&bZQG5}*mfG*wrw@GZQG3+H+IsbvCYQj_x9fJ{?6{%&AxNy%siOmONY9H zV}doA@AEL$8TYa6=H$0NLhc_R6u{okJj{1}%n-ouB6zAg|SS@sPYkOuw znsXr#Nr-lSg)!z03pL8DqAO#?;|V^&5G#?;5a=?Ifnk)Omtc&*IgfSM=0uUhGa6OB zVXp*8hli+qJX3wcVZp?5nMd0}4s6mpD-wkhw-vvtnRGG@nkrr*xUlxu_NrlayBIYU z-O#U$RnDaHVyIjhGPnH<(wev%pwR@AIUkr277_H0k0RE>{XE%73rw+f0}Bajfa;GV!*D8zj1aW=6XOXN zA@LnO#F(pAiSu@ZK{>48K_utg%76%TV&X5$FEMJw5=>=U`Shlpd&aWpAXR! zw2ADu%Ymt9cgh1qm@?nz6D=yhO6?R3g~;oAx@c4TL08))<37ag0i+xP-k03-KiX>+ z?^CZeyRp-!E#R)?Rf8wy9MOOW->UQff{TuM?RP3nu+SKNIs*U z)xn{TecREr58(!|D&Sa|w4@}0d~0wNcpl!8?+Nb;Y{Q1zc=4jQN1<0`E3^#%-m7)1?rQey?I%!nn?;py|FFpMSk226 z0HXUBqYv?XK(a7~_j`lmFNUQY%Bnw$i-tlAg(_|A-gm$LO!+tn3LXw{iq+B>_BO*!o^J3~~Y+yJ+};?KUd5F_8%;Th*Z^-2)ImB-5nkA=RqJsg*uEw37LJ}Uvu63IGAyK{qIptNxgBdk0=1#AU~VzvE)S+X5iO?QHx6+T}dYGi>ro@2{A z0yKOXht=dXz1m$HJiB|R3Ei^Cx12w2^)X?7jPzqhlylR2Bfp7nGFBtUb%=pU$A)vu zTx#*09h*peYwYKC*5@Dur*)S|ry4J(<*Z@-R!+xGS+GvCwMC3mOAGS4+cSUL2Rq&E zwqQL@^F$#|2#(1`#|Z(M)!%y#{`>VcnzX0gTV{*g_8%`djR;MeW*h!Gf1UcIyO&nl zrIS-*m|c2uy$-y>dq1b)aY2#aZ^w%j#ti1y?H8tTuj>UKXEBf7IXXRO6OXpx_-%dP zGCTrWUN51LHa&p4nDtA*<=-PF!K2xqTLCsPxYsW#KtQ#BYim*7hwWQ}GUdBy-MUsl z>Sl-EA?o~m`N&54@Sm#S+m+`<9eqDsAZtFa06|iRg+T9dw z;K%$CvxWR(Kg~IuF*6*%b=s}5q!j>PHP_-pY$7|eiGS`jOLO+HMhhR0&6p7|U^deG zb`GwU34PJATl0-rNQ_hOIh?-Vs!JppD8tUNZ5%T6wyaK>S~tl zhW0>M?Ba^y1X1^r0npS$`3$77Z@)^sSxWD@2nda&(cy#d@pJbO;`|LhpzWoi5@qNi z-LvCokOikR87+e+CqN**%)HAZpj#R_34-tag>#r6eyxw1??MINWa{HAOR;9SG+GYB zZ`@_eFOh!`@wjhVi}`nKXDfl(?OwGMDIp!rmQ9M?wFI@AE_k&VVHRC5VK$|=VFO%ADHvR4U^=hs$?8j$3S;3i)>kjP);@b8eH*?sk z(`QX^-M_nJfG2Z_u~!&%r{SIx_X z-vP)wzE}To-%A7WZ|4q&+}i|rR-2{Eq}5VVWr5zSTc~g+f%E)3F_G&&ckYsAvNq78 z+tHMo$9Ul_hUu~ode%=*Rg>TCItmH?26AipmTa&GfPpjLKA60b~atTs>3B2ux(lr}^Xg$!+0YI0HNk zfD_>JpedZO^EouOZ9Nh#kdDL3`>t|H5j)2BOiQ-MPtam9oZMh64*Jp=Mx1Ln1DMuN z=r_KgwSH!8lka;@9B~%*Eq*4EL+fNFoiD;j7Dz(*EWPK`QVe9gePU!k z-)YQvg1gQi9$TM-DFkVHDDKGgWqg=Ws%5Yk6LiqF91^_hQYd{k74DG>^sO1^DeUC6 z_zndyaP;Ovpc<)BQGe%T^+KR47ZJ36XOVNo(wAMe8m~m*7ZzQqnk6Coh%m2JaAZXU zS^)J_4y!L1|6byrFeRok3gl5cT18$}0(A@gjPWe;G_xYfNz@y8#nG-gAAKr6w^7*Y zRyen9N32t>QN*$=Yt90$p|!+~n8jB?)DP`xEGjq|5IWyD@}kvo@|ng}+A$<79TDHj z<%Vk-gOdn0JeokM!$p`wQQyfrz~*%n3#$}C%SHi7=0@EyqXa_=Kk5BVn>Pl8^jjEh zo&5o_Q%*FOe+ABE=y!ogwTL@q`#uA}}&&=XJ0vAKn#Qs=NdbqB~f9%2yJ}E;Ro%S%7v^|oayn(0n_%!rFJA5-+ zd(<(d(;Qjv&~eY`s#`uhgf@6?I1?;i7!BYu3k}c?lW{t2e;XE%e4hIga-)NCHvOO? zav$A_ZX?hc^;na3!m^R9TV*EBxsVb#a2oX1<&!DtYI}dTt~9z_yj~7$mMdL$%kZ|e zZb?6w|NGoEt^jH6w7D@u;J^S-rUb&vrkO?nx6xCzT__t+U%Sl2{t(kx%6;T{NpZ19 z|5<;4&`vtA!;=H2a|G|44_pl7>=}PfEoxdT5y+0pF9gDYyyvy=BDTovAl?YNZipFV z-c(B11e!=s7@3J`_yrXfR3J%rHEMeSA=XrPdWVMmBX9OBSCFNE8OjYH8r%enIQY^}~xEggs@HCIWgm!oYN+J2$ zNsF407$5T*ZFr=piA#BJ@9UZhPqK-wuQr(g(m^_f02aETGA_heP1z0ti3C&=dQg;b zFJ+QK8OY!u9txgZlO^Gp+1V#-p%8LJQx;mud{#}-ELSNfJPWr7i?e$~2j7>q1I}b{ zB&Y0k(u+Sok0i1`S$O?#Q$H0mb01&$<*fFc*=7t!h<;V=d$a)nzJt*wMXd4~co zC5ro6{vme{#2JLWH|xR891cr{r>{7(xo4y_}AHcUt13XgMO)^5i%x?fSugJFg!P66@nGSN%|t0LO((a52Xs`0XM zlSxC(RcU`3%fLw|3*b8OwKivy?i5xM&+o~;SwmEO6A0Oa;M>KsnS$LUJEK?COhBn@ zpT!FW$!w&zdQQ%(!ccmn7V}S#2WcRZ#)d($g`i$f4yGftatQ31YwTYe30UXn9yEeM z*wL_9w&Up1eX)0ZbHp-S!JOn(;&`^uSST1q~~q3vhm z3*Y?0k;Z^|8MLfB73Gv;BoqA1(O^@@pjPdfNn4b{_e;e$I z_Kb3nAMP|WN%>&LC% zRuhXab7cUe`DtWyaklj5&$`dr&=7=g$Dr_Zj)R5#`{`|^3Z?(sFB?}>1A7jjd^)#V z8qTYz9EB%YxnAQw$90 zqbVku&t%2vn4#pj0E~L>e^wXxMC%a?=vJHHFOj?lg!@FL53!E$affj9?cd*V8n_0wA4L(C6VW;~ zRbC|q^$VIIIdgnPap=aT9J|8KBd*H!Z!o;+8g}Krm`%`E?k9G!e=wwwDJ)1~gT9d6 zsQ&1bZN0x$wd-PUDtt zI0>qTOZe4xhLj92&c{%z$(_INJBWrqEl*70V)q;*rFu{Xpz}q?*ZRJ;$L~0U@;(qE z_g^~Go`NC8D{qZOYTh|$J`!XVwZ(T{VO<3#av?&3$Lq6vcu~EF=0@j`sDxkcgTxEB z-gV<0(};Fwx>5CHZmRsvOn1KN#+`*)Nm>^H|9Zo_S^meDWWG8y)|9RjZWQu*UL~j6 z$gJBbM)54Q4?ROg1{q->Of38x#o84HN@Lu_;u-geh^Ib`!{%#WEc#ZpbkDuP=!hpF zNR(FiwK<=t#UVp^}IB zc#-cU{@EWTcZ`u^YdUX}#k&``)Efof+V(-X&^SZIAXaaY6lb}Bb=|0O9sjzxqpfSw z7!LkI@?@JGm!rx@$E1ImKxfnV+u!~+-3a%mFa#rvMZ_QEBv4J1U-qGPg^{5VH5{ix zZ7Z9u;JW_Mvsx;%yit#)DRRL_kVi}F=QLy#DpVRl+s6VJeL0>flIsw+;I}1ArfHwi zUjix?A$buwV3ax|aIinGiOjGFOzmqH4f^%qXTFVH?t5|+)tkK>g_?fjjOH`F_#u@k z*~C6e4J$v$O*T3PE*d{RG8I(RS95AX#!Srz{~0cbC^B2L!X?D}o6IG4$cj*_CsF!h@!T z5FD|C2}S>4wV=AMH(?27VrFJ~ACk}&C|Q%@yW-ziBy)O94CzM|`nyyW`AJ?5+C(J0z%gmX%<7y4b{h>r3cz4K%_t|Wh zEl2%rnb>&-arQB~ie*EwPLMsy3=d;ymJGbtM>+Euw1_mAF)y<$Ge26#pvm<+c5cwi z{O`S_OAd^HqwS*onn?k%+pn_5{T1lm?j$bL>?nAB2t036j|lS!fd;rA0B&MEm}!Bw zW-|MAW2;`^{=kA*u~$*mLDOj{^;ol+4~Ue2&g0`5!S@PNl@0G&N#I7&D=2%c@xUI} zW{H_AxJw5M8Ut$DK&;x+n#Kue2ypnNAPvL!sjnR`znK1o%S{Z5CNx1Yd41$K{(!_m z&>LUO!RAxnF#|mlj{kO|g>zuIGE$l`Gr4?-)D?%8?9g7+qVd<1--$nhj2n%w}vq{ww9cK~iEj5IWI&f6r=tMvG^Y)Bbyk7Mv>xDgN8oV1D#x<~0t zGI~FsN#edEca{bUl{F>wo{7Qw^|-ET=&cYy`J%+|2C(81yTo|=+SbIx18rrRS%Oei zPlPHoqmI(p8Av*YRqipQ8xf3IG-5T%VX16FV7^c>neYr~Hq)3l&Yf;HG}$Q&q}Gir zk_?w4%IdY`$FuU$cOm7sYqJf}_m&)UXY;^E1)py)9ILh?^0_|d#A@rto&19$HT2&S z+)ysIEa%v|iq9*O>jyPL7%(&VtkzE?9TX;F)otfoqwU-cY`WzDWXQ%*{e~jdyN09xl6~CS<61aTSz^^F2OuclNBNiL*22#~qp=v?_LzB<)dx8& z8!n)9?AuS`Z0?TdOV_l+&ljuf5T|nlLgSn+WuSCyf4mAh2rzxSw|$%dA3V#`p-3aa zGwI1^Jxkd_EGf|fC<>r@8F<0BTWj&j$JDU(9>0WIOr4IpTD+oSUTL!whA3#-n zuU#Fmbm14)do>E~p^v3;vf>kb$gpOfujY6E94?-~>VC&7CRoR51EksW0>5xZ&W*=k z$`+Ql85R{rH!;K7k)3uS?lY@?nG3NW#WEGbF!U&ZKj#>MVbCDH(?mT~y#L<1t zZ#cD0y9pF15w|G^h^foT$(*Mo$tp$)ap`u)(S0TWhly*2`LJ(lcS!M6#VEy;#BepL z{vy#7x=<=srg?>{7}|;1{3VIuMB*A?H3@^4gl8C%8Jw&&4ZdbBktY}Pc}NI&B(%V9 z1*6@H_H4D=3Lj1cP+EjN;4y;P{JntAaordi!TAti>saeDTcdYiwK-J9W7ZLV*q)F! znNo(-(DB*Hd{qwlajP#$>^Zp7yluV6IsZpHBcH}!ZY9QQ9Itj}B);Rh>TSU^7Qg-} z{D%W!!%)4{Gyb1Od`RAK??^!JbRtwpm zr*8wSu$lm2eQ^5M-;Kns%=G=Yy@a+_yKP1@xu>uMlGBLZY;h3MVI0C? zyfatuB9Gw1FOR^>4|o!3JkuOvKF(a_{XwiNbLZu4C6qWE+l^AKJKn%)Mws_yLOYs& zp@uq{;Ur92$B|Sgp6MqQm1paEHW{jA6J#x7$9F#Urfo_o5I)m>%Sea|%#^A9!15?Y zZM}#6gc*?*3673g~^N- zr!Xb45W@MZNp7v+95lXv3BMu*N6sw%NPC8et+bgDEZ7)QV8X2;bJVXAD9J)Re0i&4 zkukXf5!x;(2GPTPn}neUkJ)(U%V^L?uuGk65`)WSnGfvThc!=VH#+!QHkWBXR&oUr z!v;35x%N`&I%Wr?fE?XZhSs&c1{0=!hA%XIsh;`q#A=Glex=L3#9iN6_^94ah5exbLOuujH(_jF$jlr!6*}(t|Rczw3bL;K_c?{h4VG z;{{Epf!haay};)ckb&HHjk4oEw(n~u0fu3a0DFEp@|%`PIu*zSY8|Z0Fn>6iOIeEZ ztb}QpnhlwJX(`9LB}_4NIbIQKw&Um= z09H;~l%ru-O0n~ze7#CU+plRbPByso`jarYM7!`P?F-0ET%88wpeu=Ks_1fu0txvI zZ;dm5+=l}?A@_#^J{MJMTSwi80%5n!_;48zGjDay!Qw(h!Z2qWyHAX2MRs;Q>B)D|-r0<2wD>;>wjdOc^01qzeU*klsuHX5g2 zbSm1G1@5hOT88eN#>55bE1uv^z#=$zO;F~%Vq4%&=WJ?NlPtaFfd0DY>CVqOwGer| zV0!$w5B#dfv2qUAi_y+oP$#e%sq6lP<`$))Ugd6AK2o3ed`1H5s)Ehk%jD7QtY>l{ z?Kl6ZP};-fsoN};$H{K948Fu9v0Ut0=n2nr0v2qqABFhjlC7kvm(Hh=4c}5Ip^i3m@0zHX<-$) z>mYziAlS5sO3Y`LtTYk#+i!LdzEN1#HPY=}r8aglYfF|%mIf|-*a~n* z46-$ntpQ;dkhi7`WmM+Y7?tmm4=NbtGK)g3g|QhBbP&+xU-1jZII`>#5ShZ(|I~1tF8B0@|*lURLqYWV{YOLmQmvy5Qi=Ou=tk@~<_fC-!7_fMBWYm6k; z7_-WF-QcYkM3g!xBYEqvWaf3+%OUjZR6<~tfv6MrtAebXf`C)XDuCVvDY5gW4Q};Zh0cR27`daBV0~#=&gWvd-7<+*Wgc zBWhvUN))Fgt+)pWSBN<3G`^DHo24S%mDFZIVXvH9(Dy(5VWW+qmwII&92Y_LwHUL{kz}@DRkA1h4?#L8ceeYdHd#Co%Rcm8gn4s_o01J3* z_%hb9zku~%Tg99JY18|??V-f~eM#^_Qb`NY0m3V6x~wLyG3pjrkiwiix|qk}-c#!3 z-~OG9{%jWXH~aFDVEM(}8_HduFsyCU=k+KjTlXtiQ~-Qt(o z=Lg4!$YqPL4X;na^tqFC-Sc~>l*uzD#L_*;+arDd>B6VVd?o=4oc#A0y-bKKb_aIP zbWsnWg7ib;Ho2+`zu$1o4&sD+duWfgtlhR@67U|rgr3@m$If+8p;YkP{jCG&MlHs; zbRSZ47~FILkv9e>0T;sP_iXtD2WhTu=|h6QAS;q=1ziC z*?mVF1T&RIL`5h@ujDuFW{3g9q46*sO>3|542ukY(#aT!COI#s#C~RbGajX6yK&L9 zX*A_zb)Q=6*(M$>|DB=8ID~a5A`?3_gfnY{IZ+A4-ei*+=4qVlC;2$J?t?K+ldEpH zKS_mT>RkuY5PGcwTVtRE5>~8?b~lbQN_x@t+`o?Ts)Qj)X2*lL(XnO%>2rQMdVWdH zRof&?+>z@}Cr|gdJ6N3+(0jN%kQ!I`dR}z7k0MObmcv~IT4X#GyngdyjMqOD5`iQ& zku$YO(FYb<03q+yi8pe>E77ojiOWPJcc|Z3#QDobT^I$)biyEWPZRuUT{@R75Oul` zl*0l&Qu^j+m++9>>5$@;R%$o0!2P^}gn8$JjWyPm`#bUHef+IX+sSg1+ba|z{&WV1 z*J?`S^v~4nmgUO^*2djaJ)imO?luD!lNSw}`Zg=WzXnbp87`f#nnrNpNH*R zwmBPwfRe=odDU`Y%3ZhA>uhI6Q1@-Tr55_~b!SwgX%Bv8-D9+8#N(oM@%S-Z5a7Iz zrrzrSx*m%@BYnf~xQg^kvM<+lN_Ko$*$)y z%O-q0Yf=B~D3Vw{Oo#5e`qRB~eVysQCVT<81QVs)O=&IWc)&m*8!$1xd>_j-)z zerdK`sk7^pr{mDG0x~L~Wu~@yK31vw7+S?=gteJq?1OM=AJ+dyGq=&B-A@|{Z-{nl zJsrp{t?hf2xWBgWr>#Q?ULA;~N3#SpPjDP#h8;sKWB8g#sz2Y=(1@_Pf~}lb&W24Z zlgRM*{P4WUWm{*PAW?%RN!LhB;X}kX@dgmOYkENGjjpdd?h@Enl7_JLa}XUgOc3EQ z!&Ea#zH}(Q{IwPny6gICKpr3kw<$3Hp#lts#lX=8Ei&fka(#jcg(uAhv*~CIOM?n_cKENT< z($#V~B};-7{qCSPC2CZL-}KsG7}8H~ps4aAqm@|G=4=mT&$-3wyxSDG9hzq^C}0lW zHdx1EAgF50~Cg^?ilbW%ITxq=TX8JZ}JbnHJmya`pSzu}2Iu z5Hag+mnKb}Kvq@|c!-bw^z8#wo!maLYf4lG&($081!Wf>ug^1rPMo=Qt5&f;7=02b z&w)>RJ#|6mzaHJ9=?64d#BQ%6t-+6Ha|~M%Hvk+AB=`%1nf1rtKA#*87ao8#Y`m0{ ze+v*t>B6v<={jvX^oDqhvpcM~p9#Oto&dOG_jOeAhw~ic0MT3T`8X<72;1mC^bxQ? z)$x2r(D$G8juo&nBHZ;p>=w7_`e@4%^>_W;O60@muBjUK_m*rrn^?;Gm z73_TF*|yImi)Zk1mPi_<`V@}wZ)51fCE#apD6(=k1XFTLPE(Wph6T&xA#QKNM= zD}f_aoQl31h>IO&(1RtXsmG99tl3t98`r`jmPwERY8=7Jt;%Be3t%3*S}6j_;|NeF zVKAVmf^-~#dvpiYZGUA}*O=q0O8oT0l(XiEtP3)PZ>Hoq%d?eOA!LUP-@ggEA)Ji zW96EbVfBwwKt3~HGCFTSEMEOpdGzJ`l+sbFPRkGNjsK_zhiGet>DajczIE=>-u3MH zA@iF6$`R~854u0LAeY`OW~amd`8O~Ua7=jI%!%i9J%SnPyABSfpf0ckz1A*WXF~&g ziX<=ZF?sOWxQ?PE7qvamyOtZ2_Wt1pS!-LzfJCG`&jXq0Dth*N+w{ z{u1i6?FWV{Zt-pU{BXn$M8R0wpR5^89V9;(k zYtVlDvTLmcIpi4rW7No~jG4n?l2M|Vjab{nr#e$;N_lP6GGPomXOIE|wGY7|Gm*cE za`Ga`G75o{l{*Gx8$^)|u8$VsExv&YnkyJz_>ja0pcNkWuZmN`33xDZ4sEUKg05Pl z_)Ry>wFYQ>E$$2FJY_A>pA>6US4HdEsU+z&T(qLE@fZ1_r@8i?I3;tL)gdmu7N-oA z8RAqJ{E<>IZ(LKriE~#2Id-M=9iJa{nbAfrNBf~KZh4>hHMW_3|K{2A7;tyWv#@ip zjf3l+eaTK?n84Ni3^hl$9P^K4} zX?9R`S()QE3ljkmf0e3$UzNv^ZuJ$RHoQ{+9i>lakeAm)^Daq7tt5e=(a9MCxo0l9 z4nbQ5H61hwq4n?2>6&IUtdMAGecHAZ5U@h!d>$r!Rg|KoNvhl$0 ze+ON!fb;rMG9v+qReu7)Pz|f!Wp=L%9K`KwRk@21Ht)uN5$k`x-vcTpo|pF>m2%n1 z=IUUk6(Goh<9Qqa3_aj+%0htfRllD7)q%rNW4w>es^i9tv-_$)`4f-xaKp+#;S{x2 z<+R;9US+1+@213we9QX+@FKA+J{&K2+}eek8ZCf^e|iC)0lXgT37q*Vxp4S;zqhHK z_yB-e2K+KVcglE?0FnM(ZN^3SV&{&*rNHBpo`LVjSl3S%?JGc3(#rohJ|fZe@dpqR zWq!Q~K4+hQa{d9JV<)REo4#kMM#X}j-DYR3r?E@_(*itX+jpO718BzUUOeAPls|YY z@%<(pu&DGFA~e>c)LCn8g(RH5lpe@N!0&WzzzhW0)iQq}`GDW>{tqg>4tJ>|Kd`n5MU8BmBhVH?{< zi8LwHI!f<}^qtE*Rkq z*JTtJ66e){ryYZ5t1aNre|b$-_dWCjZer8J{jM&))wM3-sHpsS`bh;u;xKFa z=ACb+nudlnT1*K{#puQjm=VO@9!^Y@(ImFlY>FBhHvK2Q7dcHdvH;pz@}6C66BYF8 zwfb1e|#MkHRKMvUZd`ZGcyz`Lq$X)Vwo{RlF7*1=_jMU<{QCeE82;1t0 z;G&pEx)iNKNqE@$^j)RlE{KMXI>L9-*&(;h=`!Mt1u6?hB_Ya}io$_~{)*hjT6dbp zNbYWa*#EkL`kLmGxZv0rR($v{I&Gwn7lM4CCji)J?HLGYLZ zG)y84CuA0YMuGqal810CKJ}Ml5-KElcp@w-9BQF#pYBi^Fxn(VxoJ*>HSk)s9)i?H z0qr+_HUB7Ik{w0c(={3=3Z`)sv0=w}9{-{;CNL!{AMBro1CRybnRq|qk~}Elnyj`O z{M{~|?tJ>pvjVo=`S{vE{$Tv@M~*P_`ZF#*n>b~-4y?V#2R@*t85Vckod87@DzNuy zcIp*!PxNeOk3{fLZk-E)J%b`kva^N~Fj5aC$*|={O=GA3Z$PRfe;_xjJ5!B@=XWZ$v87RiIOwN+l^&$9SE@@B6qx*x8`+KG9 zFi(G=<=}$IjRw`@(~9W^J%f= zs~T037a{TRzX^xmuqk6+NqBKM%}s?H#Q_Kv36E31v9^`!$O5KPSeY1eXgj?@rphl` zmMrtr69h9)>_9ed(Z1j-S>R7BmQ;)=RlF55JTfeEU`SJv*?#3=EJbb7F?QfZlMv{FC>DXxhs0N!C0(iO(zi&m0fv3`!ae?79<2{! zyLM`JWUU;T{uX}j>-D~P)bc1TD2f7kQi^D4T+AG*u_Bf)s8gUq12Q?|)zcO#G-YX7eO1x^K+IkiI-vT5UxLKx%u zUYVZnC&X|IfIyo>AD-@AhaLFU4v=rSL0=noI=))S(?T~RcS;&39?;LhjW#%j|GPh>Bfv4c%CU#@QS=&;~hv`Ip%^cgI)bW;c z+e|L*DD1)(ORn01A~Yh4x={=JF;!7P;xgkTJ;CIAbSH{Qp|?*O96@oGWn0Kib>_7# zN-7?|__B1uGXNG)9Wb-dl%Bf1ws0)x&*6KS7y9t8D)$sP$ z8Y|=bCH0Ym(90p3%!e(7sZM5s1J#+M>)b$4C=7iA+gHh%n*t}8I5c+8l z&vuAnqHQG;?WCe+!B<&zM}FjJUKg2MQ~;BSLYr9R^4B}%o+q1)C4s^2){DJz_8T~2 z9K~`-DC+#Pwot+3+pF^t*KZu^q~YJ5Y<|EWxN^OK{dRF3{@L>@&~D8`w^;4$ z0k9Wx984%k_B@|IDcn%Z$k1^Lo-arH^BAWyT&btV=1>wh=rvGg9f1^+wc=Eg5ZQ%vsoyX&swH&R1+0$Sbea z{Vv=3y2>-5;RHG4UqF;W_|#5F$(xLskj&23rnaCEh?;11Y-%FAxk5_satUH^Dse5w z1Auu;F4OZ$NI8-xX@?%p5fN|GN&ZyOBD5G{2|p=y`fwnQ_)ke(3@Gjr8+X7Wa=J5# zyrU^1z|8P2-f$-M!!G)D;<_tjk5z3qqbC!v4eoMtYB&SVFZac~^?^y51=PZwgM%cO zha|+5w&NHAM!}#BcW%1s2|4j75*5o>Hgaek|Ekl9d;u? z3C=vsbtLe7T&p19{f%L{KzJao+M~Dh#ho3QF($_-kj-en+!zW6xsK z@LFKMHAWhgKJ4Kf^oRo?yc43-HRA0@MWsQ!rzNH!dPcX+Msu5g!^^#Mc=Mc_Q;^EY zHn1y>U>19gT7^*zvpfO14MGGblZ{e-=vFKnX9=^bYI6FROR!=|O*~B+JeY>VBk5-Z zq>l=0uK4?Bd3mmbeaQ+otVTdG(A1@Mp^`H`jW#28X@WBE)W_D4Rq8MI6Hw#2*q`K)Zq-hc z$08Zzfh_w172|E%4Td=!9XOWbk#YF+RRz1NrK>P$YmDY@5yzZb>*QO~cjssZNus98 z;BfkGcBROPT9FZ-l~aGCCgsHNNgA5T{S1-JGa~2B3X{qKkOhA37}3T+bUb%YB?9N7 zY9D9MZ-t~J^Vh3*pdxj$vPI_#ZB$f>frG!;(iDB@8F9W)B+3szMnnjOs@RjnqtXt? zl0fRPOl2(`;otsu)-!s)wI0uRCseFpe?mb-aaR{g0ywI0^T~=?rbWedX!FWSexo!I zacE2v5fum+rn;b?rlo;q7s@a;;hv?n=tvkp>;sj_MaTJa;CV`>R1w-R3eDShppv;0 z6gQH|X(OQJd@6!U0IeoUqN!f8ouxbsQb7mR$bJr2mh#xZ+-Y7;f*QyQR&C9D{pA?5 z;C&v!mgoD!ejfp()rLY371b!+$a0xp^$e9P%P_ru7tR;;_u3Ejfa>0f)ywCj{D6y} zM^Jk|Tp#x{Mx)do_pj1~S}E5V9tip-K1t1rfAt`fR?Y5AZphQ8eEdXE0$Rt6)% z4;&NDhQ=yN(7$;FY8O0d{K{7I`L z9m-b$A9(6??}^LpzJHNns4x^*9v@|V^-c4O6RxcB$H*_pYR@P`%5JqA;NCr;Pb`tj zyhWNPAT~ecOQK#&ID&3(qg+pkunwy#3AY7eB_b&#oL*N}?XV*}hx7Ex3v^ppo(P3wPmw6qr)ZDKc}qIA7SAzN#$} z2;O*N>7FVRlOc#FLQ||}t?AxKUV<3Fr-;lfH%Gp>X3(4^OFv!uW{lgV+Jb14WFS+@bJ zTDFWIU~DJcw5HA^IO?WpVE-->uT%%>w0QvLpGJ|4E)_^Qwio2Rj^`8qO$VH8>HLbd z1B*}jH)W>Umi1_b)C$WI1fLqTJ)*gDPQEPy3p8AAa!|vc*C^~Gh87f=!24bUHri@wW+z}Vd<4=D_Zu#gAWP%63ca~PsPzx__!D(A1+}`!48zV8jaylN z9s#KihoGs5fo4CvHbe(11Suc4Htqo?qT-(6aiRL*qI+Py`ClM*U(kN*#op z_>^%FyTleCsXojh=2p$^-XXh@6mxY@yMP$~eFuDck(j6a9I3du#2G3t_t zx~ol?bZ!^ljLKW7BHLhTb7#+9n|~x6i-E+$zGFD-0Z`<3+rn=|wF9S5j2+WXUQU7A z(>`~CP^fI&E7Nc#qz~qJ^y1YnZqsKAKQ{^P7Sa`t>99$OqlY`ratr zMsAQKd^QqRDPf}FdY5~dJ-VmeNbVnsQt8~4MzGyZeKcb^GoV}Lb{|;O1is6N*Y1wk`4*!Mi5B}5kwJCy1N7gX{1xS8@|KyT`&CMQn+h( z=ggUz-Tg)HX{gSrn@??kSAaxCSJ0YON_Rj}hrxEh@}(NXNQ>TJ`F%jazZ0+Y+0OzS zkDl5#)5I1OcOuP*>tE!Tk92$EjyVz61rMV(H zGUFG6ML`dnyMDoA9>^%^Eb}XJ(}^hhi(iU3ChuFRsXA`)NH}IZgmsmPay%cL8N^n7 z!=BCY6tJ4K7!IZX2OG2enpxL*aAQb9|FuC8Xb_H>d;jl1c&6VIc*vGmesH`fKvtlb zO47p@_4lhYBYLXD_*rkMw0t15I>{E&X`Eg5#jS{*VAb~J`Ys#`dY``@YO1Q7{jZT= zOOWdTnlh#%JsI_%RQ1VUxsv0o=yy$VmHY3++Xh*9Ffy_1J2G05*Tk#RR+}<1rLeoo z&SF}Ut6z$F&rmwZ#IqT?ASnbIDg?g9A&_>G$$E!|hUUv^Ld(n?NJ^3!X!goC@Ifu} ztyd}o0Z4zJXy7@GisO+m$jiRQcJ7wPLwoJ5=I?*t>h{F#XS$1?VRP(q=TvIumg2L& zPlg}yc47u5NIb8xqU?2`ODs3{$> zm!xXUJtoWl%9AmXyF z(((b**HioES2}DoiLKbBW@`^#+Fq$7nY9;x=}zv(bZ>jz5{1@ayBSexW?>Q>9vz&|Xq3L7AuP~b$N7Ex zjpIJH*t)d9ao~zMgKGi?LmatREL9#U>Oq_fTbtDLCL#y+zfwAfQ$JN%4dNyC%8epy z9qJRM0wwSZXd8Xx#y4+ca7P4)R7yJHQEBQ2#cC}yNH8)Xl8)MFI!W`}3$+tWtfSv% zRG^#emOZ;ZP>w0F5|-7dLT_4PBE{tul!q&3Ju%wwd)T+O(|E`CGFjig?&Ic;*QD(v zY~FAkyhk+Uy&!Pe^v%otPP&IRE3^xeJp!Ml@k|**4Y|bqrWG=@xzJM6AA%_U>;rDw zmA9zFbR9NFRnrI#IbX3_mgw3hZ)%?m$+PcXhc6KVmj^>Lo+}x?p((y|budPgl@90u zF_e!R4hEI}8phe_UE^7sI+@GUol0T)HV}@J4MNWR=M>H;#A3$G^+A$` z(1HG7zAjwGW!3VlrL(~0`^pY3!qjUyKLwtIvS=Kprvl2__Jw`~ZZV3pZ?HL4J#6G< z*j$FCk_n$A3hvI=^^LP($mo>0J5Ld2do!q_g*8=WKe$Em*sMB|*khm6T0i}zA@i1p zD+O}z;t7(}FNySjs-Apw%lPZjgm)9!k%*2}W3`&OZehg2ZcB<=Ysh6{i{s{LtkU)v z&AqpGbEbq5I}g|oqbQ9y_~p!|giQyc_!W}{yQ}GmTQPp{4(^B`q+X#ek*0<%@n%dp zJq)qEu6K7<=3=uI8&nGIBk32`oVT#o6-jNt^y)u}`fq)_9A;fyh3hvr{?_uhQH4xj zWw=k5;^QKUjp`?!v-`Pm9A(fwshxTAW4(ZXn0A=WuDP7T#OLuNOGoxP|A?W8k;S6VB}ZJtYGpqyw}%PokJm$ywz28Kpun#F9N<8$V| zryL90@5h4pI|6RoB2J{9$lnDX)mQ!EE?~lSxpE&!Nrzj()Ky z@Ixeit?Pq3Y5nAOQ=BYjS$nAo@6%xdN#^0(B`c-2be-lGd7K}|>iXz%Pz)=SmodGl z%ooQ6cT&?yOXApnD_DeCIn%xh+Fx}=2}7k-BvQ2(^FXF^qduB(xwq(Z{MUVRrk5W;N`H%4C$RJUaXk#mVEOI-8V#yKHz}$@ z+rr>(T~Z?Q-H|HW`o2?LL<&0t?cYcg_n%NnIzXR16jm(2+2~=+;?OR``?*xuIOEw_U zfd9>DaBDg|s%PZz=b0)lzHkaVG9Q(eBzLf(B=g%ZX{_(Q@Z{0W#5SuuWHd;MN~|Ww zwq3Dc(>)A~D`QBd$VvQ4I5&wWz--WB9uV;mITrtz5>xr9ANl#gyX4ZPwSp}BnFe3Y zbI!&K=c|j;F^|p|$~1n5uZGooCH?IUE_p2%cGg{@Zbb8!!)xQmu|AeEgdxfI97SBQ zU*&n@Vh)gaJXjtzZFD|kQ>Z&wpukVQw~M6|{djlEWv7kIAvn5o)xE5p5FOLVkl00G z_M?S!Q?)?ftc%UYWt#N5tU5tuz`Y#mL=@8J>hExQdKdAOHZ9lRPyW{C5uBPT<7PNm zExUA^?*6dX98Yj~;Yp!ZyFuaMG>)|!N5-}G#god{HzBh&D62`kVvUL?{|oB&;)%EL zs>__UGFJE;9)SsAbuL?*{Hu>HoT%u`T&*Ls3xBE{;<{;C9RJFrOPbJ=2hQAYc_5iw z;?5P;w_u0Cu}{i5dB`N(vB&3kN0fG5dTQiQyWFCts+%f+dM27@P0_VAGU7FpbL9EZ zvMraj+S?xSWj5}3b>#XGu0*(0*2x4_X^To=ca^6znyw$;>MceK-uO#Bo@Szf#Q;>6 zdj#70x!JSw<7QFZ782uXv9w*M^O2m(16ec0db~d_KWXF!E9;aPJuB4AH_nLkU8!nJ6c)4tTotuQsc${_HGrjCA@dd_I@i+h8=@B-ym2pSUM zH212*52Qcdit(dA=vp*+YX4V~Q#COS&q|R|j!p)7!Tg>FUE4K|Ce^BnXR1vq#n13v zB7)9Nk0{kY4V4QQ?wFUu6ZdQ|2TZ)>WKJvJ{tH7FeYKWPW0x! zz;V)wlmAvrG~AWE`y&cV>Op%Ucd;N7^=orh15PSM8}gnWp7ctGJ_9IluKzm z2jfdSCU_DQ?&F4a|BQ^@!>Icvs=1tip6`!PCb(z)f!1I!zBe6#L6m}#dgu0COpV*V zIfeh4QulmZ@ZX~{#JyD8MJ}QD>1em7-`;jd?)GIhqZXsKi2v}k3oGAnC`4gG~oW+(ZqE;O`i+OtHc z>DWF=PW^P9QitjBG#z}i0tO1l+94A|^%(SQ?`S5Zwcb3s*2K@mjfi!;JdBFvHB$n%r%1$8cgJg>O%Y5n?iqowOTMVbz zX@XcSl-x7Bbi?#aqCOnCi-~;IE0_J$;k{k&4v>ET)tG6{63lO&)F|tylBEsoItl?Q z_%4z&xWLi@hVu2F3A^aPC+X>1!Nlnsh1fQdBQG1H??mKR^g4kCntiA-17x$Roz7>` zlVW8nkEC7maV>TEZOMw#8Pd8$DIBG^ZV88;G{)sw`W`-tND*JA(C%d6So2Ra?`RD| zODwh*)frJZ`#-aLIq5|Had#GoRpmc$cdq;Ods~vK33F5}kL{&oeUCXUVtSe@V(Tnwe2OQokwvaKtaQ$0Q&i zm%P`Vj>y5;yVXk09Q4$i(Ll1SD%2HUMcW>4OcR?7x;Bc0#wn5`>+7H9jET+E1po?N zqi~@(oxc5Kh26pG?;Hk1v0nM|)0O$AUHIl+`&44`-eC4i__E;YvNGET%w9g`qhwt| zr*5_%12~x89EKtEwQ14;JM(4lltvV*+^N~~i0~?-4LMrU71BzE?h2^2I4YSw=bf^c zz1zCULWsezl7NVnJf24mp$o1~Q=sCZ6L%VSZOzzdXK*{zPgW5_Mh-T@4p*d-vLk3y zz(;Gfk<6y?-ns9=)Y$+skgF zGm)Zqt6^w9cW&CgV^7ub-s->`}V038JCH8we!-qrPbfL?i*vKa0L%+ z;2|+VnEZVIh{ajA)U2H@R=CV@uElMAL`LHeM3CTMCL^EScDsr13OO$i{|yg_U0>9} z9liY-*ofiJPfMQFz8c6B*<8VsKL24eDtbKSvJY1&^IEAgCKejidwtRW@wINR2YLm> z@HsUcD4yThRXYgFNTYrjDCFv zMcYVQ`=Qsts?$Qp&nz+j)n6Yngxugrw%sR#kfJ-Sf zW~>gs^bclIIlLdc$%;oW2J>DFeH?w}{_Jm!TOusd*=r+aC=*x$E_k#+yGRX4Cx!kLW^Jgz;BsxO)O?{M`bpG`&;HLUrv=+`LN(#t zBrR2>fD|LI7w}&?+%@joU!4JhW>nsBnNECv(VN0){Nv@5R36KhBgLKvzvYg%XBuI+ zOwZvx!f4)ne3xg3{~fr+_wvtbAU4NMyXJpLR#O?%;DMs_@4K=pq{_FW!+!kv(e~)c z{(EU}S%Z_l5Z!$j@YzS2Ey>dtW}Q}+P4M^AcT{7xoYqE!0h|?OztINla$9=hEaNh> z+&TDIyb>(6qWhb@!NZ|#DJK1~(xuRzs%lPO{`!I=svgfa4?@WO-($h%u!ML`VGadF z^-_}QZu}n8rQ6r1?#{Y#<`jN}D@eE7{BcC(QEqj~ac_j|ZDJ61JIKx6c2uF`V!Xl& zM3I|CoSYq)t)^bQ3PyPz*Y45rO-+$>VT{%|#jHyY9hv7j0SdPMK$F1b;i*8W@;|~P+;kYdl#G7@#Am~f8veSCUVZ3Y!mK`!e z3`PQ0ruXrd-J}&y>Pn_}Ki`9=U#9|)P>lReExmi6J8PfN$;J3=ly-83AqS-LTJ@(3 z?mPIOPeQjh4oe{}^JeCfa+|I6XN)buVq#-vekbJiGLcbb+V6aGZ>iV(oW#guE?$nR z`B!g3z0avLd14e9FQb^)Ov_BY_i_1v==qp=tVS0$35Q{oHPO?&OYUDso0EVs_@y z{}mQ+RLuseiCww^7*sTP9sVObe!}l&Sm7{p^-Jnb{n@|Ix~KaCV({r%f)sS$_uw}g zi6NKsqWtgUpgW?Z53PSy*iGV{U!3l*3}juya#Ky?+nfu%&+(*)$nJW?z`^(GbOoVa zpqfD!BVg7MMFtRX4GtNS2{Z~aGBPJLdw?2$-A13WX22=+6F1MrY3#Me@L>9>ktbYM zB7^w#{l$>|-x^Gz$jHbnuYZN{baE2{gw;^DCKDKxrv0xkJE^=kTCh3usxp0##>K97 zV{2b+wj9AKe-H23z@gr4(ysDweN-*qPE068LbdMU17pE`V^3#eK`E`xFF5*rsMs+h zG`o=oJO>e@qZoBd48~8T$k~gdjz9kDIVt(f?Xvm{4nAq0ttAj_4a2g?<{^ltyke@-b(Ib7A8O2-q$y@gOzDT@S z-gwBG5iPmDj>RRZShc*0juhy&y_WMzSyZbHU;Y1L>X=QTFC}bjDi16jH z@E0!NT+{|YmW!pj`#|S=ivQ)hab!ekXQ%mmt5)sT{uCS*O3oM6LGzu6h$!j7Q{2f|Dv>j&s0H}A?7hkCk56WU z@UJe9XRf>GV(BGrD#IIG%6mO4;I%ef?Df6~mg*x2v>;NcM2VsLcr_5T7oqh zs!jOO&2(4c|0_*Lp+61P;Xz3q!4sLCT6jZnTEpS$?^am>gnGT#VWOHS+3*H*1FAXu z@N=nMk#?c;UN_SMyEOa6_E>2ytUv#=p)BiQx*mKv3JE6>z@R%K2YAtr%sV&LpGsIa z-|kw-Dhjjpvu4MW>~G8za*33?2zmkYScUf$A96>&(mHXd(htDvZnPl z75#){pdHr|801i-i9~?bo>h;*@R8OctQV&lHd6pAA(#JYF54{PaS zxU1V+HN|{Bov7a068T2c1GkDCA)@*g<7NzYHU!5L*zPMfZJr|89}DOO6|G>Ni&tAX zcTv4b!A<3~7+}0b8PC1A`n$O-hr|`jy?(hPu?>J)TT|u10irUW}f!QMhB>xL&NFwy~XZ|pOKsm zV;`hKN4^+L^3C`j|82b^W;?LIwVhykEij*1yaR0D)l2 z#_rtExgj;Ndd+-A;3AiMy_{Bk@0T9CbNRc7 zIso4rwgddE&i0E({PVDMRG zrSe+#Bw6+*167Un>VUUce|5GFXuW9(iz7zt>ZIt&=Nm-`KKF9L&p^AbXRa@e!8}0a zKVQFb8!~g0OK66F{00G2$=3%`P`C`Kt7S_gT^-3nWj0WS$Dq22o zL=HZkta7F>*)|{O5q&)d<#f?Mxi2ZXD8op`$e0!-?#ylb4$16rXl|ogoFSLOv z<}#cY<)qrf(#~iv&~HV0bqOZv{@P2S4|lGj`PnV?M*0q!;ktCblj|BsLNEqi^R-QzDW5#@%iDwxYQRiUY#^NCVkBpRQ zB0!LKAA|N(6?a~Sj~4l12XU#p^S6-QIN#en!h3jaO1uqxqYxFttN&z3exZ14Jugg; z(p^+ja#^{;nc|^xHGPdg z%I^?}e5I5vz>73KyBJ|0i7^7SiQTTNznX6eB6bG+ul9>Recv%SMAYUypcJ z2A$)y*&wg*;`Oeph0mZ-khKMGzjb%kz9UD7SszWD`0j~A^OM*I2k#M-6kEJuNQpd5 zN7{$lxoz&wl4?&5-={|t_vce8%|Aj4WgAHM-YR-0=Bnt(Z^jm0cC;rD^F_z5F=PtSerKZGRjzJc z7fCk>t<5XNmHTMASH}{DCh8<16rmMOCW*MDOEz7|w;5xI7mWX6LcpoOY-52<7fjJ* zh|5V5j-P^cOr%N8DpQe(P|g97}X;Vxg>=#DS)5oP7cQzF$-lPm+OWl-}gIdp03&LAm^;z;PES0r&fn_yJ7 z)obt(+zgCO)VniQ^QQuEcDZ8GzTBPPeoDZ!1$6=+X^?}A3i+Au;jq*joYHYj1N};S zUs!09wFZ^;8GX5dP!YI z^sF*m`UQ3OfdcyP|KzXm1-c)KLneu)3uSE5q4DqZCCCOHF_cOYH09Cc{8zIw;bV&ejT!?As&mLepe%m&Z=cxT2L`!3S5zv!T*WPS@!XL0Pdgwyv7WFIhS znYPqAwOAB*7g1@J6<}@L`VeKHIaf$9GCp(%>5;N@QNg{kPdGp3Nw^3|A5de$MX(rg zUYXCN9!7k+)57t#5bs1n2nqD;8g66(KQ`~BjPyJv%4c_Hd1Dny#J=h8{E$%pco+X_ zWiQYC+udxjwXQf+a&o`gCf@G$CT^{|G4g!!QYCOkCNd(q2%69QZdeP;{`l|Dh*dl6Z0|Uz&t>pA;`RMDhb+zz*T6XP%hM zJ(*^C`a%?~TM7X!5l?TOp?4D%PKf_<;y(8;o!qGy3;hAND4iWhDBPwLIsooawo;Jg z=LOBPE&090U!%@Z+=NW3-w%N|Y=eCrc2L&K(b@nf0G7QykwHS}!W^>U6mx8|@McjR`%6@LgkN9Lz`SntljGZaVr|j~80->D77d?%v`RDB4@aM@- z3Q2Hwt!Y|dAaZdQzeDC1;ln>#K-r{AU~Y9z|FqxNYe8M*m%cQ*#8TKCn~gy<+0ryT zYKW5<3I}VOyAG@miTa!sx#YjjKK$V!Mco(SD9da^M0huT2a9N&M>wIg=>@)M3@gSQ zZQF7FhgY-1B=(v3HuDxB?>1geV@5)i`Gsx%}F~`eYd8$ms-GO&QU3rMT@a>cUo&lJpGy zHrXfY)w#ODb3Q4)Wv$rzlp7AKVr@AkP=RPD3De5Ew%?znD?8|2xjHEYK*&@(iw1J+EWxZ_5L=#s%b3hGCEgSrpU z>J1E8f!%#;3J&q1pR9s{59Azs5`l<8!)36nkPN{Bu+W|}k1O{+wm-+kNd_W1+lkUI za%}jTn`pbnZGGoQ7O>q0Hz|7Y+`C(6F`{x@jRw>*2PQbCn9i z)6AD4m@lf`B$$b{JPfpDUq*DgE7KxK4b8rfBZhvDN=rM+7f_4$?PwIw2X<#_&={KG zwL7Ip#ssWz$9*X}GMjdac~*{^YEyPN8$2sN2O;ukBb1$O;k7V+x;i_DJkd3IeMb3* z_21Z0gM{(F`iC%4juQuX4E{DhXfE6>;IMs^6wM_%lI7~^AogOLTI(8l1BItP(Y#%`%7kD6mz!1PH99mXN0>`Fmqar(%Kv3}U0FxGS*eoX*Qxw~_YqcT!JF=Lfp*-XZQ%{q(&N`Nw>`8w!&(c2@F zgCL*Zv})0F%n{a%R-i@eHagBp7GIAxUbziFd0OaEx(sFHhc7H;0iP4cbIg*GhO;6d zpd#K!8-w}Sj9AR+Ffz+4E0gZK(Bce%UCdAFa8ZzY1wnaMqSL-+#NtUFJMF5;)do zc^y&k&sF>og{W6)J^#L>kV{M9{5QV$*-Ep8&J#HSg7p^r+ALFSnXGnX9$K>^!{0S7F z#;izCP@SQUQ$;a0a2JpDQ8VEPupnf%Riqa zD9t7kMzGEPICT#URH}+~L{|(F)PX#Ja~(jAm_?swW6*QkTg{E#8ZDUtCv{^91k^V; zWIR46JJa>v#;?XL4_1r|llW}M0xZ4@Lxo!9tZ79cY;-jpc* ziMFn$Sx31pN2UHp-=)6AokOPGx4g@t-^7}Vae|En7napMsF`m!^^3|PHI#jB4^Ck` z=^+ycjrV-i=gd;}=!9GvL!jGS+n=e^K(plq^W^n~dw>E#`|U_~my@cVZ3McXys#0& zIWE^rIy~ffEgFLZQhc%B`}K#91(o&kZO7T<_eh;EZolqWN6E+wK@&VskXxXoCl_y* z3zOnI{;3x=M@tkPJv#sCrxRH>Y5(TKUP}7{l?n}2{FvOK*Wuq!-YtZLh13R&-&*|b zf_01YTToQ2h*8_>5E-e%$1c9<&(vj47R*QE-h6g?Hox7KP>SI4GsePHgo$1Aw*xb# zmlzE{{wsGnDPbb}eDvef(l}}b9V9Y0^SSp3rqmZ2ImzPRek@`r%H;L*^_?^4Kbg27-%a#FTfwu{+&iKtCi3iHAhtBDq`UNY z#nVA-O4Wz%0)*8YjmQ{`JBgrMZd+5SgZ|5@Rx$S;{fl|LCBEVf$|@k{zjxelWetV3 zKO@29^~pjkSd#-cLhS2$sAj z^HmEm<(qa@mc1HslrmY7T9LkA#9nQO(M!GGy?{4VD$x&4DG8e{xCmMZUFdbdCbSEJ z{s2=?Bg!LuAXCD5IXY8oHp*Lmej&sO9%aP|^|B1PB$_jZTBwWMc{4zg?rXoi z$knK>ZePOCyS-mk)@bG2_7uG>=A6tbT>~09U`vRmN27-3&z@|gWo5ZpM0jLxAMZFIEYBDI&_peVn!I}Qx0qpA(Tm;*h25{20lCB6<1?Iq*yYYenUL#ow zLDmO&wfkH1!?@88HWq9A1_d)zjd=;o0{3Z8U~=$~FMz569eacMpvJ&|P=QGVT<(f< z@Pc)~xV>fhCp12%Ls?g@V6q`2+%5+MSJKn$YcR|1zMV-@#`QI`gLMJ^ zdnwa@t?Bi80}qMY)%yN>zP) zv-7vV>TfCMqMf)9t`v%vN_5mN43!Bz*5kTKqd$ zUK8?GbK=#?xTVU?`;5V!@!i}k0~Gjbk38PAgwu^{$<*zo(%CEAKGv=u4d#1aio5rM zKa}l2JN&1Uce0>I5&d(OLv4)^G?s%tmL8*?#iX57IyDDOktbU!8d#^H=v)sYN_;Su zIO+UCuiC|hX=wx>V{G7W$FiSVZZWjvZj;I^5))A4)vCPTI1v^LwEoQ>uqfSz7SJYp zkua6=>+YgONlz=51-7Z&APq7Nisu^s@5u-r3L)yWm%j9d+1X!cpc#6nK z|5)r;gK679cso}|jY>zG?Z}FKgx(3yE?YUaYRvaBEYMmUHC`IEKfm{4Ld>e6a|Oz_ zl+i}Pur>LAD&%d%>FFut7YKZJUvM}M=!|>4pDWG6c)(%@fvz2tJHlLk7bl;;)?kVG z-a@p7-nr=^t&KlOhSWl%0ruzW@aGhjiZLE2z2Fj0@PW=REGuy1M&4wWz;vX9bOUez z5+k1c0#IGRgRDFRYx%oao%t@D0lo#Ma$3Dxzu=1YC-v8ZYDid~MTdHm-@$wZE>QLs z1s%7or0FgR$&AZ@2$%!ql^b{dE^@3){II!kvBBA`Ud@T*3W5a6*T-8`<}m5r_=}go zCnCMTocI0@)zl~p!UU#S1%dxB=%@yOhJIiD95V_g)a$n)At8EedHOYw6CvofYkZ8Z zhn&l*wrYm$cemYb;+eCFv*#UUCgj)vj-UF8aVDxHrTa)?fjBXIo||e-8TzR5soj|3 zy!deVx=G1N{>Ddd&OWs86F7MO?p5%8Ermzh%#>HK6MEG=h>$rVLUNB-U2zSs=-yG3 zWU-_ZL-#z}>xDd8AKiHm)~^wLMPa;zyOdhxZ^w4?CJNUQ#&Glxn;AF_sU{=YWfVmNzERw|-x>^(KF(ExYT@_NEoT zxq2nJS7!r*>kUkqA`}8g=6AN3v2R{|d?$&FgWe{y82aW>r>x}byeVcRy@;oh!89FV zP02++y)kg*U1Wsw#4O{T`xm=9y1LHT_9GXT%Vd@k6KU|6(l&K>7Gw^jWY2#K{#>N+ zTK!;}2C;i+6x_F`|DL`7UP4n+=Ku%;`f*45Eet}D1(ED8;H>jQ!sKLkK}~Rph#~_# zDOo!RkqqC2^e;|!SGiG|z#_6EpLOs`=6H4ny4Cm6+ee2zy{5=blTn^t5EU+)B*==w=U8*@Nww>t^)(!cdn% zS+-e+za0aOOeD?) zsKU@mfA>8pRsq@_L(Zqp-G>ZO}|Do{g}PI527HwhiNDc+M1u;5D0~rmarZ;!)*teeMfp zFnGv8=u}9E_nun@tEd@u$E}7SOjc99FNpB@+kYX*TAgvj=0p~F3`8wBnDU3DC|(H> z>H~(tvVysPJX2P=d8VQQP;0dOndG)M5(3mwd+Fcof6ym7RnMA_g0H|~8?alK;uB6d49?|bAI)OZD2r)m;Q`Yl9gf4l;fuZY#pNanCwy1OMl z8uWSn=csrfSN~aDW;6qb1!0Up_t}c8>@k*iU#90k08%x z3*-BQ?)aeByehWJ;mWE+wkougfj!svc?DCI896yiz@~sI_hfEXwGqv7-3GXRiIIrG z^|)2mIH$xmcg9^-?J8jUyPeB0-tPw+d`|a3*5DT{vT$GJx0?v&9C&CyRRwtvQ)BPS zsw&+-l`n_?{g%5gBj*nrKOE|IJ3!yqw1iMe7HFVScSN**Nj(Z488T3~U_9I$Dj0y; zo0yV(bA8+iGTw>;VInW_hGu5^ghE@>)TeXB!HP6{y94QZwMi>zq&s<&OaV7pfYZEJYR zgf~$+qeZ`HNz6V2OLQU zNx=d0sG3e^(XFb5oeu&@F2HQ?Mj={}@K{;#rPnzKRvB(1dKF8H-H8Rtsb&K=5u)?` zM>)O%NW7`a5PpMFYx4joo)GZ0V%MDHt+CPtc==Q2)tl!lgB?908^3D1emY~KkXw*P z845`at(9W+GN|B*Xz>W*V9D3SJE=0PCj|Y}*2f?nj3#7M`?=_t*vhVN#>h1G=Xm^< zsd2Y%frT@@k)*DP&?6)*qilUsr@-BO3Jsnpw`b6c#NXz=aqj(ihUp z@)YA-vt`i~9r8Zq$n&pV8x5N)bEoWtK4z+ypvZ?`>%1`$O-j6SdxSimi`9ro)}&vV zyZu|gmSr)Oy>*xHlg59`_(OlGlYFEbzji2g zTqbn;d9^v{B8mI?HMkYKgo+1Iq;;1bg4E;*mdR$UV8h{v0Wi#)h@*}*n2S6JL#Ot; z-~)a<>>$0+bpdytEqT<~AAo%m(pLGmo4)x%Jh%gH&*tj*Q}iT~fW;%7&+gt;n<8W4 z`1coym-&?5$3I4myrC0B%tAv(CF)ak89xLt0ecSWwF@;n?lvnWyjlDt_%h8K4ncsD zh86&m;2E@$;1fQB@QuxI(OaPUnmtZ`Ai^_I^yI<0RQh+np^qvjrH_*KQ=u|KmuS!d z{pDk`wlJuVzxOg@(5kyaDnRV=o~m*}?A*xi4!S5!0F+-8Sy1gQC&9mM4nm(iL#OhX z1vAqHI$F)0S8Iby@M_D&XwIg+w;usb1EO@^QarUTSNc-MEJItzd!GGU8osyjl!28) zNR`{sSArm_Ax52ZM4C?UP|~XTw+u%~-!cZ)4>lpxcL$idZ^~4q%7;!;ne;yCSPgmF zoEbDrzK49~Z`_m3Gs-)e_fh)IiBMKoO*NbNaNIFPWIvk4U)0=VYIdLPl4?g?_^UD( zy;4L?|9iFb;hRne#JLW;8PPO0_1bInoBVLxWv2~G*mcTp3mwxwWHv86wJF{q^`;x-_1Ox*{ej%5IV8@2PMy}F;$W5ENcQUd|Z zo$6u;7f+nQ%)DCS^sffT`Nr}FZE0lwZ<%E;P;F}w=ShLYZm=QmHTu>;A{ZQe1K0NU zb_o8;RE{M>Hb#r;%gXH=mm8$4WA09hApp6Xbd5Eug6Ib?(ksLCPfyR0y9uujhSd04 zN|PvDe@Wb^cQErl!^|CY;_v8t|7aa%%I5FfPRL}I`(1h&G@7+XfM!RjF?sS!@s6=F za==sQ``(b-zk^&T5b;d|-SyFuYuJMWiBGh1Ud2Ey3)J;tbHKk%LL11wK=h0pPVE>k zJF0zYdQRVEmF^^aM|a?88Zi>byeCyi3v8DJE)^*^UWQdXM@MQzk~Lnx}Bq z*^5+)nXv6vd|c;u#@sCNu-mxvpfPUgYS00wmm0QnrE=bDRBWPSNYc2b==;3h!; zt2DI~Udg{w+!{yN=WY;EW#(vn4d~8HygE;pv(iw6jH>r2n z7*0(e227Sd077bY6eNs?uqW z{o1}>u5eXSM7ETMWd(UN1__HiHyTi@9c|oCcxyy!?tEH(lMDt|msal3x zzsdbDP1;!2C|z}|56UdLS|JC=q9NxB$8DitSA&8E=F*SKSKHG!G6y+@}JAyE~C-d*a6f1)i~xk(C@M`^*tLgf7V;pw<0# z9a6c;!Gov_1{$67BbH8RLlivx9_SQjVw@&#aVwanC8$R=*x^`0#2SJ za>b$~c|)>!FR+1B4uiXU0O!1BZ5X?~nc#vL)p?XgI|lgg-a*{DLzR9g=WJ_lW&Ez( z`_)4qQry_5VjW|!R)gp! zdrv1D_01tJbgvko56q}d{+~RleXvtz6h@iQ|NWcf1IYFZ%12644mCshJadc(g9OLT*)d9#NIqC(Fg zj001@#cDt^24L6we6#ZWc-y$%%MFP=A_wxlbr3!UB)xMEBL5*o;Hp3)+sPo)hgcyX zXeBhG(r=|WYFt||nZ}q#s1Nmi6vsgb>J&mhnEC zgl^{qUrhfdItTQH_=l^%Z~}LSl<;D9aSa|@U-;7(kq%Qe1h0?^5dZRzM8!F+;FKH z_G*?ix#sAK`wj5dkCY$or5LU@1Su-KG)>i9W%H51${3^{#(7oqa@0iP7~AOZm&07on$6{9Psk zF*AE<9O?AV;F*$W?s+Lj$qJw1W>Z&5Qy~*fNmWcECW2%M=W(U}`W5Z*pTDriNyGF^ zg{Y?<@fcYYM^mR_yhJ9<;P_w*A`)%_djPd9V8dq= zc#Ql2l{aum{iB8GVyJz`=wcQCjWmE^bR^!600n6g5H13GMgUXqZu7BS!voKdldqkx&Nnk1*A^- zqCR#&&+FNJ@L>>NVsbZ^*EWJZ9RQ3ut$=F;00u$LqZRnczzm#79{zo*2CCJfy1~dki0B8G8g9`M|fDXZ=!4gofSOV%xRUOfJPW7(kfek zJbLhs2XQGMlm~2L=xniE=Vsvg?r^*bFj)b+5RfANo2P)$PPIH7nCgHPuqgcy2IxY8 zHBbfqBY-h)5wO$w2i?W{Sq_AA!jv5h5x0z=`!*y|T{H^FI(+AMH*hGuE;kfw7e5pt znS1PlyDMZGF7YY?0;x7KocAPCsDB+-d>>)dV6=%^OzOdGOB*Bq)#XjF<*P0!@}o#} zCegx!hfZ8KtRCF*BTm2g3G3&v+E3S(WrW>sw~O&-MQgrYAdKLHSakR9~fThHo_ zT$B8d_sz_2RmxvMJ%h(N&@XV)n$Za973^EdIZ96sheT< zVj9VOZgkL8`GwEbKtRBe4ssfcguTlUPFy|5h<9665?Vo5rCW>v;iQi4uQz|$x-GRpx3iAF6l)8;m<<$g!+`zYKG z>9M;_G~4aoLGpV;*ki{}ANO6^flAM*TAaBWGpQZbHG33J{T5*jcL5}K-QnU81kFVW ztaMeK8AG9j{C#pHpVRFlGG8GiP*5_kt)Oo-DOCXztl~~Zceo)SR)`!kITr!w2*Inv z$*&9EL+n%_dNrtI57GRAjC>sQWb%xq>M2~r$sV*0zN}nex!oZiezB0fG1Tj0B-QDC za&C*kUvapwy~|RE{HmA#`PpkJzs$jC)CljGC=mj$67c4SsqkkdT4S{E;C$P@v`SfY2X zh?lhgk?zf#^M-eT=HTB@@UMJ-Is5{g7yr&6Kw0&~bo3U$j?UZ!{y9sGOspKoSpn~5 zO;usg1X~p#%zFUlu~Cu3DnLKucQ^^mS%I%oc3?k4_cwUF9k2w^Jw7y&8LxZSB?`yC z_T5+4wnQc{?7LYK0}{>tI?}?g#NL9uETxn*1`J2U0@%8~e^|5*!ou`IMtt-qj|}2U zSCCX@Bf)`>8AB7s>gc9xUb<2H*nLfK*enXNO{3DsfTI#6kqPe_r@ehslS93eF-(}# zu4KhOPbHGdrpAnsuShH)c1=M|=II?o6gRoAn#?!3U+ibU``Ma>!Gy-am-^>`+?#fP z?lDo8?*NO1nQHwz%v9|HRoPM&EB+``Vh8Rbir<|99|Vt`(b;j)&&#H1Bmb<07x_vu z8diQp<483wz$v1HtvNFOxaPxij+aEXgC2}w<(I1r@mQgrrztp0#)AjGVv;}T$&c34 zGlGr=G)cg8Np{+V;GMkxl6~n7b6&|c!sivoCsC{xpZ966@xQ3pj!BSWOuwW0eqU@3 zOoju^(nrOSpZt&c-B)-GqXAauiRgxelvJxwa9v`L<-kud~zoz)9#c)*|d+^3l*}z$a&>asB8*9 zm2B%xfQxW)V?HQ=&8zT=E)C&?3|AtRmUE(+ydIYa3D3h@<5+gajKeWAw5UqBT`#wb zWT%YB!jQCT%6rm@4f@!wK2woXQc-ebNjfpX-d*Ut)E~c$yTMKS!ZP#=m5_3sSpQBk zLW-T6;xZ$<{14LP;oI?eDY^l~I+%}pz3(m#_rJ1TigHja8qv^P9H~I44l7;c{+%Ag zF?qrbPQ`@5ztLM_t^&&EiX?G(iohjO*&gj<82Y1H%QEp<)|CJ8XN)5OwLuafX)Y4; ztE4#qga*47)Vt70V2lA$EbhuC|MeS8H@1Ku7I+axI8Z;As`VIR(*RvE#w+6oz>$LQ zde{;}@kA>zBO3=O-n2RxjNiSEPvUg86}d)S~j7 zL-2FNYu{`wI-L+G5B*r$)l~%JsSA9Op}l75n}$H(n}vk9%N9Wz!Wj`TT?sBG|M#C6 zmk-RzS>qp|o&ypA10SrL{bpmeb1e(KCX9ywsg!neyUebj#;>?oPiO+$jL7+_gxHI` z-#2vDmsJF1m>UwPS9DPM3>(#Vzj=dW2dW78n)vkq$2kc~m$KmeK42njT0j=UXZ6L# zZ6veH0Km+;9l=ocw zdqtJs+ic;zPw^lbdl7s19G3eboC;M5qdKSD{=|X15Y8lzTPUpjc>F!$GZ;Rru`(kr zf+U?_Z-2r|l%&TpPVFG_M9%sHg$;U)_QG`>pZcA>O324|6B%rG9;HoF7n`52V$`XG zTyHdVTilkpaoz#2^FQn{mZh+|qr3>#;gxxY!!PN*p_1f-#m~+h^TvJ_twgG42L)mr zevdU>xCburb$dUft$P}7ED^`~CqlHhp7=B;L8QZOXb2^|$+%n87`BN19S5t9##HAf zBfsfBn8BAfA)PnK^F?FHlDetRcuge!YCX$UG29?hSm58;GaLbozx4cPq>!hzrs&F$_I_a2pfsN#P-MoMGZxvxez?&8iIbR?>+`M7%bmlqR= zTCcgBiRqK{=r+!#IaX46Y4L$M6Dq(4+ylcm^0I%K#mUzfZkCu%)qi@RTpQ^DCSj7g zZ5jR+)x-Lroz%k{jUNVxd@PyaniDUsC}Wn%zmUCeK^MV+@fImCV~oR7AA6YS=k_D= z%N>1>#uQW6+2jf)fh)-pv$Bv}Wr}R{G48aPM`zIF0Y}48?Bu>K7VQep-;cEaIz^zj z1!NJZeSmn3QrH~|9R2$T2Xn#zpcVuSm*)~%gxY!jlA$l5QFYkIz=F8+LoIYr}Ke|E|+RNQoO3e=Je zBORH#EItl$lx$2dGxR08)7GgMTpQ zUV=avR?tKy$1;Gc12hpD63}4!e$x^G#)p4gG0^V;IVk@uUx30Pa+MJYNLNq+fPuu; z#>XT{g8aWH-(8tU3xDSg6n85XSnR`jqJ}uTG*GgCa}AV1gqeWI-F$L_7}FjbyexO} zh^+d{em@K|ptv#yG5f%kdN`*HN(G5>D0bzPHhx33+cmW#vWTGSw6ffsKbsE=afuaD z#%&qBf5G$7r*a;Zp3sz&g_ju9_eRB)nf3LkbH?_&@beqWoA<8ms`g1EFjFz63H{un zs-rs7@j7n;Nv4bi#4OW8>fI$lSADjZnQxX^&TCX)Swr>sXv!7GX!gc9IYady*x~j| z_i;{Cc0(B_+1+EQ#g4uSatKB5l)Htp;X3JC=Y$=5`zs<_hR`uKxkoRlTh+VLp-ZRG z6TL-b5b-N`A{l#OIg2`j3S>C@lL)i(0?w`z1*49$D|+mCA{u~lN9QKAzcgsHQvOO$ z*`UE!bxgr1-XePmoh#p24466ghLbUoKkM(qt}WOB5QUqQv+X-jM)IUXKdo`f$h-rH zUchtpaKH*^e*wittJJGjC%jNmJPiPhv5lb-24w93N%Ge>gPQfx$5t7XrTkjRcf4AZ z*bRP5R~S1dz$ z5iGKjFRmm}sHns_x6i9t7E(%PrLb>=17;L&wdq8^UKC)*PqsKLlzA(96VNS6-11k! z|LBT_-FA^g67t&Q(QrK-sKGVW(HS=}#Q z7hJNT${L-QPCWa^3K+D5s+6tD{7%EB6e=s|9pOY{#|6}?A-_|GCV)Ga)JOb0aaVhh z=u*?#B{qabC}|<3{J%*QU{nssJ%L{N5OMa(v|M@vG{}tNWjeb=dNZaKUhM| zq~gP$Sa;|HBJfjI#BwHp3^);A!>-qPfGKGNbKitBN7D%yW&ptQxgASP_inM&46jpG zyo~1aB^CwgMGm2WW<7ZjXG#0@YJh7fe9;EF>Y7l9yJl)9HO@{0Mj%1{{qvA(vDb$@ zP^6}I3n>Fss&iUef2n}+&x#F3(KsFYR5D4tw>%c{ui>}qlM=V&vCELfAsHgM3ub85 zaWAdf2E(QV6M1)46r7>6i24!BIhCtT@khq)Es4`lZ7GMS^pBWDT4P!czVy!_K#0I- zaMd$6EIs{=I`(6f;OaUW#3xfKZOWajy*o@0&QOwTn{}O#?>%(F?Q2fg%>UVeudXLU zt^p!T=rkt7IEi|&nP`(H#0PaDM?oHIWz<1KPIJ|tfhvSV?=6ag2RYFB8F8unP&B=D zI_#kD-8Y%W-Dfp2oz=>$QHDvP;-q+`D<#v*;wCsym(dx#;K!uEJYy(KhN7n@YR;@O z`Zcr2#0`pp+g-A>6H~)4rHh||dLX-s@s;9STkLu0!v*^dNY;YX=Ye=JWJ<7{rVINL z?|pk(haUMS5CT8M#@>hu5?a_dje#yK!lsT;TT*HF7$}7n=Ob~`?(l)4^$0x$Jrf-A zQ#G-4Z77lF6PqvODkRWXidLL_VvCsu+^v1mFi4>&(Ylyz+-hG@b?Dli)m35h;^Ck`Z00zAM7*_uy`x%evizU+G!F>l+TD0_ z3xRew;e#hY`tJipw5`FcFla$SuY1hYjE(he%e%c%L))(&JPuQk8WOaP!!C~Dz$|#s z+XFuf7l(RMImP^r>`2-53U0bB8-mFTk1@@pQ?OG?6fV8>-tpC1&3VswY(f6G%SL_w zijGR_pAxuqt|~_6muZb1;vGUU75y*x5(V5>d_x-hKQ6T&ZeEGJ3J&(s^JzaZdKDII zc+TL80j2S{Li08R32$=5Av8^Wa~%BBp;ZQ4^a$M{EJ3(1mY}MbY7na(AH8LtrtFz> ze0loZq>Dhx`on*t2~x-V#P~8TFT64LtL@uvQrNHELF(byRDEtjF+;1vo6cL&CD)&8 zGa#A1;f8BZ@@bj0#{QJe}BgKc)eS>+P!-R^;fm7(KpuY9`9F_y`I5Z z!Lq>qVTWn&N)y#4uCjGU3U$B5F_o@EFQYY9%|!D3Gbufwl>eq|G{*rofCVPLw#}HB&Wr|Qk;NEXHFUf^co;2SrHNnCSiV(5LW;&H!IwEPt}#>! zq?Z#58Kc`=b+0_5y+gs)>~^jiomh?E=sCL_nUX~EUitACD!WnITCfxMr1vV4)8|4o>MnME-xWjph)f=b` zkLrEmEkUJraTl={w2^TpxO}5^=bq3_2GMZKlh5v06pX|`e`DQha>+bJ6PUMCvAL>A z1pO-%F!C`Jos5hWwQmIPzgL?F(E)BtIAS=Y6M6(vZ1s5j)P!oxalOh)T0)56Jaq$k*hKIk|FQ z`e+kY3?+&U8#!(H`biOikTWHD{Wo&<%7HeUncvxAhW5qmfl5R;Zy`dZS^aph6zSU| zJ_%2xQp_hAc|As^10E?1@Sb2=78{5aMJr-pwW2tO@c7kQdUCCBRKCxIxiHPPeVj8r zwn86M3jx2ZCe{NpY>j~v46L>De7N34xAkFhFhas^42`FRh&1(xJROZLdE-t1j19&Y^pEU^NaJpaM{*&fl-Yx*Yhu#$i4+$OGN0jiCnn5bEx zq*IqeD}iY+#l^+PVchke?h(igj=X#)>SFWvGY}9W7Nm`qVs1x|b_a^D(Xp6#QN%{q zDYpHd6WYDvkj#jXXZYiY|AhUsM4$cA?iZ+^57BqUzORAMMC#6vH6z9q%z6FnDaI2rSf3Dw)D=v-=wC? zmxkODOVd}?i&sgwNV!X_-h2u7A(taDea*KfpN^uvr+=-f?aB8^4P=$hVmugCb?Jl;g-75 znX6fRAeTBCRR|48hiHHw#iCn3IR2Lb(Yp&K3P>izsW130x*o=r>d8q%ub4bqJmQ0r zfWkjAIJPmUiNC}FJs>H@McdsGXJWR7XL(`GJo!#1_B_8{ZrDWf{>s%^|El-7JY5?l zT&Qlzda>d!u&oV15tP#99K0DY&P3CB>WZS0-s-7otj00(7K@>IWR44%|iNa_p zx+JT8thx{H7^X2cgE|wv-5+dEJ1zmhFlV#>ogpU~T0+p?>50n1~fAId9GB z2;BD}v00dqP9j#f2pF^R^k1VtYqTN0Q*NQNZS9H@{vl~s;H_T(YNJ8jXdko4gr zWGQ4SN#iBipeRIc)L8w|?gxv^{1c?i5i{Ky--g6|`E2o})%cb&Iy4gZ0G59IBfVc* zO2G%pUGF%Rqw`sblkm`&<(p#)Jc>+D@CNHHj5sW@jY(6{sp9gdWM2_o5{bARD)cRwq+#&gNpi`>C3zuGiVR1H|B-H7C1fpy{y zN8aKnJ~C$Qa;ho?6v(Bl)ulrFA(ub)sI>uV22f?L0f`SO57aT-F|i+~I-@FBIF@tu z%fTh#hHaed6XBX!SQpYpq)(f5ZK+c zWp3;59bS(P+Ji7<3z3#+O#Rq|xtC<&PmQ1wvp<> zc5g@yvLflOh?c4qkM2^l)%s@vP@NUr)P;ODPYl)+k6!jh$bUM1isL(^$9uS4VTyNe z@X@DE>TV^wIS)IXkq84qn^uwlROB5%XnW-U+`JyJw(?qOIlv(J%=kEjo?yrqkoV!M z<>n~=mp4Rs$`qLC^0_E%vpC!?aml5FN5Wk*h~YeyV}Oh8zs@ z!OkBn5Hv>Qz{q{e@^c<1_Sa=v%v2#;Hyfk~b_272;-Sw<(2!Do-FBp7;fuHy`=&f-~~U%yOo2Pod_In&dG(2=Yl!z!H*Qek@te?ocFNCS4S3X%2v-s9$l$Vgv6mlF!O_tDH1g3RZ1(n*_f%@EQc@dP0^im_T3@zS z3EfKXP?>rzdaMci7=?#r4jWwcZcsaQo-+eYBAGtCzqi*!QM4mA*{{R%bgp6sDBBdg z@|`5?2T4s|b6$J`EbZ;7Tj}iT{b@UR+%N^(76Z&UDuJLI_h{aZNnPF-QuTn7&f@ny z_4H6IF`$r)Fa^HI@8-L7TE9`z(@V`TZ1VN(z|#3W(buk5Km$0fKINFJXrv9X&UC1{dOUAj-#$Vvm%F}e5vF|s!V)QP1#K;L&w2IN(8f^;C%NHT z1nd^vtkNUyGqN|TZ1R#Q<&{rqAt)+DRX<*-BalmgiN5>$c&w8meLvv_%Z6q!(u@3=6%OE&+elqUGo7PaI>c<$7@WCj*K~jWlkr=FspfaK7{0bf_jD4 zsz9ljN$bQa$i>CJ7P@1WwN8zXBzZHd02l?3tbuD>E&*(C_2>A@chCR5er2}*Wv9Q7 zsWCqdWs#E)Z}Cl5I+VZ|y#!UcEOGbtJzEE1W81EpK!!7S3Wzz!s~lXFrp3t3er!h`cIgeuIXP8!mu}He2AO75l$O-(81t{y&Y*|*HmNiB_uB`de&K1%e@R+aCz0Sbvj5SMoB{S! zF(5J(RtFfFfwadmp=3NNB^%eD+9g~rn`0&OsDuXmM#lYLgti^!gC$RX3DMYx`f=LY zml$Hb;f+B>0&u;5ajlr2f%gkv|7+0-3m!T1d$TbiYKiS9jGJnvX>IX0FXWsAvccu^ z$#ESE)yJJFyZfcn_|~+a-0o{LrJs8NCkbE~qhAALGc;M2zF5``=G}*>7_?;B>Y@@E z6&C+#FTI^BVYXrpTsB#zCIk_4JBG!Gs5gCL-ky@@Vr;5BCg`2qw62BRY9{4}S&9)h zRu-=f-3qXz66O$ILm4tXxB?is(~Z$dsaa&ppC~X^cuGagtP~s74`DI;=%VE5wr=m( z1jM>eRoO*9;bi=u7693tnE9K7)%76#f_k7KSyfk*oc5(_!ivo}SJDk+UpWfH$sd=N zFC23`g9!9C0Xv{5;Z3Cv=Z>x|?o%i%A^I=^d+_8IUT}febZ?YGIMiqkpZSGjq!#QO z(IvjZ#Q0}kK()=DRgs2}#xE>V2xBQXoR{2rO-O6?p5I-j&AkC&}``;-3u zS~fZATtLU8^at1vB%U&rWa)zpgTu)jd!BdqwibV_ds)QKE7VDDR+RIgUeO-+Vlgr^$6(_D-P}Y zm%Dtu5jc6&((!}9G$RepygH{AZo+Jc4!D4BW#OgwdBJF?F7k7c1t@b-j16PVC&@ZL=UD+V-XEhvM*xVP_kD*u9!C=pv#gi z1D)(|GfIuWU&Aa$ZqR;In7vlgF}33T%KGfB{+eTK^{@SpZqzpRq@n}-q%zh&goG(c zh_E z=7FcB>_2j5+m?BCj;G){_UCpRjW(2@8WldAP7|v?J}OHUtD1dQFO+3lNh&iw)DKci z{@YR?&!A_)YpE!sMlDd8s<|sV@9ip%t|g3F`d@}oWgK2rJHyxQcEh^+UkPPfvRD7L z7}66Ovm#A<{`ljSv7MPNA=WGP0%qlYTQ%RR) znI@x$=(}cX#!`)ZQw?zI{{!TtgJ>+%S0;O1otVw{Bx4oG3n z_98it`95}sS!`z0?ITGegSzQQ*mvS4;>ErjMKi<@_oO0SeQrt|M>)$jZyd*tU^wT% z_>1P#DN&hsOlM0mNm9Fh)Uyy3)NCI?(BqZch8*=Pwhh6c+8*gCdkf{yZMnqY8np`C z-NnkJj!r%*Cz#F&+mJ)|AXPG(F5(5l)>l2+Ez^NDml)F07&15T z3vnu*7;rO=4G1&UG6X=K3*)o@$u!vo72Pa=i?o96Vz7y2_L& zQXQ{8!@uv@xwJNDP*8RHQyS6Vxf#t|RV1$MV0dcI-M)N=F7->{TlNM@At3~B7qhCq zyr5)5g8qK5QucxC&i$edZO6ALNjTA^a$cC0RPV@3iGu^f^LVmV>{P*6$DMEnMHwF+ zCdrtg?FLp9n8Jp;M3gA*wTF#S`JXLxC{%UK-R6DU$`N@Zr+OdU?k43h^q}o(N%fJ^ zu1?S|svSC*E#FF2gjd0b;+pAs*7pEvGIzTa_g-wyS=FiBF?ENo!t(6A!i^MCPp)j- zGASEVi;1giC9U@vMXigWZ1L`t8ldl^8~1hudlSvtBZ{r7|E(oi1UUCa?wq0J_ejVq zbZoEkn0*u*m~)*vyU8wL2z2FBuVY}bgX9+ZD0`zSR}z0QFB3q^r4EsjyYu{{z`T1J zYxR+iIp3K<3T!-3(^nh*d7s1Egu7nQ4xz1J@1bLgW$+4YsfT(A!RRAIGA?A+fBm`e zidgB&@?B46Y}i^jWFfoMN|nHhZ~K( zY16Uqw~(m7azhE+tz5g8^ujCF1?Q{SC>~uo*GD-ekS(FBK+JtD)Q)u5UlqDD_(sfH z;M<{FaJ}ox^aSiBV#Rl=F;~(tnF>r-b|t!wQwd};W02eoDDiLbRq05QgV#4<-+Ea;`?KN;Go6-Hl}7GXQtjae}eB zCW)3r%o=Clk;gj^BLbOA{>MrI{B@~Kp@~aOQKbmtxqRue5-wn z%%#7ATKQ-9%<`C&%|zpG9G9S^*Za^!9_BZvcJOC%Y_0ywTcM% zZdxUKCW<`6xZ~+W?6koSID&I8FuVd;;Ty#%p)9tqdvU|1)L*H@qk$(>B73%V>doIK zC0-bBqOwoaw$Ev2m+BLgQ$HA#nC@6pP+*g_Xhbcu^pzAtKY~b!l6frRO$ghGqlqhr zr>~Fns?D3 zF$=locKN#>SDj}{U5s~o#;WM=>ky^xzP-=)+0QDvff!c9++!D5w7?)@_hV$)vGLb7 zDg(XewrRcea_KFf__3U&-k9_s~Hb?69jA`6}p?+wh`e~M&ZzZz}{)I=BGE!{8Uu8nZs&c+-F z@MVr!Pt(cEBy3fb7E&$DnNpGV+HU6l5j{2i;_$ZTS1iuF!dtD)tALRf(4H z(S9oH|D}w%rKp;0g4G$f6gbc#G>(pbC<_TcQ)K;Z8T*k5kwD`qPiu|2?YLls*T{<= z7piNNiN||jYCfTF)Y;DFvVEofLjh%2a0TO<$z!y$fl=HglQD=Iob#UW3?iFtX>iSQ zRB~Nwj26O_zNqayE=abC6)sF(bGuicN`bdq5-jLnMW+i6Yu zB~>%{JD#br-qu(2-tHnfyG|l1Oi}|p+tLOSYaYMM1Tt1@P|nrbfWN1}Ht)l|>h)^| z;=dIPxx(qhkoZ{CUz4%MVcc*^>ge7Fb zQ@EBcvAk*gVmbcVHBp^k7;7s~&OSj@e)dxA1rXU+B)RG3V)L^R-^3`)ntN z6@jmq8RDFb+>pv*ha%aPQzaUDK4-b_d#+oahUf@YPWKcCfbXFX&Adc$*FbC=^ch8O z3w2V{&~n~PvSTr@V72wV7(h=j^Wylx&-pE-DT`A=&dz%22QQaS@hC$Csr}$3SR-=B zF;@_sCs})lI3+NjE!wp8inLaT$MJdbHel`p=L;KpAmfcYVI;~bMu+!(zi1Y zna4oAB#+m68=i)&6`#9R*;nThT7|rkbcQNy(T(FOu@Ke#MbC<^wA9^}U9*T>?^9+; zE8d7GpIC4Wll}^X(1hu7;4xr^g1)mwMIBo+C6|d8awYpSQS>=N7wW?j6 z+Y!{X=G|8QZJXOYZ_VC|KuBZHEfe=eAPV8uaK=0!`K?&e2bb};K|MoM^hS#nPEN4l z`g#fBf8jUQG9;>mdjZm(mHEC2SM=!Fpg$rIGYNj1oW0wqR@sT(fDm+kF_bOCVEhE} zvcdVjD3T1_*r)?#)L59S_&ok7)Eq(B7r!riK+QjZw8bbEbG2(%dLxZyNXJWCS>VoU z31fI)?XkvK$P|5r590XqHLWh6w?WGG^CdOdBXNjuVu^ntwp71M_pL|wwe7A{?v#Kw z0WtR0f#H2_wOm~4z~>5bw$}=8wVHVE$%Ah#Md1A96HmxXkrM@jR~aZ9Tt(N_n36>7 zj-d&K8ITzg7iaJK`7cq&{|%B|BTfF6#51YJ|6Y(scai$qP_gsWk0~BGwHx&kdo$_^ zMI{3l#G{;z?zLzfE&Dk`QuvA|n;<$}6<^nJxPjACnAcfLiCvG6mqrmf$e|Er%#Y#l zCV{N-D9Xp;h$7-ola)k{-V*C*4a;&bE>6`O9u3Al6i4RtI0Yo`|T=!FGuKOI7x%bM|pu&T7R z&)*Td)GMN^@{#!X*yaZk)jRTueVh5NdEx_-F)U;Qy=jS7-FNB2s;I_aio=4E#!8$X zKNkvI|5%~z?Uaj~B_ei;u77~;Xf$YzmDo~!|Hxp`w-&uRv;Vc5z{6c+JqQXhvu@dysJ9{=Srd9s2!LJ_}Eo>AKSk26-|9(wv$ z!*8ufPK*tf(^Z;Ja~ z{D-)DN2E%7xT!MhrtzgRy47qb_-AjaFXqkxJ&8z48fo%IlmPoWNux>>qUeNyXiWK+ z`u(1Ve(d8I+c<7@x_8~Z7C)Zd-jKX(2BQze*!KNV$?L1bX!a%>w;a|Kji;s9Nr!>V zl7`)7cXX^@3W_BDZ7>YYK@PP5o2K&l@TK?8#J34sM#{V8WEUv>=0*F)-WoRPjWubi zujymx+OV8x^(|vr8Q=9Ij}?duD?ph}zU`87(IR=-qw=bM+tvyvfu?T~a+O}I1Wlty zYa^ar)t$7n6Nvql+5J2QgAOcYW#4+274M=QrSOYsQ->MQrQ82t+VN4AivAO?=sm%b z*ecKIL)pE?@1bv5ydQB(q^^rTv){C7^_FKeepyd*d^dn{aBgK<$orj8{H{XZogN~a z$lHB%PzKTod4XZCzQ#tc%31~!1G*#fEBn{o;b$-JHw-}j@CYiW+>O$uEotO~t#$S7 zdRsQ#4*l)hHL&>57u3GHk^PVSI={GF6w6j1v#;d}zEdig**-!hZ+K%3H zpxxua)hjkk=T*LRqW+V(QlI@g-J?)~$0_>2DwCYwwp`IEB`dN7QEinb`k5GK%P6F@ z!>g&1g)if<{%j<_RZOYJQ=*-?bb7&CCTQ zpD4-4X|2aosIgsN3^A0bSK`#xYP^YtaA&Ma`40-Jj4+cqw4i%w5nF?Xfe|ZJS#MrVgT2h-(}(HJl!<kO^y+)5k{@>-AUlQ~)(VW0n;gKML2`-?(AR~!?GB0NF}HX zlLDg|%(*`UO^=vSdnzn^YzaGMEC)&Wm~foTc(n*qvuQji9pcz3@Q|O#(j$jsk6+hc za-nitCLoy>0SA}5?n+#WNCSMq>Obj_!A^-}AjVvh0G(-xtT+M#x?<7LZ(L zs#9xY1r|_MO9q)!P*=)s09s~$9V)cDgOYx1jF^e|&OcPc=IBYv${Hs%I(RvQ+||W} zZZk}OB-GIFQzaI1 zdmNx*lzp#FPmM=*XO~6a0lM*12s-$w*^CaGgK1QjZ=@>+8KSNPdc% zs**tpPmSM}vT$xUdr>0IlEXq-Erl6A;p7W?D6^Qa`Q8K0OK^9!B(k2z(M2TWWI)-* zDZ;hGuNA2WPKW4sD(XveG9lRY$n8T>%GU}a3)|fv^^N7z323;rx~~qE;&8!)B(NFt z(r03W*>F}O8WKlIzVtyNw$kCZBzvBO>F)kgI4pQ5AWT#>!N~vO=rj4n4vlwYLQY}B zE#Y3Kn&XiYBx>_ExmK*!v?xS5L4154N9@!m=@|OfZ?ErKDm1n=?KJ785l3b=vlw4U zA;5koZO08Dadv2QkW%xk8<@E-jPX9jlAX=_6yT~r-nL}X`2mRUG+qi07kWFWIrNp4 zi3TpDNYP@USB@NfPXSe+`b_lOn|YLq5hm-TH;Jp4ESJL)3`>xfcb2y#B=!7#f%q*i z4{4ST%|Q6P*h>1o8HAcXM&?DvyQVn5@se#Wt)_d$YHV$Y{_k6qFU5-0eNlqh3!fEp zi~M{$v6iN;-!>$Tx~+=cR!c^WY3|o=+am{Ky%OPwpA;q z^J1>Q^XB`;?{cdVyFJ5{%y258bjF)L9mUD*#?qou5MKVV?ThXOs)00YQ+6qX{{B^c z7~Z935}AHgvg2{9x8^au8WD6{&VNEzAMV)0#iXKc$$Xw5?kQ^Pa@PO~B(^h4>T+%7 zB&OoFk36}9IrHJg&8cqtw3~#i+!qreq8z@+jNegY%dQ3@vCJBqH-{{??2^Wlaar|x zqJoM`OYPo{$E^zVTK4!(MOWvBt8R7fu}>`IZ#n;q%@bHlKT5uowLt7nHgMIkYBa1Qp@SM>TBt??U*Vmtssj?L{jX8i9x*P-gniP_g%bk zmN*HJ0QA_5U{uY=K4^}DJ?bRW(2&NNIwYOsw}&JaiZg3tHLJOP3j#VkKRk@2TW8F5 zSuAg2XqYkc*Jmzmcvwl}!~0AA%qT@fuHbwW*tS_}b-YqO*TuP7ST=D? zN3s8*BSDs}SG_rEVgIQwbXdaX*`GmFZba<{So0+$PvNi5FTjs=VML@!ue7^<3?}Ifht{mptSN z>J;4+tLzm4Pscm&P3xKA$sDd;h)Z%iqP{yDailnrFme(7aNe{q5>`Ap1SpdsXS*Gh zcIS41ZV3c#%+!qfCG$<@?ZK*u@kdiDg#YTuZ5|SYg{8_|=(+$_QBh%jS~`#>;R&ra z$~&wzZC&YU@Oc09SipHFGD0fx>Vn0=;jBIdIq>@FYjWxQ@8Ri7>{lI^{!mX|bh z2(=-69@af8EFaS!PH=)R`8M|33y8P1Tm}>FdkmniN2~-M@y^IQs4gK0W zHEJNOMkSwB$TVSM%p)A2^b6*W9nE*mKr#+i0(H(+-t9Y5f^+1FLJ0{OrN%dWd;^8c zcr}zt5iS~x8aP=uLkHZS@3=*Omb>0T>l>N4lQj75mqJo#TT-@$w671-Cse^hF{$G^ z+xjRDIIj(_iCdofQ1;l_Ox4P!V}omtW+(%LSn$adsaR74$;tJNva4%rMVv^u0s-qh($Nl!~cK10OMRAAnUPE z<<#iuR9?I9X=^2;*jJ0pzY5E9b6sxMZloYTfQ~XJC#N(@|3JU@T&3ee&HB8fg?5$7 z(6Ymx$FBN=Q<3U4H^Jw8rkCqLU&Ni%OL$^3{|`f@5qB;!1FQYv3Iho zwrIs(VTh3|T@b92RPmq5TVUe{&7fEJI0+__ zlfA;@sTA8qOwpEWZPylBqs8)nZO3{GPtG*T6_?DG$ZDErAO?q@J?1Rawh$9D`BDb1 z>k^mO!J-EROO$Z2;@1ouf##p=HUb)3EX<4IgU>mdwyqJQ3LX0Tilr!L=*|TO-1?-Z zU1o!&Ie{yYnfMK6{_ML$vMl9o15%DKNE)Rl^<}4*P6>OcRTdx8Tbo1M!7#=#nPxYZHGDmsf6rsWjd8IjS4Oy&yKd1ij5ag z4SAT=C3FUf`!|zLi<&wi8F;OaEs2sHMPJaR&nGSmV))YOvV-x?myydcLi1f}vNbxf zI;yx8jo-qH6JTu;{ff|siq#O;8;tj$5WuXovhat$deH6zNY!n!-)E~$Pkm+TXEB+G zxDR7Ara!#;`uHgjHIYNK*XSZRLZm}x3Uw#5$MkhqdqTFWCdf?uhOfBNKIN;&o3E2u z-WLo7MJX6J;OXX^RJw&L&FU1uP%%MZRHdUmk;U~59b(!>#YHZKpz#wr+?t&6ES=j( z*pa9L#*x`c-7Y_KdQ91z3_W8_jAoz}iM~Pdoz`h*7guV~$Hu879=oUbvF_lbVHf>f z|Ij2Z zk_xG;auU0*Dr!>{~goZE?kJs=H2nR9E~-}@##?N+Wp#w%e$xL3oK6A4;6=WL21ekVnZa!Uwc-~ zuYHACJiQS#=_)C4nb+USI7IQD4R=>&iV@`+QeLqEDy&twh?S)t{Ph61f zc2<$YjG59~_!0bFnH0N?Z`sM1N*s*PK52NiS;c5ON{nA!T)n5;h}7G2C9g}qd-d`ejMZW|$Pc#1${q)~k4N6ZY&2>d#Ro@nkY{3!D;b6{L(*O-mbz zs3%vs?+Zer%1$-yzL{}e3C6`q7_tEv8 zEn8k0&56!BDqkn3xsm_u8fP3`b?ityz(GTymuY^L_!60x67tf-&gN_2XI6#2z?99Y zB{KrUv9F^+4j%^B?%bUw!uPKzV8}Z}?F?0e5vB$F1R>;ANv?xjGR7Z}>_A&C71~2Mm)R?)diep}`pdBb%Gc~yYmV@8$Ve{S&IgPZe^(aw zCL>=xpXjc$UOqjw6Bq>tA%VsgMdNTuCihy}>!`!!JCn7(IIKpI~*YuKnJir(;F%=Y}W|W;nM~2e5AvG5z^&bBXd((1A3>2DDnWnw1BK@FjKB4z5fga_@DNZ!~8=z zoLCf|Acp;TOhPJFh6lw+PjxarQu@+yIVOJjwJ)iYQG~RBvhY=yFu^Eps&np)j5h_; z3Y|u0ju6Q&Y%q&f&T|Cb16pIgwS2qs$-UMJ6#^_BSV8YN?I`3-=c;!ATpF+%>$Yo; zLd6zrrm^L-%M@HJMY5|`)sfE36a8k3%TeF4&P18cX{D%m`{Qi z$Z=OSi^#a{hw^667M{ctbp=H{!!?V-hQs4fVp_?0R5vJ@B?zw$MeXRZT<@5flcM~z z)>RulNpFPYc7v(9Tzl=$A$&#sMUTeUFdO8O4h++>a`SD zehdM!Wt;uXO@jW1ud_(-NRnNzvC4E8?}F^A0|oT223?3?us$;{bw+cZpAvj5)h>7o zSpc%;K%ySa9nS#hY=U#-ERYSAk!Fwc7_R~%b8fP87u+|;lH*9f5RTr_`cJd;qU#il zY(8P#GOYY_AL&9Kd&%u!IMX*pq$o@q<>PR&>|>^;kAS@I{7_#6^P{@@9GshQ2T(1a zj%S)NP^p3wD4Dy7esuKM^5haf46r2Q6vPZG=Yg_*G>-7+mv5=7!JslRl%NSo(_Y%p)3%f@OJ_)u=JFx*l{CG*DYrKE(@0<1} z#GC#Jh(7t!KH4^!g}<4KzaSiXIE#PhY6^u>3Vc@>duSjsGK&=fBY*pTfP5=Tt2Rn6 z%PFCaT)wAUDP>KignEI6@D(ow>QJuTCfABotC}0ab18$#530$aDjnh{=?~BFroLU zJTAlh0Q!}ZnhcXNkaL&pjg#yt`d1>FHnh^eDx4+rWCXQfB$SGySJBVH#zt_JWyaXV zWHlXyL%1YxJo`t|&B@mBD^>SwweQODa6B^v#@z@0%y9f9V$?(j|Fu=5IOSkMGnL=L zC{y-^CfBE;Oo1x$I6HJ_-}x1c{K4^{LX>xog1#KbUK}5v;<0V`X0-mX&^9vezgNo} zOp+2stRs;b+=|5zGesTW=_1IdXJ_Afl&HJlszglTFzJhXud)G@Ic+gyvff?E;1h^& z^u79M&b&68#56z_=KD{S%V_e!yjtzBjdBLp3|2OW90D2bXf78IYdxmAr3u(R9*Tkou;cJ%(NBp{_^_XOJ z%7UA(*2>gtJayqA`9PBLc7Ibnmj_f~x85l~!lo*x{?>nbY`-wxL#~(dP0;>&v1_cI_S1#W#oPh&i_2ZvJE&c3tsB1nQ1gpkqCf zkngv8pBdY#_Ju5__RmlqnWtV#SHC^QAXey}?cLPPh#bqZ)eCo+JjjW(yykj>!3K#~Ip2Z~p zU+wI?(#}i6#u1Fw7(ZbIXgKzJ{mb5A{^11s`65k01~Y(bgAHM0`Zwoo)WF?$J5~am zpy@GjrRanACn)G;KU*jwW1bl ztfaKxB>rWS&6Fz~L({Dggqoh77bu_r+-Mec2f!oT20S$t&HHL=FMSk;TlRhv8Q9-m z7whA4tmfLw@dByZ>@Tk)q0<(|L?}ORT5OP8PGC>MQx2Bq!=%pgIMvt;$@LCutr6Xr z$qVw)?lgVD3i>=2eBE?5p^S`7lmJ?6PI0rvuQmw;rT3;^(%6h9=qpHUd%%%OYO+g#ZhzM9$Ye3p z#+pc^qvAlSS7K@1Z*jT4oO~<8$u~$N)933`bSb%7D7u+JESt>{@twK_SAWI|Ap-e- z>m?ot$LV$F$JWPeC=?%8&kZVYqEPF1U-|I{J|RR^5&D;viS3CtWwtF(;N(qlfa`or zMYLqSU4J~09o9=$Q3}arvGx<@;d=FBRYVe$mmPWk5^!7e5MUt+k z681+zv%2BTYcbo`P4L4{AMUT&uvUV6(A$Ohf`yQ6@btUj0q%)@_x;&?)+OqJ21yhR zD8i2xg;DpsWfu2nV_sLT=KPh2RCaYqg{89aVw$`;l149#^g9biZnT*phWsclwpi^5gvbGf zcEl`b(Y)YYTuV9bOYK31_xLm|*!HkXK9!z}ZS?^gonPc)5~mwL9uIfoLkQkc8qMiQ z;&@LX%}2R4(7$jiJpC6|0OViX|u#5un}oQ zHe8K-NoOC^7|GF3ayXwU{2yl@(~RVgaSqo|Q;m2{uRyt&NO{7aRX*Pj@YJBbM6cfI zHrin@%w`G@N!+=r>3DdPpy)6sMfga&fYQpFk4Qav&2*X{SFo>2!9l9}hF!;VQq^S# zGcXSzG3n{t=Og~^mz*hUM&Gt+!mC%y7nwQJ){T_L1v?9&o~mIubW}$3yy(^Ny+rad zBcia)l=HBd1#R5(zLkC?a0P<8JZtwXBgD9q{e45v>(d`?kzX2)E+-(i*`&_-<2L-9 zpZNKU1s-J%t^b?<6fA6P?OaSTD?E0^W15~d9ujAjgNGR&~8 zB3YtE`Tqk=J}CMZlUhNz55lDLmz|Uehno7*d{5G?*r!jRVwu=6tC}`+5m(eXe9th7 zf)fsvlw9$|P3LonzrewDUF)=zpX505rysqXsEUCK0w;|+pWFyZ1s7?;*t=AjxV(V4A^dZQgu4*{?h!E{3;fphje^&UyK>)Ifjh_0e>5G^fMU zVQYIjz2E(bwu?vniKN{RBI>Z|KDmng?DmtTSVwU#|J{ItWYipmik9FMA%ox`VgI{D z({4|{z2=Vt_EmG3qGs%kr{H{Uk3-rn;+daNIRb|>!GhzSC>CCrP5 z`8Max^^ftS4D!wQhPD3HfpZj>NpiYY_!NHqqY=erBlTus6*526L*L`!kD4gJ+1$Gu ztSJ&OI~#}D&3|3D(fs|5=O|gw9ors0^YVAbH)lSFvkTRw+gm#8O9Gqyl%~>ON%8B1 zUtoFTNaP%Ii5{s(HN856QXRf|y?Q$)@Q^P?P@z8`$z%V*R7wGVK`m#N)FK!b_YabX zZ97GoM*$q|BaH&Fb?=N8XLk{ogO&Z>wY+SEw{e}DBSZ7;ecFwyMsUU=Tbp^R(zsd^ z{~Mi9>KCT}%)$+`NOn@RPefJD_%%<+IAb^$)#&Uq5Xu;fgwR8ZS$2Q^j0{~ac8Mn# zShQVLHtrq}zuM2XqtikD&^vZ{B{yXGmt*sw zwm3rMM|7k@OQj93U~F5-6TjNlPv+qV(a}XlQmP;6bYQp?q%FUfk-tELyK!I_4bOI0 znRN2MRh9XX*eOL!9q8?kcVo=rACw++g0L;?uS)a2Ryr_wY^2F2=*up#>%AWS%9(77 zu^nnZe_%$#Ozz1A4`aa1#UEahFXX&my!Lo0>SK`eL}5X4RLewKQ(YAaY9e(x zhS3`=!W?=Ejn4$Sy>4$HIe0yAQU(~`LMh(!@Dg6i);o8^1 z(=e~?up-S5GGi*EKaFGD+x<`h#h)s+P%d<~?8Wi-yO>2p(FH6Y2&IV*82#+r`O&*$ zPf)4biwe2RPgn;wgp1zEX?UKO$sR&4SFtt>Nnfk?!Uj(a6WXlT#~SWz-jg~-4l}(- z`UL4oALG62HUBkp6dTGCwUjUgkE5n4#auX6LlgS+Exg@TD|iOOuP*ZoDxofw{{q!! zG!I+#s+RLcXLE{T0;6$=c=RW9E#XCyy$8>M%89d%lB9opuZKcwdjSV)fHRC=7h%;OzEC2r@$dL5;Ue{jn%B;vKL<7#HLg<$Z&iLq*6a`TT8Is_>_soH zM&XMdDXf3torrc_X>P_@PGui%N7f>nNs%RWGj+h{e{W(Jfc(a*CKKViVI$u<+AIkb zn216exV%l?9wKZD3oU#Qh@F7bzlaB1$2>z%OC1>mBiS+GD8jYm`t=ENdCz_P8+D=; zYneaR;&)q4T`WACb5>B-X*05{dXybZY8`3W!CKFB7%4Pw{S&5CQ;E3ki9RiYin*m@ zEd7zz+4hr>)h?6x2wVVcoCL5xR*nCCfAHmXJOZIHC&DfWV^c6WEDn&tPInW7Vus0> zLuph^t~2H4i$2scQ4_27LB8QXerHdQzuy`8kRe*CUZ66}2VjnDP~)(s#?xLhaY=^g z7lUB`kIuFnL&{a|5FDZV`>Yp|u$-IWa9sC?O(}g69Itb=^JqP$h2K`j1b&b^9!JtA z6Oe{;*|OK7-%^RMH}n)fpA@0Ib}H=G9RMiw^N%k_>63w_;R z+M>+KPqdF}oT_2}?Tc!p)bE3?{WP`h27-xOB zE-`=mzP>ncKxDKogT=^EI?B&Pz60+a>3qDTaA`?!c%$9G+mR)oqU0wzkGxw_1Q8J zhkYVK5ILdreY5kkDah1*B&mSwb$~NW6&Mf0)h!rFnH8a(J+X!}dsn#XK&q>eyGu7) zCDU7on-L*Ge86-a;wAEGO@au=-imcTdWWpXXL(e;H zt5BPY?(81!D60=odMg5|p+P>FZ83# zQ|YMKWGQ=Uth1Xtg-v?UI`&6IiEWC)T7Le4O@Jv&FvOt6P@E&)|5)YZ1NxV%{|35a z1ZE3wQ4q6j%YH!x>(b1l^_90@oyps2VOJzC6-4R7_JgOf3Cfvp(dk};b&F`nn-5Jw z%>0L3RX%e3A>>i)4oe6J8+f}Y(S^7J!oy1|8ECVDSe5L?UJjql#o`C!ww>wGn`Zwv z>ec9j8}-Pcsa)6lka1eo`zdA?x;*up-IZC~ORu+?v>&LZitM4R@dkL#N0trCHTEuJ z=Q{#}8>4mSZP2UD7I{s#PTX#t%b-7Lhn7$mnd2pV2Nbf1R1reH_$riPSXYJ)lV>Z{ z!3JsC%7yE$mTEe;-bmH_Mb31W2V|Mdbu4*oureyUer zQjxrU`-ypP_G)f%CRUu;=GE&+mH&O#&r=^Lrp7r6?<>=kMJB7Z`wd!>j-_ zbG+62Nf*^}c5mO&s4VC8;g3A!k0XUCMoclkX?4;E@Dinee6ZV^#ajAjXKl>Xu^tc{ zOtInA*Qfk`)zOQ<7k6VO0;N$^X4FdT^}hMib)+H0!N7U3dXIByzhn7nMMnrRU^c)@ z!iSkNe=7iaeB>J8nT?Qj*MGxN#WF$W_(9Jqtw zCEPmG=AX%vwx1M@92fYKZX!PJu$Olmx|leI#<7xjo85D&zH9@tf z7~MP-6=&6XE7doW<`AQWoKF z>3@S}4j_mD^tbcbtWjLt%;GCCJ=puKjmrme;%dDb2PyJ4{`a zToA~phIR1b@_~Ib0Nw-Amfq<`-!me|1td7;0e}OP3vc$TMwT+GE4s65rCN!oia=@Y zEP`b?L^5ACt7U}mj$*p3uyA{>W)A@BU{*5hhZ~Ms?wqdGOv;=cD5+#7y57e|hfyK2 zjO20?V0ORqtzyCxu!SbE?(?nRgu5#r7}@EI?HP;m{G%|mD@Pki=nL)SR~YEhvo-GQ zkzT7SdT(Usz?f6n$--Gm$TGNngWWcjRr?|B=^*=N85VlkG2!g^_~q)cK-*&*gX+Mv zv6gA@!Rqp08bzbRLZYyPUH5(~o#9atJkcU+x8D=ijn=YrmF3}n`K3{o3P{WCB31CU zmKlB(t$FKYFB7~+|F+^;znxaNR5Dah`E$p{c8}Lv_ybvXn=3vA^5)h#D}{}{0wZc> zIPc%0?S_#xYvZrKb%kTTCI#0A?qf!NJt<_>6-R;eeDo?2K!Rnp93J*Vl?1vNXyia=W%fGIMGG zEnzZA?zzNtLYs=@K3`lGn(^Ad^0|s8cE3X%s@ljh&(xq8IB5^`<#ijPV~1D=I=f82 zNW_b>MG9lIUrqYEupp>4@p;_+i{Q5Axbm&|45p$pEB!n&-_w!%G%_zAX-7r5o6*UI zH`h+QcWVVvVSh&%ECpghzMs~hww8kb;8KLE$?g!{SQMGk{q5HIjx)m*-Hp|+I_b`z z6sN&)#T+d=$o8(N5DEn#be&6owt$-vH1WJ!5(*trWHpv>LOVjy{6Y5`embDi^tPLrYaKi0y>MwU zbU!!$>LSRosP*C7CisWFSa;&WXQ^avzq!OsCTPd{OSAWWH;1wLQssj$?`^H=WQE?k zagk~g*2nw9R+c3#)~ESxGqW98gEHQ;HM&FI{HKQb;EoUbabKH%aUZG97Naf=3H4b= zo+5LlOom>;jxA*tNqOWXueN95Rq}3doo1yOe;x0o`J99>w&I+)PcjC7tMZ(8(1ph(= z&SUB)-e84%oyE)5#&~sU>e%B-{%~43UXnHF0=6~PQXO@|L=Vn8f7@x4#*@S3I_%ws z<)L@nv|co>T{XbfCn+%PgWmO^j%_*y&N(^}-)6vMKin?xxzm!gF?(y(S*PnU@&B6d7V^0VraFko zIJ=~qE#!1RgtQi6n)-fs*neuTxAW)^Y*C18lBhw$g)|Ujc5&i&ow0d|r4BFsAP64K zQlH@@iz^h_v-$7;+W~Gch29JyT@I(7|piB2?~@t+atHIbIZ&q-19!5<$Oq z`gCQ<_H~_~PM;uHBx}5`sKF(wE94`E;x^oB9%jze8nip)lOeUzMz{Cdve$~*_6(Pp z>%+R38!BaE&8knHXdG*`<|Fr~6Fc&&z?j{wax)M)ZP0Kh)n>tYdwLW{+I-uXB)nT8 zpugJvFtzV=JXR;`u_wDlTx)dJQ@D}($NHV?K*E9Wwo~xb!R722#s1n?wv1|p4Jyd} zy^nn!1-}4chQvD`iTIfz*2uSOPEsFt)zi_=%&0r&on5j%WWRBa=C>7nq3y4iVCY`n zIYdp{`qU-+!`*$@Go7=0(9~rDZIQYpc0hG0!dO%c_ZKVGuWCvvp>`BF$o@uZ21Dz; z9dM+Ql0n5+_B&S%Xrw^$_+&aN02aotn#P*vh4FTySi9*G(1iHU23a1`29g1q(5R&j z=%02pU(T8T_uxshFz(BkIcu+IjqEH8r1UxB4Ne{-q@!?cGl#lq2TxxThAr{#I-mQY(nm195Zqe}ed_nDzhYx+K!%<>7Tj{(n#RDO=4%bts zvuNmMTxJmC7}j=~QNi z;f$nCk^g{)I|vYQPHMKXJ_nWY^&HnKgK=vKdN@yFW(ytgLb-rrwmrvnV^ z=2I5IiitLHdclfsJlB9+Jr7`M=?_t5c~lk`M<_=vsm-ry8S>XX6(}3Gj9)H(M_cPx z$uLlmebx8cbxk!HID42uCmmC-u*duk9?L!u~ziLKF4F32U`uFK$#2up}xk zFXR7wN4Az+u`QuroR>oU4BjHxBKZEYU7HVdz07V@93pPW9wo|6PGRG5!3h{ zvl-K9@-e+!^z(gtJHe`{v6`t{UGJxXrzq%${o5Z6CXU_xXb})lN%m)Qcn62n@$2IwBjh#E{pjB;iC_Qy^ z`>&|BdxcT=M^W_i*~w`)l*Ou2|Q z<5yp@LKSiB5+dOD48Au})D<#yOBGU;6{UU^ms&18@JtuHFBMoIm0T2VJ%DhvFlE)! z3KY*VpA4LEnl^E^FP_7&|n+8*Ta$?78LnxNg(&uSA+@dw)U+>~_$|YAsm!7q*gO_~e@TX>-H_FY zNNtvAwO~Q;+0FC}s$TGW@p>O7o2$~BPgQ#E)oQ1)TGG^NGhWfTQx+H``{#5_(uF5~ z$V``)#Mqzxy?%H>aQE-^mZxI-~5{+XDI+7%$AANcW7?L#H}jC`C@cE`A9HsBxzgmSXoWl6Z*$h`J<%h^ zo$whxpni1cmKa7K!TPz!YWZF(iiw;^`PUg+K2G&ku+hJFCif_WEi@9|W8!fC0!#j#+u3Tl#S@^Tnb&SN;$`15 z=m4lAbl$3|27oF7%@TN`g&$h4wQB(3bm%+5-3%Z|7MqM@);|LC4+Wx<+h%|a^2}-7 zZUi6r*W1@yVE|B&-M8U53{iHH0Jyd|0DWRJ8O{Ktc4L5a zv;+-@;57is6L$l4+X=rHub;2b(b2(swNUl!n`qVz4-E}b2g~w5IDmi-?6KD1bHVxo z*mbN#aKyv|v`w%W4{n3t0(8Kt&jXqt0K=FAVke+tSOGa%@N)1+6Tn+rpTI6IvQT3& z%Y57TI`1#|G~?ln2H;eOGqi9pFdQX+6;rN9C}(*QQZRIYhh6C+f`?tbQd4Aj3*g^t+gqH8dT_nt}rigwI zl8H@+I&~l3VG30IR_R$H2tc-`Z95W>a-m1n{>P3p@u#+lFvIonQ(XiPZ{y=y>yhN=QGAaZuho)HO9>w3stwWW;gfgcQyI# zqt~ecyR}*!J|qd7A?@m6%aUuKfH&=N>Th3k6pW97>~&|i>XNFC&gJOA=VkY^hF;d& z?{={VhKqBFvOF$B!ADRf9hGmd_x_eRceI7y5jt;5DIVT!nal2mg)Q!}*9yw0?@R&Q z!?#`CL#-w^&U`AWx?3NXkK7LnMMcwved6CQJ376+&E7!Hc~79Estb2F-plDO(;;zr zuWi1=xt$X^4s2vB>=qxyW0qccBcI1VK{;24^~`dWikue3G9Ub~y+FLu%EvU7xl=i+ zcjRAMziP|B(D-kpHC!jihE7HK|=G!iDs4 zqhijc1~-zqLk@-6{69at<&6AA^Nmmw>ron-dE3g3U)8YzFeMG^XQdiP!qa6fx4((% zA0O_vHa3)elL0aFxl=tF0DeJ6qMY*YZ)NkY5Dd_MJKWcYRHF@`NAz_Ir=WY=dagQ@ zMggd3AJ(r&JvStOiSgTc4yT>kw$7&zawL z{50ha0@1&}6hXp%>EA$KLTq}v`;zMx0AYCpY38>%TRWiTZ6j6;vH~nw^W9;S3FuJ< zfYzI{t@TY1UbulCQwts+R~i}`JbtfxfcGBIDWs#Ot_QH>yNf-XxK94ZlV|RxKhoKX zKiWGs3pFdN(mXxTU7z1mI-6oRy1hr$UeEru zV3|gTEvMV--HmKhdw}=J_{nO@21`P_lrer~YRPsYM$($>x+cG&9d>iSPU@T8agGuN zF}v+e!cg*wHAi`>1|xd!$a3#11IuBc8F=4RW-=?b?8Uk6=X=NVl+3Lj00#jk)BF6f z=&1gYP(??fn(_V8pFh#i-3~r4@SkTbPaa*aye1>Js{N&2*HaI$&IO*R?SdN28hJQ;mLImN<_t+K@t*@0I)?3s$MliamxvIyV%@W8Dyybr=T_Hs z>u9l~v02Bb$~BSXjGCvME8QV7LI^bjou}<7Z}L+`ooK2#LXm3X{Xqkwb>o`h+f_)L z_tV)(`qO>W_;{8AQx?7q@19*!Z5r?1!C2?r9R@1I<7wzm=hLV`&-t`aGd-OSD=?oq zVj1Yqc?b6g%K4-fBDlX5;#Q&($(dWRL`n)K^%Pa)ogbI4a>*j`+)C2UC=?;hCtQh) z@D)M4&}(~)LtP<;c0mcD-xx|M&WFxndmxGg8-EY;c!6vd?fF?Zj_xz75adGG!)+j6 zC-T1-{E2!1O7!U8?f}e~*tQGfc>xiFn+E}B|GB_TG zbg^lFy2^3> zV{N(X0IK%o9^S)rsUo{(ly_N`sLqhn_e<*;Q#4HldHKgdgSuQpJ=Z>c_r)%#M$#Lb zVZH9tJg-2b?r@z9j}h-B=ex8Eg15prd75<#PD7~!Qm=C6F}JAaQ52cPZnLuJRn?=u(4(1*AjdiQ% zD_z%l4IV>WyBNXTuhTwrziW1a$d!5JSc)GGO%!_KXbyiJ%({`D@eRnZd0lF z_^wGmQZ99hmEP~8>qns`b9yC1WrYU0sp#l|&!6MTg)>9MD3z(mHDPqm6y=mUgtyPT zZRO*%BxO*sx#>TsmABc+1AhjD39YZbTP!v@g7Rr?7TBVlz%d0;6n0OKV*n=fzOPue z0bpcRZAWZW>u1h{ck!)6zCxuZ4s7BreXJP#@$QwgL3e4BUpp-C8(9 zsctLZ?@Qny&V74Uh}Ld$0$X->@9lHoyVh+L1-AER&x<1LCD0c?U$7skP*Gt9H+upA zedK)`TU$rK3?y+wz+sji0Cnh{D^;)BEXZ0Lssy(PoWz5gX&rC?rFK$LmbV7r0H=0= zkTo7Ro7cZ4B(G@@_mU9|_4+-dR!n+Naqa{ZiG3Ow zt+BtuGOxdX%)%sD2W=YTyO&92OTKm?p_s!P)FIw5ZOwzyUa@WexLMuA&;z|sik!|5 zxY~|##1`S)AWV|S!>~pn)bi3MOJS(%kvA-8Bp7A?RJ)n)E{yFUKWy7G+2E?|8p3%& zc=ZxObjfGBGOOw^dsI&Sxf0|#?yQ<^HWACj93OAHX84!y9UQa^teQ0sy@?=vch}$3 zOCwq%-M~A}`7RxZ%|HdEXxe3O2J_Lzp|wi$V3Lzd8crI zUdYHFIC^QWIR%HAA4@nOa+MD434g3%eDGaAHNCSD2G^k-y!Dz3kN%*8-umEp7{dhT zMcn!@`EZQ+yha$%meCiTxVH;+#J(GI7cZBafs7{#0Xu=qW#XMPfBvBr3gibAYH^yI zuH(w~*irY(yW#BEz-!C09|93puMz!{hSp-S&>s9yt5K6dpUy7huD6DTiP`elK~DAL zRX;Qw2-?H92q&_r*2wC2`V}pL4)$MLw;c^9CMWZ{+>L^?-q^va7#ei@)cxCJ>1#-1 zw*G03fsWfIALx~S*r`A)edoBx;YlEFT5CyB2$})>&A7byJ+nzWeIN zla@+hCfP5q@T-b_soeHgSe#*1rgLMT)dXzhn%7O%^@2=z$kx0R93>^*3<&{{H^wwh_A0dVS!`L5_ik=YF`*Fy@;1avFfr zEd%1?pa6syThpV{?1bZ0dqR_Kp?*D28NSKt!1Z*Errh#EgN&)n8m^Kj{ZC1eE zgv>x)&o=tv8R}R3y2Ejv_H<9&UjbO8R1u{msy0)xo9dsgx1UDIhl z9ert7SXeCi`c*|bnFMcn$i_8^*`5Pz_087FuN9DR zSq57iBB6}{>lCO=wQOgH1J?Ts2pa$Wv01_QS`fi99=3p_ccH77*W=INcXNOt1mKBB zxlSf9CD~(h+%}@`z5HK~e8_LPaQs=tL_@-&BSzoV4wo4t4BeDQeHe$Q5r4r{?}OTI3W-ujrxj=o=x-Pnq!?BPpgV)C8ExDlZ7a#fr7Ji zQqL*Qw$IU3M?RB-WuAS?XfylmCckuP4Q#H*3V2WVO}q)&N`Ya$bcf~I_Z;<^V5o9b zaEgjY;HT=q zQVCOCn{Q6@9qiM|a@q@@I}Gg_5kKyAD-B8=|Jz5i2$--KaoTs^)$u-weIM=>Qi{g6 zU+0bgV2ZU0@%gQCupe&4a~=|(J+Dsuxaduc29cEV{`NQbCdFOX^LR%+SeO zHqZU)@HzL6jmoj&12P^9OKX#QAX@^=jKh?=2{`eltfWMp7X>0YuqN76T+YYDm`dlZ zTNgLOMI&&TJg#l>67uZ9TA|nMsp@1I;faqkPwE548JF>^Ks@CML4)_rfG$|(nQpr@ zQkQ`^I=G(zd{|GZ(&OfId|YAb00eXwQ<~N&xQssl++IFZ;Ts4Q*}f=&({&)uTY#4` z22QTaVZ+w#4brVoe9Nnpm4KZDH7SAWZX~5u;Nn#^v{dPF4dj#348p zcWDM9w4;Wt%@ z_(%=M$o$7`w&Ld~i+K~fVz2%RkhIG4rSB|lI$Z&oaT25-Y2gg3lCmmUkN zg^?^*OW5!~_@YYbEu@p7vh6J$)ZrZRHh8-4XC?f8<7i=D=B3+Yzy7^xtAEt_-chal z=O)I*N+ZP%m^k>ED#wCbl=Rzwylz>hWz%Jv2dc*~t@_i265*1uV;mEE-yp|-^wx)) zuZ^Alnw;gMTx#w_-5!vX&Po!wuepw0K#5y`r;Fm%cmyMMz5JA)use1K@1EXv_sJm3 zax^6T&@>_GW`EuJvn5N1!IiFM*+|pIThT2<>sEB_zs2f%m9d!jo6n}M`Ehmr=@uIw zXW#WI=jh4!sk2CWmEoOm85uV7Rcih;);lRA)s=?zh%O_yZ4)(G7$JW~W!eS>+6}AQ zmnv73q6t;DS4)fmV|zpoVcv;lCz8!zOH)290GT*1S8(XtVU6d_7?|Cb-?8ab?w6k) zbk7Tc*9QuGmAWo5J_B)FhP^R>KwiZa{LalZK_12j?1rOB){BjknQI_9xJdkTEjGmu z(Q=T#f5AehT6!McA2Q7m4@yv-sMdOasw6?M8>4rDpfPQ*VCN zM8$E|KY?@D!l~Bro&Ww2IcJE}84ux0qa_VqpR8Za7@$pECMYggdwYWjsooMK1i73K z_s?cPJ_DrpCbB!v2Q&U3PiGw#RTr&s5b2Z@DM66>0O@W7=?+P0kdp3@E&)M6y1Tn) z=x(IDySwk^KF_^>sq^5R!^}Q=t@XaYwKg{^-AG>_H&8{s&Uau6EGh|Z8yk=wDQ z3<}cPB^K2xJQ)+zF_;D$-#UAEBg&JnZu5%){VKZ2II(u`X9JE}#E*RM#R7yWI|RmP zPv)8&Dvrx=Yb|wc%FfR=%08KV8_9ubonvng^o-2_=IxbR$DN8WhQMg0E34o}BAS-M z79+$m?(*^E+HOvULY%qZYF4M5Ox>AbLsx+(ydV2v&k5|-C2d0AQ=t6e%)c) z@W?M8WzNd+R!_(v82O_Vi{H$M!z|H4(dw6H125k!rRkfE!R}u~b*|69h?&T7Ct)r^*v3L}oS}!7g7zGCsg;i7<)`4SAw5pfpsGHknK|1D5Q0&VA_0{I#Zm+buec)w zR4FLt3+@*?zs1#{4;Fkzs@_bJGP_8qw|@qf-s`Rvd)m?Ho}6F@9WyPBVUu2WW(H=n zavw3Y++-jS6*Sn)sl8j!^1Mo_U8k;9f4}TP;%M2p?Xs&iKzq$-)ky%I4DuT{YL2>N z=+oV#!AX-Qq|0(uefV`zlr-_3ONHG;T)Iezm6p-sqMi9)m{iz$F z7yaKdC=r1hq^o_+W!epPpjjDQa15D{UQ%(nwbRB+5S@r4nbStspIKO)9$YXCI10MW zE~f_f#@HZ5q@$;=%Cnd%34>$bR2lzl`o-Buu-@EM~0OK zFgg@k35d)ujPmNcMb$2Gwzi7X6O1iM#kgiuU$LxAB9)=6*C_f843x(_Q^ttiIC{N|At>as7CVQ@3Z%>h;h`=CY`e3*5I? zJoo6!ugh|mt4rQI;+J{$Ld@qKa4;1Rc$^$H0x^y_&1n4T?YBMdmXbzVo~Klp_g-up zmc6uE%hcO8i{qBt1yewQL9gN}x7~J}=|uJR@O(FEc7J27+mWNaV1YIj$`HVfYP)?K`}Xg9%|j1R(Vpa>&EO+`w~x zJg15Ia3u(AC6L3xdn?uiR93PS+i0-7p-3|!`ubTf7Qtb+pe7X*UchZya#=@Dx@^Zv z2@2Ue1~ihQ{-4Bi+pj;T%k-AV%_C2XbzA1VLPXAJ*v!W%H|vib()?Ao6NW_o-lW~g zb)93>U-I>Dye%`GI`2JKr;Uk$WlCLqt9vmKSHU@@>((nKnXFd@pYxl|01KiUA~j}W zw`2IhqVllodd943MJ--^(JIjxp*kQQzpSLwHrVA9f356oW}0t^55vW_2m5&jZy+Ay z!?4ALeMWQhL799wa7@IWC0yuGxgx7y!CRWcnA+{BevI*h4}g}my^5oAZ!0$1^*ycA zySp?cQ}rFi{ierWEHcwzzJ9lz2BB{(Rb$v*2(!7`UnyrmKZx)h(S!R-5Z@p_dC=|p zPeJ08x`gq<6eYjsyztA#7qOeyv=a0O!YSv_B8KMdNW$ z_3zgHSbCiNGjeo(E}Y4H#Shw0AOyi7VpR8qf|w*B_``gDO-AlQgy6^<>;s(OTM8#+ z#isf9bCBGhm3wT&M|>LO-@w_ca$NF$e$=vQ!3#>E5Jy!r1$z@69bM)M;(DdtEI|t| zBFva=vHz2&*=V|OKpkPCxV%&5{@bUt7JAH^?^4erhgP2R1kOousaQURe~#6R-R zR*4}PhP>5pH#7LSAEglZ7A@utxnR?m-y#L!<~Y&#R@>J-E!)Yayw^*|T2-CyYxhg; zN8BUlu5srJ_sRAIWJI3D?Z=JP%soAfDpp2lM*B+vIv3H(Zv>sWAiujgak~!NCGEl) zGl|A0Pw}$Nqhl!bZbFydK3na&EX#h(bW2jph$a4`P1QpYZ*cqH;IMB)AD4S;Fsr|A zcsBu!-r#>ROm)Q-=K-#x4`Ku3U1aN*2~&se^@q>yyCKNy!*dlM8VpKx?IYnZZ&kyl zblx3KSWkrqId8fD$LnMXeHzy%8+N5y`jwe~3E{uu{q|#G>z-pN@34W|VX?ZxN5Vq0 zdUvmOyXCoXwn9{3ij2;MJM(Li+--syUFmqaZMzI@g5w^4oHan<6(hf2c{HTI~IPii)qq4>u?w zcv9;hl*kt{<=k)+aK=Y^$MhAS6v~ZrJ{Lx$4M}CD=$O~()ecv*`}r0%imWyEEBaw&1C52NVgV3=#Oc zgHJ#W&JYd4momLNm<9Ta6AmLN?cVaaoIvi6<}*KsftKwh&+Elu+J!>+Cyt0FyzkrU zc0Y&=r`h_`?K=TUwIKaU$%|FpZbl1H<6c>g+ZcG&6(TRC;!OyNGcFTn7iVMx1cW4k z+l`oY0)03kTiestpK!d6d;h@ugE<&$g+--ynfl&_ALJ)x@`vCs`99gMi8<$k$_}`H zFE%w8;YBU;cZD?D7Bfan5!i5cR|^o&`@sZOOzsV=!F`~fYP9eqd;tDeQgf~#@E~kj zE~mh0gHpMR&OLfY7|ru&<)BWu^(?#uuEFVLhMm?*8Gy5lUQ^~7;I8S~ffmm>0$7$iVEl>A}l=n}>9Gt5P z@z3^n;%$3NLXQF#;-R!lsbRgmr#36j?{m4SIq-QJG|fO zrAgP~3F7M?jL%%+SJ}SL-Gvh69qN>shZH!@9;ct-(OKWxvI0u&a^!l+ErZw7KtN>+ zD(Jomg#_0P#L+LeOlQEKk~|Dl9W@>rz-XN924OCG9!i?(?PJI`SQNWcz`#%| zG<(z!{D|N^?9YmTj<%e9BfP8J^R6s@bn9Bqr9N&b)u?c}DS#>_sg|v(o2YrzWV7|H z!o&X!USK$+Q*5TPSNX3wgCFGAZ?jU~`1F?S&A^8ZZabaSrs;%@Dn9X{Es7(tUW(q^ z>0)EAJav~9xGmkNVb!Fg@WUtP z+V#v*%ad5*dr9`jm5h@7DATz~vjj~VxZa-l)RLJ=uF&Vi=7awT^&$DAeU|H93yiX; zhmh%}N5g16G2diwwDj!33AVr^J4nLkP3efNem)OmWAB~qrs%4xeY`U0gL<92koU7( zA5_p){$WfUhHJh3^bvt1-rJS3#d7g(y=vk(ysk*K%(H`Jjn1>-7oFuSBS?g<5-e*pS~54o!)#NL#9r~@-D}|?vk7a z(x%QMDpT_cO9H5@J-|vkeHFhS#$GUGNX5$cJqeJ$T)!6G_DfE+8g%puWYo#LiP*Zm z+mWdk942I6HZ7~VMGl~3U2o<{Gd6TGgdGa$A!;@{m<#j7GhRShoury&1MHy9<2ul2 z9E2kL)xV#2+8Nzw;b{EJ+C!6VElFbIyH)JuS_5>Wzn@!LVxw*+H!qtbzUy@+OIKPw2oW~BIvJ{NywBgO*vqxmfwOpURyy^9R>o( zyl=#{m4W>puYC`5bauM%d`^tXRjd&498viYCa?R{lXdEJndnaJdXTR$5NFlcVBTD3 z@+=FX$*jJK##8>XgHpP_!`2n;K>CSI?N2j( zj`3OYWvKcsGxTL%HD4hE&>q}Yrs20;OkTDCi0T;-CgQaA06+#~%f&d5etKJMt$}<9 z42Z549)~Xi+~vjtadk^>^w8S#FLm>e28(AS8~Fi&#LgTc?@g2cGgmB2{)`{B)RIn2 zu4FM8OB{!su}fD?k1aH-xbdUaj;q$J-FaE$1sT2&S8jP?AfY2lc0r- z(kq|UJl$IL#cpfo$x!1{@tSvzg{tbhzYqH3$R?JE!PO&+yA=_9J`V8;)n_YrZ~8n* zG*N#KXZC*Q?R7_0&%70pG;DA6j@@eUd34?WtUbrtFIVT9jAlQqKaTfD>k0BTbQr&& z{6IFTPCkPexPfFe%17_^FMceTtFJ|3R4CThqGW7Z)zT#W!0G+dUkk7Z+g^Un7f^ABx zZ}EqkbHau-me~R^9n@VHrglEgYs(?z#K!tE$(yc(gVj2LG#z;J-|nHiZ#_{_LsUEq|V4rw^r9pSM2_ESbN~S5$UY zCyMi2bk?gHF@RjHty|ys3=yp^>uq_T%q7?92G{MgTYB}bQs>nP?v%&i54y81rG7dp zUY2v2TP(k3W%~IE87s6%lDf4@1%ZrtMmOH zE94@O1h!b^3+ky7{-Qta7Z2)LcE~$c8s@oQUP0(YS3rRGIuSmzTm=~%$nAKY4m7Tb zKlj@J6;!QQ0~SULKqd9IhB;EP3u6FK97^V%C@=$SNlrSJUj1mP9XAD}R}CXK%1Nd1?$)8^ER;2F|7t)8kcp*J?T=?7^B$yrYNWA# z23bHe5dKOBzye;2OzxI=nb!pT3wv;HK%aT>13_Q`8&>Uhsyhe3zkm*VLxpuzC!Zz& zNDJomOgcZo>)Qc z$8PlHWxgww>t+EMkJDL??nV=aXSYpIHX%Dzdx<_1?sD6-_y{7M_dV1-{pgsrcD@YV zyw$6ka!IQV!sP8a9QJVPr!u{KgvO`-sOyGp-I66*N_?!NA11n~>p6Cag$qiql@F_i z&r$e#HrveA0qf(3kmk|5nn%pLc18~6sWVOc@DB|UW{=%+wRYKaF)gjuX5MYYU;3nI z338Kcak;E`l;n~yFL1eFn{XVn!@0v%XK=O1S;WZO^h{KCA}?dPQ0{dErr2$5x;GhC zP}9K>xLckG+XkkjD~e8|AX3TTYE1ua+>y#Vy}^z!+@qfNFj~I#CZy{>NNF(eGbSfd z%EIJFz;}IPR-mlc$3e z8iYK8Li?+upY_-lcA#i0eFVnw(@l@ya4J8M1@?rIxlg~k)dV1?>bHO&0OaX}BE1TO zE&!dZ`-{~q{-IS~8`%YIi2`s4Xk5G5F)x-JaK|u6cIM(QDzv;k>dPUL#a zkvhKj!{iV&7c;6Jh|YY;M%3}&{>jK2k_6v5*X#-6F3+*Ef`>5FWHR1R99`h%B!*tK@L^6`zWH&ks{5^k2PmSvpXVnq zN!)n6%7mo2ZvRQjYp^MqY}yDCt_IrEnO7p#HB-2<`{r1j=t1FqY5XcdfSa9d441Y3|=t+s2Gp+xd!3)}i{ zgkM^R*I}aj9A;`PE(x2l1f)6ZL-nyI{0X6!=G6ng`2yaMQ$|zHuQWC3r#UQZ zpU>v$zvFD?3E7`Gs)8R)%#dhEe#LX$uLWB}lSw?Z-i=GmB~2K>caoiUoBC%bWhDM9 z>Hd{PubL~-G<^jsAAa-kY%tXF0)Rzf`+}gh2f7xg%~CU0*dP! z0NsH(cDu>sZ63~jF)Etoe$-GuAj`RU0akdm)w~l}`0}7a0~D`vfl`iaQncz_5Sdp! zF!TU`5e_w42EG^&7dFAp85{|?e&^mO@)wjHP!nJ?2SI}wTpIL=dVwdxma$s2cq4nD z22N*Iy;R%vVrLAnlfRs&i@(};6Pkjdqv-tune-qJ0Qh|Zvnjvkg0wm_6L`I^ zT0t~R0v|z>vrT-?ry>8FN)Es_-hoGLEfK&M*ML^G0J-bS#1rtE^;|Xby!BdV2>34` zq`5bzuI~xn?PPrb1T>hUNy1|E1?0-(#$-T?0U?If+l!?D^cocd%`Xe&r$EO23NolQ z0ID$Q5{E4Vg9(HlX`Y$fXI{fRqxsSq8vHc0nb}THa%%HU&>k7}7wrfy$@Y&_rcG)r zvBccXarv&tVxzQ?pWUC~X?Sg~7}4?Bzwh;&q7@-Aa0u^Rv%HJ7pNWbzbIA~7XZ=Dj z|BnT*=Vnh&M1?``aY`eVzuj;^4~=RwkfIwy=U4=0ax&JRZm13moX zS=TGs!VM@v82pR;Csp5;l1};M?#Z||DwpM8eUwK?IImmchayN|%1^TD2)r(8KP;6R zmMM$tde3qrb~V{AJ@l-7wkNx>C@uSqj~7eAzm?j3U+sC5y8o~Br!?+oP``5aMAY+P z8rM{t%@1uF0-eD9`7q%a%3q~(}-GFzS$ z#*7+BajA?wq= zU3#*+lSR`+HqEdwU_GYtxmH>zgCq=eT5bYg)8l&48TN%iUhj#7-t843Vi3PQ=LfkD zr{&DajA5VlB!C_W7_~unpU9xu?syK6ee@U5-8A6_+8JqVlK^0022d@Wk21kFqb%Jo z+{LMTlxl;L1JyE*LjLe6>hhYY9n80~mMfmEzlI>YLVy8D;a>qj9`8=*RK1{HFFOR^ zsTi6U06mE~EV=fJtBjN~`xe4UJ^Kfml^j&83QmNsnm;afGX(YuRtS3R8F zgjqKov&Y*O@3on~vxw^4{a8Ef2^8lYw|WSng&4Q+ zj+pNS9L5F*C-87b85D7{3Pg1;sK)#m}M zY+@dVeD@?`dOSK6v$4$2xuqv?+&3j~x0m6|>0tZ)uI>%Mc2fI6W;9hGTfaCV#m<{qn4w=Ba@B8o1YBL+=H5 z+0#7;GCV&0c7GtdM}VFUu)+a@=hvwKsvR)XK+k#65uiybvM?|&ZDt?<$|W7;&Wo54 zto>EW5;}?KL})PhNKf|_9$u(9(|1?e>32|kJ7?zMZs~sbEu9^#k4B&Exvu!Y8(8@t zRIHXORidUafk4OPn-x z(h`LN>qpOx6!Q$F61T&OI)U3Iq7g7qYU_+iV7AF?!jt#4mFzDmZb&JQs2eABZKyjh* z{K&ZveKzU1bBqlwQYq0b()`84e@{Htth()h0>m-ZBG2ovy6c%D#r&^+-6eE*>a|1f zJX-GPIhCe7n@qa(j#Eyj-RB+ex(ErvZN@#8>mk~=q1)OXTFbB<+mQRS?J$9SRrlkm z`>UPV?US|sNtkr0nnWCG=;fJUE~75n^}gz}<#@5q!v2Z=cKx)``T}IPosBg{#2Yo*$9QL@5N=h zPYQR0YiM5=vHKCw6P;be@39$FmMOI7B!Fj5&*~D}X9yFDvuG04Xy7-@QiykemNOBP zL<%oz`FrO5@!qeFm4KZ1l_vLHaN@@E)8jVCTYxWlS^Ed%H7^6oiqv0pns`PX@Yijz zQk8;nP))qJK%hsG5s(F7>kN%|rc=o;2YD4>3SSzYL6_C?3w8jy9}&>sebTu+m^}vt z_#?pMw}w-i47x%=0c^F{SOb0l49gYVh!TCNtm}?PGdem$`Xu^5BYz=XvwE3ssVp$H zjgr8P{37Ll$(~Zd-GTN=0T8Xggy~K@c9403MlMirC*%S(sAXLVJa*WBpIYHC9R`$v zU1X072-#r}-9X&g0nGf@`XBm$8wPYTkzgeh+%pOu-OGsw0L#+=DIl<1LB|-rGcicH zMpF9*J^|FLc#Rt%l8ESM_P*DAUj4b zn${R7TK89nO~5AxysjDWb@jY%jX@y>IvLcOoDM<43E*DR4}5FE9{@)Dn{m-vT_M$Q zfhj`g>Mps9sv$bObmIcbzS_cU)!nn*`>jpkHrKr6O#gD#m3_O zS2*{N)D7|&2Q`8%^Oy_f?Mus>Q@jTul;-}GMZM`|`Hh1EX`%nQ4o@#S#_RBLae5L6 z#&jHADc`)}feFd?Il2;cyD9j?VjL-bPpY=gpKN+Si63NRlBcE9DpkIIvTKmXzdqVc`*B*O_51k~`<#r| z!N=6NxdKy1>_U!UoOYUsdw%(G{)tx}yuz4IP5&g*(XF>t-em2%OQC7N-B^;R)+w?_ z*8JH)8Zy_1q?TSCpl`LN?#WDhd~LvO?JM=GP-84rQgVvM=8>&7CN+ogH?X`*u5Gu>;a(SIw#_v)(haj zwuW(sivW3RaY(aR7%T)}iuLb-e}gON%)X5W{gI8T@2Apb5?E}fGVF@A>W+fne9+hg z230Ba$<0s(z_@@GmACc2qSy0N&e&jaf>4Ehcdb`JZI!`bdL4s2%xIZHA zh0Wv}8<%FZTGVg$rmcsH%-h!j$y$=)qF9c+h^PB{THSG6-33ioeJu6xbeYAWNmCh@N^ z%}Y*xQR@Bs)4ORWRunhRmRF6d4Ro(}P?Dy6%Z3k;Ej}bx)4K$RJdbPOnb7eZi}GgF zdJXdu0GE>0?I9mas}E8GB-n6W6{3bTuF9DT(1IlRzvYn*6It9kb?(>15x&9`*PGP; zE-M80bPyBRnaa^##2Nf<`)!6(VW_tq&LH?Cc8s?M6Ha+Y0V=JPCr5H{3p7@N7gr6i z;b5rM>Q<6X1ZE3pJfKcXKK)voW_x;yz$6z`rfwV+YVjd?RIJz&{}t}SqxK*8O_6l% zK3fl04%n-4!2~oZH9KNb>VWP2JsrND04na&%s?=Ogr0lFw6Imu`XE&C&Yfr>%#KfN zeVav=V>vu}n1E*4pJ>Y>KI(vfN@(j`J7`=<6Xidf;ivgPvME@hFb*9M^fPK-_8Sm% zx2~tD$_Rs(OSCn;qPs>|GcFM?%q07H{E{9k?bX}B6E4TKw57Sq9Nbhn=NGR;!6)%W zU9p!At5-GNk|bwL%$CnER(vA+5q^b-+!R*+Us=-E%>F&u%9D*5l*3sqCJG1i6RDk1@&PKd z7B<8=<8SJV9K7RUT{*g=Ty{!}x~dY7vW-7E6A5|}*9xXG4X~W?#jLAy>_}k@4Yn>( z$KN4X*JT=mMl7`e+$) zKF5Ky6bziATzZjBK&L6$cfztO7?^u6IdfRSm{(@#V#OPj@?B9dX~j8Wts;qfeZmC8 zPP#2Rh9*>^-u4;Bj%i1Fw4_az=<7z4_Hzt2lbo$0_#$%I- zX|2p@88WNC$2YrM0RSCH0uNzj4f3_OZkOubDT&QCdHMO47Wa*{<7X4}2S=ur#Ooq1 z5toQQNS`?xQkBnCN$L)NM>!%lv>Pf|PNoLw(+eih8H|jl+jFi;SCBu*xc8rP-0l6s zYw{midRss^nrdAsPfYGAy3b+z&0+K#o-iiu6e8F`m2c~9`}hQ$v9wO&g08v{j&q|e*X zP<=@K8^Lsz|JX7^ArF)p!t!loE=vg4^g@OWX)XHL>2PR~6;vV#YP(LmSMqjEV5hT< zt^yue7-M5TNQ=Nsj2R$Q6{!@Qfl9|4ROvR*$18O`Do)E6?F{6EM@skJu+PuE97gy= z_Pw{=OfE(^FbthX-yuAR9;lo%H{oh_M(WZ$CmVT*8|nTLl%5VpC?~k8uOk{|>qbLS zMi++pWBe+``{M`2Oz=37ko4RKp{qe>@IseP0+^d&sg%m|Sux)61YgZ>XX795NpG(# zMvi76BrsFOcV`Jb9|JTw%$K~*x6pHzE@K!4(s9cRvx?Pyesxo6#eY#SDn5aM3A=Tu zL3}pwzpF(Z{fZ*xT`F!Sq?MzXYA`1`1e2=Eh7{^yuW5GK?ahvacUg4!5(_ASBm_PN zoeBeZpzzc(Jf%j@Rf2R3RDf2^WGq*@p))LWTiG6yjC2qhI+FGv?6zJpP(ksJvp(9-j`vz7V&qWi)&zHe zvtd>9#L<;}HHlg;yf-C6@^%0&Cq5&`csf2OZA9uK4XD z@wf_1j2blizWG~%v@a(HY^Ppw#y~X9A(TtSE|8DBAI&Mvr(~vkm0rhBd(E>6*GB5K z4DX@y+C+Uz6Jb@&jsnk~u-0Yf){^sEK~E&9 zBi)0G?1e2Gxj8Tdk1+UT{iV*;u6Rpm6mDrKMR8EXPX%W53^m}!r$$NNyRGD@ zx3X^FeljNB-Rxl6Fv=)v#KoGh{TJK(n$#Cx(obW(yAh#vbV6L{yW4B6AU`{fE|jyw zBo1W(V-2tkf-AU6K|9>Kh#sQxo|yv?Xt4j>;C+H<5wlUmEXEu+htu_t_&Rjf_>A>pCuiK2(@w{N`Tz&N)WQyZSRGVG6r z9BCAJ@+zmvUU5Q_!&!W3sJ*ZfaS*!_H9_v0DeJ7R_>=qCRy2B@ozR#HCb46oDAobU5~#7_ItaO&f(9d_hrAJoR^ohHjfm@%4tbk@w8K!~GQ8l;EEqQ&*cr{Ly&JJ4Ax?RnKL0KX?| zp--_d`Wj}@%^{kA)6Kvy@rzEw2k_^IY^%q&ftTV+sPhQ61byY#u7W)NY4Km2y3!^owPR8uI#afa zBAkCQg=_HB zc4dCDH-v9C3}!S|r`h=WLnCT+9L}$3Z%NfZDh*Fp!N+pQ#(06LLh^>UJCyY!nI(b>Un2$rGg4@oG-ezmpv!7EI zq7|BdM~y4E!!M*JH@$jD$oiK0dtuBMl~u3v_agskn|VxHa2=@XpZ4v$oA0*}bk|GXC^< zc-SfBIB7fbU+$eq4H(hH-Wq)`L%a?X|7!<>+x{>}V5{~MjB}M(D<_*A;nvEfDts@` zeZTj|$yAF_J%rcZQOa2rLdtcn+PIH*;3;b_Fw9w#eVS7{+($6cCUe#+Ei~&JLo8Yw zzHEosdx2kGgQH+Ew&-?6Hk*^ohglh$XLNqaYVnt^pTw~bJyxC1H~So9TnOjJpYk`J z1l)5SJ13nh!^6p&{xVq~O$L5;9{v6FiEt#ouoLGZ3i$(UAdSixcVe5+2|7FWlmf@9BREH47$yZ0S72NIgNDZi7jj+uD_+OW_>Fk#>P6kWOb`-p~^tHK;cBQUtymDv97 zpD*fH{42hYk)EcQ<5Z_*5@nclkRbrCAHfII%^9klPI`T@P`WPE`!z(NC6d-9wxV6S$1Jx(K*$k6|E%|BK6!u zq)p~_F{R%J8H&oH&3;#uG{IJqu7}sy3R^qD7?S-z9lvZ2C~3U%kCIpn_(=60Gs)LL z>5Y-O*zYgR4edT5+vnfgq!MnFBTmg}qYsMsjZ+*{x+;`V_B-f}Q3Yc2(iLTseFJ~_ zd~**ZXZxlqfwC)nJ}-oP59YV9m3r98Se0oC$sAGzfy1~5Vu$%$X zP9)bF63mU);$4;e+ZKn@GnsTgHj7jFNJQ>eGL+Y^3Mbs|mw!-?(U`Wv(R1UV!&9m{ zJnoV2s3yG4R6+ac7=}iabF#i?q|b(RNC@|)YYc%0e;2(Wf^GQ~Qs1y@3J75Fh=upc{-{WZ)^-rTe-(c z&+?HPSC&Td$bWlv{+iNsh;fha=U_IA!89`fOu5X0=QXIewKjYJ)9?;+T{+!})Z6&)B{=>RbW) zdGweU72R$Eu%TQ(-ou0waWW3ubJ*OSKD>$dWOM!*`0Jj?F%~!Z^_E=8p2CpZ*i0*| zSR2+IJUe)JNAR)>dN z0pI%gyAK+Jf5wcG{?ogfOz6?sU`YmJr=dXl0iNL~m^llxa__i5!gG|#6z>`POV9AX z_a7SakIF@uJrB%ew2@z-bFvuyz(w(W-BA*Jf|bR?BKDT$>M!cXXr!GR7UVtij})`F z;RPr3DiR8GKqEa$Ikyh7vK%^ z&nxOt8aK?;?Hi9*{)r@C;gArz+$~DIp{~K{Zd~%;P7*(tiR-1 z=VHP)k1taAKVxZ%wZ3C^D{-{$tzr1{B5ZU)SNt8bG&ddPiBl-NM6$;UG+^Qh(x*u}>#+qEO zsHivw_mLqI}YxDsRTA55ur%C;l3WZ0Or& zk*w662jTb>oy2b?KW)enbJvzMG;;0Ck=ueR3`7$!80kL6VxAkL@@gad$Jzy$xQ^0S zUsOupY&FoWt7ik~Xd*Vccw?r7vS-cyi;z7_T1wm}Jxtp8KhK~;Dp|MEgZTV2oqhfn zxqdqxd7rb-O_>a@>4NR-wuV$b@x%5Aqz~dm(K1SW7V(VpXWv$T(g~jazzVGSSUPg_^@6zs$R$>VWrqR z>gR3OzCW`oc-4-L;6Y>#XD+Ves6XFk06H(&`)WU7of7OS4l2Djvn_J9l18O)DTM$1 z=CA%O40+sI)b2+PeBtaJ*w(+^kM^5{9TFuv=jfwo)QpO_4Sc4Di8jMrSDD_PHLu*$ z$!|i0*#>_~BcfBEs_Tj2QYG50A`pgY;hE(u#-6aAto+PEIUqlAk<8*b(-Z%qjr(>9 z_Wfw={%gfdulHY~C+syagZW?WxN$R>!xH%`hI~(82yWV?;L+N95)#ThQu*+Qa8eLv z&bZP}Oy<9yxKtR8ns*a~4!+^NMj9!Z=g-|z82Sn#5GuXCHXss-eLIi*8G5VsLy*w5 zO1446y4Wc1jrmB7aWbbpm_9gRE|o&O&LH9u%Z>K6?c9;h;N3U$coz961ftW)E#yDL zMmR%1KVOs&CjG4a;`gNyo7wN&sd5MHugzC$*pwpsWVt>ueCCc<7x})zZ3L&GSo#7( zzsFTJu@nqP#TzC`&C;xo9PS7`CwM;3r7pUz&c^rX*jf-TfU#z7z>^wEU=8}604zOV zQ>`D8P1$hm_S!=pwB!pjcJfI69xqy?%Y31S@t#($&4I0r{rP7$vl)1CO{rVq9vOBv7$^*T|-C0`*fm9j4rBd3|s3H9k%Mz_%!(OpFK3@jRS7gF8v)ObtYBCx*=9UgL6e^XNb z!7gxRl^;VHPV(%yXy#U5b+r%1i)L4c9N8ZTK`^A))1IU3;U$S`zjUgWW=}Oy>T$Y3 zjwXk0l*o`Ol(+K!x^?;Ejg_gx;KQig;G*Glj~iC07qjAVbW!24M1{0PPw?75zKO<$ znq<-aMIN;`J=Q5i_gW9`@5@J!bJ-*=Tb-w=gV}K7-~2M=>6g`uuuam zjuJm{f2Nv2exodETgdZI-LV<+cffxNafb16yMC2qsq@Rt3{gG1N{BH#hHNJJ8(L4I zCdKrPuL;cJZ6E8$K1q1lCd<_!k;oMkQcwLB$HhNY8x4oCRDP$7 zp0}+^xkN@=;Aj1~lUOmqz^W}Lv;f0mU5c1%{Ay`L%{w}jcYoF1<9DAY8-K!MD0ZLw zwoz)W^k3E6PZ&WLr0sD%7TpoO)~!fP$o&!B%qBv~tv}R^2#|X{k}Q>Q3?0*bN&X@; zIYj86Gv@^V(ab-D*P9yejUb1793{O(I_nCc)ddn}C46|=w_ZfFpQH+@?WUD{P->B61khysg-kP{pAc4u)MkI$7S4%y zUQjtmSehLmrDU(dHh%62TM4IYsB0iSJucWLo-UY&Y|{TZuG9;<0i#ZzO6qN2VE5)R zV`zkNznG4@1K-(S(VzXPp)sAjPoJNQDGUDrXctlyC1KxpeJiDbx$n^oNEgJ^ zHKji!nAJR5q`!WD4AJ5?+=xxlh3pa-zKQ#9t?3I_E>dNIZu;riH~v43Ra*wQwOi;i z+5a+U#`Zt1F@)Sr+$vqNh%55wY4@*@!1Y~MS&;@n|A(o!0BSSX{=b8}I|O$tUYz0t zcP$imDee;7-QC?OP^=X9Qna|cdvW-u_x|p@?@VTrnLP7kpWQv@>}JpTe4Fi2!LKPU zDL}X$A`RU3di7*i=EoVX2ez_o#4GN^(ur&B7xVL)?MJ<;&?~9YAA7b^9>9z?Ov)_2 zcvr*M!3lPcxv;WM4*A~%zcnQz>i5oTz;qZaIV0wM?_n3?bh~IDFIAR@7q) z6Ta8FZj5JvNXsW<=J4kNYAACtjbx?WCAow|uV_SoLp#ODxv0@XB5)Wp@fpXljcA`9 zE4JuK@sjxqUX4HduR;gTobg>l(>pw>@k?DcaY40~qW+r^h-N@_?ek5OtqhuoO%EjR zEjG|asf*9x(L?0L2VCL27gc+2A8FHDyIgC=OCr~d7?#uc5ceVW zSosz&hN4mb1}7bj?pomn{v!9Qk5EZc^E4fRS;kZ?2K(c>Ra-bgClsIC1bHA%N31U4 zph$arLHXiHMI%>_L=%SUp@6d1cnoMYVu5f$c|uI=A-))-nrPPX%=6G(^DdJ8qPx%) z*mGUAi9ad~nyws0y=n?QXoXQIljRsfxOhGs^~0_9h~}tlCOq&%9T>b|Mz0;2#>!I> z*~VpP;!L4BqtuFoeAG}oQc<*^jGrN~4pafKir%!Za?c*Q*D-`FyW8Q#Z)FBVel5fbU#Dw0Ke zpda9@GQ@X?0)|^jk24`tEsn?|A~JU4VI+kqFEDOE$RHb#0=t`UpP763;fPc4|Jq%^ z`&2LaM!*v_RYNQ@h{?5d1ZD9M@uV}PnW^8_c$9KSB>16+un6Z)0|$9SbpREc$W@1E z9E*TC_$vUkUM*JZ(-S^RWN|4aC6bKe!QS>^iL2zx{?uE1IVjSr2p)B>_c3Tyo>3^r zsT-g6>A;hCV?E*=bp0@#fN%A83TMCtzO`{|!KfQGOlI@($(uB^+(F1KjXF5I*S=be z;@YWEhbLQqs$s%Qjox%I^i^O~- zD!FDmp&*AaTp603Y0PGF&@d+Y5Q78W90rLx)g)#~7KYUzm@-W4QkX3+SzOv5qyQW^ zg{ODub#K@Q;?$_w@3u+k%)^gHWbFi3aQt=Jy_GbZ#?0{tgL-|iW%t}iq$@H2e%MgB zls%X0eByGo$Hg*yoG!;KXr|rGfuHDb>AFz1@Wm#%MC(B95S6#$0Fkh}h!BNiEW+uW zyrPP6xU8v0N%M*DU#fVWi!fd2{hlc}Q9Z`#hRmIWZb9mVN932$_Qc0e(55a`Q}i#r#Ky!*mpKxsf3B`Ems`b@zMMe|hgoaVKB2T$Zg8TFp zFFEuvToSzyO|!ylTRl7g^|f;7@1xyYhVDqepx&t;(H&^LW!@S9|Aq*$v6nnF^yzFX zT5)a-+bcTpriftG-z5Sc8|7bmId&-tFF(++UV=l)6KVQCB$)+CtZ6$VP_0I2w5BGc zGAIq^1@+o{B}QN@x=;qvn08PPy@o(vW6OB*vKI+y*a)m8AAXPsO9|a0aGNmYUK*qj zt7)neKl;!?CM8EHkk~g16)Y`Us9mT_d&iU#BP8`&0vVi*!Kp|x^~Wc*?NlI#t)#my zwN_@Lgt}gMcub z0B__V0ws}l;-&HY2mY&&^daPF44afdR-IKAVt9M!tzCb@p&BLXvTd241Mg>0HHw86 z5?P}G3Sa2)3x`0)cobKGUu~-y^03@7+H=3)3h)ui3h9&<4(4hos_}%nWfsZyaTL&& zsjU@DRs*S_d3%>71jMfQ)kLnK?pTQ8CJG}$2peW=zvEKlO#Sqx+q02g458*)DbIm- zZH(ZpKF31NhX2a>Y9%el0oalrdDfB0;2gw@3bZ0n;WgM+sin4PLzK+?ViN2z4Fw-2 zZi2pVMbch9dQR(phE^?qLl{f~bq6)9@W6hf(KQ|vd}Hqb6+gK1x5){iObMJ?gP5Zh zr)0MR9wb8C9IG)dX?!iaRZ+%cLla3Ke`g!?tU| zny7GGAW@Iwu0$NIgM>bVg^)NgLM)bUw9c%=u@&1^dzw9IA&bfn=^u!(ypQOX! zBM|Y5%ZWv~sE$X|@PuYjJJh=$1Z#U!X@+4RdBx=i#iynMK3$MjkjdH2MM>xuJr!MZ!>fHoVO z7MJ@QVJxe}UalYv5NZ5C6MtW+E3k1&<)WLER;^AP@fGR@f&1sYidMTrz zeEFbnM$(0MjDlMfv7<|Vxcaoq1bM++HV8B0Q;(74a=kYc6;RCqM98dW^cKH}Rw)NC zC(9uSo|3Sz43fw-F-5-9oWa-0ejEOr#V}qVx4$I%SM3VZ(!2vkTxwpqckwz55d)E9 zx@xdzhinwD_L%FUDrMTE!5gifbDQm|YU-2F@km=SUCN3bd9}0yN&ByW*=TVY1}U7V z1>^S1d=1ve;&lT}drysZZ!&lff+PNQ_`aVxX=pb6LgAh$PnwgGQN>|y@jQ}4_#WP4 zx`xl7-WurI=742H(fS=sHp8wWfg_wSEvXUIgE?qTT)OE5G*=p~Rjdui1R0|liFPB9%Y_vJ znr++6297W}i*TZXInnDV7=uyz3*9Xp$hx5?SSCSChB%wHk}evvV%kyIx`?w1|LtsSGy^xnszKe<6f|=#mJZaoAk1sFNdf1BN}bHvRflm)M0 zziEGrGKQT7q}6cibM*Ql6{EjEU4#v*K<4~mD$ABE^-D@GXkeyY4Mh^0>!dc1BNCmT zk}aC#p=V_Wv7`X*3Q8H8pMLsSo2dslIx$^-^V-0_h3T zBE=0FX?ycR@dUn7WIw+#^T#lc?r_-Y%cf9si|u{$kxs&rV>2fE%D#Q;OMJm`BY+fP zCwZXYE)6-njbdW94`1x8!?d|18~CzQSMm8hWy5@Lim+bMV1f}gqI}kVo|8} z3BJNjQz$T!qmZSN+`bBx5!Q?HN)C<0qpQYb@lzlUjS@Y0v`-<1=~568(L)E{d*Jd< zp(g3@O7vd!UBDME4H{`Qm&6t8;YD|J+tOL2d9MkbP=%oe^aywwf-;5W36g!sdCdkBB8xVIv~Us>ZUn_HQ%&{eUSq-fjs_ppR?C| z7D~@t>DKGoldeoo=6h(~FqcVHkPwv?V+TxWUUkF%v6(k|0E%v;Vs&LL9L*nkoZyTI zlN^n*K!cB8W)k+3XtP}eLRZOZDas0iiV-g?1(eA~QFC7f z7p24<%t@)06LX7zD_)G{{JIXDYBLJmCX$vO3)xaz@J^*4S-7H<7zXfpN5~U{v0~1u zA7pP*u!v*oh{Igfh(mzRoDW-DDZe;ex%u?IJzLh&aeNWjnnSE4iX#@y5Fib<@tun} zq)Zy-e1VW|FMHrWd3sG3l%Kar$5=16HA2Pb79x70${zvR&EWz={qGmjsan@W}apQ2MTC-8|APH zvgbzIsyKz}Z{g8U)KZdbtCVhxAwJG@791f!rERXNrdwKE=dYq`DQJ%BJgFP3h`yXi>*U+gLrBe{f0&a(0Oa+W* z%C{-6IisQBMg|K6JamwI!fh7(xxy#km;`aaTB}kgsRPeD$G0J5h#qyxo~cNKYnIx`symfD5S* zq}C>wK%Rc1GF%blpyHSv8U4)oYcAY*5w0*W21tb!_OjBjOO{G_c&LMevo z;YJw{PnGIDIehH=P#K0R)flMkmuoV$s$wJ>go8?CIYSXPLh7L~z}|Pt?&Z2x+`&Ku zsvv%qSX>+#SXnolQ?wUj;$jS>4rqB>NARN%dm+umAbp`buwtKpJj;9VGYu$J-L ztGl_~zqc<(pGc-}3-)v-6$nX$@hHLxymL=p<{DGy<+At9rx7@%LI_8KA_t7%CRJWsx`NdE+#htA>$0#pEgG-7%z=?T@ z)tB;t9>xi#O#UU>?Ssb)HU>UDe^3vRgjrcYCRs(~rTcn-#;Em*SJdfe27Eikk0OEG zgU_H^(5$akYPP=JvV}i|cP|Lg`w>YeUkCV_w6B9r#l8@EkyrLypv?uP`GPFxMvHJ? z(wWmygjA*&zB;mwn(gXmX+%dw%7A!NH{3s&WD~KK8|d)|%!pln0vlOmT(CSE9+;7I z5(pYnLRckK_(cZe#{5wfjQ5eD&*A`cRqXtiYdoVq`wdwLUSS?`C9iR^5l17sZ)ugq z>CLjvEKg#&O8=xL@9UfbsWxmDVCPH z#1k^KX!WDnwGZ%w(CNg{QH|e`(a;3;L=TixCK*F-EWUB-uBQimsP${8uhLFnrJ4-> zHB%7$IAl+#hJ*r#2b+KtPQuNXo*^dlaY%40POdq%YZ0t*uf{&olG2=}+Fzm zCq6A(sRmtY{bkcD8}i>bJF-3`&RPn4QINt|5hf{eHW0#l#2CPs!!gg1g2xD~s?Wj{ z7;`@L`G|2dAv7lS_nLm)$(!hz`o!Xm^@~YVqPmmOiA@GcKQbhH)}VepLH7Z!_@pvc z+R!rzXY2y)S$4d3DFy9!t!!Di<=ky%_$igM_gO;{%)6kAH#?Os9e?L$_NVfLM`s(Q zuMBP;3Y7^Z>dq$#r-4fdoZ2{Pq?Gih?364ua?;}$C<9<*Yz8-z7zWPYW97!R)3+I5 zHV@5WEXOJ(wS7jE!G6s`af)PZ&?k-B3^7M^9xyaHYPxIAE!ajKyL{J-;9g@NmJfal zeWFDFAO)K2^Ne^UvrrJt6}rT(gg=0g=S^ZRf@ipe;oTfhNoJ?i6lH|!KGP<$4#<1z zInXrG?E1)3jDQcxH5q#OUbDWdDY6%=VZ(UT`zhvAi0&RZB|em1yvmqe6!#3iK~--g z(-Buxz-*@nj3p6+66tM1GvcQj9&{_wd@&b@;M2noZZ@iM&T03x{MYp%w%dV~CLqCr zs2+{#A!k?(j+n1mbMiGP{?s42KhS)SG4=!N*SRJ{M>YD zLc9yzb%af%d^Cqmgfd<6>_!oJ0QT(4q7qEJqN1Ma*8e5*S-@TtGGy+^kKss+41lsM z1Z5!hB=~Te3ekyJ?I|8pi}p#F5$VMeYmf(SlF%ZOZj_ks-pRs70XRgT6dnX}Wg;gV z#dC^e;WFCVBU!gu2;2Ml&F+aMnk0fgif2#230Zw*C}#`leMG{a;g4O&o==_y_iWzQ z8ySVJ!e9km4;Ij$ej z!vCB*!MU%$SbF74^X3iOCp&;CgyF~uf@EjxcM0?E&(~U7=9cxCeqTG3kZCMeou_zx zAU0M6Wwmob3(tzRnh#dap_o0-SnU_%)7zAy!7*L@0MYGq@y-X~3JE}|#WSXC zM57M)(;#r0h?(1-;J>Je-%1*ZI^r%jho>#5M9)pkg}Ny$YBWVUm*?VVSgrvpV1O}6 z-xwMiPisleiElRWjZzt9kwxgCFm5$Gm$vbFhxtSK`DwKx4T`GC-}$msQVdVae0}Zg zmze!W@wcEMB}WDoDWA^?8JYf?!F!4&1KRD)-{>k)K_sk-3*&nwK;wUoTc;r&1_qGNF|p` z(EFx{Y1p7{ckw33si|cAydFB?6u&l^v=f_RDHp{P!yStv3iD$TR5Gwd!Hz*G2H#fH zh&_|AtFM9#7QP;#oVp63fv`)SQkdqUawF<<>{%su)Ldi#Pz@A@WiZHa>W> zFK}4mHdq>;3>BZ`-X(ab;N6LG+PL$y%hvMe1N<(-Cx8NnFK80KBVKy=pzOaHT6AJOB2j;qn6JU*O>Dv#iRhGPij2OwCXPuBZw8=Eo z8(QYD1z9T+q71812h$y4uGb4wwr2w}d8i&po($pFIopQ&Z{OZctZjVV?6|wJh@2QW zuolP7FGiJ!CW0OoKI3TE4n4_yq@|vI1Y9F9<2MydPtwf@Xp!Ovp8sGIc|SHR7k)m^ zCU-qw_dnX+V;nPtq|uGw1ibck-gI7Uc#UubqZ6&BdsQ-_6pU2v+*{J}91F;lJ8wl; zMz!e9`0i?bv8ILfqS-u2MzZ*f5Vw#Dj+{rb2PJub$4E zb^N34Jzw5n>)dmcpb3j`Q9Ta@QRPB<(TGb-F7B$N+;v?ApHut8yv8fnWv34UgE*9iEwVzme4$tz$Sb|C!73u)=!T$f z=$9a}Ncr;+QKE=>yQMC;enbjs#G>8&>U$NC8nZv{W{gm_QDFrxsdSr~)C7NMrwcAs zORSY9O$Qbk)}}#O9_VlKjR5kK(bn6hUnC=Fz7jXkRAr0dskf8k8Dy(;N3mqCVM3)5 zS%6Np4%?`ei#$8og!26xm0AI8SZhch3%Iozj6Gx%^XS@p5HtA2=WNjC@2ZQk);Ehg zJz8i6i4f;Ma9fM6*sJ$JPVAjSWK;0DO*nnn4IO_-~JNATMuffpFJNz@LHnJc-m?A+$F^*2c)PJ4JOOqryop z9j;N;Ab;D56Ruy89XA{|m)OxOMO}cQlZwg*?D=3r7-=wKyemy7&17`NkHY-0n}-!gJGq0dw9U-9)@;1CN%1OM73 zlJp;js3N5Mb#Q6>)ewnHEHPX&L=*C;Y5YS!&(HRA@giLy`3Po7Mk2nae9=b{6*QCG z03B8>ZtZenc{CjGSp(C?Acn$e4|KaY*1*SU=#4VB;dsJ7O^Pn`>v-Dwkf%nqsKim? zGluY?A=y9i9*{~--;3rIqFN0hog3ek3VhX?Vm;(GD&Mcv-m`7g`e-CHW=v&Xb(-(t zjKo}Y^91qy1mwi*Mm>;1%-^>G5G)!wME$RAe^p+|lJ9!7XvY#Cn>y<~)nJehN0KKe z)CeXj_E?{{A%Q7o7`Y-`X#@6>Zl24@a)Zgye9uZQSEaXs!6boD03VuA!5{UUuit1f zUZ(gku$s1pWf+yx5!`l#x{iksQPd@HRLN}SUShfH>8jp_vn5kn@-cNi^MfESlghNr z`(M}L9}o=5FwkWpmUkVoS>I_8?*~L82~xFRO~lvy{T^Z9^EC2-d=~)2qa0}7g{fKS zNfkt?=uDM`6oq8Oy5RA-P&p>VMm39GOT+u_!uc(OruyF)jHm5=pNA5K)Rg}tT0_Lo zkO3WUMXyo|dH7OTV}GfC&&v*3sHH%5(zf27SdUwBn+Q6mJm0q3^HXDvFXY$zZ=yl(WtM}N%B6NHf3JOR@#gG@ zR83YvieuG~3L$JQ2tdc8YpqqHI(;Lc!#UN&!Zaq#cF}%!cGYO z<*4`FzV0eX+zG0&UT#e282ql3cYh?zzM?PY@5Hy;lEl&6Ys znGx`rle_)QxMJg2b=X|=R}yyFL1~`@$S3^LF<4|3jcxo%WXBm{&FQVKMM)%XxS>$mK^%zM~>okn2!9LY@F>Xu1P*q2;zwF=%0`* zL_^C{x?#uPo(ps z{CG8KE0QaXHO10&sS>&e5nG}CwjLfL2RSlX@g2*?UMR`Vpf>BSca*S6ySD ztYN5;759O$`g6$LhaMaqSruCzr~xywEsK#)M~A*y$(sm^p^)wL`BI`4 zg)WrOh(@y1Y>3k@lzzh|expvWrD@y+lC|QoXdGVqS>{~!pD6Pr{{gUpNU8ecFLqK= zm`nTaIZ3=xbAl+8ie-unX?2PHW3>389z-kbK#qy;3GEpxg*KwKsz3AmIv`1rcM!DC zPzFRv(-I;%Dmo5<&;N$p&Qt-CDa`8e)@pfiuFyi|?b$u?BO^a5nG9Y=CZ|qCceG|q ziIw}dylxjTOm{0gvCZHF_`-CYG~x%8l%r+mQiMUCVOg5L3l>GNFK0|4i8L`GysoEn zLY16lnKPw`LNPlGgn?@|W`~~FTf~2Y8k4?pi6z>slXQ327n6KYUE0H>fnm%)g=G5z zW_?8OUF!K5MLd_S%Y`8<7SVrXz1=hqX~>+3_s_+GoW010VQg9?7>flsi}3o-6^=v zWI`@fm?x9CeZ@*oM>tVx%_>+lORh0;s`aZ7pVL{+=rsv}ggJ(B9Tk3N(yojVWXj79 z;zx}02@d;IV9;GzjYA@L6GXAy;4+e>AIz-Q3WFuT6nE*3KJMZfHkWK3*Yrs0FTFE8lPx7OwsTlW}T@u$d#C}Q^-3EQ{~hyrWhtCxhjfc7VO$Fi|{K2 z1Ex@n9TN!;HR8(%dmo_6AZSNitRUhPqk-<}IG1qSf%g!ja#$T+C|xGuO56fnS0U5~ zPdbo`*y#mO9Y{dV z8Ii0j+%PEz4XE@YRa5tQ1)EzLq51>bjhs?E#Y}EwWQArK#9oq;pv`79v|zFrN49NJ zzFYR;Ua{hPvF8o;EvB>s1qcHV!T@m!bLkJMMnC-d$;}M|_okF-Kg!iA0U3*`U>Da( zyjTJgZF4s%n(VPbHQHNz0!IJcRY+@~@3No{>JKPT0scawIjYfF??a@L6>D0o0PD17 zFJTuV2!jO^=}um_!Z>&8j}q6@sA5nrCtWMGHUVrdonSDgo>-rLy9E&!jlNNV^&dw+ zbh7K{S($gHbqGjIBj*6(FB9vx3X*Znhh4>~LZ}30+E2w=6cn*GNeJGbetyi>_Rsi@RqyT=-c8HZksO)iKyi#Zg~i;fo1&Ky>sgjXRrs1G{}KLAnF^}p z(Q5TzM$vlpGt7-W9>%xgsPO6Ef%FCW!0uem?vS*My0#%vb=YL5BZ7CK=f&v_KY53s zgYRy~)R%rPzb)ahGo7N~N&SMh(U;eV)~Li&?hCo4#n?84p> z`s}tLFBa42XI&_?>S0G?bw%=BW`p~i9}&!Tm~(k@y2V8Ib)S=y;=yF@21H=T6L*8p zN#R}058JDS_v<*R0shJRDR@B*-@-zjPOvdLDEa7Y}0oY<#?b-AE$>+CI6 z-+oFbzRGXjjF{N_K67aA(8OIzer&Be@kv9?&;t!ioelFOElyv5GI=5fqjbVGum}5!C(PnuvfXFGQ4Ve#54I?rPPURjtjdZn~Fb zS#WLb;qW`pmY1S~2FGG0&!QEgAP_^u2%-+13<*O5g<82;N?d1x-+#QmWkdXdB=%E- zC~lPB$S^aDAE1nhRXachp9Xb>y9-}1gq^SzaIHI#w-}8W$!9|10zMVB`V1$hHNZMTJ~@FO-1#6iJVL0LfBwt&?Z>YC-G*<@E8{A-t|Lyt2kx8>NR`CIHv0G~n2n$(WhMZ}qf!b+&Q~zr zG=JjA(6Mp+C7S%Ie6du|MWj`M=&ljGnvO`-5=jo#Oz)w@*#4$~p8GW=6>E+D7iCeW z`-WEXr&AupCcj|0lA&+iSKZm=G6y?|_VBj}u8f{-8F?6gQBnwDxiRzRBM_Y=eMpgH z_4$x|Tc}(j-95n9(YGs1oua`AQzKky6TGglr-9gN=jZtKmwXr;`y9xc#0q!xqq871 zxN5#?m5XQ8r3$i<{5=<0yUe3}R?YvFu)Ldy!IdXvz5+Josy5y$_A?YfN&$@*#RgPM z%Dr1q?a;P(_QkD#FgOgD!u`vllt2C65zleC?rm{HNeU1DQe7i?n)f#sx)&0{B!<_s zlnrTNWu6mNCr*q8!>#Q>Fs8r#Fa~8DMzO+h4^wf(_Z||g4b5AMX<0gBr^CAQwT@tO z15JTbvuLHXWKMkIRO|7QBsyM7OESj!;KlyIIrd6u9hm1EK zez>=n?bYR2)GRB4EBIN+&r(3*Tz92KKS&ZKEzM0-Nz3Fg)YkXFSJV`LsNv+5GP;N_ z59Ym20EvPU#3&Wr3JRnmS1TAq#w~C2aYBJb!AHd0K0hDz513e6JzZbl!xoRZXI8YZwD8y% zGx&S@H2Fv9P=<`-db1u{=uDH5w zi8jh&vPAsB+i;^^c}I2j;4ywBQu>>$n+emN)_)bHE97SZo%zw-^SPbCA+J5RG>B4& zugcLqgaRZxaN%O48V#nP1q9@a29UorP#_jzG0IkkHl*LP{iwB@uocu!XC&+6O{4hM zh}eRS4jUB(+{*xF>d_ctdvS2S-XdTC60(xMI3h$)AO`(=jeJL%m!=5@gz80B$j8OC z!2nVeKuowap?l$b6Kickq_F?F407cLr_6mVf2&8ViEf#B#oyO*QYPScxHO?n|#WZyu~OKX8FdZB=M0HdWPOrU+&Qql5*P1<-kN z?rzbLjSVXqWkvG8?ChAxO3KTx6ngswb4owz>N;d6YSwIdk%4#2XiR7K*6(E%^qRo40no?jNkqmY!`vg(%p=4T<@JLD|3nTQvT z2IKOfXuqvAyWJP1L5n4*ji(r+B`0r3>mk$RH#?pQe|-!TU!4Apqc0IGLeLkNP#^fi zbS?H>XykWR(=uO9!$R#qO3St zxg^fv{g>B}O|Wt(HPUq|;-#PV92EtknoVRRvIOMootq;!%G1yTuQ^9K?l+M~zcdvc z=Mk!U6@d}1?Z4TnuEv+}o~f*i_Xf?Xejl1u*9uZri%gpdMV zy`iaHW+cc{jmkO^D4zpVu7jm8VHx=V)6ytc1nA6jxbUZ!pW6sMHk#nCjAE$hr^PrM6Wnz%3v)LFX{f%|8F`x52 z?;`_FRq=>!ZoW)W9t=GOX|SA$3^$g4vMM<0$;g@?-d`OVc#EP><9!k{%$Z_x^qlI7j?3gF=H?hoJ`$aZ0 zHXr<-DMbu*`Me$mL`y@9?%;2*>bcZZ3^I~2pz?PKtxK0hT{Z>r1ugRlT>^h2{ec1S zqydL6VyUa6-00M}WC|4v-4OoSbpkecA$wSNN!KpEt}ZXvk5z(t0x@bg&}Sh&9QJWn zp!NVPJ5;t0)&0f6V-i|4G)mK;6ZP?FCRGPF4PvIcIkLIRCZk(cp+Guq0BcQuLNu)m zR#Y))g>JTTDk=D1Q$ez3dwqqnkxpZ;X?u1j+*b1e11zpUaqg|jyB)G~UgzBXW%Gu{ zO0R+qAGN6drBWCs{^OCKv@!)T4746Qus^NUcml0uF0<7YB>vNLjE#~qVXK`^Ci@v_ z2<|MbHoBOwTN+lR!0MtYCsnPFsvIhz=p|FJq7Gfg_!(LJR-^y%U?q|>-6sv#A2k6K zwEQ9??>nF45i&E60FL`3`(cWLe{APkE8t)O5pk%G59hk#`YEYyr9;0uaa;HzVDBI~ zSyc<-^A(YiUdG7k#0m35TYLP_Wt216bW;Xb(|p*^7jAZ<7Je&aqJS++UaKQ72S2?d@M{wYIS?m1K7 z)cHnjQJFtm<@ZzT>=`dkh?XpF$Mefy9MKeJ(KmJDQOc+2ktFK6$yeUdK zoC4baj%iZ7weC0+wPkW^ems4CvU>A1#RzZIB6oiYJSGD}*R3l+C2n+UAUY;7%v6To zsRI2!ApwL`uKy@kl9-x_g?-y@6)KkS6*pBcta+P~-++-W{B>~LZw=R&&oyCacj}t| zzcZbd1mY^2YfEblHdipvo^rC1c2$a@&bGWnL_Shcv8X8jj4^LVrYpmT>sBw3#KFS9 zyjuGf(ExCf&$i(q)~HgGkgwjnnAi>#x(yqxC}Wsq{5ob4)YnZt*NNQvzgz&(n%xmh zJX~`Y{%ghA3}CxD@c-IN&qNi$-Q3*#i`CvLKquk?;#8q}s(c3x=!K+g&mK(wwz&8J zj8>a@gb|eJaz)4uUxj+{0Q;<;rutgGFMPfyC9!yX?yBBcbXy`n7y*74Josy;OGrv2 z&TFna`KM((!N1>vFgxRHmG59|Y0dTUX%J?Cjf~=q_C$%tZn;qugY>aOsLRjd^{XkM zXQ{M*=B}JamzI_1@$wu`d-5$gH&s@!c1&CU@_EZfY0^-BY8$LSZe+8ASsFr}hL^b& zrYi4A8y&5NY#n5V1k9UH#zi4a_^-_zZ7x6lD4Fgl{Sg>w9B9WyGqrNF*I)YW`Qf<> z2Ec`4I}>oV8|Nk;V8jFbH)Z+#JWWl+Q2_&z;rI~%kSxsyx|j6)jxR#(!Sjv)3MMMg zw-hB_HgYDWUSycivEG}pXoV_(sw295T^cYFN|m=v>p#X-LbHAFnr3DuCo6||{eHRm z$J+3M*Uakh@DZ*h-7@}N(Cr~U=9at3%0ZP3K-CX(Z*QpJ4Yc7tT(->V7R&rjc5f*1 z1F|YJF(^a^v^5_)7}n-}Ux+@4f2W?+*K?Iy9n|JL7@;`+R`$;rv(7B7cli(DO3Q${JXQR8ny)iYXV zAZGp|Al!8p{{X#`s%47zfRm7(^M5^r@##{NN)zO)huLP%ow4GF=MS$5C;b(r;Ep?z zuZFb_HgZ&eu7KBwuEP3l6`!-XY)C5rQ?0WU0MFN#4@ez{i}tVpeg4jD2L5>)ZbAVI zF)`UtT#5y@^Pm1sKPzUZMs6?nz%TZfui3+F`RmXC7?}RM&U39A{oLE{=X^#>Id5OT ztja6Pue~0;w2u!6vcjh?^(n^8Bt zPye6KWbEA;jhzaTuPv)B?zG!^)Q$F-uUVNWh{5uDCd1y9kS6(f2I2GySyY$DEvz%Yu2m)R39*cfEM33HxP+p<4skr@6S*R#V9ue(6+JDBA@%RfC2QR$D$^A|ObrZ#gC}CR z^A3u4^I}VpzFfT%=sHWv#r26&E9aC|&@-{GJN7*lTaEnJN0)H?EuW{lqvk%mA!nxZ z^!YI|q44h4yZJM|yT}U~IjyX=8|F=K-DpV?$>#O4rk4ONb(%#GSE1hSu>QSxN!U4DKdQE+In^jkoknN_msQS3Eg>JpLGK>2=C$_GB~^pVZYc z*fe0~XOdwg7JkTTj{X@5*+8yLWYl*5zfnB}Qs7XNo8266eTfY46l#CGGkIO{Ak~^2 zIC~~Or>9@p^U2s8Mv9%x#CVxF5iwcA#2{81F#U!2k@_-roRyyk;=$EY zF8TSfwgM>w*9*n+y+nrpOin{mFTAqH287FuMn^~Eq7c#1{vx6LRg{b1VYD^Y`!my& zQ)~K5dhF{0X30bU4S$K48b1s`5uBEnrq}R1Ge3Xv$#>u5$1rf?{pZc__e6u!jYhNe zYZ$mr>XH&sCPpYxeLG1#np%BFpFbgp2tiE3fPU=)_1D?3&NL)w8z{)dX!={e>=6>BlXAJG{xE(0q^$@K z_*zc@*XdW%)*q6xs8M5MWtNnbg9zZQ2Oducs7*&|a*uEVMp&7gzeTYWXHm(6O$5qQ z^c6Ea!APUVf|*Z63UNGQV?2-@S4o3or%PPx=a!3MbaPAdw|1NB6yr(DIJ-1dLnVa> zNL1MwpuBUp%Sg@5y`wc5kPQ8mKXY_&WnZ7OL|eH95Vdn%oOgLoNODJ1Q+wEa-2D~p zQ(~~xO=oR&a+sB&2LSxOA>R--Ke!RPcs_CqJngVpD=SINp}ESWQvLy|q~YSqZEb08 z&TcsuTb8-6la-feq$-R;^?JRvv%C)BLed3S4Z+2Im3qh z!d_BN;*?0AsJXac067NB3pFb-fAc?ev(oa5DJvWLJ&lwMv_;c<07UV^ms$0FY&~5O zsi>%h&z6d-OU@4G8i4AVlsvT7dqKUBn>w!{<|n(Hy{uoVyoWNs)9RL7M?ogZ_yKxE zd@Syrt?kL;lAyKq38b$Fho!Fh`2i{_TGX(Q%O&Nd>zz)%?|*5{RORJ6CB9?1HV4W- zBk(Fi^|YIQ-hXS2_w>IO`f1u<6MvDn=VrJtc@Z0le8%fr78po*_Cm9PZu?TQrzH~L z`N$Ik0%561pGV=75{>=Ecrspp{apTylPd|^gy8pKyV0Jpd^PfE2*k&x>`JyOSy_ua zjBfoLNTRMurvA2z_ze03Efoag4mqH4+T5z0jlVF|;L-cpcoOgO@qqYo_~xZf@dmQn z2J}pglqi-5&V_1Y|JHUG{#y$Lj!LzCaqBUd|E}ro{-?ENwzyavs%LtXC>5yLaV#N3wzTDNPhN?$4z>rN6ky(ON@2iM*^Vd*GYO9m68eLu)q?Yk!(0uupsUd^ z_OV_K8w48_Bf>@ItXpVCyvD&xFs{ben z%)2ZtYXO}@r<#6$%=eSq{=UAmGj{;3KS+3BC24Ey6{A9QJi@4?`#>MN4&5ASn)f?7 zi7GI?I|SrKm<|%z*x+C??z4UlWWlz)P?_)2qkl!N&#bjX#ra&wq)ew&BC@v8pB~OO zoW}0m&`eFJ_I8j)7)WN(%VxeHJ~&+HGk(X^T%b|H15wW1U@^E}x|zm*=nJ4&yrJfp z{KZ5kc8c7!CqGS(x|U`%F^nDp)8&|AN`5JP^+7Xt<++%o*WLq-WCcZs zsNPD|ColjDMik!7%Uudp>-Pi^hTfhI?W84lk?=fl%EkSoS>}9-}2vzzq3>7J6M5HnlGo zqx&uUgpgg#QL33UQ4RVwo1TN2IUF9Edak7+V$F+1%(8YA2v-?1)d+?6onfY?B6SAKEILo^5b!NMQZokBrRKLlcj@GMl3#w zx39?mr!I@p$yd<()!f<8DrBI%;bU;zAJdy}_WWfsON0HcEO-z5UBNtl(cvS=$fLBL zc#1@GBC=o(N*bEPcDwbUn3!srg5u1ZlRvnpk$LdpN@v%xi}tNlQaKw`G|yKh7!U{` zwK78m^!VP=DFz|BdVc9~gW9F^G;qB;0Ce*km1jBuH;qnVnO^OkwL8oOVdYbU%O3cR zLT+a7T4Sn)!M(dXh!=uPSEuYUuW%4|CQoU^b!A^%xr2Ofy@@o4By|&mhPeCfzcPtmSvz zN;L(#wxq*tHaFBi6HXFM$|)q@FD;0*To-kkn};No8f|PmF43F8Ji1fPP^HMJ@9kt8 zi*K+`M__Rf(z+^kO)KAeAC*Vo#QLFnogsG!a9?PTmg$D@zUxZBg>a9W*PEA2YlR_S z=%Y+dPHJiT-~98N()@z#BV`)8sM8kSwu;B(*! zkB&y5m|9ub*eJ|p78-A2^(yKkB8o#Om=7?>WkTfBuxu?x{c9UtG&gEE=I&Z9^3i%QEIWt!Z)aZ9N2fbMNx32=FOJ4yhJ{#S-a&JNLE&Z&A;E`MDGe0G&+uGQIi=Ct4 zkJ9mcw9WDum;0Ge8|8e4j(zHLCVB8PR63M;#CB)vvWtn2VR*X;`Sg4{;reoLaMW_7 z5=Tn#aAY^%f=a?AI8yTAnXbsMl zy>nKDxoxv28=9+pID)3J7VDty&zJdS$9sG3f2{Gw87YBBq=?^eqI^EUKJI=^X!*{k z7?62>sL1f@3?&hj)BQQO8J{&5Dr$O09-cp-NF_H#DkW#b#heA*1m6<8Aj#)?Q#}lD ziRJn^fCsB9M$>8qN>%O;V`Fs0m_-fK|9<@1cz^0eo={S?m&taIT5^tY(Y6*Z-TYa; z+3skbD_J%m@9F8*h^nxj%GU~+dUq*^3&^bbcmFg+aPd=Ix=vo?-*g^-d`Q!MS@;(& zQp9THuSkn^@HE zKr+(yAJDh#8ysc5-VcD)p7rU& zhoL`&cE@MjC#g6UVIry%n1m(;oE&SOQIu{Kd=Z!G&Uf!_O!XV-yO-YQFL~Zd-!@HXzg7~$mrzUPOq^BWt^Q(6Dluc*`+B8-7ynB!sRAW6b(W<(d9e_S~bx!yCt`V%>r=a}Al^^V3zze_}Vh zMGfWt4a6m+K=M6=zz*a-G1IKtJcKcll%B7neOF7Iha3PkAfR0LCQx;j}* zWE!h&X)))&{4Qx=Wie-a@o7k~(X#I7VF)WI*Um&;#QJ&6&%2fC%K9d=zBx>a5=I)T z^i}T<5M+9->1miWg05+LJQRKwKKQgtGb*+yD|6-1g?n%Oz#hu7tLVk`;woUEzP9h=md9q!yQQjm;#O%J-52~Ze+$$QED>{vpc`VncSE|tC5XV_$s9l z3^3FdcTKupOFOkmX23NYEk*ECfG%I&pbm_(nayRJroI*8X>j^wE=E7agZKLnP<%X< ze$2EfTadxG>EweD`p4oF*SI* z-w{Jn_-jAW%Le_Dt69SvONtruji{(8NyUJyhgxsW93u;Re#+#y!ZAd|`_UjzWwt@B zxo3K_v0nZlyn2@6zPU1QU-n{9Aq`*FO6}~kLbk>>DX3GAA?L{0_D?!ggjuA4z0sbd zkT>mN0@)F@~bRLIt#>S z%Fs>l#$feQpFS};ojx;z_D^iEoJ!rLzoyCw5+UhzJy|85ZQvb{Q+=FkAG}Z9qb}2} z-vqO_?M;6nD#BN337hWwCKm640^UDlxptq1d7s1ViT!#ai{Y-Gr8gx47iHn^p*6T@ zie0ko!Z#s^ zlv3Tr5aBxugFO_?!BMu?g1V~ByZ`<;*EJ^rOI&f26b1iCs-WZxfQp~@i-a!c@{v=@~qM_Be{fl=NGn z)o>7ApiujI=y|#Tnj1j!Ox9Qrrf<$loK?GIjtQ1|q8jbbGo&5!+whWJM<> zS-HY6<@t~aBWjqCmrI6b(|?CF8%=D~W0c&i2i>dke}pN0l6RTCcIR(&j6t3LnOdgG zQ}MCj?B>TdGei=Pvq-y2)u)bv;rsAVdL52i3K;av?uV-Q07B0Oz{q3&H`n4R3-MJe zrNKeyq`FyD5~v-wFY%w_8Y>wx@=yuVcK|iW5IIIXU9&b*#sp}k+&llYh|P444l109 z;foN^vzGfTCRD}na18DYF`lU8#`E(pHwYR^5st;m*+z_Lty{{!&AQMLYQd#;hw3mLmXaXI`x`SkdC-n75qw84r>Ic% z1ky(~|1lW@$ot`6eGPpWJll<8h!@$hqGr29_gjt3Cv%G_kz&_#E04us-~HPh`EzpP zm9`eRZElMB;26TN6rU^kv#yph7RY#(N)3al>%-p_5^B*(FO`I-954CC0o5mf`biLG zCy+esI^mRnjH?&UmE!$LI}92Zpx>nmHTLv%@i$sto_wEnjGwRXSkz&FEPPv!^t&)S zV!_A3T?5kDC$l+K@;5ZnX0a52A6`D%1f@7nK9BI!k-^d?OP!1Vho2W z-xa&H2d@fv-tN=)ujrynn=Pz_u~-DdA;L@NN1{fx=~npLc2t3NVzB&IAX`_0>HLh1 z&eGl!OfQXTwnsm?i;GCfPD?`omuVU2dniyY5$-lw@4OV=U8RC2rb zCctS&o>LP_d%WxS7>i*&j`6McLs$5j}!CaRn|%8hu?m0(0o*1uo?o`7{=>72FT3izIy80;Sss& ziV}^!sdudW;p>+L@1E3)m4dn-Z2JK>zqcsCfbn&R0y6b=WhM8I@l_U2Jy*_ z_7;zYi9Xa~=Yg=3GOdEc%cXRLnS){@pdu{%Mur*q(cWfWghxg~85Wgum5UrkHFs!{ z(3d0xM9ery>5g>ZYhDXpgn z(YC~Uke`R4L8>Lc4^?&-!dqh+-kXvJLwgIsNx#??uLBvd9C$^dBf#soXK|Tr(zFKE za`ib{UM!S&7`#`FfE%HVMv$zlQ^21K}B+1=PmvD5}OB%>8HQWbe1fMtxu@ zHKpa|(^4-e`s|Ye^BxDDrXTWaXb@x${*pp|$?#UCN{NZ#6y)?uY&>-n_8K*V5eGev z6yf3HW1!~k8Os0auVvE9P23T`qI%w*H24(&e6!kEW=bIZmlVWV5+`P66yy{YiA2$= zaJY;N07nCrrCc%6n!QOFU>1*>TX!I?WKiFTH;doRpo3{2}Z%a^kl9C}({o^WvmZ8CbetJfXJ9WYo+)TQ1T42%reejuj_dA$q6JD5d+l zb83KeS6)#u-&iZ*Px;Wn7xBp2hECoAw0ntFo0k`J-+>_`kllza*nyQwLRTzG{Ap%3 zB^}M7$n$Dcvu+nJu?*gN_p?yvJiV*{WQeSQxOwfWTEES(%c&;RfdfUQWOCzdlp^zH zWLH|%1QjX1#-K0X9#;+6+{@u>COwZZSh~02IM6u-Jx=;%a&Bd%WZxtomPKfdeF}N@ z)M!5WE!bHJZ^~DGNZ`UA#_8urv(QqJLVitROeF9G=`BfJK5u>`55!vfBbKpcA}Pgk0PIIU$40V(nGx6nD6}J!-!nf z*amIcvVreDA7&`VW%UB=c5#<2Q>U*U?!^(f#a=XZp0Nw`IE&`hW14Cy8&TZeY5K|= za9b4U|CWequ!&I;N%bnuu&}cd2=0vw?U-vBbEU@$1lKa4oqmyb#~IEGXa{oDLV}TO#bDN`gX*Nqgt^ z&M$Yz=X9C}xoC$S7gmY_QtK*rr1{Qmbd>$b-+mX7m31+4Tj+3=YZbYcv|@VNe+2IwKe&;BHLv%kW<6nAr5 zXSA-7PqS46$1b46og&71m9Z_4^Tuyv9Z8&cU4fQ`y_q(5!lUy$3+L~QQ(r6dWInCBxVh-$IQ1Jk1c)KCw8B>;5N6kjW zZf8GkwfWf=82pUgc3MV-ozsJ7 zLX&DM?aKO-GHvgfR$6-Ng}J56^?M@FH#f2xIynN$x{yJ6aB%P`*2TqTYxltC)OjvF zMU{c)bVY7W&SAkAG}fAL)Zz44RUXgc+!p-`Nk{sij%;khVj7=lV`cA}T(?V{$^b22 zIG0XJqSF-La@(}f(YIpB0z{MErHDuq>y07HO6t6VD(<#Kjs$)IwS=lYbCqUtSk%d} zx5ws4C}<)qYxiP*BeFu!pDL4r=kesvu{pB1H3~b)D~>vTDdOYu>abS+&v;NA0C3q9 zGKeuU{^4n7wZGNV;{+KP<}zEHlF)1UROH^fxe;B_V3^Sg5!TYmu|Mtu(qwdjbbu@Q z_fHOr*Wl9kb%fq+eUc5retLLwvA4N8%*S{7n!|ZcPmJm<`8ht$IxNO#R?(JKL)R?5 z;fI4jMGP_ZiMTwM@&c0jj&t3w>9MC(+autx+B?d5)E|p2m2XC9IGib_rjRCw2b$32 zb%aQKM}my!qXwN8G9IZvlI=E8vJ6Esv zTXvtPm!4w&Xacpct&udL&|R&2oqpgm8STteu;lwcA%`kTeFfA8D}Gt*fnqMIYkR%N z_}3GwO9+xVGJZuk4m@(HNytr`zs?Yt)Ec<@b2j1K)5@f*?%pYx`3enEfH>YDPmQbq z$~F&%$C~=eaqyEXIA*^|i?U$J1lIZ?I#d zb_zf3W0L|KM-ilz)NJwI-n6QS^2J-%r|8liSV-kawW;_2q622bR##AT3KX^8c;b1< zL3nh${qoSCO(AydqdWOGL?=g{(u5Uw`ze>(^Ga)0>CGLGgFnX@=FdnfJ_EtZ&_}Zf zwkEQ&9TAhzXk@~izi3Bns?k?1zDu31IbE+FS1cKU-@51*{5!#k`+N0AH0A8@nCg`S zdM2Lq1Rg4Q7giMxjv>l#;HE-gG@(LacT-FO{V2Xb!6va_@9gU;T%LIX(Qc4LLs0_7 zgcxrN4WgNZhp4j9NefCVqk||rI*e&|PMvCHXHq`!{ocOH{QSifZ#{rm)JHKfnNq>U zU=LEdDa+0O@}^KzVaq$#*Qjtvh}C;#0!ObdRpf^_?+BJBD77vl?<9)8KLW;TAT_C$ zjHdN9;tzmIh#qMbi$@pDXuiO(4*aj? zgT-=Hgnj>@%9tSu&ZO6^*Y}r#2UIE_rJKF(nMQ}Pww3bG&W|+o~1xs zy$R?aF`^Cznvm&Udmdp`iK;0uLB$i81K@4{{srja0a2GeF4M{N7u-N)9rG;ZhsyAv z=z@}Ymt;f@V+&3(7RB?cOo|yPfut{ec)=G#8!+o{Ew7 zOx_sxVJkZIVxO*=X zDtFm){b$Cr6r%|%?*?_^;Yxk=iYZCBr~`EnVKvph7c7y|8ueq!oj>K@PCRV>hb zs`_-PvJ!04$?D->reVeIOiT< zwui!Ed>UUFmDmdY2=xF1;22RD64<{)9yC$>eE<+bo??+~lfN}+r`=4ia1q}!zc@o^Kz;9C79F*X$Z2^wOjWO5j*e5vzfWAyn%OD5mv1%x5Iisni zMxScr=q}0j&>J!ipqwXc-@6L(Q$Jf<&U+k$a2A`NoQLDfeY+R9aYp6fMsXEjzu~ z6Hb#?O!qN`#{tti-%AL!d~iO5djd~B0qK_96fVQlPu2ixQ1s`u+}KxtSa#WwxBJ}z z@Wcb(|3$ALzO`|7Ln0b=cC_XbY6TVc{jIBxXr&nEE;b1=gqoUDIW`QeIT8~7JCjFG z6GdM@c6KafwMwHo=1|eR$s3R`gb--1xqUpRzqG`fm>P519FizFfJf!rTFYtouWrKO z*9GOA&7}D=`WBJPn7k$V#5Fu#XEO>Zxf1Var2tOg{>6Yi6z?PD`H(IfyUtJp&^n55 zS>R-IMY&riQ7qQpd~;kACgeWx5Lm%kTe)+!HM#37BQbmn2uXK|)7LDp>k|&Ajq5{e z;ONXgjQ3l!c9)H-OMz-TnXQ)tmaSVuZL0Eoa4j0gyx09}RnbjXOR>793i`>hX2S{^ zXTzJEERNOnkYtbBU$>O6=$59q9qw55^EnPhA@-j_|E7juw7tmYdd>{j`!DE1#|;QL zAfM$g&va!Ot9|PAd!vU1NGZ4TeM-`dpJ`<>-8PqVuSTONrKDt^3rtBbZQq#m{W~Ma zM1z1jTgE{?s9< zD52zj#ALNid*H)u_K2MD7gXwoZ29jmANp=Fc1`^_*jqO;A2o{L31UYjmT-`HIy?WW z9_PO>!jn+*r`5Ps=;(^&fx&@q-U*x2N!jx63DkRhSz99FwvZ(dS@yJ@1l}e<(m+Z* z_k@o;ueY}{Ms%PO5@A30CRl=3c#^mslXXTle~*y$0A*9q^$pC8$x^rY{1;h)asdtw zt~u^4JH$&#m%U4oSY08&weA91e7t{Ou+gi%8cqRu zjy(bFuE4eq@y$~CKASTae1^O}^ciRFg17MSa_o#4D)SIEKvK_5!)|cNh~iUX>H*ky z=Ccj3t}{_&1lTUc40EsW&>x%T1jb!_H!xxKiU7y~mSUb7M6hI|fS?Xc12 z-8da1frOH&j~C7REyZj|D$@sKKUpx=3{2+?Wpu2BbCNeEeVUY4=p`;^f>jZ3ol~3vU-1D1OxK& zBUy(KOn@e&ovAtTIPv#wBs}SWDnLUM_->G1$5~JY8o?L_n!$*-(_>Jfqqp$`m_D{&H3P$Ouu7F5w0pPbDSQTUDefW%)8Eb62Ma*|ciMMtbtKgftWhYNicbI}{?lGk~4dj(FU4gS~#DY#N=D zqq_M6K|S3|SLMP5zx*GK_#jI5pH!65>nYUTMq5*9 zP!|JNleA{>bCpV~anp{7Sm1Cqv$mAu1Er5Kc@){1qO#LkiLIq*Fmv27ZZ6b+ zjJ;}sT9pf2zP}0zupt1M2e=l!K~aXa-Q0@NXF3!ac?9rgM!?~;ga(mEI9nrXl;KV; zn{NnqmsX4Kt(VNGcr!9Ad2E*#OgtvkznSYLXf=y1Ihm;3FEi4p^$Kvpcz0RyI zsoo0m^3dJZq#{QxCrrSZ(iNEhLJE*r&D#RmG9`n1=gg#&6|BF65Ott2JabAP=+oSi}(&z9GxsAIj8`mzuQdYaI=I0;f9&Dg2w^XOGr-T zcNqP(U<*~qHa>PZR0^J*xkB4&ZcacL4?M~be50MI26Lq}G!%FIE(~&3?4G#3*Jy`m zR732oiLtm@^FfTW`WZZi*yZ(q&;f+TA%vW}BaXi*hnl@kORp+5^#c94HSjhw|8WBt zBX(!jh6@=O9qxCw1Pxg5cyi`)8I+e^<{TKqMmTwOPRh$deIv*GzZeoEAamKc`>lHdkzx~^$QzK3$w3%D4cHK?~ z@!!Z4P*L6`k9u?^LE9Q`r4GLi4{Zs!<$urW^(YO zXJV2GoSA7it|BN4e~Q>3zp}DsvzH+*N&pfw+|_&DUW`$AsN#t6^0Kb%3we8=?)2?Y zHE`4w64S7KJRy5811-{5xD`+_Gq)@t9a7{_&dkjZ$#{)3(yoveC`r%5kDX-TQU1yD zh!F_27L&Qv<7a9)((K(m14I%seYBAu4 zeFy?6fEOw){;bzhX8C1ji73sv0|j5!=I&yT0CLgo)s9Z$@e{Dkl=qvi5quJ%W)Er4 zQ9^d}9Abv7qM~8jiJzc#A(^XM8jJRks{h(?{*HL6&Z=ibr7lR{#2i|qBo8#ikL8?L zi3-q&wfOZPOCZrA$zb=FJpE%1_xDZlm|#qf{~Z|8j{-5uDy`|UB?pF?MFd)h&R-0a=Lqpq`DV6(y8s zVG3574~>tHP37G$X~C-H8?zF1f0uHt75NjNU4(F(^_=Og)Z4t|M!ejJ+NiQo+?%>dS@Qjt(S!V@xmF%6)P~N_ixt_-?|?+-+%-j z^i~7!J=w}#+3y;?Eh(93u^f=(joe-5E(Q-X1n8|^cWZQTeoeLhU&alBTU=(A>$N7+ zK9p*1K*%r)4>Nm!y+pHlbyVpAY~}yN16KEirCA$i;y{*pB^dHkEuJ>f7vda4oX={m z%JyGf263;KL?q%NM1aOYMfK0LHm53xUAjCJ0+r+by3t^a|;B%Gj*C&4F}EEQ!)a-x-TtU-J7b+2CMO4$J(ul;`1pcJf&j%|_u0RD!$tM+p@plG#RS=;P5R#m| z+@~#w*9N2<<)`J!%AK8L$(+Ev9xYZrKAB7%kT?PcaUyeY@Y+xI-@spU$)v2`2zrpt zRgv#Y=zV=osnA`-iN$O_^ujEXwE`1kcd*g~tE2l^^?&0IUZL&Xy6)`xbSQ_YIy#WD zc;e|4ZqH1O^}VMY0Alg+`QBS{yhWge0cbBJtFMrZWd7IJycU<&vVZG{0s*rd z`U%JD^9`Ho`--|=C!k*X0Q;XYh;(l;shJy?argyXczK>%oz-%3R;yT!*eIqcRvkQV zX&5&~>!Wu-2O&J6cbrRq19!p7az)+~a3QBoU}xD8CEVUB%CfY(Qu{-%~7Q9h7Tu{x}kId0wXv z5fL!whqUJ9g|t3|bl6L8qCJuh<&RF7fw$U!YJQ-bq63ZSg#70@;kEe?Aw zgB6=dC6QT@tNdI{qX3wBz@rm{ml&k7&0V9oH$FJV!BN4=QIQX*PxAtsE*zk9eKywF zOclZhd@@w|*Di^`s%+*2uEYQfA1k5Ao_d3tCG*!$-(^50|JJjM?+gE->J%;$gy0IJWT5f?V zx34QiP6`EeanzEFfW^}!GDe5$92rt!vdsZJ2LNG&pr2x6o=M5cP~FtKqvFy}0jyou zKVWXbj>g1fZNYDzvjXf+Bn@GO8a0z31Eed|LWH*@)0j)IV!t(D?N;(d$;=2V&2rxU z?T1w6sYx*O!N;}amtaUzjB0d#)fT7KFsP#RrnmvTbaa7`H|95v z`MU-{hk}5Fau9aFFu7|io?7FRHAxGCEsNz&MxL*N#xkM&$CZt8D!s%CF5j4iNdH^u zaOO12RC}-z-ShPv+nicp{Dio>rtf~klj!`2PFqOa^|3hOQf4egxq?H*vTTBHEcp(Z z_Vc7NZeyS}4jlw*fU^?(c98|~CZBmvsZl)cP)jh@WqC~p^5z{UIX(HrTw3Uc6rnz# zHh+UmUj1u)9MTtfW;W)AD$VKC#d?5il9LTWgIH>LB)rxX1+Rk`r8$QeB~w7Rm_EwL zcwC>{VUnF&hTnDk{K*cn1Bxko4%iA%EoWeH4hwzO+bLPJB(GJMv;n%Q(N2EL!*?no zK>3UY*_zp-&>7s&j`07hx{m=?%O}-snIpBf zDge>Bo4>n2?q)9PlhMNlpN|c;N)D~{+C?{a+TdjzM-a}*0Q|yX!KEP7(OJtlTHo?m zG6(`ozXy$z&;C4dfd2Tv(~v+OfwCEyAK%KD{=YFCU>?pcpeKc#8>FX-lLNp+oGhZM zDtq-1zvADhhn@1PUpg=}mww zO$u@sr`5vX{+8<>#2BPDB_gM|;b)|%3WCq;NJT5445K}7 zyCNNWlRWK!z^rgdpBa~-{Y3L|IiJBUeBas0xg~{P@((;dLY7ebeRnw+)MEbC-*vv; z;bONNDaP;&P6NUO4}=#*4Tg~`lNon-y&DSX9>6vwK9%DNgsJ6P4=y>uM#t)0D*EO} zNS_A~K^mI7#DPrC&`)F$>nh7_HCd8~h>jhW$!swWN6AnNL;??CdO-nbC!t4d7S{tm zUs0T3{&a}{Wh+cb#-HlxWgq@kLayc0{YWgyJTV#FU(~=4rW`9Fji))aw8WwpT^n}* ze*Vs6xVxpCyD#gGC`LiLUK6hStTqCS2)-88OZ22UM$^G;BxEG_C2VK|82GQAT}st? zzEmV1_%9!xeZ>PZx-w%J;?j?z^SL=}rz>)^gr8@Q7}%IA73dy=0Zj-Y@E`bnp&(5s z&&~I-8TCe@=jtfrhs84bHz#g7Yq2jAaFHDW)?T_W4oCKNB%-@HwUNHvU1gy)5|8;%q2fn^CGGRQC4zSW8|A37T* z%|H6K1@sGt2cAtaeS4Ad5!Ihp7eOJt=eB*NWP|khGjdAK6R~Ff<+MtRa@+npa_Rn{ zq*fOu$3#6h)I!)_F;BPT@zP{H12XtWk95ZdN0;?>9tey5*cH$K@%m(g-x7|rrwwxd z z$P7%&h?SRm{ZBeHN=ke=foGn-dCnFOx}O3P9e6;=SLmmMmJ(v2oZdg#q6F!S11@xXGMIAskde`z2uvI` z?#pZ2<1d21{{o6yxNzC&_PAUEd*DeJ2_GjLn-_p!&8g+Y%Of)H%<~W)x00QhV zNhsVy*BQ6~G7)1UQT&t!7?_@9J~asmQv{p}0<01VhGO*U`st-BW|~9+Yd(1D4#4<$ z6i4)BKGB53(Sk>f;m0!lB-jV~3%|d3ob4=fA8vk>CNK0CdE<_CsP*4qn~w#mMWLhu z((J?!1|7kOiXeG#9=2L=`)O2|6lvF$=dRGvTX5Jxo@c2=W@K~9rZDor^5cikk$sqn zyf5u-ziBZ<4!1I`)K!|4Pm>I@MUyLU}Bdw)MgZ;xK zKRO^bkBO8div%sFki^juW3K+2@Utr4ISXHPp8!++b|zi^=d8YH38ip9){`Lc<= zi8*>5di2E1AzE}@KEp$|&0L%Yb(;eeECA$!C|3La`$7ezMi+P zv*-9XU8v4xWnPn-V_mYNP)*QHezRQMZf-&(+T;q!iKgU?5P!d9@zUP#!WDob z>Y0Rd_`Zv#Yo7{Q960oBx@*-~ud?vjPK2+bH8JTA`vAb-XBs-ZexW?tgM#&s!*Rz_ z`N1XG>r-rzEz(uoX%kY#$M}l&cs#Ogt!0v^DUjTvca?55e7y8+@9|{wiF`Enq`>ED9L$n1IVwB+kd=?Yc1?1_A_{iqnUMb!60O{ zXddIuGAdezDB*X=+aUgPD4j|cJudT+^HPAqS%|hf1|G#~yCi-8%A<}cbxo`_jL741 zg!Fny-Tc3K7Y7W{btW~Q@8>xdZQ@&=7EVQz@fQ8oq${Q`k{uC-FTBqt_Z2Y1Wb9e! zLlrddUXqlYV0IAmH87lEOTAkc-Mv&0RLjhALszOtQ?8m6;-`N_AE=ZVmfEGMb2|mQ z$h%d~^zQ0^2zw*%JQvkh@$Y-AUmbL1UM)R9c{6oKAg_{P^BX?Dw)8fKbQoP%>Qh`T z8l*}RS{f&$YdxMPRdGEe1*Z)sIN|sWZ0BVM!`piGJSF5o(Z|IX{JC^Rm_6o%MV zLI?})Ld<|H@0+U;+ur3vBFEtJVIPu`oE(ZMjV;X;8QDv^v~|@igS@L5X*x~%{v7dg z4CdGq=XJ4p+dA6U^d%+KU1?8XaSctrs}WL5F56qQw7|E!pF&S;##NRKw6n}R__0U=h@w_|^=xqoDZy%XB{_8Bqv!NQ+!;7TSnCJWYVJZ??|979)3zuy{BzDy>ST72X%`gv@eQ&2ZqIbMQXTB#WB$(Gp=zt zlF)XIC-5*dnk~OvkC_u*3eK;!oP4{8rgE?u&5aj0OYil{FH!ZjGMMVQwe*{ak;5ml z;X#IKJL+z6jb`eHe@u%XlrT~_ia#RLI%{p0k;+^0q!0WZ%=v2u@BBlt^`d!?k5;7D z>N_FHn^#F_pWn4CG?uxREFjg~a)Z?kh~0*rH|Wyf1$um;G6|ku5E% z2l1RZPW_Q?rOF|c=7NIy5>q%#$zGgN&~T|lW-wY`$gc4EWh&Vk`oC1ntmT(F0=m+k zgXdJ(&v9QIx9HV0!wyLuhV9V*r5Ku;#zwZz*I$30p7PqG$Ku zjtBX6)A@Nv-T4fUV~mHbALC8qv|3L*TeoX7o8#VP%4B{UtC9t4=UbRvf>A^#U9$wk z50)37j8zAUYR`YDh~0RWL+5^4`9(vk-5(1K^nRevm7-E+hSyQtVt&qw9L1fTDvFgT zVZ?sxmn8TSxhmwnFuR3^)!zs^Zob-Ln7dg`uQCnOWmy%)5U2Y{**r9jm4%a@j+1}j z6dxDhdmUDqV*SXy+xVAOhz3ENuc*5$R>;h_nCyx{f6pTKASP$b;A=x`)~ue)xum3L zOGH1PS=A?f)yaMHFw*Z2jDqe|&$c+1wuCZ9gPR|mUOBr-!RwJL9|N5zHA$&n$kE3g zN{h=K9sY~Y>!eVrRE$Ps<f@9LRsquZtrBcXcYPlt z#1_qoUyG0xuO}Zcm_u{~=}@YexoX^$T#}>kO*#2jC$T7-e}F_RrH_H8NsCdAiqKV1 zehF^ev{Duof9+Y-Isg6M(#_sbHxrmE{@BJ*ieQ^i*@K7YmoOD&k_FSqZA5}5deuKv z5%x)@Z25UCDntvxa?#Rkv@^Z5F{M@_DOwojQm57Ggrd}5q((%AJ$#PST(E(^d(d?s zc9U&guT?{4Q@P#0sql(Udle^QU!rXh+HCbHrL93OfWR^Jvr9yMK?B^4@Pk7DYPdZIH)p zd`CWocVTtl#B}jdehTSrMo2qPb`H`w3}V9EwEKQ%5Z`f-L-d@VQmq1O$fY1jv_x<& z$D*L6e39&KbxBUUld5zI_a# z4liShs@g7=TJGEanNZ*|UKO$(uY_ByEv~80gv^V4*~=g#^Cqb&`(n&YBQHvkSCR;8 ziizSBm; z45U4?^yjWQH3l9Qi6oIxf%}k1<|MSrInGhLHEY5K~tb^lrcZtK?)n_KM^74-!lguW2zqKQ*-njiGm5vC)PMkpJjunXEx2wD3nlc!b&xM#pv zhKdd47T{wRQnEk(MVX0IOW$VI_6VjWEv%F&BGV(s%@2vI|nN00o zot&kG%hD3LgNtESrbD$i@|J)-4*}+fYgY=A&!Xd^({xlBmEREwpig3b|+XA7=&FNzIEsU`8qXBPeIPEl7k6 zS%4&5aL*sd?jGeB#T>6B+u+T~!97Jbw?_Cl!>f`%L9?4&9& zRB}4>`+^LnjFTUDug`YkNGvVBU9eM}7KCTvT{xBX{wO()egD2&LDEwELY3qVachCW z_A-;VutH^%iXArWV3AHQEwrt&4h&OqygY(WAp>iv1eI|49PFjQ3X>-8rn5-Bksw`G zp7>iMu6Uh0)x1*iAu{a4PJ-!w7j=;2sTZP|ulA-^o@^?Y5IAuVfE z99rK(9ce&FI&CmnLTMRFXyXn1TW#^h=mu0ON}1@7akXh`ZE&JNg^KxQx@~DL-nC{c zNrt_rICa!=0*ODH#f;=F5iVaT_E28Xv!;r9ohI>#>`REeYgJ|vgVFOrEn{t~~;x^PC zAt>a=3-xH#(mpb}jb_1<6xzX{g)>zn;q3>*J53b_+m04lSJ*&=D)T9_Ot1Iwcpmpx z*40&B_eb@h!5nhdUv3Fxfv20J8`)wP%05Fr137e);QpMBS(d(ca%1XrP+tWv!hSqb zbr1AJ%8I_K?#QX14-86Tq4iLrysyZfViK+`_(nZQt5BJxHn=~jZ!VtGr0gIWt#rAA zvb?LH93wz0eO^i1W;+hAsc(TRP533rw2(87YB}K~Ecjcl{Kz#^oGxrspofLUX+eX@ z*UATG0kXuwDe>D%3Qds(^N&uf`@e3Tj*|K@R+*WZ&q2qU@E<=C*{m1a zzr8NkZZ=_y@VvhY`2PLZ&1Q(>?y!2LK@aH02Sun+dOTMeSXc5W9H~x(B0q_H9=U}= zpA$8+>bcz+l15u}7yDQld2bzk`!<1+UsUNW!wmWodlGq(8tq{Vs<0b{w)pOMu|d-O zbXZe*STs07BS3X@TJ|0;R0q8sPG*<=frN-ictfvSLw6_S8=0B&K$M9QHNuYCV;W)= z6c8-eK%EfrmZ?9YSKQxFkU=E8pCV>n0V!qxBjgT6fI=~h?IGar@HKO5H;iKR=Z<5R zAerPpXZZL+XeDCrI?2SnuZaEk1{Ms_e?TVC>$2U;QBzTIdE8VS_E`z^Pdu9D`cjT% z#6gii<>h8T2f#C(Q8DUBc#e3CXBa6I!z6Jl#n)mOM?ZciFQ@#>#&r;8S?C+k6&V@% z7c`L>Sj}2!cCoc6@__MvaMmh7a$FNAV8T=$f@Aw}mu44Y)8I~|;=hL+ZZ$Fk4n?1G z$i+=8q=RXcUC%LTHcG-m_muDr@hTQdkI3g2>uQ4IzQdJ?2`7Y$?gsn%4 z&YvO_raoQKGm|OTSMv~zl*~ed7>L?S5;Xo;;2yQ_U_eQd&0gZw9l;+2m9>Uf`EP~MVvErw--X1hb90-@BMT^TmROBf6>=<)2R>~ZWv~qVAb}aF z`LvsYvq~}_reikj}z3N1QLpjYDS=VLU7RhC<9T%@2U#Vl0%RFG6coBaJoj$BIl4twC=# z6QQS#hz#CChEq;f!pA)yXtg);q>=k zUGkKbkMd@4PW%R3DN0^>j=?+Z1A-jkp1$(AdW+u+jAE^ndJPr|#y&}524$#p=~G3Q zH|1GwePimhr#dr6|C;QObxEMWl?qEfD4*h=L7m-@cmBq|OtN)d98c}6iM5eaV@$+4 zNnr9$^=_z0>L8ERHfSlD#n=eBnesNM&}H1FP(}$M^-dtwlZ1u1Q6fWwsT3~8Y?GjO zmdIQ(B}b>HG}W`{hd+b1qmpI**Ww2LB$D_)>Q?00?(qwRNCWDaM1(n9j60-_9Zz@F z5lB}-M()Uw6vAa%(KFQkVrjwtdqJ()E3rT9PvsL>>)0ODE&L;^CeM6ZI)DEB>FWzA ztc@T5Raj7f;HCKRTH}<`(#q;#>2B7nwo4R#d)|RjR;E%Y?|rny8%6raiRuBrgII^Q zMJd6|yMTSw?@^s7A@TDmR+~y6Xj72;ay4pmXrXV1S~j5a3&pAkdsj)^y7b-AcC5L) z$CUIA!X(X%a76Lk3_i)2csYyUW#>|(mmyp3O;*1qym0h!WxGz~GNmC@Dn%&d$hym- z?wS_eGYwE<>@C>T^*Gqqrz&e`3rR>6H)hzEaGg?&`^Lazm!kHL5mdpm;|dr{g_oPP zo|dI0!oADRnvlOm%l$0~BR-Cw0Y&I;n7%G4fH~qp=PpCyr=$FCOS?xkoi3-wLjON@mzLpjs0RtbCJ}!`S6x4 zeo)OI>-V&iKpu5t8if$p80 zfCj;5D!-ggW(kDA6SMD&Iefi#u~xna-P2k*RrWPjj%!Ju`71cWC4b0hG1vwvUoMp? zwwo3|n-poh^7HSl8n>QQ;FK=)eG(Dil^fnyTz2{J#d^GS|Bx=*=3{zUsP!)24!d$% z;pFz9ffY8>`!=kl7O6%dc!DUX&Z%~iOdT^}DYr0s8!OMCr{E!(XI4Ji02u=sm<?{7wbds#a5o$QMN(g^-4`8|WG6zFg_h&})5LxYxUlA_ zV)ZbgM#zTo*~&g8EY1&f(`qehhRNEC^H%jeQSFaMHlu>(45@L+H~uIHNDelggm?}lUY<@u#=}L{(UBjWtL^R0Ybx5!rA)o0>;Au5fD`er zDJY*&ZScx;OSF{~w0cd45jPixR|a~5#+fA3QhcAJuZ3(Cd!xq@3)m>LP318H>Y=g) zMjkDatIAWYI^j8J(N}g0XM3EeY#h%KMo%HC(89yu zF^VL}AB&9xvZt7xh-|)%IheQ(-5iSq$kSLHQJNQO32!zF_~RRbls9rACzyVujn#gh z-ns;VQJeZjfY!_)87-WEcX)h^PX!_hB&NUOEg=(MSP!Kn65AJ=0KA;Z`6T)aCZ-== zVyI`z(w{lab9M2P?7uF2ihiFdBywsv%^~~k<3dY7LYgZ~d7^MY4_4TBU299iGMiDb z|Ho-Iy{cSWn%K0}9i+qr=iLZ#lBH*%#?kJInAJ)0`txh=3_SP_gjJnB3>g>u)kFP? z-w}yV{43&?*gME)&^=HvQX1(*wUIOoH#+m{F=RUK2^u^pouAps`;GV3e`)MvEX)9%u4vPAN`0#dN zK5rOih!KLgsOF?nk&p+ijLrqEP;BzFs_dgEs;I1J1)79<4SJ?iR>C>!yIh=Dvl=_3 zQp6ycJ=rLxv%Q{YHG&A86kcVGrh}X{Jt4>>b2zD^fk84Ekw|I!xUj47kZc9oMwK|J zdSvyey@(Vd&K_k7iz%tC7$@31s8|bWrU~KLAt9N10Y$_aG);;ke5SMuMZaT9cl(`6 z@!)Q>FXBW1yJ%&3>6L}%rFEDPc!fh7N6|OSzfm-BXCu}i-*l!p#>gU?3_v3Pln`7* zQVFN-87w@@Jh7+K$M>=DjPaEb@&rFby~D2(E3#5eqBMcj5K;3)54FdT7}N_gMU=eN zLNubep&$zUTt2-Lsv_HgX_n-M2K7M}P~16TZEs{AXw2I)hLXfXa)&r5@@7Jv7lA7; z>1@(mObGc4_3o)?vFbF6G@)9MD`PQLaZudeWlhd7eC18hh}jRs7^cJwjw zdGZdiF}I18uoSlLSHWV=AJxToQnw1;J8xN^@b-ghq<=mg^cccTwlR@bveal8siRVp zkvxq|R^Q2^imA&CMnr$6iLH?Nlp7mw>0A1z^TD8%vzAdwS%tKl%pqmObD)H%O5FAW zMWXeuXIk0kDf;GM8|hpX7{hN?R;dz3F*S6?3$azvN3c?FZtJL)_>w-*M2Y^2_`y7d z98IjAWf-3_{6K9#aFL8LBa=u3<3foj)JeH7#L|>oAj+9aK`nsDsmwU`sm?1Y5OPKL zhNZZ;7}|Q9q-RYkxXoOqe%*`Hi{ap=qu?$cl83@ZNAgkc*!w^e1_A=&W9q|l=UaNs z`a@)&{R#s(WO?UN0&+wocQsy1{--`>6oj6sR z7Md1E@%n*M?{Qlp^@f@G1*Z)46DYW(})yVr-{s;mxsa%({nzpFu=xe52)T(7*I;2Ni+uIlA-j9}@{?OM+O#lAaM~|e> z!NAZ^nI=W0RHIPz`tRS0`uc=cWgQ(nZ0xVPonU0wNUMdaT;!g$r>pwAcqj-6A1sW4 z%2;$ASN)kHWmZJYIJoU3GPN@4Z0t@CZEK zy~y8V>8frSs3<5OFc^HU8lCo|K`X>yJsuDm-hi(3{zG8WW$M);qz~UeVn3ayKSL#e zZs<2>J3}M8kloUBe{i`vdte4Yh#6bd@}$zrV~`0nHV_1M@LJ2e%U$~cQplp$_SDl$Gk zzsYc7RTa}vwyLtSbSf7r8XV~Ck(HHo()R2%nIm2!O(f~=&gI$Y_2^!zQjFpXn5K)| z;3&t7y$P39xBQ%(ZX|O_Ny$pVY5;#e8%Fy?FVbA7ZWshKH44Zh=^<)8z*SzUS5c z2u3wqZ0|4p2&RjRYaNuTsHpfF_PvitV0yYPKa8xbta;+u*;%9GZ=E(TZA!%6enS5e zjamt|ZLs?NC(=Y2GZrVSt<)dEBiscC!}M00j-|awBJbBL;s~D3lTf1ZEMZvEQ*a}n zy!U*EGf1nWZcn$r%XQmEmpH+~J32eZf7DV`WH#*mpvD2g!bK*Nasx6;FbVFhe4L!O z-9gAh3H15aA4*m3SHGSdt+r}5#U-f}kM#HJ@hB=N$fWb}nvD}IR+p4mfw~(7Ot2eW zJdA@wLoP>)!uP<+(Chy4=>CCNQ*e5E3My$9s!e4D7aCCMq^H(pTm2n$+o}5H91-efawZBr{@EI3M!cro56iQ zO}!Ht_1ri!AtvJ#4rD_jwXb%EpXdf;fo z=Z#P))^2t#wGuOjnLME!&Oivu&faRU-%9g-a?1q+gOOiP%^ahzjwbPW-hr8prEoGE z55}f)yKr%FAqjw_0LZ6YA1(+&ot~dB)tDpG;6VKR0LH{+vziCu{WiVX6Y7^8EA&>`hfi4y73Sd`?b|6EIZ{8`!KGrB^`ELV|)Czx+`FUI8ez zot+&p5T?V49rSTVklxoTE}c6-ASMCT2=WrFma`AHzth2BU~lnxTq_XPk5`)0fwF-n z+f2;NE6YGl|I7j*!DKe+mxxYJZv(XK#^lNhFAM*EJ%E=xWbEzjfn41Kin-1E8N~fUbvZ!C z_k5xh-;X=X7#VE?%0(^WRdQC46W`H5y<}YW_4-N^bw_d8#@A`gtXtl`C_HsFc z3BaP!01_9=vltY*1EE5s1I7sm`I^VM^u{zns}BA0<7pp$0~r^AbV z2c#Z&z}&d8ye3P zOxjpzXy73Gz`QWr)g@@Bs;YYa-cLkWcp~s3a5K$QcG zkdP1*Ah!KiQU3z-m&ot^YjU81Ts94d+qq;=y@~B z$wy|mbMNPC?+b}EAmfEiO+VF#&qujW#uI?-PId_b=Qnl(NY@t!{`7DQ6sI~5;hlkj zK~`)b^*P|R#7GkW13Nvf6t@>2%0Q#tq|^f;CM#QBSeWVQ`g$GMNPO>y%AOqtLNzj; z(5S9NMqukCTv)NfB9*F5My8oZK&95x{fRQ<_)sg9@pcL@&m-(>N}M6_2+;`%*FYDG zVS(lNh%cpDEa}SqI{;lrOiGFb7`+?V_RuCVmk^WC6ktt4ak*7EKGZZc+!UAp%KF`M zdwa|KcrhiJK$n}3k(-~-JQV}fjtvySfdUKW1zbKLG#g#7-+gdBs-44lEtTqazN?Pc zFx%bNw>E|bBOf>~&LSJQNQ! zAO6FK7EqnHKb3F3WpsVK^3aJ5JJdgPz7dE&8RT@l{Ib2PtE=+z@&Hm~GdR*SF)Hq9=B(Gcwa-?QhhJGI*Y!AEaapUaQu`SD z4MZ9Dk^vgNX(a{B<4D(xonfJ{UgK|44%r_$@=j2ppGX_A5W;!XV+Mox- zC>c5!Y*rYdIC8oSfeuIl{^FuSRiKGeIsJ+2<&3IvSPG}(?%%cDfXJ%-aVcOFmK*GW zLt3rS7c>Tan^p6tO4X~$*x6k`hHbczPa0Z8n@S`cIJbD_{mVO0vpm6vOfF+&0AwQ+ zhtmOg$tv^jfq_$|I6XZ*ry}8cy=E*}`q64yS~#}bN560?jEZF5Yi`P}jlba*_-F<$h`5bsSFv zecT59va&n(^ z(a_PSejoHlM@78^$m6c&Rn_Pa5bK11$OW-w27O#!F`@v#lA;k999&$#{&vE4k{WKS z@hKhu`O|DRkp)84)1?L2z0142N(sB8#oDQS*>Y--PVZDhl==u$fWax$(x(ADnzZ!C z)_rF+H8l|Rxw<$wIGCGLRwuA9Z?KV*n*!^T?r|orte+1V*znf;dfQvsMVsbz9<67uzH+JsVI!(zx~fd>1nJMK@C4rQ}{0dw7;bK`pF zkF%ST>ivAb#>vUqnHv^1Qleh1<8^BcP+m_6Iyot+x)K%iu$U4s9!B-QK=4KOS_H5p z8Ekt_T#Nn${teB|@K`Kj4bv92!$xc`Vckn@g?}WaVje^JB4D36zx72MbAA}+BnN1C zvRVQg5rDir_;~Q*#d3vy*JC+~0gJAZQa6q1X68qD^}QIk&CN}JX!8e*@bvU_yJ$-0 zXhCdFR8-Wqr#tG@L++E7a{xdjU8w5OOuk6JXxa*h+gTX(w!!3P_MDoo*SynjoA~WF zh1};}6{-{Arem^ND9Oo|XFl|1WDuAb1|2QcYvSR8C>!^MPHN|}u&~hidQ`jSbsu(RHIdZ=$^fMA8Q8uj zB%oVEp4#EmJ0EE*^uQl{h4;AKN$`6QuhZgcFCh_J5((Z0;D5Y8K0-!9nskjA1Y)Ld zVo_)%Y<*OBb}^Ce9|;fwk?|s8{W+o^%HNF!|iMv7y)7;qMMMJ}H!+p`j`6(2k%klEWzEg|2hK7d3$-D@Ze)tIE5YX8= z+ci2A5JGO(>A=OvxtgaDjCKs*4F6qv8E4h)-MpW9=v$03Ze|SaIM6hs{KmF!1h1 z517P(j2gq8{`rbie=aXzZGa#1W($F_v9Vz`9j%e@c^A;y>Iw93-%%iePWu3OEk#98 z>Vj!vAFEQF3b5}FL}H~~_X$8ghU0LajqyLDc<}v$u^;jI9zX~K=n_YB^Fq>n|G+?I z5g9eLL-B&?9WP$GLt=TAB#X6q?KW z+NlP1m#Yi=)4%ri*A8SB&F+JC*CMNzu-^y9SG|t4-@jGEIPE?j;wlYCn5?Y#F3vD9 z=~|N+lxd6=hO&36{PQeybPak$vKRrTG@f3U&tY3^pB7fsAyzJ_EIuj8e5MK+8#@`5 zS_PDi^-{gU!jk2?-Ja9+rqu#cc@65w6ku?Wkrn0S^fG~6y#3ApoD>;Ja7YL0uDl*E zRZZX_sz2azU7U~c=X>r=V5_Rm!Qpa|kSBZ1-S95#{RP@!v)aPx$+r(gjoo&YCV2%6 z%VxU@6xI|r3`d|Jgs^T$i>&VJ#WgifAmC|sJ~9wsT6zZw@hLld1B;XN?M_rPT~VO@ zrIY(v+^ZwOQr9mW&HVe5=3Vul$W;T&UcaVcj47hF47XvoigWz!ejExPSR6?1k8{IF z#$4-zVDMWHmKe||z^3<$;(Jc#%i=FI55HjsH1qByJ3D)QRn=ri$2SBbD`hKaIyySQ zWyZp>uyXAVCshOAxCns&D2zlPD?sziWlgwWeev+{01@RmP%{-JrD=^$;Jn!^XQk#H zeB@vqTm=LL&4~jSczz;muCM(zKYEDYFeHxLBC3ecdyLyZ5OFk=C}FNREdvC^;)+%=X4$6;-1CPySnj}=sV zfnA#eSXW_9>{Ipxc!}r{Py>p2Qk7JHz6bgFb+NB`5U&u74h{VQn(qFmw0_m|THAdk zT-o;7AB4w|AF5{ zd|^@Pw427CX3?fbM{RZn<4#(y( zElhKR2-kouGs_&92jK2zJOgF16y<{5+x0Lua9 zvDxk11hB8q`zs3zjle%YJw17Od4VrP+Gvaz+S-nh9B*!B5rrlvV)P582=nv;(=kGQo>02)Vd2O>)M@va5 zUOaP_O)(i=G@dOQ0SpJ=luv;m0D1{<{s8#^4Dxt)SzJ_{25z>^yPE0i+_+YlNknM_mZRx5 zOHw-K_h^r(C}i)4-G0-q6s?`3c(7SgB0X-`8a0gPQ*aPwP;cJ7T(jZ|05`U#?_q`I z>GIb`Q2Ha-+S=+PGZa@;Rx&1Ygq4<-DsQ|6dKY{6N*zR5g(P)Y!Y|Y-01kp#rMc-CB4WXT{R zI}b$p2rnR}V)5BLvtvHvy*^U5P4Y-^Qke)8=k>jEs)~tAHdx?!V)kK?|I$-D1X$I; z@)8>D8BGnmzD=^Yf29@1MhlsjUoi0`!9Wb6)2hv2YE#8rLVBELbyA%xVr&3jUar4B zW)~tdLv?wzbZ$wcRD|Wd7_$R*Y+BbH-@J~J&lBp)O%f7*E>Kr|-acjCU8si11cd8F%L>f3X!+*p)Y(;SRZI;exi$1XK z#i_lpl&UP4*3k;K-j0EAJoC)VdhS2-L+J=e%R@DYBefW;#G?5#u!6j0X)=D~iK7&H zb-{V`UF2WBoHAf=VhsGt*sW(yfPqpjh&qmDCY32Mthe7>pOO35z*y|=?sVUi^qHDj zZU_265ehfv9nER+;`%k+*1<60tVCkOk(JaBNYl$S8gYE*DX<(;`8=W(CL+_*T68do zWf}bT-%JZg!&du)xSbX4T%cn3Ni&+NcJn44{$4p=Tj*B=xLO8669lZXwSz?h?_Xr; z|FLM*@wRExzt|n-n?g$MkOibigDY1OQitAuIY{_`Uh8eGh~{KFq5v05E9*OcLhp>5 zOIVixO6!hJ1)QxGR=j=AOvoC~PWo!-{iVF5a_kdiN0P>-s=SsOs*Rk_(P-8@w_mtB z#o!2YFoynbuP9jfi1UlG@(7AAX5&gyhh%MT4-?m)_bwRFT|^fsI)7ORwV>pCno)<9 zetsPk?I`dz2S^d|MsG(-(b++RA$#_7Ls==Clq)t{PepMptnqfN9?3sx^a=E}_Jq|# ztn?=9drB%n%IK?beoWt^`6^jH1$z&}!Ljt#$%b%wa(=Pq&+*sCPHh1(H*@2=Fdpx% zUz03Mfd5_ze1p4yHfEdbeB3@Gv9-W4;;|xdSI77!Pt?Iq>Hk?7_UNv5`{lo;0_iF2 zpD5c)yRAAP&G8o{=52olrd2xAig@*E1Q|795q67oeZ;bzH=*UP&M7JIb%A%Qf@Rz# z(1>~cUxL>sP#~M#f$PEym9H>a0%FlAQyR-p_GH%g-lFC~WU}f-gI0;TIk=Xl2v=HC z1iF@zAg88g&ML|N@0mYuC6>)n6<}JYqx@cLw!3(DA`?DFF-&=WVJ{#2^>*001NX~% z5rTZXNY{RiyMUOpMy8bl8F3)-Xc?SuTWPCFd*W!HGZ_@$>%M?_Gbv(WxssP&#x@Bzw{-dqM>6oPdbVFY*O_viGH-Tr9M<8@yYP>d z6Lxi1Kc5VZWcC?H)TRu3vHlGK@7K$yEgjrDoAhzvT&`IO8JNojV*b)}*#(<5oj|BMf#`4AHSjV z>kaH*bvR{M5axgf)*oCxfuD%k9B{OSU?0a0C5_Np#~o+5u1we?^7g0mYb&zbfp*ri za&5#p%4f#I- z!EH2GmmPs{8jldg>aZ>9pp=n~vQ_Y3Z2}%o*cWNIh~xyt`i^hVeIM&a3U%oNKU(%h z3}{6#f9$V;zH)YUl#}q!H?)0TQU%A(&LnaleXZGC+1$4{>tdz)G4{GgOvY~)u9jHXC>Ums>O7&qdz}M=-_>Xw+-J~5&wMx_5MVSNn84@l&U2SC=MS<=YM{h5|@?L ztu%PMbqnzS2|yT6$p3UEe!L+c@I*iB9sY<(3v)N#BM}OfPlcdutLbgu)Li|w+8At> zgdJj5ID^)|_H?#&_7;I-!!v8Yxxa62Y@}SlWw%LLK!JpWjI{xztNK3L{}#Bw5$jki z+%HD^lNjvk(aNAU>2_q0_I{W38MCAlSF3FXy7&H`)mm0qJ}KuF_Kf_z2JMNBJqU?V zca&eHeq@?ou(V=PS@J&~NeNJ}V@q+@lZya%v|VmE1$hs8N=ici;Y2_Vg z9=@6qdBC{8vy1YfE4-5b@>KVw+Jmq zqyhr=<>w4S2w?Z(7Z*4|%I4+5f&wf?WQPX#dr)%#u1{UBKoQKDn$7R_mx>fX-^zia zmrsb2hdBQY`SByDohiutf4<-|eagvM7_{(*pRdFzfjA{??+=?O1Rl%9EhnHaiT(fn z{{N4aI9(a_g4Gq|BVM1%#Rm0A%hiC1{J;NKs{pkFDxXdSA^v+(eT6rh$f;Sn~De zFTd2(fqK^T40#!k=Tk-Ve}?Y=Ly6G~<@LYoC(8d@TU&cz1I9rWGYyA{~zVg;$S`cmBuYf+zQRFOi*8R%bhPTD9q5X&CVF zz{oOxV`U&=N+S4J9qCW!07=J+sTumNjhA!f`jg{z<4=oHcZ>J+=}bK%lq%5B|CX=8 z8!IeIQAg!==q`Je`r}99D?7tsEgm8dNNXp8?TMD z*$o;o?U-PZ04e&+e3Q~ zmprt}=csd?^_D2_Y(C3WO6dKYMf>AaVknMvm~-DVoH};#+&Tg_wjd+Dh&DTSGtI8Q zIY$24-{VP$x0@4*DWQ=1wy(bje~s(A4(WBfaOZ7%{SU@MtiPH7qX_gD3fi0Z>Z8Hh zU)mj`p!GKwQx(yRk{1WI*@2gA#n87APMuiOUN6ZEl6|@PU9F5dW92lg_OFg@{qrP8 zXo^i|6IQC%eUaa+NWOKP+6<=Uj&Yp0eZ6bFsbl?>_7cOZBC4XHAu7zVZdgilb|y^O zQWY1UMTburXw1j|o(Lj-3JkP?e0NuOy1Q~}3i^{KS}0%qif}?+wl5NqSxzeEN)30p zhCAuB$3Z4wFP5`wZs3MK_Dq&#ARK#it8B;^MwlEyQQ$)8w)~@DraAFGSo}`+yb6~uvv|iT-{^XY@ub{J1 z_aTS@JjB9H9oa=q1N+F}*XgN`mCuMK-uv9u+2FAiAotmKLgMX z`RKK^d7e+Ye0{DvXkX)haF;2rTlJk6y%JK_G5WWYPd`}z>_uRDtR?q<6`$_SVKpOx z+VP3VpJD`N3%Abs=lCmJW8_Tufm5?x2`IORq>eggtkvJ`busR(PwUQ8uQHHL!`A!k zZrc4LTZRRQcGs__I#08C-#Kfv?8eFPLTdA-q-x^`y1#pL94Ss4S-ENEo_mm2QLu4~ z`uFrVu#^3vIJUaw8?skBE5f(AZPYk$SgljiKzFy|7kE@sc+S8&c%K`)Ur0=sO2 zzTy2}K2FTH`Eb3$$03Xoe-^xrC{C{aGGTCPl$O=(;ic|YwobHkEzH;-cMf2M6)51O z;`Z|Mhq@%h#ZGle|EDRGVZ*?9NF|*o`epN;tk&%<{fYTqs7z>0FMPko zs0=HUEV;3P+uZ8oYYszGe@C|Or0kabOT;-mCK?TTl~BZ-LOh1F?9~Lstbs3zMPxN6 zGF2#=u|C?Rp7rlu1(P?SIOs0<_RQbUj?bzFb!YKwd5(ReR@?$*@|UC)^s(>G%Ufjd8y%V2i8}QRU^pfs?qk7mqTQ%lx0#(JJ{1VQ zc-FG`uBn6ee-*RS6{WVXEV>f+H@qJ_zVj#SMx>ANogO0408ho^ zsU(BAoQPKSdwwuXIw2UojAT(BZMVBgS~UR4p0p0 zyn>kMlg>Y;__&2|3Kc5qs0vbS&a_`Y9Fk~paVu0Z|E#m!Irl|iiZz_;}b%mF*ZgBY^g*h}~$vn9!7wQ}8WfIX zE29Cv_pH#|ByE*RL?+@}s>a)!P;Z;W3pCzOLJ`|FCzBd>>G#YX9{<@}h$0pnV)BY? zUGJ^3@%Sn7Wut-a6e*NYoHmGb73)@Iq9R#UVt+fF>(xmpTV2@&&Cayrc0VU0kU$+87bt8M5nz2T@lZeAlA}HyLYj`A1GPO5Dmw3-z*<4@`4$F~p3G&rAR7H*7v!U{! zvZ+)m(V|?gD#^Cl^xk zvITPFN~)^f=OgT(M7yrGwiVP&oT$socW_%-Th9c7Dv+?$R9wQGnwpEk{IpG8Ybz^X zKR-}e2-1aUbXqqAJ@41c1r!+^w3btQ`Pr@REhHn}kosT1om5>_^ZHH*K<(d)v0qYL z%v#hkI|wz4g^WRb#2shcOohh&^&nk7g)ATeGxXvd^CHXb+W!}c;KAkQ<^pDVcqDEC zVWy`i7vz2pCIEH5RB!7ID!@`@RMpIWHhY0gu7H@mJtHtm2mX0_K<-5NnQwq4nsjWR zet}2R#0-?9HG{gI=VnA~3`Q=v46v>6Vsmkd7ggt|6sk=%)^1VovkFpeS|OhsUfblg2+aJY1fc*$Rh)zuw#$wz9I) z^SM0*1yjI?;DbOw?b~Xup}Mf}`@gczL-WJIV}yf=pCI>3bw@nv8o&~{nZC+Ls;R6^ zJs%zgCQ1KQ(4MV9B_oc}<%Ei+jJdzOPTh?=G(2c|rPy$~ORi4u{XiaYe-7Rd@+of{ zUulpGn&QlUJOJDA$XT@}Who^c@O~sHO1eK)q8FsO^1#dc(%`#X_rM6x%{2yGQ1_;0 zz@r067(3q=(NQ4405}f&H;&POO$LhiWhyEvXJ-$s1Kbu=j}Vx$g^3>SRg+XOTMZhi ztIDrmj1FhXc7jHZueH(RPWA}>ikYq;OIz4`r0)7)Ck}OO9UXio^>@fqpj@Y~zw}VA zwVD~PvC=BSS>$2i<-Mw3Ph(_a0)RUpLEeEfhoQmfhKv8>pI^T;BZ4a+d?EoQH8RhG zKY$zywj@vtIstVvO(^#R_499m5};CCvOQ#;f0;Mr9)5tJ+7_{X;5-1x3+Zla1bw?H zfW{>D{ym_y^8v!i*|Y^nZ1Ww4vi_ZH{)9b^-Dyi%dM5?Yzku8!7$*og4RGtgX|7m5 zmHzpkzc?0M4uJR%3fA@x4sK|)l$4J_T8jDjgz@1rU#6FrmMT8|@ddSkWRk()&``j; z%6$`vi;bP(ZBK{rbNs)3A$6?U+S-CQ2cq)>3WS4E;fo-LUihPMuJwUd=K~wrLw*P< z^M7UlumUFY;WA{V#>!$e7u^TnfCu11Ta$nVcQ!cZluY?><~xHgH4YbtDF9TszPM;_ zZ|_fm;69uvvt`+!8H7Qy#Xt{E!+Zp~F0Sw%eaU?$U=t=JWC9A>V1Nf;FeiI^_vJ8d zkoO+g=2u3gfWh~Hx){6wNB|)c$aO&xu+WAc!PNJ_IlQD~7YrR}UpYFPvB4EdNlBRp z1xQf2fAnC8AYOL+o7-DGz;kGj`WKh~i~=-{BfxS5B@=) zv-Y?F`3rn5LfU2^dwYB3<>gu8VM%~%2WaG^G)~4V*?AA2hiw1h6P5}nd1w1@cxUGt zDCM6r_uVcnE@ES0Eo^M4gLM28Huy;+7&$;$b&C5>7H#^G33Rl86?w}16ai1c1X={x zmJb&*ZCzPkp8y%Tx)e!SPw8TFP@r(~Lk?-24O=EK#M;gn$ZIkg6=bXP` zy){1i0N8+~buW^u%%?hw@MA2@tNbI^1a+mGuYVa#I6MB}D5pZi zmd7oW#BSdD>ncj!Hw?Y@Y?~S1AO@L&%=_pnpO)zRrw2=z{^G!(%B#p^xN^M7K~p<+ zKt(+YnVr?~yw$2YIX<))t&ldAFj1pD-2lCJa%3URX+RuGOI`WnY;nDbv=Et|57~W9 zNy7;;aBChsIUgz51w8f#+WfJ?mvfsg48200Bq7g?-!^O=qgTo6%6&CqcbTTWQoiwZ zldjS%S#T(;n(gU1qzE=8ke3Ku4@=okupwo0{#H_w`(HEo$`hyCpBK)OcL4sEKc1oEa(mD=_@p+EPx_4k54Z^N-b z1GU$APxX0XpPw;&Kq%qPAm=Z9xF^HuUD5cXlE-{i1n2z*Ft0|^W=qU;fn-dk`}1=} zA6k1)K2}mV6t1+L`Z~dzfW6Zy} zu<_|L>T~)D8!|rIBp3Nq&J1VpYIlZS#+VpuYT7wUq(Mu~N{@aeD&wZR;-CaAuRax_ zSa?xg+7EjHw$;h6qse!*7X2$xm3ltou(s*}cHWA6WhWKReGCi~_nU%6?-DJ%m0I1FQ99mX>vSRGmKOa55AMBGo#KX@x|B$l&aU{Q2 z=N*zA%?o#^Tb*(4`iqs9BVYGTa-2F-cgFg>dEFOI@`~`r_lj=L$b6Pbz=*Y8Ih9nu z4)fhA+|sKdm64RYHoq0x_dzT!S5nbvGCBTQ6$_R2xsXVu^}g;T`E%P-7*O%VwALmU zZLjuD&}8VK*@;D5hYhT(xb%{V?Ne&89Het%5Buqje=U1}_L=a^e8seS+Nbfd{?lUB z!KwFM9DRhmXZ!d7xvh#3xU=HA{Jt6Q8%`?bzPz07ZhCr_>vNC3r)acvbyn>+c|n3) z?_7&Y>B<_t*l#Tj`}bf-sK+5cHlFv5vs9F`?N;k(tJ`xkA5X`{=EvaoO6?t{O|ZZ{ z&5o}7wD*Wu{>KIIzCdE%g)wn*PA>-y9PLSvk{PPZ2KJ9)FZ*+^wccbx%`}3%gJqh` zTzBQZHCLPl>E*%_)qNgozSniWJ3IA|{w@U$2>}Sx&G22dw8?Zz@NMA9YyXW?cU>Z+ zGh(R2Hj5AUnoR2ey2rLBs;Zo`;J&Sci$W6RkC-^xIzQg360JvDc$loUkwc%naW;pa zV7(3II7-`#ER)j5G{mm()#E5928D;W>FC0eplp2l-TpL(mgGN+b-cAoEM z7AC{vgf(aACuZbt4x;88qhhd9f23 zkL#e_AV1A_Z+=xn_&b-wqeP z58n~$;Y-m;bwAluL}MdF)fY`h$cHl|vxq4z1Y+IM%y*mnye zXxRPry0_37Q$kaxzb{LzS(rU#`_dIl^Tx8?oQ4B8z8??s?_0^%I;E9f{t2{a28~G@ z`K}y2Z&4p2!!Lm{PrQm)m|1XVPuyRj(bK;b^Fc0}AtMorCAFA79rZ?|GMhvE;4vy(L$RTe`bA}7}xvY%0aMNaDFz0PWFF8dt13>X;;iHUb%;@NtCr(I$R z>>AuStLyS(vGh$$W~xnSi?Vw^^i4yLZLOWXg>|FAr628T)$YFr zeyu7(-MlgC_SZ+&Wbm~L=qtD+3=7X9;A|6g^HzKUZu79l&h`UV3sYgP`1N*;pPfR& zlbPDY+s?{NN9(z;kcN~LM@9kW(Y8@oQN4+6!q3GBOdMm<`9^1G-LKcMI&K7t*8-gE zEL(3Mfcx!{ECXhhcf!eT{ircXgu6MDujl77RUBHl-cEmzt_Kys-%yW}-PW07uo(uU zOrTIucjN?S7ZyG3L7}*N)GWxHtQRUfGHx1RErMK6>;fC@)4|HM;`5<_0tqL9WFL8H zSoa|Puj)#4OL&77bo@5ZU-G@)k>@S=49~8U1f|IFP;B73|5LYYs&G0zK$zTSgO}M^qoU3%qK-jub=OgAB;|r%|41X z?rF66A!(Xbjv=YTeB)s)e$KGV&BaXF494Le+<|26{>HQ+Q>1&H z?2buuZ$O1?rtgnw+Ft2M3#FF?8HEc}<9*i6DM_`OA&*i}(9~uyT~5@%)pgr{YIed5 z-^b&!=kc^l7S96Z_28B>pGv&ZKT_ZO{IiiMj`916`BsOy(+tk5F4kxI5YG*kgII3% ziB9HXe8jzO)5+}L%0kOYX#H<|JkANm+ee-vxJF~SBW*UJ=6^$-{HRcl|E~O}{(k1f zUiG=fW$Eg)M_K+i{jO5=U&DMK_GRCFja<1de^YFRh>Up97$IcL4Xuocyx5sag!n~j z62);s@$C2|J}JLB0Lq|El=NcvFJ3hcc&JU4+51NZgwbQz3pSw(R$Oa~$h?&>U?FMA zu_*?xgH;S4E?mJF{I>9l(&RsEq99{lZL42Ub~lIPk*0GS!IyWAOM-D9byb@kr)icp zZI&~JP0s}QmS6{E!>uU+e#G=<^l1@W+h2POOVe7GTZf5eu`xt8MrRfZsPMYJo8=Eq zV1BNn?Hb5PEpC)C(54okE+Q|p&6wa#DJOJ^j(w59l3czzwtW8Zv;4aJ*Nj;VTypXQ zmZ3FKO>S4^nT_VWZ@AXsX|1o{;h+$Qg>mQbY=7~U5RHgkRB(iKOZSJKR*wUn(?t@KLj-2jfRO?VkI@=& z0(y<8jPiCm9)0VVAwBl#94{zh6H@p0ao-W|4~-7=j;GM|O;AhoaC0T!?#w5P@(nHu z;F?63e+V#i7oUz1i#ZWTK%>95!r}Kf>7UGmx6~OoK8xe*9qf%Zs^Fc9(7n?mGud z%_G(*?^IL63Ek9b)1odh?+DZ|YI_*LD?5zD(cy{qRmYL^R}T*4OL5{e2y(_$A59Lm z$x}hhR*&cpxtu&8*h$@KDsS>dyttGw;U_uzoaS>YEanNJNUub`{a zK836lU_4%^U$eJIZ!qiho%i(EIgh4G)+DsLw5fMKmHljXIM|`Y)hA#fO3~#zK zY@d8YSx+bL!A$~(Zt}FK1%>a?MnjnSu|ozw7QCQh_o}SvoW4q`(J9Og@I)(2N}^<@ zs=mL$LnFfVRDEyTpR?TjK2kJCA+Mjz59>QdNSAQ90W05mu;|uAcB%!&ZgE;g2W-qC z5?LsWWzGufSSyAc;1fpUOH80au(TT~%Qh;3tn!w@!Z(G!tzNC;_Ox;bj^6bRSwdXZ zeRr>=`-z17PLt!MKdZ>M4#V9kN6-69*z7LEXA+6xH8r!Dv5}>jLE+EWHr7U=SW>dR zzjZ`u;sZSfEtQp3$+WnZ+9{?;cpI{t$&?I4qg5I9pwo= z*5JVY9J1+OatQ>;(C=9JE%6Er) z#jRe-fA2+I&f|uQDGZV}q@`I!4C zTz{J%xSOK^|Bct$XGwg75$i6~CiYxzybn9R>&T{d6l}h{Vy3fPo;RAia$Iw1@QuD{d>JEPQkN!VzdRp?z(1DU zp)XW_y|tWN1`RJ;cForldg{hGCp#irM?e9C$HK_iIYxPyMI@5EB(g1kyJQg(9h$zWV#Y7KlNTB9wz|i+f^=;(J{`C5H%Io&2%n++;F7UciFr z`>((D1>4bw;jcJuk0!3$)=kI@TI!vWMVW5*%&CRTIo}3CI2DQNPPg_ytZ3h9Tv-Ip zR1~0fVw`TUGEUG*&Ud??IV?afwhHdI=e8qa6I96k_vT!;HvaYm%QI2nt-PLQC2hJs zqKLhIWuMf*OKny8i*5L?ndptVNXvU!7oE2jJ?*5o#OAV$A_>J;dPSL;A{TuR@Yq7c zi1xExWJy&yGj&tY70_AU*+3wl zUnIlsT-p>U4IvJir9_1lZRfY>dlo_7NE7LJ!3pY}%yeF5@ zP$^%;FF}gvY5Qixh%Hhc73^f?ENZ%GPTWwD`XNJ{?pnord*rtK)|hO{t2r9H3biLR zl?w;H5#$+uL;u~zH-9BY{1d9Em~4Su*E!*kQ` zM5>3ecWm?Da=dV;`mty}OEXADMaAf4c@8!Ak8g z%<&w|cah$kL#CrI?fphhA-v6(9rVUzG25Jy01`N%-v6P`I<27L7gzQ>O3s2iGLPFK z)2fWDYaMq-)3*7X6%9!+LiaabKZA+|S?i@JzM_9-F^^ab@^FN`VC<>mJp zd;12ul1Y^U(cCBpGLvHxC771xu~sZmAC2`TbI7mUp^7RAWCD}671W}kCDGv`=tK(w z51sJ0vSw2O5}-ymtD)rDG>D&h1NUqmTe`5SW3 zA@{upoBT>3C?mJV-Jtd#agbTL55u0Jmtcjd)=Cns5}E4MgEwtmHb=3W(HnVR>C*wJ zV)GgpD5nie>XrJ7?{T5MhKlo*S9YYcBbzD;B@UJtyF7lCFeyoC!lW(SRsW6b62`N2!&MZX1ekG*A1;-{6M8erM_mp@ zhRSIWTc@#|-HUh~Sy89E!h+U`Pu4n>?+hEU=zo>}VpV_(2IAfyym*Sv?9|vR1^gsD zK4wL1_44!E#JWNovbIC|m#fdyu8HV4>6eGXD_tuAHofAc@RzgDdfVp(UXV}N8qaxK z5i5uu^3QXoEYCLYOL0O*Y`e*G?#DUv6LUnZwT#`Zn^zNf0@Um@ zlx!QYTDY+i9pesfO<+o7p{mtP@}j-AWAs6verejdyxl2(*fO_EvX;tI>5%aOWcTAR zWREl<*W{?^>fHl|qkdGpn3peJ-5iWp3h#LZ?}dOJ1W$hI3ufK*A~9PA9V=gYBb@c% zx$bB2R|W9V7O&@B$(CGvLV4tDEFJ5!bKj2&e|(hEG~YYwS*dI|+dDagsbw~L+(rqQ zaN!sr1LI$(9JhFms9ij>?Sl=p*q#Ko$p5YiSSEIN{+my_qa3N`y7voHu)n<(E1_qb zq{;ipdogLVnnk5Y$zGqZD=#WR35zV%ak)`wK#Q4?mfdU7^+-dLtoSjPhM*2CSe zq7X+U|IK|@HM>h``eBiT^j5E%Z3J{89Shby(4D&0VAz&My^@CK|8eWw(I@>&uhg1v z%Qn_xmpILyuFJ}nSxdqE^0ZS`yU;-ynFw4Ejk4EWoEN91EQUylWMzYB`|oMNLlKo+ zyt%pA%#8BK0#Hk{A#jBC_di1?{Ci-om2ON3us&l7e|4s_yx$A*U;n&7?U-zRC#--$ zp{%VO9h0bbbFIAN-ImfRL!#i|E`=wOUlDzIGz?9Mm{%}UMq|XKCy?YKdbxVv-P=un zts(~`6EY7ULdA(6AblR2{$!+*^CY2W>-O$%z3LIFy=DX9drAXu(de!BDmrd`Utboj zS<+gkX0lyrX6rzS1&T@ z&i6LK(5Akt1M%YW6>NBXB72m&xhoIHpyBlf&Z#bphLIVG0u0dyrh?autOl0W!0m-R z5ALu_@EQzXd1PR2rEIur_-)qZvOeIrv&5cYzHdlbURYtig2jNUETz#aAU4UDw#vuo9eG%z1{=|ivmukN5W(>aRIuk7-5lB&ZH$sOZL3dW!J1Bn$$gWf zR8pgG0nwEk%tU8A+LkJD>#V>E>$k`uKI`3S=8OBb~aXwM85W zNo)aGLRi82G(3RVB#a^_>KjIg(8XZlDtk@Jq({L#Y=MTFnbT%9mQDMT0SrhkD`CXC zhKlOj7~ByM&%C!%j>cgnYZ)9^ z;ENE^6&@IK7|NFvg^;GBNh*G~{!~1Z11%|9Cznm-I?ez3roCw&KpI#gOf=iD(TFGF zXQeJGKLFg}F<{;3y*2>FPp@OwmmH#Q;B(m=o@kbHTFNi@3Z@OSl?+~Sl9cYxKuZ8M z1k$(@;onE3D27U9oo!eMxg_F;sBt}}INjN=b6nopUf(;qQFPiKox0vPo7D@n8`T+& zW>25>5}Pk*%R6++DJ?KJGuUc!Cu}K;yl$rm%ymzW-=Sw=_`?Uv@-ZRHDMm2|j>Vn1 zOvu?J>1QHGGzD9hbpGjGq3LVCmP^>t{_O!i}`{T&9d|BA0EhbmMy10h( z-+lnaw3P;kOTp+WFfRg|y`A0R*;(i9n+Fj_wXz1U-MbmDxZ*zCMZC~$wxs$&qu}0$Ii+}wLUTU7Jx8HXrj2% zqEB#j9&zu1fG{V!TD#j6heMX+lq<>n0^8#W%vKAJ?3RS$Z&umuxF2H=EA@NYTRH|| zKxM;Oc|QDEFVnX3X`gfM4b9_K!_^M)*DqFX>AD$XqvL^;!AAuZ^O^bQCg$SEARHVT zRB)~ITCZvEQdAiDh8#ZC;Hd{n*{KHCU@JMO`@Mf;kBW$xV~<+#tg}I6AS-{X`kQ2L zPcQqMVJrAl(yL;1jh^S%ieu9lI>kNgh5%>6-R=RCuf&QOQJM6PnTDHhU*hvamTyKt z|B};4=Lz$lwn0i!{g$r0HN&-XbteDFR8c-H4~suGprWPs8vu^q?XSdJ2N((rm1$_R zJGx%48m$v0ZMke%iuHf`to)|6r30&La$3v5lj73Lt^2KFr5#d&tVD>W&&QXmp^<~w zPu;|x)(6DJ$xy!(qkj3+%YZ}_SK#O~6CU4+5~70Y#77F+p=AKxVqZ~Lo_-tp^ zBmb1zO@-6Q!NS143!AHO;(6w>DVV|`$=Uc1)T2LB?i?cBGcz84e4kd6KEHst9*{!@ z3iNDo@bI{Rm?#hQW=QlbqO`qir-9rLDU z=4NO{1SGpg|CIirPH68#035z|`m?3`Rf?i*%=e1l=X0P|B|!=WFCL*&#YD%y$QAMb z=%~(5O33}mADNdQWD-Dr5%Kl=l;zb`l?WbMph>mV9t@g^C9oF%<|YpGTxLBdnM>c$ z1Op}5c&j}ND-bTGD=aCSs&=_umO*&GbO+0O|EJIo(h~ka<@uSY_3P8u+FEXfGa#mU zS|YkOIhZaK0h(kei7A95sk{MT5wjLb)2k+b$@!<@NW^{Xo{)mWdisL`djFH3-l)eu zvt-cxdF}8 zrt?L4x`yTabU3qgX?tNxcUY0Kw@rGAYMEOA%16@L#^9 z0H-4d{#VGC{F!f83247J1-1ruj@a<9A0TaDIpTfD?7;Twb#b~r3)EYBW9f~IjF2f) z+_CK<9e@e{eU0GXNwWU20C3Cl-oAQ{h%M(8%+zgXYkP5Vn3Zq-P`pGyfCT|LGw`X? z8aB6U2arKrhd#Ik!gq$k1Hh5JS9$rLOZmL8PkDkZ*CxF5pT9Am2!_6C&fB-S%3=0>QZ%_yVg@3ab#@)V8P_X^|*MSt_ zfs9eN`b%VrJi$3JoRv2W1Q+(pG;ZHmyXm`NKi)dvAVQdx-QCwNV-m1Lo?bpBy4PU* z@3b8E^h^19p8iCyg|@d%JLpFe(S*B{l>JtTh4=eJD0CSxU2>>%_~o;uN6??2hK8cc z_R9h}J@Y24%S2vopaLgq@PrCi)Idp;p1wfS0=M(fFE1E5&r2eiB|P=u&d($KZK8~S zWKLYi&(AEB^{UIkA+mKY*N@J4Kha*IOjzqZSQ$-wUM-s^xh=R{fisjPz0n13kDKHUJxB{AiI?Sd<4|a&BR2xiC4` z93o1K;(mOKSyG?K)wkVR&}YeYeN?|ax-K`&J%sD>uM17Vy!jlF4DQp48Co(^{gR z3)DydZgUSo78coQVlMa3Zm+}l@c#QGzU@y^=9@0x*i2LAv9JJ(nSD}`p*imH%hM%g ze|AEQhmWhb@6M9i1k@KnF@JP|gr~~qiPl)Zm;48NGJfmrhE(7qmaNfC@oI|a*o<~$ z!jXft%p^A_2LH&8W{-cKwq(s*@VOQ?c8_-@dRQXg@DL%5Mn;CRuQW#K_ zBo|V#KndJypZL3!x6b1wNNUA1T4M#(WRLw6qm_c+x^RaZaC_*>h1chpr53(~UWTkQ zX{^e|j*h;i6rpPIymDSVdCBjwG8L0>ena}41fuR~WocFA$N~5EQoRP!GyaaJPQ=g| zT%*r4g`9!0D}R+F1{Yx&R7fRLjnQ?iwAVL>+f%ssq;qIPOYQ|`p2*Lq8@x>vgcTHq zBbp?fgBfYx3p^)Y6Uip4^WI!*6sv5Xd+=B94DZ^>gOc5eI8V9Efi{?2I{RfB+^@Qi z8^m%nx|$82#uFwU7#ZO6^gj^)WL@f#>-wj-lZXBSy4ib%F|2!(PFEJ@o z2@{v4o*%bR^R***rIq`{c#+NEWr)c#Y&*fm8|8{-rT+6D>~>!-l)t?Fj5bOcBm zHwP`8@E2v1cc~d^L%8>^=5deO4)spH8>Hr56yI_4w6f9N(0yAwn`93TI zX-zgpBI!&luy%X$U)UIcYa%Fog+Lk=3jRKoUcYH_6)3+vTZN89EAp3Sm4d`Fum6Xu z>%E=+@t?V_?b?#uk95^(uLyDEUI4g2oF;G+A~Jh^)98uo zfGf>-?ENI=l6pe7+EwT1{DuF}_#BZ;e<-#HSREJI&8NH4`C?j?+b*n2R6e&1P}emn z*AMSSZqEHD5mh)<&=`-MNs9AeX~p^3KVHVxKsmvJjA?de|_8hm0M>G3m37}SxLl!Ntl$E!uMSH9t_d_%%# zSLFn)5mC*|$qJsRIfB|vzzqB&X*)Y;iuKA9>Co=XUf|)uk{iC!B_}0YPF0h(C~GRZ z?~cF*$y(m~ivjmrP*{{-tFUhfw;TC;*p81vaWavzI;8*O7g_b zW4)G$kl?&w)7-Vrrt33e_Rkz?96!?H?bg?txLKdXwB)UJi`c=8(g}+?i3UIWBL7Yy;&$YI9H!F9!Ph-nsn3OO-R|9 zWfg{l6*=avs&1OEo5zGTR<{G}U96Go7h?Wm;FvpYx*xhfyvNx4XpI84Z^*IHa(c>f-G>a%>~kDM{NQ3no>S~5@1bF=lk z5=Y)tsH7?vC!2{YGU5Y@qlxmWv=d*}7kDK3MuH>GDvD})2Kq_oLV|#oOz*AYLZur0 zO|>lg0<}|j!2SxY*`tu)G1L=6!gO+`9Sksa56+$^atKNKJ2}I>xAx7pTUWpb>T^!< zXe>83JC)B4`=R*nDSCOTbP!+7*Nu&#@%i(xo`dSj`s%S~wKtGT&kIfMvDtjhmF%^4 zafyHH(O>sI0qA$rjr&iA^fi^ z&T1e0c@-ja7A-9egQ|jL`FPtwCI_hUO6jJgt4;o%j`b-jIpbS9sd_#ekx{^$^zoRu^rq>ik4&Yh&x9 zqfHCm)Ol#S5)#yToXg6=;fNHD>3X(8ZTI_Bn)_apO;07Pl0E1E@cbH)aULN-jK?x! z69dd#K*g|T_eI=`OiKN55`M8+&iJ^vRDOHs4$Y_%`39Q_irH7Zse%UcFMz+-(3tv> zU$-12hmZGy(qGW)!v1(`S7BLO%xU<${j)uS1#)U;>OzT&qC3qXKSQIJt7Z<3U>|$+ zGPL_(V;M;vd-HFZ?!n=KW7+BV&Rp@3_(3@I7@@q!xbg|9Ae!q%acT){E3EJEtss9` zD$iBRJqZovcQ8keh+#lx_T*4!%+dqI-PMkT<2X8%mQ)yGDLU>h+nc7=pEcFhW253X z_s_{GIYzqML1*XBeA}sWCXo%WJTZPHl-w1?cu8c+w2Szg$m^y-bSr<3=i8Nr&+UxO zB6+?qh>BLag$i9iw_Ux;nda3b=2R+BwjwF1`s?Wb(O&WU zmxu>p3R7WC@yJS(u1DqCYGNu+6Nv7}^~B;{tgIaEagZ}o3&g%0GY{P^q-7xYui4Su zhqy7#6E8!|??OA|N%_T)o7@O4ZhEBu6#e9i1_%V>y4OK&qMNLIxC*oe2AKa(;mbJXLAj73=H6{0=TRN1x3?Nn>=0vE5dEs z2cPOefQ3Q0_I9XXYt#F8n@J|BeZ|{Kd9otSWyPkjX=-?-@mN6{FnI(qXTQ_JD}9#U zaM(Tgh2nnCC*q^yqeEg$jH^I*RzX4Y3u+MjcFxgQl`s$J@_o$klW{tTG{q;8Z$ zl!HCxzuvBgH~)|sA9v#W_BKES;%H6=*&q~{x7wiP2KTx;qxPqRMvzE`+hU1u zpiDQE6ah?B|F}~H9;jDd3UY2$7R?Z(O)xMG&p|=W>G)Oot`8C5BmJ@HMAVj(mBjtn zGT|uS(j{nZAilM^&A>n(IpNx5w?DG5_=IP#9>yedh;CtS@7^?ziz$Jb(r#AlOm@+= zbH388$(!VWH$dKd>2_I{UMM!Hb<6K7mv`^tiP^=_K*HzrY8mpKPQqUdW%N`k<70f8 zhUeUUrjChLctV1t z(;g>7^I4BEHbz_UFQ3UF@TQjj%>Q_k^@<`kzO}YROi{6-H0*r~{0MBwQQX_K{I7jK zOa);rC6vpGO8bdtVN(?}(|%)`R#s6GaJ9!AOaR0+*S)fsya>I8VZEDq!po3zfpTk` z5*MMo`P-Doq=GZ|Ep6BtoJuPiOydCJY3}k7@>~9#g*CkN#zU@>V6+bux^4E~5dUT(O zQ*uT|c89yFsrk{T#JE??Hvq|ix6tO%kW+GK zQpEcTb8E)uBYa(6cW?4U=xC@yZ4f^2oOA~OT;}IPNABxcT-@;5~Cha4!gl=eh6Tg)%uX_+XMK|3c`#@@G=*bv|zgx zfmqd1=@(1M+3t*V`3}xJW(xdsWw^3dvH;*qDAVXKYs>!i^Pty_ES<5IkgDgjFR#q2 zbQab1BBdK)`~&LtMNlMmhhuTB!o>a$$0t{`sXFd_r<;fHdkkE&F64cGsB= zt300;5iYxd{>Z>Ck&9iVlBVL-*(zTq|Ev>o7X$Ax9t;&T?78bj=%kFaEWFD2x_J&M zZ{R42Yo@3)LrzBs52Wc7RK)&~(G@r>#}+@LrC@CQE#4O`_)o>=n;t|g--SF84kzEt z{*|rcu^p(~VO$w}B7vBErdpkr`ST-tA<(w2qEM4)!St@&w%l&FU_d5_lX__^DANRS z+#ab8dZM7JtY~UsWi~pWPnW}Bg=tMUyj%kKG*WrBRTzj=6xCZpNB!-l70x zMa7+?n$wkr)KuzB2l}@F9Uumy{Ob$X9bh*V4|CX^f)X9}Uw4eEp`+NVGHHfT5qmvp zk{lJ#k-(&V_80{H3RS`wZihK(O;Ai1BFfg@j*uG?qD8@2szWs@fq?TCm`GJ!Ap9Ja&xkJj=MhOD6X@nu{*9T=xCh_Hj{lp zas+8}JtnEo@tslldy~hJ_e(U!o%;8ic8n~ggu4aE4|w~R@y8WKhfcQp3Mpkh8Cv*9 z=-L81Gj3Q3*XTP++ehRZ=Xr%W)aa>L#Tir2^f(0LU3%x%EN23Bcv`1^dBN@spdj(Y z@>Y&O02_S+fR7cpYB`R*9GyVS1xW6hVrk(o<&zXgP$NxjsoOc{v_J+Jono^faf-<7 z>zmp7Ws3>9SL z&q{yoOv+2V7tNkmNpwWXgl@9uFFvo zy|`*R5RwGthC*`{HV5{bkb4j@A3Og}`G*obLxk}En%c42c3B!W0-1VNSN#~YMoN5j zuf*dpH!+~V;eNg~g^rmy>>4+d%3x~_p;-Q2_dRGunQw50F23YS+z;+?ZHz3@@pvQw z!lcg-e{co6&O=_sw$}08%VR$qhcyjmmq4BEJ%hxE<6}{MyEOd-L0|BfMcx z4FH>l%sn3l+l~6RK9M@u?`0I^2*=vM>q3qTtE+3?RK9t6ghQIPw@e%HC)r8rBoqm1 zY-GG_+mwim8T{MGOR@N&=dH>VYsI$EP`OmIaWBIMMtqWbT{G_SqHAy@eU!GpV~$6e z!D&BAr?pHXSPe-^O2*o?w`|_af`-8o{E<3Jwivlk@_Jgl4#2+E%-6lY>oFcZ{x)l| z2_VX2e#Ljx)y@7Up@1J-f4E=L%Kg%UhH=z^1H3@x*`N&oEd}b-P}&l}X(+bh!GE3u(ppGImd7L3^cITV6@JTCet+m#S=Zet(4^VzQ~4tnAztA( zqgs`o1{xZoYIt}V-3O-Hdz1Bkue2Wa$!}PQ5KeU_Rj^Ztrp{SwA9nBsHDEY9V;3$5 zJOQ}Q(L#ar;AhtpAraG>bH_dEnWr0*1-TQH4BC?eDJeVJ`j6sop_n9mwy692OOpjU zr|Jb{VjUoPI@IzDNHO7J;d(UTeCmriPezLK^;}QTMfQSQ1+z0-a(=dwkgIGyylch~ zSUgd=Rn^*DS?O{=D_B2kS&kGE!VO^W?y}zQ-torRs~Krzim5F%$bbm4Y$^mrLJ9LW zy;!5+t6m<`C8ltm^oS{kcY?*+l_}O)7&+^E2gCcrw(E#vEk9}@*7SAnii^5-3a{C5 zra6FS4(|T|*at)jZ%G#9yWyTg{%p7s*Z}~>)GsCs5V0fM?p@9-5B7p$oHgxr%^iUi zU{~K{#itz~S0Z@~V`p$6_&d>FY(6#+TGf9@MSHycr^q7kFfinMK8pPg?+n-B7}!w1 zh#%1!tMv`S&K;9m%VS{15`hx<_ku?CWzzc~_e3sbW}pm&a?!*Pkm4L}pX&xlj&`!@ z)?^j4m%C;!uAVuI&qzOnB5m7n_3eqB-V5kBLj1(ymq!WTLgHg=T z1RSD>NbkZp^lk(Ngh&@5NRx+BOq3c*;O#i`-tTvQ+#mOT-~Dn&ut9bY4RU>txN9C{$~fj^TZxyUu#7s%NMEtU_1BGq&8=da7QiUsX$7G8sVcCKawPnDo@Nw4srq?zer$Y=>{tveF)wS?`E7ZQc^5XwFwsZ{v6pUhhQg?&m~2 zn|zaq4)o72E#ws!ImRn;?C9~WMf&FeWHN(xq&fS_`vQQw^jg$WY0Bq@|M17sa*ujN z_$lY3Cd}|X(``7NhF#jmTD_J?uGxS?<)_v|rP*zj9ud0`ej`&nBCy$uvO_k*YQxnT zn5v6WGb1mh=-AE0DflZaLnmPwq)eB-?I|j%G)G2M29~7s0#dxJR}$ETY!Ac3avx41 zFR7ppy*Rdd7Huj~-SCbHNR?kH57rqVoHmDX+3N9?5>%m^l8+nuJ%HP z_lC>Y9}vSm1}Q^ye?Se3%)5MwgFtd=K)XJ#6*xFmS*v;e>3;tdDL`CiX}}!EV_5fe zqW5hNpVR9pR+c1jw}VmxW($Rcj^eGeS+$7`5lUy}Kepaf>X%VdQ^iXbKYpaywfrSk z4!_#so1k8+4#*L{ctKr+H0lA^t9^CierEMvXYTMhiW{#tJ1_15`4KOi|2|>t3c!U= z;8)M7_h+Ouf4h&uZiW2LmUJS!IX;>r1U=^eHN#$JGF}>YJ`^oiC27rReI%ORmO|H$ zr(6p^sDD~g2UHJ@c(BeMg8n+8U@yGln3X+I8xl}L89 zx479BwZ91&=;R;5^!@jd=2n%DCnW^M#0AA-_GaXN>Y`rwtJ%fcB&I4VF+%3R4wGze z#8zlul`J51R6^*ewJVz67n3wHJg!H$s^uG9G7-e)Go|@i+n}BQL+*eg|wGsNO9;C)0cS`W1Lo zUm371E-kKTx7eDO`ThGX*uhS6jMc_>5Yw@+w@>7L*Gua1cmvuq$BO3|c@0@OMCn~I zI5M7oEAA(6*)@xSb529wG$EzOd~Oo(Czx7{CGO8&iOR()C@FDA9W=fO zA%3rk-jJS5`M&Hp4pBs1ojl{{ry;=cY;FNA(u%+-x3ER6&n+&Of3AWlAY;v)Gw8#hTy&>e z;ZyGvx7QH3D%$8_V3eN(gfVnPCYLZK=~x(;qtGX>!Ihr#v@vQAW@7)-7E|>{$l@&u zIb?8k`~BDFJceEU5_{O5sOz)Hqt%(=;eYxUSLES%RMqF2&0BkCV)pw3DzCLI4tFW_ znV)2+K-~9Eq%4LMgXRX%T$t$A;@TDkev;gQPn5l~+J2h?n2&Ff+N}hs)V6M0+fBM+ zY&$E$O{~!#2WhabTuNgYiCRge!~S74I@HUnaF$}EHM9dT*2PE+77&MLdKhrgR+LjR zNmisSt_+lx{CQui_5B@{YxGPBNz6Mquw?79HfzzoQC`Q`nG)SgO3Q;eL-GboAR=Fg z7_0yyMF&GWbMmxcG1{2TchM(Vf_`1+a{%+D?)fxCU(Ws`!^5BV(zd>gekBo#Qz)#H4NIbXvIQmISYz-o>JsxhQCTkJ zrA~JKBZm*z`}(XR>l}<7SS0fEi%DybKs7=WmMxrax&dkqWBzrn^_8a3c?_t2m!YEg4+RCW&y@k!PAY)7Yt{>qd{^4?kj<&@+1ES>P)$_?pu1Zp+~M?-)G zp@6-By}iI}dV6c^ZcyB=9d4j4zd62{ix<;DGqW;N0Jc;`Z4P;FvwHmo+7isjqv;Ff z4+#VllQ&CW{L1T5pCa)+o`uCl#l>Ese^Kv`SrCtKB;?6l=S*C$4Xe>|z5e1#>MtlL zcz5WK2aX4X&Q{1?63G^Ya&WMBu(9D+R>@C89TOBW$i2)gHxhv6WRW>@!Z7ZnRAilS zde83(cm<^@@NjiN*=IPG^j|*%@|;CqAderz_4RAn1;HY~g=<6A?^gY?Ro5EX`RPRn z>kc=c-}mxYajy!f@&KN{>P$ymkm<xq3OAD_pFH{bw4_H1;xvLzJZxE+QRH?mo1{}|nr^Q5)Ccfc~f>(oK9bPCn@hk`%TI9#x4yALx+1^%S$M*N;D5aJeI)QSlG0*OUoh)3SK5|h5#x|H&%@+iT9fz&&qTK8+h8gA{Vq8=%GRS0{W zg?HqrylnO5$SKU%_Ii)Vi(ZJNH-AGc)5+YR&-&f`hz8 z6aD$j^t80h_Q<-1CqFba!l1|Mn7pfof6NKsLald2fCx}FAl~1hxs_>H|I9Q86FQmD zFgP^GAEV`@5WesMlQ*N`VfggTryrQnp~P{YWBmLAo5#lmU>-@?ZSIQX&*kNpKhN*I z@Z=~H>1{fzHJ&+wEf?i+^)>mC_p9y?znb!?LDUbbXM|-QUAACn&KrcOcZ#jAKOSLn z7S9TXIvkcuOM1k&F!r%#$&mdwly*hR) z7m^Ts9THxM2U&wOmmcGx^SLko*p4vWe^yqObE#$-;i;<)9`y}j%NhSw88Yo@7Zet5 zQKyy4CJ=)2;CR{CNQ`3WKp9Qt&g|cH(crM!;kf@_y%*D4sp&g(BB+)z(2R!2JK4Pr z;znUIW8@f3TIx5R2`~Z&qw&sOp_Wp)XPs5Eqs`ww-<0dW8T diff --git a/doc/design/img/mmap.graphml b/doc/design/img/mmap.graphml deleted file mode 100644 index 3ebc3a2a0..000000000 --- a/doc/design/img/mmap.graphml +++ /dev/null @@ -1,553 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - PFS - - - - - - - - - - - bucket -mapped -region - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - layer -mapped -region - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Process virtual -memory - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Layer X - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - file region - - - - - - - - - - - - open file - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - offset - - - - - - - - - - - - length - - - - - - - - - - - - address -returned by -mmap() - - - - - - - - - - - - increasing -memory address - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc/design/img/mmap.png b/doc/design/img/mmap.png deleted file mode 100644 index 66f24f25377737effa5be16e1e5f8e2942222d4a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67976 zcmeFa2UJ#Bwl#`vmy`+0vJ?ZM6crT^1(6J9N)gaf0f~}9qU4NKD2M?B1SG2n2na}4 zqJoHI0ZEc1XURFexe?{l?tA}t-?;A|FWerthqvK;=R0TbwO5#Pu7&ei3DLEywydI} zqFQ_EkK=z*QT_B7|Bd_kANYyh9x6^Ms%uoIjvu*Tq1an*?JsTm*Tnp0$IbixvFqsL zBR6$_f4olCRegGFz_(&W{i}bt#i%*5#fEDu^_NP=|6HE-ySjeVzBUy_DdxNn+#JrW zJ2Q8wS)AJ0>Sb|c(~6^S8>e$GxiS?SYOJs<`HN2|$livrBaK7zazfI$j>xc21kW-& z#OX(S1@I5md*M8K{B!u!yZ^WS^GoA4C;C4V+3QnGr=9F4oUXRKt+(olx_kF-jozE{ z=g)iC)zY!a+zZgBJv|_e^={6i|3yi${i(@BlT+qUK5I?D@i^wYmxo&Ox*7}v0`}#* z%xs{d`qb}LxNIe;fOxWLh&WAoZ z@1xq6bz^m|D^q5P@U%&9b?i{C+~@T1hV>G>$^)NX*Qc7tL`B(-r#9NYStlxQQhK7u zrYyI`Rm3iTPuO*lizR_V*|G{ z^7LGAbn536o(2o%c%#0LiF8UfS1pPat@|!YODEUrr>@;{u(LDRRk%NGn9<_XUqyaA znJqb+z2%KgP3f()FE8vJYAY!3mwO?TVxr9%SoO2-{4V=;uAozzejklXggl%C=VyjJ zTUe!@{Sg%8W7QSnT^4>ZQfjtWQ^b$=YV#3lhTk&Mn$gBFfa`q{f zfcpwlUWTm_(v?+JPRs4ha^&W^g66a*huXTDy_M`Nv2HadHe@Lc2e9j38SSnJ;5R)j zB_%CA-^O7d?mycwp5ZO5`QhogE(=9wtvSD$V$R%rPuB7d9uGdpwOgh5JlD+)Y~|5E zLzn3@57VLY-DZ5)rnZoNbaZrnwsBs_@Y5@kh$}rM?N0?OHR!E6AHnE?Cf=~bt(WWh z5MGY;FQ;Bnq~;b8#g!W-JR06RQt3#`)bRfOJvQO-I^+0qxkxs;+C&RK7oo9glfjX$ za@Meh`ML41^Tr*;;<>|Hjtn=pntSm{Mag;!&ogMceDXIgKK0$xM9qk^HpwfzWHtWB z{2LbG!8hwApDAV9&CN_|2AEc^<*7T%C!m~lT{F;9qsW&#xxq@OzahPR(9lbQX;bK4 zvkMG{ua;#_{V+k;%W5Xx7z;BW-bjjrHr(UjJZJT&*8nC z#Af0$IA6%a^kdS;K4Ph|R?cTke-^;z&kr)q5AtMAicjTMN9WZ3w(*JN&-S-X1a0ig z24S93b86xXYwiC`doZ5QS9^2O8QbpjK~qyznT#1B_+s6OTRK(Iv)N2@hBJe?VYeGc zqsh^b{lMw%D=sep`cf(M@|B_13vec?M}-ogJ+TJO3I-$z!m@ z_Oo%;`WuJvmR!U~-moqNc>|(B|;y=!_N@0be^K(U3k_ z{|?)JYo)2&iP?Vppfz^0JxY?)Vd8#LFNI^)9h?`JNI2JC9hy1YXAmG@E>R_d1@F4h zpoH_T<%sj%rtT80koq~f(N1Y1i%vgdviZ09{=3+X{Wb*+^9>`1>rQyyco`!G|S-+;pdYr&(f}-p`Vy&aSfwwjLxwn=4vP3 z39mBxcB?_1Wk*2QG2zKPNsY=5?!VhJ^PKyN=5KE>7*q>e3)$-=bD)29vNNn z+iTx;dTBZPEZQvwUQ=_$r@l+&%UUT?pQx7S&Ki?8B&IY~_Q;8GONLZiP}8gBv^LXS zwV4)vdqpHhr8B20=I0vc1=97lunjF7q2srGT5ixO{y}#k-P7(@^SY~|Z5Mp`-rn6R z%@-Ip#Nj^v?QjbO^7%6Au!{JpcD{e)DKb1X95gQ&5D_t<>oRieK!og+u8XAoRC!na zaIpY~Ks|rcKxQtZ^^=lIu_`CbPTYX`JhhoVIk7yf|8R|Uf4bjXt5^dMmqA+D{XOT) zxLE=%RLrJ|c(Tno70xD_?wf3;%aq~sv@0QpF3dr>C}@7BqgD0DmMvTMiufHP@rGds zT#lX_A1AwN^BZS!Fnzgo+KU5Hs}!w6TG;18Jv~{aGv=mxTt((4@QxO5H*4--HAv|S zHMx}Q!cZbIKNA*cb!B#HRJo0-M9{cDecY}?ppmcpX>i7LegCeW5`q4-61%bZ`7wj} zv3Ja_Yzip{{j0NscF>o?_{ut-^k2n6F4EG~tux4+En^RdpXW(EIPd4fquA9OUm{V} zn>jxx=Eat0bMN$vsF^*FvS|c*Uv>npZmb?q$e+=b@a3}_>-C|T>&cw!>8^<6(XAe+ zR&~8g&ljiik_1YD({TMCw;Nv%9xWAR3fNa2nv#;zI5(E;nfmt5W)Fo6%6Rjx{6b4> zzl_NugI%{J`P%OH)>V$xn>QX@t6SCFZaZDuSlrw9{?W0vlF}zb89tURo}8JfJR9b= zclqWs`DmZ?<=boTEzYGzLq%16gp>MB)kYs?3%CkadzziWnk9kB^=5J!1@L6M#C=In z$uB&oq-0-rHCJ6)H_Fo|ST0!~;oW?ywNJJ#t*yY@90A(Mx*?gTpR7apqeseqOmwn7 ze2NxLHyYBdH8*~!sIbL4htr?p%5BcM&D!|hZK}$Cwn}Kcjw{`hkBaItHOWJ^s=K_w zYY1I!esjcOg=tSEyZ>ai^7Qj;8ovXoZwojw{qZolO!hOa9+JWnpB#uctf;Ka=n(4X zn;FQCdC`0;RE`mm2RX!=t%r1RU}=9B)`pI`a}9j=MaVb8xF=x30@m#T6*%|N5U;w^%`B!AZv+wuD3 zAr>8EJ$`a_ji)E5W5*k;oeZ=sgytq%=r|PX3OVj8#Hr;uvjlF+m~4t^jMpp)cH!SU znO%Tn@)tL<$yE2R|L^!mUFxFp>%V`R@1LK|EDaXzhaD=UnlT95Ok9*!sHm>qf+?Z*vpiqzVWSq5EM5i)QCzj%&K`X+1r?AYt2+cej`#$4ei4 zyVb|DG;;(w*;Wl|TE@C`>+yjQa&nojzF9}7U{VUlW3NkllFZB%l8M6$q}BSv0{FV#WG3Ta#}z3~~1OA*pB ziikYcZBsnS#q=cPw@J?rIEmD~H123kQO$Mn5gD882nvc8Y`VUFZy3CbLQvr{b+5o` zzrkqpbXI|c9BiS?vu%-CJQSpEYLYvw3C@ z*0=40lgLaHOOOrc8QZzxAW3TL(QZ%Sj&?r-iCmp4S2PMA#M(>_O-)U~LN!0V`m5_h z4xJMHtXQLA_Sy{vk-N@37(h^bb}7ESQqCaChUFcS!ZLP)LwWQh2@AwVj`mcQ1c}T) zi!WYP)%5`>N~+~>yIK{pgYrQK8vdt4f1%__{oVMmPl(glCwL|CT#@OTlF$}xAza$2 zw@kJprMDO^#$-%g;hgg+vDw8Sh$N!ChAxl1_Acj7Q#~;; z)#nloo3czIIE~ZB)BEdRTvg@wev6Nt!>RZ`Y?nAFkvEoQs6z3)`|v!40fZ7kZys4d&2!4BE|3U3B4Lo_*<=|C(X4%yxQAoSK9J6Wd=h zq)bgs`KOU|F?C%~Z+{?cYn2l^f57~6oyn1@BTxFR8>ijfkdlN3{Dc%mRE=hL{KyWb zIykegi9KQd?k@IY@j>FbN;ZQzcuz6!ed2FZO?#yH8t#~gwOxM3VOyi_9n!NtwLim} z-+HpmJFLair^QQ7KfHQSWXsmA6)FSd;8n%U9zYn)=PQx^p)x9F1y`5b4WBglU4yrfVf z*0@T^e%41b@Fl!k=5Rniz3GQ@htAI@UFa|IC5kuBwDXjYdbS7*g>2or z)m3<^Ge11EySv*C!7J3&D!#&}1&6a_rn_dJLdwLzIc!%Z*{=NMGcE~^50QSqXmN@3 z*Nb516diz-NzZNhcqdTEy1}egudDeuE0^egtC7xIPP6m<8}p*E4^s z2c*pm75I1>=u5W0n4gSAOobJ4^}qEeOIl8aazUrr9!c4I30OsN<} zTO=k=4B8&>8Ecrq6yHa`qkq*Hoaez3X3AU|CK-g-r(T6k@4HpFY(^+w&BfQ(7l&1y zj{NK{`Ppf%T#{0fTo|ce{=zG6^I48J-M#YvQ~%sTiZyPteyAR3>Y) zf6l}Ic#Z$Rxk?AKUZBMJG<_e{Xod(O@XL+DBi5rxvR)!XY~mJ(UG2g5j@cOT{WnhP zz6$S*kq|?+H8C~L3J;b%hL-o-OovESWu>F4vW1?gLFYY+&-^(g<$^pXA%@3fDK0sT zu~@Q-8dfQo@1*N%Tg$K%j2_O0&s=aB8(m5Hvw%~L`EF?r%*Ls=DIe!dd9uMlh8p#} zZ}HAyOwF&qzPkNW^r`mWT}ePZ_l`b+#nYhfMMt1@me2k)jko5g!@h>ThxPB$beQRX zE>~5ywmjxU@k%_L1~pF$>BTk8#VIbg)ta)qd!}70;ItT%a`<0*Lyp~)qm+_hzD!8TX1Q%MIT#w%--&yhTzzuLbqfKx@}9Bo0(b~ zysy^4<@U4S6u!MxZ<(*9`ANlgEsX*R7K+QYi_*B6IgHb4dZPoxC@xR&wO2aH@EWb%o z-2RsAu;QJKT%tORfA05j5agt$qB_3K_&KZ9XoYmPWq(4sM1jNwtpmpYY+9*s3g zg~g4+NG|Gd=0=}fx?O@aC$%~f$}eW;S$M?fn{nDAN-Rl>#D6Ja6Tr3j)7E@heVay;9=!L9IDmJ2~^j5Ue&Kdgq z9mVtY#ue;aP5qqI4-AuFBuC4M(wiYE7H;*{X(&%?2%DXYpI{#-O95onWxV1*o*W}f3J40P1LD!tSt@@$o%1I zG2v@1sL0RgOA1f68d^{HC9@=-YVWL!s;%9+Y8%Cd#L?OfI5bg>1vAa=mN)pA5N%oFvhV4m56R)SRdua=}wJwHBnXK%VPkJi_%{`{(_PF$_0Mw0`}zVZh#M@ zdw2g$VFCr+Jw3F1t@k35dt-AMZ*OEjttrXdbbY@?Ut*Qi_m@T%%ZN03W+Dg#fw6`6=h@7ey3dXR9HaPvR)hoG0_ab1vlJK6n?HxWuaTW zk?m>qxqtdNa4zQU4qgnvTCk8nyq_>TP}4?-hg*Wigb@nb%}nekOwTQb6O^WSDbsE? zYnd)6vAjMQYIdKSD5FoEFo>Qevsc%a85r!oPVuq@S9El;m+7{=y_?qu?&ueNju)T) zZZ!kK)RY8D>c^|yX+K#IfT9-QBwt+vXwW=8cRm!2V`Gj-zcX?NrkTsU*Je9KMOIb$ z9n$Lt`i9leiAnb=gzb1Jo4vcYO|DPdnsQZ95)>{MaWuh`U!!v-@pJMqUfkkP|ObmkCxoB`I z%#nsc_Wb$bk#b3CHjeL1^^krYugaU7hASc?BgOUsSmo>RU$4)R()sv@?W@@!Ex(TD zG7uky7lz6&iqu?yVR6(kTDH3V%VwAx76u9lyu>vKYWwQOy2@D?_Bc{JUO|d^KO@a$ zK!=k0aQTlOZL1?BD|60w2BV;$zq9q(jRZMpEr53iE9{u zwXVjK;X=hK1_3icfKie{qaQSbY!9W{%T0~;(3wSDir*y=d!F)c=Y6?Rz_T}yYm}WCK*^%AjDXmDt3D2D%qx?O@ zK?88M{`42MV=X}D6Sp_22gN0VegjZw`TpS%!rv74AwGW%uc9@DemorXUf^<`uibh` zEfky-AzrW7Jl|Cpt&}-;?-WI||ZsK52+GY&oD^JijPJrhfSNm-k>lCY*++;{tS z?vN;xtqCt1@U+YV3EkMsum9dZ9qVe#_AgR2cWJWKe*oE4;?Fbkq0h$n>ZsCmvX`&0t-A)9&AB#Nf+| zC3FWtcW@Uva_8y&DP#tPMwE0SE#+&b_1pHZb_!vd8*r*|W(P^Wfq?t|b#nk5S0#fc zeqR~+k(%=E#bC?~Wj^X9*ApoP*w?U@CNNwzAR5-jH$vy({(63L7d`RJ?QCmJih;?U~VltR(!V)OoKOxX&DDO5$#zcYA+z`;$Y@HW#9xS^;dG)QwuAvF5~z~zq#+D$$%k^mv= zvGyWAh%jvN8?&x5^)UL9CCB9w*Hsmx#LE^X|B@5m1J4V_^mzn3gfE%k1mXU;(}@4s zOb>d-*4EZcZmY*q?wPMq_T<#XSbMFVGI(56Y?C{#T!@H{W)v{%CH&NPKGG%bQk>c; zS4JW7GCjkhtn7+u6YgUBCWpK;1zy!Ub&mdU9@Y(wm1Le1@MTjdcsS+n%O;>T1X2hT&M-I7#iL zT{^)eu1%K|K^NskdF}2gt<$N`A3FoG=k|X+HP%N+3BvIJJEx6lniA@_l-xA`u_|L@ zrCLzg+offtkv#-oXGk>-LM#nnf8ht@DJSo2TBm^&&fDjT4uyH7@(i>4)VAx*-q}b< zfF`O#_H#46Z#L2j4jg_kTb(QH^}RWzN{@OK>J2B^om`E?%JsSWe`0*K=5^|HCLiQD zlZ$GnHnTsCj6{B(IX8;pnH|E34H^Qb-S@w{JvIB*gIZ^KuPMOVh>I%r->gAGZSQf) zCDf`(&@Rtf_6+0)=vR~k#{7)YM4p2xyZ3&J4@EdD_%GVjoZ<{+sS_2CeJ`<}8$R{n z=>-J^AsC9ZqUFxiN0jNiI=SeF6gc35GeCVYj`ReT$+1gm7K4zJ41syI1exB%EiVFQ zt>(lDHwqhSC&iId$hYFiJFV|AgeZAAfB&?4uNh@IC|W~k$foBb|7mL6M^Q7LJ49+D zr;z`+Hd3}01ZfAdXO-qCSMtu(lqCoGj$Zw|oZMC#F(MDsH+a$HXlVWVC%PTUSpK!i zkOtXVAM0V0e_=Q=&~&jV|ML%at&>M>m^RUTJFhQ~C5T=h!iuq0(3X%BT3a7uWs^&K zX5QbRtO`+|$ZVG+%efX%Khy7>gzw9{ZsEQHIRs^tB#vgGzw>NAtd#Y~z$H65cc?r}lGZ5%>>*Jn2{qqjjz~(Y*dCDH z<|UW)uCp|l)ht05iMw(Yk&4wxX}ElXAS60s;^l)ZN{4YCQi^xmoObNoj>93ekzMwF zaiC$=O636%Xi?RrargqMi#E@+hg!w`4(&{Cnrs>)oW}P>(O{zIv|l}qCc@3neJnezHjU2fG6JLBVCw@Y;VwFC9{ zWwEHX(4x_FbT1gqC9;JGfKNOmPV4tW3mU=h!rOckaSlqv{_jraxudJ;_7S)K=k_7B z@q;>0?0K+A3imvVFLNuG6iFY_h8|J|lzfiM_HPTU+j!Mmp%FE#)%a(IVlf1lnUOG& z0JWP)r#~NKnmTJF1o;<^qDHb_CRlqX=9){;29}HS=ryGA3Di^M%r0?!eF^1R%Gx07 zAx*Uzj#8TQJyQ>Y5z5I`=OKGUjTW@zQ(fXVbE^Hp%T{kQT$HD$-u`njNa=-x@Prxi z*KbMtxkDs3T0Cl}+N%?^&l{y`l|D9{{)^V;Q{UP?FUo6E`;z2%*^owTd)bHF;e{ogKXsUk^f{PzrP$=SJQJ7q*%L>q1@7a5x}W_+Q^SS9!cNK zn4_84*W8nws-U@%FkD=Aie+aq#E#Mx8v#SSdFRGV$~NQs-#x|;dnL4o!5KoRXOA8| zQW2%BA7&IDh}hfl3PYWFiKpJA9GO6Td7sD90vstIqC=iThcraZ4}FcEt9%^1Zj@Ic zOD07klEFO!$;01F;2nL)D5-A1217B`I8~V(XIvW&M|n}sr2B(**7cRc9VI2v)tf1{ z?pIkjHlRixN)hgVd{?R()O<|TiAm{>Xv%g9qqUizZk$K*w*QW%_r^acR|hbW%ccX{ z{k?%#xZjSrRd}_Suj6{{?R_avRdUTNfJ+edF|3E(Qg{4wJx~cv2W1s4S&rn*Zc7F&* zov3D3zY!_=YMrdAB8fMpK&syQ2LN~KTfp1Tm zv(Ma2ojNf`fP)n1G?#TMpXEvPUZ6-3YZCBF7icaA3ZbsjU@_bKb;WPJ>rggJNwVn3 z!R85=e-_kw|8S?6ms*Dzm$KPg!oFpd?Fp|1I2;otekcIi?HewYy7wlTeLJj=*W@bfRc};rWBxux*NRzM zT$-d3$V0i1A_qaO-!uxDOOXWKw_D)IChDgRz6DNuL9g%Ha+(*D?SAoMt0_?4`$doo zin`GA!J?2WD+?Az%fxOMD0P3bX9r+KqBj}vb&-cavSzKGB;~;wE)fx-_X-vF^9rbL z-r6{E^1ICw!NxIyFbSjbBwl58%Enmzt5ku1-&pGTG0>L!Z50l|^?$CQ(x;IDDk zAen}86(T!+PO?SiBtRiG&W9)uC2`@|%Gx9)bY@4uf3cq&px)#0G(scqBW7s#t!+^r z8_1?Dk@yZG&@F4c#i{o(0npBqmZmA}jd*C5PlSqSK{UMG^GFDzNYh8VkMa&IW6(@1 z%l8QC{r3`@zl zyx0cr>IY=#s2`63X@`I#abMxKiW#w4v0Q0Y?UopbRY7Th58*0V!%S4L2u$%?zspf& z0dq71cHJczCe_Ivd9Q#FPN6NpiFupqiA6Y35 zXu=`rl(8-V7qps^)TEQM5UGhc6vtiBes+`;ypJgN@hf_9X!hr_(=|fuFQJ8+9`6sb z&998JUV{xhN!Ch!=G`y#9stMK?4f5&^8T4o-T*ksEP3B8y zMZ-*jppmQs?&7b9H(J6>egqeKnTy_0{+?heuKyA;{=-AyFm`&$t}Y9tk$4Li1QaSc zV>GH)zG=yxXk^uq0YmT{1K8e5jGxq)DE7mG>{3)LzvD2IfV4w)ew|AOOoP!mA3kJ_ z{)X12C+IGI&(ak{L)~sj)l-|?55aD>-Y#PzTWumu3O3fqzT?}xzGi;Aj0NFInvx`g zguDJYK>2u;kZT)K$z|AB8u{hXga2ttN=iEP13`jR24oph!flt$$Q$APGK6wQ5s-}wTpF>uk&c5?f`p0JBP%I z5-eb^V-m|!uJ(Q}#M*T7hDSPz13aSPQweQlW=gqcN!bw>BZ;#|=K#4APBDNz`6S&- zX6MdPEF7g%{LUU8CLS_qm_bHg7$7hX*Ax`bj==s6yGO=wkz@>Wv(wVttf^s3u9ASu zLJKZ$=GU5s06LBtf1mO=c-=)^LH|nl-}<+Q>cpi{KW{{%T#a)8g@ry;7qS}pnff_+ zz}`0!l)P&XQA(|h0!e>F0>oMRP@Gce&TA+0grD?Djk(`AjS=l;f z+jDPiV6~epP;#v5J??@;0ZdFighXzt;FfV>B`u01T4iw3NW!&nA3(GNMYQDh_7F%1 z+K$zbACH58&~)-m1pU!_HlD;`{L^63a<}&vs#p7vRjPl>l%M1j4x+zc+vNUFz_H`Y z#o0g;02orXMG9HdwF|Yq2*_pbXMm0Q1?t}bV5x_XcrJ`orPpLV@Xerc0_HCMZ;rHq zqYy$zCVi+tm$1G{S6PVd)X2&nkSoO!Jfu;f)%=eeRSUms%lBNqV$IL^olbKW@XmuM z0lwY`%vJth)Qp=d!K2_l&7kE-pM?yQVz#MSiZe9mwr_5(!$azc_mTxciN}a*EAKx< zVrf6*5wxZUNp_9zuL(f9Nbc37RQ%=rkY!#4l1ft8FKp)QW04+~iQs_aV%_i}K zv`r0s^nDQzbv00Q)A`x)1-zUr(ZNH9i25EJqT!cC%JLns=UCwV<|VX>rY-|sXkwBg zoAm;+1>w0tS0}QeggyKEbA)S>hmzeIB;9i}*{jGdCf6Ai!2OY)Uwi=HeZyWRYJRX5 za`b6*wXP)B&^+wHge@ccOpOPMON^!(JXLe z_W=Fv&$P=&iB~8eUD(jE=uDiT^3kZtzN#h@?`VUfQwG*mN_LZNT~iB7e~UHxk18j! z*}$*WQ-tL;!~7m_>qw={QHx9*!DblxidQAKt4$MFRe^~t&`fRStX#?*Nph6tV@ymi@3fCrye%tzIBf{sjZ(QZA| zdz)y+PgHpQ(`(kdogG~YT2mso<%Z{}p3MZ+lVv^LHvi0^X#SXywd-1PaE*4Y%p!)H zuPV9lWIamr{9j)A0%PsMZG10GtLM=H$olnnIt%8WuWC#_brlU(Aw3H}sfpJN z>yb0=ZEScp;1M*lWASh>Sj6mAore*Pt@I@Z`caqqfH$=lxmhm!8BEsb`QB zS$H$*l9JCJIS+-?Zr{^bwP%*L}MdotRZo|yh{_MXQltS^-T$8D2I~U3%y96&II)o+#M)= zb_DBrCTnx1ozj;!+1Z#MRkSSqrP=ZNrt61Q{yw4gXr)#D?OU?)u`71u8UAzCx$Q=; zRNOW;jd-6u|G|QW<#FIosRzp(!VcX3^GLVY9*#ZR|9G_0a*%!1%4NT5sAT z7s)Rr^o_hPwHujhwhvFL7T0%JnwkB|R4=Gx-cL+3%-`s=#j|$r-=Os|g*K1$zpQbs zm?=9hvwkj{AFq8n85Rlb$S1iJwKG(drrK4`Qi45iWREuFP?YMt z7}5~s+y3h%fLy5X;*e#u9>0Ya-|kR9{fN-3|3vq5Ov8{@^6`1Q1#MP2RqwGck9_lU z(K)3UADil}R8MM9eP+{&)Kye?`cqxDSEK1hn!y~)gZ7a1oYbR8s@!lH)jb>%tDVO0 z|L9#yMfDESnRItPtsP@4sHoJC-$5C5@KS^Eb{=E$e zsi|p1^KkMV{RfoRCshon22#)`d4gSthWFDo#50z#WxjWo?iy1oDl%2N+xVZr+fGR= zKe&16+ZBn{q!~a4ZKEiu3tH(954-U_g}U?Zn+B?fCu@`R#Aucs+`06L|8vhvMdfqv z+ij?>omqNRs>8ef`+D*61`LggZuC}I5=3{)iA(+g%;$zt-0sC4#L_JJ9~TYaEUIy z?((6QuQw`aQ;Dsvtlao7HzJn%^|}T7JZwj=eoEB<(qZiCJJ(<~?M0JbHhJ~_&b#T* zgu@Vi7jcPu^7Rya{dQvO*JgP0-QPB#V+t1X^=bd>U-IwF;OoO&DaC){{`DnI%*@{U zFvD{#^M$B+@o5jU=eHTf%INCoto~+CUvESV|Lb-CTi)t_^(AXsgL{dBO|ceLuABe6 z(!&;OXVri2UrV(jKTSZx-ySf-`T?54K2Yl`MRpYSdcHIh1*)FixEt(RGMyM;UGGt=@CYh z2Ou=Ag*rqq1N)5lXDw0Vm5+>#)zTB*50#IoV{t?8P3rz$4Ng{uQ znnUVYy(_P=iHQmQduI6^bf)G`ub=k<`ZyyoLNd7r3V=mN9+8%VZ?&AqnB%1DX zouwa{r@CK*Y23u_g~rW?;cz`cf7#r>`|B|hSRbcOqtN>Kyz1LK&Z1NI`G_#+o^b@%{q*X~|*X z{IRYE`e0F48fMZn%_>EvD~+eya*3K4xD4#6X!K)#wniyBq{D>~tnhJ_8X(}0f5!N& zm>r>GqY#D4kv(ZgpX-8U+l6^Tmm}>GBb&I_D$Fzg&tK|=TVm$AWA<(|>;IV8gRwvT z89vOotS2TjLuJzLLPk06mG*0!Cx#Cdk?BkdFWCf}zoVrf3a;1;AWH3$0sme!vgszBhWV@ zNV0PDO6L+CNVcC&E}i&^PM!=kxe@je^&6SW1zBN<4;PD~ln-Bub;_xc={-bB3bNP! zZNc9-C?_O&t;c$u7_GEGwt^*;Pc`d<#L*CnKt>$X!F+Imf^5?sB9k86I+YiO%bC7- zOsc2QlP>IB@Eb!s%}J5n=d{tdKtv|2bOulmVUU@#dQ?=D{VLS~IEk9;$=Tf(iY0sx?g?bzB_8MU zr`3&?wY9a`$!P-*Cibm)ue-SAZ=IEMZfzjW3n%E)YX>ZuHTnTWh6}m!D(dSY zR3D!!9}{zho|vV>NNp)o>8091I-Zz`Uv!CEevf#WyTp2V^Jxx~mZyWiE~GjFRG8p* z@Y=r%sK2LiQy$3d7Z)W1p#-XaxvQ3}xy)nHq(`4bls7GB}189Guv`m#v_Wqtvgq zLX8e>xW)wYbx%aLKP+COEys$;d3sV(Q&Up1lO(OedPlNFKnLE!uza!7#z$z zWwvo?&z3js-fDWiU|*^()O3THPTS9io~Z#vLp1-JIDT6N<|N9-op@GfZnxyLHg?RP zdn#X(-oc8dC36_M^JwtNyH-PaEK1|e7*C*dHD`mfx*JGJy&BA+7$le@=@fYx9n!`X zGTler1kdEPrr=+nT#z7^E!s}t>mgF-YK5M_IZ{HYC=0S^w&_==2Wvonj7O=9Ei z%;pzYTQI{b4$~0yYLkfGjLeX6SkEHyY^LU;FndOm>8a->GD6 zOODC`#m!{2h%%8Jl4S;jku@xcAwlJ#;$>2_@Sok=M#|WMasW~ zru?aM!a@M~Wx<<})on5knwD3Y!2lSSsiSD4P`LtKigGvRQAphB?;#`MWHtvz810|B z1%LCD{!Zaiz#hFFtaP+t+(~ph5iQ08+IA6!{8D*hK+nKhuC3(BaWe5EFdtisvqUoe zFIHu1uVbF)zEjx-A}Qy=->3BMlcV72K)8NTU1Ts5$g1N$zi*s-;K>>s^8$?v`LF-D zz~sW${=nkNR1Pw*wM?Lr76Z23PRv$RO&}_2?Ni|_NcV1UBvX*yQ!T#bTLX3XO|G1| z;H7_Uski17Z3`ooV|_ZyZin549qiBk*~FLq;lTl<3*B@jI6eIwsZ4_$pN{ji6UcRL zz{Hv~5y-NU7g@geB68O4H|=Md%EUMTn~L)2ef+Jl`xzYB6E&ABc87ygIc|3p`m7iB zM<{8C94nC|pY>S!?9VYGHl$gxKfQaL=uo`w7sfWeIa`p<2eIaF!o-M_5H!;1K8xP} z!-cC&c+@WbagstwwA|F+6`{0fkYizDrO zfd7gK%FH&yvez>3C*)_AfCUhZVb9p3t%kTk`uh&<{NbG@ip?|1%Z&R=khjFhB}ZS_ z3sL+KRHG#4{jd-t23vADzz4~>q7_32d#F5s@s$m5Zas2>1`TOigS|EJOi%_Rh5ZbW z6>I|$aa#!GGMLM!aqv6S87h0}(DArOf465ekpIr3 zWaU<1;?UyCtwS+9pvi|i0s=&27Sv81=qXi{maald@!s8?FDooYCRPPPA^6^vLEM1a zNM{GUWw?AzRgT+YRBXz*v*~~ePQECE>4&bcpbVHP)|W*}zcFJLD0 z(>jEZ5068OLV85C(5=-P#^)NRt*o9omGHvtUk%m47f!%~RCvSU9lI-C!8Ga7cs#`NSJF=kYszOe7> zm;TNg5D-u*GV}Lh;1#X!)Jf{>u`>Gf9~S$j!7(+Dai_nQs`+y~Q@-p|DS`A9<@&$5 z%*}(qZs=EpA9)~XIrNuMs6@aNra_uPxt|wOUk_@!DPg@7$^0-Mln`JT+*U4@^wiGQ z#y*#XJ~HqN;uOaNVeh`DDM$-pnNgzysL~n@KRAS#-BVdPtW6D;aYM4;SM5t@W$0Ke!sjpc+-AV5QCZx}{ZE z;1^w?etS301wo4B7>*XsBSNc$WUY^vtgizTT1s4OI!ZShW>CK`P-1zrT^fQg|MRTP z#VGAVO9+>!L^M>syc>IaNZd_eODTPe32jN^n)3}1S_S8LlGD{+wKN&=eiXjjUokP%P6to$^iak?!^ED>S1OdmwA@^XK;)rkmvJU}6Vyr|) zOKLOJPJKX07z8Z_Ado;RRR@=QG1~bJ$29phBHrVwL(n5Vw*!ruSFv1Dfgp98Ce&O6 z--yUuMa_fJS;43+Zk0>|$_d$SJTe-_HIgo}9^^UK|BLf2Bpz4?*=a-f~5cQntsj*OTK znzAk}po!^2tK%s)8fHn6Sre7n!{_M;$dKoQC&zn^M9)flwjMoKDb2be2OlDmz$Xo!+~D z6nXlMj!aA?&AqdE{R$F^^K+J;U0OPFvZsZ#2+C0Unyr`~8Y!2P&3?8g5l}F`Q1tkD zPp9mR=>DC?&(KN-{XaW5eFBMiBqV8)F&Xa0))y8Z+#|V#@DFz88sQhLdgK~Y`ipxT zSd3>d=K1!mY8V5n_YL-uh3->KvNDy4O-CA=BRnP4)fr z63$Xdy~ez~D8xj3)aH4nK0UV;EF3)YKu0^Or0Q#45IJn zEz++|ii(Wv0g^^0S!Z|9z`|21v~XU<73kHuECm8N>?aGFV1TX-;~jUhPS%@in)Q3| zWej=7T&E?V{H|y68%e@c$_3gp^Coq$H%K#SX||bk=;cq|CQrLFbjAG?pxtX^O2<3@ zSVXm9O;5~8deLaDg8IhU?jM<)#;7v(%neSTCj7LC8!!Eswr#(UD$2B``!bW2dp$|t zUW_36L)^=)uMg{uetFDb@jqK?2S<#m9)LKWcaa(ki_&+P|d|3ao+F_NmH^1dp|kQ z(ul?Z#DGdk!SMA@?ADqtm#4@2h;~6n5I`wCzubfvZE$C zKi>>AWxL{#08g^Ts7_*tRYV|bLrum5@LLYGB2?_RsV6inP(#v}1IOYA$1)aoD#BhM zFe8Kd>^P2}mE(ZYy?pN2bHOKX)Sx+1h_S0p9S)z&I&l8$$vzMV%cWk*@7s7AgFKsJ zwHwQ5ekTP_ck}LAF^;EmeHK$aVSHoLaSMPLBT0nzxx}T9{N3H?66S4fT!*ff3N@Eb zh3i(CJh=pnuz2q6%}#REVUjz;jr4`kU6?l$MA>)e^c#`+VG(JU{<@dL;79L-T9wcf z8p}G&lL)Ktkde_b^!;t>3)O%issTw@ly%on^|r7oSG$p5s=6KF25Ma8CEL;OX) z^&mFy?@n7wLYQ&bG_CHbim^ulP1-Us^I2eO1k&;gHf9DKOZeHB_v`#zP&B^1qBP}q9s8W$tstvR;fs1kbI5;l)N-% zixvV#qlKOr(1`gyz>6Zl|@nEgq=*gO4j#0gE|0;_i-Y4 z*jGfc*xDUhb%A*A2F$tJe3Lf8Ir-;$X^wGc-4#V<_h((_CtOLV33f^Zpga)#v83em zzQ?!sO*H~hY~}YgK%(?&>+$w?C8-Ug(S~9cNDmF;VG+$!4M>BR@CO4)$w?CtNk48>zMZ1W^Vg;G-d z!G2|cW~8OQ&6lWkP#+MW;QaUeLGEh2v_Vdz#a@gy*duF5+W%>pFdbv!{C02zccpgO zO*|i$-cT~v&XWmvLSiKAq${KC=a(JP@-O3tELCDAMQ`Ge-|(o0bO$&MKNt;I^DsK9 zEg5{6)7({Jc?tQyA@}Z9^Sqtjg5at>S+k947zKIR>@d86HJ2HwfB!Q*+I)KLn7zJh zIeHP9oi&-2TT$^u+eGZ0|JjXz&n*6iM~xK33P}Fg`8dCjOhbiOs&Di+cJ@Ueh?Opg zU?nyH6g?f>7Shjd_;i{W5k8)4d`naxc_eC}FKIo|sfth|uC3lc8`@clh7gP{zBr$<_nrLz_E(8%Ac^G`_mI`Z+? z#GSlQ;AMY+d?Dw}1XagEk+i^cra5-X2%}?MX@088Ioz<=)$V~xW22wUmwZ=NPL5TC zbr|(Mww=+eS{cQpO4~#-+!J^inVH(%wo}YRPj9nnW8=4Un%m7Qf=l$#SqlWHy}3aE zU5_}dxDhUAke+BX0|&F(L23G9)n00%PrLpaN+iI5u|_tZjF+fY#bYLHw46+lCjiBcm~H^lM8K ztK%T7W&hLwP%;{(xl*Rt(uT-xhb8wjxW($!3e=%1&!fTbw8WFnoyN!e@1_{EPfV8B zGqySK>^2_ha(&MsQdC-RTOXgyy+c#mP)q^SUHh!B(qD}YpzQ3s7B%{Aetc{N&&rk3QR4OBjl9dA}I1PnK z>tm}QzX2vrCgsfLwW3SGUM3BFfUU>P$4O^&C>Wo)2qVL)Xa!_1*1)$s!2@4T^?YP_oA!ZM^OHZvvLpg< zsd_KQ`7?St=otD@c`fFLrpT}18ExSyqtQE$7TRc|L5TVprRMU2X7YfY|0*JM6$Nd= zNZPS(tPSZB0R%)y+)~&5#~U6Nme%KrWZ}qfl|px9GZ|fkymyn-*W^@5)6<7}6O_N0 z#*{qz0)7Q^tbbWS<^e+-wL>HsewmYgol|!U=y9q^GWq$pFEA9mBB`WK zik$fKD|_S-^jws8?nD6soB7U;ZUm^W5{U(iiv3^lFDh;5lqW_21+YHJ3ax zvB>|JqDzVu_>``FvbXb1N3%U5z8O= zh`&p60LF|W___!tsjI?WH3N>>^)e+3t$D#5SIN||@1FLEEhJQSztH?eN)F@+9>$~D zi!zeko~Qfz`feQECb(;^|G+g3X5+I=a~RbkNdJ%dshD|wzH`!Fqmzq^OJ|3o!&eq) zxdK3o5ywQ5NuYr6C^yadchVGxzgRQcL|chUG5RUbSFY(>4tiiw#qs*p26dOCkM?6+ z7wAjj!F_3?@4npbzky{sGFB`&TnJPEqOiSFhA0*YDUYO^^o*p_9b#AlnjG%TPG`wS$aAE7c}1lj99Ae@_M5s&;70p3?bn(fO1DGrt=>#MMLzB^L3=nZj-^@B@ah^ zDFp>+_PIW=_Tu2v%}q?Mhm+>LJZFvOr2us=WC477I!5v>;=jpk8{}^e)U7c0X|htMZA^W zz_I`m_`T9*{gbyARU0a^_;XBTR!@EonZmFs2wYIx_6+C4`K+5rW_|1upVY=uPh#%nmY7tJfjh6o;{{2G7ZX=EW#_%24I zE9>h2$^I*Jm{dhbn~(jr8VM%D?3GixDD6tkg8}S+A^Eo`p);q=LaHP_hD>BfwZ4`W zW;>Wa*vYza)eYEkgwYJrJeQC8QOONkXeIEM85*n3cHytRTpY1NXPBSA4$apC2Xc2L z=GpIyr_If2_ePt~#zA6tfUy*@3$1$;8OeHp8xYfeQ#c!G8jVrG>Cq-yd){YjBro!9 zvduxu-36NegS|Ho%W)0cN1wC^5u!4@3W*d=l0t(t38|!#lm zqJg5h(4;a}8qhpYh6bh5u+RGqi?!DGef#_UzP*oQ@8jq254}C_^W4vUU&DEw=XEKO zUQjnlM*~Mw$SaLnNo-LVPlhq@4S#x_YAM+DA0pFE9>x!H54iqXSXvV>QCQj|{cM)o z;3LqSA&ayI5OFh2<80@hVJa@FE1ua_?bcDeZIs6@D-gh`eX2Xu$Q*qS zdP*=xtK!bHXQ#CYu{JUZj$>rGR5k4vueixHU}ffA$1q@Wir0TFD2?lyfN!?-F>Uvl zcrhjiuq(lL>fqM$FgznF=~F%J@w6*p-+6Ss>ds*vm1CQlW^>4F0{r&Xpx+st!88## z#P-(H$ONhRLuC-+dF^%{df|7nviX`e?LaNhcQR?~^?^-u;zpekGup3oH=$ca+Sm9` zK(6jb7oQ|j;m3r?;r zhNO@vh6-YQXi~kPMwqoVHzKex3>hXF*!<6*GREKGK7P<+JN|Lq<$)cmf>oR3>|l?P zWg+aC#r8}JC>OGOR-yC=K0ezsI}KBe3vs*&6ex29^ee1)57M?>8DhOjaCAnaxUDMy zEkC_`H+2`3_i8qgmw$xL6htCc&hRMypHLQY?xb=Ti8xi zRF}^gAwFa<2&O;DrW>_dHbQ5+MYE9q%TTi9kaSTJyb#o$!(s8>JwhGcy<-6evyk~T ze{)Oq*}3JL7yja>>Pu*~6CUY@CH$30`-AvsJ|`!7-|~MjB}pN_e9m_h^uJH_h_^w= zk%t1`;}n)Cnrap|IAO8ixyw4NWAs}atFJf)DIG4&Hy^WN`E+9zo#MVgyF0wH$lu8yFbBD z4b)#tnm&#@sE0SZZOE3<_>hhvRC^6z6KzjC0TS;3cg}X?p~c@fE-zH+|gGBFl2U~9DmuzcrMbJ|-dlV55kRk`LebG)WWqeltt z2JpN?r0>Q18-WkqK9hzZa3*{)O!Q1XTe;olKklqGNndCj>XCZn`ZklO*h2tdl^OT< z?`Iqq;WEhMAYp!aW8VJt2QU5t&%|^7og#0&1J=W%M;QKI5vO9e{Cuv_zu4_mCwN>F-zS&%7rULZ zQvtE-$Ax~|%)egfw_7)FI+;m{f&%vAi9f}#&L*Y$i6{Fnemp@I{~vhp3FXN5H#i!p zC6IRiKG*cK-Sdgzl|yI1?>F?0Yw_a6Yn%zYi&A&TR+?Do;R!|_d8@xZ>1d?B5SMo$ z-1R>$^xHdAXZ>0+*#oSFAG^Uyb{{?D#}ZTj#gD&>0{++D_-_yX>zjGU^}C(@{hhyi zces|p>ZfnOh`*DJ9rj%KFJJZBlKoCg{N+NZMKJUX-w#K6xRBiTA3KPOJ^Stbq=KJI zOxOEQfBfIx>i^+`|N3U+^8QM?3FH3vcm7lOLRDgt&%_Ri(}6`t8;$AC4YpGX8n!FBkgFJVkK+x2(Y5jKqIr-2VFy{_C50 zF8sR&>i2j4Q`YgD4g5>|^2?Jx7ye!R^2>!L+~WV575KZ6n80Ef#`u5l!GC=-^phF_ z!F8`+6G8&vVE+EjfjV83{-+JRdw1fZwh9ug2}yGydCF|MiXk`ryC4nJ?bI)0BUI=f{tyeqhAwzu7?4sjb^do$S{q z{qi?8+%FgU?VbHMt-#-m#Q)rb|N3VBPpCkG(f@yi3Ud$mf{`{RU$eJ{g9P7|7`!&3 zaG4oVHASiO!pHR)j`th7vAloCw)NOThB-+ zEO0lu8H@;me174?z*h08jM;jJF((B7`FsiEb{V&bV}JSrbV=Z|5?1w&_9FR%glCj^-0wCRxA%KYhDX}{Ps5AvpID5)R*?RmFZK6T``26h@$kPb_kXe@eleHS z;OTFl^Tt0%n7Op_@_0^&W~rR%8C>?I%cwgeh*NIcK83| zZ-jm_8FfH1TLIAeMoeTy;e4;29+AWih_O*}>+sy~SV)7|mT!}l2xrfrXsrV(`yd`N z>I4x|E@;v@W6^%h0R|mu&52r~`v{y%CA(vNo%TYp^d)d12*N}$cEmStPLUrEPwsOD zNg_gu^ryF=<4wv+y=XWw%j!kh7>ZgKLMgg5zPN2 zX0`uZFYPke{cfJl`)i+F4%Io1WV{F@gOQsa6Xwv|FJ!y6K5-}IX zt6>oBcB_igv~8%BI)qql4(1TEG4}h1ew5UXL%1h*@WQi-inS6F+B~4ah}tRSf{dfS zrj3=Ab*^9L@%Nh0w^ka+w74BHghCYab~-&=z|*OAFDxjC{pj-G!Gq!X)tH4bX_RBO zVZu=|9{tjwe^g{-(C}^(v8@IyUr3+Xr%%Ixr(oX1iYe8%_Uu}1KR!0v`7S&A)~#E} zBBgEtpGDK3plOf=2d0@BW(ctcloS&g%8h{WrIJe zU_CZ6R20dIev1Asu75C=8DHtaGlUxGD?y1O(|~NYSjWO+8Eu>gk_QFV)mYhIyMgB$ zWE3p08{)Xr7kMThJb19ZmNWlx6}bENS%s6k`zBHPoT9Kc_C!Qgpz#W0i>#UprkRX_ zhPrw4=E!x>R+@~WX2f0y{Ba>32ggOIGu(CHvgUfsnYl9G_Vv;G95OGRLE~2>X!!}g zl`CV=U@9^l61himI*^7&cXwM^FIh9jftTdH!M5GUX9~&(!h+FWkH2)$y{d0R&V=SD zyH!=!~ zt5F98fdmF;O_LK7mFN?Koz_=yJtBMZEt||U$I+t6`xO@3WHlsv3iE*GE1IO7$sFPN zz|PiG&}1klNB8r4hDv(;KyPoo>(_lf-H^w11`iH8YRqZb*OAYy1dU&G4b@qll%%7| z5lEyiV#nS=5MytD&&VRIrd(kZlD#THG5$HPlWn%X003Be76awC3tyyRwK?&kZdhcV z4d(PtU0==xj@g_-h{nj40Y7oon(_K@%%A`d!?SnJ z{H4fpxyPJZZJ%{q-1+J(xlw)G z?+Y4eciwLH;Csg!l<^n@)!RCan&2q$O3={~cK6A>bs?l5S|mF+ccRX5(w9tubWsUv zDCN74L)aKQ-qF&+n~Cl&)q;F1RyJuEAFh$6@y=;qV41;AE^0b2oNRtJY_a!gFAz6l z&yT!La&&U?Ozh#FjUIfJy~39c&)3!5=Ku~XSVY^yW%hBuFgtg5)G~MIV9!UG2UeL# zU2GMO7I44gB^npXqZ#~^WN&JAb_AGa=(kulJ2J+Fc8sQOlo z;ZMV)6WR>jDjn-LKV}Nm{|H9ARvgz!gX-2h)fJQJ4;gPZ34B_;vD-YAN(S|4_Oo0M zlr&LJjK;Rp07{*eqAQ=HqM@N-Y=yE}+kJ?JpIeM2Sy*p>_wnOxjv0oH8`$U0ZS(I& zf%8DLmS`=RJ^zT+?co84y!6K0Rc&l+Orrq^N8)3NW%qO*z<+#or(d6 zV#rwd-q$D->bQKv6o}ULc21h^AJ)eOW^o03D?d~$d7X|<20BSW0xRY&x_8md*(A;X z(t5WsmlvHiDNzP-hIzTM3d+ihltx#rT&cWbVNsVKWAi6!-@a1E--&}MQAqfZnLP zzn|kYY^;F$8vS(j#4Ttlci*Dc2FCHrmoE^*dM5Gm@?r~FZWsSDeje;V|M@qoo0dU< zv1&a>j%Iwwc8imG8uz6+A>mq~N4$w3Ifb#L7I;p>jp#mGl@4c*9cH{@bV`Uj9~p*o z+f3eWav5aWZB3W`-QA1va&P#IAYC#2iY6`hqlG@V?OER9^j=CzN-^#Q30kiI&l$3y~ zQw;ZsGE-asj`P(#YozFb3Ht^u1(ISrCR6E>3^y8Z^=cM`YLFRB{^O6qZ{HdqUjYq> zIOQs9Cns>7xr$AeE?p||3jI8#QRIYP{vGLa1a_*V&^o!3ED^YYM+p=~nPjgPD>=oa&#$QCczP?$A?Sv>FpQ1rjb*yHzs+ew1PtW7>a`DwsHz+lk&mbopqlN2~6Td@)-)X$! z>F3AA^3j|l`2p!NE^PQ|YRB~(H*b0cuNv9BoFynIX!*`4^{0zU@2*)^=b?c)o zHbS|2Lsr4%_Pr#2{J=wMUURZOEz|))8#+tM$jG4o z$g!&UD#+U%zP!`0a> zx@+0g7zvT5*dOx}E6!~n)Cv2z|LArhCdxi#B_5n_l7t8z;Orwd=ShqUis4^)$D%UX zXS&``q6_ZF3X41*qi?w~J;(s7P_-s5$h5{kG}Piy{!(<8R^F!=A$}~UU)wH0AUZ!k zUto7UnJ`*Hez|K2_FBjnFfu)u740HtN_wgW6H57eXq%?8vfh=w&77D)r_&MTeS$!& z^XWoLe+CXmes*YRDEfH3USJ>i+nU<5oP&cx{Yy>dt}Y{g(c1{?tDZo3ZmK8@ZeV185iMWTx4Ri z^x9S7BGmrV&Bnf(lW7YUEI`;}gJa1!4V(Gg7Z=cycUg_&InzcCU-F#!G;)hxyG(1YvrV$t?<2MS586v`PQRS>z$xpQ z&bV%%X#0{zigG4fP)nTegZgXwOw)3s4UHE-J)g(T%}t}NCUNV>1krQkJ_3Chmp)0h zPW3r%``DTJQ($xMt!(7qBKBbrTF$B|>I>2NhqsZUi{WOd^o#J>mF)YOm0#P?j=DA; z|M2FFs-5&cDp`V9aVkIKaykMpT@Dh4Q;l!78zR)XY} z1V=QL@w*hYe-24Y@F#y6aZva}SDXzviTv?q?7I{Y>)oh}BH+!CW}#?`ziScwu93;P z!h*uWxV~BWQ^*z87PM}`H}bECRCRX;4}B)N+QE{LD3Ff*nAC(MC3P*zQTx9-T{7eO zbM=Ap>?JXY)Ly^n(-~zRbn`s3$VO1U>5N6|MS8B+2x5+VK|d*C=JxnnR(mk z2e9VcjQ2jV@QTaYt(5+`{!wsbPdz5ymowQ`<#=iM-FcoNj~3wNlh-zju>lyr>$_}@ ztt788n=W4S2eS7?Cf>EijfbMjI0Y6K7RcyHCkeU>I4t2{@FKLB;a4`d;RM3?;!iax zk903-?%E}|JAQd4tnAuXkSWnPIkoQ}+h=L{8Fm#S8jqN#rKMuNbM(^8`M9&dCj?bS zW^Ay9%nEXW06o}9$opSzInJuQ2i+*m7322YZHFVBIqS(HNvnuhWd()#-Y*gNuGB-% zvGONRf;oB$5r0*kZ9meG>uO_jr>9A*ok%xEMn=9Eq3z;&U+V+^wpM3Ne76R!P*hNW ze5C1q!^yIe5=p`(4;Caj^SLy$ilXl4*%;F0X^fPUU304Ojl(&h{hT;}B*~^5EzLma zzI5^8ii_Dunn`kh!qG9k>BTsY?bP!N*XaTC($ow{Vf_yl*dL_M#q;Lf;28E$1u&*y zjmBw3&1mQ=tMR3$M+#G4Zb2Qa6n5#l%Q?lGyNCSl-@kwL>T9SCzeM^-xwyKXSA6&V z7{ZEE`8bPsk?_MlQfCM*;}g2kAu&Y~!;&>Ds^WuFBrR%0G-zU`9tL}o_y4K=&UE%Z zXq*%rknf*RETZu$`H1*DH2k+KdihA@;<+*WYrT18NkfC7pvs52KD@hj?P6wTR&+&( zc_$z=ls0?z5&@`btK9MMs#*3D4gMk_O(QJ zH50t+zb!#f$QGJ~hb5*JYFSl=*WEk17XhM&&hD~tpT_`)3b+tpscqdV$~L=RB!G`@3u9D{-UI5xIzNtJ-r?W z(1xLu4kgy$;9$*3kuBvKV*j#>mh-g_V%1Vdj+|AyD)A9WCY&*z`06I7J4s2F(?pf+ z?d?@mJlA$b4 ze_$Z*tm_Kgo3~7P*PAu8u{Q2F^Q%E-<50BZb zU-74}?+8ZsB9%mL=+v0c)FSfl-(k}qK0czHosEVd&&tcSocWLZ%j4+VeXie^oekD} zXq}wYgq8fw!nQ46v*sgEJ8B z_Ff0D$H8}WpML=vJNW5vVp?&Ug)n?BXs4`qU@34-q^3aGv(TDiQtvuwbJ2_kJ3&*a zeL zd?na)?`!cfF)<%4twz6nt>&@;tnSJ8uEA?xqVCOMN?GwoSkN?Gv=~D$VLHFm##bWk zMoW{0E7Jn9vcAD%o@aGJua{s^O;`sY6Gv#59oiOPetlb4SAtIx5DI{ArYEe99Yf4u z5^SUiD}@~S{AtXxbt_wQKek2uD0;K?!dK?|OPwrV>R?#-9!7>c4|lEIQ0R2x#2Dtm z6}6$Mkl=6>W>Q(_gk8LNaQlLJz~9v`n)pbCxs5pq3k#3#O*tguyZTSvD!_+bv&0GJ zy`k>UIX{A-bouh7XcluEt&!juVs?g~gllO#qdTvV3!P3s07WSX8b+db;qC|!g|^n# zcdVBoHc(I~z)>JDypN9$2(Lsj3`o}46o~zdX9Fuo$;&45$$@}WR#sxYs{p>uTeQx6 z`NjRmx&i4biY{5QA2Zr)&K6!-y)RR1lXm5!N00jbopgj^IyySQ6Lf7aUyu?CAT-so z!9eC3kX=cMLw-_c0KIAOkkKRW>{0~Ru3nJ@7jfbUyzYq{zz`>LRM2e7+j^FnkJLQ0 zb+{z+As&!jqKu52PWZKJx7qCW|2$NQvu)v6wwzQ@j2T~fy_e_sx38Zs^(Tadx*(#7 zh*% zxV3!w@{%L%n5JAK*^3S^Vsl?0Pbh956jerv1bV+@q%=TO*8!|rw{G``V(1GrT&F>8 zaBb{G0OT)wk`~#TrhI$Zg=eqi%C%`z%@MLc2h=rhe9T!-?}}q4*1i*S;}5={rD5!g zz3sV@L+~C_9mH?%A6dMBx}+uYkG3}(iec3zqFAzB!GBJw1SXl#*1>9Ae=8Mo?xlJt zNN9APB??C48@fR${NX9+`BM~$p%qJB5Y*Km#kCvrc2gtOxcwnsSds~7Ik*A&`d*j4 zn0DAAwdy6hFI6jAvK=;3vQAn9&>CPab#!$1f5ywN#6z`)1X_Z5rx zzHkGc^~XXDxS;H}%{j1L$w#ypl^gX|4sQ$pD_8WA45VE(UII&~viAL%cRk`T_Jsjk zW#^TmXj%#4ihWeCH-*jRokM|gWZ(f6dyZQK6u$ITzk7kDGingo_%$r8B_{$Gka zA@+>$GrW87yg{|Ff`GmCon`j7i>?KXG={DD^6d6}6W?VH@e%_1LXTfM{H&HXXU?4F*YF)2wB6!Q+tu&e?Ih7Dh_0-W+%gHv1R^_OThe?2AYOBU1Hxah7ESol7h@T8JGZn>(Xhj?E z)f|I!8)nbW$!Te6Q6xp$6QUv_>atD+4}RLn*|z*aAto4+)J#Uu&Z7l?MJcM()6!qm zJu55ACg~zPMcJP8o^j@*JT9m{^6>CHx%Vgx!zN4YDuh)9qGMuk-23mqccAlxpH!>s z%^N6CZ$NOdcC9Vc&Bi)$)oJpg?%k-V&Sz2ioZaUuhCo)PA}cH`3?pMy<>gP+oQ}S6 z0}~d6oBJ<|nkeBI8#;enH@eimh=%k`;-r2XwDZCJ=p$9yrMZpt33yN}#s61w%XXIC zwxsr6$k;zZ<1yKb$am!7A<@V4@)b+?QRJeLaF<&|cdcJ9DJG@~tq~(Y7QFebIlUP# z6=t5jD{90cc7#U-hYs3No$VEcCFrwP52s)=jX3O@M5l6%(C+T;>{AZ%s|c+7;ZD%T zyy2lCj*N|(?19S`BvD6qmM{rP=M!$-WS=v)xb+Z!Dn{kV%K zIO3MEFPgb^`Sthq*5x?g7mIKk9~-8F1@Q?1c`2p5<%enATz;M3B_`O%XGYIT(z-x& z6nzTs=pHmRy%BQCyEX%v!BXyAq#Mb}$vEJi%4g4z=TEM$-L%oz7ZBT1$MyOq$0Il54zLyaMFD4ODof7)xs{F-(>Txx zGg>8jp-;jx04n-{AyLNqxh`nYR4OKXxIR0hPHE4t3EtjlRUy@N^L20l%972?0Rf}c znZjl3sR;Tyrv^>_d1+(cJGhx*SRmAn1 zJu(!wzc$$fvS*YN!u(fM{Qm^j7u9U{>CK+(c09W|^T5)7f>|7XAyJ{n+q*`QloTpT2fP`9DjwUfLF`;Yir2dLF-2)IFn-|A0W1W}ms_4J&R z=Zy_0ESz-V!UZPgDH!Rpd%?v+x*w6*D=8?HR3&IcotvL$*&t(=N&wWq<9N-*ZWBP2 zDfiOKBtyeg8(Uk1U+V~IqgtW=6cuH?dlqrN6{$F#Wx|OIZBcHk2mUpg+oce3 z<$2z|#9Q}tbC6UVNRBNiE*5Or2iW=VdFX}ratH^&S-BPzbmRWbo#8^AOiWBf#0^^j zMhqGqlm2QrQyR$M*CWcr)W67gpFhi@oeYdDPPfJDdmo=}{2atuZQ(&CR%vLUggft< zGhB0dc{ys5aY!h7o6sIgoJ}I}-r-S1?8ZaLJRn}Z$UpGz#UsfYyMMlCww{)6!^7NC zHU+w}NOv~nnZPm^WhNhb#%*b7xqkin(WN9zz5WO-B+m1$6%#{O{VBBqQtgtC-4VJG z{3J9Jo{lmu8l%qXhj(1iWs`@jf#wIEVRJE_F4N|*bet1?@53>^rAh7s^7 zkDhs*HgV(7M!nwr{!Kb=VTu<)hDRmQ%^K6h#1NUdGFwm%YC8E3{B z_`2P~rB!HW7-n+{u@EV(^`6Y(i4)9sodsF(Z0s(z`o%@8)D@D>xv@I-!v`fa1v5LWi@udtcOL zIV9CtG5r%)*ix2vz*BP;D{Ekn!ZZv%Siv$2>9@;9gjlD_1}jL2)fiyHmT?BYppzpC zckyiw_W{Pxp!{_HxZ0@{bmM$>Q) zKhGe6Bb_z_#g2v3-?(w(UNa8uhiOuzJ4v{WK3{u?wDUdtv?I2}&7Vj~uX@VczW>>b zZGWpYc$KpPT54)(F=;U>f*mtu??DZIK^M3Pu-AdN15A98x;KAHeF?R{2HMZYr5L<1 z@0eqoeCQnC-+=SNw#mqt#jgJw0JbA+Qw9R=q6eTiaqu!vn$-R*JUk?XU+E+mNxiXq zvE^o?0qQa@1Sw!P04p}t)_#TlBp}PJ;mI{FoZiyNu6;G+wDj85ShC+xAR*zB@9z4Q zfD+17MGp|r9C;CWc7sX%+g8ECwZQhqjT>b277n*!)0S)h@#QYSAaTr@^8)_Y z&CN|#mZ2o=tTaPwEZ_j@N;hxb%tb>|v}cZj5mQ$uRLlDSFNT~rckbNTQnJj%H+v-E zJVf}7@bCyK;(^A&Gvlb`IrSYppljfnu%rAz*aA+ua;Cp5ZMtK*Mm<_~FEcP`n=8b> z@}D~{YQ)skG}WS3GdmkIZ+AQMv;50vwuj2NW&$fFy_En#;4lg6r*|XDL=3E*eJ?jR zH!@a+YWtUs7vxFIuwu`bFQgG`)r)@v)1q$O!WjZsh!agh6JVhTiAjG2l58$^w?QcL zf67@uUFQE#5f8;jz#b?*KCQ0Su_#AKJx+y&*7($}eYCLP;P!H+3P$yCgjW_NZ;EUk z5mGhXucWX4Ml1g@Iejoc#w*@}9*D6A+mA9oJ=mE4Cs?Y-jva%1SHqb;mpyQ%-5661 z_60?y$D)(U)-u^gvM?+knbZzr6ohf$r-~TAI*QVfN3=@mzW^Nr;NqLNZ}Yx=+L*>7 zc^%*k9QKY1Y_yvLRDLj{_^NXp1ST)$hiHwB0&U9P7 zypuUM%|H$UW@tX^=f{u<9YgU7-PJ3nG^PTGh&0m!Sc?U?q`+8!ZAcJLoXFIh99jX# zv~CCMj2YX(+Qh8{kmsVPL&5(3qR9P;Upq?C5#QoDv{$WKMdtpXItNM8FnHuU&f=)Vt>Oy+AV(r1QX677}J{&bgNem%iE3~#j z&|ama;UPsU1%q)`PuF2|xB-o`jyY5t2?wxwK#`SGsB}hk-U97Jb~!y=GL3~wBzTMZCn3XJ+T6J)gn#{G@?<9` zCz7(v7uk=RW@$1C=bo$3H?s^#0T(y7&^afd#BsY00Z1wK4ogbQnZP? zEz zxEoVRve|a>B1PzCdvDYq2$L$j#e7hHX+T12WMSaq zl8v0{FkvDJhe_EMJ$tru9`OD{^S{L)$p-lE-BI;<^O8&YNC6S^E>y<0(621qzgK3h z@u6~!cn+n@^Vc5qaeM&8;^~_=`Y;1`;fFnsI-_=r(+gKg@#}!SdNahEg@xtN!Gi*~ zfoUEd!hjr|J^n@tD5pv8_e1!xg$R`e!Xgip>MzWAkyu-~xHU+Yn0{ zKhSVOj1bsm_3ru7=-naLM3v*T=MTJ2F~K9aU%FI{Lg}pm>JTRv*GhXXZ0YSgcj`D( z6mmy?rN(*V?0X;!?VK z6<%Xj*U)SCLv57A^pv&j{8iH*odoV|+?Ry}xQ+_FT(buo886;_iOAAIZ>**Xzl-qDOdVhh5nw<@9>)Zu0_ zd$&(ztSL3U?D1o>hTI&KLDpQFJBhkPiXQyR;6@g{HH!wJHL@OE>vLio%oL|B9{AiK z{Kx~%Ugd=GmL3)tE6+pW$9(=?ab*uF)wG8qN5Lk7F@K{sne$*-+8Ov50<4u*qy2h8 z3cjNQY8SPzO_zg%(-1g<*>GeCw&1#7kkJ`+G|)gH`DQ!O%N#x&%MS=vJ>#|ZCis6o;C8V$-E+Nw#=XjoovJ5C#du(WIqhC zv-FYDz@}^+{*HCn15m*9-eQSDVurSm6_UC?f>Sj<5}XdJuy{&Y7#fA}p1y%Z2zk}k zkMO3|jP31ETU(16L_{7FuYJ1{wIzV77<&#MvS-0f_c?f`nXIfZN*CwT*xALvt$G{abL@&N2)lTs@KgM-mgG65wgQC{9HADOg! zz?cR#;xHZrr%HVpwiDH8ipH+j&FaWl=iuOI+zkB)Ir`MoR_se?7Z9+Gqq}7lH=cx; zXDa_PM%!$uNCgKYjY+rAbM8t)vJ;^4B^W<7yQP<=T9HbyMj#p>_j&=rBlKvJ%3v_- zB1V+faRC@Nm_<7((2FZ(fJ)FsP*hX0MP5Gr%M2=n;FDD7BTT7)Mg+S}GMtC=RrEHl zlxxb0ig7b3`cXzCH+K+&jF3fd+O*03)0;S4X7<(lH_hNCw{G2vBeQ=I;}PFsw|q`- zVY43mis=yhJ~DB#b8@b@*M~kmvAEU~#N5`i>n$D8knI8GxO!q#Wl*Tj#x*$Ch>_|YD}D)e6OIOfKI2@ufGph1ivO{ z#&~F|S{JAPrWx!kR$_t&O|f+X3qf^5D}mmI5Qxa6DpDun8Gb&-?MFmLMi%@@n0+x4 z)R9`iqCi@)D2S9IzA(O&cY{r<3V2O7%?PL5ncfDvkPyf*wEcTLK+{VXdN zzBGUJ#D_?gc1Hz>x354L!rtG8_^99&_ksm+_>O~Qy~ggKS{RqVt{-g`oxrEc$N)nN zK3}ikm5_uTSY_Be@o@W7e4QE`aM66jg4e-QaDz2++ZR{W-e4x@Vg%d){FelYW+gX} zucm4U8KLoAx&%w&ZS>cCgSj4Hv4^5MH8eQr!Fy(9xQpgTCVF3E9XUBvFBVXIN7u_} zZy1#t5+p+SDT(MCG5D36m=j_EllXzCZ>YqSp&_!Jr|egRCAxAm4e|U@So}%kB&T1* z6iI_PO+M>N>;vrDvd2%JKw}bD(`aI1f)f(4n!IQFkvQOO%@-_DkTGXYj{NZ91DQ>R z1Ho=cHUk4*rmD*U_paN4fl|F#g-RSRx%XtvaQvjDbHMbza+BmP#8`(g5}?HMqRHjMG}353`N<9>F;VULqS3R8>)dUIX*VNCpwKimaQ}%;TMvTPsq@ z@*Jf&*BKW~O?l$Yurb>iU+r}dWO3_k9kk!=>v#t1>4lK-V^>$#r%z!QE=0gM93T`8 zN-GvWMo@I*%01J^?0EH+C@U07G{o%WA!ZLG9Vl|;fl@eNJQ|t6fc4ZsUkrrVL1SdkhMGM`);gdN=6GBW7k zjoGBcgFq9E@6MVw?Qv--QYm^b@eougV9Tz82L{oq!H^eXIlD*{w)f;jJ{H6-LJn4B2?F~&V)pb>JaW z^GcP9R19Juw{e!?(5TWP(BSZC5EazaeDRnQzo32*RnmS8L66;?zb+NIQAH{ce3*4P z#ihjc954gk(=75CGc&-Kt|}}n^gN%3J43ohHUSbyY=X-}NHz=55T6nzajI6q}2(velY4^21LD~2Q z-r`29Mmey42y_oS#sNMn&d;9!hO4Tpt8w0u#FA(%f)OUr#jF4*9=QTBsUIHT$5k)-fK?9_|Sd=?n3XNRMEzSv|bG%p-Veg3ZAC7S79lmDA0f3QD&$f~h%O8gYG>eh;J{9X=fN?`v^#>m zo~}ezioTCfT>ys0?arSJX_>^)I<1P-iMVOv6QH1D^h}hQ-r>WC8R6PL9W?zM**&qMiC>QT!oYvb{VBvh=qw=rZe0S zC9!qK4%CPfZ<@jQk?f4}WxRu$R*W>mtHg9b8*T z8cTO~P1=!5xH~04^Eb^1;|W2LJHzTE9s@`N{VF$Jd>y5~6^O;U9p0Xvi0Qt6!Xh9j z=y*@FV%qd{AF!d}(b?GpCc-v0!rh0D%7l30@D)xDj%e&&Uxs=is1-$4^J!Mn&|+7L zuMrUuS+$B}0kmo4f%#vdmas`$I^6k_6c!{TIay>!1(-e&_YWI)W!2dr<78!J$h-FSv#Mp)-%g^UH! zk@exP{ndupv%$+(tnkX4{4-jjw=fQgq9X?QtlP0j{O|=V#&(?R!M%76ce17od1VAx z*=drA37XNRpt2*bHYTMs6z@p%gfJfjdK^d2jciBdK8H# zsL9(yB{eEi$%qqFoyfV)Phq&iJWSs!E-Nd0^av@>V#W`eCVoI5{VO-mb4w#INK_t? z0yS9-^?GbX+I+H}gJr-YxiMzp#*O7yI~ z_oFxOLyWwwsOUpriW_h2CYQ0YgHwMgejA?z*u7gkCFf43Xr#PECAL9- zIGnO)`*t76NPPcGAW1w3u(0%Ti!{%BHm&S)ZW$0EVi!^FlA8>#}O)&2;CGGIYO za|?iwjO?X%rRxF(=uB8Q|`B8c1MzTfHYRMV|F4e z%k%WJ7}UFX_#l$L^{(EFWn9w7s(a8^dxRr)wZRA`lfU~M0K!{kgLDRMhB~UMlw1mF zND#9NS#ovE9?YzG9%DG1YJu==`1nW66}n(6X&n^%>?2ayn@`8oQ@{>04Y8=6TRa}g z?^`-Oe0$;|2qDIf?9N7wY}nEe2>7N)^KzyZ?Qu4O4E36fpYWeGl2so$thfopd;h)s zbhnxvXl!8KpOmn~I{(&~tAVI_C4Ya?u$PSMaO}5JE<}}V%@dihFY8`r*5*Qzxwn0e zgjE~<+;!cZ`=wh9#s{7n$-(Tv9Sxhb6ZKcEzky@!ny_u#wq5JB^QXa5(5s->->#+& zcf0(M$Z+TiP?6M*qlfsK7#VP`F7+)nS>6QU`P1wBPh&F6X1AGVF&3b!Vn-ZEio-W{ zt%rOi&v#Sr4k+ozgWrD{ihY54>6eVSrx#~M7Z(?^{PD*hEG%<5ser*K>;ZSZvx^Iu z?q)h~a2Z6%u*<=8(fY)__toCEtuYvZYccD1mRV?&3Av~x@!2Na$QmCsF*#oX=QYq_L`ab6ZRpKJ4Uy)Rw5 zB-3#O1sTJ@2bdXzNg!V?LnA)7e`_nGJYR;|kVW-Yf4SSp(Z4fnQ{31jDkA82T!(&# zm#wbJIDaFY?^n zddwBTH__R8Qdd(psuf!F<~C`tDTZ0;sB}_vSw=G#@MJi9( z{RZIbbvoafOLXQkhcba0x0yyF8}03IMU#;)o;t{2WR`xz#FXfrl|DRCC5TuIcTrZG zAQ_syQLZNmAno%PQOa#u-J~f;-bObx(;m(m*!Vd+D(@=vS|#@#N4MOPN1)QSeF;Cm z$~At;>qIUUu2}x6z;6KE3+8vM?NxY+X&3U@3bG^JryXCQ;(t9i!b%>tNM8~0o8wn@ zmZn8XZ(eMvFJn*>XIdiQ|2jW0>1OVh4I-loo**N2Zn0L#sEW= zEqDg+`el=RHafRV7R+=R>V;;d2dMy5I*289Uz2gIL!AEht@P5Rn?Ew8H=_<^^Ryet zSE*+KGVZoY`Et=#(?^b<9;1bfV8`;|m?NMERBrWQ4y=A1fCgo=W^u{BK%a)#mApVx zW+%~W%tSnGw&UhQuR2ypgM#N6118^j$Lh-{PdKuW>D+}`8UT)4cQ7p;+T{KG`STO! zb|wG?-=)Te`Kt@~!Y=HZK3Zykk7ydQHkt#q=e?JS(+-fD+%w9hdq(yFEmJ;<{FD?5 z0v9+vywwe6#N%4uZ0QRpd53WJOQ&hVhto!6BU_8jO;`8U1YiB5Y?oo1Vgcx%vuq=e zknE|^1LaEoh7lt2D%26O0P%GW>Rk1kptRzxR~Z={t#SxCvRU?WGvd{(VyoS^;nR-A zH(gxNOY;IIt(}_`bK^#=>{|7wcx_89*C&H$kj9c8Uswli9kD&;x%*u50PI?S)I>s5Ht}hR?LxkB$$sUppa_tD4gIuyvK% zL=AX#Cwkl%bAH7cEZMpbF?dKLu!U{qUlH*xX+OE;+zkI+o1H3XkRKc<(lr7<=^XU< zC0pO4lHIyBmoTGrKcWzczD8=JwWKmZ7rPh`u&hA}j(h-=s*;^%2z7FZ^dmH?%-Q>% zS%p%z^d?vs<8?*8iv{bKVy5Go`e&qOs7*9&?nBg6{d__*ALEB_J3caKP1YNz5t!mV zF)3)u0pwuHp{Jp9))qladMw*!&oIyuk3DVa)-44y?gB(MzE>N+TTrxbo@H8RQ0k_) z=qF;F=nU`YS%vWmFA+VZ0H{d`fRA-(8^3@&NZs7b%&fQWEu77HFRz6g?JXc^DbFtZ zc6xd5+-1GusBR5Q+qc=MH$Q2|xRjSB-e^Z*7tO?V(ck|C*tI=e;?4trG}4oiN%lCr z_=&>UUq~qdrFP}P7z_a$LX-aj7dzNZ<4?nwjN$2YtER6~aD3Z&#}g;K4rPoEyv7QA zBU4x4O3N>g1Fk@Y>MPz+r|RCRSlq}12M>b4t8TDx%ZE#djOvks;f3g<1k%Z!x*AT< zBgYEYIIwVytUxcO{_U!tgBdCGxv&3n!}c0;;H` zpA1v6u=s!|XJvanR4u6^a$hl)WWDGX4E5BGd3s>;_B`aVZxC-5_kMoz6(t2S!VqPV z9YCO}U)CZaP88zhHNh?eBtihMmD@v}=edoek~o=#C1Iod#*!ljq#9VS)Cx)r5vZVb z&m*bEs!#oQgk^}&=O1- zCKVHiC;H(2H?Cfda{j^H@mdcZN!@w1rdt{XsDLZ@!iwk~8O$ephO8b;3;)D$qDe6wAIzziH}5WW$bmdO`^CJ4uiC_6YH zATn2!I_jaITwsYvc3ZDq3GgKVA)C0DOGwxQ{OG-xWqiJ|qvKF-^9dkaGw^5T%=11z zs4d$ew0I)yh(?)KvvHzG-h_L?*PcFg3K??MoRaB6LP8)xVfLWUtphHYRS)C`=kOU4 zj)qx~;d^^07~~A&2>|_2^sAsiUPb4Xm*)V9VS>kXl$K$j=B_%Esh6yV_#Vz`c;MqA z4h{|;9=FeagAwISH3^E zjDCKy&@&H96rLAj*}oG=9^cD5$SS1)fgy|0F7|Wr@Q+2TjLZ^DoO*2fA&7Tub7oL- zp6k$sO;l>2t9mECQnv#+HnK6WuZK`mf!hGdKs|P?A;jKOX3S4M{2~%9bz+70(fDnt z%q#@#^70RX)@edwV|S*c49Be+W(QFM)@-AwXb>PhYeiZ?6T% zs_}82#*ZJ?>N@rY{=mHAg=K8^juXWTcFuaiQA#LTvJnn zoIop9ee>cO#~cuJVMMpgrcL@5!c(~V&r(5~;E`?Y?R`Bw)ECj&EujfB?gY2?Hb4q~ zZQKV?9|Mdu!5FHA?Ci}=P1l7?j_F`tP)h8cdlt1QBUM!kEH91 zQmD&7NC6FF!-+4k!!8uxYc$3@NzD15&8VqrzorN5xbYM^3A%?b3}+m;C0Q2Wb0F_V zQ4i|#Y9_FWMiM*UwD8`j+ZFp&^`?l;;VF@Y2eQBZlok6B-X6i;ktr) zQwiyns4vbs9J7b^WeaDaSiL48ei!Z|B1lTLE%f{=Tot_v zaMuZYndO9lIifXVD5}74|CJIE1+T#_hzreI7LrGor5c55fePTRgHFHMrqND)(E23i zejNw`FeMJrojJ^MO!x<%k`FMqG^YIXi*Uram@xQn9zdb}M}QK8e?r`EywI6rkvQml zWn%95uG=B^8e3ZCE?fwf*h_SW>mQ<2?}0<2@55eSUynkI`66t60{bGY=m)q0h$o*D zMLlB^6CB4j3 zA7R7XY^ZX9VWAjPf4DTz-CY@p&U{wHKPk>%j|nZA6(omZ=0Y*ksWJCZdT~YCyc(Ml zhitX^Cv1DW1-4n!IRr>}Y~hBNyvXUc-z{bGGMS7*!@{`d&h10L&^8CTOOg5+HYkud z0>;+T($d!tov**z5CvZLHu&51mljW^J2%2cKr0GRo_oXb!@ye@;^^xEmmm<$fgTm+ z52C9ym$XT*lpV;9ITdW=0=yFN`ONtu`Ols`s~Ph}JA_G-Cc$3GKv#e>Oa>>zlyz+M(T(YAz9y zU!fOAXB2hJ%g1Nlu6l^DNCo4TEy6G0JDx!%U{!($0e1p)J3ccteJ;4h;99_htU*vD zExpgxFMFPmt0{cS;RoC2Z^KT)_d~o1+SkmfQ~CGayVY~*tR}pksAK=9x6Z(5P&e`R z_C~nU{BD-q7B?i;xCh0WV|A%pk1|`e6amgQ6&k}Cj)%o(B|Qzke%`5~G6+sZq49Er z2v}KoNBGE2^9R#Waz~Ajgx7GQ@Y5A9Usis-d+Kuk99S9FgAXNlg0TX{HW|GQ&C<-7 zGiS^QD!aZ6pg$@Ntkb5wZ^#=*lHO5%dG4Y`SvY3+YhAS^e+B7#S= zlzsMWK-_YkK>-1G5WJx~!(>*@5tW=82HVAw`KW%z;7Rh8)lZt^d4t zLHV$U7)!+DwMG^>p!4C_3ZpqF))}{r;NZ)^fJA-u3?80vB(mU7-*FosgB}r@PWU<} zW8-GTG@xL{3^3EpNSy=~XItAAu}5-YPzB)o1rNqTBM!6P=z#`girPfr^6uR`$VW=$ zm2g-To{odUn(*G)A6oWk1X;}o8gSZS0Y(6~WP^lP=Vxqu(3P=)FN;X5LjMDZ_PoxY zA4Nc}n!LrsDinewkTW0qK2Nl6K&l3_5g0r#wj>$?Ax&w#xi z|K>Kn^N(r*1KeXOAvwt$UecZR;K7@9Q>cMcnqEaIH_6KnmEpq zumm%rkSwBWgsp?a`b<3^xLMw9OMA0JFlhY$D(=d|sovW5R_CN7GGr)2nG0b{DB+OE zTqHw9LQyFsDWM`WQDjq+l!}TZM5QuQor+S3*v3TWIp4iH?_194bl&&7zCXVGS6$b> z?Dbo}wVw4n_jBL(60}vTL>`IE2>aSrKV{(z9`vT9*%XipMJvWIo~i|RNcyoIP;*4V zb`g1b6T4hQR-xvt5Srz$C`L~~OSKMQGE83snMapjZa|7RFg&~umpuV{DT+ryJpwzW z)sU#T?$~T4m#K<~$KjcW>YiMXhWDyAN>gGZ9T z713KeoDaJ$7mfWFml5K3+Sy$_|2p*BjxtvySID9?DWu#39(MKWRU%DtI)Eh+{PB*u zTsV+OXB+}poW zf#<;plbA1TxEmNA&PAgmhHIHYrmGXV9nZJHE?(%(0@}{rgt4u%AXM~*WA6hTY|v7~ zt5^Z4Aoz2^h6vQ{fXX{d(j^^UYoAf3NXO3Ek#{;S?l5Sga$iVBgETHCI-%H(&^E0qNcVg%!hT4|5nTU^+rCA!7<@0y=Sb4O3jV;dK1sGgYCWuCv)2wO9PBcug+d6Y zOLqucq!@GUUA?MiAynATN%A1XvmgEVD&;xhv|hjl%~fi9$wL@vBU{jW@7%fE^ZPCp zwE&VAI|oOTxI+%O6>@9Vf@8oBiK@C>c2HR}F6v|FSda0A)u%2}?s=WM+2~7sBAP0$ zwh6Fj99azKg_u`&^MAXwSX^xhddKzr`ww8w`UVE}K&>~@lDd~7g#(@RhH$10EHiDaaZa&l!z1eORar&e!uD)iXP1i#-3XIcnyo7`*DFF7=q+hv)RO>~IGze5i zwqQN6U2uW1(Ek2%%hQG3ySQS#`K$-)fP21NrH6QQ7rxm}!Ua&+Wq~>!W zO$gNEvydRFsnM8l@41E;Va)Xg%z7vZKZ zquPu#T12&vuuuea3=O$f`{gCni+*26b!)_p5;!y8;6~!b4!03^7e+@0o|E5I=m#$E z^Pn;r2>d@u%P_hGKn6@`9fq!L}V1#V7}e$g?gL6D!uKLN)6k zvcE*PU@%+6pxcIZmouPs#L(noroGUb^Yp*L2^V_Jz&y(-TJGc1Sh**$^Gd=N?W~L< zog2AhC+qZ=h>Wi?PMQr#NnuGEIz1CzfIiNQuk9A~+#Z5nYU%l0-qxM{yo(T-)%WveHW}+C%RX$ zfm$5B6YfI92}>kqC)Ei1jLOZM*1Yu3Z>)VR!rLB$4J=r}U_Id{l`^wg-B1&-U5j5I6!#0=p5kP2)Ynw# z{&Zg}%YI8*y;9!m5#bWzo4Y*PooPF$PEC@hJlNR^nXh{LNXQB4e>-!1HPQ;~U2V3! z)f3ntM8qt3E$Dkz$|B2k7TF0(`k5ddf2IYYf+A^fu5ppfc)H!QE*S5@N?1CkkbB8P zHa!3|xzW}#+v?IR{r>*gdM>(8K$Mi+SB>cGhodUuu)Ec$DOf6-RfUD|FJ|2C=K$ z1M28rymkxkH%*L*U&LSn{rwHi^oTH*U6k7aYJ+RrC2%~VgPws+8k(GmgmB%xjQRk;zSydnb%EXGFY5cZ}-s;b&hl{ zSfhEjA%iqsG~m`YpiDP!*e3Mdl77XXl^m%*4!W8W6!} zTD(fF6^8$m+6 z(D5}=t9|zEA>1LqW=6)8#s8v5?5$rqd%E$RKqG-oSN&Rk`0b31aC+^Pmpl&qmiJ*z z-Xo=#wcYtc?{;?vEM*}YYHJwJS%Hu4m5j{0GuxIwL)X-2o*A;Rb7LH9%UU}-okr)* zVP;<8?MUovCgxQvD%0!RCJ>c>DF9|Rsyl{=I6-`KLB`rxb3qifcWlH$8R#WCjQ*r) zN3%^T3R2mKHdCZE6E@_XN>%9eU3bspWPb^LW5u0K5~v8-9#bgwzz`&x@49Zg=rWVV z5$dp*(~nV|NvDLOF10k2D9_4Rs!k~qdR+MLT$@>4vwVF=`GP@zG+Qf@EzH5#wKrRX z-mPbgSBv+G$HZu!c0PdaBdhFVHEVywrd!*#@&`Fp?E)A;b((|VWHu`sO^O$;JZDrV zK5N?F*^*)FPfco7eftT}OJ@#q!OiJUrId{0Erfw!a84&2b9<07Z0WT`fQGm!a#z!D*farCH)AzK`#(=BWq_w z!77|aozGb)HX74nvG0lXMn-2n)X8^{dObMoA#!kcr?70A+IDvkFjhD2+PP{?GS>T-mVaZIUX}Vi^ExTO!boj{TOt^7!t6 zw96Nm2+{?J-_K&oP&cwO?hL}oMq!FV*VPda z;B4AblPlntQ?$2#pG})LkPp29oDon90Nvxri#VcMu`vcNwL zF6jBnds-}mF>x$`4;h&Elt%UGBYp!+XKfWkURTh)!w8fiK|s{pvSkZ!n(F{o_yVU* z+WGR*QZ%$q29A9dmn`zlM?$~OPapLHfxdQpuW@Fko1pK`y`%wO$Gb#Q=DJ=_uODUs z8!~slSq?4ShZ$uN0s+6?q&f9`D( zMQeg6iFUHz>MKxHH84BcM`xoDp7tYuT_ar;P_>a88IW)RL7(=`u2O7ZT)&3?;}LKT zvgWC)o@C))h$9!IIbb7vXQqwkeraVT;L3}$`-!vxUnMnIQgKBEnB#in#r!sH_11(C z*-+sjgj5L2(ohs@lk%uBA3zzFUQG}R@j9JBY1B^_Q9lD0Ak6|CdJ0AvNFRkNQgo2) z0-hi2@8^)|?(D>s_bo{J?jR}y-r08Bu8fe{#M{fiK@H62O965l)EGipqmy1U2PdaU(z?QDv2r`A5?`J)H@!L&p2q1{9N5^|UD0UKSfjPn&CcGQ^k3`S4|{5R!qwkRteV&Mq8wMuL6Ml-Y|=M?3xjxTH>a8z?7ge`eJdB}CO6+L`ob8j-e2${ z{|IHp2KA>&IpM7}&>R+3oLMMmTiEd`r8=xD`1`s^XYa__n=x(Bbc;-2C?g=<5M$RX z8%N;)WdvL{`~Bj!m6D`RX_=mqh zPxBbnIXu!VbjYrFnR&|gnvO~JU@)5F2i97s_8Cl zwbyE*9yHAw9K=vXRvAbW>(Zm)H~(SggdWEEBVoWYC;b_09PFkBleFBG0l(!x1LiGe zc#emUuM5KNUgxRit}ZXQc@8RDQonCvGa>$L=^-`aU*Z_UTkB^g1_@P31%EzM2WmDP zhU*}hY`*LV4C`90W7)K|9(17YZq1ZB+b#j6I1g4|v0&p&iN_{=ZlB&M;DBwe4vo*< z_3S@M8Sy|Z(AAo%O#d>wFd3X!a@T6GwB zH40#qhUs4;aV8XF}{eBPXU2M=VB8Q33NjOBYJQ-jeu7xZD-*_z*T#5?Gsqv$uN#9txW!7-v0J) z;LO?0?0i|mE%EaoyXeQ^>o$IAfE)7&7~9VP(c2RTwnGV`zAA!utXYO!_UQZ(25cVw zWrw8x(me%~Ffd{ZSfE);WU78WCH{1#c7Hd6@8n;W{tfn=DQ+qA#j37A|Ef>s8@H+_ z{B$@~Um}8osl_Yw&>UCnyS9rynwXA?31~REp$cRM0OSNvtwp(P@K#EV;BqqXg)2ve zaMIs@O<$>SlRKi0bauuUX%N8&K!{ZtYrllQ0gp##pfO%v6G+XLsMU}5w}nafPbj^C zRZ-6V?dZvX@s^sp8{f%u|2PEhO|9#qvi2dAW2;~!_%<7xCxjS%R^Q-{`y;FtcuzYJ zB_qRY&4S)d6>V(AY{CL4d5m2d0#?@#kADr>*{1h;E>2!cz~9Y8vw!A>|KSiI=OjO_ zjb5g3cwpCP#PJF~Xr`TBBo*y_p5Fm8Tk6WwcG+4D=%+-76$yjoVRF~Q=8rJ)<1!W9 zKji}|*5$rcj-noO7C&b!YO+>o>hAu6IS>&S2k;3$7Pxf|fWgTgF$`d*>FsFL;0$s+ z;~+8A)63i=U1fCu23IQWP}G3X3=zT@9(y&|u;6cdO>Xp9=H_?uWUw~Us=|0khMUk5 z1L*FC%#ZH2!`3!bdSBi9y)OoE{apxmL|E8SP}qx3pzDxTSCirRHIn_fPofJZ_g$%U z>8x7nIZ=cOkygx$benk=3tvfDnP1I&i(|RL>$w=m*(1#w_lsm6P6!3GlN!m{6DTG8 zP0d#>F#hVGbV#AM6c}sRUO%Mjp3m#i)Ur@ro=1q9U~IqZ>m@xxmE~hGZORHP>wmts zedO)Ku|7hkR>C9Pb&)3eWO^;vmFVkV&;1H6UOcr=4nS(JHTEa2a;sI$m)rS_Px&yn zaVxwySqpuRdU>Hk>$FE0k8Yqki56AyiQBc<0GTzPePA-*QVc25rZ~RnpDm<0byE)I zd^oJxrogaRcK(TSfC%w3k z4yVVUDO3$FYYI-$XdRuM%Pa}|R7E21K3D}-we=$RmO&6upSn^>;ve?y(=*D4hq@d? zgGXtr+^@Gt(#vw}gWegJW)rk_q$hp*6K9{WDPZ-nbveG&0I$P9_Cn`L!BZmSHMSgY z1n2L2En11+`@6=(pKkibFN!9urvlUlI;$&OZYI) zbgOoc9%87!QVBQlJjTBbv3R`sSM+H@+C?}x_toDU#3_m{>lz%M(3U}2u1**To7(mcR0c&{qZ~Qez*$-66uR9R7oiIQb-bZg0u1zRZK;~>2l|n-_#;SGH1z8fBOjFqp zCvV_Rr<*r#f=|`)F6$#+lT>yjQ~OV!wCIb6DHwQc4QbKhFA~##w)$$OSUM0|z^LYK z=mJ-^dL<|0Pj-YfFgel_0mY#o;ypezMG^imI%^z)L?vQb5iKb$V0yu~>0jy057S@t zCEJ5Egyf8^TwPp7fm&AG*r5LpV{3#z?i@$QnO0ByaPV=B3rfuR>9l>zeSbW(zNh#qb_doSc!zfOj=17 z_9!Q=XkfTF1?2D2C9irL8ygk3&$WG?Id$xD)G^^J z%2~g$-adlH=3*l>ibBG|=7EA95{DdOYDg>ex`sa!d>z zCCJq+b5jiCi0gF1ixK~lzBXJ)IDbBLlf!81OIttiLWD*hzR7x_XaNv*A)D1uw6HQ5 ziwJpD1-=a8*;>uOP2saVe0Ypd63|?J04r_q0nu>033=JQ*=5>zeS$~2GO(6Vn!tR> zq*ITZ5%YK-U8XJY_9?vy*Gy};AiGcGaLBR$I%^h8CwIV}5GV z>>>&Z?%3tv)y83tfbD^R3(02nvFkeMdcg?fcd#D5Ope2Y=s7eku-62I0!(Bj@H@fC z#iwZ5nl+QyvLIOCa&d6pZ%Ww_!8Uz@RF`oNxYwh70_0I_+YlN#UqE!iIxHd11e3p0 z>{$|%A&469NVx#snHmNPPc|?lF5+YqW1Td{c7~HybC7t#Dfd&L@JmyvuDwk~KpD@L zoB&bA;u9B!;LC>!gRqIpCO}v8A+J7RIY-W7iWPC-R*Mcdstq7cwDezolflgq4aM@>3 ztf9g$7z&Hn|AjnaBlo_h+6+UHxPvv^Q=g{@g-Ym2^rj#J-h~|nJ<)nOChkvEEq|f+ zmw{Tqo&qgNPL2ZPDup={kd%|ggFY(^<+x3Xw%$`*fI;ooCdvTb4#2Hi2cd^?jg$}h z%D5NufNVFUbO+{B#?=yNg2F3lGxYh1W7@tqV8B5(4@Wtq&ZY-Ckf9S#baD`mmHV1WY6Z7jl^A>(&B&$a6^fv-m|Ve8uN@2R?Wy1f|Ka3P=kj$*E0 zi}w`D9L4o(e=}v4ds|psys#w5m^&# zl%zgyS~HkugDuy@5AO6A^wL9|floO?FM{S2=Xn(@9l!$B5gTm53*CR~@|7z@7!djp z@Lv(oRw;)-T>LT6U2mn&iX?6r_nMSdS8vi^aJyA{0L?gk9wJ6wa}JD)*ynWK6|SxG zO-L17{T#e&N*HL{W_B(B;;GlH&NYL$EN{3Va{aZ&j~}hN-sGnzx0m^f;ajBQ*nSbCKMzJ~R!XU4J5XRLyzBWyimD z7Z?uh@o86vpXYwM+1`&(FO%{d#eCp3(`~MsaAjlE7!9*#;}7w2xt|k~Y@vr>cbx9P z{nz(JZ{Grkx!f)dU3VeH-Ed*jukIe=6*2>fjN`mx)z!#$E0lZScZrOtpeb9Ve71N5 z(uJZnU3Bi$-1R>nQX7~;99QO+Cq9SBpE?LghSrR zd}-c)N8i9KzB?`MI~&V0TP0aTfG$aH_@MGT!7M`m6fv0VyR0qXxz-^04$z~?!Lv*1SOMSP%zv!YeHk!feJZ zoI(wXBJe)PoMs-TODaoxo{A5EmEJ0%f#b+7DXHucr&-idXH@+ovvEeU3p(NGX_6k zre}dcpMgDW+^Msg-qIr4=8p1N_%|f;sg;&=m*|&JOW3&z6J8Ltd3|N>!EF5##B0ty zUjm1E@t(af|G`+NVP=jva<(apibOXrfjYiiE-VPNbp0}OkiWAT2pkH%lRmYY#F=GzI_fUpiGJrt=9{0 z!c|BRd-2?xmQ7h2&IvaxHRE^RerHA~N9UgBIED{^3ByZEl7&C=l-?3I*!=xKx;R6h zL+JvsV$xnYVJ=adN-<7g1)3W&79Trz60GcsIW4$dT%k1%YR{LkC!OkQkAyf?Qxmmt88R?x<)bZ85pFihB*R-aqIBPi}i;m>dYYNGUR%%#*=x#MSXh7{Z zfg9&EDuwTnZisA50#a&?F3FaL1WMAS<)z4#cIETmO%1W1u&cC>I=XeAo*-Ns<%T{G z66Ej=7xq*+PXijYXr!E;oeOW^(XAnZz943FFTfQ0dIf=65D)I|QV&&7z)flG`mwT) zbXE}nEIA&@6m>mv__#RD5AUzz3gF00%cwV-D{kNjEB3g~L$xct5nb{rw0368PGmaS zT#BA=;dRlQTEW_8_~a%!K8BM9ZAB8B+B&SK#?CI>nk{b_Q~5N<(9RiQXj94Yc<4>4 zKL|5MyTtc{LHmH^lE`*X=4SM@LiaU{_>&v1)lvY{YJKS}j4+wJKjR=sxRl0i06>Ey zO9WL8{jQUV9O00prggr#RBYHC{r4i~POq+nMINk(Z{{I#oGw;TtL{sC(m46}_Doa; zClU?4%#gEe_`u!OMC~-Ty*nJy<7z{&d*QMTlM3eW9CqzAnlBm;GGU912o&xkm$LvY z^>It?6Ok!9;;GXYF7OhN1iW!UpgzBap^j{@#`Mqw#-wW$xm3*ZT&b{@mX?K7p%C=* zjsW^H;Z!$4kC8-MeG%iL`_4UK}8~J - - - - - - - - - - - - - - - - - - - - - - - POSIX Adapter -<<user interface>> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vBucket -<<user interface>> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - MPI-IO Adapter -<<user interface>> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Hermes VFD -<<infrastructure>> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Hermes VOL -<<infrastructure>> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc/design/img/modules.png b/doc/design/img/modules.png deleted file mode 100644 index 3a47248cd4257b81724bf80a8401b0aaf055fe27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26821 zcmeFZ2UL{XvZ&i+Q4}KxNKiyFC`yh7Pz1?}WR)OMkem@waS2KW$qJGMBxev6XlQb7 zvXZl8n$)*`3~Q}-_I-PtclN#ShP}pE!$F6?XQ)|Kv+Aq*LrLN4{=Kw&Q7F`Y*=tu+ zP$;6a@IUPyLU^T9=fVjT%Gg)-%0)Ft-I;!;(jy()_=OKTR0PMGPrZ2bhOlQNJzY)P z*|YdqSZCfNO(kOKs+TWrX;)rlztzJrs!X7zwZgf2Qgf;+-PF+T?Rw6s7cVrm4@xTw zT$Ey}@Dnx^aR`h$9TZvbI#wFv@qMTZdI6#?~~&1(hM~l_aSEX zKQ`2ZBUEb03mxsjovPepKvHO5j#9HpH!QL*zjl^TgCHZC7uBI{rUCyR_LaKz=-*%e z_tJ3+`|oEEyM4*_pXolwcyE$oEZrlaaB&y=4?+Z*7YVjl;6ul!eqL2 z4JF(X=P9(OB~I>u!R2QZ0WMR0MK+9KXpO-=U%Z^E%d(bDz4sZnNabECmv z@cbzMo_(QY2h*GRuQ3q?za_m}xrmB8R!mB6@QAOSiR(54e7cLtlS1hk?Gq9V(eqmq zw5RXf`b+;Kv+?A7ri=98i^JxIkx$)x$#%wLL>?ehLlk4FivaN6!aj7AwIU%dT6 z?2+-)jEjUcZz%r#WB(JEv3{)nd{c0c_-ePlm#DqaO1rAzM?d%F_@M(eTgyq3AvJHW z`Y}|}+ilM|R0Q3<5h6GMD-C%{hL4{@LDw%=GlK(NL|<_Zq{D zW9;j0viXRJ<**l~Cfg5(-;X(`Z1VNf(@HmqGUt`RbN8bjKEv_ZGVL>NPY%e9bWK0$ z6qR~0xGrV%q7RQsA(cg+(|B~cX5QjPx?YL>tWjgsB%}G!t@6`ah&*DCYTr~V$#UCV zRfra;y?oAk1U*zvKj1jLU-h$}#6~h!<@?taVwt*r*)7d%gIdO&@7|veh}QR+7d+^f z*j{ccu3-sM<$tasoF}$-^`K_%p$uuCKaQ|Qyxg3xz>BZ-<=m@!(GV$Ef#2CwO_ax) zWtORgPk-H2gw5Jt&!7?NHYvSu@nOY2fF317m|_k+D&(sxva%pFlilYh$Q-aKxBfc=;>rz60c72a+FQuWP3`G z?|e^QnnzcpWw}$Q#(leq=8N{V`(K?9Ud(Bdkh_0{c|{cP!C9p^yxW6I2= zAvC{P=j(E-9o#tC8q2F+yJ-LHMZPh9V6ntDU@1k@XO*z;3Hgz!SLwswe0W38-H5oL zQSczTAnUJltkalP@eI>-PSL0%@MdtH=*ctN_PQai=lJz0?ETs^yUEt7Ub_})YMcX& zxXbDmJn3BqVK1G4`TCF-<%XrCPMxcOMUO4|q&G2{Aq9ijdPTwOnqYQSm7;;S%+Dxj zPL-5z&suH@Qn@;cSNR5)rPMPY=?f52=wP}&+P=uvO~cU>S5{_dKlyE3wSh|9qUVAQ z##8SqX?+AghGKJb5I<+Z^;cQ5zO~sPILJi@o0c@4QSzWRkOdjc_s^8bDFm@q&Bs`5 zkVQHx80rQ|*o?{JwMfcz#GRM@eK?9x8|%Y<9^2D;k$O`Y8C+Aca(ulGmv&+EmPiE% zOK-l#QvYT0rJ~_iq6R+N)wLd>ym~y;_xY@Mws2v3?nU_VZX9|flHcr10h8hUAkUid zz`Ld1!JSSJoX^wjMk5m~@eLzhqK9izXs@J1X)=(AUY6F%^{^DyDPG5^%;(n2 zTO94MA)~gFfn%^C7Vbu8uq}dT1%;Jlu175AA8pGz!nHfzOKO|#DXNB;N;1Zol5qS~ ze@Veih#aG(Ps;e$&)%Cd9ay`Lo9ZO_iZNosA-JLf?M`BQS-oo=ZVg7VOEi-1rLNda zy$V;$&+HRfH4NwdYDMQuhb_d$Uh5lR{$P!npj+q1e5O0)zx|E6;%tpk#nh?0Kb7O% z$&>EKOj{;5CR$kZT!H%=OFjL)p?9CbWhZ>W%&H)-l~& z;}1-Xu`HPtu#~&grQ^_ltb6Ap5hWVYoh11_Lzz4MIC!=Ro}W{ zn9do8WtgF(;7#9jt%5liDf{4FjjVf5ag*D=H0P&Yac_NhenI0j8ls-aC%H;FckM6g z=9tBwgDh^>)7TjktXL4B`l|xRc1Ya$h{_S+-je(xup;KRVQVQ@sgu}_n4FKR%db?{ z7*(cRskk-UD?B_@V`bLn`dJmC3Joz-mcG&CG6N<44~j;qaM@tUu#$xw>&ReCcoq+c zeg8Q0>dV7aQe}>$(5sb=N-b2;jTHs!JUUaZshx3U)GHO+tGzq31f|K6zpV)j#=QG1 zmd|QPYLG3`b;TTe(={bYF4pfPwYZ<;&iW0({`LaP!Dr!3+6_2`$tYd7m7A)uO%j{$ zU2zST?&m|C{8X&84O8a@Mb(Rhrg%~*Yxo`>7^#tNuM zN8D&wlgRXYwJ}~a6@IEkQF1$1Ht0-BeWdTm;KFbX*;Rw1pr1Vhdcii9*r(%$zkyvX z6c*hn6!Nz>cRD(A^2gT6O|@1vCDn0j%D+BaS)UHB6qw!ltBh{BPD@5b@vv0kl6j{K zteV8vh+lF1U2m6sdD;?xb#st;nALEYF$&q&>l`b#>`THs>%(3p4uvb7T5f_G-F_Zh z`ZjNZ&B_Io!GU=lfJ5sSwb1qipIF5Zt=w~lbw`(6YR$oM zYuM>MJ9&v-3oHnyqx|D1;Dm?YC^Qw+b)HNB_hYtT(4AW|*La{5C$Txx1`{qi=rsIN zaO?A8M~&j?Vb;n`!%E5E0Bs$#i%NDp?lh(2o!4DkNDSnMp zE0T7RfQd&wD4uM-Oli@Tp%ZlGhW()18nd66^`|G~I36!@gA}Fzg>EP_W|Hfblq2uX z+=w{Zr~Cr^p_jD$sW4seeXY#~*Q*5|Ge6q!N7G($lbu>*tB`ApQ>I6h@@n3qoX!cm zc4hY1?c_c^BpR`R41HYenr>oc@8@4s<1^qe9gE8 z_r*eDau2@O3Wa$n7fbz5$^I-ql$PVhk~+&`RGraFr4%!;1E@08@{ujcnY(8@Cnq1io` ztN~ddn2`jl;bv-)Y#7=dj%@+>8Larxbb@xDdAg=Cc z3B!qq1hjsRoOqeOldPV<-7x(FbZD2ZO9KUGb`Njq_fO!6ySOCGzb+e(_1N+7Tfn-l z_3MFAIxl11lPhb=F2tq}zJZZ^CRbjEP~VaPSR6~FDq@X&Z?I4TgXQWaEFYKf8FT~2l?hkjp#DsXu*T30y>98gZi}nb_2F|re;Na4t0k@Sp;*oE+ z$C&+^`W*(j8~Wv2CA*jUY?@vkKX-}eL<6^=Y5Rn?>=EBvVud!UV)ra9B4Q^ z*FB@6IDcDrwkL0MBuH_q7jB;JSY1`sVf;n2N>j_u_t=v8a=N}SZv!0MGSJG_B1RZ# zYMs-1aH-0k`*?z@!!TBMA&+&o+SR}5^*Q~iR}GbXvESy;SwwHug>W|TV~;3`9qP_= zT(}o*3g$E0Z_yl^E!|U-k`yx~E;v)V)HojPFaWYXn#v2iVVO(ES)a2SNswOGi=qS@%nf~ElXAQmLvL;~{T-l!Y&@1)| zmkQ(2DIGNH4zV#$;n4JI&WOXrM+s;(W=(1%JAsVcHXpX)CTxW|(~>)O@a<)9&>kFx zy{S<@k5)fO!p=(>IJ5>~coJfgv~Ad{eRJJkVG;}%jFli>0QT~(LM5|bJ`JSlnv1o5bz^+0HP@KAO=RiCHx3$#dHEhj3{!@h7k`&i0FUaTd>Xvr*s(k3N8F+P^-!wSf zMnPT2E=16}B~BU~EkOqh@Y3AAefb02r-lBQ^x~QG?B%PZ6vvtL4$zC_&VB6TT4LW} zXzQ$v#C5-<6XbEc9lA2zg~v5{NNh~Rmys2%%OxZ0DQ$MJ=ccVwaGMcESRgu0PRp5) zTSVF?BP~i_4jjngV`vK8s^1CWR+oT_H*UNRv$q`m1|&J{ z(*8@C99(`@7M*+C%5UtJqb;lap1F$wS! zTfBCsZy3W}p-*G0ZrrvzEqa3`%Tj`p$XNw5G3oy9MWMvzI zM3j5t?}q;|0b!V#vTrXqx9CQBN-NOQ9WiTEG4~{H^#0T;3cYYY-u!(0+B>C~#mw_?(Ksq7b@A2c83JZgY83-zYFu%cV=(<|S6h z@%s=s0CoJ{IDfhn{73J;+heM&;ytXA_A|F_&~Gk!frW+RFH&?tJKr1;pOFQ)SrTn~ zl~Gf}O;FdlAx_$7%VqW-nR#TZB7WIMeiA-?FmaGu3Qf?HU!_DlnA@i$rDM{{Q+0Uw zA(7NDwJ?p%87F6r@?X4Jq8&#20Kr*Gj+>G zL|Qj5o5Z*-$%a|`k+AZ@#f4?^;^Lir8fj&%dLya{$G(SHq`zjcIhIe1+%}d2@=oDN9i)z>e7WI&7uxM;NrjX z;l}qGIq%v>2Zs)zq}q*J8c+SBjCSK!lD)rw>9dw2iV02xd{o%mluazjBNH`sn|hrTptAZN0tlYasuT69~6)^d$VfiQlOhetq;`y;}c#A9j>ze+>kM zdi4Ad`md{6FI*}`@Ox7o_fh`(=)Xen{<>QKT%n;QOCr{@1S7KT*UiWZ4=y zM*TxE*qr-hjC=AxM@cVX64MMr6uaPocP}z=N_FANml{QH@Qp}Tx``g$x@=O_x95># zYe#ln(zL73WX6y2DLIsn$GBw3-`?3j$EJxZKrwMlY2FHNkG90^&)?FL5<7(63!gF6 z6$^0^dSskpreUZ8f39}(QC^LL_nIBcN3|tGY`7kwP0Te$kw1~Uuyp;8_g3ZDF#C?q zkP$wYe94`5g0`t^z~slCyjgbTR#z4r^VsJ%%7)@#v(R|UF0Ln+^J8xwwJzmpUmH!c z$DBmvD)nSvQgtp5b^ozPsIu|Mc})(zym2~8sY7MloG0Lg%+Vj~C^e58D~W;}F^-g| zS=th__bR+FUHo&hZso$6$=0`A8Us-6%Kh1wl;8!rpF64+YuV_NI%C=-Oz}u2RqNKC z)F#3E^FN2Py3|mfQJ1kga}E{UI*@%mVcRvQ_vaULV=Y$`ZocWsAwxCGxnMX#;KOqt ze{TOr9BD=dVMk*gwf|{yV+|JBkI4QUMDM)@@9=!s+I{Ej-`}amZDeL`d4zeH@iN^F zY{`BR<_jXre1nOTjb=P@iH~z!29?SZZ+MIs{>-5p?|Y7*YyPicjIDJ8gio)0`O;BZ^^i{RJrI4?K=fsI`rWmPb=YqF*qRo7Z!X!KV> z7G=k$b*#k!MA3F3_#^mT=uO|#h}=H}xj$@QK9-sWKK<@nA3|>`eL{{29~={MQ9T=y zGxxtB&m%X}AYqw-jBu)@OrOz4z!Rg1B9lGHwzyUy<$yoRu?3=^PXL@X>2^)srw$eMv z=(ZdnY!5N8O1I=tgLWp4E1QeoBW#ANUrORef`;aNE4D^?jeHA1*Otz*@UPBz49&1;T%AyXpYA=wvQK7Y|jo&(61tC z_VuU^38VVs*g_0X$u&D%fe%F9FBmsVt&WvC_#pRlqXYZt1h#;8d0dj7sy%|FM*F&Q zP1+ud_J#V=UU_P)$FMWDRWi8EQz$z8E=Xaqx|=ARwIplYyof4v;baWcUXCQ?q{)r| zAO{u5_%d_zEeClxcOcOjuib}UHEM_`gN)3;5JF#taI;^3;Q_gpNI}a`f}h}(feQp@ zv(IZ+xNjS%w*QQ`La z*?yf8`;giD(RQI8E*Bxv$R}xM)|U8kOVoX9eP|Eo3$X)%%3jF_XEC2`KdsL8Vk*4p z?QdH-h-_o;$I2F)_7~gvP;sji5o!dy6u*zjLkmju9eJV?{!-*d<|{RXMeA4q`FSnv zis3GJ%?hyzkTZ6PLhm*$j4nLXzvkA0X9Kv0lYHA_=Kj%2yxa(8EkL{LlBcl#9u; ziUG$aYC8cSjF%xMl!UbJVGTXs9nfL0={9w=Xff)Vm&a_$^?wl$`+hrh ztleDEcmyv5rXuUE#JXZOK*;+c+iOs-=&_+{IUKvQ%!)nLZvg4B@+^T+b{z;)rqr{< zhkLDI>JaiwDED9+y$tLIz=NWy=_w@psEEl%uLjgy)(|H`-W7Jd z(Yy#ekp@qkZg4~cUjhWD9j4mZLnhDYxh_1#85+utT9iT-%IFq;9=|P#TlBwtAOMf> zT$K-LAm`{mpRFCvZ_$$rG0>V5Hudp-5)nCQ$Sp!vaV^GqDuvH6$ro^#q5G1X8mrR- zUoMi-pY~Ral|a&y)^*m0v{hAe;NZnL3^?s{$zBWMcR1?5-BVIzGk)Os**E$%uf__F z|H;X1>z9f`9&}Q8Jc^kzks{Q)MN%f<*O4Z~LQsqiFN{PKT?Sp|#fwgaq_sgj@7W%M z#4NhdN?}7k%5CDi>MHRHoRb|@7p81gob7hy?a+E{{)pRB!gf*v1UC6T@9eBqMD9u+ z_EZ-8=6JX8Xk_35gxOxx&9o)?ZE@xu`c^y+XC_#`tjcXVy&#b|tC@@Lh12GUwZPCz+4q1un=4yY%{V_IBA*`_8(shb#KuYugQ`fo zUEa(*fbK#Ve~SFc{j?XZWsp+wXs<(%yF+}rB0EidXeOsgK#+~WX(V81?$XF(Yd`T- zfBnp&G1{Eb?X^L{ijAmq`ZE1lCX>&yb*^7ZEVCZHUK4Gef+KX_@8DkS0`= zz>UB$VrG`y`u>@W(N&43)S!Ep_UgzCcRpgBuz~fT;B?kPb}c-9bH(KadzM+kOV-b+473tAHRfwEIUiTrVgK3 zJjG!-?@7kMru8l?(PxhVj@G5IlH`XVD1#uFO>|TV{uBgT5&zNlcd|z^KY3Ho1lE;i zXV0|=UwF`WTc+;uQvNmO@9Ixa`Yb}E7PfWV&3osneI6Nt(v8Wy5?cgX_LLZ4!edtW z(XqSU^k=jb#1^g$eq4r3sL2!_G(eEYCbgdH-Ro>Ozf2znSVMnl=MB8jQqkY81xj_h z?I5V$2q%jfNFh#IZYSd)!G&O-^Qi+@)Mz zN)IL%NE(jfqebijjal@{w-zyrqwbrt_hrWR83^O(N<2O!jFKAo@s5+9iv1}bDSwKG zBqZYir168qk<_h7L37A(r|?bPh(IGc733lLviAYcl9I{9$Svl!qEzvZn@667tIwN}Z`UoGo=) zY87=48|QDjN2t+QJ;!KnQbd3q#<)I4r>x6FcQ&WVnYK^9Tb^p>pRi@(y!|3!=gNj) z1@4B`-~mS0@09@8HA-xnFHQ1Y%7(<3+jWFlDmu(3nHA>YkCm)`tf z`Ms*WOX0LTu?exDl~Tm)*5L$1)RMi>PGl|00V6pUKwX;B$*^pwadzVEuA@dT z+O<5IsT>P)a0izp$2YTvbx-`ezt0Jv?*-i)+-k2S*%~f*Iidl#G#)A62@oIN9uwS| zt~FkUvBoWqg0fQ>-JprA0%W$HB1f-CLraUZ7n~DI7p;1h9>{f%(5*1;;u*udi-0$W zzZAyl;R6PgEgkyp!aU;7K`d8otWCU>8}~w+;`mu6tE<>jBhxvx2Ni#tM85TC9YVH4 z@}ZVBHh31`>O~nD2K3Z6dE9>-y9SVX8s~P_#tb-DbQ)M=9Pe>VM>B`AgC8VsU)%B4 z>1jMp`wJvJW*9$>>+3$gvj(d5btQ|Aos$+%;m~8SgYPx->NZ=Vh2x{7b;oY1JJ;v~ zR|siI?Lg&`n4gD!4>8|EFxlaa{;8WeLC%y0_SoU+e>5P@kJ@2i=1&gxiX-?-D7RMP zb=;k^OLateQ-$%^z#<-bobJ&ga8AJxZ)PVr%nx9B>Ct0uw_inTi-j*FcCUgfGeg(p zuJiakF)rJ3u-uN!y`i+}!u^2p876F|oDk$KRf=9{%kQFxVlo&yb@5s@YsH^-+<=U4 z-s>1Gzv=Qq4WmmhGo`_#v)f~~Jmn<*<*7hgI}!FW5yb1<`&!bY-07+;erH?IqUUHg zr*wCkdZyY~LcDd}(eB7L@OF#_yxag@#ubc5wE}M4;Nm-a9bK^vIlBwI{DnYeZui<0 z1LCL}PW>d~cGjsv-p{UYp5fk0;}%~E4qP)_*E3@mc+h98`#E6o7!K4vw;qGX*1}GJ z$)L+qYbqzuOu$Xe{9uA>uzIbF#%L-N5lC+W*|L~;BQo)Y){DPQEgPJONNO>9oBDI* zo>_YgR@RA?O?Ndd3;%DEdraW&C*mAkm*t6v;3>ERZv-Cd4miaI!gGKb8P$>wD2W4( z2=x$p9@{I(ac%bv2BgRx2iUuOv`EL+kWPJ!c(2E`M)v*S-3Zi`5le{kmJJgf9__*G ztPVl_L}XWNwCiFm#8~Ph1)Jn2Tdp$lYbi)>FRLZ~u>}Pn4l`U)SaxL8O0NSOMrX*?G8(Z1FUN`$QIetnd1n|wCH3R`i|akOw*r zAB)E^VqHMoN!D)gxpmH4@vLs`ObwPGa*X@z88H#KQ-|b zp+@mC>vQw3wEqtm-qG|OgZM&xV)QiBjHtR)d3ef`tcgwXdTQ^_S~{-VTM_W;j9 zOOMT7wXvZ0WopU9f#qvmn?&m`Q=FHJS|0FoNoV^Rni50ZG`qZ*Sz8v{kFQtPW85z-xnW#5PuEk*kg1fa7PnAm3AKr{hVLPxgd)S4)NRERdhlnvjXU)tcsG^2-1j zGhJ2-RJLr1Ln^_f%HF7Z@50E*94|^;ck98Dslia~mW17u4hZwxtqNOttV^$nIHPY< z*%lX-k=n2~y3J__Bt*=oc1&7VVz)x9eV?B=lb$g(?$=R}&h7-Ycm35#jF$fefnUF6aYVOCWn)Ld}8MQYz3QO#x} zHAa!GBVmOW3WVVk1Io+ph98$F0EC zwvG@Bm&ft?@bjRmr!y-g3N!ryVz;|+7kh>+uhOjJ+cBkP`go~%%zoI?~7l= z@Rk9w^A`Ye8VuCm3CQ_o{A-A9Jt80yv7cdMHgugIC}X`50rc|N$FuXiYJUMPky;+J zJFoQ`W4&DpmcZ(O?W^ybhY~0IjmacljqERU_HfgEy|lL*>IU2dZ_SN(tfc9Y%rgyx z_v#9}yi`$2PgIf=(lOFnHc_p!9r`SPtB78z7R>n?HNf`vf1U#DbXiyo+s#4zs zb6^nQR4^v!`27w(%Q}TsoJELj7=YxEr{J>f9RuRte4w;QP~JJnL;g`hfMA1(#@ zzc9?(vnzZ~E_sNqsHCR>uWA!64^tIxS)ja%wuhR10h}G6yVaT-28A@78pVNvG71L3 z+JO4W^0=)4I0kR3wpVjP-49wOpr$o$zXubi`ws4mrDe;cO0jJKw}?(%F2*DW`Rt{x zJMu(WbwE4;B94W+C;%+@DUBhb0hLCGpO0CS0E$n=kGs4l!2ulx*xQoxe)LKU^4vo-U;hrq;HCx}6SA9}LOW{d}MFg`Qx zC}o`4?&`w@+n5Wds(LA|+=oDWYR@6}f)G#u_UBZOog`WGSj_fn&? zAeBp&xjAYsP9wvR zjX+_#>U&*}w*LrT+Pj;`C1T+G7g!Tlh)*>{i>mWXK8~LToYsm%$4e=_6~n9@$^VI7 z&gTG|$^q$Tz>CUEE@R>MomXc}7kh#Hdx!Sy8!U*ntwOC|=X zfXe<`AkVIOn1qi!B{QeAMC_()k-&i9B0^KTGmF1D3^=Nu zh04Ilc>!c1P--wbXpFu9%qBG+ms;k|YMHVM^Kyn@7hEpbmBFwfqfvwZDD{~28V ztwClRcDo)Das1X2_)mVqAKrzrCSgSLkB6jPJS{aE`U7V-X(3=~7?g05^HHGgO7Jmn znLM1NqqIx9`y47J)jx{f3R7C74?HevSQm^$PlR~FfQKyasR9ygV{P8@Ma{%O+2)r& zjue<EHdlbnW={dR|hu{@3&fBRi$?cP*+(Fb+2oW>bU)^5$L`(j_7(s z#i&9XPRq7Y2KTV>W2pb^|UGA-=r&3ZkgeowRNkcvq zb;1Hs*Snb1(v}zZbQ!(0`g0~unzqHkzh?4icb3nN!0J;2RXTv5we;9mx>u2*5)!`0 zT#4nxJ8iwMGB}<7pfiO%5u-Ef4s85m+#hLciH7W-)W(h|&1C9-M}`fqz%HGcy1EhV z+GrY9#Y~r=ydg+C#y!0wWdwxgAhQdG$F7F~w~B;12hKkcQ>vkN=mV_&zDGzFz@ekb z3i~gmOmX?C^uE_i`NspcP}~(43?eQNf<=r2Qd_-bEg28tL5N(0n#B(_b4fGwrm6E{ z?W&HcuM0kax}D9Sv{lov04R74yc<~A5ITwpiCq-p_!= zQq)x|B7G|t5_UDfLLljsUxGAQ@C6S{bmCI(ge|*HF&Yf%!=BxwYdSR`$8(Nl{)qb~(~3$W}4qdnFb4b2G{&kK#Ft)9I6EXrlB==}Xg*X@Iqj*P+ajUb;+@*QiAiXKqU z>{{iU{qyCxu}IJ95a%e^tXPQIaV&`h!cmePMSm_as6AXX3W3S+%@s1fhYkE)uV=l; zD0K_1-fH)~rYs8BSo1*NkZoJlz1LUn=3=r85&XsZJRtU+58d2<>1lE(QjdX!6}XST zma>l1n!6C41mRXQJ*qK#))asBYW!jU3#zzXDQk5LR`7wI|8kUPQH=6Jx<^`%&R z-_~H7eUWyu=v2jNw@pjcIk}9bElY-lpq?eld}?`xQT0tYzluw(d;Rc18`EZs!f84@ zx6RDVyb2<(ugqYVo*YrQW=N8afSlBf2K!u9JQmtw#GDqvO&c-g{lX}^J8Fylr^e!R z7;ru2=N7?S2oT1$>y2!2J47_L#kH~6Po9F)A8Zna0NMGvQ1xZ#`Ti16zII@1;lerq z<ar4jRX7N#hw&(Zzo^<#*G`}jnOQVn4#i=HAbi^4CHDzy1fR~k~<52=NpCeA#RvjyR}jPHSVuo?e-6@}H|ykLx5Eh*5}^+A?=n9BW)lDFZ$E*2Cf3ty-vfHr~Qjw0I=IeOIT$&Zn@vLsoGH#@0f61{6_VC{nBVbiRQSkub}DG z3DwpFx?!7prO`B&G8=v`h4D{(ByZre?0pMtoW%OBFcB&37c_FeW)&Y38=4Y%7S#b= zu~IMUlq(C~MWq5us^(Sf8vo!EN)7sCC6n6WWcr^FBm=Xnw0~;xWidIC(n~4QH?-}e zbaqGJUr!1t{$z~c)EW4fkmix8QkZ4?jpL6@iTO|+Z+Hfi|LG4bjc*;@u9W!supdb9 zU-?VqcXb03{v42{rrQ5MBmLV_>|A*Fb1nWsdofrx-ddGEM=8ji`cK#4Z%_XpvCwuN zv?@Qh@~`;y&$WhK4>R~v9Qq@vI{s7J_&;X&lKWGBZu~#!*p_;+UoQJkVVk?*^Z#86 z|7FAPkA3z_Ja9|FccpqtmDSFOtc!vafw7JcF_#_M{MbE!By%?CMw2(u({b zl{m+u@pBrsXFa)O395h(b&1daEkeyFopxVK`+d0n_j!dbO>%z`LXg(~BuML@j8Yl+ z_Gwy?p@N@Fk0kZoQDeYAPStKm0ru#s0`{O6D*w$L4s>>V}Wj* zY+2v6g+}o;ZKo$t2=OW|#A6?7=;Y5&Of?p4*{bYt2fwS|QXl(@Iw-~Dbzq9{I>MG# zKDy^2d#my@XC(u{?2&xNeW(OvPwWU1dcjbx*Os3`FQb?)z*m%L zm3GO|4~KhQgLn^&MOu|Fpp*{$e5cK&9JhGgeCbB{mDe_(02q}D-(%2x(Nc;Wfku6} z9ySUJU%uN&K7N86HOdU*Dx*KJSb0;7I*b?2$%c3e*${2vG6?wv)*cgHe?% zW4JS-v_^_TE)=MCYUGFLl`b{Amt(QJ@6eN;i4sSlrVhb(j49a?TH&1q7Kj}lJVKJ&FGU#Pi) zf1-b!k>E51JnuBqo8Z$xg!0ym^fDAh$({pDG+7TG=kNPSrPLtGvXL#wM&H%IyK06T zr8EyyJxN25;P{8|;LL@fGL~C}PUprJ4oTgE6}^0q@Or>un{wKvmy&$dViJVcC}1kr zvOM_)jwyq}N+KL%M90<-(eKS9+0Q z+MNFToaiBa$e!3+1ugx+MNuoBuu!Mt-=VH>%Sz^v+CszInJH-AeNZfs zkmC^gfaI1{Q3Jq*Bo1$kfA7b|W3Q#`Aywh51@Y$fskF=kkSH<8hD1P*V5!O;gM}-< zcUtYDJ=4TBdxn;5v>5GUn@GVViaV^Qkhc0q z>6Z1H5efsy5Q{@*%)kXI`=N)3j^!p)%aTzXZw4v}(nydHg?<|@t?y(Z>%B?}9TCKh z_y`~(ULiZ&E&Hs0%&^_WBKrX))9dp_R*=n+9~Wpv#0 zeYw)_6LJzbSdRd*big~Y758VlGGk|G#hr6K^zE5~uB`N!Wg=}G@{yhmFovc5yxjge zkccpf@52D0Az2f_q)7k%iih%I%p!R`P~ zwY7%!t@5G9xYd$xc;v`3aWekaDF+*ob`XBOvaHb~gF=u#QyW`z-g#+a#$?uCyz=}I zGc>x`hhP6jgaxQcH9WRMR^~l)SK#%(5iOGU@YxpYS($wFYY4gvedwP+S}c@7db5n1 zI2#hWzx2@8()DjXi-od@8b#I#1zEroWk0M7W-s^D!|a5Nf@DZSFFa)DKyE^GyiZu8 zA>cbY%PrxL^ru=Jh2Xok!<`3b{DEke+p&OI4z0a=hT}#WQv7UCHn-bQiB9ovQ|m!+ zD5r@t3Q@-8QKAkRuCO3WkaTaSf6eUgG~hg)4n*j?sp>+Q9TFCi`qz=<1_=Gi zu~d8C5&&AOFR$owAp;a87$bEHQqn&aGgGb}exqx#nI0O=jA39$1w&RM96ldLw~9XQ z2b{K)d#pnT>nYzF`NtJqCOQdM$J8&e{@Y(EJ%peSf@*ld<8okBJkO2!kx`rwI{QJK zkx&Ct%0KnBS2L0_TKb%h;BuSmS&)|y5z0_vI?pKfuwD78lNQqRMy+3w?>vI8NHLwy zJQ=jWQua)q6hCoQ^17j}>uReUFRo^n#;G;k(R-_T;^CJ=#Myj*Zp-L8@a0^710Vnf z67J?8ZmSjgE&AjJa$!9DDGwn2J|MHim&ut?_(|~lbQ9hA*r6#(gMkt*1Z~F;MQzPz z2bV&t1Sy<#n2tS?g@cYSULe5S?y_jv1lNiJ={Z*JE- z+Gs)Pd7C{Jx1X6@)HrHl;N~T8t2RQov#G)GMo*j( zcV0*+)3#Ei4_oVN4NQIDACpW+J?Bd{Ylpvn6t^l{`#u@pRHNtIxM?$2<1=5TS0Z_! z#QvtbIyE)5mSqbE2?>dcFciN-`=8|{X?~$*uSZFH49*gLW;${B;xkIdjBh?qQo$7v z;Uv_4;PQ1}-|^L_)Vd$K%p5{f;@MLEx-f##Bm^=!y^0Z=iBw!gyLP zEIK+`T3UKF^JY)&gINsZiw++CfUj6m@a`VQ%{i?8?Y+jsMztfHbrjA&w) z6crs^WJH7rL~x6XMaCazWI##6*4Ea$ckdtofBN)k=%16GpPxToz{L%)^J_RQP-U>~ z?s4zlJxxtb6B82;_b|h*ESGJ}r_@xA=^;?Ur^j!+K6`PM=E#wtsHmub0Oeb^pgoj* zZ>z_(KumqTub-cKt*;anhx5Vj;KvxqsOj*~vpU1l0$C#0$W zcowz%djbS+sKxgkOliFSd@6a{M{) zi^FFpW@g@JWW)eP0lDr;v92(y&KOttcp3c4BAdGQ!>2GrDQW2{C^hd&*K$~%P#!NT zf083`>cT3M6$cYcS@YRNef@ai6fmtDgh*cfzO>bxnfC#D4l-yqVJgXHKiiWq1tk%d zn+u(toj{xOaC2J#k>Xw<$iIT zY&qUCHHL)j>_w#26A)9t=yR_Ct!WdCUD*gtu#1Mw3=1!>qWFV*_r5`ftEIX5PDmM~ zTc!p_t-d~`?%`SWvs(24YEYDsX$bHYC=lgj* z`XF6wZEX#29-*h_?y3N&SMK`t)fwzC+w^!o}M)+g#8=VIaGH!Na zvFuq*8i%Za=D1Fk#ZmH#BmxW<@0l~z96{^GnGdbV4HgRnFTI~;vmOD{q;9xPZt!(p zQUWMuS~^buet;+&Z3xDpZ>__i+x?32iHQmSX;C>M$eo*pJk6a2WCepaATPi5a_e#( z1_Qp28SnH>Av5NCOfCn7NZ@QfBREbvJ3EW8x{%HU+|QQ(`?qi30Gh~g-*ygfXq=k6 zq(i5|6!e7k>OFn^QA)lLN=At-m^?HqkN^+i>C>mTuzq`<5gHbEz^;0-A7saM_tk0ROI-y5=UIWt#x zOn_F*Ggsuf2h+B|fW6APQ%N|@QU-@?&~>RHBqZdOx{S-Q#vUsQDN4d5J~-e*lV|h+ZZS9)!x>2U@f{xjIW`LBc_r`fZmR)ME!D-3^ z&-Wfb`i6%s26`x^>5@c%{lsRpkcCO*0}H-+Q9A!*(oAfFHMzl|z6uv6tqONHd;8V( zg<*Yt{d7*wMUQ$8)mVW@@Uj;|%E;|{-8O(83knMA_~c)HrS;fA&(8P1jL#!P?%4zY z+6ajSO{7Gpop+ZmJUK0`1iJ01eNk3XQGrte?VjTHW@l%=SFH>V4b`Mp^GwXnZWU=| z4U_7ewrhVdJw2Vy%;U-Rc+-L00K~u1rw1uOxRub;1eXnqG?zoLV8Q^;(OH&0{JdZA zw&b^+iaHQIf5&z&)X|@kcSjB6%I!e?{2El{l%=NDZO?Lye(8(>hO~TuPy;k;u|g(e z^MeNubU~X!W5MWvfJ4WQu^2*bS}kdDad9LQ+VISkpJEG>I^vsz42O~mjg(xi^fksx zdO#Tq=+;|N4Yjokc9aJ2=hDjXv23M++2()FxK;pTpTD=Lq* zx{ZMtLQ*uOcaEbS-SOLN^s#}VHIyt%C?in}aNDAYe`~JB6UZ>=i`1q=nksGFUTb=7ku|-N)9a5Mf z9#;q$Rw%UFwZ5+@_gB41%KceNr9F^E?s31xVDzS@-4aLmXrQIR8|X9nCNXggZf!TK z%I7?h?yjJl*Pu6|2i&`$s3R{o7Zj2plsF{uexG{2LVynpJe(&WD(V7WBKVc&W@f#4 zX6kl!b`Kvy(p%45=L3ennfQo1lsdEdGedjPo-F;VO<}>O^6+;&--F~(ujlHSn4~uS8IX-wW2bUZ8ufq*}Iu1L3nW{BEo%gXX+$p~~>gYso zzb~=slz9@$K&(IyC;5kO04#R=l~Zffa?Z#8+j7?Syqz*d#57)alBgK89S9tCcpF!7 z=h9N|wMLa|7A;cx{r&eF6}iZ%vFji~4O}#K#r?icpY>Pwl>Y~rBHi8HtEzT=-enh` zmGug^kmJ>B!4tqjVU?ZIwaJhD{{FpR25drxg!aaP|y@Z$UL<2{m-=g!sry6yE};A+fV zA@DJE($dms&BE%L-U~R@{y1C{t*AFJHa51ii&Nb^Z~tkdKY!V|Q=I07{_Ix~pZ?rN zPtQ*eSSnn)bZOG;*{^|)o&s7w3S2bu=yvGjDVfHfBX%3 zmpacqV-es4$}ljT1>e*&3w*MT3HZLA4DhKz7r?jnEC7~Q!0UXBL6`b8q!FXx=6^=N XAGK3do?iO^G@ZfI)z4*}Q$iB}bJa2j diff --git a/doc/design/img/perf/dev-bw.png b/doc/design/img/perf/dev-bw.png deleted file mode 100644 index e63ab702266f91437c75b943accffa1b6eff843a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7054 zcmcIpc|6qX_n*4C3K1o@#1zREbF(DdOtx&<%2KwuCCR=F*=M>mNy?HWV`~zMvSuHY zvP2jugc;dp#yZT5Wo+~N(D&Y=dwEj)NQ^5NNl) zp7uEqXqzwa`Qy%Cfxp&fs{247K?{9t4f6o|DMGN9`6!d!>sZ{=SWvzG!4oo{_ng2# z`cqGayGyE+!yvZrW^^YeK;I_0>$&U&yP-t2M*Z%2le=JaP!)RPcNX?+y@1 z>Tc{_5a_TX7QYSj;57CZki?&w>>$(aXl~Hs9UMZS)4TSmfNt$r$fX$lRaznJF|>3%ree&d`OaaCby*$aiPoe9k+Q!MfhlI%hb-%M7Z&R z0|(5ltqo*kWQb~@8|HC7=(P@Aa0g?oQ5E9cH*D&9joUS22+%c~$A^b|Hp zhoO)?889m|GmYK5cl%EEoXC0p98075F!z9NypHmyztPgtk})^Y-FTegMIaC!BqWRy z;H7Q{WZ*Swlf3D}FX>tl+(~^?Q&S3l0$w{~IsVT)WlT;O*4BeP{(l*f3JH=&}Z=~kpCnhGG z>7-WT>aM-VOc97;$!Yh#QnQN}lawIp>Vzuw-+w>t?CeYoja;ZND=ih-8OCK;ju`Tr zx8y1)=3j~yQl-=^Pshirudc2tT%Wu^MH(A(&oLN}`f%trn@JGp`PCUquJSh=ss1JX ze%*FikJ#VZj8#-r5LbtWE{{)6DwHvqIyyQVmgKGCKY#j#Ci(4t!1As&dlD9R{zm*< z=UC%6VVRSYLnB#OS)I4D6Y}%(Gjn&Bv9PqfrM?$%zINH=JGU(;Y;0`d!me$O_UF!N zCM6|hF=B+($6NbqLRIh z%q$NsuuW@V!#f}5by<+GGt1i-9^>CQ7wuGJ>krpt&wkJMX`dGlhfvBfL(}uG)q(Yx z;~GZWz6tI4{}D1V*uNAfsw)A5!HrK^9UPM8cD!=M)Y)0Gg z(9&4;SXQ!!oPaG!F6H-2+X+4`GK|UM_K#zo*`w$04`swap8y{laXajEbl4OX6!3)( zjnA^OZZ)^Ggtq@exU$QIqg=d_e;D7>qnBuR{Q$wcQHT+PS_$VCQqfjdSAT_KA(#3F z)~;sJ=hWdrV`5D60UC{VZFOnx1ZQelQQ+|x)zzt4#;N1M!jx}T>UjBEM17P%XHDqx z$jDCCDi!7YCXCIe$a6(Eyk3g2H8|CIeB;HQNq$YESf1rAD7~)v zik)kWY*^A;jtFZY;0aJ1=k3ON-MpFEQ6%|KkD~Qh+Do`YKJqGfRCR#KEuV&+6Uy?KyhWz?^8$B0~AG`x+Hd|4M!PHC>*s3{n z-{IH|*+wfYik>|1v{5+tZnUtvOSH#edOM8PB&tQ@G`oR4yb;zzfPbRekF69j{TM} z2bIvc>gChOhs9U18M9Cnz=&U$W?kN%V+}+)2lCe8{(4Y6&jGX$9xStXjxp)atKGd8b-C9GF zu_=92EK6HeMv$7aPxCeLTb_4|8^-VRGSNi0!9<2=S3_>;$TDW=k@Ug4u_2rGYRRx_ z^8?@4$}7NJjyegs2@?$wG00)a<9?s zOZ#dJF?7LhoJvjPDJB7kXMXP7W$x!(7yl|GWgd=6o@ zPwPL!e{o6=Jp-v-kt3j3tE4CR;l>yxm_$x~el}{2o=zKlJ1$VWh-PN&^&sK`6%eGe zh`TF4)))lTzoD@M0NvhXfulc|)eCD_b$v?C3ex=H!>h`c%-q}-TjfaB*PFxV_bMnU zB@Yox-31d85(4RU3ne^UT&C967=QRs=KbCI(RsK71nc>|*WF9h3hZ>#FIvd$r&&L5~)>+Agjl_TV16`KkRalC@MEP48 z8zQk&*N}x5QcN@Ft_36E8reB<%qN00IML+cXrT-Y=AaY=660zg+r)PE z8;Y&No){%Wks5CT)%Pd(39K#;uud*_sj8?@(t@Y4Kn`0%eJ>ohDZTp;0+$sR#IYeM z_#=p_>S}`RS(P*ZdIV&@0ogh#K6p&Gpa4jl2_(Dj+2Q5A$Ijkd9Ssc82L8v|V#^b4jjIP@t~((p!~U zBDgu%v_AItyEO;)mAac-T9R4~O-vR%zU8~ME&V9bWT3a#Ma*aL<36T+gfxIkA_Eg% z>xerNEEMfUe%E4!rF(D?N9%Ng-mnRnB9YM55ph^#uS*%Wwsv5JQ{h1oD4~7N?67ur#q_$K+@BK|veP!n_UBWWO zq`{x-8q*r(e_DMVBf$5wd*Y87$yG@&va92&w%>aAlhJ_|KiM+h-l(kKhw~m7mX=q$ z7Gz!FBUZgY45nd$VhI>@J!&>wwMzw#7LG=pWPJ_i!GK45dL!~ux9&)ef3I8cl?d!mLT6=lclOkaCoA2`Yn(6O5-yY-L#6#Kv9wyfL5U#aG4 z_~JC}J=$<{%6Z~tHL`!Q7M6XA&$oB0Fn`d$2s8P75B?;q!Ufr4w;WA!JQ{$6duQ$) z)LGBOR8{|=^=2YmDDth9#E>-qcNP$^4lG*?g%=bQY<29rV%DqT7MCyCEYSBAto#EX zB_mqjo%=>cNjn@A6&20x?PbAyJGTZ)Yl|cC)2E~2?3(`mN58uV zrwCQu$%(0{M%C5Tq?74a?wag6q`mcg^cp2229sgUedyF-r;Ao_xJj~-=lnE1?o?}Q zYgT!gd*2yADUl&<2|NvbW4bT#sQT*>r)_09z0vOoJYmcjS-S6deAKa2jubFWzFRr z0a@E?dA9Gw$1$h8p9CB|8om`BuyDW67L1REkfc5w&yrFt^WK-xH&rtKlz+81Q=p-h z6`?a5W1DP`4Db&n&uX%UHmFk@pL)IJIoR>pqVz=_8RsQ zEPs`Y5HNfX=(CQcZ`VJ(`QnDp@XVA!9@t8O^26ZK<^)mzb)6Z288d$~gUy!alk#2L zW=!dRByN(XfEGr|?pwRO@4m_A07x{&w!p*ECZIQOqG302SU&y&{5AbXKpe1 z-3$$*Li7&*w2>3QYE8AYei2f=e%HmtaT$k8juhd7awwq z!_{+xVqHurPC)y8-pMJug3!^S0o3RrUJ&RFuu(>3C1!kl{8?^pl#8qD^P-}2&>f(| zGo};$9t*w6tMP zxtV2^XV>Ko^gh0wMoLL=H+aAx+T6lICp~ojVn2P9+`z7?pr{x~S422TdZ!%U0rhW% zpqPf0OXInsSrFTD{FRO;I=r1sCNRJv-g(PPJ>`A*(}YVSbs;A=?+Gw})G>1EwhxwZ z|57A&nV6ejV*qC@b+=9hKy|yJ=f?yG7Jq$dUD6zKlAZg|V=!y6k&yvjm6&GI=zZnh z8ImXg1HzP8;}M@DPB_iG2Nj#E1_%Trpx?`mfvr8NS*(X(mXEBg^m$a>vqv%=5$UWJ zr_8%^m6+M((5aH{YXJ2y>pZ|}Ig|D>9#-a*7TC}qO-A@m$tTpzpds3B-t_#kKtnYm zvk(fC^&_6^mR(8>H}E+E3tduBpYL(-U7hO=?z9gWWUB=Fiqsi+eDYnhtz8*~barIuwuVB*==B2dv*N}nGs6CP z?HwiU=GFy&Y$G7r)k!gr`^A21UwH$qeUs(-ge4UAh7%DG(6Rz>Ld9&Aofj*8CDqp! zhe;M7BG>v5;k#{O&v49!=K7Dt(glF>uNk*G0^_@DZpxvcl3`0Rh{Bd(X zX>AQzEY?QO-{r1T6fbUqiNB8)XnJ(Q;R6FtXSP%Wq_wxiHSvq$+x*cw13uZ6RT^x# z;=%}nQ01SzI(c^?HNsP;fv#LRpOm&6k|AyR;<0pIxn`T(+f%x_@hH=qXZyX-Efq; z%TihGs(Z2F)%nRUyq&|$L!+)0w^&VRswfP(QvStLn7VQy9n7kI4fQn@?b2*x6Yk|% zsiua;q@mWBx0$GwVelBhIch3|f}mi#*rX(Vpw;K?v^z2Y0Z-N04Ev~{Sg&mc2M4qY zfCyz;=MAFfR>rMrpR7-fu5zxtVFxQjNH(a#k;o8VPVgw-T55!)2Sk}Wj#Jsy)m45O z3}cS$#V9!6XKF$YQ9UYvV$%v(ZXjqv29aE5aPdX==i^<=qnvx1#zNSR6B`9p+L3x{ z>7oPC{8lqv3k3fG_h{QeYEW}?vk3QI_3jX6#q7fhPRF+Xiv}Rj1Fir2E5ZNzIpN;~ zy?;A#SzG%*BT3pNG8)c)IH^E;+dGYi)P{S4&J0RNjiJ!tB;bj{I~Ltm{wdH%zk%LMehh#1k+<>k!&dXO96vP64bMPoDDd_+nBRo7JYDsx^RTNkF^1)FO=_A10 zs>i~skDWf=VJHdGjQgD)^i$&?i)&>0Lqc(EBz*LV}tFZ zAMHzA_!X6uI-lyt)05-lwb*%t8PgG5^B7R!J9~p|hc(sEX-P>3X6v1SDwKp=VmMbW4n>W9dVb#b zi(_Ka<%jLnU&A<`0aMf}uw0Mp_IG^2l59727IIV<7M{2a4g&Zc@Q`p*eF%X=uL0*^ zq@<;*od@%onUK&(ZZLF`z@yyfx?OMMo^j|l-Le@oS=;d=u$4icF@XHY+YA8!cgikw z?lTdWofg!LJF__a7LJ%5s(*F0^=n?WhD-SZOxq~{ zbYudPX(<_*Odjas>*R&mMsVjdvkai@v?TAhr-VE(1jfUoqoW)`L5{889yh5+UlGJbnpjMejogT^jzhbaP!^8~jS3sn$N3 z{Z3vRw_MJC+htZDvu@+MEskf*6C%xujbnD6Ky1zGeyO(SM)^p%)>h>4p%z^G>zbX}J;!CA$T z&nodiknFAZA`ql`^CgxCa=A-f554|H7u=ANMnF((!hd$N0%7k=CYV+S2}|ypJyZ*Xz@#4kZMn*EkKNc=3=(8jVIBE=A4O*M>_@ zTk%2(trsu!6ciM=&a|7@=0rVxs-vf;N9b*9+mn)#;)UUX&fBF+(@zBo)$z#@!tmrI z?a;>Ijt=8daw&@J072WH&k@PFHvaSDwi*tF*)LyeJ&%c@4!cx|h9pc|Ho*JK(0K@w zqSn8D8;ae+rQR-*r8BUcI8U-LpCakig{0)199g#*N~QnD()EyLB3_2hh{ObJm8yQy ziX9ufk)EC&Zc`IdSK-G8Z;n-2QOHRTRU7sjG26V*!#>}gW}Kdt74ZUws#d6|oWo38 zE+48lQW1hI`BV6zJ6j{bSN-!n5TtzEo);SU>3_-1`!u3Dl98}!=aDnck?1+LPN7p@ zPfF8wfg$vMCj|L@z+wAQlQ3ab562QMr~I4L)6w$qM`Y2a7cg@nIje*G{QP-;yjd5m z;+AU9FKz5Rp}yC1-a4rlJM!z{J-sFrbzZfxE~NgZUk_=%@ZpZerYsEH6RbZW2uoJd zrkp7*D^s^3mv~}=L{*5`_1iL4BMrMgK6i=d5qEb~lTzo7BysvJQqZ*Q!uq0{)MNs| zBBx~H*e6D%pn?M&4kr?cTjZ3E9@Vk8PhZ}?IP9U!9c?j@mS~FBRzSMddd_835U49?MQLZLm95mo*}K@Sv8+HtZKIb#}u&;+}!MBJYM}qn{;|+ zW?+keCgJv!fe$tU=*=)^2`|0PqKx#x?G@cDh9az43lb0~CbKb_=UzK~`6df}tSQh2&y+E^2`Ueu64)=6MJ51wH4VXb?4emtb?Uo zex_W@X+cl8F|XzDizvP_Uw7vDin6;?(CtPhb#5&%so>yXVu|M@@@h}9JMHRC-Vuvs zOW)BBi10u+C4wOqbJM$j~Dx{I}Fs_fU%I{{GNkl#I|_%Wp&B8iicT|NmYA_f=yB?b%4dMeqMyY{>WaANM{UminIRocB$xa8 z&a?7yh2CgppmhXl0d-I5UQ?24_Tw{)A|&ly7!;`tDB5nVx?)+&6dr=x(ggk=8ZAM#l@VyHp+;I z=Es#70MArYm0mBDQkDAd%TxfKsy(s_||!DW2e;FX9>PP zX;ysoGUNFE#Dsf{CaXkj48Y!*)YMdt@gXQqZ&g%>tzSI-*qxy-dbDR0N3?i|X-<8S zaKmW9YTSaS6~FSoozZKRwPwZ;%cklNsH%zO@Fuu@6?4EiE9}j^4GoRa3h+-;!&7XM za|1BlOcQi#_ljW&x~iO^Ck8`rcUVWvO_Uf^hlm{jfgC82k!SaA_oo_-#YAU;RY!S6 z;n<6FlMTIiw-ghMl^RUI0$OpQSK$qIIQ!XQn`GamMR|DrHdCBqG=W(y!uy$&4*VyixWo9_imt8;Jjm;$q)b{J#;%ZFlUr5J0ZHyga0qMeEm+|5o?&Azs-~Ri0#G zQLK@0yr_C&Hh^MKe*gY4&VPp1S(9Hqla@HVzb$!~>wD?ar891BZUoVgXO)|8l{w(= z&%Sad;ex~No(2+6^@rZKsEws7=r~RDldy|B^ z+T=HgPndS8+Z%Pp#wLk;`#4bchn(PBr3xH6>tVCKk^sbsdU3KQ_Rf99M}01uBiHYW z-7HS_jhp;x(qoEbl~#=P$BlQIQ&y5SbKRF@-FvYIAM$ExwtPuvl3Y4-9r|Sj492=} zr=<*DL~NEkeG7SvAVYDdnC4#}mEa9Kv=Z^tvUEp$2$sj<*OlPH$_&J=MqhUPmX_^b z^D^Anvedhv*Xv{4)SC?wRh`9^X#q{~8rB_Irb*wTmj?*UVZhetI6oSQJ;y@+)y&

z7n4poGBsPyI_l&TbW7+mmRy7tVVJMTg(Y$DYTyR%>sR%jFDDmQP$>5;FH9kKZxc}T zKu(cXMrJ0QS{>{erp^ z+nZvP8Ze+pa(+87JUr}L$**+g+_~WS>G-8KrBgg3V$Zt<{a(a$4gUEey6f?EP{@>D z3TStfbKc4ghATb_Bqs*QSdHKVWW-0J17z$Y&)ApGbeQKkcFRXA-@H113A@iQwf52W z1J`CJ`najmQ0a-S=7d(at>$X23R}&I2g8ZY9U!$rVeDrhKN!zF!dq33_|1mJP-TB_Cot>REX*HGM zGpBY~ZtG{e&-QtGDsxX?`QAZ_k7Cv=KW*>HA@mO z^%46FSs(1!ZEWwOpLa?ex@-SFwN6j2 zbX&RFHO9c3vB({#Ih$Bf);=mi3>v0f7O>ssteKH_l5Y5d0;6KjNhh<0y#$Xkz~*0u zE<2m+bFgDM3Bk2`IS-ew_ZO8Vz=4l;T1d8NEZrtp0ES-9ZO2@rb!0O~KWM3iQL9@a)j>rGIVIT!SJFnb^wrwSZf0mC~;w zHmlDaiw$~EvDU>?`wEWQ>V}bJ{XhJ$phCsp{Z5u$yX;yelCULfF*S%d zv2IQDD(IH*t=(Rl{{OQxuuPGcCdQ2-Wc%ac}0>N|AN*I*EVn2g5-Ne!lhr$rQqf3%B`8Y`o#l(}UZCug`X&JHMq5~ycpysXY8`R+h^Gh3rk-MI!5H-F z=E?qYYoB?lL_??#OLE$q!yASG3giy#o2cLmZCb7db8I^8FIW+nCcy#1(L1cXhPQv3 ziYgbC^C$5O%kP;E#WMOqS>7w*Qng?BlLuxy#p6MLj8dXrjFH=}$Xfhpr=G(J0=RLR~YiWqgSi8<;J zsW#oHz;rFLELt24l&hvu$|>ybC5)tYNYy%NIk_k>C8>Xxq^;mEB5LcMRlq=@m~VI3 z<|rvD>X!Q~43#e}U~aSX8!mOiyK0Dh=Yh#Cx#8Y-Kc2<)u%jgqq_ zPb|4lmd@6Ph^Y>Nj+80EGu!X$d!0Xy`d*x-NXbQmQOR(%h;ySOirsAsCnoDKr|$o< zbqN3QMMIY97o8?v|H;b?<#y%TP@{5!em$&3_NK{9bEAFITd-nv8)x(N*=GN)=Zs}y zLYA7y;`bk0=3YHT5VfkDA2slOZpv54Ct-HvdgC)BY@8R2sXR*J%UMJ{g?^kTsAHUM z2afss#Ln2n#kt%^lMsb(Ly=ByHDaE%PYl$B<^1n1vJ1=OO!L(`ls$Vjuw_4I+Z>mil3MqM^N?#GD77_(VaP#O!vuMgvhuBN8KVz<-^%vm@owf z4mD}UCl1>#-Nsmz_B=fWDA^ZFFgW)m#~6K^5G@HmQKG{bDZ1UEM)B(CK{hU4gJcuz zR#WO<utEBDXTb{L5kx=EpYMc*4<6@dYfzzQj#!-ksL`NQ#thCff7t6% z$q%A;ey8?AuE!bPko0evoE0(_LS5h-I_$ z{$r;(*nG2RnWQgBGE(-^glGAaNuSvgX5MgaHJ(a7F<5bdCFsMrhdhl$iaAaV)zQhj zZ{cPbddto_cV&mag|CZAwpsfX6!P3L!h`hCXC$Xt5KdJM6loX=c(xwJ9uN4KAgX@*a!%3UBc!Vtj?u;eEL^l=*E%Zb?8*LkyuiTtvBhehJMX%6OTdtfu7BykzJ2zd zB^@0djqZ@`R{{6d^FIa!BCCB#uCrww=hl1&r0k4;q=X`U@p=>}OI}lTQo~@#-Q=YT zDgONgfW~1Q$>~X6o_so(LVA+Dq(0K2E`~Yp|M=UKswLhxN9)tu4Sxy-FZm#yQ~D=! IwEnpJU(fYg_W%F@ diff --git a/doc/design/img/primitives.png b/doc/design/img/primitives.png deleted file mode 100644 index 751b25ec37f7aa64de35d980c174b167bb70accf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43788 zcmeFYRa9Hu7d;B4rL=`A#igZaarZ!rdvTZI5P}8w3T^QZC%6Dc(+D~ z2!a2YdsEin;f3HS$-dF?ncYQ@_~`2Toghe9dX?Od;^?`6#ROcYFjk zMFon_Q(R%u9xeVM4Xk^kY~Y{$}P0;q)RiVFSi6-h$9alNufR_7ai!WXiXr zQ6vtgD3Kpik>?-p^J?#A`G3o*Q??aTo7HY-{F)6bfC$_p=udi3)u>a)2}$_O>>XurVppp*K% z6NfsxAgSyV-9mGT>EW)#DbI}I)|;Xi^2;LB<}6$?i&Ew&)5eQ|(t^Kb|8rq`gFg4t zj$v?Atz??0AN6ASWYdqmGfE4G8sQzAIZh1K*g1~6T4IPVC1{~B!sC-aIAwS#BG;{1 z9@*}N^2d}}CGydao=;auokWKw+87`Ep|>G|JKg@VcINITE{5l+c~WVp86^JYNkGhL z;A9DU@606qAcfj+*n@nj^u+tEOZAx5yqduH#OX`$`5TmxxFy8J=7Hzyb8~Q6tM+xv z927)`vXWR6h^ZD_IEK7}Ya-W6&o+(qd@+(=j+nr$#$~IAnbib-wAw8TQ-K!fIgd-B zFA$jL-pm&txX|MH@zU4hI(95heQWin`NQ4u{(582nyF2uuAnl`v$Hs?(BBkKbbpKt zJjmN`A1dzNq>J1%L#3wE*ro}Bdh~Oo#J{8Z<`tsgyNU2}jjhYwiL5=0#Q25*_ol7E zN$QM7v6)N8)snkiCKh_>0)J5RZd?6KC}={`;3Ie$HlTHDGh7gF@nIcMAK_=cdYcT`w^ zP-{57j`{Pwy?W8vv{00eAfSzp4;N(jH$jq0?>u2VGQxC2G>Jt*-Z>HK#vOW@iJqM^ zw&671*51#?zGbkV3~k)lCBIvMs-k_rux=vKa-wT$9^Lbp+<0>VJ8!q1iP6liubjht zw2mIWLT-V@doX|6nljZsv@GF1^TJRPgwxF)tdrO$y$0vB=9QkUpqz@8x2Cqx^ia^; zPqdm#{f^OQhho!ZWNCd;;?DUl)0OFs`xpt0IkZs_kF1|m-F5g63lT$I3oneVLZa?T z{&D!PVyeKoHU{6!jxuFoD0G4S!4GUwDDU6#z&gWCE2D=#ROqKs)SERD>TXf1StC8N zJcxxipJxAaI;2=WVW4Pd7;b4c{U+=)l#i%*bq1aL>Gw=nS7dP19h0#KS}#7N_$JoV z+9M4cKC&lqjTK4)2Y{Y^l3EgzipYp)3iE%a`20&FXs{MceeG8$n!EqkHCZQk1tevTqRYm8{rP_dv17*o&x|e%&u;dtJ*PB1v zpqWP2^0tL)f?}uwqAR9)4AQ{wW1(*3$Jyf&3GG3n2bshDk#Dmu1$^p+ml}T--oA0q zFW%auL;mhw51AV3jXM0rlEKi?*C_GDB4se~K_-#yuek#(|FT&p?_?v7n|?+<0UiI- zAv}$fU73N7Quqq3THYc}Au?cp%GSCv1}8=?kKf58m%3l;B9HNn;lyPzHcg#P{bVak`UDt zf0LXTJsLSg`2%Riy@H-!)};?m>i_s!n@#37GogbXfuJZbTe?Z^oE@3kR>Aw9$ zBxasfI4Te{^L{+7GAvVzp=3wsQcGa?+Q53345#O{qvS^!8;7H|95}K<(fh5VR^?u2 zPQw)Q&(B<31iZiWNRXX3PtUAYL&V{t<J-Z{`3=LJ2K-I@6**wI*mF^Lc2e!{Tg z9Wa;ipebyQyJ5f+ZeCH}{@OgzGhoqqQbA&KpmGTF8Wj{g+{i;ka%1kjHp|jcr^w&? zJhtp(6EgmspGM;F%}}3X@LJamk*f7;$+uD8oQcv1QAQdBGe3~o^((p#q=eI>U$XV6 zGfyPWAJw$uq1`T;1jXMI@AeVdS~&uilXw4Ddu%DokPwl+VeeaP^D{}VGAf#gP(lJV7w{QKTvDNN@tB|<6pU4eW@w!J$XYd# zsGIc2Gkdm?uc$Ju)Y4aWX7FT19L3Nqz|&)}KuC~|Hj&sD%SK`DWnmys&3#|(QF~;3 z?r65>F+CucD~g~g6b+jNvA`qIHx@r;Ar%``%AlgXc5~&Vv}h{*38@>nX!Dndnni#J zeadvdRPppfC2S`tD&?lC?xPw^=n2@q_~kjTC7)7QG`(+D6fQzosp}h|HtXB*(yU(N zcr&Rb34fvULm=5l#?#5Xw?jeae!a6u8i+x2RG<~Boyef6llSRv4@FTit$j$n2>I{= z)yVoxMX>PP{m;B>xKqI=zbsNPdd`U7GjuB#dVT|w#!vd}akX>Misp6R2s0TcKf7tf z!{*V!F(N{*QZAV`zBF)PpKn{31aW(cN$d8w*%;F34K?wB_2wVmt#we(t_Zn8Kcd?w zdGoBkN%uU^+bRXzo&Jg8O6|r6jAY!RJdJpSiTQF#a|Fv z`Xjqv?xdQK5*`R5oFBRc*|1#LHW5h-x1zjdHHCEGx1wH{oRuA{#nqOE`jOb}zUIu+ zv4_Rzf|v)n9~^sZ{&9<1!PyKgc=f0JZpl1IDWsfoBmChfSWxiW>Pp`L^;~RFdnCd1 zBYwI4O6bS1jmgsqm|@+lU8|rvl>38GjG^QTZ={zb#p3wJWO?fy5ceBf5prfVeoy;8 z>uwqUyhhvCuf*M(7aCoiDwQ7V?rGDnN^oa9G~9gIBZjPtonJK^o(M}=EhgH0(c_Q_ zDjFyZSmYQPrCPoNv|_aHlNBE`!XL*eOM9U1#ksOs7uCaCS?63u=36PJ>9m*bc2UnR zO(y?EOP;D1*T9cmAj-2`x%Tz!Hz={T5oQ~IhY3v3z#pe!?j5@jNT*F&&VS?cW#h(r zF1Nvjb{WO+C(Wbxvas-)C_)1sD4p;Qgtlr&YMHmKspi9+3(N(bUK)$D*}w;_%pi@x z=D3TNuRpLC8xIzj4TlHFB|0|)WUWH>p!G~|17|UOtu~Wg_lI#ODEEFbTfc%tx@V39 zJ2Zv9T=SP8tmXbAbo!u+Ay(G{hL%*vG*5WZ4qLtPj=1#rmTD*q^sB|YNR6|Qbj zWTF1p&o;b@B$J%PtKMU-4y3y1{Nud4%YVtA4TK7y6p~Fkcn!Ii!~X z>#Tg%NY~D$UW&-raIU#C<$XDom4`H6Gog6G$NW2>{&!E2zUbAHygPnrPzIVjw!(|! z=$Lf0fSp9Sn<>%S*G=iBI>m<(VEI2jHn z+nGpTP3PGhM$b=*u54!2R!kZp%!^~Hc(7ti(7|;d95}a1RBwUHy+h%ouWYrfZlEqO za(${+zq}r~4E?bPXFHhpU)d>=N4NIPD#)dP8$sbviZQH}Idaq%G&H)&K8DqyTM@%_ z(wN(jE0S$IQPzZVVzjb!sXJ1;bf8_@Cam!7{z8Xkj9DdZ9Jv^1pA_ zbS3|YOSb=VuGcd{kro~+XQbBin=j%me0n|M+=16aZYx%Js&nVx|Nf8ucu>TdPP-Uo z$>;AWPiE9y0M`7B|<)w7bXov7xTMhq(I8_j82prZ6uLmDsnZFtG`ioURO?YY>+lM37F49C zL;?No*35}K0zo)jyQA=7Gqni2J1o=&L@~~Dg{cS?X!zN`V;;4UYOr4HkFM1k4@WU( zBh8&m*##w{Z$crPoy4{BYWH4@+X4SeFM@e9T&7t||5;$6(8D0Y25IX3S9G8Fbh*WI zqUOvlC7Z;L#1PS?Qs#xp!=wE;cF1JG;v~t*6qSX`nhp+AvbgcG)Ff*5B9vd_WgyMs z>W_)J*wp|PhnTBHqnH)8x@fDy+UpEXisiSZkqXD?+NqXDTUF?qsk!pO=<$!0izUdV zyjG~fokO3JK|iEioXR3VRqi$Cffyp%Pz0uJ#E_-_wndK#+$q59_AzsQGDY20qk_mqRi; zU`w|kx4rzMtKS5B$7(F-o$lswnr1^e2+Vh&Ek-+pE<;_!V8eIR%!VMYaKB8np>y|d zJ)h`daYCO^t`pGksZ$jGV)o#?2iUOItgbE;YOq?|Loj~#skePArnaT#=?2@_9}^AUw|}HuJ}q_5m|d;1lz}G^~EF(b*z&fcHIwZ&$+*;KwX z{XtVh-6S5pJ6E(HC3H|ycn}trxe1wa*)2h17F0p>dzgKsfxnCWX?17MlSJ;YNStX+Kn)Pl>!Q-02 z)8F}0p&9D@SIF_%Q>T0Y<+l_4-COdIGNtVnNrCv=z_q@hv^bsIQk$eJMoA9r3o^r| z@LZx?P0XsR=kC#KE-VXp%aC}dhw3ey!6sP%yshR7WuS<+=i}Ji0xRwKHiIcjP}xjDqw zcFrjyHM1$Qz#7f3wZPRPOR%d@HtsIu#H&p$iEZi^n@TGG^!AeW8N?*#n(IKc&UMc5 zAY}vCed06II+yC}SL#sY{wxnSm?5JxT#w2E^R_oq*sCmAKL7oGt0JMnz_vPAT;X_g zu9IT9ga_#*&_l6rG-J896zLsBB2xNmWs_nqvv>ZHBk zCluP^eXd~SE3xulAX0qr$);R}QwaNUWg4TL*RM&6-xj2{tS00WlMsP6=FwPl>wxHS zcY&qm@Ae~`e9MhyxxEKXT@FKknui-jI%ebWg*H*eWRVcoNc>Fd?iBxci^(laokmtq zKo%zRS=V8m4b^v=K)NX-<_1AR5&OiZp=xxZCe^YcQdwYk0@-J>*}?nEOKu zR?w6e=AkwjjQq-L6CwF!+9kb2GIJ748Z~3IJ@$-Xd7(HN)a0DTzoF~MvJEJn#^Y`c<5)YRF^&XTX@G#>?$2>=azC_RGCOP zO>=%f3u_iNhKKy>OGwo3;oFJ8<3aAjiH52E?@(#L&n&0Tl#eeX+%_u|eeDprr(PP% z&J2ywAh^)o{KGM__^rzP*VNo|xPgmZJ#=MSVSF81%wucF;C*`5)I@X50e=AhjVG=S z6*jUOQX{i>Vfg7spI1B7C<3nJjUJ^~COD%ZbC9vEB3Gz(k&5^F)hLoHIpdIN#zI9v zGhxFs{`Yn##r!DjE^HaB3?emE#pVBbGeaX69mX$skiYN|!O$|!@IU#aH-({%Npr`9Laq8a z+iTro>K_pt8L}s|+fPX>G5~`i|Wzwo4;^a?Tk{0Gum_bBwtxA1#)7sq0I1 zteZ)GtE%FN2anPYX&MZKk(z-n)@wQ^*Y*EhC> zr4Rg8X}lV;M3~M^Br7yV-O9r3m&r&yS1>GBMl~m%{V?7W& zZY=!&_*rHsNifNWvra9P)=ZxHmzvocFCU6Me0RYLIYf@7Px8{_gzBp39)Iu&G-BO62CcW?9p4WRFDnH3CtT&%sF9Gxu_?d z{63K@?b$|(7d!39QgKa^{tl$~WYhTka0+D^7zDY0j0-Qr*rE&5jc@(i>{AlTlhYER zkNydZ?nB-W7VTiKcy&<6V&^rL64}>_>%8?Qy_}9rLuhywd$heMoV6y=ZPMHP9wbnwH^FI(YMBS2-Ttq6sf{=xjAwT;YSTV z_EuXvzlUg^#l&?sOS2MNg>>%yF92S=B(k?UEg816v{0+j9GJYn*{Rxor~^3o7Ml`*$8Av~1(d_J#?lh?%(GTOOAg^&J!zQq_*CS>Am)0g&#U+UQHh!UUTkH-nJkh;SB3_sJW^U&t~7XKJUgmhjj^Ndd~IZ zQGarL3PDVMoH;&^P6os=zb*7_E()tDA6dF5&!f4* z(8e78I(9xp$Y|^mzS!t`0dC+LnFP%Vgf-sUa|)4xJApGcxJEfW^@WtGE*RQQ&#DW> z0EuqOCM(aJkAlqNf2o}<{>*{+CuMb71#)J)dME%fL=%G_#2#swNL1x3FZ1Dd_2l3#X z>=_Z&(h;I@<<-<2)tmZ&piN5JvQNLJxLg1KNex9+xsKZCA`N8HRRqegRZ4WQ-r)Ftx z^Ho5FIfi%t?XA+6EEu3b6U_Sb`tfm&+>hbqmmf^&N6dad^{jO+!+$6;2*p2VYUIO^ zAmt5{Ujq6@wP~_{#)Cp#$1?m(aH5;lZGv3`taqE(6@b~tyQYFN4>(F!_5P(; zi0m8UQ3i;QV@5=wMBL$bk~55Gl4$usig^5P@rY5YSa_g1(pI7QdceVbl2A6${$o4;ZAw*W7?J01^MIJLr`wLy^!2N}r zkD@GrQ}>*Z7&$ZH|JLFuvVKQZ%YJxJd>?P?mUVN>n)rY7x8rKm3qp)nUn%+M7iQ_7 z>QMh%EPe8QTW;!LrIVh6n|n#fV{63x1cgLL93Ok|(@yL6{r)X7-sFvbMnFjX*JdH>v%s2n|igib}lO2ZBzjNzDhLVy}p&xb!t0^bfmFW_MNu!?+I^UyT48Fc> zHHCX@ zzCMW--**K&I?gvYH&;vg<0NtiFs(&dSano&d%diuvT`54viUd9gv3qmg@y{+V7FPQF{pIDmgKU1Z8Z@}B z%s}SqYE=voMy){z@%J5{FZ+n-)s&Ub!kz`q3jDJk@8Au5D)qDm@?5km4S5z9Rl)b& zyRfj3o=)6rmpAOd>vPz#ya73<(b_`_o@E18(%IE?bRCW0>FH@OEF&XuxsSax^swryA8yyz z;`e_qTlHs!U6xX9H2#T)l4l(c zxII#1Yp{P;W14mv9u*c8w4iDp8|vE7a;|#xL{g5kF8pZ9ZDt4y4?byHlFS(zc4Jb? ze*Q=6U-t;vSNt$WV`E}YT0Peqd>vDu3OhOveLzkZTV4}nW!DMgpwu`&9@l8XYz)o? zV<+{|gM)*Yij=EXj_1_rDk>w4noIPyc6RiMm2guVMhK^Q%f)BDb_Jaovh$P+X$OaG z{*q?92|r#%7GOGSK1ZrlH|McO+@9<~_u6oEMGrj`miXuqqWk~|Z= z1#0%#9*e1HKI(gv&`#Fbpz8pC7HC=g-yf2&GcVawuE}@Nl>8f3x-sI%5w{DAii#`_ zmfKskJ!d_3D|O8B?$Y^4Gn^@CYiT*J3jOnXh^!n#qNjs{Lu+zzFd4U50K5$LbuRa0 zDR}8}r=2OR2{jMod~p#${Ihg4dkGsWV5a(&!Jv>Y$2&OFEkX zm%YhS46ldoU&D~${Vk+YlD3bGC@ zgLvEjxBQ*8we^6yii)@w($rh9+b?+@i$Vrx_nm!sjr#&b7?aG`hmw+n0(k-IKMWF`D zUuIvM)=es-(lIk&0I_jz^QpiJUO#tx{Zjt7=EliVMeEg>x|-{xnNn3>Umqjb!z7`$ zxp^M3nd$#$@nUu+$NyyiviMox7v~#OzkSr=O-wfRnnCje1NPuOz}F{0kwKrGw_cwI z8rM5}1|LS42jp`a*U^|XdF_3TAm!326=G2Sz6Cicm z&Lj%?j72>F&3*NU`$}Oi_MWLhm6bl|?@H$baxRGjyI`DMFa;>!ugjzx z&HdZA%#XK5S9^BOCxLmu$G5_K*=M1OHkz8F6LE2IUn;G7Kz=|TNonco)>{7h_T;3m zir1=p9u#!(*WoP4z5TUT``@k9aXxIPoVRZ=%MBKCxmUR52@=X{F zRtUf!_nXonoj_VdcvW4Zi9I#H#+ujnntU0L3BFi;bauRrsT1m`E#uIypVJK z@LswcrFLl^bzXnUF9DgY$?Myt*HBkCi+Z9?5k85<5!ko}=MSM~fFqFSt&Sk$bhmNL z$q@H8i#zl=+b#q$x3#sGCxTa9L&Jqe$Z7h#?{S4zYfO}Nw=N9#tMv5wV|J~mvRx1<=nl&uB9zuT()X-->=r?vj)S5fv@~^ej#qDrKF~}GCu?^> zm;rugJ8RaY-DbXuAG_L#D^6x+i=&!(^1U8{p944cOY_yNdOKN_{n24k)0kdFgDcIy&xL&Ckz2;%>b-S^#_<{2Msa7u`DW z%h#7pM&4y(H2HsUd<%|QF|3`gvITdySwA_bz(SurfBrl!+c66YYD}WP)Ay}K(tYi> zhB*)g>TJo1I~L1r9~7#qs;(O2%_Q2G%vyJ@8p`a3GwE@Ab0=#V-izCy>kEvjvnpd= zwMV`laujwG8aJ^;TM`hQALJ5Q|5=~4j>+_Ie}BK-0V}Ze5Chr8F%A9MqJiBNd;-E| zDd0%2{kC$xf{vIIkCZ{?)wWSht}7i@KtV+^vQ566Jh)b*-l`HxVo3n%?oflq0EE4PGr0G0K(PW%L^upZpK85TXjc{w_6&?z4$htgEwVktzV0a`zz?Dndd3c2Odj#GWwLKn8P}XO8c!TJ$h^W z-X@|5xi+~9wK5`>baZrRl4W%c9Sy!m)D5Vx$3^y~hMgF$vWLoH|7_Vx<#5w7YL51HwQ)n#Pxb^nT!v{ro1LjvI?JZ^y^Q1x*!d zWX0G7&6Ep4xLf|_`gc9o4=7hfUvH+}SvN3sOTqZVXUMX(!=FZ;Q=agL>ZPj6Ce&*AN8W zBwadyRuf+E;L#hLn{R~1Fte~^MTP#ZDPz+vXz-qQQy54y_m^;Wb}skeeB?Ic9eJvz zqB3{W>*%fEFFuQvBijKujOd=2(D6-D+x#~|y=N1ocCui=pZfH2Z=tE4TIC7GUch0D z(R*toXV`VSzV%d1LnC@5$}#glU^ z>Lk7;@Vw=wtfIMCS^r80g-=Lb2u~bwDobgT>`UpRlG*RfO;|pnOv3>1W+2EJBXCr?&Vv4trx;f0c#+NBhL&q3j*azz+MNzT{o?2ih*e{bIY zZw(+zFSsa>05>~ND83X{O34e75_J3X3mRvj)Ge!oGlNkn!pr+KVy&r@){hz)&4FB5 zR+$jNdgm9C^El^WBF`Be<-7jJTa^k6(RUtGITGewRcl5{Zcz$*WdAwsZm?n&lmws^|^-%OC$1Fbqr{DtM!pk(i z!l3BjJ(%eRQk!3V!?JoxC{r#& zFVC>lgBiXK+|^pz+K+lkIVm20hMNJ=HLlQi8|_N*2LLxCo}b_5JR!LI@PgY$Xo$ z`r+NK{qA(-*mFNY4WPAz3>9X7|Gt*^(5 z?=dudGC&l2h~xmsWDhtQ5PQ_BfsphVtvEM#1zSJgm!JM6!Pi&&k(B)E&&kPy0erxL zaVXb9x<09Qix_Cc&4M4j7kC`LT|RQ1vp?A-Q4SoB7?Djt4>>7m0GAcJxPLl2pRqsA z{-@(WQw0J?Rq!*dxUOyr=#2iYaI&l2;j3MH9tw%1q$FI-P8`=pIZk167!A_l$dh3-6m!l5PR(vx?||23IV)e zC+=>PH?;Fq&3kK~iIRYe8`Kf?BoinH$?X>JHXfA!QJ2yH$hO!sE@fqedk`Zm+w*T*-yEG^u+4%LxN_q)D4;Uy>P z`mkn30zh?u&oub4%);Ng3R-^Kfj5zN-X7*v zRq?_0A%taQKpd$^3x2HV9UPSWbSk$X$ESBukmf8_pn7m!WSpHXzId`T9b5w-lD)c+_V}i&1atG-EiFNp#pcyMqM&J7HCI2sGkPJ&?B`-> z)|PJkEUkJpSt=K>KZrFz6wtqQ6jEMJm|`=~0h@||BEX%oKS?WQPI>)|0(ghY(b#KI z(Nh8=?wbYK&CM5gnV6V1txG5(AMXsow+qr965jeywy=fc9_sHuza{hGUBEULY3>bh zWKixMAP7g4fmVILs;X+}_XID+i(e_r-t##!UxRX;qwXxMJ$a{VWOT;%Gt`UyQA=prCF6Me!D4Ovv^PIOAy}dViyuKFwk;Z}`&( zq+Kfz*0oy(s+Xgk>+|B380B|)VOW1`@{%#Ht z)ju}3rD_I~v#q?sTOZoW zZwHZ)Zf77QJ&>$d5vH>|PGV)z1{$c5P0B_m9$4~;92A`?6vvPXu80rp% zJ}RCo6!_w@Ck;e)EpIEpKKX$s{QY*7Sqi8>xq^UyA2R_3^onFP z_=#f%P-9+U@HL`F=lo27q5+y@1ziC&vmUOAfXoS<>^GsiEnbkW#;qj=A1ef};Q-Oq z2M|#~oK!dVD5|MRR(qoKR7u{JjrYQdi-mJ#% zRu98>f?RfPaRx`+D=#jNz3tVSZFpk`Ecd2=cLEP^B;c?@Cw28j(cBNsqy{`Y3XoRI z$B&gQdLNX3{CMjGW|!G$aawNLG(R>r1_k(xq4YRVr(X~Q8zqDt-In?E4n=aXo^0Oo znrwd+$g8gk0BTjGYNNRQ`KdmiBrIjAht7ZC1Ve7-^Je1^WC}oi=?4IbX#08-tXxV0 zJOV_u>h<&P5os{kDWbgPD{!_OfWmrW53Z7U_4$tmU%8Z&)DpxYpf&O?otVc~>v#tc zT?fK#Cu%UhJHQ6mnNFg{u}VMtfGy4@Y*W4@rH?yAjIGKtv;x9$Km@84ka73hJ`fBO zQwupUm(c(&e3YM`@2!?D^5kZ{AW&rZcgRY1`p$)cqrF}_GhiAy%-Q&sYF ztSw0H8P|~7yAk0Y)wlx>XkG!zbsErzB68joC2DPG6T7ODFScNB;M3FikavHS;r!f2 zHAQMmoNR2O8EI+ap-BL=X$y^{1_g+GAMVW)EkKC8*9A1=3CWu34auY& zhN`9Bc$RvU7t?#q1~y2u>HEu__dqM%uW!L}dXE>VrGPwQfnKK$BinwTDh8Bs(|fl+ zweJAstn8+Y;Q&Y+lskZSm`w4icU}mue8YMlff*8a_VU6te*E|`mkS;Pjf;Pm0j zF)*GO$&pD@wYE!fn_2pMxH<<(E7dD=?AxUt6mn@f-6*(J;bS~C^;nG(7D5mK{OxO` zlK1x0m}b8l@e$A#%*QYAfpMOEtz$ESaXgb*XwpEKi1l;X##uC=} zrpQL}==UpUKpC?$^^wvqIzL_AAMp`p*3V6UByrHDZ`2att1UYf&;lsI4FYXtGX=oq z0%SRM@GpB7AjepojmRxze`;S=@(d&$A-#nc@;L_gT5Yhe@9Z7rd>#o2iCUQd1#nM5 zS9ibjkyKhNZm(%Tl+tZBU6=LGV{3<`7ev;`4r zPVr{-(-hmDCUF|K%tm_$R?k<=9iL%Qt$fo}(G*{k21-;_tm?KivUEZ5L={fx!yfw9 zEuVF9K)q3$YyK0?o_ZB6kOgHV;D}SKN_aB|IiTKSm?<~$(Ju*@(!mTCs&qt*My7KNE!dM zXxGmQ3xo9HMZ6cxQjwholgi=FEM*7X=VZfqlbnA=7lLbLGt@*ch<|##N%pjcc}OLh z3D}y3DfW*%_4l{*+Pf43Ei|r+&XW;vd@;ibzwfwBOb=H)nTN)hh_l;M;p0i?$_MIO z+u809!}ZpT3KdFVmT#(2anP zns17%7+1GSOH0#&lK1QT0y5s6m}!#Fqa#`|8EBetWcS}X$XrW!fO5T^A@hdXRT66f z5CMOLhU?PS7&Y!*VB3O39rQ%zfp00=;9_Ib)8FxPB*btqMN4 ze}8f3qkqc^fL&+Fhe_OdzeZ1Xi3kgyl(H6>li5psV%5|guQkZ6WvELMGufBSl1l;sZ}}g7nnXv%9IPzzHDie4%nEbqodDS2g4%g?p^Bph z0Y^7-`@3kY=0y}fFk|tLV15QD65`JR0F|Y(N)!L_U5^q&t7YZ|s5-Z8GGix%;Di1Z z&gB~CAtBrs^`1X&%U%R%@SnOEsgx!rDeQXUHD{w)RY-9_S9efuwuxB~WY!|C>SgAf zi|6b0vE*+tfZM9*qg-uugM?8q@~4&_dy192{ft-c603<(ACl4iEr@%yH#zT?^er=1 zwglL8f_b#u27UMO%L;tG;YeY7At19&@~&Wz>$KL>H0~R(r89PjTmSwKsIRe8nj2dS z>Or?vX#q$_|EE#YnWgiHxiwtqK}}xMi@8`c-rNHJDo)}j} z8odHqzMdD{N~#{l1=>gKrHma_@j5f64YV3xwx(=9kc~SJ%SHZoGlz~T1+U#RQU`#} zu#OEdJ51Pm#(XCydNd~JIIrNBe%xVW{%ylP4#AT1s-OHzAD94KD&I#oOakq-;D{Y| z4NF_&mr~NV45NsM#rFGgXub=BhdU&!y&Kyzh^#zqkyYQrc1icU$J-bM37eKgG4{Rg zN5ndw8Yii&A-N(_>}KknG$PBFc$Ga289J(lY8=K>b0=4Jlq3Bb;Qw*Wsebs}QCw28 zr z(y^Hz@wDmoGr6(&H;KI!$)Qc&+DJwmldtcS4pN*0p{6vQ*WGGU(~(dTFcBA`gn&%Nq7q&Qrz3& z$(El8NzNv$+2x69)%eust?`p(ZNm;Mivh}%RbnSg^1q$C1Bf6-u1}FVW@oOxqJr_f z+*%D{Z|^Ez+Fw5)_xe0j5UkhatUm!BR1k|Bm-A6~_g#0*9z)`(|=M|=h763=} z11>b=+~sK}GKDkdq)@zvWh%qvd%z!gmEw});*P-o_BotzbL9ae%l}ZFR6FDM#H$RO zZku530#%242$yl)0H6$dm)5b(nCf=sJA@~S?>7*9qP!c&Wzz;11gn*Og{|TxilZj zM-XaK(QQD*pE}}J_UJg~1m;foqogjkHPQxZWo<5j*y3*kMwOS2X*VMpxt_C|jtkK) z7^*yBYwh<|Jq7v*x5&Pt_QCJ7F|09svJYMLh}d=)4Cn)Plcy!Se>Q&wC`DqugHOC! zc({pbW2~LGQ5CH%+xgZK))91~?%$8)VH(vpL7id4N5?j$79#5$wf&6WE?Cdpe5u9+ zj-$d*_2Nlg{atN&VqvJC6!yLrN&d=ls-EO8Ad)PsbbdYeDyvf~L*n(@G2hR~%nWKj ztog+*|^AwfHCKz!b$_B7XSUVgTb<47 zH83#bav4LIegVb55E0+%Sjq!?H>`LExY zw{Q@b9&$ex4u{7ub#f9C=BOa8VkF6`nGN=#*kElqNPol{P@jlhS^6k+c%jT-&j$PQ z@wc})c-z5KKMogy1PBcnYHeo#0D3VAkJ~nkEVE7%(zLKLU{On*ARXl2K!T0bbPRI) zx>Ndv)_iPE0r`CaD8vQ*m`6bepCWRX_cCBqwXJJcH)F|hUfet@(d}Dv=f~UQtn05B zBH#TV*>(+H0iv?&YpHtYe?(3KU3qyQl?#eu@j++VNEXnksOQc!J=;bdwoUnTA7 zx`pjyxWF?ScBRdh8?>VW3vd2)AVc+6%T$;& z9;xzB%(F;>>|X<2kwMeUz`%#YY#?OYkAqN)!77^NRJVq3R5lu-4JQ66E04fz`XL2Z z1G}W>P~os7j-eG|R|o_$?j0Fc|IBe_^u7NM^Q)$Q;KH2&T%_L*AkF=Eta9X-eN$w= z@upTT+=18Bc>G^Xon=^5-Pgv45RpbyIuuc)Ls~*X5J{yQY3c3(6cv$@25FFx8bUe- zgYK5@9AM~yA?Dqq&;PpKuN+S7efC~)uirYO68?x6h(7~YR>y%9;^4~cJ&)tH+LNpz zN&TCm8~WwTm&wD1!Tpu)zlzt-ch^*!IO|=%^d^4RlHyOO+PMq}enw{65v;gYJiq>OH^qMvj2`cxyO z+)ENHS99Nb(cR_GySEU3{?KZQA<c`Dc7hy}GvuOUlis_UArMsPV7L5Gr z3!6hUo)aQgf}XK(Qd)tpSw$ICSUG44$bFSbRPV$U`X3#HO8~M;b(^ROq?DDLfy--~ z3Cs!zLa-hawzjrb0E&Z{@aD%q_SSRSFS4nWk?<|N?bmF~UsEk6Y&D`xyX?J+ld$ur zTFkTfTj&-w5r|W-4?B*DOr)`0-xh(D@oyfxOlt6ip0B!=78csN6tePfyflsueV6km z<*IISKhks0{EkAsRO~iwR$qJ%C-1Lo;^ro9YkiXO(b2cf12&EUyn0Qo9G&mg)&H!m z@b|;#zsJ(wc1So1q~3kWY1DWgsY6m~@hah2c@Lb!${-;b`5g7y4@#9AFaOECHNmFH zAx7nv3B2OV%Kq?9HP`U1(2WFvXI?fE8dYf>O2?dtj{55*$EB=!f6~Ptq*|Q4BwNLg zR3QZ{TX?-fcl|yNKZpbG-~Dm(1AbQDD3D?$9uaZzR+rXq7RG`-j@kIU3)IqjVh}z1 z^>}N(Q{M{rQ3GPxaBEbEDJcHBGQ(wEj~$oX{is_g9d)MQqpif}N}(ppwzo z*%{5#zw-SpP*4oAFDWmD4HM1YEou#s8eV_jszDvtkaAx3>X)$BircG`-31Jb#+z4i z6r1(o12OqolB+`ig{(=*(|HasC=XB^ge5RBeGdTFk`VN13BtL*ILOX&LsdZ4pq}D+ z+m^TA57y8gfWJJ1aWWHaChuOQ4f#m_@&KMQ)VPNE!gw@K$*YO2akM+0S({J{ytZEIi0 z+raKGZvwR$s@Hv330e^dQ)!@3o@5$V79*Bsxno7k%r#8f`f|b|-P-bznuJkCCIMc! zI?1!V+p(uXCZH;Pud=~r5I6(_Cl7#ktmwH^*VdlofDG+Iv&4;+8(TyW2DL9&;(Ih~ zzKQFbo11y^7F@}up~_J-{EKzE-ee%#&niu%G4bF{*TBT86D3k*&F^<{hLS!6u%zh|ip@nr zthfENKsDmel4etIVLT87O@? zAe#gVmFm)`oocGzrd&9CDZ%siu~Lu_jIXV%q}hTH8N`ooUO%}u&Zr;LGY#ZlBW;4@ z^alyq@F#%H{dtY*9`*^IU_nU@4~f#;2OxYH@VW6dFE7BiUlNsCQdsEf=IT1FS7}w( zFLlPXdW|jo>zNC3wrK_gs$Z3kP4x60BxhjM!e`EID^{>3Tk+o^^oVG_*mgo{p%#h0 z+yR41lT}ZiGXh2VcK%`Yz`a)Llsv=rIe1{Kt(~0_?R~D{?q{AxJvTnBJ&0Mp5|y;j zHw~MX22Np;w~5VLl zfb~$N_45b!P6L6v@0~nm{M_6I%bgb_*Pf&mVIh|90xi7%J!ae~bcq*GmuhC8)V*Te z6+!505rR}`8$tnvSUQbn3vCaI4M}!b>egzi+9B!sJMkAw?NJ!rnYSL_pMg0PPwxUS zrL2)8H^~ztwv8b;@R?*&mmq(g+1j#yJ^f*E;SC?C6OZP|M~I(Yo*~VZDK_x|sMM(f za+hc)BaV`l|H9FRJ^+mrkwRLp7I;vZBt%0ug&9Q3H+m{E_|7~=Pj2Kt( zvs4E}0P?^O53|A6^W7g|OEK-V4Po_Xx~7&drzBYqv4oJ7g$08KO3#u(iv9^u8?Pdp z&m2jJo-h`Sn*3yq#r=@xQ2t0-cRsQC81 zgRD$Ik1|tUlP_3$XggBC3JM^BPu)O#kxBJTQE}q#3l0*B%^lzU&ihl`v}4$M?t=qP zw~K^(E#`ML)zx>4z`Jr2P$B+sCF^(?M11{))~yYy;rH7?5#t#RD2ELm?;MA!Ke7cf z{19pRhP6S>-lQE&@!mKq7KVyje~l~2@dSEyY?PRh5vXr`!_<(X-d!>82yhOp*Y>l4 zGi2dm2j#0%hJ)Y%A4Pm!5@UnrGpro>B4Q$^OAe+yVx!uhE_ojy$btNpU2))>ANxf2 z2(pB2K>>RQ6wD$)9braR13RXJq++^r=2!j{;;<>lhLX{xpxmoZRaAPjMn*=Y3;YVR z=^u$adH?=>6h~#C+0a+THDm9EDUV7OUWSa@uXy%ZK5 z0=a7FR{(>wQW5OPRvWQR1OOV>R6P0GGIL z?9zIZ7leZh^nfWz2C$F8PWIEVY3mH=7!-KAoKGJ+HTgiP&WqV^i(jYbFhXROriS3E zN8iX}z+HqjbRa=VOYPm0ETfs-_aGRTQ7-can>B<_iZE>)peO!m?jCA6^)1 zYu}scoI#_}_gIz3Uf)pNcg<~Vd{o+YUg)>z4SdKg4{3ZKAl*USNl5`ccEJb=!Uw*< zPR)R7h49lmNI2#az#b}Sxr>WX0UH6eHMJ2*jqL%4@{$hX;6+nDb`M>PhuF;r$~h98 z1UTj)*`wHo0PqdRDwh{y0P^i1PG8#C*ofN0Qc~;so0@oLynpYqk(;OrD&esJ`S6Qh zxJa&7Se00#GN3Hq0AuxBY(tVzH8o`iUJv#Uh-xEuu^jis-TFo?rW?0 zE0&P&N$-*$>f4zaZ`23X`vX{d{wqRomFy{Noi>&`OJd4-gxyqN)kvfrGy$g$VIg3D z4wxd`C9s%F^s6@UJWT9Y1_w?L!OJQG(06xzy*LSz?TeV=osImA$U4`*T@gS|$ffQK z1+)mg@3LBlYUpy=ACc*7f5HGVFz5h4aD6jDvEId^K@zha2Wn@0368pt4o6#or!q> z1^T@N5PdL3JE7uC#70UWP*ins*@NBfc6xX~m~#+=jyX-FI;F`Ca{)E^=I`NF>a+`2 zK@fyWR<06qn0$x=brdfNj-uMexM;4uqobqzipt6;4!ldHHUEl6{~bf%a8Fp0;^GcZ zS9??VWeTP^g@lgC$*$R%y>9^(D-kSpx<$;}_}N)rHQd&(&+U9bmRzVeCw2YB0YKHx zyTqi)YnYz`F6Z(Us6D}khKK1nIXL>Bd-;A#QArW5`sR2yIX+yd8ByBM7s(*DkFEQD z1Iyh6VAjRzXPhg-Am;n1rakLEtfCz9uKtXn!;aFk3Qyu~Xn6n^Zo0{+<65?)?jFCenSP*|zn39GBCQ<3KN zFfd!JmtExZU_940BJ1W^co}or7?yhA<`R=rlO9H(~flt;527ro*-ku(Nv?A2KG2ncOIRk7; z^yhY&E!M$#(2dlX0X&7O08_hLjm?9G9_y*G-&*?@K-&~ygFd%7Xz)2$6$EtPRmQey z5Ln4b2#T??E~bGs3#Mo^R++w6L-SU|WqqYt8dxazNmv(zJLB{T#ZY z--ZO$^y<;MCJ><*CF8P3_rg-gK|TJNYrUA^5qRVQN-jVb{=opy!Zf^ISAakipjxlH zNH=qqz5@zMMuY=1c>MH~utjgWs#l z?5$;womct+^#Q=1AV#fbqv{gp!x@)1Do5V?0+nwDpf#J>X>2p+@sDG~p5AvWx__^= zwUsrTR=dBVQovX{JCC<<_enmD*hK#YY7QB&L_a5*8}48ZGj5XV=;>+Hxmm*Zq}&d` zDpJJOZ>gYtCY@F#y+|@+Wnd;{bohkW*j5ezRRTJkoX1~-{yA)mSwWa*oB-Q>06JZo zIPN*h{s8TFNPu?wHF+UtO+dd)DlNeK>#zua=MCwL!(o+c-zEi`hEklGC*pqFZVctB zbRsx7I3hWz`~X9JwgeW3y>nkbx51xJ6=Pk~Tbw$j=M@sYr<_Y)fCCMDcF}*DTo_D| z$c25WdcXK-NG=V9Xrj3FDwavigC$IcuGBltHSa_+N)$wS4uAmD$TC3sYxGP>r%r#B zt>vdgP6sC7=|jMVH8}?D6{-3Gs{qTBh5`yFeRmjGhe!bXG3X$eb*5b6CnOUxGT;W# zHD;iSQg?;Pt_{T&#urs=C5yT&=37P1$)z9YNG|mi_)2en-UhoGV)l_}wAN4J1`#6$ zFq1PUi($NGdEzfX;ap`n{kQD#CfEtQR^6x^=8F!%<4R|hu9-Q$CJb(*dGNCE6H8qH zkwiTMh#jS0tfDjBeN=yo1+pzl0sS*a;8>3lDas=WZu0Tz^UBIfW^#hyc92ncZ((tI z&_na8hE0~ssK7F9v`D7`U|ulgiv=FN3VgSzPaD3juC8B5thf?2`pV^s%>SGV1H7Xl zVTxw)<3<-@#@0A&F4;R}^+60!9@+Z8n*&eVFQF`pZC!E+vREqy$c_FGV6jcuHO}9o z4eArp=>#mE#l{VO{>^(|i**fPjn8BG>7)Z1{kpkp9!9)|0^`P4Q=;zRP$H*fRv&IA z08&=-(ew1q{{D;~CB!Zt?etPRo60~jf)j(@}~;3_GdX8-mz zmyV+62NAT|Z7!8^s#r}>Bt5*$eJTBs&;A1sUZ$PFoOUmUJW*iOg7EJpf!4H8<&EIz zriWOc62CKAuF13!kIhR!hC&hYEb4ullvVj15fLv1fI)SFz;`=bPRraw z)nsF{cn40^Rm-f|cHyKs;U-!F=I!$?fZvVsm3LopNb)CrB5YG6KLhvCj)LOwqsq@rPw34-9^Kf5hM z^jgtQKiDSciBiq?>^5Po5u0zhH#zl>>A0m;IQ6S5j0)Xz4Kyc}nq0z!IA-L-xVxX; ziue4M_oZYc7v&186{#x!aD3WJ+xH4|cnM+z1Uv6$pb#F$Y_Z8}m{_SfF-(2(^KVX1 zBP$y1Cn|9296)oyE^x;)uql?S`kdU{P(cR5^lK)bSEhYBUNro)^m~i!WfpT?Zm{l2 z+H*gF&j4Nnkx=q<`Mhny<{o#xRg6&6xUpY}4rt^VUtC<&`8l;8BszW*EJ)M!FW^NA z0$d^>m?gzG+Jps!-_Hi@Z-Tr))Q;*-ZIwRxL4GGEOzq3VjefBEL|kibhjjkP zQvEctS{O;9E($ttVcn+@D~)Z5hOV6{%qJ~Z8$qZm_hb*Su<@>c;xpoh`JtN{88J}b zCV(HL#%DEr|9+^m;O7+Eaek-{Ua%OOx}o8Dh?@T*)w>BPGGUMvK5}9i&6@+s3+z4K zZWQQkJ&du>4z#@Qd)9Wk~8Q#O&4XCC&LbX=Wu+F*n(yR?ssfweCTJ?XQKC z$uOS!by*Oa836qw#qwInidy$Lz>WiRKSpl$L$tx29}uY|@hxChT?rGWW|>u>y%2Ug z{WCZEo`4#V682BsKgD)cTGy5WEG+$k z^M%R6e1q)QG36UHA1zoUxozY@BFo1f-$CI&u0-}C3VAUIM+Nm#LDLc0%u z9IXM&VG-M#n82myY68Mpe@AALC#$L0gpwxwwl_#)VI3?FDQpE|AAk~6nTW^cNWjca zLc%Rx8WLYDdIWa}wx4|jB2U$5LlyfU=cfnbTtJ&DZ4ILi^aHY@8W#PR1C6Y(U((E^ zn6peSr9_^KcCfmp@Ae7(u%Nq1Vin+a3G9;T=WFj|`vd_C@|7!EED#t0z9c{!`a7JT>^XJhmDODjvLr3u3Yp6IVKjEx zrEgltJz-q46>9+E4Zp`FHwUOYdrma@Y;bXMo-Gx;$T4o1e@lGHKvo7P;dQdEEhbM?Cz^UmYaD1D z_};WG!xb|$y5-Xu!)ARAE~C%4h6U~gcmb4)n}7v64S^I`6Rv^1dWqaRlQ{=locxOgjn(U3|7 zvH^fqF!MpIgn`_Km!crt?D(#@!NjE`{3c!4;S1KM`hkudr*KAzl|n59Vg!Djs3Yt?~7&gyg}GvZNc$Xo&9heyM6#^ z>h!BSkUE5}L-gMzu4}UVau)qQ%MW_Dks-l|ZdUp569nkg5vS%gKA!|4AI2Z7HXkgs z3QF_@YL^Wl`P%{kego*DIZ1R5lA^yvZaD8=02n4l;%9duNg~^#p<5422z3Yhzk8tB zGvC@f)D{yZesGq#JOO2YI6H=s$79j3M1x(vV}_BZm*F`}&c8XNfh@E4TjL)@<0k*x z3oVeLyM3ehc+aH&U5$WZ;*maCpGIdEWa!gh6C8wN6D+EuafL_miA!=hRTs46FCHE$3RP093a8sjwH2JdM5KYio1tZ=ANWO$93o2J& z%_)R~CuOv%_I3y^9|~(W32>c*(M@7nYDTri9*s5fwfTq9k8NW`b?6^3KkU9vSnR83 z898245MnaHHC9~yfWKNtjaM@}AQ4twr+4$Lvb>|e^!gxO4!5nuJM)=}I`1a^_UB8e zE1RhOqmoUU$z{KolF?5$gXoL54T}PJ)X!l33QuuyL^Bg~czuP%Na?y`GK!8Sg#FUk zt`V5lsZL0v-!1tLw7aK>=h?|E_P%;QiR|4xNMRV)Z_%|5bs=@)E9H#-eMB4O@05eK zkM+1nLK#Phi@0WpwCO4&yQqv6phW1$-#yYsjM$Z)y@Xpw>`f31`u*O2zH}6^*%Uw4 zRQ{Mz!p9zsw)`Tt=JZY9>pkEi#*k)L=AXA;ETK@vJ?N=L@xW3zWoe84My;CBxRv5{ zRO|C9=Bd$B!acYs*t);*c2$r34seQk*B%euv{?>zsu3OMI?Y5o)vQnC50kanjrtwa zzLCH5f_)SoPw1^|&@>$(9~r$f*$>kI9qW?p%o^Qs7|HBQ=%lZ@S2THg`>23gLBHp> zj9$6|{i8`T%54L0^^4N!Kc(ijV)d`PvW~Y)AHx+U-wyonSZRl)|A>c|Vo%(O!*^>8ymg+{>mh(ee8cmYW{TpD0yTdhbkP z4nDdVIeu@h-5Ze{_*R;zu~{`dCP|4v7b}|jjTJZyw!ef~!#yH8>~dQFc=5I+lA*gZ zjoLgn?7W)lZ`Bw3<^2?QLNip+qaKIU=WT;N{c@G5l8eok?Q%}2>KiZ@zrMMk$(@Tx zl;d?t1K4k(7R0FEcse3WV|3xS+Qk2-F$Tzce8r@+scmX zqj}~U+}GlGWn>&^Z>_>5=b!f>XYWmBt`3vQiZeW7ZWgJSS4$L)Hw295wsh4}Tms{Wy|~MvJgRg* zs^yGw#0{@TK=R=&gJfdO4BxaD=29E#I|+^UQCharPi*0lAJd0#Y`D!O{S?~ya-cJp z(>jJ|FGkeMIy9VmImU`N8*L(u8hH;5acXrLhdr0hSLjU$N5o6b;kyRNT7{#k2?ywg z*&H?6X+R#K6i(&=N|KNk!sh|GvWJd4sWy-?=_xn&v4S}*-N(JW=eG(%p6c|j(+xI+ zRx_R-;?)~b>2NNNLqv#B3{9TguS-~ElBU*t|I8IJHE!$AYjNPOPJv2NlS>__c8ed*PD)!qD+odq%9Q$d6GYl zYKpEx8|tbw*|JqWr4(1p>}5J$g+H&_a;ys1MBlzubX8|2s%qvwWa_qQ3MFLQL3*C5 z=ZN1pzCq>>rv#Q zGWoP7^O@DBDqXkB>LRy%p`x>-0-eG`~;^atzwD2k{7Mvib@L;rx&u{s8}bi>oqXKzPZ$W9IM~>k}Z2j{HbOmq&TXxlV)m%Xi01HRW(a5PAzvRWcu|x z!G^q=R?!dT$3>4%#)E`&X@ZW9>myZ4J{m32T`yO-<|J7C;O@D?m~hR53B+(!U!>^m z>S}VFTH*JQ>AUA=)Kl#PP9wX*tOKXb9}JN&e~_9<@^Y^-u_$!$l)N@u9)8T712_82&gE4D+m^^nAkFkxOz9C(XtNO;*d4?xDxEtU6(T$oSln@cBMS2gntQ3b!ZJ}|C zvQxz_ZU{oKo^C# zqeSY=fukGf8PT8k#LCSO4)K?nk$}c`q{4GK8aD?kjZ7xpe~WoGl$dXOr3#XlYmA(mNyVXj%?P5FE}T0OHB%Li_oCn z&vsITT_H~;pO*(@l6|!4Eh>5Weoct~?cA-5kgiox@(<#FYFFv2&R(Fem`WD(%4axN zs-29#jsLb}81vgl*&0HociS_A6~ES2-*T?MX|wTWk!i*FefzxYK?w7Ao04~qq*~ZE z#qCrCAk=q#pp%Dto<}%ic%QFSFa((&;mor8MkZ}h2i3Ixff@ar;Wa#I6amTb5#d^c zf1gt$-8@ycbs!>5>dcbdab^6fXhA~F)SCEMpsco7X>X9Zo+p*_IMhYvaEG~B{x>oN zv=Z?xew$yg^4P^L9o1ux;0&3b!Jb7e!ldm#@@1Qk+93o{2EOy|g?;7s_}PC34d z_Gc|y--zH~cq8?d>1~bZlLDp)boef^As=tx_F$!_Dmlbb3jgeiXhiNFD^>_GB_pMOiOc!ncNweRRJ*%YXW+AYq091bq_UbR2x;0!jJ=wk+262X z%tU=kXWgzQLa`5kwLyElg*f=QguW3uKl6Gk=k!Xb-Hmm7eABGR97(iSRlT)gj!X!H z$-KMRu~NpF9BvtiDjxY#6sRuDIi1BVA*aPDx1-?gE*VI^z70CcZTS~=h7#5z0SR=0 zROS)5+#`Pd1sBBnYxWs?`fomZit!<{mTZ$PH=ByW5v{3Gsu{@V=6)%JzkyCsaCKfc zCr4O}$=;)~UZYJmwMel@j2ukjoGKz~a`c9C$jwH)JCmA+;riN%9G-6DvTet(n}gr{ z505u}Ot$NUjNHq_UcZZqZ9|Z{;Ff9fWFNe(4qJ!lAK!SwBW{y$U;FLxO}+cDAptkioDQjb&lTE+t}t(M2`_}c-Jiry%tyAzh^|SD zNll=7_1jO68jO2*)#x-t)ePhoFJATfjlwV}{r0&!4~`B3^XVEi-PG4U>(G7|AR-#S#(-t|4A=pDfitlG1e?& z|NR4G0$!eG8ua)-5ez~?>AJj3%0NK!GPDn?Z$V562Zk~DVmD|kVv_%g0WB1_*Ug%hzJ6HQRw8!(ac-kGY5EEE$3$_U_+szJAqC zwIecW$&w*6{r3paJ8SBeca4^D|D~1#_g5CD4GSNWT_NE6_m|JNT-CE2z_?8RJs}JC z^WSxoRTLq(cohEo+jgIU8JvMC)-cwizX6XfcaBesa|Gv+TJ@&mdDJKhGj4J47 z>_uKQ59>_^>@gcVpW`F<@&h-{n26XIS5oFK%0+y-9wL*eU|$=pcP`mJ)z#K%9N1 z;)_zWM@k}TIml`NWN7(}eRY+!j#V1R* zwn%!6RimlPcoVs|R-&=0ZqC=Hj@6@zlQiXQOxTF4Y5x%Z&yYDZTL4{pmsMMEGkN`)`zU_o7-!X;8@5 zp@+k9uaf5F6%|ewq2#{Aw|G^$g^om!4kr#Lu8xb5*mB%_(dstRUe*0G=LAXpMmLIT zmL1BYCXI6a>U3HI#|(exf?3#ZC@pl?B}f3j4GDtj9Z>gYs6X21m3#6bRY37WLbI}# zY?hp@{oMHcC~WR?g5^^*hK%`A|HPGN{_i~~H11pkc>qAcuPJIK;QsB%A(-3c5C*H_ z$~?cmeU+=5s$YtkwhGmm!+U*|{X%$+_W~$M8-Hr zA1HRe8L>dTaql^`OZYZy?MaBnU?ZewzrX7pB!I=R)N4V&7gN;|Dx8wXH?K9AR6e`A zaBBScN5M1Bx9$XXqZpBvG@$%LSZw2eufU_XiwV|hq2&MYV3&S8&%s)#ubQOUx~H=W1i_ zsT42fE%BY4b7SWpAD_+j1dz%LKKE5_kX2NRZuiK1XEL#De1su&UHkhw6c^S9XVjRW z{NMQQoRK0v10H2<2A-d5@+*2e64MC#60>?I3L$f?1v9e0jQF*`AJEB&wX!UhE5wQC z=$$j>=i;9Y&ze*2;JltbSqy;T+=Gx`Li~r@9b<=?9o8a&z61GOu~{>+qnyjUE5m_5 z|ARS53o|D)T@H>wUMAQ|o8t1}B>kH;S5=M+AV}ex{#z{_VinB0I!q_GaFPUA@OIG# zQb*aD7;G*Pbn|CZZ8h(QK=MAV)^7le2a8Fpaocmq!ban zT>Je39Oj^qwKcB~HM#F3Zi0W55nqntqVYK1SToK`@FE=BCtHS`caW$1K>w7z;&l%D zJ4Y!Mr^Epb7e5pTTkz*`ZU#yFPFXJ>DoE7H{@nn*Lo10FOENoGl_5-DaCDC_M(vE) zkD{)-K)^~LVvfZbKK>tLqv?m&3;eLLYHHCb;Y zK>sX6X0kV3((5QYmzwI|;V7CHyY0Ke?`j3HPiyhy&DPcPo=_b$+iuSnja%1@;tmrk zI&)lkAegB90G=~U^KWi|zd~5dt>Swnd@$*^Y<)u1xsfs?E%efUiCdV46baB44UuU( zwEq8Zt|CwR3*()bb5UXo}$@Um-7aCh_JyDQYc5VPHB zpn(L!vc$nt=(}ZJGp7i-2SF)8jH?@0brw8CT<>FFK}Rj%b6_R)hpzw5cK%WDQ|Bjw zXH6BLsgTd4I1}V2&S$T|LD8a>Q0sJQ3Hv`~I<;5+j=1+Qxl+}#f!VnN=frvV>8yE% z?mU;;T-|Rw%zi7sH2mz>L6ZuS<0zoqLvPvo)TB;X&kS0+#F;23h4MObTXbBi#FPyA z&F9t4Qo~Yi`oO2F=bTJ-{9)5p@Qdi~y|ZakqLz$0M3L(*=YPvKy{k|Op12**(n(-S z`((xX{CU7EAF{1R*^@wl)>Xmy^IE(LlgDLMcUM~~#=g-5R?A1@Gj}<~>Hq6=<)B@> z#%?HyxY2J`uxQ+Rj5~wIEl@s+VaLH(2&#T{K8&~N+~+aI1dJRq@z7~u5oHAiSC3c5 zBY;1ZVN~h^LZ%S08L?-#18bZo*B{dc z1$v)caRS>46JkIP9$jWk1%w2Lzvlcn?&FnlReaFG%x;nUcTeTZ8^^vG_YV}n`9o4% z#MzD_EcB&4-uHMt^L_o?k0;qEP8}j#({EobnS{H$wo?aZZS1?X^9VZ~dO9!awn7c0 z{d80b=5tJR>8^b$WufEX`Pk@)%2yGFbU!Y6SNJLc&pdfSKf#Fk%i5V1Akmkcr@o*BD8><)gJHkQ5DH_eYdEK^i(byT`Oi)R;ryn6Lq-UG)=kuj1PtMQaH7>8Bnp-P=)2ZkmD~%yz7skk1d`&E91vw_YbjgZu zI=LeZNn3L8-o!3=XuGaq7wJ0rV=-puYBWw53CUYFyfRvM%Rr+G2|=2CP}ZfmIo6`N zDqsf0t8I@v!KR%n%qNH0kEw$)9-*_hFlYB2#ju0FxR-Jfbap*x#imc-SyvEe%DwIk z4-IRvKk=nmqrC}6I6@PUV%cmiSA#7&Q?WnVjhDYF)@AIP^VTr&3r3EW_N65$GTn5q zB)#@b|F?|?kQEc;x?g=LxM;bKkmK4nt=S-R+9KQMc7cJi$;h>?{2S@yYoUiY)ioDx z<2?+T|1;$1w5_ZDtzBNL77*dHv##dKeIYSEt$Vm->1y7X?VD>RvA4%l13J@Xb#?LF zTZvm%*WBlsxNt=IthSWi{__QBy}fmr`7Igp7V_zbBG+v^qVY7F+y+k$;E6(&o+V`@ zMFBW}3={Nv^4bINY?Hh7Twd+^o0yBe{9fETs*?O(D5Q)wX#1$^-n5ccxD@7WhB;l@ z26uP3zljT2xvmmyXq7pqlKOC^hqa`Cd6`AGBAl6VTaeSv!UGcjYJ?XOMiZ=Fw|XuV zT%P*&^>b&rY0HVK+EDOqYHw2&jrJYVMXy0Sg}(E&S_G+-i{4ZfzVg6 zQ7LJ=FL5g}?0hFUG4b#Lq&uMYiI7Ki%{JIUPjO4Omh5mue(PwJvmE!QK!2O~&&%g54s$~$u*kAy{4 zBOyk;Y1nBnWnrgfWcM}CyfUvDwiPDvy5~w?T@V$w>#~&|E2mP0P{jlfo*0DwBJiND z7yLhjgplgK+J#2m84k#;yYDDCBp}XCrt-cXo&uxQJrVu|l4@87J`3#g@?X%?B2(eK zHg!`Lqvd*O^3RCUh;~tP)vatJ zo2>@$=d&~kTp>&%%_lYgtqy{uf!b@W)*x1xcaOJLZOlHy+;aBA?j;)apE$LZx!deG zNs!b8S7fkXAhWRoy%+fI93%+GdiL$7V1qA6R^XF|U(Wg*dAMHA-P^&^g%G^HR3BpD zQ6nhh-%|yey%(K+A_`Xeus(>7!ty&aACC9>|AG^mU4aSU;vTu<^8Jh1NW7D`us&V< zf17}yG1>B9rxA_z#kwsu%c125{UM1a?TlCtY0AQx@yJqN1bZu6vFQMP6o%AU;_mUSE}+nKg-a+-KAgYcFG3_*iqyx30kma(Ux7uG8W z;W*8{ZB-+fso^JW$e$2PrzZ0g3#sHS&LhrwTw3MAru$prQ8+Zjv zLomP61Ev1w3=(^i4FV29H2e&><@;J=V}ZLi-gQ&8!QftyFlf-fi z=Y<5^G~S`xVZk1)`|64=7MlS*&{0+3G|2N03divP7gAZyF zwumtV8HjN2@6-z~&+H$6o2_4=Y62eG+ma2($;{b(5_dOTePVF8e$1u)@1t0bx^;8* zT@P{-fq>7u`5}woKhCCj9Z=29UU1aOF#Isx``8BwLQ7QxNFTk_Ip+Pi*3WJ1T4G6D ziK5XPNQ*3!gY0BLQv0IQjaH`SFdLX_ur>S|Z1fnED#EIHjNJJ9CR1@v^4g{>4%X73 zwZEVLVq4buROw%rSa9?*C8C*Kdp!DAEaXezWzmJc zpSvV{GeOoOWP3{5X6Lc_rKo4mO}FrPI|xm$-evjs3Pa>4Akn7)YfiE&r=&mi78Hpo z!rQ$)*l^7~mhx@QiVdVsvqWGK0n;C_zdEn40Ab=fY<8G;1*6gvbU^nUpP|10J^lHT zLrPRZ$b~z;DLkkEZ2W zQr(MG1gu`mA$U5GF2iF3Ri*p5+gK;Z86`2=UZ|C({K1B)PwmLuDEdN zn1rt{nOGXJ)^BV%M(a`?J~#Jqr5_ic)i7f?fNsvUJ|Hmlg9aE~d%J14t%aDjH$+vn zq4mQriVy3^ml0DD9%B-HzvMljak}I3Mycl23PyH6@R861-7e^q1=KUbe2(K(S^yg( z52T(-NP8dL$65`h++zO)%0ZNVzAjGfEMd!0q|AaI`AJn9!s^fGtex=nneX*eL}GxX z!9m3NZpW=pkoYA!i;U0i5F)JxaV%5%kquO$c~xzmoBfPs$5l%2KU~-*-(W!>5Nfgb(L}R3Ckw;bGP_A*2ssKbpA0c7G zKqdHj)2_^*1opJ#;H&63L+#`3j2+%NEATt8s&Hsr#x-5k^5dgs&!aQdquHY|YSEdt zQ(K|ZD(~BWnpF)V9EMR7R_)EBdw~|%G@*((kqZIx(;kTb{BT$!tl6o$W-m>!3BjAP zFv^v*)tgLr`00LlzYT;*>JjGqy3wyVTgc8!%-BjCW$m$UhT=U4bwOFC?3Vly1?<<0 zd`gmDh@n|yI~km`&DXPNz>;86sLiw9HmExs<@i#%8{*NSYZ8Eqqf79Sqbe%5t=aQq zI+TrbBw)ww-S>;O`uzx&msTZ8>4@j%MmR_M4L6JJM(U=m88AG94Y@6D&^z*k?%(*N=$-vLUzjsY4Ubi+XBi_X2EQP^asD_#(Zm_tZFUMH zbw$pp7^AU%QvTznNzu7g@$s-(ZZ2_#sTYN%HDq>Es=8}RFcIxoOXzm$_nkSSDl)g^ zQYf&99kp+pkMZt7ta%`by=e%eh8?_F#te9StjZ{gQFKdkZ|lHVeAVfga}K&6n0{-6 zHd<(wMR$gS>&i=#7Ax)wJwMHopDn_yTN@BA<}GqMoC(KU{vF;K8$ln(REZP{gk`g5 z^BC_;7FYfz65qq!B|{u}WP3T(Rm)zK{w8{TocTm6+ghc{Sf}!+nLvS1C^+Yl(_<}n z(keYpU3&(vEL^u5=W&MgmutUBS`-+N5>`gBpg3oBB-P+k@<`ZV6j3CHoOE%BMb8xC z?$$-=NYmaY^J6pvrCERRmzKy8Gs^mLQJ#D$koURVU7)iCPKU&A%eOrM=e52BT zW=%jFGV!Xh-#F2+THpLVjK_mPSmJF!1G?f>O-|}dxL}P_RR5_G8w9B{bt{a_N-J`1 zgO{}WE~sw`EliGk9yMfA-WX@|mIPpfLNQ_p{MsFErmPXKuphi9fkI*ANGG;M5-!w6 z4dHzH-4Y)MwDKKh{C&6NrI6ITOSl!}7?Jb(!+h5b++D||D(@c1w8Oby7%NK~XE|<} zIyvU5zm%W)@6yKy#8-Sx$iE%cJx|-4jidpES5m~#ni2upw+e1FmPpsy=HYy=tzxox zdn&CBGJR{Q8zS-hgJB@k(|-K}Ib3I*R4BKi*%=iejrjY#Vwaj6uHlLqK=44eh0p-P!tOj#%j-={$}@_^oy+0)<5185!oDt(a0mTS8BxT-IafnwPsZnIhC`YzR z8lF64Am`B08Gl}Ote*Vi?8SY~a0lCGw;`|_JD2t1WlVxUPaQML8vFFC=jz=>-Bft* zrYJDKSX26ty_D@GXe?oa+`snpifsex5S$b4M!M;27sXe6Ht%?I@bTB`cEZm>0B*W? z+Dg}60m^A^fv34jHO?X^DU0dgQ2hB4srEZ2jFkYuq2+~K9TD?nByZrY-mO!u0Z7x8 zzl*1#QzP%9y;LDk2Ew$6>R@p2^b`B&ZM;0X3?xBCG5CK^%Chub$t4L1^ z9~O+0iN0;S#`^&nqc5mT=e3Qfl!%%P4D239K3o5?Ky4@QkL}MbRN#Dud=*8E_VobvA@7|4i&$bA3lzIXA)LFp`XyC zdU0jI>L}fIDQN@^7Gob7Wm?5IUAg?4$v6whSi3A!hE|)MukBYOmz^`fAFL$K!br2% z9vb{L3UI6Vcnpv@;jxfM0K@vp)=fr?A}mcNY}v<|V!ATvPc#0MAn8a7r&g}^YHiG` zqG{jE8a!upIlHW0{pq7@)b!7pY_AgS;G=Y?yt?ShpS)r1%xCu?2&klmy;@@VHe2M< zxEX`>IDcqW=}XEqrRpidBZskiZMyct;XmU>+17%l2TAU9-Botq0YUN@K||dxnm2k! zoG)4GmTStmZhQ4;IsB}|^1cAMbCt`sm%hniQu|$&?B3R1k1y?|GIIG_Ln;?Wy>Pp5 z=+?tykFce5U>Or>5!(WhQ$Ix8C!19p!Y?MRXApnK8z_J4REZ3qX9!xu3tiFs_#EW3K((vqJ9)>!lruPc1Nt- z(p|8K(4Q3_^m~5Wuj$y2NK8=B$bsm9ikt(RA1;nYtB^U zyO18Kz#LmI+WviP4PE5EWv9;VWQp58W{A?yuFWnlEzX5a_6sUr)9H`r8p;htOPiY&+Nco`0Tkp5n%|B7YkXTS zw%10UyzL5A#6)TWeSk`UluNpF6nKJsuUFa6{yHarPQ2X`)_z&gn=ek?Bf-?nY-KRV za9I*=zF-Z!%;zA@Qf<@2r2{>4{Nqx-%5t3w;8%ST%ZDC^1s}C!`66gu)sLzO1OusT zcj99ZJ$L{47$LH4z}Z>n*hK1am|JVISxjP2{sok0SgrnOx!M2hI-{rd%t(0l&_@w6 z?c0ZU7qtes`+ihQuyU;bkfw_ZXFm6JcUMXWqb$+fkft6voS+XN|D&PfpU7lQuhYY zj?N2mFkqeErADx8_TPJZy&~=`m0>K7U9=7>vPXP1J4gC-OBquk(iGzBcpv>EKRvH`#0dmljH(RWE>kQ3+ zm>fi#59JHFU8c7qRbe44(4sgMKDQuJ!}a9;LrqFF}c|Z15n8nb+40 zg7Ds89=V+6)xzc!LoFQV=p3!}p2~j-szP1tbvi#mAPT^*@I-1)koANy3s4MGl36$@Wa!E-q@etAJ2nt|MXDPVbru88AKYYNm*~@_7 zA#cMT?_7@`D{`Va>dGljAiA7hdpO6}A6NGf2A95};ikR6B991;3!X=tb=YZEq4@g% z8ijGRMZhoa`tGkaw`C$sti=$n1uFYrQ=zcRaer(D3@-clyW2WY+2Gi{w3THkWM({M z|GA6I*5|YSl)>GZ5@I0y8Ia|A+#H&bLV!saxs`p`t>kCLA&@8w%|wgjN@+q4KiPS@ zxPxBd{*l>lb1>!#D1yAKF{fOCYJcAK{-Ab-u$QlLB-&+~Q6Q{Fs$et7UP&pBNft7h zF;Xl&p`AO;qR%XR`$d1wL-WKC1MsAiC+$PWf+?8 z@4@VB96mKl_BaJ9V0gtrtku5H3V_N>F2VnNsnozmK|^0ZTQGIyQIHoK-?4nq&_ii{ zuDX|X(oGOgNI$A8>6+~2SqmALJCB_Z_Wz4)ZP;kN*3LRX-rsN5rrNyhSMS-GF>;sdbV$pW!Ro&k4mWUrVj^M}k@5|kZo3jBpZh~ZnZ+lTu(ZyR4UJ%qy&U6eA*t;qDTXLd+i*cVe zj;OIfuxW-DmUOXodSJ*Hw!nGJEG8uR<6TGW=(KKbZdkYbHBFGs^`|Ly?yR;*pC^0n z`RBZu&>c*+ZJ3)Q4)5%m`hVp>6dj@Fg;;E9vT-@rOKz7F?HbSy(Lt=Y$LpK^517Ml z`voNAO9~UqZK^qX|v=sp;yRG-u_eL)xQ|bu-`-s)PTD)ms`GVJqr-$N9I;jU1WwJ zP0u%tmGAq+u~{XYb}dD_B;d4Pr>=yG*qYMzgfL2`=J-dPeGQj}WXmltH^Y#It^&0U z=ODrj6>c8}@Y*>an6@})AwkGzp~;G{$}nErB_BV?&)cdDlLVX7Ur%->Y!y>UI7$I9 zm*w)M^6<{uSMFMpZt?S>hQ&?Jt6$R$RHPIzm7lx=N>=xJVdX^CK%B!IAo&oFS9!96 zQlL+}T72AM4~E4m+C$Zv?>zBrBNln5cJ#H`_fxjs&KP4g94C41hh-rGQGk9iE3^S;Jfvn z8}YSwRcm{FPz?x`*dy_zd4jABL_8!}V$=MdQW{9&sbFHq9s?|?b|aOGN&nT#M|{Yu ziV6j+eVh`~L;Q85I}8RHEE=npdXMTcsBptGGv^TIGO@-Cup9Wz?h6g3E*n*GhJ=}5_x9{;bY+Se;;mJ4BRf{1r4OQ^lMP$UQ zrKUqb93O3Kz8YU(hWiNP@A-z0nnO&g=Hj zDi>(04;VxmY}}1@<_!M%Gv8;RCdVkL5o~ zoWzEEt*g{zKt;*nhxBY1%ow?bg;(OBtYiF zSIosf{T=(L9d_vy%S6v<<{zFoCAL>9*HEyW1$Ec?Rv}2minKQIqXWKj_3jal=f%`@g7RM&RqTU#k%2$gm;sba-<+*NY-T|CDt-kTUnZHIF} ze*HY`+p~@050=*M4IPiBB%d|`E+ptoVQtrQIL}#D2x%(8u$AU@j|DCQM(75=VrbTT zP=+5dJkh_Lxjz1xv~`EsR;Y8}dfZ;5a=>9KdDOFA9}Qi@ekYqnUlgCaHoUKT@8FRL zqc?(UXG1a9;r-Or5IGIvYuwmI^GohioxK`_bLguD`+^C(%+VYkG7^MA1JE3-v3g9w zB)qHP@4Y-jQ!|L%J6Ze@>L25^L|(MTLs3IgmA_RR44tzpf&J2~=C`8qRHAa`u&(&a z^J@Qcw+c%r!+Sgmrmqgta}>EUd+z-}D`z_s>=JMr9#0ZDuQE0okUp%?_ia}B_*Z;&1QG59t}*;2pn~73)jZRVTb+*uE&a8=Ip(`ZI054{5&~wZBr+q*?7&y$zTA zW*Sx6Wu}LCM!m9j5&G;zZ#m|Mtf-P!Eo5xo=hDe{R0H+orKztf0j&x9QT#LfH;FCx zsLDTf=eP^@SE&ic?56H{F)V3TZ5i!+Y94r$Co#%Z)^F3S*x+@ zgk|G^Xb^19jkl5Tu4pmbD4ZHDW=M*qX?$^ML1i*+I}$GEqS)^E(^-=xp1ok^5-n{S}SI| z3~JUxGeI3@sd{S5X-Tw}ZybfI5`2{Pc^Z#2VpkMJ0a)R#4`6H2=2hRw>iXQQdU-Km zoykZc2`gqS&ro{V4G5TbA;X*iDh9ohD;^ME zXng7F=ix4<=y#Op3*)^zmv~!YIt%qF@y?Jz1CI2dMcJ#!@2e>mwzK*HNd6p2;Umk8 z6~!7L<-O~=!m84DM4O`xBNGM=Xow7sT9Cwnd}^ux>gt;l*FBw9FDMgRk)DmrOvR@U z4ubKA!0qRbN|yU-WkU9gXWj zts1N~*8#%^df&8@@V1N+yZw&51jKgWESE?WJ?{nyYVz^6AAPAu^N9L>a*x>NGAD<| zi~nv4)5#S+Iz^3S)uqil!kp=AZ{Jh<7mMpO`0wKI1t9m{-e!N%FDR)j9HKKs)Bb}7 z#XAA!FZw%_rcQ6F#B>nu9^Uc9Ytc1>NClv-uPW4<8n?olSL|^ASIOsvvF4ZJ^m;a+ zpOtwQO1rY%_FEYRv@Lz$noUIkLk|#=H+}5yVc$N4TsB%Zn<7<5sGd`W_BKb;kMY92 zJ;pzL8|d_@wD3+5GyRVxzj-&%>5z=E59ke1qva>G`P;KV&ACwHKh87g&FwndaY%mn zmP5Bga(bKv2jxcCbh7Uf-6OsG=4yrxxRs9GWhp*^As0T^(Fl4lpn@5YnvA})-QL|_ zmp>!rThbSW>MrvTGL@pYT!x2W@`49g^UiA!X;j0*ybSND-+SBHdH9;2WO~Ga2TnX5 zt2+A1siH>=W51zH&`YNOF0%c1@x=*%ETj9BcUYZ+OZJ_RCkx6rq>SksuN0qpdsPG( zq0esT8<`0h$QGZ=T;xsZ_#tR-WTmKRnRAZvxO*8SN%zIMRdxTeDyO#%2O@U)BtkN3 z6m000UDE*qQwm&bqT#_ge|A2T=`D2Y`o;quVck z0J}&l1C^G=7M(pt4rwJ%QeSn!kQNBDPM+Kh{NRx-)0s60NyRwiEcK+!9hLlHCP5|W z!(U%#WJWm6;s92j@8SI-DFE1QH@)MyL7FEhdn<@BdSA_S$y_a*(r5oeVcfN-t}?6J zK^_;LI`SnD zP|}rp@e@TxN}qmsj|w6IBB+__hdHrLWED}QkpXCTz8DS*^Ke~#ElkL|Xo#E-qXKww zi1?bxfwKwtseP1~TZ2RVK6|Ci<+$YmWm>p^-=Mdu-Ot6{0)hMj0sv~vk}gQ(syWcr1KAR{W!HsV-_^{Am_@LCYCVhIe%4KDYH)v4-J4h zRH#mvdX<50wU~Z(z_C-kd6Pio($QXseKYkNoEQ39x1*Q zWx`hmyXS{9&(@qouJh$VtA*hriiV1iOj$l~7Mo3iBVDa;@xncl*5ChcC+%awkpZ_?UFW%9tTQ-i+9O_*UDX45|kdV=7o0Dd&KV4eu z>gM3TkJ}Bw>Ek=?y5@QH_G-LC0qKztsM-Pj`23t6n+^DakN7 z5of|T#bhpL9Pn4`J|a#y@rACrwggQ)BD2dZ0t1wp$gewV(g9&{o$57}GY#-aR7^^< zYN~pk36B5>dH%l&;~?8>Sg-iKTbU@UhP|JC>Doj-#vO&#&2+}#>?q}lx`pHqV%=qh zvEqL^7mDmnqPV6!J*>;)i%b-)apzNj(YojQd4*zBW5O&kQ>C0~*GJd#h z6YjT&-TW8DN(JM3^n@2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Low -Cost - - - - - - - - - - - - Short Time -to Market - - - - - - - - - - - - High -Quality - - - - - - - - - - - - - Inexpensive; -lower quality; -longer time to market - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Expensive; -high quality; -longer time to market - - - - - - - - - - - - More expensive; -moderate quality; -moderate time to -market - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc/design/img/quality.png b/doc/design/img/quality.png deleted file mode 100644 index a0d4d88d846c0551ca03e14a11d6943f70687e38..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 38953 zcmdSBWmuJK*EVW{ASERwG7%{OQA$#D5|RP}N=yVq5TvC`>BgzFgwh?-t+b*bNOw%6 zyL*p`KI>iYv%llpzqdcuGH&ksigArN&vRUU4-}*b@Tl=loH#*nUmB@=;>78c6DLl| z;huuOc_w{*{=|t_C+;J~AKIhlhaBw5RI5do6{Mrz5c?vLxR?0Ovk8P;aYx7K7xs#e)0B|%%l5Xcrf z(Iw)N2y=P)AFArw$&PQXWjALwm>`%D3FnSyu;zqu*;^=44Y3HkEVAoE?7O5E^d%)FO--{q+`6+K z%Y6Mv=g_tE8)m7p;p<4mfIxU$a`0p6-#I`Hb845f1ceB%`j#Krjku(S()Odbg7}9Y z^k-&dJPfhO(y2kQ!WxSok9+^J16TF7Uh>M$RJ{6@m=(rX*~&<2xJ`6V?s;xzri#ej zI7)Y4-@qprrUav@t>#SE2)o(N^x4C6Ba>lP*|Q{8e*ga0>WC7}5;Gec8yPo`ccCzI zk{wFyk_y#G4c8nUEXgp+Mc&Zu$vQD96r{jP^gHafy$JfZcDd`7hQR@oX|6E<~k~F@1wgDw6MD_3PzD&MCsd%5APxp+qY;6ri)cDYzgP^xj^;F zcAN3Y;26Vj6{;1z`pQs>u_(0 zh*n^Eqeo}tlR?YZcg$_D<6s4k3!XJJG_az#7Auy*=_SG9u8iF7!Gv_IMCZd)N)=>%VUM;=fT^2@@t}Z z)4Ff}>NE3}Ma7smO{ge%kgUv1(go!l5&Pv2tHiO*d+$f~CAHsPxGcQ=J?fT>VT5F1 zm$CVc)ytyJ0t^h|`}~}oW=Dqy-%jab5AKxHVwsdS0^!ehJTJRJOP{yP7fDIOuCqr8 zJACROAfyw#+MleIS?ClVD20j?bt&)YOC~Wb>Zb{;oL=)0+ZTTxC1jWL=}zhXYD?JL zFYj4=edqnL+l9DwOXa@(LVwY_mMI2GwxK6;-PsA+%ssQAu3Hr&BKymAu(Z2lKJ;wt z>@6)V{N}^Lw)2mYtz~4swTT^Nt#47A+VpD^47Grt8e!|&To~x-?)Klr`pgAb%-o|Z zh=eq(tL3wVeP(A^dMnF`ZkhTKbxUgIoUiwvPgM#!ZDv{3oV*go#*}T8*|liQvU(S$ zA4W*;y!_h8dEAdVv@rtZn10CU>zhCbzM;=^m+w_l($zBOxR?4KR!5l%IbE!(4XTmLR2Ra(V)y z$xZ2?o~^=lE$fZX8LxNXw^_duxo(f)l=-~hoiL%)vUuIWIoV(5L&VOs z(TPK-iWw>}RGey!?PAL9RN*(0(9E`tc-^e2uAX{EYlWdWh=!kysBCYsViJd#M1tNL zO-boG$sb57~`mbF!i-YtFI;O|MOai21XA&B(~;+*;keH6Rz7h&G zTzM;&WJi zYEz|UB`uK>qGO^oH8T{ zhjVplC^{ldR}ecRS>$VmoHCc4pj5J`+JyjFD)?9mJ$C#Mtt)>kBS|A#mv_dMvD?Xe z>GAOyK6~ER~33;XQSp@6NZMe165^kSusX5=E)P zMY5wImmKad)0Pn6EquO)pz$p>;p_|Sku}8q z=idb3=W1WeGpH*m_?eihRt;FI=(hX?M@B#LCl&0+*Uu?G zHA*c$TE$ElmdQPpJ09UWSL~PLt%VVefg=O|%e|`@byK2nf9dGZ{KbnGa&q6@4%g^y z6iMV5I51&xT2iRtPH2zYEy!5)+=!At?<6$`}!p>b{pb2{yBBO`t3VU?X=i>^Z! zTjaJsL2qqstr`%@xCAvvz!A!R%xm^3m+w{qd_KdBr~1$~s50llg%@}q!KaMAYFY$$ zQYoh8v9(O8qAepO=ihy!_H|RemBQ^Z`fXI{SeXlhxM{ z6YkDGRCQKSt~&;0^R|$qu)ADxsC$e=qSP&pPts4)dQo*z&>{`mo2#q+0 z-&Ff0Zbyf4nanG+di3#PT()AjVeOpIEp&?}Yxjsm3ZH(xn623AK=>YJ z+BkOD()012PiP|~#EC0i>N=_#TEBb5UQl7ZRR8CcH@$hf_aHbCu7ZPR4ceySFp}En z$w8hwvlGqE=>W-+`L7g$)(&tcZv}mY#uuyi7JvVEFLcQvouLuV&+!S*Cw3Ssn~D>n zoNPTZ`1MZV#=+jm(W1rCTaH5^y?Wey7CP(S-<2-seDD2G$b!TSeKH8>_~fwm8(S|F z;zPKW_#JN6u@PALv&zYiC!t__DVo=0dE}aWw69$;*)!~L>0!97pHoINv$DQ7H-C-` zTlB^${6MF=j+Q|Rf}M~@R8}mOn|upp-PVPyM%m~4cez5h{gtM`%BP;`vWCQ*SoeG# zIhB6n3<*Kf6dMsr+ZT5 zH?>Q#k<8$(@~Zn)jw?ERqe2nW_m*yt2e=*X80MlW7j|}b1Qd&n+7Y6n)$;H8V5xOV z%wo$!K4@hCQeX*QIVm-DqP#C%h1rIFB(<35ljj}98?!fyslUAql7rvyHKn~=D2D%{ z!+LZ#JE*3yNBeRCnsO!oo)4it?zvZ`={?q0%DxY_1n9Sn#cgJO5i&cAk5)H@S&fIE z-gx;8#mE&b5!3!N_*UZ0hoU`Ai~bopqoiszUw;x3Tl2c7MERiN(x?>Ejd9 z*SrTdbol^MrQngzH{O_4u1H#1NSEOGdI&-BiT42t>nhw7x4nT8B)%($$zW;zis_<0 zra>@Zxm>RSFS*^i7Z++RoPK777#43l*Oq!*^lqf~Q65vHDP*WRM_+Es?z>yIOHiis z3rNK1HwF9M-&$Q=#YE#}5N>sPPILA6R-T95qjP9reoODNu9VE?-rS?(wy;X!(15vW zX=#aD>a;m8e*NAVT4DRVI@6hsWEsVt9oeutk@(->Qj>(n0~Jlv8#*tZ;`x%+^4&O% z-}mspC+!N3{7woA;(iKZDgJPs`K4KZ;Y_jV0I~&+ZT$+{&_F8f;3Z~((w3nc^9{@u zLN>EW)nke=c27Ffl;YhEZ#Q02V1k{{sB+wZT%Yq`?5jPd3aXZ)WgoH8`{;s|d4Xt4 zN9tS8F6@3t-8JWltCQS@`gg(N=%O=JbrU%I3fFCj19NM-BhTaL4W0|`?oRZ56Bgzb z8yOYlr$apn3o@=Iy&hqL90sS58C)s0zwA+Wu=8vMb&ivG+dP6(8>z^;y4ah4Z~s)T z^XGdHhIh-LRP9%Q>_P< zCu{8e>e@-Yqsj|bngd3tCk8EeM@QjY8wnSN!PCea*axIzAePyGE`qtQ! z!~LC%EyII!sIP*Js8u#S7(zrMV`8$tpGZ6HQctg(M0ysjE?)gRgOO`}<`>S<0laW! zjWI-c{K1bRW(m~F451c@u=!J+3fpXfe8zk1LF39cOI2GLf`_$R=^WV{UusAL%jI&s zAALf@ti7i$Z8$FQ2{fgphxN4g7-MYCUfY1IlKYYaWCWb8Ok$+7i^N0!4rL03e52>AysJhU>~ns;-x$}R@f=ALxgT2vM3{GDJb z*LUGuR^&XFn3zt^^E-+@d*3p6d;Q47Ma87iTiAlE&SlP6TU#68l9e@xBNF%hE63k` z;~#qYZabG+FKT<#XSZ%}T)F7Ums6@uI@_W}$wf@tcJ4|MU%QB9Hw8o9mD;kyVq9qB zp55p-wFp*HVb84tEEn(2tO17LWb6~_AD7{P`RuK;QN9S$XR~6!@eT0b&8B1gNgq6dPQ@v5?v zlgQawn-Usy`_AixkaV+DAr8ON?ra-c`I^>p)`(6r*)uw1xv+wG=8sM}>>g9Cw;_P$@GZ;Z&-`g}XX;G9a;)dx+WS=)EHRZR~J6nCE~Pt?{OfC=N~ev3tyW)Mj~9JXzbn60tlp(vmqy zHr7#fJUuU_ZI}N{PY|vC!#Y?gyVy9)adR^ODfapO01rJq@rZ z-2wQBDpy@q>RPYnDZ77|^+Z%E{aMr!V_%0$M3I zlw(b@?qV#9FT1=4{_9=l#{zLEq1tXs7QuJ^ezFFBtN6b@X#|T(xaI7Qd&cU=4Pgq| z&sKRnWs(?*<8OA2-{#7VoVW>48HDS-%>^My5g@T>FrCYT1g0yIrWBVtzF#>Yo#~7b zOf;$s_Aw6KGfJBX)nYSkVx#(SvdGw5ddA!qdiZPvOQ=8!q7@Lv`qaoO*LNALu3BV$ z%CM5vNE9+*!PfQU|OL;Qh49xo}%j0z(kb#c6IivZgeHGI^9H{wOz$o9GkS`dWzz{vv7Z&p*mqc-B zSOd?-4Kmh1oUAT0Oy7PO8t`KG`dX1B|LST$!TJTA?wDfBl+9glGy}7 zMj@3c!A}l^^aF2fywq)->t61c7M$l#ng7VEIVo_>IW6?rpyF`WNf3y#8wcuh)OKe)iA% z!BG~BM1G2Y`}QJtQw2{DS`T15?edr6K43CpfVVY;&_@V|qh4k9;;fIxEi!?T}P#I)}?q-f?|a@goIm+O-=H8p+q?AoT1np(J)H9I>y zsY3}M#2sfuE<>w1lg{wtt|xam_IBA5q-7IoLYK;S~EKrm)q)B=36&!QuCX|L`OTW{ov1ils^sy zFCqVi&TV;-SVAp*shg& zv*(nK0}$N!R+Z?$t0()Tne$wBNxh16-p2tuEUMlR6=ZC#>YzLlT& zcrH1#8T^Fr^P6;kk8N`GqIkY3pUbe+jAjH_IpQ)tw!&=yAJ;Sb>p4J)6|)(cNj^e6 zaqSN=RZG=-p~>mmFWV)swask%l>HEOBO})!p!tMBRNGhvMsQH)ezLP5YHsfWW9&g= zsOdv>^#JqY?_a;7O{NHF1&HauESM=@eiXax)%4A4Z8?x|i{Ds}4nr+L`mYf_m>w)! z>qwT}S{ezd+=f86HXgtwd5ljk&hui19HwH8C&SbU=|uz2>`X@Jpxh2N^zsSK zkB3fjKObt`NADop9-{&vb0AfbKDW(?ZB^ktAHm7>#1PS4$T`}p%|}GDFy!i5V-{{h zp?_u+7bM8f=WExT=gq7b7b-Kpmlo%=csm&g5O+Ce24>fLQ0Ro3mb8D*gT70h1XRq9*V(n-b`*?uU zO5}08Oo2_AFXbQCzqri-fb{sWI7p&A6sNGre?y6E&Ef9c>?RON0h%Y04SC2Nss!zq zpMbBUy=-^I-i#nONDxg~Utxq#$@!4qtT*q=*kSP^qf{_UP+%pQ98eAZK|vBX8roTU zS~c_a&TeW0h&}0k(q(9*E`9CniaknLsQaV|qKCz}cW<9q4Xu8pY(Nv&W0V^-R>S*u z=r4h4R2HfYzvMsL=PpbXFct4MJa)KOG1Zi*12{nb8SvH1GnI~GUX(wxVcp}}N+IiL zGh7#0tpySzI1Xql|MGTg6MNH6`rEh_X?g`|G~gXV8oBiw(`XiC0$I;|h$`JcvwRf~=S934kX{kz`mC8qIEFWh779U&j-;G~iUngZ&Xy)}N zyk%@@yRb zK)Tji-+1pRnENu$5)k3IHY2`OS2*Orh0j941jOj{he&RH@AA|saL8$5R^M7$8cct{ zw3Cf)JFF%>GOUc*by>I{n0_uDCl>9KS0gI6>4pW--&)#-CVqVX%EZ{%aceQi43fhz z6EeE`FDat?^PXt_ZQY>M#ngx2z(J zxnEM(zRll?h6L}o1P9n}n(;|gv3F17IVnOEzThBarsFlR#CV5Ja4Nc&3^euRRlJ36?mCQXjeZa{z5B0!>F(%n?Sg}+8UNQB*yNx2H>Rfx&?$D+>rH!@1 zu-03SE)nW-8$8Wi zt{5xGWcP`6FDTuY&*$`$?IpL?^$P>}< z$r*noia8@&lrFpLN+HxM8ke1y#Wzf5 zetz^wqmHpv5yO;y&k7+_q51LdSlIN4vIUb zt=gvhfaOqHHOCAUo4$XP#Uq6TS|4f*k@%HplN|AuP*#l za#dS$f9L(hAVRxXufabN_$Xof$lF1`(LYSy9rcj zVN*i`!{}mP0gifALPEk951at6OvX>?e!M5~A*soAhU42{JU>uE6d}yb-VTs?Sll<0 z83-%b`k%-#I@(rcis)@)kCab(&=a=RXve5ye`Oq2>>3 z_9wXn?4I=I>eL*1)0h!g^o=bw;{z~cS89056~Ye@gdsz%60;!)hEp3<4UbF*OS#qd zAS)1H1Zja}WhNU7;}j4`by(E!_xGp*yCqbi{tL0g?TPc}&nKi7H48oOxLT;?ve7kA zVxF6xuC#GL-akh7xN!~?kq)~eW#mGM?E(g2DAQWFHBf5{7x zBY|LY;t}s{sNwx6Rx~Wo${N&3ybCF6&9{m~HNKmjQ9<{xXGhF91a0=G74dpnL~3{T z|9pbsMT?6N;YT>Ra_8?c5s9q7**;lA!WIju55*aiNp(%$NvbZ!DPZs*<;fmd(aIsG zzhsvtNvNPYo0Ri8N_KzTjAK>rQhj4%1|`eT6(-+%_%}K*(VC-@3PzCpNg#B7OO}FG zX!&!Th*O4CImJ53Qq-7|RW`%8Ja1NNcW&@gvDX4(zAZj4MeSm1sEATc+87@XUMX?j zu}Yo<%R$OQ%oDD;VgZ=_xYj`(kyz<>#jG+w^n6N6n7sg_?f$sSm}<}QOziHgO}~8h z8$)B4!pI?9Y(?1EvR*0aWZKzJeOKT+r!h9IjBfrA=i~JNg*vP!Rn&5vN!~s8WwE9m)8v4V9zr_<4E)R1Q`6!;`_yz_dbOzwhk4 z4^ocpijf1t)T_CSIAU#)+@}L1+Ax@>SqtDtEq4J=h^d<(n}PJ*RIAL&02upx1LWog zwZqoJ0CRWhhYxph%KY}&^Spv;Vkt-<6GWi7ZjFYAh3Pg645q6ejUrVs+UxvIdt6V3 z^1ST@y47;f@@L2QuW;Urvo`RrxjK63UBCE#*gl{VN*bRE3fkQRVl#TUmf54WJ>U5@ zKNGV$fU_zvxYWX96<`x!Ng6xWXBbZ&+h8JY{O~d-x9Go2qyXKs-Lmc&wE5zH77Xcx zA&B80wm&_P0hEgs!;?aUIdXe@B7|;nDS6Gg?arN1+ujvEo#6C0YX?rPK-I7+o3H); z0m_o+Z6|#x3Iq12ev%#}6Q0Yyxg2Zplo>YLsA>%XVv&-oNQ|l&axh8|OEn3l`)al| z>`^{r?xy9|^EB%GIOzCsL*tCfFEe8=VU@YWhIz>w5J)07v8=`xBNBy4}{~B1naXEf=t6QsEPBl!M&U) zC`qm@T3D1G75@TG_&n*gCx0qBrMvX_OG_DkEi~O!CY(LZwx=vzfBO5SAk0UHC&^_m z-?mE1NW7NaGk#O-U+dS(*?-04o(H-|9f#;Lt$@WS@Z}-E9YNx(zXgDh^7mPQ>WII$ zMI((cz+Msa-(sK`Rew}A?=33e;}}cM)L|*-rKX?|TzAlODO-#y@{{9+aSTC>Lh2EKWu0_th)Hbv|*|~|IjLEp6Nf|XHy}zZ|${! z>PcwuU92D0DJ>3Flz>q1oK8`olhV*E?y2!P{sj&q~|$N665LV z353$1b(_eD>t6OC*JlLJYGY$#N@}VYh)t^Z3F*oNQtnPd>8I!Tp|VoAjAApd^^tW} z4Nn_cpWKyJA`Nb+YP=x(6c{vL{QjOFEA23Bjp>6-;|;^TdIM>ZzIX^Ix|{QT2`+yz zU0413dsrsWabp&%M>;r9^>;&CoumH*lh1t;G*7Wk<07{BPXh&pZj0fwXO4&I-21w* z`M)2zRpOON@?nF$nj8;}1A5;Tpz`vyKsgOv6;czB*xB;g%s$)*#Wlu#B%}C$f4wvr z8JWN!e|%VIXmb(nYu=_^p~9&RMjBj*2kh6PgDN?<_MDF~kt*;;;}a8b9z|aC=JA?z zOAl#XW9}y7GaHNqj>YHdwQGFOJCgpOmrlQ<3<2p*S#Njba8G~EuTAhH_4!km^{9T_ z810Ag*z8D_OomqFi@JtJ^xL=YKjPx!xq1+I-SoMqa?j>U=b$;3_n$avp!pG=7Pd-Q z`#y$&+_Q3W6e^Rsf&D9^cwL|U&R)V^+bRe_-$qnS*^cJ-EC|rRS{m|WJvaH&wW&6{ zQf75Q=j}YLfIIf&_8k9y{3DtlbNu*8ze>~+J_BWmoIrcc`Rhg(f>~DsKybJs{*3#6 zwa93Zz+l0vi7tRLEQG~+QS}nTH=_Q;v5i{?OV`g&aU|CK-X!&HHqoV_rd7dI6d1N` zPPd=d6uPq{==o;bouyK6G9aS)vb}3w6>d-^L;OEK{gbp+Q_}6nf)AR%UoKvh?tQsR zs>0;tq!mZ)82OmZSl1XYz!OjZfhTsOynd2iL)iVu*2M&|fhNoZ<4uo==IA{^BbEDofdjpq7N9Do1ao!S9-s;T|`{c4xJ|3y-5?Z$atl(JnK zj*X2Sf($zKSKlWC+4T{~?FI*2?mPV-V2is`6j5LcftR@NUM~hkLJ!J`qk9vybISY% zy#K>>C5q*+DD#iO$-k(`tfV+PUuF=R+6&)@swhu>nd}iZWA)3un0=hCb9d(fg@{GN zPjbmSl-7^Vy}k6h!yU`1`(1$b3g@b>2aba4XeKqj zlj*?JaXPBW54TV(HgW%XPCPh1p6f^7l7D=zU5)yMg8d216I-p{Bh z9ovp@xv}#X)hPc$f3z*N;vu+-P}j}h5IRgyj2D#_&NzGYi5g2LzPlzLY6fD(2U|*) z&WprpN(whX-PNVYaK$E8)Ws2aSY@XdFXjP{2xG!cK}95e;pHz{D%_q78wOFYGW}RW z0Ou|!HakVM2^K`#nl&{v`m;6pc@TZ?MV?U)dTW41e!8A(Dlwd| z-y}GwQp#|3p|2pdyu3UuP1+fUNBXN|JqZ9Hw`BIVmZZ+?fTF{N8jM_VbR75Z-&X|25ag%qY-~_gZT)_4o|2sW z(FMe<8E1PmgA7{2V+AblYH!FKDGtDV9sIG<*9L!bNWDQyIU2xK-3!8!c3*lcYwLy# zS0JNAoHmh800x_wn5g3OZIxGcJhoQ(v!hU@Lk=OfJDmtZZ4x2sT#`1<>I((%t_(vV zDYvcrXskfRw7*ci1v1m5t%e|T%~;d4hS5lbNrFYc#(Q~T+Q|gW{)aW@zUeV4B59wZ zk0eRW72t{>r0`TP*Wzacb0$$uE;Y~El1On@F5jKWI=~7VhGt%}eZ=f7)ll9$OM)CC zb(jVjBnguN8EmF><=7U$Y8gUx;T)QIkFt_>+xql)LDZ5|V>$Pe=y;nYK!n;rzeSFV z(wm(69+haOj^a{#^qgl8rl(CJFjlw%^SQKt3r)IAcZ8!NX3~1p7HUKDZq^`0SC=bX3GG!+8j3{l=nE zXoRjC7Ga1$${eH+!V}DhqXb{xx(|!`^G#hsD7aj69#s`-@2ESs8yZXHavDSJ;OpJbj zkBKR{u<&6{A2o^##fuUuex;S@dy#1hM-mlLVo$90vRn9S7D3Pzpxjo_pqRMdHAl(I z%e%6&61)pGj0R`2(BnGgSL4CXZ>A7=2&J#h}M@j}i~aD%un79ZcE8cKu9ndt`4$m7Pi~{&UVJ z9!dOAtbTrORmSAUk3}I4SBK=Xva(r9CiL*#ZbwP+hme5;yA2eXq;5zSrJH6J5!hU! zRe37&2MOnUfknatsO=YePxTVUxt&auCc$pS@G_;DiJ7U<8P?Tkm(&N$S<>j9&WcJs zUddqE$2E!9{xyml^rkxS98x^Ig*`oxi7| za@jrce%ckX-cp*;XvYh)%k|{_e({dQH!5Al$7U`B$cm+xQXP)&_|dlvJVRQT@dl0}p(@ za7#3jlau>}kW)}ld@Ew{;CQw0D*Lr5pr04dtnXiGeJ!d$``j3><FLl36Y#PNg2M$i~%)NQISa zWB0-i7ErIH`e1Wl)~uNu_&Z8)Paxr@ta(iBaNfPu*r;qWTvJ#lrPjK|NW^R9%Jj{n zf4t99?n^AP62dyGwYWn;Yt{`+-hu4~gf+Uw5J49>MYCop#o66_ow^Vp#V>8^Sxv^o z(3Vkz<seqg3lwT~zeWF&f)ysO z1``*vNY#>>=LE&LxNN6%4p%8@iF*g4XDg_(Og{h@-luV{6!{ zZS3%BU(9XmH%QCX+UbT&QtKrlXWvRrJ9o;h8$71{An_Jc-QuY>wJH8c0bcpSMLVYn ziwrDiuXKLkq~_^?Gn$e)O1=2gJK`~?U80VcP7av7GfW;l zdPYVxxBfZQm2QUHx2alUM4b2<_rdkjUYrvC>Fm4&_z;*Ru(EB#-skV*K{L z0*-IG9R4=V-PE1KW5FZ)Y7a}`+4cPo8zwa?B@A-%x>Ol|P80dO)^GK%?6H$`NK{*o zfiwM%L6LU9iP`L>Wk!>Gb;als(Efweb@b4%qcL30gp)S*Hd9o*P zVZ>Q+?8=8JoRNvpbh~Z2sumMtTj_{R6KFLExR~KE3?K7)fbA1F+EJ&No4j6=P-fOA zKi=mhvL7$Ah@yXQirKbiAX2U#`a<;SNS zU6(}^%H$Muu}wcWSJ9{M>^fACsnL7`JO64UxXAHW>VQzV0$xh{4G)C3{_ za(@Kxk*EaR2JT7x;dtYdOHRR8z^=BL6r!SQR{>lezt8k7FrlL)mnw#?Bl$q%yMk%D zSbI|Xv30!;sI1N{mbLp&u=oQFgc`(neZJXhM#736qHK&6#LZRkK8hk64?(yUqrkk& zIK`n;tyLzZMCWkUqu`!#6-C6?bPnxv*P?k!Wi$%I=}Vj0w@a~WZxWM ztStgPSq3q60?Pl8+Q;R3_cBd`bG>;+0Pa5(DHfy%JquEEDk`d8Gss7*f-aD%tn16_ zIep{BBl@rB0K+8QqGWe$J#)WNcTSf^@BjGNV`!~`-YtdH0w`#5(D!@2d*AlP^=4QF z)t+c0LZXhrBy5Rq@C_=Uj*gDJ2CTe~`msBQ--2JZzbVl1oqvnGzY|h{V4PQW^VThq znu5@*FX8o@^=b{pK&2V~?-5#_J$trS4Rr8KOiWfm&aaMN7r=47hb8HMztBTQRu;s` zdD*LLYm?MF7c|KSC^Rn(JkC*?{<~(nOsh~k*ESr{eeCR(fvAT-d^*L#{3^Gx%-@?-Oh zCaJxAi{=E+Ms@d`QW=HR=Qm<0LpAf+MRd&yd;LpjL}b2tb#%TMfb23!S1*phg~03} zQ_db1vS%9QIVx|?B37}A4$APIl9~f~&NuJjym3Nj=hxvZsNSUtok_dP>{TqAr51Cb5DxpqGb#KhfLT+u!@UchxG$y|v#a z);$n-Ni6A$Lg1}+&k^Kb9Z|OoksAP1SlPcrqd%ggLkqB6jvou{N=gUuI7g>$ZNJUXCbB+8z)}wi z|A$e_w3wFvqS+i7wubEDJEcme69TID7KLm;JZk6+O$%LZy&v!4*r?Ju$jZuA*v$R5 zP@Hw#-?mJqd+ZGD5#KI$g0jjZbY2g`+jx8!6Y}cPPu~qDHSj!b(ql6~N>JI(+pI z8W*}2UXfc1#)4P)wo2u&@y7#@p|*LS=(_7TQvUtqZ_s7171gYMV<+4axd2JJ^7)-! z-FZcdk`Y%SM?Ixk$(2CJ0CUTgJ|c&qd7z;TS_t2@cu4}Q^9IZ6P;~K+IIHYOpw<6Q zO7|)hvdo`CLqoAbeo1kks#h+rMcfY{*$&xOL=3R|0yu-u2%xYGTR6c%2NP+ zV}d%+IxPBQ%n~*)huJ_cACAT;dl}o>N z5v_jkW^izD?e;JH8IGrr^&pD?PTR&d&taj0!8+4=Uy_ajdu6k)PcU775A~z8?Cc-k zzt=y$&dx5Iqa>{x@}%a;QkvMoJ@F?CCN!>_qq^}0*wgmg}z z-ILByy7IUpXCO}P1n*^s2JeNQTpcLz*Jrz0bXivuG_s>TgzVQ6)g9-E&FtpaCV)ywBTWPPhLw*LGT%thH z>Tw$T5J5^%EhRfx_Y)8=MX`g2L)O3y=RN1@G2r~ z*rJXA;gX>Z+%lwbw%ZgAUOtx{o5$k5sf)b62mMxn7Imrcr12VeO3#$()_M2#KCp2Ah9tJwxFwWr_NCTj{ z{dTaoW!|3P)m)9zQSUsaYy4QAD#Qq&!GqtzHrPX6$u!UE<20)>Bc#qs3w;p@sTQgkU8Xj-MfP zdXR(G`S(){dC)G@+4(T0fPzztrH5r+H&=H=mq<_S8}BMTxM9S=Uz0v_j)2;elme1c zGKNwdkIxYNnm7O2BVV^9d0hpWzUVF|o7bPo;eS{=L6n19&X=gMg|)mmm=!JoCUE5&*E^IaFF&I+s(C zb2y-hT*ohn_k0G|L!4k^Vfh>;bo+K$QW9iP3r~(epRg*HWlPnJn;FwFZ`h<^a53UMEM&Boy0 z3lvW2LGMASe~EF|?DAM0e2n1uQ}j&$eC~oR`4WUhM?+KnJT$ns7$8>ZVewXL#>dA! zJhkhEqau8Jq;wz1hJq99v%Cq$z36S~9EZp-&}F{)$=BbFOo>OzA$pAG5*o$RWPrHs$%;c@8ufWwEPUH1{AdrE$ zUZBO8U{MPuSpBW7O)45#4pt71FcYMi0H1~DsKA$YL=OUcy98(omtG34BXD~yIcJ0~ZIoAT;77#MV6VpxFzEZQaPojL*j{{C=c zLDs6P`%#Or<5l9>rVGKev|?5qfk2*|&x{QXboo12oMz^N<*H~Ycn^+^b4eedhAh^2{$!h|uck@CMQ80p5Rv-bR@w2P5 zvl6fsus|>#fQ7&&ArigaBT=f?gQc);gl<#0pKxhN(wQ>Jrab)gB^_biwN;1zF7kk`goYV$oA@tVT4@bQu$L=E`o z0l3}+C4@&!Gf5xTxlk4Ti4>k5Uig$?pt2U%$9!uoy%J?}&JKfOa(Vu*{nLCL2^L3U zuMa{id7LA-EBYCG5d+Q_;hbQ4v1+p+!O0qM3y+ImL}e%-i{btOq_^khkie(>(fy54 z00qI(m_s)O6odi=mzBt1k_p~k`*E_LN?r{!x`L=5?i}(xeF@u3=c<0=&jaT?zcSB# z9X`cPikpBHf|6dmi0q{UX&iV<@Q!43u$^fqb$Q*``oi3%oQ9zY085NjV5Yc4LnHKZ z`X{(!>~2Iu(T9#(-VeY2iK)gpa|A@Mfbr8&Qu^dUi>x*1HNpQS5W#Xh2b2_gIPbiK zVgwG>$yba7O*FxRE%7&mTy|f=2?rhNUt^E(W6{8_Mhf7~m=Lp}a)3Gk!#eR9?L9#t zMHehmmV)hNob^MB?oKnh-U;%(ighsBy1KeR_xzf<&c^l@zTolaVcOSLKk9(`u?2sG zK!~mXyk`~UD~-K#rV{REdZeL2{T2zAd1x`>-JrD**t9q^E)Z%m zx53pXi{i*6tb;&-=z`(?73F2kN37cxoqhrV6CjY+f(guZWk%puKtB?=E>YM0QkZ#~ zC>YOMax$_ja#xv|BS1*OoABXtz7HdD6GO0MyF2YZP`ls-!w~bu5E!7F7Fqyuj|32i zCF};o0e%r0N`~~`-*+yPBtr_EvZNh1s^{K`KibDKAn;SB1GZn>&w}e+J?-mT2&>kf z`!_f}JXS(TlD!llwB#9&bcRmqm3t73v1|-njMRMK>r0KDP))R$TlMz46}TB-l3Ixv zUWM)BZM5`FBx%Cnb5$8DU`cE2SH>Yy z{AoXVR}7fTRGbsxeg13Lo}uZ-MP3 z`Gf3sg>m_*_~XZYbasOb3C3DlTA*xWhC8UNf6MI$#J3}4raj_TJ+yPcU5(;`+?ZKiMI3y%d*eoOmk|18IIwGr}GzKBNMd0QG>JGw)AuBJOx?W}^ zULjpL?RO|M1Nuvq6uJ8{G8c8XAXC`KMj4`mvqVaWRTAjlDV)$W!UTN!bQ5g{$F~e5 znPpk-Y`eal0aD!QVH{hEVKO%hLjj@4rP ze_DIMyL#u6=&3`h+(jFAxHJ8t%~_wzj8^ZmZ}kJmqbtb5(}bzj4I9p`x-=W(5)A$Gh}h|qM1!{Z)N zR$;RiM9H~}D*U-YVBKi+ef=sRi@h&AG7=V!&M4F5!hwgNI=Q_hDbU}j*w}kcXJ^MRg~eAb!~d#WyzqwNiSIjLXIz@XPd9$*%b7Tv;`SQKS^0gv5v z-Ja(oj0CZIlZ5r)5P@-_8U|~G4MXpso#V~UgOy{U$y`-W(Lg=xUcnH4L^o-x5$9a= zylB+UcQrd+QbKz;HDBe3LypFP7=a47O#lw4yOBkaYg4e;*jQ^*?_EK;cvpj-=1OKP zr{K5Pu0!bm+TPwSZExc*F{>~1q@zX&aSOLvx|w@IGd;R)wX5aQbF_H6t)UR6h921UYoabUif$gE#DIv+GGBhnYj=qqvi80 zLCu@1Zr!>CRKeF2eEzBX*CP~fq>Sg0@Yw9qPci2M`tWG zSiRuos1ON1t?EZqe~q{V0H-4pqYW82Fp{U)m6O%N>#9mXV(eN|rQhjLC52LJ;Q`|D zZTI$Fx=Hsv%Y5s20UTUNv(E#1L#2cW96!a`}OP>80I7I0(1DP=7&+ACEXiJYwIHPqTAi(Wem}ur{~y zunM(Gp~>)vtjF45&O*qvG+^1YBqk!bDDi#J%O9)|VsoS4B@n5)C}$*UiC zx3sk2-4v`+tf*EERzqk-t_+sdnmw@G<|WuY3@j`#lW25k!8&!uNk#->pA>gRTn~b!sFjBeN|QJHV{&+CyYj! zi?=R+N$2H8RbA}%!e!qzQ-bF%S{^;fYKXDoKji19^tet$L?kse6)n^%lPhy5yHiF3 zbXZi~WKLef^if|IF}-S}m9({%wXOAe7@%A@8%Z>sB?y1oeG$U*Z?RhZ*gBlJU>$M6 z&8Vm=8E#-~Vbn2+evyvT3#{u|m#l>@JBG1R3qWA%!3Rzj7JU;FVRV8$hbf08#|e(p zXpkdh_k`O3eURp_?Q%gj3Vk=u5-Jh8D^w;_u25Z-a)Y=6etow_EQO+LYAS+TXiL73 zd@b21*-gw^!hmH1S)^mh&7bMuiU8sX)(ke=h%2-fuc*7X?^$;@1BUUDmwg;nHw&v>!LHx-bv-Ic6l3!Pa8OFFb zTpc31Q_R4~2wG}#JlEl=;O+SWb+6`JWe#x_#p>CQ5eW>0l{eQ%_??L7;h!mzqEdUL z_DdasiB(&J?yTD(CKh`2s;hjQnwlDhsf<*)U0t|RL_Gt`=OD3sY?dM(?s7b2GMTty z)#B$nDl>6KHUuA8ZA?u~pIyg>*4WsHcczP;T20*&!yV@ z#rZYr3I1#684tkCm-|rG^Gi5UC-7q%{{7KM!Rl`fd&#y0R5%N7vLLgcQkS z;vWC+D?Xxpn~R9HV>Sl9iWa^|e2wYzCjMN``T8JXy~RJB=LK>(JML<;ieQv4Ufwdc zJy-BZ>3&vb;?D~#=SdSYp6%)xlnJkC=o)PtSV2QJb^RtjNWb$ ze&lXm`Z&}V7x2L$V^zUOKbwak0lA)7+{0EW$En8~jt>El(2HQ!sECM)x-I4?kiR4r zH+|3MWZvUj>`BBkAhyW50ecFPG?+CA#1U68E`1)m8Vfma4JL(3FIM{9sJ~ZwqvU|p zr0c3uvI^ExFna0N=l=p22pA3ZFa-P{y3alu1K_H?lbS=Wv7tPYU8Yb zxxGvo##x)DcK4T*WCX=`uV3wVH;l&X^0<_f?h=;aeU#9ZLOTNx1;B_!f6j6yX<)$a_-hyDTZTa##)f2?}T%h4ov zo0wIQjo|-4PfzdiNaM(nogyNX(_BynyL^QpnEvf5EN4~|i&b=4|FW*~7hs7l2{StS zFEh8!kT7!}wmVf5@{#Z=yut%x0EgMV81`fBTTx8AxiHL(%b+?hqg&M}Dnrx( z+`l%8ID$~@A;}_FqnCxiFBD6dt!PRA`Sa!pW=~FePJg$?RZ~k>xhTCPRsDSxr=hud z?9H12*FHc6v(M~++;KW)avLu?nj)@q#w~s54)eqOPUaYKf?8 zrHci^jr8_x4DPYkF7cGp`zBE66!v>zxSf1n>+AgL@3(3DHy+);CTtC}&#~fEJ=UBK zv!e|&EJrm$%uc60>hIGF_sK9ED0*rnE`I8f@1sYPzM7QaVDXsC2fs`y{%H6h^iC`~ zI`mH8kNH=pzsxFpeY02St-I20_qWl!iu(^7@I8ONKkh7&p}4lKogJ7AQB2>w#Vx2NvE}MzmSAK}+teK|Y$2&y ze7#Itx5+J?<^zS-^JWt&PP_j^>fXV<{=Vkn!x60>IthBI8oCRxMb&3WohF$wG!wh` zDTmH0$gZFmAK!hNF>+t;%5{4|ckW&-i8@FIMn=Q@w5DnM;szw>4?iV~Oj|s2**CI` zbX{=DQqc4Z_M@TJZTI0MK=Swq0(M$j3U0kL(7c*gp1saW76c2ud$(?=g@%K`zukT0 z;Rf>x`8}3)bas~7&q7?W>VMQ-v0S> z32Mc6+r7EzWOFk%R_l@DTj}(WbE010G?}zlRS5|QTxzO`*ipq^Sycc1gt^w?!J4-& zTb-Poy1KfcT=W}}hJH(LEw)Bg--q_l69p#%5f9YE^1oX{;?Z<+T6a7LhMCHt6hUO5 zo%X1NP!Y<77>B=(&dn=V{=}qEp6;YO@cD4G*H)_()j1X1pvCXlm@VdQI zSy`EE5fsf!W?boOed>uHCe%%gHx@v%Y%xlDa&OqG-}vBq zUWI0K{OgDv?4kz2PxXKa5xac?X|8tJZC3KC7TGsVDN%Zkkee6 zUgvfydH0U{DJkFO4#MqtGXRRhmkOxl0Vd7MB)5^%3f{CKYKwv+ht8gL?=U*uKTola zqz==OuD|Y%@jp6*tU>n%)#x|jQJY?EDLy_mHpcuVF)0%zfO}mxx-+e~JCp??xA6SQ zHG}WA40;Dr&h^qL-EOA|a$c31HF8V_;801ho86scJCQitmE} zLHM~`Z1}CCcov>C=rB!>jtaf#d!(DZDxntvE2_nVBb$wQAPXjOi;&~UJTe+j9)pO; zm)+H=n9Cxc_PwrS*UnPKd^&iL2H`tCSyDjqJ*g`d0$hQ;PosJu-!HXs=Sdd}l;28U zZSY40OF=$n{;O*ZLV|*UKMOil?#+s_-o?iKqqmwxqY^dOjlI7LsrMn^Xxy~d{fh#0 z%#i}?_Po#w?S>aFUCU)Whngwu*VuMcem&3{-Sq7Vl2LxxiyImmpd6By@h}>p#qmR> z9!oEoH68y%BSN(M>S}Nfdk^=*>}JLT7G+!QGxBhEKYq#1#DqgpT<4z3&w-EMmU&v) z%Ul&2wvk+*)`1MDcE&SNfVQeslpniIgj0Zql2o&q!1u(R1?r zm*Z)VQGglzyW+bL?{Z-pJumiacAa>w;5!p8D+Qn9xyQG%!MR*UHEwH! z;j0G_wq(DrGwu0iC_~MTc9ZYCk}ZPO)T`?++OXVNLtz)Q_3rf@e$wRi4%LCdhq3?e z+_{r#NZp?SEk^1*#^Qae`Ps7uJ(XcZifI6~E^OMpsmpZl6!bZWJ+>@Ss{MY6N$F;* zV<{HoyFF^Qwkd4d=Br3C`+Jy%zesQDYjKK?hya!61rf4@Kx;=w$9abn{k4;@=VW`S zxvK|c!v@M!{f~KAw_H#;9|7r?wJg7@A!Y%2{n9~QU0v#t)rT!^Akd2!QQSdCM^{nA zELZSy`Ct9-W%>#v=@g^0vODb{g-Si5NcM$)81CId$j|(ceQtSvEYOSJ8F8pU&Z8ri z3p4Dsk37D;XW~^rvtI#S6;r7We8?l*o6T8Gk|S~XJFq6T$Et5D9PX>3(X67H$CpLU zAq@RxbNP2@^YB_cXcN;KGv1j$q_Pz$I^_lYB?P7P?usgeP`~^{+2ok~33F@uKI}%B zkA2wg@txl&xtU+Wgf>At-4ZhkXgB5!?!NHP+$!n2n1VNb3qPZRZA`Uev}D2JPC+1o z$bDU1`!t`NY28y;8;Nq+fVknN{FC~APkBgk8@U1<#FCh=RtjR%tNY{Bm}t&ePt}j@ z2fDPA-K%yQEj0|$*~?ewZ7)%bGp|t!unAFj`YETa`So!^o1C1S{HyI>>om)Y-Wc-C z70v6^D z8oPJ#wJ{8V%3&Ag>{~9B(lR8wqv_s1V$!nzgJMjbc80<=RoP_aQlIT$v}a~!CiVm^ zt+s4yy61!90>>U|7M&a2srBGg-MwX}XVCD}`yiAlww7<=R5~wLn+}Bq?RW)BtK-Hq zhtB@w-Ml{~=8xq8boX+C)wg}BI_Yt%HFKN1yji{9sSUs*{_}Gf75HBDgNWqa$B6~q zk~@O-qDPN!T`)bnRdQT3)JrG zfJ`e+{s}lFS-s}$s{?>JHVS0%!JYcxpe*F@DL<_(GVaQiD#RmGqozME%nNz;L@``D z=Vl$m$Q|i>Di68fR>J>kZ!aqOJek{G=zhg_*Jd6bt>Z~;45YKy)wgj)Id~*VX*lJr z+C+9YC~!LFD*7J{^%&iKJ@a1Ltf!oo?73Jmq^JaH?Q6UgBk!%@MmQd+3pVz5k)oXYsKR+Ke-PU{jb3fB!?%vh7 zawNx)oj>cu2J*}Oc4`(u%TMFH2?vLVpzp-Q90d~KmUfWDG!polX!O`T`d#g-R|gb6 z-$T%ddbM)U+@kDb@}Eq1-$AuCfhQ17 z%3c^pFZlXJMczI>QLxb1*nmLmHghmy?sd&o-SH%bmgmG6tWCCKOK$gQL927`2?()n zWMeY~;|UJq2P1H2lasA6(1gM@M6=rx-|PyelDN1APu3-lwVRdLj=lc2{vOGy$-%>B z>UHXvh7%o$1@JWw+QrAub)^|*G}^Dnf?SXJg4&oUZc=G{J%^eATbQx*scJe-Qtt{1 zdu%(1MX3qH4MiM3Gz{?QavloI3=9kg z`O1lx8`hCXN3eaAinLM5d01)a)(CFgJeGtN`x5kwRqJ`6P;=Pd-~ZgXbI#7A@LCE^ zHxWX62-Uhdd#T~1is`vu?uGOuD>@JV@_Cd|xCw>XfkCEM0JB*Wmswjk%5cedUvHrK zUeAH>5jnx}iHY+Q{S<{3&AKd0<@uxu6Q`gdWF?OzD2W>{Bds9seLTRcfmO8q+--uWC|8IRMRwCLxRSMgnccEME>LPKp1tD`eXg5O0toQ zOLiL_gh2$oxy~&+0!)U;Qo{of6}rm~f+?9oUQVcQ-aeE1?Zbl_ zI7Q`eHX9$_@{y5QU=-TU)mtp7UKG#fBF`NXjp>Hhlq30_A^U-`Mi-=`84q>%G%}Qw zl=%4gJYpfd5Z8R0(vAKiDAQ)?1|qVuvhoxC!>=BsQf@av-ZS+E(?6j7gH7EY5z|~- zxG~7(@mZ)lj+_l@Q|Vo)d496RzT4^0A5Ys}kwUVI*tfW#b6ayytQ(!{d`ZWp(syL} zSy9oT(6(t51UsbMXZ%?$lt4rCGqtuD*K&Of+aMsZ*+wI3XDu)4!@@niP-SBhxIAY% z=OHWkbi>1!F&?`8=VvC;nk_@~(0d!-u{qVw77Gi6!n$v(NqcmGJ|=2Ppy2V?gN9b0zsO08oP4Zn5EX4|-48$?RV z5uQig?__WFZKMmp3z_P#zceMGml8bPVG%<0Xao9YIDAlwDAq3W6&?@y?2b6h1n5TR zoopdIL5c58cU#fIgxR%-mk zee4!p0EVejTx5NHeMse<4yi%0?~lnzyC%_tC|)=2zPe~gWF-G(*z^OU&xGf8JSmpcvN1g>V_xOc9UANbt< z_?An2uS6&O%9fGdid4CNY}0LD(@xX8LK0DW0S;7VULuXCl_$1%`rqY8WrJNOp_cc> zlhQ%ctU3F8#%~m^xzP?- zwjRDO}>kcWaC7zS>LGS?8x8^tnj0G%kDd zbEswaq`38k zada!{yhjb!aqMyv*uDF>@@lmv&UyLSmWjEt!ZKI>T+iUK`CcWZ_yXR&Za^R**FzhwY^FXu@DJMc zyRxgYe3+pS%a^ow777h56;0A6fCD7b9wrJqNh=pNZIXM6ZWY4@|Kn>(q|!;g@#O(& zjaioOcA7{}aI4|Ur)zQL8ypW44@Z*7aWN8KXTU%4!^6WbK{6R3^gWR1F<%KVDg}IP zl?WB=9U;)NaONrpn3FmUfp|%S z9B?F4JI?^35LHpScrVCcZ3P@9 z^|CEqj35#7#*O>a3f%f$<4cjm%Z{(W{ix=!n?y7%wF8@zd6CB^Ajg@r8XRWsFMgJOCvf%f(3W%rGPIBbD}p;o1nZv z&@b%(|D#snb;gg_*?mC{SNJEu2r_P}i_hANXKl?od`I$vp0kyem4-$L923&_jToiU z9mJW23CjK!kKv6_c`yw4>Fgw0$2JSi8)RBhR|PzlIuz4vkGs8M6|>KahBc zCI=u1fLopIE?*nz4f(5K7)AmPlM0Df$?Cw4kCcJ%&lf1N!FdS8S5!ubIdT|>2&UjjA$=yC3-ZQ4QjhUCVNsl0s;aE_91bM*2)KoeJTXm5QI1cta*tQ`?*N9=C`RlyvK9Awa|LbNa|-S(NLX%c z|GIGXP2Xjtt+II4(ndx9c`BSHc!agDU*96Wq=iTU>FEfH)#BshLz3VZ>&!(EW9@bk zADp$qJ_{rr?hS%(3~G~PF}FF0**MvS_@$qpU${3={hjdeZHu1}y>qq)N%d6CsQV4G z*e`#BV!qg;%ZNs6CnYEEFnFkvRsin}r}BjpPntk}gY<_xFk;E?Vdrpf#M&e*sKUez z`++4%vp+zJ#jMaDKz?PK;E@p1V~BX6or;i$Ma+lPwaw2XZND_zzQT>|6s)mCHip3Ta{J@=aQ4mUpH zeslzb+_y^dPoR;LyXT>RMd9WNUTUt@*S~I6w1F&$XHTwM>W6zcK8GETjw3AA;-lRK zvR0+BmnsqS1hYQ&HqH&f`&wNgZqUa-DZrL%Fb94y;xDjBoE#keU0u468%Cgp$XE_l z!?I-oOJ~|A?hk}NJO3kL0XYSOq1~?Cs&^k{Wl7W;WkTC8=*ks+mcv-mFGJ3A_OqSj zBa(FPIQ!QFih_voiywI#oy)+ryJ-ZWckF!+VP$J?7Da&ckLiR56#Fl{Xg;iv!v}c+ z%NOS-CIB35wrrP?!NDCD8;V?e)N`D=%S!e~sx9_zG10pbKU6yai1a7*a+6+50ZUuDH|&infTa(a6f9SIg9ucb$kwf$ctoa6W0`)a-C?L7GY^tWyB9|? zdj~O+Zt*ltq+xNovk#C5@*%`gn@DA+*brsJZ;9h@*oupbk$Ca%Dhou?2=$8eX9+7T zUu!vZern{{p4+Hm%y|s}hls~9b8}QavgZg{uDbgU9^`EhUvJ+&$+?`4uH_;)JY#R| z#4mk2Eeno7Is%#Xm!OD-oaTE+5K|?QL)cAp^MctvPNSV*1j z8kQbEb<=y<&}w33#0rh;+1yU5M%W>3q^s*?i3_5x5<2%tMAA@navPvoYJB(!p1)}jK}Gx9{wN>Mv>DlT=fzoozk9F2 zoh^6rK17jKRBUf*iV_V^sc^#Pk`kCGuY9SjEwJG8Qctq{)ss8?Hdb%jwk@0L34CW{ znr*dAk*L(lJvRpqBF7UDe?@x_J?R}Esu?xm;j!hd2Pc1DBrp1M!Kskpk z+J<7CkRAWWx$BByAt6Z*5ZwEfZ~nwQKb#$jzj$BXfkVZ(^@&G_m7||-$vaqx(FGnU z+?NbJ!A&xT4|Ogm%ADiWvV4xB=+}J)Yw78^cG=P}l#IvrKHw&f);Yo^D_em2HVcNe zYZJRs;$5@j3(j4%jacdo@?V`Xiv(b>T2Npa>Jf4uz(RkiesdEm$2Q^0DV#x^`2>^A zEhds;rLj)^FlL;scI~lMADXTsN-_gDcf#{NJiYM3)Y~4O0upH!#qw-vyM2mKMDJ~O zPvrt80}#dz`KaXR=ijq#vMt%9pe37R z%n-;r&N?h!;HJf8ZvI zR2M#fuKydDW9}>!-tC6jJH(wmm+oxz_nlE}56EP9<|e9lZ_FU<_#4&-flM5~s_QY* zkU`F2z;3EXD)t)VKNxXr^}(EYw05g~Y1 z@?4iC-zGBf!}~gK0P~6t<>}ekYppCK3%;d|K_l^IdtiXa@F!2uNkQ=VJAj5_Sw1aU?^h*+nz4wsnK7+U!9qlo_JqIY~kj@R6JU#>;vHlV= z#0NJ^+hJ~%9BQdaKG#E1=^#KD8-La<#c}Wj!*Dh*^;JOd%#V0XrV|go!*`EmuLk08 zK2+SDrJH@C|8t4*>;?mae202)N{-#Rz*%!o%6^y-+s(F`JPqN=VQ6cpr5CbJ3a+=CMg zpZM;cGh7SOQve-eEfd47va>1akeL&0(H`ilaRKy#k&F^Eec#?*3{MZb@AW?VyTjJj zl%kW6M}YI~xA1#vhhci}@bF=BoKYg`kWd>-Q;I|JN)V)P)D6^5B68cdUDCA+hDa+f zsbCqVi%t9Ez0FE1twtAj*jMfyk{h++S`dYE$Wv8Qt7e>JKgv)vs_snBk;wi=B0VC>-k8Z+>>$zex zvZ>$O@>6MLrx)?oCz`I<7q_g-)Fi$s9ZX&RyyF3hhlyy+(9HKtS)U7@(a$K%MU8L>z=NP{&&6{ya@ zCit(notv}wiK(lpiI{bnd2!1(svwiNHw*S68q8=%Vjr+I+Uz$k{f9=hu%8DqLlS_!z zJaS}mY%CF3qk`9GCf_`s5+PMxB8KC+lEUD9sjxi<1Fu|JS3f9$9lx+S59O5xME1PG z2CraZFV&9Kh9pdAh=Q#C3GmIj-xOjB>D=%AaLPipgE-Yf<-&66QRH!&|7?Q1K9WVh zn$BBt$a@I;Sh@B>USdl^4DIvh1suy!)*#v*=IGs<+KA|HeGiRxP-!m#2u>ySYjfXb zN0z&KJLm(~>YC!4|6_dHrPaMB`uC|J%|ibSp#$Q;P}Ddep<;5=3_+mLLMUisnDhdT zVTRNqN=sE{rl6JH^^@R}i9(tZMUqx*clR0!JD~h?9vKL@i* zqy|&5iQ6K>jm>V;QATTPir%hARyfl0L?f!ZbmHyu^xT-qusDY~Cy=8r_WLdW@nrSM zJcCDO5k!%+ z(gv|8x2@B?9Ua|*BVh%7N`1d~UdYJU!8VFyj3}5{Ne>{b%D8_0MT zJom;Sk(D}=>vOKn5L|(q}B*i4%2rd6vgs4Mo*yHAdw{lql3`2YJ7Zk<}I(aI$R=sdVT~ z&DK1e&s=T;hStMoobr^@&oShku-rv~uw1!=ukxHGLqbDS=miZDn-X^I+SQzD018{e z`SG=em@_Zq^^7LR$GxHQL%>BP$7V@UJ#V{&Z{t7ATLhi{6C}O03{js`^Z zj#{;!rr#WJY*e3geKq0iob~l;#1T+)DDDa_kzb39z5yVP9dyVApV9=pq%u-83^ zx4@zPA}{o$bo@?i(Hi;mQLnz|NSkSOHO}zx^7pG+Wl_z16xD{SXpH+qp2&@Z z);ISqwZR@xri@|2ajDY;^Z6S-L6T_F1BvR13B<47b=2(puSGRSB$wUs| zW^FGK$6JlmuZ`!Vu5e!iw5C5?LYsaZdbrzg`SRt{j69)vTgP+RM2sKDUi~`0Nx~vl z6qFBE`E_FIm)+(Zlat^G>kh8ub zgZgJ5%m8~jM1vvl&f9Dgc{L-KY6XV*c86#G@9!}osTvVUN<-o+e(Uz~nms$jNit#o zXE)ZJIS(~p6USTq$7}&W( zF}r>_n*;|jaF+3+Catju4noTx>4j&Jj~tS^K?O!;1YZPWpQ^z&d1-T`S_gu2xiLKM zLrqPpL(1G~M++V|loO`+Z~N<=7o|RXq5I;>b;8rg*EY1bD{kF-4{=PyCvO+IV!O1H z8U)$P#uwD-%y}$^tDYk$>Zd@YEHM$Cx00S!*!ZzI(&|9oXTQA{%!xTcF;4!SLPi`( z+8>+*Gj_Ajs1CzfBRQ#{PN{QKfay%`YjK+OAy`?jv02VVWpp!l{}^t2@@YdJLI?R? zs24y2E-W}$M=XrWHJru_dNN!#vDpFj4!Gb?CA$yr-ZnHxudV$Kj#^>uk$+}$;H$iL zDOU2C&q$x0%n`XhXkkv%?KB#i)3W?GNY=V}HjFwH&jAy8wQcJ|{Rh{~YBae_l+O&DIxNM0^yYMKOCa)RKuKYI;ced&Uzl5ck-Sqb^H>N9> z{srjhuh>67euIMOr;M<$13JWe)bfkf|7Flv{wgo=^PQEBoUPVlOn)4Wx;o2B>IGUQ z=|p>Zo&92E3`UvR6z1w{Mzg2X6QL4?J!DQw)5S9m-AXG-fWxrN+j-;9>}@V~tUPc- z+?lgSb7T=zE>2i21%+0(Y@ZgfHHv%6{G!?jrW9gUOJ8Cp*V6xwe>19eU{*_*j05)u;LDjAhRQFoqM zY}Aizn<$CnGcx^9US5u@Ie^v+EVK?YSk%o@HYE-41RPK8jvgtcv|AGb3g!_(sp7)mm zVDNpglSCC!V9jz@+w`WItL4>pjA=vuAhBHF&Gfs9>Hg#!SZJlsSFT=t%~wu;KiT|bxVDT^=X9(0y9u5sxT@J^?&l{`H5*w ziqAXsz48 z0xyBaP0aUQW8X8mhaQ_3B7U~N0tZg4-7FJsH$6qOiL0Q%is@cv8w4FOrRd6))l7n* z))^cKOVU}v?4!pks(+v1bT>s7Ia%M%0;HO9cJK9>6swKgq&FuYBd!iVD|Pf)m?W>a zM*)M+RX~tDiK)Hx7LHvHw4(VNf6csK{v)YyFf}SsPMwv8R9ZI; zS}%du)1e2|_^UFEJ98JbExgTMoF{+N*B-}Vc{y(t#+UG`>sH{``?fdUI!Y2o3A7>?|A$J(g7UQ=h=^&ARj4!kg2~&3)l)kjaU|jNpVb z{!B5ws@CJL1O~xi;mLZs!rRfUjdgpZ%gTM}N#jKB_aO|=;Hu|vkJ6!zZP5So0dgK- zABwbO%jLpil*|;DzF7TL$O&dXDju9aB_D~81RN0OC4=?TpLXqLmNvMsHtx?Q>^XX~O(?sU&Bl@J9L^@FAM0b~|Y%^BxRZ$R~{XI8M z|N5mnh{PpXZSK8k8Nb9rZcHs1b@DAfX-M3e@^>v#UovejPyZ~UR7jS4_1CxW3bMGk za$hmKa?V!aBQ_(L(hk+Nzn7*o&d+$JxAB{R?SU*>b|Wn3hY~p4K}rwzoNnd%6r_IV znf5Z34kG&&arKIQyOsTjP@sAlg08hQ;Zfp*Y+mj;X&TLj!$x7E--+{pgtgHI>2t*s zyYhR#%nvhE|GrE3`M=#oH4q28!L!#5zu;Zv+=pK&vPq55u&ht7zZ4Ka`gUY%sOkS6 zIjjU!%_YEziH$rIrbg#L^*;^0!RV~c6sc}0AXPg+#NWkk}ZWk%jS$gBLiLDQ% z{QtcnxeI=afL3!6&?eC|ALUyhN^F=Pr5TU2KnCm@pUUJjhDrXH@ftyI54HW-o-rRI zbNar+sIFjR7@^Kj=;Y|HFd?|{{qOg6p6cOWQxb$5Ff z$gViULBzsIA}RVuJXi8HYZ|%@xseL;-SNTGMH1HD8*H&b6^Duyn)?57z^&wdxJJdf zUFP6YHILf#ZXe$@w7_t}VQkg<{|`HS2ebx6GjzN}fiRvw_d#hDVolSKv>Mmi|_TkyyV1JX)2Rh$eb)q zH5&vpoo1FcRLCw6V0Gn5PWnw^2Y)~icvo@iRe;q@X7bwKf9mBcpUbtY$iqG_)sp z1fs193ZIL8h~yTcJwY@y_W$&8L*G$6^6DL_l*_>xxzp^;L~CcXxc>^- zI1-$ihs3E;!z`kt>{XxgdgB;L76fGkKQ1n~8b6B!7W^)KiH^bVfJ;P>8$YWF(g%Ll tuLsn{&myM@zZWS#`2Bx=gOP=0dh`cZR7q;7;CrM)YDZKb@3*@6{{U12Q|ABx diff --git a/doc/design/img/sequence.graphml b/doc/design/img/sequence.graphml deleted file mode 100644 index 22ebed503..000000000 --- a/doc/design/img/sequence.graphml +++ /dev/null @@ -1,1267 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - :User - - - - - - - - - - - - - - - :Login -Page - - - - - - - - - - - - :Login -Controller - - - - - - - - - - - - :UserDao - - - - - - - - - - - - :User -Session - - - - - - - - - - - - :Logger - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Key (UML) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Actor - - - - - - - - - - - - - - - Object - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - login - - - - - - - - - - - - login(...) - - - - - - - - - - - - checkPwd(...) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - new - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - register User Login(...) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Lifeline - - - - - - - - - - - - Execution -occurrence - - - - - - - - - - - - Synchronous -message - - - - - - - - - - - - Asynchronous -message - - - - - - - - - - - - Return -message - - - - - - - - <?xml version="1.0" encoding="UTF-8" standalone="no"?> -<!-- Created with Inkscape (http://www.inkscape.org/) --> -<svg - xmlns:dc="http://purl.org/dc/elements/1.1/" - xmlns:cc="http://web.resource.org/cc/" - xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" - xmlns:svg="http://www.w3.org/2000/svg" - xmlns="http://www.w3.org/2000/svg" - xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" - xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" - width="41" - height="68.997391" - id="svg2" - sodipodi:version="0.32" - inkscape:version="0.45.1" - sodipodi:docbase="C:\Daten\alberts\projects\yfx" - sodipodi:docname="uml_actor.svg" - inkscape:output_extension="org.inkscape.output.svg.inkscape" - version="1.0"> - <defs - id="defs4" /> - <sodipodi:namedview - id="base" - pagecolor="#ffffff" - bordercolor="#666666" - borderopacity="1.0" - inkscape:pageopacity="0.0" - inkscape:pageshadow="2" - inkscape:zoom="2.934351" - inkscape:cx="144.21983" - inkscape:cy="28.533711" - inkscape:document-units="px" - inkscape:current-layer="layer1" - showgrid="true" - inkscape:window-width="1280" - inkscape:window-height="968" - inkscape:window-x="-4" - inkscape:window-y="-4" - width="48px" - height="48px" - showborder="false" - inkscape:showpageshadow="false" /> - <metadata - id="metadata7"> - <rdf:RDF> - <cc:Work - rdf:about=""> - <dc:format>image/svg+xml</dc:format> - <dc:type - rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> - </cc:Work> - </rdf:RDF> - </metadata> - <g - inkscape:label="Ebene 1" - inkscape:groupmode="layer" - id="layer1" - transform="translate(-29.5,-42.959476)"> - <a - id="a3142" - transform="matrix(1.0873906,0,0,1,-4.4741999,0)"> - <path - transform="translate(11.586889,5.2908993)" - d="M 47.02914 47.36993 A 8.5197716 9.2013531 0 1 1 29.989597,47.36993 A 8.5197716 9.2013531 0 1 1 47.02914 47.36993 z" - sodipodi:ry="9.2013531" - sodipodi:rx="8.5197716" - sodipodi:cy="47.36993" - sodipodi:cx="38.509369" - id="path2160" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - sodipodi:type="arc" /> - </a> - <path - sodipodi:type="arc" - style="fill:none" - id="path3134" - sodipodi:cx="43.962021" - sodipodi:cy="48.392303" - sodipodi:rx="3.7486994" - sodipodi:ry="0" - d="M 47.71072 48.392303 A 3.7486994 0 0 1 1 40.213321,48.392303 A 3.7486994 0 0 1 1 47.71072 48.392303 z" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1.24319649px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - d="M 50,61.33709 C 50,91.363211 50,92.247838 50,92.247838" - id="path3136" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - d="M 69.760668,72.362183 C 69.760668,72.362183 69.760668,72.362183 50.239332,72.362183 C 30.239332,72.362183 30.239332,72.362183 30.239332,72.362183 L 30.239332,72.362183" - id="path3138" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - d="M 30,111.45687 C 30,111.45687 30,111.45687 50,92.013532 C 70,111.45687 70,111.45687 70,111.45687" - id="path3140" /> - </g> -</svg> - - - - diff --git a/doc/design/img/sequence.png b/doc/design/img/sequence.png deleted file mode 100644 index 3b23890241809581b04a2945c4cb09802fb97b27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 64902 zcmd441yq!4)HV#F(j_1w(g+xU2+}PnZ2?MyV9-6ZN=l0gN(&+?0*cZljUa=HGB|{^ zba(yx0rh|<-tYhZJZqhGT;t61+_CSyuYK)n-zP{@U6GXNAQ27@4ym$|+*ur)J?uC* zyPF7h!6$l8%G_{pd~lTIWX`)9jCPVQy6eHaIcVxW^eoe4Ufi@4g#(Vp5>5JZ8Vv zQgiZYCXY#(05S#>Q`2MGnI0WZtrCmhZ^J0v&Yu0$S>hhfAm+S0dyUTm2F2vPI%T-A z+$y!&O-_1HfJ-|wj-WFrC}`yCOELHP43Dy{!TLQW$Li|pc6o2CQE7;4rKvt7qB;7; zrps%2c4%c`B6_etIx_N=aiu>cn}UGVx7kNGGAZA=f6dTkjcShlC@dqS(-Q z$P<`-@0_%>Ff;RjC2ckI_SDnUi{{ZUT^>G$QZV}T>63Y5j25igjm!e`@8vgHZ5sHu z-kZ!yjD;_L%eU)&nb&#r?!3f8dyeOdGfLqEk6wPZS^ZrOwMX~3FFXHePJXH$f7fF3 zXB0=}sdBUQ3hv&$`{}7hA_jv&R;XuUI^?^4vFlWCxK!qa7ZZ92^5KWh)(pP9uHBIZX)?E6XEU(cG7FVUd)Tm8VZ^{(wbmx3QR* znETkbw@$Du1|`cObo_nY6(SyslWgJQ`)FxhTJ>Fmlz7QE*TATZl>l2C&cwuLA{Vsp zu-n}5{e%P_a@wZ;hG-$HZ|AcNBqIB0kLxi1`nqO-6{SC7JShsHFcL3V)ah)Rv**rj zOh=d9JFfd$F`RJ_ofsXkb)t0BE1fT0`t~BL^dJL+XDx$^Ua_kdV{XO$W7?%l1F_F9 zy!iS;8$DYawE1Zg? zo@y+{$!Re&@S*?x{TnUm9~Zj3dC4zxN=U4Y<#mPKxigG;W$NslD^7xZ-zQ%7U@rg3 zlMi_GFJfId)8iF}DfEAqbOsEEfcagF)CKzH+8uHT!|1l~~_0=H_@1xq8?RAlC{d_OR+H#&c3^sjv zp$+!E(9efF6K9nAQ?}>qkB_yC?in6S({RqCYBO@ZH`a?LN{i0iKWbCDlqSCU`pw~z zGfA#CpC-SMuG!rmZTO%z!+G-iyOr^xEc^cVpPy<(*KR&E`?t zecJ{)-d-To<-sp1TIjwo9?fg`AmC+GR8-EDh6hJ4JQuLU*uv6JpP=wbO-^11T2Zg zoJX&A7NbHBo|0UhJdJ7;g#+U{-)7O<)3cZC5LY&omJ4veq62Q0$6){CG5g6FLeLg)=0WUS#_>h0vw%g3Y!sgqX zXRx|ps0o5rkHkrKH@1h#X6slG67RAZzUWtILAIhvgHt3eKahEe(swK)BLlvA5dy~b zFVAgC<{tQIaFl_?tgo$r13ySb_4(!HJh0a~ejlHQ|2m#yx3alcS7*>v9I9OB(o|uq zqwe12(Jy326BwwgH`PaRf@8b%{QMESyki$LA}ny;oybd}IQ~_);#PTV;f3V!hn14& z_muN!Mi#udC}4b9*!H`0SE;8;G>>O*s8HtRyv&HQu2TE{J4)Zc>dlv?`)_>B5V7l# zi!A9^mOjxq#0mh%AMPjZ>fy4`}$x z#BZq8#=^n^37(%)m1&NiPrvx`8!>Nj1h`R+UAN4v6$n*bJ{kg)zisEY(PhleLqJzms% zeI~J*62(X%?jT0=&LsZuaf0G!muZF*L`F|vz!}{|^RVc;JYBBC%%|y6&y~I8jCybk zA%+Zex=-jA*!Oo=24pS27=kz&C*kf4ks3DXA_Pa3dt7!9Sk)3GCDJp<7LEUX(1De? z-C!O$7U=bk2`_L(1rWxiz=UBMuLwRVh8judh#PZAjz8qf;tMN`b+_ zH@>|YN(q+~z4j@!I+&K%uyi$;w{&q(YD2`Pvsf!d0j8r5vG<79W8c6w22sbDQ#Za2 zeSQk?f~fZ2jX2-es`80SbgoSokC%TKgP7pe<|MFXaPu%HR#w?n%gD$`@L)#_OFdp* zdYf4?|4q_!xfnbnOmeye=i{w5AFujnq$I4hfa3)5M|f@cYm91Br#cz z*HhH)wK6s>O#sM1Ij(Yc=yU(g2!aoy$K;0gBqvxs{^P7|4LRM2Qe8HuiAc*$Cfy=7 z{SQm`wJH~K5Id^l8`YeVBYV$)Vz`at53h94n11p@dl=D0Vc)37G>@Dv27V%zmB7mV z;Yryn@k+rVy z@LQ;ZpYQGMO?mwI`NfxR6D14aTGSJw0Rr5_OG;EVuF9e5x0;K4w` z(Apa;;^WzfX?euKH<_52z?UDVq#Qe+&RnW*Z~xl;(c%AT*3X#%y_Q50fkW7EWh57m z3(Wrb%2cseoxQg|Ba-;btue^8d)T2-Y>mEz0)f10Zd5tI5}k^{`<3JMB$xaSXWs${RV<(>p2>JISg7ruc|&||bP-qFENjQ-~I z)n^~PjHXsZ(F_mjF0EVfp+&?b=Ztcu7^29N$ZbM$^*P*3Qn(=g;3epr32` z(9WzmNv5a4pv0{|aWQyqq?LY%HNtipLPLIdJ`Jyd9*Gq;SB>$+F@cc`iYK(bZn;5! z1Um)HUJw4flv^}L!rKUj`PhsjrynfjaR^kEt=v)4jzSo zQPk~Ee$8M&FL5`xljAUC;Ji)she0U(xgT(-DjVj=<0Ia zEeLrgn_`&Rl`B>qg>#uj<0r?R0m~*tg`y9K1qa7MqOuznWf&wu*JwxAul|Gzv#Uw5mD50oI9F7 zTyAb~+?I|hy8hD{G?pxMy7vTJ?<_75*K~ZNH&|gLvy0Y$&XUUM1*>Y4-W)*GCA7D5 z|B)Y3snq*8F5prpH&5JpAxNipzu?8sgIJ*peiLlEik#-+W#uv_9`^n0(%rh7p#?TT z&36odOX>9cV?yn=ryriY0wDm9Xd4Wy)xdoWXa&Z+mHPzc$obGp@sBw&kY|Mjs$&SK zi%$XzQOD1oJaIdx}tb{3L4q3bPN z92^`hEXg-oGaI6LX}Rm}*UqS2xD_9FL*C_W#9vddn(D2EOd5g^>qp0v{QUAWmtL93 z#|m0iX;9PCA0vAgzrD~Cac!iMMz;%@FxB%)+^Z3zS|gHQ4kHDb$e;z`)V0AIow zR~I$i?)X`amC#*XkdX|4(i@-)#;*a8s3#q@%RWHk@JP^V7^P`uv!IW9uJfuV*9t3c zF)$#}IEHzZB7O?JIMw^4exA_o0*-NAslKKrJxOZ)fm_$90khS3d#EUc957z3=9j`h z)%JY+I*WsLXg2AhxwWDn;lc*x8CEv7$}#@2dj=L_+032P&o^-hi{#8L#w&a2bGux7 zNvE#V-#uJyNX8@?cmKXqUtQ$e8?AQ~eKn-k-VBmfpVrXsyxb2lP!uwU3O{0#fnwB5 zh!}OawFDR04#8xW!t9|Me^Q^D$*xQf`6ZDO|4&GsQ_#^ zFjRMV-WXTzrECY!u(~9YUVqU^n;0^OpSJJe0SPTQt{afg0FPyAYC6*LY-X~Od;|WS zE`FT0EP6R?&y@20%?L!&EdmaCjkIk3E)$o_=Z$ms&r- zz`#IB`O164YeA;BJBiB(ecNA}5nm%t_~YVGm8l`>cp1Q!+S=OShu~10nN*dPkNlV! z6LT3KonDSF@RZe~kv{O+hL*ZzFW$A21u}wnHlx3anNfH_S#i?~;7@8I#E=cF*uH@% zl{04!rXU$?y_D5tjn12BW!b3m zFh?hhP0taVoyO)`=`S%|JV=huWTjIz!TrRIMXIO<+BH8sIPnaV2}nMAM%;FX}s+0UtYGb!sIa@j5b-!?yczmrc2{Y9K}z5!m`LP%yn_H z`}LKEJ8Y2NxeY&ix}Vp;1JK!O-eNUSw^eb0fq^}!JN-d*y*xqDhF)zFU9Q55-)>}~ z+H13Rp$@GAWl%VsaQ+B`AtZ$m<_ycc)_{U3EX$I~D09DLV)9rRsDS8=j%guaPqehO zOdQ)@Uzv^}{T;rPqUuRsYmpojQ1WPdp`Eo#<^9TQ%`JtvV8%f+ihDG3|8K;6FAvtg zB=v}jSCGXQ(HoVedp_nK6^L!a)5{q}PZPZ^Er%r}xQ~l0G(tL- zU)=fq12+qcObCxowlrh5%&+jWYtwd!Os9b>nMN;HLyjE6)7ONSQOnIPx&(~M&phBS zzJ(x)e*^MdPW9eMMaD?{X$U&>W!xz6lc_9IbrT4mh49VblcIECT<@u-4*$geb z8J0A^Oa>y!Y0Xg9bi<2(8uedv0tBLV*Y>YTGI%KF@W9VCy~U7(YcbAb+x052McH(f zb_k#JTy~tg0zq@N><#tL8HQ;&* zO>4|A~z~Iu7uEi!OnCm$7>G7&R3Eec%rSH#O0Jy^WDz%5R zWcXb+&HLxs7h~dC#NiryMwlb}YPrUue(s+AxyVb*x z+X2y9YVkYX(q2)bccL#2n-N12boUO;vG?~EV!^Q1fx()#%5ZMr&I_D?MBSzy@4@uL z7oLnHmswsAHTW$y5-&H>M?S{By}=n4tY`!NudBp;0Z!eg&!2(t3OQrjzcGmkV08Nd zrnl4=r2!6#B~2eHEDaC1*H#v2Ki{F{9bYoPW?CC|%GwLYoCQQGz_Xs4z3~3h_2m&- zLk}%C&$nQOpL&h#Du8dH30E6zd}wM(rQJDLo0gX5);SC$ihX}c035X5D$~y)AtZKG zQ=Z`TLt|b2t==tCLj2=Yjj@HW2J{~Acb!12)bs)8;;x>n5%^*k!Rh(2TerR$`521{ z)hQTzI6CX;wP)z&09|$X;P;wP`cSVMwu6I4oR`1oa|4U^6Wt}bN)_%4?;dE1zahsw zb#>_byW2}P$;lKMIhSN}m;t5vPZ_qJy5154#G|W^*ZL|dk_!Nj+-%8I4SjVpkorfv z8n63~;rOK;9i`X`Le`y%8uaLl`HLAke9ykY*7ImAe@Io161TWhDq-KJgchosW4~K4 zXeLFDzqZ=-;@AWUQd~j@B>Mv;<|?V~1gz3E2-SK9#+St}o$?Bu;5KXJ66(8v>&vp@ zEF)vp>>Wi3MU;?-0Et>!&PTfUqj*}+X-HgIl^1_=LTD#`*z+W5#1~0P?bw zPA?N8ma?088hF`O*YGGHHNl}YRHDFti@S{LEu{m?fJ-EE0mEKpVbGkSdfM4}9tirC z0IAhvNS;lItE#I{S+~0a8Q;nKWq&w97ius*LBjnGp~)r4pCC_}DBD;E_;H4xw99ir zAE*V$#sN6!LXiO~CQudX1{!V!ph=$yxC5CXJ6lLpAP5Y9dEv6q;eteDw0E(&nOUIx z{;o@_a$t}9yG%wguX=Dt0Fvn&z%JWRLWJaIR1RLD6@p61nTgI4?JNUswVVsv%;Ec{ zIj}G)g=`1uJ;~L`q2&w>zIZXFzTl$ccHqIa$InW}QAqpUMEr}WO+(0LMhcuD9-z-{bVcNbnO9GoX{|mLPWf;rH^%+dFh?9lv@6gGR_88581B_@Qoo| zZwR(QV7sCXWep7`2{%XZE3i`g_U$VKQq5z@p0DkG?aELf#Iao=UQEk zJpKCh>(pxxh+!wI$M>}5ScGfgeR5A+bO9iR{o;`GpyPru)6a~{*;VsSvpnfSW>$KS#Bye1J5Rx(jR(sk0PH!0Pm`VL^2 znfy6kD3XOX9j4ylE3gL zSEo&c^WvoO0)1+ARn;M3+j5B|@Z}n+sz_B}VYDqOGSUv*5%5m}H@>Pi0&V`*{n0zu zvRZ&JKv2x|)km4E1EpFed6MlQN`l|a20}&~T;p*z@r9f2V&!A=o0F|g6qU9X2 ztNr?OGf@Cn^-%@dj9i#aaLt>x?+K7zEK}rtz}JQ38QS-IlMl9colQSB5g%3E3<;6^ zEAA~`izf@sHN7EVk1kpUSDngPg70-v(%~)o_RWbs8-wN|MK3BOf3i;BMt3co<2c-7 zI0!TZt-1YyZM(NxtSb+x7_;XftT-gKChJfx@4$B%#Td5-5fCyuc%vSxgAIqWxL_ph zdVHr|6u?<-adT)~t)}1DoU>`4Y9{V;ejSKFRXt_?U z_idM2tl*rub!s!5#YAz`r_mDF#H$kWy>Dq(<6;?* z{q%+G;qKj1@dGgD>$eoky&Ji%?yNRF2{ZVuPfW>RqL2 z4=2sf|K4cxx2DYD{|J=b#0Kv7?!TUQi z37q`_HBkwUbygthVn&x{=cFseIvpm z-iy}h&vy}6IFIfUO|JUZ|KXJTFY;HD$nAr(%*9MS1R@~xg_-@clf#T7Wl$6yo)D2XTwVs2#Wvlk{ zC}xkNY*2qk<8MGKLwMAZp6(fQ0#n+zry5LlW{|5f=Bi%*Cc84VyxI&yI%Dh%e<&UyR&wGf7)ldz z@c%ND9c8m$0e?4RZYv};;$NZQRSWffN00~r4N7;qCf?P5EXga!YSlj*GLa=tfAyc% zJKj+gdXQi;rR>%+gMDTf^>t$m*>FZNa;{{?_DBty<8M1xTjj5xNi5gR04uwLCgy-e z`);=D^|DckSLPKib{vO~eYqK+oPSWHBuO?YvAWx~Dj1I?>cIFOLAS~5v`2y<}Mj3>5)|BQFQkeu{d zz(UxaopnSBqZOGv1ppaLMQgR_XF{z5?%!^~~b%cX20-D-iw9>({x~9c~~J zp}UgfI%R@(pGTH|9#lFkVS3(cbFAX-8|;tm!!b)Jj@&U~MmzsxEm?I7I0yUsfOh^g!yPRVc_|6 zBpX^+gn$td^IU$zch0N{Yh9$45Lvq-zLu&vk-Y{bTFCwyXUOPJUGHO@bI>k$7q=O_ zjlR3T+bGcGF}q+avV%?EO7&^(l~?8>bz=2rAftgS8v$G4b5NE&c5-G5Y+Y6YgmUWo z`P+L*0ipEt_JZc>H6WB;x&bIeelqI@bf}`LO1)qR-V})A76`JC&jG~i@?IX|L_L4< zGX)D|U8Usg-* z1S~i4gaSAh>ciJl$IC07t;;R?`L$QXuM(K?>dm!hAX(dke<+*Nan zM*W+N;b+p_eVmOg1pq=%0MUz=sa>M9`~EI6_*KrMf{Me$*O%DG^1%?)fc^UX=@Wo{ zV-F=AOUn#luCAmXTqRA;Mt|2#{Ct^l4=8ka3?a7xy$~;&=j;|2>3Z%Ti+=a=5TS{D zG8ij}*B~49aCCfAhXFZ?)tBdAz|Hre{IaiqIlA< zT)?Pp7Y%T?S2}|R0&3`&^Q`qjS_kyEeyN9U0xqVqJ#0D`HaA+}NpAat(0tmvpTZV6LH#(T~VkZmqD7_Q%qj5H<*DlUP%8cx^c zzA?zw7NBa+Zc{1w9}hbQ5A%eO1}Om|ivZu}y*6UV&(ANx8CYBbf-EFYEA|9FAj7aM zp_nxGKQs?1+1z)XuU{Vq-K7?sMUWDO(+lx{YAoSiG?YuuLJ5R>+8Lldl&nS^4r(g> z$G8q#1`~Kr_aOyZI3lkgEeB!EBvrEf@vgTwv=b#g36vl}p_c_zRs(OD#~03k;9^9}0kz(sV$Zd?R?fqRl{JL%iR%B|5iVxLz62by$#mjRBYhPjo5IdJW`fck_%{{@o(w4}nEW0yzS(U*0Iw zux$O_k#jtTrN=dn#?)a!c)}!Xo0N|N$XdGCL%eV~8qe=PL^)iQxk}$aM(8>BMH|7e zz!X9;hC4mg3KT!!lf%WQ1M9FzeFg}sGoAj2|6Nys$c2%( z!zX7v#KhF~=GtdC!oU#8B;f7_cku5@Evv{E%1gj~5t1x)v`%k= z$O%&l!or()_*FxQN)IwDC^o!0dW}BpkjQiul?FRIJ1F-i5zUVGda*bjL1Rp@WMDHv zE~1e;Epxt|qlKU7v55rU#blx9$1t5VpWqP8142~@JxD`K+X}wkrOUJRTdI|?+w2eE zTnc-|@4cavEvlIBHEb+!pHg zU{lg3Fdq7Cq0q=(`$q+AIGCI#u%c%k90!`a0jUo2XE;DL7GakA#t^CL9Ep zjJJ!R_9ihLJOn>H&Dp|_!72Swl)6D4MS?FY{h{qm*DAj@Zm(1u?5l z>+=gYKn@A#>%Ik!Tk<;@`0cNESIuX~$HybS1tdTqsxknC1c>^+1!f2i1LV7zn#r<2 zUv#>SyaZQY$Bz@*{5^?km`*e~0aqBeMBT&&F;ST1vN!0ugHk;!NrhHHQF6}$!+Yw~ z1-%L2GeM~ZXWCFhMJ37tr!Z;f0BR6NWp6(X?4A54#Ek$~%Y!Dk57;r3Ny?ya40027 zaQGninI@4Y77+g7>*lYXZh@mNofUIh4ZHpkfKTo+_xW zUwC%*3b<6Mjny1+JWVE2G8lL=jNgZt1ET}> zRUtvE1+?2~8i@uV4g*bj2!q%>>;z0*cjwe~=#Z%cW(Mv|{?mBNv(F%e@2BhD)Yh4Q zcn$(K;FAzP)Kf(Wc9RA}``A%2JjCvEZQK68Mve#&-VQ+BMRRFAhRsJr-M04gqiuPe zZb)5>kdP1!>j&l7e!5N3Zp;-mL^cWva#M)9zzwzmCOGBa*3YvZ9s)bM?^4B3zl{H-`Qy(d?4&FC|(5Os>`;L6o3*QK3NM#w7I!C z$Kk_4i7#aRh)(;Hbg$JXAKK{?8hytQF*c+ttdWxQ7}H0dvU89a2wGyAK^Xw&sTBM| zj|w=vn!mk~QzPN{>mN}Xl1qSU8cc^9nwmOc+)NvbwQ$VVk^1e4fB4zP>{F?^i7qK_ zZZ*c{YLM`PLQK-A9EWJn^*tquCS09Y$q$u5X7Xr;Qo|8K_{r|#v{?AYBuBPu8 zAgUmFo0jxQ@$bnHe@&VIOS6*W5FFU)u6(VenRYq>Wbixvu`FiqtjrD*gw<$9gf~B| zdo)luniPL|Ai{<0HLxx?!$lAwKYsj(pn~@NDd5oxjNaXX9vl@UE#gp(X@vVh zMXg0Gq2TH5O!K{()^Iki05CH+Ign?FLwzcN-$9~c9!gHeaXH;ki;Xl#nyhe)WEcGr zPjP@QACN!L8)sBhis0~pEw8UlctZ*8HpDtdbUUaT5E(@yaQ)mm+24WxggzJG#pxtH z^6khWpg3tXt#Mm+fpiE0!#-ShU>oV^=n|+ly&6}@BqgD?w0EhlU zW5%Vo!yp-d!qR(c8FoiaL&Jqhpdz;!lG_)ZB1semoOs&}^;o5|OrzhJP^W7p0n#JH zNOc8ZHviojU_E%l`v;&hau51ALZEch;@QlQ>oWGOpO03PaRP^M+hgv+V4#!;DSFR$ ztGtd}s0_Y&BM?5cF$1FUU=iR1JIlTb2Ch(3!PfOln)r^vtWk}-9C!L3Wv&WmNUW+K z3terNW}Iw(EKkfg2&Ke1abehOAomNV1BJPJmVxA*{iw(U3aEOCLD(7r4(biHSK^eY z5EkRQ2$u2K1I9&=&5X18Fzz&QLU~N-V1j$qV{M?q7$-HM-pMxzmG_9S3CyeBHj6|M zn7y64Y<(J(=|VQ0ma4Af1%vIiYp(FRj)kG6kA^<#e|sXMpk>Ul3GL6FOkq}ajg11Y zXCc1#z3PJQ8#~~QAR2jq*r}=!tcNbEKS9*#`QmHn5xZITh0d71@bM0S!A@=L{_fVE zOz}jI=^*h377!F`e4)%zXKEn1zG!MXjnMqjQ0j`G>W4NeZVhgne|m5H@z1qvPFhMu)-*yCKA1Rh0eTydfR0X(0PBOzS5a05RMYQ(mT1k; zNW*WWhM(zzNDu4Yk75*;5(&vB z)&IBy1MVx#<{pKzaaABC@NW75fMJ1n)-8eAnCz~E$^(nWhYu2fUjZ&rQBfhh14MGr z*hRiVS_R?2Pa!B_wAe)xl!q7dtUI6;=@0`0HwUQ2(m^MdRXqJ3@R5YYzTD(vJA*(5 z2^Qzh{=fT4kib7KQs#Y)LoNRoSSG-3o^&0U0`do+>R%sZWE+beIdUyT0l@d+4PPHo z*LafcD4Pg43u&iozIjczKe^&7ZI2(+f|gm}AP$F4##(WW9X@zSU>iDz-{JcHL)hB6 z{Qs}UoB!VhQcx|S*YS7l3|c`4jbj`3?!FbjYnxBp9J({n_KiU>0OrVc#D*-U(NpPd zInE}S_}_cbcd8*IS@@OgKyXFurrgBW3i&V2ap3Eb|CXD8@-1PF{Lk^i)O@5T4^eEl zm;aTJjm+L+1MlWeP$U@o>#e7u* zKT&lF8g}$S-^5i$`?m*Q7D^^5PhK@3*!}18JA6;#1WdcLQ2hBCr_(sjCnX~pcMhjx zt3f%bjxyJ&m){aHS`-vnfo%(if9~9Q4#x*nqea#oqG%E8p`js2JFP+HqpGS3`e)#$ zAS3hwX`OZgM8E~r#I6?*g?qk(5EAK!jfp{eIe+~40XQOXA|@_wRAe)rJIZ6ehPvSF z?=FD2rEva12-J^z)5?_8)T}-}_7g|{4--{x@CHxxtVA1u+mL?tE zv`g|h7aYbDMn1y(=;)3-p4%amWW(XreO=Z2wE$9gYJ5n8AgUnRNC@zjnCxzdvQGE| zIF-2|*~-sftF5rdr|~?d^2@<_DtiNU*@K&Wgvx zF6?&dWt<=HI7a&J=$|0eiEYn!P$o=W0lWpOIQjqVNzf80&t8KR(Unnv`gc!)kNi%T z2um?Dx>2r-i$K1f2H=DSVrOH0eRuN}8ok@qtFm2yr-y$ui9%eH%?c5?3%Mk^l za!{2X|Itwg>cOL7Lf1Z3d7gmU-2K5{YJWFG?axx-ENuFRA9_hYfzw{*?G4f?OO2Ql z-N5WYh=zDrGLl)OhiG_4?M4ixEC4WD1CWLTx{uEt8g1cu{2WCD*zV9j+*q!4G2SEq z18blj3~#-roj9xp_Kir+(fx?3Go%cd?ROGwEAIXKa)YV=c8(WG$^mfzxY!?A1Pv5W zK!U0cNDvo5%p)^sCF}^1#F(5p?oYo{^W}<} z*&FCA(z~_Ubw;2AO;BU&8LoeG8+{0W20~8mOK-RiA3h8`I5)QyG+|hMdqX+Gv?T$T zF4sfJQWPMm2DM0q;IxSIXiOyGVSvq$y+m8E9Y`2a?n8X^tOCX#G^>(OSvQR0P=n4B z273C-u)?5QK(q@HgDgOrVWl^}?D#|hGtjot>~rX?x2bWT`HpP_H%xj}nyhU}cV7YO z&l4scxXP^?M!# zLtGN8(4V`uMZz&?i#$b z0;(-edxD`vUr*A5gOrAMGR*CaC-@QdA}$h}EXuhNV=MVCXm zUO5lsbXCOSImWuv82?Klt@t!=XmQ^DB-0@3v)tv?pe@&E6kH_7u7hF#@Lj~##~0iS zt~Uy91);;64~>2)vHL&{>1G(0Cfo|%q)K#H8nexx7n{?ICeQrlSYaD{_7ZIlBA4>% zS~#FkqJ+=80hw$PdB{u7*2P#cTULrJ;=?bum{9-=vZ)b#SZR#E!z8&Nc;ZdA7m4CI z8@PBd)pt^u-}DAVn`qJ#6uI4p9K9fPZ!(M!+cn|eolAmy|NqG^1#M(YSbY=VV+Zzo znj$y(0kDQznZJmGUqsBl4P&fSg&-U&ij#mg4*y)oZGYTa(gUjjRt7nw@!NJA{sLz& z{5Y%>)}p+qYGGg$dT2 zn8^pvxPAfA?Sa;jjg{34DSS5)A&!Q#;M!*nlr-(?ZPH)+4YA3g4F}J-C<0gb9$d+A zR6QRw71KWQh^BFTqjY6F3g2;u86$f}Lxim%M8}J1GiKL8x(IiXK-(w6bcSEM*+l;9 z*tA87f`(7i;1h}@~$28Jj z#wfHVfBpJ(q|pw9tH9QVl|iJV{|_F&>$Kn2MwZEk@D1E4WjogfX-peYeKH0_z81!mxVxb1$)#4ag%^LMguB5Yv<*EEm#RSGiChuG zH}`S72SQBP<$nTR+6hpRLV@@uvIyo?qv>N@1m!B$7w(KmTTAs6V9<<=;|uSAE4yv(gNpE`Zcs4=R9E*Qvj)#68c>>>oNkaQ(+X z;Nr&E_`&^Fp&dEvav}Lw5Re^0L2*Tb;9Un=%*lU5X2l!0ZRILDgwC=1K|t=K$vh=w#}U8&tT?Zu*)r){PI&KE^9>1;~EI znlOeDz6LiaMDnQ{G_+d61u^M^3Jn&9Hgi?r`mwK*-*}uFnG4XySZj6wM1DaOZ zj~%-KWe}GRr)bIqxE$!e;9v$B7HM3l>qX^R1Gjmq^qQMn;l^#RFyfkkJ7pjj|70gI z8OQO5f8@tWgt_Emts5GWPfjrLpE~sxta@5JdgR&Dgp=#`-4%Nl%qAx8)0<)T7dj5H zk~KZ#uboMHXW=!EYDQ1NHdNKrfOeQs#J*}KUvOuiQu*(({=SaV0AJ6uF%M!P_ z>yX{CW`u1QbF-2_>AKz3zaTIpf#1_B%cyd`mdX1#S?-P2rr$5Mo4Or*yJv7tJe_L& z-S5&LKzh`?l##7HCb!;wqJ;Co6J#FJ$)IH^fZ7)jq~Nk5YBQSC)adO7=mjk!@E!jt z;J&;c8Tk=t(O1VS1IVLmGbSN+sI;R%)zEiS7K(ch^9O<2evfx>8z2BYaE1Vz#vtWY z4B#DHA&|`Vjg6`*DwmNqk%n#zM#m0%b>)8suPY7vcLXm0S#_rW^guD^#(otuvpzUq z{cQwze|n6f6Ot>VIZ!}51;u+H0bjj(1ucw->>G-h_$bC*&8431MO})w{|qQ>s`=BH zg9Sr79Ycj+g`C0H_4{g(v48(T-;+^~uXv<_aX7SxJXrb6Z3WNHm;yyKVnXwZXt{x zZia&t+5t|Umlc+YvE9>X(I}{iwk2b%$3f%+n_<-U)`@N+pTroeq+H?j0j_sFTRIw^HVSY{EK4)4?<2^=!0_9LUCP)BI2 zrgwOnzX$xsM?W0{a~#3}#@)tG`t@K0v~}+Wc>d>N|KIx2$CL!{8b_MUx-0yqrPFtN z?Lw+#$I76~K6J7Ms=xsevY<*4_ca(r=J!3m~^@`5u>64{-3)I#;K* zZJL`eRe6%41b%{()xJeS*8r+6wQ%Ye~`4hya@kcE6(9_hMOF%LWwIMM@@lvAKKK{4iO zBHh>v9dE8hiBrfTjJsZS;xrDJX;O9vbpF}dg1#jI|D_Y$7X~-eL75ULMC7V9j6VPXxo{Z5z4Sp~%K}3{ z+HV#46&iH_67fT&1Kl{#RESS@NJ&*S1HuNhK=~2Tj04v0>oW^?UrvNO&_JQw8lwwW z`~k27C5i+hqKoF7q?*jvg1bvB5NHsFh|xFJ*J|Fq*n$+Rh|Hm90`y-qaVN~`D6|vc zIu3V6TB$T)F7^7E?|#IzRZpPSYNx6iB0LC{Opu$kTVSB+CQ0}#^qx)7+Fu?C_g+m| zR6tbD;*iW*K^ltWD~sfu&x>nlPWYlEp!fC)7?gtKJ;#a84v7zSg%HDNU+F&L2(yIV z&0ZdF)WbJ1N7VQ?o8MX7)yal;gNR)z8#Ji2_|TX%L1#itg#Gln4%$BLrWQC8Jw7?8 z0}c!kNhr9qDva=o_(;wn=?bY7a8yk(qt~FyQ)4KuF$6*g8pTJaZVX78L;-pUv;pO? z5Q_P{D?8pV$B1)#!`7sVd?O{-pNg5w*Z>Vo?fV9Qo=7cZy#Q)N?t_}3IEb-;JzNlA z8zF*KLUhQ`7gPNPBn!|d(RN>oh8S8KY4PEPw5oV^PR`&Jar?eHXn_Yd!`y=nT`v>w zBW8)*R0mlSjpBIOhSyMD|DP>D(uWL-U0*>}v1gqAV3~I&+=s9@131WmX#knM+q*lmd>Xa)h31?I5=MJ6y z5hvnM5iUH}a_;k6(&~(&N-{B|S*D}5Y%0_yM0Hj9rFsu(J-=MD?PTFd=13=9JIG&+ z2SF7m%DXf^x3?*Ris zq?Pjkbt)rsnnYjRn2=tDx@Wl5+BciBceD0sD50C8d-WA%yaNz zU^CQNm0Ja=bH!axPMCy&0W$i-1Mn~70erwH5SLpic)nNL{gv0N% z%L+DOngDCCMy?eF`|!Me4!K)VRByw3Eozm7&-(0y=Y;#vC2o==Y5E^dLE;s;0i=(AE;I@NG23T%-l;w`=SZOO3b6GdhBYG@ zUr{nnSwi0zye1YDI65zS!jXF__BGZELlz`3>M?S;>)`rYRN|_c*_vXo1PHQn%s&uG z*?FuLCAoaN_H^7P#0DeT?Mqcit^hI1s5rdCSj=nun>?+4%Ue5UPzf@4FN4qPb$q#> zgD#15wh*9=$TfT6n+B%}5~SvyX@t&BS4}%s%)?o&jIKT8(CA;3_H~3bzwfR}PjI12 zCu*?oR5!zCax5<>A1_|Eq7m$Zs<+|#babUu6G+(dslD{-uuvgbG+ggD3{zd!%ri)A zTd$N_t&CLm7O|vYbp3vF@pFjCU3J*P0S)q=AS%Ee8-F+`Wrjse+Xc3)>*8DERZoA^au4*wbitloLPzC6K$QK z>PTBk7}N0Rq3W(fu1*jnt6C`ASffl-AUk2?rZ0=}j}s7ylg@gihmI%517YK5IK^sG zmH{In8nhaN<~E(A1j;A9V%kOw778z07aVSDJBN`R$;hb~haGuO1f2#8p{&9s`7V^v(6&G34;uBx^m)N=9D0%G@&Jx(ox z{HJY>q0FI0OEFZ{qiLyYeXI-g8ok|Qogm+t>hIBV!5F-~eql8*fBZIs_l%G?Rarm0 zpYoXJHQ|*Yj0)3R2yN!q`av_R%WC3ADf!iun++FDLY2JY3T82bipCLkHio(cXj8Z#|4(ic*Su1i@LB5I+iYRNCT0~N@M-Z}4dHJJtVPrm zVsK_7fv+47gH9+$($D~TIRiIwm65bt+n3Q223pJU7r zmMmT8jYDQqKPCFm%tx?!1ZwhqsiJWqVPXDHY#;KdrgO)Y3p}%^;Y^V20?Pv{aDo@5 z-tc_AvAz;;y#SW{n(&(XG(lT4;>=_y77hkGJ3KCSFgJNBT;R7H>iszZF z{Gx5hkm&*LXlNG>Nes9HN_DOvx3pfkSvDS|B7@{1a}6N40eQ3#=u7(UPvU3-vu@8C zr@3#A1c&-~c=p(r4T=i^R1i{E~WI+-|(!AGApEW$2RPIYRdrkQ8g>K&g zO|(E&o$pPB@qU98{5L)Z(G+afQogg#cq6gwcyDg*cuAc= z;|crN-^(f|oXieGKJYO2LsyPV3!`Egs0r)9FSG$N^#(=vc!2#h>f87 zxiDRLv$LvnIhB#TjjCLueWJK^mtb{XY_KfLUJ-uStE*M5GA#6azh(`G;a;$iC37=~vSIL2a; zikSIqnO~m|y{6+6JB!9Sdl^~D0|SUXvM6v^IyZ*3@cXnpK&-}JwRDtIM29QEKjSGINXzyI0shB{1yRiLB?V5cknCR)LbO&@il7gn^Og2`T4{zmF+)% zT-70p9;(<~c}OiK;Yg6%-UIaX#4jxem_&3*U`A&p8ZYoSeiRsfh`a?km{lhf;R#^C zIP5!Kp|oYFkB5z$+;`OnyQQ%Y(@9@fflqIyyTt_eUYdJu90unBMu>NO!L5%J*}Td_ z@F3)q?dylUX={Ooy#xQ6k|ifVAV$G$pZqUPa&y^`iA8+dIsENwy){;Vg@27`YlUI{ zp}*YFiC^pB0~}fNd$8o@i-@Oki+N!llL7+xe^`_qkBk6YbDxuv`%)RVu=_6UudUd$ zz_}B&wQq+nR37?}ytxfJzYOj7_$hL!b}WB|Gx!?B6VYv&3o-?aIxM$@<+e|*BUty( zNo|?w|F+ru9?{l{!u;tO7)VvRPwK}Q{ay#?T}Zy~*Wj4^63HIsPH2BPq4;^r7?pMm zDdM+nBb&sGoD10Y=K&%UgzVs5w?+w{?gh)@3A^#-g}(?KWW>m`+eadW*msz{CFc&C z^c_Q7BshVSzb!vI9@zv8GkkJ)pk)SbQAch$oaLH*EHX!O;`nc)P^N+#NJrr^hel|S z8_*I0JRo{TKRhYXx4*K_&+do*EJq*O}yF2{qyWcJq(~4 z3m=FCXgeG&L$UJ0Dhz!{X86t$tGxz)2o9dkN{XO@6Klkv@^PGRqtG&0+ za9!(K!+X8gT8Qp&2868`9UnZj68Si(=WbsSGh|R^OEa@HlX)yXlTKi-L&vG+;Cxv= z5~~nxT3T;1Crj|U^~%Hxn|_%!`t^_AM$<4q;0_bHxw-oiT?#JGe*7oRg;#R*(&wk++OaU}W$Ol0-qUg0YG=>d zF&FAVJE7r^(FnSM&o@5vr*D~c)rcr4Ag7~2q62H!6ADh*vsttF?@XVvto$!^O^X~K z2V7wKFWi^dJkzL;bIG!jQ%g0r)mgI^{msjWb)CJOyzTVabz&xlifQiVX@96w%kdD^ z?HJsVKZ}I2-uXUr?2~KYv6Gl5HDl@ryCX_9Ap>5}z|t}w@K7IMiwFM{% zEc5qw7}RxG$v8O@IAv97>CycvkNuLFWh7%|BCsweN5X@ztdk&q($_USW%K6mC!2n3 z`m~2<$p|($1s^NkaF)Voff4{?hp`$DiBVD-Y7Tt5E4EC&;Y))nM_uX&xMPe9(6+Xw zOkS*$46;Fc4iX3ny;<_=E zKs+ei=071{d4B&wzFKQYTSQk*z@G-=y+y|MeL{$N(L`kU(TL`Fj;50%Aix{N4=n8u zbNFcMpE&tq(F)eC52CG)ym(%$su#cX>6C8J;>+AGw)o0WZOpaMU5~PdkEOP+-H)hY znLpa*tor2INmkK55e+&#<=Pn8>&NPbGCh|Z%1n0v4(Q|nET)?y@XCkX&3!-2veF%X zIC3~OiFL**wkD720dK8{h(5W|U`R?!`J?P_XVvSWJw*dN-{D@sid80+2UAgPciQlc z^6&%fL@U|@m=5Qm?25xXlZnA`7p)JqTOi>=#E(AC%-JE<(-^;>kf zA((@pPD|4WL>n^ossWmgzJ6I+55#5hR01}tsRvbiqLU@S4$f$E@}3P8ovT;0=$3M( zh34`TD~g`az>>5W^hTR&C*blN-S802x$9h=Ce~d@AAA!wPKYFJ9{tjnXVXqog(KE56weNNW2$#LKrewZY&n^X~5*hx#$_OtqS^aPOBN5J$J ztTlNHF_c96ws@4TqlHic7L=fAk=3hN_FYPQT(1hHQ5usj7$j{5_YM7hL$rsYXurGe1~+g zO%~3pFZpqCnPj|VH4bL-kie;6WzSVKzk_NnWq{T^Lt_|26sZzPfy^_{c?)W7KAe#;%`w67jXhz(Omt}=G7`Cv$=0hld)bj^tYyYg)(Z~x-<;aWFyw|{wCtE z<37IU*F=o2h*J=0aQgAJRbnWS!7>QhidVyH2S`opwpz$3?1j&?j{1vv#9vO~c0M7}+CjH8^r72W!d)8qRKR25#<&PsPuk)!B%{R3N@dt*tQ z^d?|Tt5!yw-BO1xW~I(_>*VvM^t>b^(`!8J`=33`??xu*iZ<+cLjkD+9er8LmMPSK zmzZQs|fihE)SH!o_^6*l6E$5yLj#c*VG ze1qw0jwyJGm7RR^Dw=J4v@)HPZVM0dzOI9z?Fr(w3$rmBHokA9+Kt20UB_~E9NDPS ziJqp~eSF<@+YDl4{ths0Nc@?K-S_QaZ8XAffyGX z##QMc7mlM{vv&d8Wthaz5X{;dCqp4v!r#M!U2;J5K5pWS{}0aQ{E`d4Iu!RU>wkVZ z4PVgFym)-y&TSyAc+hCZ<^A0hz=ib*E?QV~Y*}K?j3Yg>-99*`vSm81ouSPC`jS7Y z6=`uE*R@Wn%CE|L;D*?1=k9BUEp2-jtzZQS(0qJEueQzMnYGu-L7j zlm`SiU0NWou!H5yYZs?ZO5?ZHU0FMM3@Z0gQ2-La(!Q_ib861yuPGDA(v4#s;Oi|3 zyK7TvD%oFXv3Q$zwGe?QblBy`**_9zpZcunx-|=)zmoa4Lw*59s#C#Xr`*r45bL3e zI(j)znKNA`9YDGZ9tiZ0Mgnd*vtQh~llz=SdFaUjh_pJ~E7!HA@BHfegTWynPG7fG zv5f@D1CVqSdYt%nvAf_z*)>R3qK8JQ^J{_dEr)|>8D4XduuUW?VmTzsZ=jO>Jyi77 zme0?ohB>CrSuglh1COlKc=dcO*7*m1om)AVPT-&TW&0OLCdPQ}h?jA@koB;KuR}a? zG=c0_hjR(PmwZGY(I!mq7|k$uJywKjo~|8`q6Fv^y02Ll(Us8Tex+#&63ear%z`0o zk?O{O0PWPJyQh^52vu;y33ZFA^9?$MNN_@o4_Ah&?+<@!@LwN~yV0qDSoIIR7}cEW zaL5tsMaDbiBC924Z}F(celvuzhgTej^Ntn}QRvJJ8oW11Nd-PZ|c z9+L}rkO@SF=5?hZs}0KtloXs}?`>)cKNyU!WPzqw-}@kbt$Fz|5-Ggi{!rLhBf*7a zLr@Q0f-}|RR*!%ui_Ew~xU+#F?B?sArr|Dk$4L7CafYMU@HBbX+SrE%0G`abQ&2OvDr({l+oZr|XoD3DG=4uB&F-_(Ep61^ZWRFm9%B<$UF%(aDd!Kcr+By65gX0&mX>6s*xX`0ealO zYWRXKDlA47Sxk~^O);gae)1ZV?Vol2B(|#cJ4Xw&qWi&ekU(BgMZ@YVQF?qm>GDa! zOh7hhbnJb#vIOiOn7l1rddz4c^p7O_KWuoi7;YQ@NHQ3T~u z0qQ~I*cB7B_5y2F7Zp_L!#TW-p3L$h>%NbFZk32{$SoI$mt(AfU#K~Jso~4KRca+~ zE{!A0G|?nE*<>;fqiQ+SvpUX_qUG8=udjYMS2(lBq=_q?KFz);?B-u>GnsPkEE(5m zIgQ{Kx>|ng;j?}UJ^2QHbGL1)7#7&R;B+0|>nE_z&tf_;_K{H<9&CD~@q{JHl9(ch zr#0S-yd9k;yX}t*R-3!KeOu!4wZ=$rAkqs~g%zYioz>wYiUoUe$&^ul$DIa&rti7& zVF!ZZ>waXA8`qJ6i%QD(4%4zW|9T|K$pLuoWD`e?r#jP%fR1YKd)IKOjX!Xt@LjgF zce+mIN@g9o>tc>blM5RmpISk3$tKz$X^6Hx_littylGN2HF@rnA9*|*IHnEn*14`N zebZsN&&We~63N`IwYSj+=Lqk_kX?uK*fJ`gA$akdiw4T~4*VC-HGc991ZVXrCb93) zSF{P&z|RyBn|Wjd{HcZYtj`Q$k2ZaU#IvvL19Xi%7kGny0%zXGBU0ztzeEz-l`>USdg3)}N z)I0dBYMP@bVge@&&IKN{|MtYyrHPPSk~|jI9H9B_vb3`_4RAq+rwzO`M*HZg{xXFr zZ-)_Y;=iKy6BFMBBu4k_sHhdQGwC)d!ax}h}Ti8wm1sjNja>q6jJs;Ry<`@*3Y z*06p?sRN&&fX5Ujn-vC8%2+>^T3>?%KAF-$dZP!TJ3}|@K|kxJ&v0*1aZ-!Z6JMp{ z2OUJ1_3*n&07a0#l@^0=-I%>KtlL~A`k$D*L3Gy%(^c2wZn;e(dZj?Ep(2O_ zH;VTq!$|CJW{kH20|L-QAG-Ei!RBqM$#piQES!OB=;^Qj12sCchYDDh^Zwfim&?uo zUbM7~$+9Ntfp9Z=WUelFAkVGBJLAETq~kBp^5VW1YX~f>ZciDI{@~KMm?89LBhB(H zD`u`h0#ec z`)Xb5LQ4hV7Gi(5VUY_?dUJ&rhbA;|KWla`kdQW zjp`<9#IKtXQCWn{tJkL9YoYN&vgz(4HCUasA?wSxAOTkzswE#foe-EW%^M_2Q;3)L z1tBL7-*JbCHFwby8b-jp1-V*;WG)#Wk6d7+>4ir}k$@y?M(?h1*lZdsEDxUo}fVwG6QsPe3`evN1v?|ndH5wX2$_6-}6x? z)grq4fP`o9tJEHGHe9_P?{wU5-^&*26|ee4bOmJ!(I2CwD#In7SkFX^Gml3*XUzJ$ z&!33e^`vwrXFxA&&igXhyHt~=j+FwBIx|z#4Rbd;D%Ku}XbbB1rY%ebmr{dM3UycB ze^DAL)yFf_c7<>X2k4YV)*;u5E#?!cNG-RC3j<6@%ah`97I8I8QPV8{yH>T)=`bh+ z2-S`5*et#XE!jtB_Zndwi zm3f0Sr|}7iCnwDI1py6tTc|vAT#MD^5Y7))C_?l}DPo*6E#Wp{(G2@ZY!nr$JR^=S?6cf~of=r&CL#PFo(`Ay2=056h!=4OKbbpmZASi9GeM)t-NJen`@X zd!*#pZm-ONy-MEJ#X{z|25O!_%uQ5bRf{Peio` z_1~J2!*YH3P~>6eKTK@b*kX{i1ZfJdc4mD+tmQ%RxrJ)xyes0gu9Fb_--l@$Qh@-7 zNB&(T0Tr-b$3FJlJC)k*bQpL%RlDunQIQ(!t8v(YW@T{ZR>;32ijn>?z5IVTJEf1T z?U4lnYt*0Jd2%Q9G{mYp*l^Rxtk7=Q`j`BV-WGxlu)k0Pz6sm}U3=xD8MDZ_I^JCs zEKiL=k@F12xmOUmX8%v7fCsb=ixIE0(N7t_t5spPEG8>G%kwgG|9$fH30~*?U+Qb1 zGUj1XG}PatQ5$Z9+dp;P^AChbz=I>BsLSiD|Lc0z?WYQZ>7dEv#Z#9D|2klN-|z<) z81V=E-!vALh5S^8ENUdp??!H|#_O!{bGZiM1@a62Bvs6gbSiaMqIYJ? zM_RsgTE(>)<1%~1{(ayk!gY7D>c#wP%XNOy1yo)uci8*;O?Xvl7ES68s&q6>0=aQv z7s23U4~6r{q%lMgipi3+0)ttOUeXkI@u1`a+|1Rh$i`zUeL-P|Xzuqj0IJYI%WmjC zZ{ToxY(LB`LenWQI*CHZ{j+;Wrpr&$sRJi;n_c{r(SM@hkNLVM z2ez3iN_x>}Jsk}7vZnIowZKPkYWQTfED@OhgRZNSJM(vJ)A!gF7Ej(2TuTt!wYUK~ zGXv{-obG$H*#Dh)U&LRFCO_@WI(w6x3zB*!zVXB&t`gp&u+}AsXz%RLKJQ&1_Zq5x z7U06s3e&y1mtTsi8EB*c+CRTid5}-rO^z4as6El7*$>+LgJdTJKBm`d$#4}K&5u1m z{f~bT3;qW=3)7EEu*u(O3%|YTv_Ff(b*uWBd^?qQH{(YBwq8H^48QHg&!oct{N?@w z%HLqj#=k9CuN_G^qAq9PJD=MrVBs&`7>`nq%TKE65bL@6*#WeDzPKY5@fic_hg}^n zE^ARFMn8JEunuxL^m4WBa~mCg%whfQ36g6`pktRhvz$cBG&p2P5EBG}Prc*pO_y~) znUC7qF=mwxqY6{pwoOyxfm8hIq0Z6b%zhJ(xzhUlyPY~jG3BVtnI#!PWtJd^S>evM zeXA4&=sjm5f=!%${>e_AMT)W%Qnob+oEe$DT11R(s*)t@)o>6|QJ+Jp^MLHf9S8o2 zrthV%nVLwS8hRqjmpNFX4)S$_`?izOf2x1;!oYPAwJB-9o|m=66$NlU{w*K#cN$E2 zPxWfL9DjvB&k4z!YT%e$hd)MX7KYD%LN$s#c+zEpp|z$_&nb6io?3^$5S{jS_fz9G z#A_IDV-H*dz|aSh3hWX`hsGwfgr+9wgAk}Tr6z~n$n|Nnz*&yD}Xr`35g2Fxlz01Rd2_z zmkM5*Be_b@V68UEao4FkYrCTe4A2mgf?T`8s>c>TiIUVzLk!E=YC1y9h=@$m2RKn1 zu};l7=s#nX9-e&7@m)n6qb07H8AyY)pjgNn$zL=V%t0H^0m-S>skKoK+KRLU>GLly)&gl+QvZc$cK|F~#kFskgK|I%v0P(C8za6To++!xOOU6 zIBluBt5`sa6FDfHUK&C5mV~C!)aihvTA84tzyQs>{Sr6Dz)_{7=*e?FLqI<-Ywf!& z?*G^v#2Hs!96(DKrMQONp7e>DbT*%(Y;pVlVvuHp3~>j_e-B+)BVPNZ=lxjPPDojZ zS-O70*E0GVY9&vci7@(DdTk>UzqK=wB9yEfqb@yr_dAk;d(#mgaHa9a6^3;~794oV z+N_N9vE=j)35q3h9GA!26enlzpz0c8i+?>V=mxT@a`Pnne2Sfdpx;ty;OOd8-B@8a zKTus1TzCl~!uSeXIu+S3s?f5*$m1xR+K$W&NcGp}Uh+weH@6ZA`JgW?Md@mr@1asR z9RNOyWFM15O3>C#Zk3tfTP~!x$TiY-dg|_#u$PABm-BvJ%)#sn5e1W*U>2kD#!cV#~mX7>T&_&plAgcdBvspz;%05)4PYztx zvL%HRoPMQNi-t&1p=N+dCLOCrp0PMO=3X*0r!bOBf+w{t_=O(8DgsmDu0bCHen|}T z9JApE(@F1^_GUqfUy7#S=iUo0ox>s)4yLE$h0}W6v<5h)dN}fI9J2F5tV_0b9U3n- z2L^A!@fj)R%#2$+NugtB>R)Q4?y8$4jilRrxqYo^z6jMq>+mZ*_aENSx-)OB7msXE zWveJxYovR0XEJ1wMfcGq&>sw6<*d}I0U%a&>g6xlaQLynVM!X9o!pk`1Q1e4V%0=N zl)a#O#LQDZ*&QSPmCc9GfEW!ekt>*8ZNc$98S3uOI{WS_oo_TxjSAMpuxu%nUnr$@ z0nZ+JCFYWJj@T96q;F3>=^%QwNRD&l3|EhAJ=&8M8AAV4K6l>g0#1ti-wA3Gms{Vc zoj(!Y@IV1=4sozJ{j7@ziG;fiO;Q!2%4bjHceo^aZg{#)6}#JyOb(?>zV1=r6qZut z8lct=P`Q)_E;nOi(B2lNN8i`^67m%uvcn+`qaj^Q>ta%y*t)MD!NJ&nhX9J19|#D{ zPIu{&_Id&NG_9GFgItk%3NHbWy@E)d1-ZoTciHt58tA|J+V{}3FplX;j-47$pXDX# zHjb(qhP?WLNXwiV1+cm|q^3?beT89Ust~GS6|?`t$hwi?m1G%YPAp=#I6c zs57O(g;gGqI|v#fHfG}y1?1`LrzYwp(fnDfmGBP$S!G{bU8O`}luAI$W9a>MRDxN7 z(LvHu&>|D(IL1y=R+_4`?#}_~q}WaNkLn*Dz^@(JY;DTR7$VcU6&G>us(W_F4H)is zWY(sTb6gZ}0Vt^MnJ2^4vVq~)?#jo;BLZ0#ON?uftQyII4T5N$LKQxn<>#IZCi@+A zDWHoyjfn}7wzUmkklN@j%Mu4h$qGqfp2U_f#*;0^jZ6$vp2AkP?;Y8CJZ=VpsD#Q4 ztV^$DE#dwHJyc>Qs9~W_=pnHt=BnO%it=rfvfqu?F`66OgLl1eldQ1Y?MIhOWqLT) z=~@UWd{8I7=;qgd`broD24g)Q4r}Xwd|vvOrS2Q5NfK@$=Nc2h0yPGlOtRAYgLpTN zg;=G}w<|bDi7fGH+5f>q;iJNX*y{o8Jw0LU{Wrz<&Qll}MVs%KjNd)NBjawoO;Uk8 z@<))12^$AYtX{mi%!Bp6`+L+qbpJaOi67-vpc!-`O|lEvHmQ);*1gU{%FW&onjR7p zNT{;z<4Hox;KmNO{nl-DqN0}0%@%Dz-uB=2b5z4!Bt!)mt`x}cNpuMZCn-6tJ3caX zuo^5e5j6+dT2zGkwlD;IunR2x54;GIi&=(LQiLdr`;cjR?gFcV5RSM!t}fC|9F%E7 zS2bBc>Oj#RpcmZK@gel>AEoKk9JZ>Hl*v|y$yO6va7^-h+`hJQdnj1#xarWBGR8l$ zELe|<_R&Z`{B{EonhX{9Q$$Nw)ghWu+D9n*1>wYTpF`` z!1H=8&5|-r$!C*K$-55}Z_PJ6nr*R79dmdG=#{dSI&U|Wot$f(olGrst8{?t8nT$* z0!w$2g2uyzXw1+Ft>_?-5 zTMicXC7!2g2p^R^?$V)xV!(GwBwG_@*Lkt6%V+0Rp}qbdA_8&?DeH8YcA0kk;z<>j zprE*345_s`Xb?hSA#N8)Lc|N3YUYxZ87g`>J9eGEP4*L?`D}>;RQ@1EB6eiuzy9v!Z)T3*e28iFC)_^&JFt7`ZZCyp`&qz8;A6zivh{_W z@zfCcZy}$xpx1x8bWx0z=V@x7O&IVff&&Jw#wfb*P54=sMQ`$SzKy?3$oCv0b5ACm z-8#xoCzWh`lr8*lJ|^w#mrpy&@*AJIMxY4Xlv&@BwbfEERvG2}tlnf<8HNU5D941- zqVftSLtmgQ-xJqd)eoq`Dcl`1LFBce9vj@9XMPw+bEDzsESf*pdQoF@;RH%%xXS5@FM1+CKDP~hUte;j>_(NRnWX|DP0lAJWu53@ zQzS6)kWJKbh)_)QOsHUfrcjAMYVP-UiOF0D0=tE-9daZ^c7sdeSXL?uco}LGx<5*` zxhqYvxcS%gSF~L;;{WFjXq*1AkosWEaMYe=kb*dZF9TqSMj?++9`!~f=|Q^-?NU8A z_#RqO@AGppNWlcqk%hnt#ryvG%$rzTq&h!pey9~+io(4$q{wYrTKc_duPkHk__JuL zBy+xWRv2;_^}BRV%6|OqClQ}$Ul9bP>~EKw<*0KrI2|wk^MR0O-PBeu=WM zNVgXa)u#&eq-j@|3>7A2RWs14h?v=}2R0T>uKkz5lZXB7p#AqRDPy+g^WO{R9CfIA zbmsEMYrp&~x9E?^aDGUP4FK}u@o>FF4ALt94Q&hl9Nw=*)qAYZ6s(jKLIHEt*Eh!7bKVa5hTrgwWS zjbVx1)xk0ieNs0aO61RQBa>Io%2Dzm88@oRp9NUEY&zLhCG~jW^nrCWGC3<`zUH#{ zeluiySf)?e{K_xc2JONL)-J)QX(f9{PFn4zB@9n-8G69vEWR@T+^&Im@|}u92a9jv z-^Bg7CZ0T)ZdAFr#P;ti&u?tPgs1zj6dkw>XEJo@SCBNfZvK;fl%{#A+M2Y zb*Z4$?%h{gCva-pl@&)O%GoCb`%c9wSBTtY*V|w?=c$jdNU=eU=989*9-mI1lU*r$ zHa+muUcKy%v_p~YN<(GAMc5U4K@lW7==PoBsZUs)lZ9H?jcWU?gGvs~JlUw=Eo#wm zo3SHv;srkcKpc6hR-Evcy2VA(Zmd76kDc5EMLhjT&8pP$Uqt@AM1Y1B(h??=FvNbS?6~^6b1^P z=8zQTFIh-r;raCx<{zHU!`CSNcD%Lyh4q_CT#SEPfE&u=SvK#Vh|&9mJLgB^dU6vq zsv;l##g#?(pc7e)9FMFYqDiYHHL}{n;v#lShl?f-f=@Ec*AEp&E!skj0I9DMK^LQU zkamCtLo%G-p>rXrIraFU0K*3}@-3>R@aZYyTC7>y`J@lkt%X$8xUHIYYCp2?l7tZ< zkaK$B(`l>nzx#(33`R56hx-<@u07QkuZ4^_P5+#VB|3UqTh6`ufY>9Qk2Gb@IP0N= zqQJ!Ym7O$|J?m=Q^w*?1t&Hwz`}|frzIz8+LtwYzG4*@nYQG)BxbO^5!Qx!fUEyBF zuQ#4v*S$|!z;W)Z;9}a>zgwT9%YF$pu7)MQ7Mhfiv+_i0b@+VgPrF(7L&T2G z|MkW+63O`;zt)9Xi%IERTiW2qBJG?m4Gt z5}UV2OtM_|=axRN&=Q-eul9wnCO*G>L~AqcERo$+T@#mMz!!wO+jWocIPgpEUM{XV zD|9Nt$mD0bN|o~Eoo_nB5pohs^SIaK7CjZ$$MBjM?mgy?!~hwsIg z#&DmU);-D$?k}u_T%aHy)8p-zufVh444Dz`JuJ8}_Xa}A9&Yr}ujPID%7Jezq9Sn$ zNL!8e;s_1(x?t?|gT-9S!45jVB#wB)?$@4+nO;t&#RXz~>&7{4t5Vk}7*`~5n2vG9 zq^tUmA@W%5j(kRUlVTNSSZ3*yDoLmA!;uWW+a@hSPriQ~!1AD-dyO>;(zG{-mmKoj zEoDWjtv7ah6dM$0_qt87?|rtdUa8gf_VnImS|}`{t2rjs?84_H@y52Pv9As}+i%Mv z*QFlUd;YboWBkrMnrb^j>5@67CAKW_!pi`A#m8{XX7rm=XTb8+ah#rhE30qUFW1{f zZIeKBB(GL7SB^{Pq2^#^p)uikYYvev08~9mF4jQ3*bSD+;62Z%eFj+I(>u`BkUeuh zV05$68y8#nw~K9GRO9(hE~AlSS;wTJ)*W8jmsA-LUJ)n#vv9PAp<+mAo$1_JM^n-i zKE|q}NIhES5Fjmdh>1{o#;Q{*VumZw7x^ zK}&yF?iwT=;S}}xlwA$Gt4P;|i_Fv2*>~ushI}Ck_{B9oCD+c?ArT0i=$s8VyUw)p zxW<uMMg}>Tm_>-0OBa!9?qu z`OFrTnj*>SEhd#AFf2#f#Lz7EF)D`mWUaj-2?b~>oR%!!>{`DZk4ZHWGl}T;4Z_Z} zrzh`6OIn-$uUyXf0^tCW!#R^goQcBxPNx!6{WZf6LDYXBjWTXVXT8 zmIVL$!*3t$2ES6nqT|7^Yt+Y|Nu?iUOy|bmP0KDVO@Hry<#G;3yk=G$SRs{WG;7wq z{N;q*GACf;MjrFM?-J=%JKK$1+z#gC3HC}&_wx9E9$@8O()F_0ABCRdcz5Ud% z7`G!mi%5Y^#FW-9DmpOjDwNZTW5o@2u>DhDusM@~Yi0i_G$cEV=GfW?$lRl=63m41 z^thPX{{I_4b?V|M<~#m>@T)IVy4@d|{6hOA-CZL}RxUYQV0Ujn&#@)*L@$e$SFA8z zx>UXW=G(uXu2)^PL@i_9-lfs(i!&SNM2??0dSc!sDbFK`m-ljeAKv>|?f#sgl2Xoe zkCFEx6Px6?41EQ>D*Kq&JBP;dH?ZiW_1OvgkSVOgm~@B#&6*_%$C&*8i!OB=X3ZL( znQj(H9fC1y)(?K_rp=mVKQpeXczW*H|9_GHsWmA?Cw=7m9OjYr{-c7-m)w3jc!AI# zH>a}V#zS_Ik7s3^KpFufr`miNW*<}4O2z=d>Of`cSJ#$ua76W5;QMzqd-2`r$qF&* z7qdUc=qlu&2OXT#c)?=Y#(qEOOy+{hlx{L!Fz@F_h9fF`LRMm%X&ZP(_#ov$V1CWs zy|g)G$va&Qxva!i&wQ&&vA7#Q*PS-)&Z3Du&cC_y?Ru)bT&=xAUnAdZF^eW7Z;ci$ zCjWd)o09U@D8-3f>Cc%f)n&zbrKpe2{8P#W-zMLi`P&!d$H@XS|J+9XwtVJNFUUhB zU!M7=?ILm`?mrE@4{2!B-B;1z?zTipqqSxzN%MSlQ$rD&^#ZKR)uT-p$qP(ue@9(+ zM(8-8Z^M}w7>m9^ASe{q{QH2CeU#uRtKmj#+yr)2{rYWuIx znH;O1^4)r5bzBZ+UOZ0#!yqv+(VBz2P06v8=*|tsV+vIaH4p*)&k3*E*@TTAeLB6e z@R6vZ#%Un@k+b^nVH`A%gmh;&BSTo)JpidTu)>!rpW{?yfjDlv@PJ2G-S7b{% zxy}n17+1aCh!LyR1^2fOp$_+9^84#4DJhu$`+n@M{&_n)G#LC(zzH9^l`p?Nh@Z*gHM^gGis zQ!%J5wOB#xbfy`4kA8Jsx8LO_>`os*U#0 z2k5ABboM-t!4Zlg2j!rY)OO9ZlrxF)>;ZS~lz_ve=Z#^QN8Z<_JEWf=a|9kfTz#qq zV;^3av)Ch#fVv@>BIKiA4q5evNKVMyM7NQbmI=dK4T^pErR_s8cveRzxb0GULVxLz zATrZn6a_AB)-QAI23B&5i6vtsc<&u_8%{-o=~;WOZ?+;5b|l!qxf9hJCqj>AH0)Yx zjiK1ZMz9yN_GHT3U#d6hkrW>Q9{OW`PuRMmYK@KV_y+}XqqKigojJ>rPd2@W-z z+y0o{)xNw3qpdZ%`?N8}x4mi#43IWEp2x!HPMsKs8qR^!lV;hiksA4!TQNAaaVh6PJ$3Y7I94B( zhF+_&k%R_OO7h4Y($~M%#c{(1Q@z$JPoZ-op1>Y#4f~OxQRQW6TQI^EA!br6 z;+Nu6i?nP&A89{^>T91ma?Jl+KD^UFEWak__=(K82Ji!1HI$AT85yCdGw3x&bK@ng zAd5(O=&4=PMyuDRDP(RegB zK>%GAxNP42`_bDkn4o`bd%GUy^r^nqzM)~A0j7<@z!F};cE4uS#zlRSFmG3K?bxE1 ztMb}8N#V_biF^0%)gMmf>$++;{6j0XI42#XrB~-(%Rm20NIK{+t`v2t&oJG!6Vn94 zrQJ9rHKrfLIw{klZQZuzVf4<1BLh?DLb?TA-_zfx_r_>+@|m^B4lKEmw?gfG@;$SW z0f)M;NUN5!`#pFN+;vgVwE0klX-grC`DBdEt5vyNH7zAZ!94anY2&>|Qj4F1G8(>( zlQT>2GE+?>GJ$uIZBP7)LutA^MJb^^YY@U#{%lyAcrpu*m4zr&N<-UmN08K|lM9hS3f6pTF%iz;u~B#>!n`$`KF7cbv zvN<;KfvaU(Y0hw3f5N4v{`FC}nL~)WPA&e5u>tL3olrGpV>~+~_OQbQB-Wmztn6K{ zY?E$3QP^qERFmu7oLEd;`B`oXH+Uc2{}|nQ+3u36zvlKZb7aLGkWp$sX@?1MM=%;r71)_kQcmc+74oHNcn|WFL7NfH`fGv@k znHE~(>~YY2?LRI*GbMNX!QJB=^O`j-2hHg~Fg)bC)ffX`(J|h#%9+Nw9;)H?03u}#!AqpI+nRCp$$Q;W7_r;h` z$;!;!Wa6(Uem46$4zbMJ2P*bo-d+Q*1Z{0p#mCV2AxL~jf|i(=*y#zkCTNxNF@r6V zl2aH6#Ldl3=EM|XdZ!l1Gqq?Oa-u#+#%mMlGXa7qPsa~(JfNMKI3x^u?7!oMu1*Z! z8_qoC!&uXZqCrJ2p+J&GW^IdsPVcC6*Qz7Ga&}2pfb0@=2$9I7WIhiLp z*@O4>9*DE-uERisK#@!Ph)E=6&}b6W#pE!YMVjKr=*W#ItlfR($LK(Zg8!}{-c@+H zXu-f`nj(u$DU1)^T&xU#bn0$pSl94&QAtc~=w2*`>9Xxfm(fG0WbFG!F)<|X$LcY| z3=`kkSFJ)L+v4ZXpWBTMwwcev3G0G+StDwy4PgckY&~6_aY~PiwrMIkrO{D$4%wdH z`_QEmLrGmb4B9c%=bPPwX~*#OBe40DaX;owvUlRk2#?mTnqy0|q!E_8Qp3g&u0C(_ z_t%guHZcoX-YuUY|KuQGTHB`x72ZvoNCRXzK@RrzX&S^zQjQ(kkoNk|wONDMo^rvr zU1?)Rsm0QG;s;*^^h&-UnFJ47)T;M+Qb=`U&aD9nR?n$0($oPSA-XfC;o`R}<1Iou zPR~MP7JaZk2)7LnugR-Nd?BaM*MYh_md8aUwQq5i8U9?o=rF`r5tb{uZaD-H8lZ8ONA&mWP6o6=ry`krZ_q*RVGI*4wRW_Jw}WE2zvg$z}(v{9&A z$0Ui4+8m5W;;wpxDdHco2k^?;YH@+9srRa~RX#Z#qZ|||$j?6oKL!(TaP+?oEv{yR zK!K}9mJCf)Hk5Wy|HAFg@V4~^oZB#?OS}0U#*2C)N>PD_Sf|bh4z za{4A*@{b1pPK-Xru-wUKNy843>aJqXsZnw^LU%s|6zvM?s7#$7uSUJCAku z#bXpn4`|L8F+v%9zn4R^XYDIP?;!0HoKx*=3(py>SZKCE;?}~N=*05!a>O2(M<)MB zg#0BAZY>-I5XCV@{sElJg>te6U)*2L(Q;RRyFOFMpbMrEKEob0wa*xX+%MrUh6d=+ z9x8Y?47!9fyu|k*)mldB-Q%Xc zQi0iDNS_sOR>=yVIPn0I4u2WC^J0Bk>VZ*&nbO@ZY+=zwCs%EI;qV7NGi$=Rh&>O_ zt%|-L%`gVJnr!w`m_Di0!#ioxULFQ(8ieSxQq8Og!w>t7C`+|?6HZPoT>ZC~*J^;s zLj%8nf30KNmBF+E+w#XKI+dnq0c$ zZ<3XngJuW2F_ew)Dlb+NEf2+5xz*M?Fwo&U?q4schH>!?pd${x5bEzoD+NFbk-5OwetQ5947|sXN za3szr{n2P)s-AOVlPQ2RfiGTarcYu7$Yp4(D7D0198oEI0e^3A%z+Jq5l$Ov@q>-L zefxF_nDSnC@7lg^k7k(Ogh&3Y0p)*k=8r5^&?$VlPQ5p(u3^3UubYHn?&F65@iBDe z428;r&$xCK+z30mQb@&N1qJUf{`e0AQSaX;v!gI(s2gz|Y$Gv|5cdeBuu@nzG-EJ> zRv9&2DzwlhBWG0l?T%M;3c$o}metu*>6|%7e;bErY1)P(-IP~NXwO0>lHeq)U}LL= zf4A3hek$MyFgm(mg-G90w2+X5yW{{mIw;U_IOS3Y4<0V;f-6TF8a8Ir+1*&kMcrJ^ zGY=_FNa%l=H}@Gdl2bDN4vulb%9>=$ZWxLLG7j+Kr+Z)2Bh^H!?ver~Fc#3969AVQ zDs4!8$Z6wL^YP(JcH9dk|I;6Aq(1^aIs;I zzj^Fk&G~)ast%6XS=pO*JF8LB@KS3vFrB;2gU*h)1nTvwEO~GBppbri0r{FVDj|Lb zD8jPcbJc!ZJ*Q5-r||fD8|!WRt9-q@%ZUQTUt{AKgbYC+64{rm>wD{vfz^~*P}U;h-lbo>s& zY=3{l$el0LlTT>K+S=M8M8{57V2&V$U_B2GUNa8_g5GV24TZ$Nt%8U%dYiR=$d7{0 zc{E)gcI5o-=h0D~tu3h56N|^f_X=%=^NYX?XUqo2UG)^y&esO?|8ReCGsAH~gz)NB ztGIG7HLm2QR_=d%e}lDg8<%gL{A*Zw^Iy+R)dK5#+gO zdG`L6q{$apl23?cjMGMR-u!_1Tz!9w0S0cW9Psu88h)L^J3RP@3Ef%B;J&lZ6E%d|7OmKFXbgZ4!Z|fScq0U45~y4&ZY{va zcf1)p@K;LzyT+xK4`D72N*kj_||d{HnF%fFC&$*2I03hE6X%tG`XyEx^26C zt$C%44Q!s?I;)i8@_@q`avF%c_Per-uBAzTq+!_n< zV5OC2aJc5z=4PdEB^hhU;V*M5o@Ndr7F|@H80s-W05Z$B&Qv4vY8Ayc_n0xpX2RkXFT@KKreF1gYugQA;>Fp|9qa za;Ab;i82XBbU$OlKi6~Vjz=+U0V%GXd^R8A08uwa?kZ5V(=fEHyLdEkSv_hHRFkEF zgy&Og4~r}_BiE7gLSN5409`^ z{R0B(5cXQVLuc&g@^u#JeQ{1FFGQ(C1z7OJOSeDjOh^zJ@(L0>&Uqj1m*=NRwD=WM zE7UsOa*k0mlZ>%nE9D=vy=-^IJ_?V}zmAzK9hcBb&O0_bM~PbqhX})MA{74IxKD4M zRBwW|hue-AJ4zaM;`t=veTd16OF>4RA8K%lUSRZf&d^ykH(Lzqw$|JW^J($0Ygh2< zsElJf3<|_d6Vz2de@WZeU6-019pQ<=L+Rdw2+~%&;q|vRyj?Bv_Y?o?QEm@ESrRul zw@%9j0pM{pBn2??JnR8nqHw<>9%}^OtBGo5{_&=hVfBtPh}J?B0zbTekEyt)h%I_` zuI9}}T^@FRo!eXF4;`YUSjA*tF%LAnQQtCV5Mg6^io1dyN}cDLdj+q|qn zxX;hOB7D8KymNAPa^Ch4IMS90T`}TtwBR_L*VQ$yi9?WPq`24t-hkiD;VVZ3 z7HOT^uUi+ZrCHUvHa2Z!!Oob@MwVoukzaHm+TrwE$-E=SlFm%H5Ei3vKib zWh>s#zQk+X$~d=XujltQab;y?O-)T~k&15yq|~ldc5xXyAQf=;DMB}v6)RS-uvEbd zL%KjNV0YEM=TQCk|40;+8Sq zZg7ltE4kz6XVcr9U%uy5x_#Yl_N5KPT^4d^T)L1y=QV(@ou!Z?uWHj z-_aB-$a45;w7{J1eLMzpuPJ3-+iANbZt0p07kE!bpE>lnzq>vidJjXJQTG^Dr=kzg zsCKI5Cf_ZZ>7fc@R@}-PsOFV&MR(Te z*=EN(OidEwnUkB7Q;R4R)@X-#zwk=UJ=c){)wo9!6DJoS_lH)#T(ucs(Sxx4SP*c{ z2GWj~0c8PZ_^tah$8COZI7JsynR4K^NA8K7g3!x9+_CpBr!_1HENbvi82^CXR>yhG zdwNq3Sa$g=I4_4o9JDm9udk0}BU6iEnR!Cw{d-?GXRfKR*m~d#e% zQOZewhTCl(z0&UxAAsjPP8968r|so z_wgA(hSIdNIg8Aa%ghIQuU}ut(us+zx&qc}uWg=H8%NQ+!M4!+?#`!E>^>uY-{JU5of8vxCQ$F|)K0f~U{8cg@zk_v}QSaml~Rq8Rr zcxVb3#I=(4pBr8yeOA%fv3Ma~=It%o;bFLP#xFV*uOcXOSYG~w${V-q*YEgvX|!@? z#_|8D#ia4anF);jC}`LLvdUclsb8S$x+t0p~4^NTzyC`=1{^Pg9lUdHYN*Ok8pZgWNB$a1Hhc8avEd}QsJuEOy8HrDF8tN0#L;P94t(= zoW2b$y&0rY7g^44wGBH&0%+ff=57?+5UouLspeEawHT4JC(GIPe%2i9A!aUQW`>P8E7uJx; zz*z%6ewW_(VPXWOp%{ zsn=Bny%EY0!IsPK1q4LKS@f+scspX3FhFRuk{7ody$o3N`<(VUi?*xA?l5lX^2rP7 z*iBkUwkJNfgPu9QjaSBEz152i>!00@@?7AFhs-8IhFd{bfdOD_5w#_vTGE ztPxUd;MJ80E#ZR!dv>f~H$-m)&_wC_9Eke%lM$Q--Um*J>7JgfuZVxJ&1j0<`+J?9 zxWU-yoPGC(qKiaprcuhP3xpQd@D*4lpx-0qvjif^3ozX}z{frg1O5$d1nt0AfAQCi z26zei?d|nblks(}mvIu*2QXbaPfs8-Zm7HCfnX5Z;1~4^QBy$Ur1LZ5@@_Gb&fbh= zRxI-IR4LK~6~(nfL$0N^iZjzf2KKiJhX9A;iwVn)GjGvm6LH`6nmvaPzXwW%tXfT^ zx|rE3ax_$#YUDX+=YqJor@tG{-#qqHr!!>6?P^ur_r^jh2qcgbMRy8Y<%=JQmNg{B z8fdAgf4V|P4MZS`8Uzs6P5M@jqj>wEAhRJb@vt3#+_RuKTH?6dO{^I~eh{&d>L!H% zgDvCtlP(P*lVcF8N0pHX@sIB-7#7xSJeGOGt!F4ZGqkv{wk#^u+`x*=(O(Zf_O2)`6D#}<&GG>S)E?=PUrm3hxgn2`@MZQ$=Q29`?;Tct$VHOy4G61EaK0| zil?P&8axV41gk^y6G=bQXJl`1%DJ9qo=>ov{Mg9Uo_g(oZXdp`KRTvxWJY#PGR0rN ze%)-V1yk@orWEWD01^%Yj7xs4wg2E!o89wY&-a}g7+z|_<#c5pfdV{Me)KOO==r z3nm9HJMo>zei>JwcL(}`K1$&S2`K$p>CW05WC}>fNc?&odiq*SynjA^#QSLuaLf0- zlP<*{^Jn=I?U0l?u?r){X_M^Z6}L+Dqp-uXf&vp@K?;`Z2U8On{sW?fnWJKnUDkY~ z;)08PUEnWC|`<>knSj_gFrea1($&Z9v6$rfMY6L`AoxP1BZ5n<`~u>wU6v{z6^Uib2fv<=g4A$>vADzkYXrt$~8) z*WC=p{EtR}r|$)~&599<6OMm;4) zyO(NfYriLT+7z=FP3Lzbqk2G%Hi+vBe2W`$oh8S=4L%?BrS65yEF$17{~vA#>??I% z1OZwJgGfYvm)K%~!01+4qZ#Q#f;7=-0=jk-J{M* zN&)q*R<$FF-D}HV&2(~!Sp>EiqpcGF$%64j*?It$_bxM#ahz=#29J{W*|FWV?CAi# zObEFy(6NQqSEGw?m~FiBS4h&3jJ3WIbL^3scw_!3eSarQt;-1I&x zmHCt;McK{3>-f_|7K-3f+~GlDN<$bXS8qL z8$6noTT|OgD6s-*r%s3eVAkRNVd$HFY8pY^M(VZgt?gElQ3~~i96G>cFCg_nwzLa2 zg&8AP)Q=%xvu)4xMMo?m@>~!-7~-7#Zkp6huou7w-NuKYKH(P1 zhekqWI%a0)N`Jw>biLUNXmcWKoaC#%>&+ZJArEY=I0>6k8N9S^+amOHwM$aqfyC|N z_w^#T19gLVw`KB;8C;}U)cNI9gSql&I@W_-TFZh!Lqv9OlM-+M)vyh`&rQyRD|M)d zb(~z78SB*Pdx4{Vk|~Cba0=Ez4pS@R$y&YBy|j10X*JD(XHHx17TdhZ$@2K4dc*W= z1<0**`khK9nIN&~m($D)(DaIT5g9Rj2EH=PJD)$&!~AFxEtD`O3%1{G%+CJp`95B7 zSRUq3X?ii|)hq;|sZ}IbjhOAfuI$#UGuW~te736#GM(0}g}Iq{GuoBxN3%6fk*`bo z7V%jtt8`?_Q`&i8UrJwHa^A7$$Ijj>E2Hu`@i(Y$ziwJ7W@Vedib@SAh2hh_S~>nd z{Yp1l)er=&X0-nlQ^(brYz1&_pQ7-O_@}`RNs66Z;+7YyDrt+>*98`52l;eJeV+Lg zpPkV+h$bQrJ(w$TwQPr9ck?$|*vT^%ly}(dCuCtB2lx8-e~aj$3aEoMiF`Hp>B_T- zNN(tiD+;&OzQ+&9qlopIq0qYDA-9M3rqNJKLnA5^0D)miYC!yc8}mO+f|C0zG2ghK zekz-FM9FtSvm>sPfM2;lKiAhEDBZXxI|yc3>f>P!ITt!@d^z1){>EBDq)fqY-cu)| z!6I>@vR66Db6~}p6lY%7;H1ie5-zvI4WWW<|9wM2-W?2X>Hbuy0(q4Mk1-CLGjq;P z>#D@3H@B5R)dDppk_-+JP%4>4p7jZ04-|NL%|S*X<Vjmxts%gC!1cbll+@gZoRTUxnU$^#w52n6owl4UgDCZ-W*85$abet@`!OWxfc;YWS7>H5?lhkL`6 zqvihxI?ohTMbBiLv^8|DUpwzN^W^P){xoNQKEib|T6(;{cHTfJ`X1p?^$uqm7m-1) z1r`G1ShsH7EB&8R(R2eSzp&$tIY zq;r2GPNp6e;1$}ScRartZiBiG{Dt5X zOmHPCf4>I&FTMp7E3mp%o=vxvX+Oon@>t!1PZh9rY@oRQf1%k&&L2?Ee)Y8GKX0WO z?P|WYS5Coy70`n(*>Q#MBLnr=Ko(R{!^84j&^MU{JDtv958^^ zIVx5P%|S)Cp1j}ofDli*e!T}uez*)pbfqEm@zi@d7^K>uO{<^~70f*9b5P`iGP{UL z@fR?@rEW(n$gQZ7*8pP%?nIw}uvDW8orLddOKf#OO*$^ZCs@hF(%jpI@hzsNor zc~-LIf}BtOVW!@s2%43U#sWl?w0^@QCI*N!+;?{JN=Ywl1_9OOLUWZrDS+SP0yKWN zqX?2?%Smbbj<1+RH|@&iv+9A0?qeNbrii*1 ztUerCt!7}H!6LLm+rRybs#R;~xbD(TcWkEUNap2tLMyRjOj z@U@WFbYjz?CsN{j4ln z4LUlYpTSpjbAQfh(}c(Z_covjcZrnr6VnwMzz^acwtLfP}QVw!2?B zs1%UkuF&I`C0{>6E$1yBV(~6448MQ>83EGC5AX^O?U}vU+zYXJvEXUKH;55;1dB38 z(=V*bBz;F?Zf17)y8#2#JiepI*xqlkxp(IdF^LLjG|~N#h%nyX$pEE}iju*oLZk(c zFsc)zb?MTA3BU&PEEQXSGR*ojqO{b?pYNNYa>s=m>P~GbEhMy-^Fz`Ad>YxwJgg`~KPCbZBNS~$%L3FV6GC|BGcH4XjU1tv}b z^|=QRGmQ%D6HV9fLy<|q0z)K#ecSl>Za`#9dDm+4yT+8GrV-1d&oHlg7j%Ee9$5sL zt}%!zgb*_baN7x&&?^0VJCKgO)1|{W)=?5Rl*3Uw(f17@nzm zr8_3q6=3?J^u8*vBm`0%yM=Z8gABRCob@$AltN0Nh}+XC|PKA`7tzFZ0kDTZ-E?Q2(> zJSL0Z;4m2&9OQE%+8`pTYa_oghY~&Usz*%|5p|u{)S#4{y&d^>@uq9&;W^@k44aDG>}tEA_1fly zG8Uer6xSDn0*ucSuXht_ADo?ih}($Ts4O~8gsz0!!B^Sl5i8T3=AMs52bXrkr!HjGcx$;Lbsd^FO?h(YW zK)_TZxrECprk1#uP)dZq+2PP7QJYN6f`*Z#&jghLCxj2|!yF*&L{vOEkrl7E>VNGg zG@RY`x%}tix(k1(zzUpkdAmh$63g?K>YYrfNU|ukLPsBl0D)_ zs)rQ7oA^Zx2n&}tY z4Bgw@)aXp4|NLf2N!=`_2XHEN5o90BNoC)UAi?ADi;H(=f|zLpd0ast_}cP_W2YB_ zBZL;j%zq)@bmjL|lm}mf=LOBnc);_Rw=Zt*_uq3^%%6IUgr}i@{t&oZ)DztD2xcNa zsVLY%8Nydn{{^>&C5j^uJP46%zvNd_Sf4fG-ZP(tujxVCiR^`_VS}3i`6VDL`vYmI z4`yiH(WhL%UPxFt1NnMRZ`3G3M9xoQ z5!9Q9RDnUUQX1;13ls-nj{J^^RW|UWuhrTK{iii($jZ`L0hWK;!K+(#6U8Yo7e>83 zADd7GhrM|ucPm>;o=6fBjSb|D5_kbMADRUZA3!^+V#>T}L;ATlx@eOMP%0{uXdmnyED8vR@t6I8@o5Popd#<2Yp~^qPM!p3 z1SQT+aq$Ow83LsUQVt8(7@vX26G9(HoO1j2?SpC>&!-f=G$3-Z6(&7Q$I8uZma4%b zdlLr&5k9;R&HIseAjK2Wql>Y9)16DqD8udHNA(kcQ%C&vgoVXW26XIZak+jA-p%5E z!_5ZKjp(IQLVbC^z>G zu0ygp3&GK}eCKzZDOV|iK4y5+JtR@~!L_oAB-<7*|C!bGQ`0cCVGq(JP=+;l;ZVwwmvoekL?bJ;L1$B93)6-jJlnUcU-*9gI!9@X z6^II^_LV!rsxI8dGj5VZg<;D6@Nw{*>xmJ>2M-=RbjTl^Bt#FR{4#c<~=b?CCqq{|(m=y>;kHicUhYXng`b+iOVVyj=RfHEsa5KW2bNbAqL$61h zqcXVlbd@fmI*uFRCjW{dz#%U0xMf|2a17HcveStnMZ|ZD7CuX0BEy)PDWIQuJ&z#N zjBT8W<+j$>v4DNQ0JQj1tJW#(84gGP z=H5J;=qfk$2}mtBhTFzOhKIM7rIeun)+X}e#T|;-QSChC8iO#gSFxYdMVq~DKSp^4 zgq=HE7cF21wp_b$!kqCz=?gmXZzCZ*mn=NyZ)z5AzmPo(__Z!;vq~7h0bNH0N{(A; z2u3pH5kjOUoIr&M^u8Tyl&rH|W}QlM7E2U`+iH&oogNr&GXvxg!;IJSpZusj)tTe$ z64DP@Y4n2DwB$r4>1{U2xo3xWMi>phJo%VeqcJ`)0g9^%CvOX^a^>&^gXIDIV7wvOP~U6;k}CFOQ;I3ZVak-ZmmRE_t1zaeGeY>yc`>=o2_@sH?e`Cn!wLl z7DrHp5|@b>@$v4IZ+wps2~=3;EtZstbk3Rs6s{e&gZ_+Ru5Zce;+Dd)aM2?I2ufTZ zD|$%2X3qFR$17nVyC7|9=x>^|e$+BsNoKo{y_sv{e$<6LG9|eZu&+0$;D5lr-=*z+ zIHa$a{e5E(uJv1Y_P!T?<^1GBpmN{{N>59*e)@AAxcHSu4Vg4u7+Ri5f2)Q%~QvlFw$ zmZQO~aFT7H4~(u#*bu{N%h9iOJW2QK-aTq^UgMTH3N5>##w;mPkd(|v9Zi!WQFYol z8?jpHd~CD-VC>X~DVs-HG=hQaNOnUx4o<+>p#5m#e^P^|$q*+WymI6F)4koP6gb%S z%W9B0Vj~M13lt(YYMH4li{YfWDldQmI?)-|9E;UQkC?4FivdY3StBVo>qMTUA8*Y{ zDS0jrxPh2vt5XievzZ^Kg!SPHfmPU{pYjcS!}K~*y<5T+zz|vIc^bw{6A$fP)d)1~ zL*>c}DTw|?kq-sEywOb+35-0o;M7c_StOUjodw83q@|7Cu2HhecJa=bu{ruJhl57& z>840lwYt}@e^zlXzB_GVs@)yc`r=(Lg8Fc8lwU{AJ!!Vfdag}pMG3FRh}p5u2_kkj zC?kuT27iip>O)ZDe7JPqo0sX4edVB7o_>9>uV6;fp=19zxk) zT2+14K-V&#DDj1(Hb(V)%LhLo8LhVGRwp;Sn*NNMqE;>|D^;#{Jw*j zuL;v~i^W2DuaYZfu4^abXC})MV#F!qgWwA`o#lehd=%l5k<;jyk({&Z% zOK388Gc&0~aNIBJvAbsI+}3W{$qon3i<*z^6CT~h`w_RS*_6= z3QTu;etj1wdFZ2cKY zL3{3^)_LM|J$O}u4Tr6JIX#utl3dem$kZ4a75A+8^8>}YSPLS1s9q|bETswvsG#MS zB8SD-cBopLttncvoHN&yYH!86Wz5Siy-v=i>yFy;)7InenDK#qjy#tu&5d&!Hk?hE zTAKRm<+~F%S9Lt;OSAmCcdN|520i^YgZ! zsSmH!qv4(qvH9YCCN;W(Hv0g{u#0QChdG+B{?u;f3ae5PTb=vIQ-)yYS~*4IrAupY zjfH4)KG)H34I1vJ+x&%nE@j!%hh}p!)%SPGSxCWV6GhJ5I{tM@2)SOjdu#ogF8Y7` zo(v@6_j|kKfB(J4Man~~cP`%X%$~|>`A!q^ZGWdGvc!Ue3Cj8Zc>VmJuW#(C*Ajbz zGj%)ylNN6DKJmc8yMU5SM84L5z9HKuTFY#I@%nYr&Qn_OyF>+8S{nZk^UfTr*&`zG zhX;{a4)^C=h7wKF86L{ry@1^Ahm!9Z+TI~LzbJT5Cdi&SiFQSYBnObnpsR8TM!chv zn3LpE%NM_kf|p6fZ)am;L+U{QKsXri*A&np)4tH74}P{Gg2?C_=w1@=f@Y`5?g*vz zGlrp6p|Rpg7kIX%thRqfbN-O|ybQl=2B}(4Wn-unqXzG<1#8)J;^e)AjI_-!5f|9^ zENgy)I0&KYB&l294&7wtqxdxQhF|>g0rOv)1!9pbym;eu^rV#)l3{gG7j-qYY||7* z)Xij_z7Iwg;pMAX7<@S(ktw<06DT7ep`BP$J&W&FEw$66JWQ#L{N1X>N0l!}Yty-# zU#qyc!;dZIdfbdfXi9jsI8Ewh6^aGDcYfw4c1&B9xe=5Lkj&K7R8C|}h`cio-wv>i z;RdTkh;l(qy}Bk%ba!gv;^t0@k3VJF0H98kM3H>!R0FrTy4&9mO13)-s88G}Koo^& zY92irGk+Q))Uh%9`^JTiH^Roo_qEx@5YQ%K-9^lu5LBJTiLai>K|YX=#I_UT;WJT4IW^5m9m(JJmT9%8)U{8uR?InNuZ=iNh2jjk1FnZD(fPuu z(+|e)t+nwIEs*Il5IS*~-i92|O5LVme^O(=pHmxtw?#UIaTyKcVJClRdf!hIQmkok z_C3`mNhmDeOX`MQato^vkCXvuYWX(qm(Q7 zUU`B5phL-3!*ITFW}u}|cdC>9Ra(!sgea}GS}`utk|kO?88zQk;&q0U)i8jJb79!i zLN59Hr2DyZY$&i0K2^-e9dx$qlrWR%`a1fQmE>hflU+SMzblTrV4|Uaty^)1pZ|Oy1Vvm(7h`bnNt1V*vWv zkQx|NnJ1Ki)i^Qc2Z{NAutj(v-Rr7D-ClG^1BuSPgV)?4-b%842B08psYgvx>X)Jq zT*0Fj+ulDHfb`)(hfMPKS@&~Kf$RUdbiKsV>Ae(U`q#=iix!CgzVqnV{A80Expo;C zVbqPOj&7sna)KmELRA`M3bClAj(bz+WR_$LkwR9@3_&CV{y%{BauL#~MM|o2J^by{ z>%wXud8&|mLn(*9O&fk8+z!RG={Si6sO}0a4U<}5w;@J}lY-1$FpJ>*+Hl?@Uvs8Du+{h312b6)rd*{^nv4B}ZLnE|(lLPom1s#6@an3}^bQ zH)qt(t{$>vUaF*8OUrla(u@U7>eqc_T>El={N|TD z#Fx+S_ah;DsZjbrzz%yWWztZ+uAE;2eDwWG)E%NXQ>zt+w#K46m4W`acG}Qf`rOxn zD|#n=>0HF=m)uSnIvV#DSBPXNa)KI9ef&-(Z1WjrXb}H-29Z+SyC+bAIXT^^{9Z!194~gZnY(eK% z_y2M_zaG6CC5vWTluI7^@PWWPL^W)-1}{m;fppq?hMKU>DfWA#tBT(%2iyi`gYYUh z<=x(4w0v;c@oGP5 zp*S)AAUO&m$LPidJgfspAbh^v7MxvRVW!zBLT8R{S_?T6!Y4bIet)^|_a~_k+(X zBO@a{%6pDZ&n?(XjZS%I=#4He`pIyVm6TlfQ)#u9@Kf}PI?M5no5lF{(AAY6_;yX} zubVUvddJSpksA3Hh=nYhARG4&^aS9jkjvwf}TmWG|bJ-Sxb2P`SsaLJs)%~>@F=UySca` zrPon^)y;CMTl|8{TSBxSY^-Ko+kIHRn>78|gFU>)rD)ggm4~mgEVZrXZw)6do|e*T zE6Ei(x8QfL*`~0&*U>{fF};0s0@pJqUoSuTyOITnrgJgG%R#Cf2@%~ww76B - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - off - - - - - - - - - - - - on -disengaged - - - - - - - - - - - - on -engaged - - - - - - - - - - - - - Key: UML - - - - - - - - - - - - - - - - - - - - - - - - press "cruise -on/off" button - - - - - - - - - - - - press "cruise -on/off" button - - - - - - - - - - - - press "set" or -"resume" buttons - - - - - - - - - - - - tap brake pedal - - - - - - - - - - - - - - - press "cruise on/off" button - - - - - - - - - - - - - - - press "-" -to coast - - - - - - - - - - - - - - - push -throttle -pedal - - - - - - - - - - - - - - press "+" -to accelerate - - - - - - - - diff --git a/doc/design/img/state-machine.png b/doc/design/img/state-machine.png deleted file mode 100644 index cb31c0755984dafb437c36be19584203e9c6088d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 38789 zcmdSBc{r8r_cttxG?7ZiRFcX(hYXdWl8~8DhKwQe5K;;eMF<%(WXe2`yCOrT%-fvI zY_@4{v2FWZm+JoB-{k3r8rFd{3(>^jXvV+&J z$=xO++XMgYYS~K;zbPZHqb4JJOmX4up1sAfEA`OukDXK1N?$LQ#2RhI?l z8_D;}DDK|<@XWW&vfZ={b%~$&&s_{S9pD$C<~x1wd(WO-)QdkZ?QQ1Yr>iC>*KpbF zo^Rs0@jXeWF_h(uyw^!J_}6nkom^8h)#DbOP)0sWevY@a`#6ZL8&1xeo>j09GP0Am z+}9{~exl_IAnyE7$73wB^TWd+G!62D?9gdE1@Z%J>HnWTVt@C?C*KpH(|kS~L0eT1 z#}zgoeKMB4e9`9S=FN@c4WaX_$M@Nd zk#W-!cEKkv(i#eK;wd_=V5s0KiZ>h#PiyD@>@h|YHIECgVw_p z&Q?}dw{Epbtq(cEHdLRkFAR!Vb*Gm+82t9*5Ua%72AlqZ)WpP-Vq)4AfjSfyrQ&aif#SW5N_~LM-4~e*5?{MM5g$kF2Oqvx& zafg(yd++_CS^|l(MZXWao^giH!NEZ*ThAGT7Pwwz-TSerwKbw1mfl{frSW~5>a{RX ztQdMigr`Z;6K^y7t+DZ~w3mmaio0g zhWt!DKZ=Tq_-cNA5ymU}A7Z`Dw(H#a^Br)4fryxxm^W`sFIjEpX=j-(+9!4-QHC-K)eFyY2GR3`<+Q~~&a+BO zd91w}`T5frH!AKp)#yW5n{~uv`u9^Aewxb5lOnEloftfO>s`9ciN{$xn(~;S5f=40 z+tsalb*eQyM~Sz*&O+_!ac-TvOt|oTlZG4Gw{PF}O7M`rF0J6;z0$iKjkpOF7y5 zwyJ94W|M0Gf@zDmrw$zJEJ0AC;G< zTJihJ;ToWDTL8cU%0;!f`b>1iS{v7N}S&57V#;DBx zy;HGO&jrl<(x?x~%US+Yyp-pf^pV)wFA5l{q-Q(6p{%_uoRniw*|OldzNpETZB*r! z-(hi>Ky%^}I`J~aY@E2`hY@oXGc&XC`j=+Zxz+fX5p!cl9dy8d>zkvacCMkMW!jB(6 zhDvXqv$s!28)<5an0-6CnoVdulg`Xeb$}t!H9^dE@lIOpko>=nd~F59$9wLaMSp?W zjT<-Sd$YkBh~NUAF?}qKeK1;6otL-Nsl=zOqGE^6yGOty!jUp>dlxFRjb?ECwKQhd z969VwScsEw2IIM=to)!aXZZX3t6&>edSS!&7Qx^CaWq5boJ|z6JSTS?@FF{&BH-{J z#fJY0Jo$YIp=g?u3V*EkFkx5S-v2moEO9ETo7DmUI%ZJ1(vd3?u?DI)=fk(hW*>$JC zKC#SGY|&x7I8?r(G9uoH{2+sD;W3%mkp0t}?~n7|Np~en=b=7KD771nfBpJ>!#a~i*N0ZH zP+S`F$+k-TfVJ63ncK2)HT&nfTbSv9llQ**V0^}3ih;Qm4H-HfH*o6c&DO_pA_42s z-spY#YylM)Ly3c>cA;`p+A+eGU&MXjPrKzCH9!ZuzdB-M9{tLzOjw}l{-Dg9vkKl_ z_T-fh;O3Kf<4sWd>q~`rWV1fT@U1g>$GEg+)D;q%c`I8kVTQ^bl_I%kd$P2opzcifH1L0DTQ{=7Xx+i>H@QyTBpR zH#&!euBUOY+`(N+du`6JKf2!@|3FXY+WLrR?)J5W1R};9RJ)f_TT3g1S?m_uoY37* zWNEdTnVE?53tgHWsjifJXp$nPgK=H6KSnUiZW*ohTK;-G%P>~*{Q&zawznoSceMy$rJl=?Sb-cfNXez!O2@AkOZg zc?(V*mcxo!ySv4g83~h;gV5H8q+!d(SZS$alrM*$cazZ?V*GJ5INWfSoqT83sZ4op8r#g38M# z?K>v;b!%f)szxPPOGCrLK)=*hv#YsXns9TzV76D^F2Y}ifGJc6+T!MEjS<=ZNJ1#K zmYzp%g4%`GV74RKfBK8{-9L}riF%nHzLKArHv=vja3r<`)}m4OYAKB}K1#u@#NWjp z0&6{a@7ccSBqVHr>r8`CWZoXHIVw|aJ1kMn_leoAO2BF6J7fbN($b7B%HS^H(I)l$ z&bziAlnzh%nDUGIzhsr{9VgUT{|`#TBXQfFC@$H1+N zvWj@ly!Q`&y=Tv!PC<_~i>~CS_QEZCYHF_q415~z{uez^n^HR$cZHf{&Rsr!y6nXMM7K4 z6hHNn;783YEf9MT;Yo)vs>@%$&Rg+cw^oWKfmh$wIcptPv5|JEvgQh$=GrwPVSRn~ zP?iyhTcz81+}CvlwOr4fN?d8?6Dy^Pk#NpDz|4R5ac_+`(L+R1gwt@Go+xy`g&#DQ ztrwy-F3v=o}jA83_8YNBc~91?((BPV(N`f zRXR?+;hXjvTQScYif#L@a%ou(EM>MU&2@1|-2R8GNa>wos|ZoU)o)z6ZjW5wb6vO) z*1d_?c4XwOSW)nO$~>2qszNWV1X!_&J4{fqJ+hzc&RAJlp^@2ArSrab?|s)jaE!NW zpC4k~Y?H*YNVKS zKd$qHLpbm8Jh|HM5R3T14>zNYYdWfE-3EqLphdg#OLZ`_%2-N zH2P3{<*WDN5)^r4wO~b%PIeVpsz->n=8w)-dX6T^2c6k6H#193Nhz}GvHG5PS--?a zAT%HHYZ=6zAZ%n@DtS0wPZw(58siM<448ojIbH4wa_`NkEu~n|2e;MLW5w)7vh~HH z;YM+NV+RLoygWf%A7wo}JRk}~%o_!F4$Ay!Yq??So^QTU{)T-J8tYth9fe}V>~g5q_Nr>*N-a9B{MVmR=XiO0 zg*fW~%&c}D^R^0&um+V*_X(pc?-fufGu z3MC}Ak-n`=>aKC_6woI{>l`u1C0!S*W|D%|8VyY@tct;5B6DRpelN}5W`aUjIA>M# z*XVbf6U?FuOo>f9e+uAOW+PPBIBw5%Z9}B?2yi{k&Yld{e`3^}Al|-Hga>#*UQ}H( zzx7gb>)GM$@bK`Lp$8fr%HokSmQ0oN%ngh-R9-!18yA`gCSDeh6$p}}E=6MJc1ri| zQ-Q|_Um+|{wZ#oqI9s!$(^67|ZP0m_pHj}DphA1eEY|kz+YKcpq54KQ_cxccj97ke zn|sU{jYgf}>_HY~m9U9_f;O0s$#H7v~ z8pNNICEhxPC$zp3S2sp&Ww|21$WDQ&J{=Est*|?dpD5 z=Z%WqNpHAe0;zqlVw8KO|&3&@Dy~e1~ zn>!h^aqau7{!{GT^0+W`eSoTpl%MXY-duw3Nl;E;c|Fx46hm3bS^&CcIr z|Bo?$HmJv4lNDaJ*zIccNnL)ec;Y+{PY)aA$$DA>aih;>k2GpMU%I4SA$DzTe!RhRwT)BkJuac^SC-zHEK%SgazMy&oF8Q&`LLgcor~ z>xLD+PRgy>oZDEPW|MHb#>S)sVQytwhUAVx=QA=gS`~x4;L9>FG=v+YeNg8uvm+9# zmuF0yn!+MAZ6dcW*$$O~bmzy7^}lET2q($jggb?L1JrLnfJp@i3Zk}{8xs6zHk(LxH5&?SREb*u zoSwyTDk=fDSeERyf^uJ-S~2pXecZ_Y=aICh0wXtCL0<{$*xWB9P`q1l5~-nNwAcT6 z@YtXoN*UdftD&jcnjnJ$O)of-Q}*)(l!i0xWO4`o?1HQhvgd5baxzeY7v-L}J;0%w*$MG-J{YhWEK$bRx(9A8P|7|y{a5sk%~5<&x@KBhy}lGw zYtnZDCTRYngd!SRT4#NY?nlsVJP%m`g?MtM+O5eg$gq3k-D1%nM5x<6*tn5Y7o6VmuNs^zmO@qN@6Nu|HxR5X21_1TT3QxRY%EfIf)e|; z+I$g(LS1^q=qlYWsTw<}31(Wdj>FDHbl5=M2B`=4j2Lca0-DpmCL9R1Q;*r$$0 z-Wz5R0dL>#gp4k0eV2X6-!n6uS!fo%9fq3lsiwIXEnxB-(`h?gp{c1k>qW-$SE97f zy@{Jm!Vd6o52`QDKkEV^;`C>I{rc5|jm-b=<}PaL=>hiCovxO=vf@xpBXZK$=+UvC_U|0RM>K|@)NRpuy{(e!qrZkt{W}xUgeyQDO+p(JB z@jqsZ8TTYueTF;}pvCa8FtB^BYMRGVe?_+Q=LSgF&l21_lH@(8$UOg^3thL4?_r|e z5>$#>%5^2n^Z$DgUG0v3R}WlRxVs1EZ+_eX?<6o??0cLJG6?(it6+a z@Gj+;1+c7HRx+EvW^`9Qir2t&?$?MXc9u2Z@p<(RZoimhPTu*uHyf+Yl)#&;C~zp% z6o)_c{n{n+kEjk^CkWcTnaFiLDG1+5Jp0e4mtnQ0s(f(Q@Z9IQ%M^f7{Sj{npnB0_ zhZqIru(EYaBHprqJ^hXqXb36?qQ!0{NE`gmKN;8UjTYlIsL;U5o>_)(x&8C44ge=t z%ohm5SZ~fsFGy8Ed`f?Jk7;af4tF1ifPp(Ccl-MFKnx&KmGi7*+<(} z!V1U#ER+5NqH{WZc>PR_pxFxyso{j@O)MELIk?MxhrnrD30Ma6E@qQml*g>Zep+Givl@SJ_wQS3wV?k9n*!{%vW4;uEcu13&x6DI@}toeQJ=b$!w z5X=SH0R*3TmM7p#L0(TFw*0e}S|7+F|I3$na4YzWz^+W)s-q+g{x#B$E3j0DwQV>=|yZxZav#wfj1 zd0rIbE>G}V21^{;^kQA%`?*P9coAUKXu&S18RTMnbNVUAzLKdUUeW*k0iphU(`C(x%{Bjo{LbjEp?QEc(%TzBeNy zLtI=Ox)kk5_~`hs3~(j0h?NGob+bo+wg>Nm(&YTPbB)jg!2dy_4vGU3WyvTb;XK!^ zt|EoFHAZ&!)s>YT)b{3{{q!v24rhYzUDnjsSm3J~c&V3f{EYK00+}(WFKqfD$s~tT z4-k-A*quH#NPPAYENlv^LQc4)GIQVS%Y7 z8(;|nD;v`ZQITtFP8(xCK0i%UNr-$6XpYwpU#hbz@far^HyhreZ3Xu%!Ydy)1whK0 zZ{E5UDT36{0C@x8va-6`qoxB$nfw1{uYk#`5;RmlYUhpsUfR0giN|CZp=z9}m%i+u zauJc{;n{(pN|EF*?jG;?^8%$oP@biJ_;8kkgOP?&#=9!`9sj*A&nTQl{hu|oePMm@ zMvOe)XSBv^rZK<`Rlm=(l2jl zcyym{#(!(U?nA5B{*x~>zLt%LNUj~qb!~igs*@wNX(+uOL9vF;0mOA?0s0`l1IOGq zl!{;~;U4NxaA&6_Ot&ZS?YDtgColi3x7c}J%cvFq z2$E+7ll0`Ldi5y+a~F3%W(O}07^S@vVS3yX2xW8Nz&VutxxSI-QYBoVn*0SixK7XK&0bRc_qK#HzP*HZ{0y zAV8Sa5_H01g)L9>K-O6e5V3Z)-mzab*~EKe`Gxk1%#$x=F?`(IZ}#Bl`&6(?9_{f` zv0-7StD(yv#B`OFwYvcNAx*w+YaNQXL*4XK3x9Xb&wHoHkug)?;*4epJ{Mi&NSz)h zAadr&-eU0rTOY#UsOhuj=4P3#rK`F=(M(LWA4@Bq=Z{X^9~dnqOH$Gd2E=)m)e!qF zf-5x(ex6boUzM9uP=mIURlu{Pq|+u_pjVKi-*;y9?S!%mGy?p1eUV0G?HBchFLAbS z=JS1)eteGqj%9@wsuEU-uiCW8PeLNyY|buqLq*7nantlIML#If3G7ssx`DWJ1})dhYHU z&kr)6@Yz}h7)|M36vqK3;cINBo;q&a^ovf(cS4C{aSJq;;MoqU6S5Q2)AOwMwIt#w zsmX846-9^}x%Z`tb*Dty2smtY%VbqhLj)0ANX}ouEn^DXzpccISkt>bORG1|sul?$ zx`=jl)a;=(2ziky=9Ks5%Bh8Hz2bs%hk8e#L53UKkn+UOc%23FQPeN%dbv5t=QI0I z1fS(<_%U|}`g>Wgsz_CQbGy;w3W-(jU=#KcG&$AP)hYN|1kll?Dlx*>Dc?jz&9ln> zy#5Bw$LHqW-yCFw<$wREZm%NehwlFtS6%h`2-kkpLdoD-2pPTpLiR&UX<>8-p6Bt5 zp!5}_ui!`k>GUY37tu0b(7Ex@;(MZRT7}FT`U_lK5T&@4+!l)aes}bed4^^caM;n& zBldm7m|p-OYD^xe3Mai38#%WSW#nD{+uXxPSwgV~*{dmd86}+3F=L|u!+SKi_p}WH zUZ3!jU1r8>XX|#;5i0)(Hi-IF7nf*O_3rMew|N=I0DCVQIsECjL=Bl^XPrjL15RrS z$l2O1LP!WDd^D&8kglW?4aGPV+IvE1^zL;(Xz^=l9EFw%_+U*9jcX6wp(sgHj&Y2rAB7rSxuE&pW&m)TDF zpB1e%do`4@Z#y`w98tNXrlsYwxn_C(%N%{4aZje^)Mt9Qs#)mp%)PYRxQu@tb?n|P_ZGDpKKuj7^!gpNy&8BwUgW#wG&Z4zh zcz4@%?vO)Px>;J;6FMxH+*GlTm9{q$NV@nX=tJQSvCX?nPS24klYiZ|G9tYRWq3a& zEXsGa_$`@uwXx8vxOt_8;QZ?bgQHdMtDm?oiO}owm{NBl8Nby+*qD^Gs0m88speW-TNhFkLI66!~BEXmo7nf2< zh(;gO_7D0C=rKm_s#Vq{hp$+u{_Oaz?Va+B*EGK;34(KH)^O96-a@<4niC29#y-DK-Fu8e z)(wKO)N${2@;iOGMl$_P08n>Ia!)V3%c3CX(884FkRx)vpMJtK>b;(d`GQft0rG;5=$vmmDS6ieL2hgaqc-j+J%9NPOvCx6sh zckekifh(=54-B7MNyQcWeEkroB)brt@gB4&!&2^~yNk(NoR$(fNIx8J-%kyId){Wh z1b%SnCV@aN>!hoFfNabilizSjR65b0sRFvMQoD?t#}4kul2+r(SMMM_Supev-mAqz zam9{*9`NuhV1XNaq!m7Z4PK{NKwyN{_KO!UYPL76O=OGpy(Su8k8VwgL2*zp51^Jy zGf!#bje)9gRB~%kA>!|gLBc=wbBg2djQ5>kOk}#XC}1~8$6gkro|XtnQ7U{E8a~y~r#G_$YRHXZi&dAlo?X3kVt6Vos&7c=)@BZn zkejSxHt0mxljQN9i{kx*kOFCudZrM)`^aI7S*JX@7e0G(8&F5p?RP{7L9Y|UY=_Rt zjM~>8+&uZ#ue#w^kzSITdDDF ztaquYGXqT9P4>?TX(-6vUh_G8{UL%vMMM}T)+yuH0^MtlPRS&hW30#L)qgM8!K z8X9|0i`DpY;-jc99Lj2Z6j-SvkTe0{n(fslV9M;Gwm{vR9~c zcXJfvt?Nw8mB^Qr&Rj|H0;Kpvm-3YA{%_6A;5y7o?FLI)o8gWDk}>2v#}+8-MVJTF z7k9{W6TdJoAn?;G9lvz{)$3Sk51zC@dIPe#BrBy?Y7g-LFup!~_y8T@&zJ6ln}m*K zd3kv!hvOmyLhd_dOVApc68H)JPelr+0cK(v9Nf~}T)DkD4{Rd)iN?dLdBMR4+!X?E zPAM!y1(CJ60bL@7m-TaAKr3<`cGd>BYt~e7_Q=_rzeY(!BSuwCP289Le2o`?3n9Nl zzOv4(o-#@gY|T!034Lb#(F(X)&_nOeoSK+`!cU6o5X*PU!69lct&ggt9weU78%*tq z2U18=7zc2&tibQyv@^~AQ>OJ@s<+nL;6rT{K$CbO6>#n5yZtN=^7`=kA$C+WC*H*M zW@>T)`XKB)r_sMa+G;0Z5S3ugCn-*aUEa91+jZ!nf5Tc&ZU|31Lur(&h^hEO(Hem+$4#o8qFS&|k|+wuXm z94It3H^-v$YcHXJH_?cvH>mFnVUrn2Q(^P+YX^jd*zVyyX~vhKo^ijkz`Xs$>Rm#E zL=6~SPK}Bi+X*>0ASyvh5dY>)k;{S}*sJJaf{*FDu_)Byo6s`X&9&M3m&|D&KAezr zxrr?t3-#GbzG=;i+H#0p7_m2h1ucu>tD4NfN>GeeU_(d=Tw|&gXOaPu`mYKSD)irk zx|9lNJvL1GLu>p-e#F1ZM;pDA;kqn_EY3an;yy!32rs3h-jY7skDBx{)x2L#W;utO zZI8=rCl{3h&!nKB6HwOdbZQe#vwTPs!cc-?6@4Fd!((ITmlq6IUBm~d=_C4+l!6%q zu6RJN$qe`!_Od@W|5TysZ%0`KNKDrVeMUY2+VFCX7MLknvOsg$GUHH)^N@8#;{~UxAe&QTtjz-wLX<+)j@+Wo45W`zt z`109gZPa_t{;pZSSAdXHwgn^Kys?0ab>W7xvhs}^f+s}ao*BjOeev5*MLBkEIkU%l zq^f-L8tUH6_xD!zKm{oG!j4pD8uY3bUmSQ5qnf1$dInb)Gq{fFPaa8mg*vK`oE&#o zEt;nSY?U+iEW+gzDh2XOW>zOO3ic(dWj}(zj`TLVbg<2QDDH)Vs21(3Xqn!Jx!_$c zhq9lZ`{YY8Cw&8)kj-g4NiJ?F4WdTu@zWl0MnUI*G|5Hd4rONROPBcN*|Y>Lb%9pI zcbsJ#KmCNM-dnOSz~e#tNgnB=Rvl)BiE&<&2j{<&1?&>J>E}*$grJ zKk*aiY(7CQ-OW6Uq{q0U3 zd@qkCuMOmAztF3OvUHn|2k%y<(9=3+=aSmStoQgrRW?n?TT45PNjeQu#o>TFxC7LThx_@q zJJ8Yju`=KQ=UEOii>5r1i~DoB6P|siUjY5++CXmk6q93eviX~g2u`&B#?@l~Rg&%{RuJ01oV_Dc8^v$M2#BJ%W%8Vxl(372_I%S*L zO2)Ij!FxQ3`s-AFh$k2{RN_;h!+<#(reiT=w2jZuK9ETQ^p4;^Li5sr!XE0ZXBw`A zTw^5;#U=l9V@DBGzo;3d5Y%OU;20JTqx>MI)mpB)Sn#P+5u7Eaf-BzY- zb#p=g|4AjaJ8xXz2N`%Y6%;t#MyMTp!GBR=5DQ!@jr&14{B3xu zMRU6VbgqWy$K;^R0|etjX$sp6HH?hM+RU5t>bw;*r*Z4l0;$)-&muszZ6gbT#UNlt zN=7-~ohPYmD#kixIXP5XQ16|zNxBc~dE|bsut3VROpb1Eyk5O`VPb=XUV8gpm5)80|nRu$6$ z!-h-I{{wFl<>>?Yx(PfhBY+I`XqzHka%^E*K|uj9019@mHax6!oc>b(jn5nmHgN9IL|Vx;jQNToK`gAHSZn8b)kz1ZQ%QYe!t%p-q(S_z z%?F`b++L=aY8j)Fy(VYAcZ+w+cFTcSg0!>VLQky4vzVUIZnCIOqg0 zKY*#68jl=%U7{cH;};zM(1jIUu7Q_G&4(wZ-HH>Wggb;J=oK^rDz1g~Nf+Mj;OO2~ zxpm9?XCNmu%#o<-IDN}GSvPdHw{YY;p`ZDq#h^@Yap-1nOhEDGfijMFXg;6lZqIb+ zLHGbo6Bo7L_&E!!GhHHBlJjKOjgU4@Pu5%N-6O#3eQNn;{EyAF&h~HrD!W;9E0Hp3 z$~aS6ze-OY+I^tUEn?MT;ZcSED&IwTI=M0`0oWK|aDM>+vJMgBfhk2D6M3(NgGi&! z#>)>RdO;(jDd(~yP~m^bc_+reY}+x3lTt$J`)bOCLK<*lVmciqJ|C?aaf~9z)p3mW zN*D7<2(;C(eqs(tdhb!Hv?db_#I0$WsBv5)d*}?6;92ns+KzeC7>BK252O9Bi}prh z))=pTC8#dkQ;)sB%A{s@$?z;wVY;uqJf<7DP=veu|FVLQ5{`ZS`gKgq0I!ZzOoy!58})5-&e3vFSHH zF@CT*q5AWSl4KKu*!1?D{eR*KF+puDtB!VmoBlAjPyDt+m8mH0uYMH6#LN}B+;Gbm zF()ZMSfsTxTp01UqKO?4js+(^zAV^*mOJPdG^gc7Y0QFS_ZBZ|apRI~L(O~zp@_&rh;2N*T4(OErzRf@433GD>m zxAI=kE7RKpI73WG5UnQnEX^B{^nA=tK5trH+5Ik9`CK@sN~F{hc>8(QYh+L(52Jig zBTT%oBY7`{Cyx1Y>Iq}Kb3#9<^JPw{mhe`u-R`GIc(f8Y;=B7KY6Chy;0;Q-gp~dd zUwGAc!tKYdX!UPOpCY0;L}(Y`CO((zBxN86E$1yLYZU%`m~e#LY(Mp_sn>$BPG1 z45@@a>aI>k8O^EC82ebj_h=RR$$x4dG>`0-ErVe7wkKuhNSX-Xb7714e7lPXN1{9@2;ep1zKXLlK!n1C}r zO(gg}!fHMrSb5W-CdI31YqQJfTJ?GU2Lr;Adv_DJS zS?Kr9H~#tpwT;V#WUbfdJUa&RRHR1*39!2v`RLKS)s=@quPc<4l>84e_X4m6jPiHy z-gz#TPebZD@B@lV#;wm(Y!(Yh2S#s|&HLBqA#Z4Fr^S0@0t6Nv9bJH~s;ZjLMR|XI zvIk6Bpq_#2q-0uN9zP5(DTvyQYym6tjiw+E_ZTf)N7s!{Pm=%K7ey~7%3@AvNV9Lg z&_i9ea+ zyPscghVhf&*MZML7;_%|SO>@^}z(l_wHlykxRD zYW}I!&D2GRV+%v&T|+eh^Z+STsc%1D)m>qU7ep8~i;9HM^_G(Iy;dUhcOXc{l@mb| zBk^!s$WYke5h^gEI^BNq)2+sTpQrx*hA@tC@kQU}7VsY?r>8w1O;GWi=zDcz7^$$K zd_ZUq70{iwD7yH9=s4B#IudyQ@1#6?g|*VO?tXi7UXL(V>)#`Y&*klzglCoBl(QtBO*AM-2i_tf4ikQ7a5TA zU&Bxjk0L`r={2YG0n3N%i zV}ra~!(gQr+}9;AJO9lC7md}Ddi2TQiaBabQ0KU~!dtM*96qJIxYBFG|Cy5tF-D=w z9j9BxMn85|{;Na9V6wc&d(Ias$4*H)3Xo!Y!PovXpAH47Uw^mYKM(GLot`76kZ7+x z)Zo25>^hn;f$dG*4r#-~z}`Q2JsS_6YjSduS=7c2$h6RAFaQ=Yqo~FAlgUwNwtcYd znJfL|caC0+ofzehPN}x-b3PX}mgVwm-dGweUK{j_0Gm>+V=mgetDTtlP*?%Ol|k zOokOZRl8yhBPx#b8QKWD%=aqmf%2O&&18T4h#4%+ZAcqH^F2kHJ?uuF3nD#NetQ$T zOq*R}`SHq35bD7r=GETX*=nFx0@9*9n(*m(l-C94AQ(S^*%r2S-AWg@#D67Pkca3q zF$-Jd!N?*Yy;CzC$-okTL6|7LC(~cheEwGre-)%d5B{5=O8ljAu(ry@3Z>`Q*sdY-oBr)x05B~Psem{%;E)ETr zx#aD8qa*FtK>7Ymaivzdfz?BaJp&BM8*B@>6Tm|kLv1JnALRhdKlbyK4ix}bBGGjP zhJ}_2%$jY1s|I{SGgH%)4iH#m~N;KXKCk}$Ga=AmAX3u!@@MQ^{v3TiqEFHxA%6tH(scJOgTpQ^|D~r zcSX(<5ZTXF(J6arrMKH&Z`tJb9~qYvVjI!k)J?IKJC3N`{eJQ%`|Oa1y#UkMX@0c8 zg%p3b(h~>u#Nn#$cOvu%tMMZTag6*~*Ws@1K1f$yxHn%|k&kjolZp)>3Iu@>(v{fk zx-YVGCrLh}?GvM{PNsC_kY)inm{+$T1wy=4!A0v{?$x^@*+&sjsu+BlwvktHdU`sv zZ99=}(CUxfa$b~gVnAD0@6Q!x(UnM(aJfDa2iu#QvobR~37J-UX4y!~Y^l`%_5gDs8{ty}!@}AYNWjk}SOmF5N;#S4gnnqj zIn)z|Bl2*%oi(Q3{Vq);44r+YdO?M zn5n180cZsud}TIQ6G$H^n zj#vNj)>PjkMB153+rV2+a|s% zK<+l5E3Blel0HHhKael87stv;$qqSos28GJuyj^|_h;M^ag?uggbh9T6uF9W66|tEHC=P{aG*y8n58 z6*zk8k64DC)TVzuvqGv|=%Fk=KtX?gKnW9?DHn)vD1Y=SCRd-s8Kyx1`NZ!Q zf>|jO27p(K=Zp`^asL)bEo7z6+9;Ubo@f31Q78XrTCmgm9lDlD-S%&oA2K~L5s_kO z(YTJdn_F6DSl&B+U zAS0jp{9d|+EwlPoqRlSbj35!({9q{6pY0_0N;htp(l}v946u7Y3>;mMN|=BS!8tWu z=CeFN0utcCrhWd)5X0=W5E??+uw7}Pq~){qt$ROeH&yg8pT=uzYBp`ohc3Tn#Szy> zW2qY6{0b%VLAjEi%dctF2<<{iM^6N7`m+Z|5n6$efA+wkmleCwOygm+mp&nSJzixc z3E~~NcE*6KTkMQ@_m2A%39|5G$BqFI(%9avP4oI0 zGkdmnQ<9hkMmyngazgHwh2OFGkaE8PyjQ&>CRHCEkbi*<4BvK zW2dV9tz>g|4$1}p=dL>tRw*^YDplMIy5=d}Bxc=<0j{)ADGHjRo$)a-R@?8YnBGN3 z;=dj@dRxD30W7e&qD5%+wJbzLM8ME#1fDCW?2AQ_AxF^2C$tYxB5)l~Kd}}7>qOT_ zH|taJ%fm3!(o-o7wNZ!UBP07TIBPWwhxz?qHYVV{Vj3EbT`IiRF|vBy`)_(GCGA5Z zjJ{M5acg-i0PZ?k#;3*!#*dT!!q$4-<_TUO^x!&^#Q5=idPUdtBwQD5m?qPBPw~N! zkEHR{tp;!Bo(5&H;S|?j>HC!cQn+8__hSA1!2IrS^gkjcv?&gc|Mr@Y5cPX*o0<7= z>ZDj;H_o7J`UR_N=b;B(C!E<9KtiLSd{lK^!44E>6_q|1>>WCRz#+dCCrOh6)OUQ@9R^}v)&u{` z^O2FC3DaJ&J;Mu7jZ@eLYI!Lf?75P9kOtR3e+iE;VAuSvbr?1y#Z^0u%_5sh@ZQX@ z%Kxn?d@@Q(&~SqB2x}%CF%ZUa=-G!mM;d+j^5qLO0g?9vz)ZE8-9kVEo6pdtiwI(B z$f;UFRw|c8Nf~$SFoB(C#?}?GR(qwQIU%cb`U?--(%TKE^HGqp#6M=nFms1hC8J3o zZ@-Ei2Gh*4yl8R<;@K%oNPw|I;`+nFzb?ctfH>cmD^~MM?MT_F9l%Dyw_r~hhn@)k z%*OyUo8@KK${jQzNi#w1YO$(i?Nu+lL?{t3#-(42nP;0WExjp|4|6?A_-f!f%z+Fx z|BvZMTqVrH!FVFLzctEwZvjZ2a7X_OAw_z+kWCH(c3e4^nbq>toEbIb7Yscb(tyMG_(SGaW)$?LnW+IN}_QlAg84~hd<2&KTd zZKOk{@q8|F-h7(}W6G=ilQcfRo^E9j`rEzFvYTzeC2YS* zNXVsx5xo?6^#sh&h=n5KaQDD3Lw~-<_6zjPVdh47BbH|{cB%_dP0*a?yIm_=7(U(n zLf;p};YGlo(yoHtZBGaI*^6tN)bpU?5xhyEz-X@i8? zAL+E>7fM3r#_n@|?VSxrf7mx4~^*TE{C#Y~=IaYGHFQ3&!-evMUup?P9>ovPVuujb*cet$JM-#6V z0b!Y1_Bx&;aJyAb9$tV0pfAkZ^31DhXml4@s#{_J?6u60x!pI=6bAjhZQv<>JG(WA zA`Zd~%j&AW{pYg>SS5sIWRS5>pRI8=vy_idFUdpoZi-HgGPKg03uWQy-=?&6DH|M? z_o5u4o84rgRD9M5qSv4ilRRH?E6T=)Iaz1NTH{gU!@pBs5lDRj1{|X6U@9~Iy3aZM zEpS+42F>VGr&I<$p5w=lx(q((bO2T%GtcFwM@n84tz9ZI+0MYf;s(#Ig%oJvR%${W zLA({$rhKMegBHDVVYJz1FbDFno=Ws;biHlltO2}JhFt8_Io;}(mYa;_79FRPJ37Kx zA0&4DQI~+;Y5t2B7pXA5c?yhQWRpR-hh&8DFAzw2rXd_$pP84%3(+bKk*#S zZog-*5g-cU44Hc`l8u%yC?GxvwnZF0R9l!`*SEt+IO*U+0 zsE&3p&8(ST11H&-2hc0Le?_hDj^oQ1K0`1^1v{WU6%7H0>w}`7sn6MpeupFE?ME4z zo#(SlrD1vlp@)ye&OkN-_zTveDMa+Uans-Uby{!76+CdcZNrnPbysQ#lk)pQi*y{2 zU~$IHOG}J8GSF|tw;~Y10jg|v@xHJ)s1YSiS)DF8lNw5&KYH`5``l_|J(XM!M+_7_ zy`^NTn07faeZ%7(@HQjs*u)!MBCz4HPiz^vr-U24K@LT}x4;<();i5KftX$zwVC-&QMv>#%*l zWkkXmwZ@8R8{Dh&jy6e6OKWZ%1pqS>RLu}DwEXf@c=IIVjg?6&DK@V)wXRRC0DvK{ z+bW?J>{#3Vh%3Skc_Gs^M~o|?NHb2QmCf0k+`c4uaAbDQM$%)63rRTf10nl55WlJR zA9*1L5XyzyfcoCy2f#&YtMWZ(ZKsa=K-BsiQDFANhK^1*#riG~O)k~&$wu;{fE5&N ziKOM)2F?*Nbncl!MDX)nV6j`dq|IEI4go|FUT39SU>dl2AgzU|wRAl^Ae|mYs7Ige zKb`~SYy3RurXjrk3%V5uPthcW7@cnXXnHl}dN>qVoIzdy<>$bN-zR0H0o&lpmky4( zd1x8eXB5SlcFEJqDwD}FgcuT8)!+2nZ)&b?+U)iY#d6gk9Tu_n%uwHE_dXx z5s2gcbF=18{bx>gil|QMSWTrd3R}F|{49ZY1@6`p|L9ckb&o>^%L11cW@~tfC6}ozFR|Hf^MqsG9|CYzF>Y|K{QM?kR7;x+XClY3L?jD&0(EHp1RZ}e^GfeI_FCV8$6hZXqe=E$_jK#)kN)tBmIB>` z@gHk7pBuMU>peFoBJW6I`pm5a z@4nn*DfeI~=V&Ofnsdm?!bhAW*sYIV1GdsT*12%u6yGQ1%v z74SvS-zB4#-K}BS&30FRB9cWOvRVm2*c|Pfr4JHw10SRsQFtd%L`f+JB=B(DE?@v46nay#53F$-NEk)toBvZ)#7W zULIr;MpA3wNg()BVm=9Yg;AqDl8a?mDb0V~ai5F;ns(v=Q64O3-S!W<%#`|olyV%&=N;e0jT{Hu8H@)g zrRw?`wJlgFHUe~{Vxhq9%58I}>vqMU@13{p<88X=y+|EV1gD(?)IfAVcO4+vIJVI_NT`)x&qC(Pc|3=A2NlZE_A$OBi2St3gsZ zEv?9b(@21~e!EdUZGviSYSM)_!68jEHklfLpoWI-SVHq|9H=gL4?yW#B7FaUPQ9(@ z>`c8`enx05N+y6naU>waXOV@#?O#345N$#NH@biSenk5>y3Vr$6Hz<2X5|}~nGW>4 zm6eEx^JU89xH6Vcbr;>f=Rl2Cgyt^!Jm|w-8t8i6#+8HqTvowihbv3rO-q8=+d&Mg zHCro6O=Ixx7OUh5e~^_5+fm+Qg!!JVQeb{K5@54XvGW=5RV_WSdOgF-`2SS*<^eUW z?HlkWQW2E~N`-_*l%h}>%FsM&KogoLq|%56N<|}4sYKDVo1|#dNJ*mxl};m?CtGvh zwK~pw-t(T{_x=C%AGOzB>v^8_JokOw*LB@ryJt_Gm^eWOE`v%N&_nF7M{?Y7KJ+VY zHAK8U@g5b(q+fIgC1KwKr2^tFK|Vr7b@>2UcK<2Mv#k45weFu;uA)xUDSFxc>2?~4 z!>F5s`8i=4HF+;em|Wv_Z4tzdP-;(44+0BWglXb8^#C0Mgm@AHwqcZqw$;rFDq-zc zad5KcD8c-{dNr4<4#OfOCQZD0;)2xwbupY7fr-q)cc0z}^=Gl+80n{gyf`sccw(!) zs;F*;oQ+Bj*{}`!tJpZUJc)fQvo{}}JehiPxv4B(L87vz`1u?7vA)kg5T(977BohO z-$a2!5HJ%si#xo|AXIP3GJ6>R zOLf~<#rKg(c5S=UC6lceVFa>qKvPRA8*3UkXWb-6nY$nrk`yIS_fDss`7!p!HGaVW z61em$Bt0_;-LuVLEjcfCeVUz{sQzqpc`){%%lOLG`F97|>zCh}-|~^T*<_-;__(j( z#cfVHK-dI~-6_gPePJn_oP&}EPb`GC!>un>b6PRM0^06K{1q2HU7dJ%)j_)Y~imLMmm-s>ql>5j4X>YjUbGHf%`N*}h8Ci%BsD@s>)aD@L-2Ql=jBjeO~ z{EhS*3@g}{McOm)?0m_&Y&{E0NXnmHk5`_%ZWosj@2KWI829|?NOIc9q|z_L zMM0iCnci4yyCVyho+>J9=dlHmB@AAf{jj8{i$Os3r;p0n*UtB}v3XhCnh)vYqpsT0 zXwzxQ;`Pg3g5E8FSEs90m-O}R1~xavTk}sKWg(aISk*nDA4tSD>j6iJ;SDRGP+U5L~Cq*ES<~%@hz06s|)Sb=$5Q&XklK^N%hoJ->Unhk^=d;T(z-)&U}hg^$9C zY7dkFwl6bYU{*CQ@3tK;hmjn%Fn{fhWKSIGlO3Qdj){E0-mM4ezQA{lo6H?H#&Yz3 zFiu|o!yv{t{QHY#DMuF96!s>2USwFkTJOv@2&L^P^SRAig~`pTaQb_LMZ@BU^cA~o z1OxEzx3Y}JAV{P*T)fzaqX_1QK!1ZbNo4{x2R-u&^kWY*U^;WcvIXe~$2sRtAY4x( zP*@coe)ZUa>k^tcN>Jv}n#E-piCH*tPEU3Tr& z0sj77TyV(Q7%MRPR5kX~0(4D|j*hW1?v}YT+Sqx);7{2?RXh^OdFP;`!3MSEx;`>Y zZk3vgACDR`Zl;+RpNddgR1!V+IuDr=rlIPd)(+*UE0AZA)tikz%pvR~@-@Uiuxt9B z{Tcfd_8F)hw5*3lfhS`dyn?j`Ga-~K0SLLH6a-PmB%)jfk?L64*oO}lMNCahSXfwg zrllLa9L6ybMdP_+aW#kN$A|lZ_3PTYySmJi*@bo0;K30co%;K3KNg6r*@+Rs=wL*M z(fLS>PRK~?2Uso;&zk=o)p5dL3zdA6gxy9Z9^~cA8OgH3@2?OK`d*Hgu8_Z8cB1Aq zjq{C#jc%2ei%M&GWey^{tBS+BYkc2>c{bdqnU}q1>AOA>--YWF-*txguFNLcP@IaR z1pL%z4^u8rkiSg=YJ8g0g)7Mbvx2F;&@_}d(eAwCXplWp(*9!^W;BMF(abMPUp!hP zd(+OPUiS3q)9Ghki2-}K+@>J|A46;Ac&6F#3sS@eb%)xE+@81$p`Ls8MqyT^E&A#1 z$1CiY=IY$y$MrM)km=kKAr;Zgn2m~z%zncz2zmrU(VDW;v^Q8yD4D|L9@o~@6cV7^ z)F+Y;c}GAEZ`W{`*XX*pvm4wuPlfH97q9;QqS(xye)H4@Ong}S%r}5Gh1uC!!YfC2 zRgx(%z1yid8K>b#x||1> zJ3iQ{Hu00|<+{8O4q?@oc*$gXCKfhs&cNM>L5P2i3vY@eVBju;6|6^RlW@pqQ_RzE zAE=W)qdd-1@M?gt_3`wCPG@poU}A~F0-NwnsYHxm?5NG6bt~h(Ho3|SXtT&$nU=Fr zXe10N^r6k^37)jP>Q6jrN4FdxFGAJ{UFhe4TG`Fvp?Tf zHQ5yjQU|`r7AwI;$~|LTp5ylsjG*$-S(Xn!0ANmzPUN! z#Nr(2&;E*icTwfZCSZ?q2z)^Tf+=OG^c7?#;DSBb@Im>3(gY=9cB{w^sx9-ZxF9@nHw=>;cna9BFud;QV;&j_)Rv zpoNWt8pO0orJKlV>gi1*aY@r9n|RzMykLK{t1SAnz4N869J=?oa3m>l%f0E0;Cq{P zqew+KXd|0y@YT-+H32v^LKy);!OVuT^&grkh(YxSkW@;T*Lcy|pd_b@1RE95K-#_+ zN=NrSZzr4j)y9AZLU$k$b>??FuN=6al^qn)D(UT`#7pP#SZh&u{VWm1kiT^L3#B1n z=iLW?y)+9T=?zZwL&-v|kQmf_4gV~VK}8^HhX6P*0w@66i*Q5dre;rQI{_+d7-IvR zPICKEmx#>6@KJm*yqJz3{$g%XRA;Fl)s2m-*+^kGjB2Q>D5B%7RPyug z+rPVnfud=4a(NQyNZFuP@$2Y5(^_2f$7PvY^KqTX&G*Ep_GcConw!(i@~G4bFuHm;Dy0R!{gI7t4S4Niy?~uZ{bA>e9&E5 zT%*YWR!A^3!y559R|n6JxIF_=CqU+!n3%v}hf*w&&g10M?1axqm6MFEUi6LHnCIYb z2%UKxYUsR$x(EiTWo`V*aAac)ye6V&^6pk7KXeCdycT*zef?9R?f`MjuO}ZA5(=;z zyq)+tdv7_w49zXkoJpq_G56-usC+eS$!f8X0J)dya7fczyuvSAqEBe_1fOHXHZ_lh zX|t=mey~+;3SPO)%ddIyA((X;;{5Z9o&lf#FpTg$)>kYuy{sNSbxBgge;n~dxbVpL zT0;n}5+L3Cu%>b2vilf)ATo8eNHcPLr2DsWkv!--b3WQw_3?B|W%Djd(0D65DeMBz zY`ZjA0wyL~-<+Bn5*mmypF69o`#y4{gCZ*^=>Hr5z`MwTOzD-4g$Ou`B1e_;&N|Cb z(@{79dtlP=R4KfWE){At37p7sC~_TnQyv)#)>Iiz)nmur3*Vk2mH^n)-zg1KcAm3b z9pkWqO)+TP=FvKGbv7?Xpy+|Lu|YIHeHduOwd_fT6q$KfAcFp%NpBAiF0vIgsDi9O zLtQ00wJKun?umWeR6Wk!KnC0!E`7s`3TaFSBQZk28lsI!- zap?j1sG+j+KU625!}@I8C&h|W#%;ghdw@3)&!N&5)M+^H$dEFR3s!M7RycMBHltda zd2Q|Fs$c#cYH@v+_}gkwYjbRu6-*jFR$lC%EiL9?CphH^AwZFnHG7E1v@&{=`Htc zr3bV28kxjyL_E~9 zl9wg*f<6M$iK&Pbt9TjjUFUVTzJau)#6K!U<$BIDrAm(RZ5%PmI}RO+Ua)D)8$+NN z9r}mT12G>?%*-*VW75a=ZwMs>n5B`g?iwm9UVgOKBBZag;gwCRWFxe7A+6;3wWDyAhQF1@9}^WtWo_I|BRLurXJ z8mQMq)Ef)$x*+yQ7DVunlJudU_0Ee)x46R|oh?_QByCPD=E8#?bl$%0-Fsn9;2N&G{D`8Ma5qCHk1v>%F?v3(!=_ zbFbZ3Hyn2gsatmJ@UsMnBBZ0cjS>~nXO6@raj2WB#GJb{>bDo!SXpcGckI}~b(Hhk zq6TrozUB>H=KaJzc-}UBs6b?eX9zFLkEg{liZt740{sygjebuEhVRNQF|m-RyWyb4 zWs{N*Gj9o@)3fbqV#fOtE(OjTy8jF(maN&wrrNK$D@mUTdHw{`tX|S%9=Vh}x9BRH zK_{*QXFX$kBR&hytK$`BMV5py<_R_*P3S3W1CIo_8lMx$40SpxENaMZ83eI=%T$=4% zUfs{CBz;tIb+xMB%-*Z+r}}Xq#7daBO|QWtR5Y73DKdzUcq{@5M`;VJwbw^qU?q=Ow=N2>gv-! zr@yZM=fh#!2nq;D5O|GgF=@XbW^;an(|C1N72ANNnBIk=;Zr}+Tw>HG`z{nbO`EJs zJenLQJzAH0F#rsBDrM{A2&PJ>=4;Set%3}kdwP3o4Px?xXjZL-!r>YA=oE1Wu33rl z>Td8JG`lhoAt1W34BFM9vCz{>?Bt=JYU+ws*T?Y$yT>JC69-*BuJ(?wZid&JgW}>u z9VSc^FrR?8_1D)AwtpHH)e%x8wmrx<8aAmKxp(?-c+8C*x5P$~i+3m)%xBE?Z67{Lz$Rf70;#e*1T_Uc!vGm3-av!+pU0WpQFz4B?wXZG3NCUs)$GG!L$E>negj#^p@zKEY}g*v*>AJ!-n zNVz4}kv*ZV9~3`)?#PiM1T<4+#Ib+U%-o{0UBwJASX8%q@?7BjuCw`ow$_dP675yr z@j>}5-_ocMm=}Fmd87Q&Aaq}QN@w=x3W+t(csE@5!NZr&+FMn2KJ2S7nM~fhkUS>l zmBCR+?)Z7Fu*M-zl0+fz-FE4~8S(e)?n7d{O~;V>QM~&7%P|GKECnS!{)>W7rv>Xb z?t#m+vy{zs=&kP0+wlDLWEDFbfn09ezCCl!uEgU=dnt|t>8pb$%!*+h-|zN!hKc>C zK>~<~9b#GNRwkb#VO4sxe+(r-H$~MHm3Obj!wj8O>mM9&5)CgeDM>l^CTiES*N5FD zlwbW{iSBsSL2`anez;VOtf$PreUbWgCiNRbrKeFlP+wtkNZfOW=k(Bj4@YyLAjYzI zVIZFws%Zs&l>CC#my@pTFZXB)m`>yNv*H?Er+9R$YOzGjM=vN@J|4s0E*_36$;yrZe5PHl&jHhr@|MaV*By)&AVB8Tm1f(Pj z<(V3mQ(r%H^h;M*8<6N$e*jQOCQDPGTygNV zwHy3MqiEMs@FryBi$k6bt%uhZW|CaZ;U*D!F2ST%YdL7-qxr|UPR&Mjm&cTK3%1#M zLy+Rf0EtQ&$8zsiy94KKcXbyG?QhHV&iDPtXSqdby`>eq_6_>765!-1d=z53(y}sz zW5?Ve*>sbV+rPTJdE^IYVQWyarfHJq29{-{72H?aLo(uSM74IB$l3ed)smno%Hv7eq%hfYO(wxLhdfDB3wzlYA(v0ncU$R@g&U=5Le^mi05K~?iMmL z#`YLvS9do6Cw$JHh4@_M&z!59a#V=#BOTFAEtaRKo1N@$KL_*N+i6 zA3R)GK&OrGBM)u!KRDl}KUj~jIAK@bz|33=xDVPYy_Cy$+vs5F6bGsxzsX9uX1d7c zT}Ly!y=qV#wExnMs)dQX4g5kCcip*j2Q4uiL9|t3BP?j-%+<{Dl0JnrM8k@!afpd(Y`%<_DD^C1cBd$goY-M0mAey z^M%Q4%?rqKD>xdZ%Za_|@(x+qjwh#n$qznr{7_L5FCU*{)soqZiZ&3B?R$d)0~50)^#eqI^Oq-|TM2jaTb zj)MfCpx}E?$_0pFk3pu93km~-7rXgMelQWfadGYR)VP^&JeZST?DF0Z)a}~&O=M5x z?SNXDo<4$?hwCrfLb9g`<)g1ybL#=DoeGVm;a|Zdx!h~p(untO*k`+GC;(h%^^k+G z-a>#U--b$8R!QE)D$WG{)e$|wS)Psqh(*DKn)>qVsgoz;iV$KrXwFyRpiJtA325)? z-fwxps;b)eO;^1b+|!swcpOYmV{wP&TMRXqvu9Tbe-IBL$HH!dax!(!x6WpW8#ff5 zEPh4{sXrapGC39#%6jRw@|%#GZQXiBbEJSW0}3WS5iQmhMK~aD|8a1L#1{y7Sc@|X|OgM=bFnDRZR=9p$hY<9NK zEUnf@|5~lyiHa3EH+%6CCfNfca*IopiaPHaD(EVFcF3+RkK89pCH&~4ZvG;7)cMw> zju6vd`tRw_kh5B#x#XXjQlgO|3}zLufKpTAioYJ5JlVQ{V@ePfdQN#dlrI^^GzKeW zsK1cBT;>?usG38JxWm$jtNv@myRfgu*m8Knrc2EQf#RJbitoh0c(@!HFH%*zt@WW+ zOL3#9n3%?`aw1Q#D&H{yn^Qp-SexegesUeOk}#E>ZM>OLNc;c!JxMV89In~4&JVth zFv*niE`aR-Mn7$Uw8c_P<*c>G-^{NV)zZZqI~vy_|*pWCFI`Q~K!A?qQyxhff- z(|zvky<%abcfKhb@?h_a%*@QF4D6nL7(9=gaY5MGAtoWA{Zz-}=now#)}EXi_W1je zk;fH}n7NR*K1#|uKw^dcws+7`0pOXq?F8?W(~{V9QBVTwvN`iYK#h`&t}POcg;t2! zqEa7i`}XIHyrb(?$9C0Dc_n` z>7VRY; zS?W2a7N}8Nx_i<8fJYx`^Ez_J$z+eErWINV4fYP8k7nSy=T z#W;zJ#kdyFX|5wFVoM3@FDC*LIf+w@9K~}6!^R6-aPTRYC*S=!sts{}*zaj8{6>KfmVV3B)Kp!a26-*%AV46E3|=A> zBsnHYojKnz5sOwsy3GFO62)d0mbtCOC93|s1Z!NM{T??_FTd)lIT>i@r7+!joOMBfHp&pGwBg6n9q@WfgcFARw!sCG4P z!*-YbN;2&=yLc_|T1T&YN8~!f)9}>O_UQ?39@%U7`S+@LJZQgXQ{g<14CO)%n@pXY z23`@1bx0d_hRiX~Mvo3Jb)#7?xQ?Uc%`nJd0y^Fd`Vu16wtdgW@6n3s4vy3&th(UA z79xuS_v=~W28@pMc6Nan!@18^Dc~0>C$}8m1?%|h1nS(?(M(w_FUV`L`DUrdOk`al z!v|iRCnKW7$9Bg}*^k}c5%ahdvV6F*GY#C>n7THVHI*~p*)VtIPem-+?7aRwj#*}Z1HKd*(FkPLNzVVfsr`E>zt(YM|+#q!;iqflwG4cyQ) zUvz%A{%7#s-8)J=4%Phkmou{#%U>NR@H)0S_C;FSr}yt`j?C!z*sd(SN4x!uDTSwo z7&dPw4&tKs>HIrQNC`gGWX70~kW2jy0%XxAOMlr`=5fg0PWA7vKbEnEgN9ASjj*Rz zUpwii%h7+M_XLgV!BUSya{u^4CPt2FbLKmd6)^cPfK1G&<%0$ z%KK`?j?2r}n!MnR1CbXU+MY|@;gqOY=Ws&#u+x+4j~k)L+c(WeJ{lxuCj;gEMi zJ(m5xiLA&pp%ohBSg?4v^IO(i)eo`bl;*9ER&?K^J@^10+w+f)IpyV9)NiM1x=2t) z?S#*vcp2;5bsI>xzethND38V46R7hU{$maric4`G2~7ETc2?3MPFZi@o%k4w@+Hfx zBjq1Mkg<-#=MI{%@sNEyYnShwI_b@DWBvqBLvC`FUBEAiejLc#&RsMh&z)Esa860(-qq+~9n{^jxl97UCj3e{El+L6VIDd_M z2eBR(DN74-^DXa>_G75t<3R(1gUy!tF5HI{T6S@|8e$)Re+F{SuA1`}U3*tYG0Ph2 zuemTHwn1(|whrb+>vJ_>`c@KIAM_A)&=*l~&q4)a>WkMbq`FG0QDoXj?#e`gTkU4Y z{IHY4ZSvEeD66#2FVbLuw3!o)LdLWK&214PNWD{+t_L=!H(eM%KIk;QSg@K{*f&nx zV&w37F|2g{{CPYNA?QX0q zO9-uJMZCNQ+6(rHe8t&X>R$ zqJu6{9bv>L8Ee(HY8k;MaMNsf+<3d@&T$SAuuy z50D1XHtff`ri3w2brRrLs*ur9^m?*R(QOF4guS;9+8v+{IEE5$)7;|HhUm4KWo>b( z{;r0~(GRC-iykZnXM@F7zlmrBLhV2VQBqc>aq`-=8B|ZLah4*@M%26;G2pcqg^)>2{z8Me$5O-c!jj{=u5xst73{ebI{GUMxFiU}$WK`Ywy|<> zpe;!jf()$XsO|%fecHLdMgUB>TF&8?0uXr=u8+ZPdDH7Sultok0ZXsDBkL#TU7u*{ z=)lv#tB**^X6(+LJ9q5Ziurv)LIX%aO{_j4iE={#eGZ;k1dJ090A^vN{Z=SpWBdM~ zMvfxM5vht4ljIgRiS|Zk%c`-Bnn>bxzrE$G$%nRbS4xR)JtR8;Ha!U9#&*RWRP{lI z_!Vwaea>YoC@a6e4GIc+I!<(bY71x^6{8Lmst{*@hlkTLuS2<}#_jY&&5O8D{g>L4 zXH)LhQ#?Hbi_1M;;sNd2OFgJr3X&|BE}UCU+Cp0_Ke|qq*8=Cf@wYC5m%wvK|MIb3 zqMwI#xfB{4!8Ehw#R^nCcSk{{DX?dc`oyT07gALEp;*9DReOG+1#DWJD|VEpWw2df z5fU|Jl!nqbY=hMdY`xO-vV0ddNLYrcyTYyI{{8#S&CPh%vEXl67F_vZ(h!29m>h`= z>ESM?UiZN!_o$_XHHU@eYF%EGzy1r^6YX%buM*IQk2kPu%b?&+=&?lHSVod7s0(Pj zOOkzS&pcOLeVo&^BD(Lyt4TX-uT`-sr}gxR)(Dm@$R-K@vZr+JXn{c()C0D(Cd$ML z-N?tSh6~wLvmTGbrIdq%gUbhosT^$R-Ehz9do3E^Cx-LP6ENq`)sJDZ*7_ZMfCLb;x)>nTj>187kH>Q==QB!Q=sXVvP*!B zq+SWYCKzrsS!$)j&Q8E9xcCP%UyW{f4I4NVAuL3949m`SuG88N0e4}|`m2?12!&Qk z-XM1AC$wlRoTpyYm<^W0q!Y+o2-Gd^7JHOUrF+f?G`@kO%pk`y><(9=ry$Huy2~E2 z>q_Nmvf(fFrOjsoTjC&@d-Xe7YH{u|8roZImr%Kmlk#0@;`2?gb(RAS&E`nImy>V; z(a{E#5B;jj?lh07PCv{#>Z4GpG?_%FQQ$YY*9VPlH}ZqIlgMIO-7#j2-c4Hf@7*&F z({%(*tjZV>f=P5SSYf&}oF6w}ZwO51xiZigxZsLWpP7ZmzEQMeViCpeL6rzFJ@=j? zMCI~6Fv|J1x_#UjE9u@}D=A4+VTJVn%~Nr)1nIsL)*~-aY-zZ^$ju5iN+pPCz-pDX zVY2a~C5zy;j$A8_!8k*8BZz|U7d4{U7QBGW;C_n&g=Ew#{*9}DY&t50E*1qum4Tlh zghf+u{pM25tD2(P-!%Sokbp5#?WU>Og1q?<%AE8mTZ8pDDdA4@L{kyu8DJDcw$jG% z5>__9m@*DSoY>IN;=cQj9&Nkc=uRSW$elcPSW7AJ$9`r`aiY;v+aZo;jZ!k`JZ^oI z9%kdp&#(NLAE>C?v+I>7N|4&$#vQPeTcuvGY*(Y@a$QB5Tlaf{!1;^f8%sk4BDzw^ zqgge9wbA#`FPXY6Mc2baL2NHJHv|pjK`a9r$BOG5+4Jy|!xeqFA3d)9D0ub44Rp~^ zNV)8cT~7BVoQH^=JCJJ}9!>S0gbu`yQHp4|DM?BYmv9x4wHA-zcrUxQW_<@DoPodI z1-WPw<{M&M1qK4aVj1>aER96BCmZx>sl6jDqoe5F2ATbc+o5kD4~Y-od43?@@l=xu z->QD?RLO}ajk>yL&ngdWRrz9!#+VA?6FCugeC}8gq=ac3tN15Rw(;>Db$e5c9eFy! zI@}^?@JVSx2tFqa223r@DiGP;km-^yN*jExkxirkPDGlwFdtK(|{Ms11u| zR%J}Td`jM(cJsFzzWiv5;#{}Z$dpA|Aggi#?aeubK(!Q@k5o8ZX5~2S;AMUEfhk#5 zMy4=*q^3El_&QJDgpE;vM-vgx3L|_=$VV~xU z+WQqSA0T>=+Mbej^wE;v^8@zr#I^?qXS_ZzO=gSA)R|D(0owZ3aM*9&TG$VUvb_At z9O0civC_*>*G!envOzX%?P%VTYuBN&+6`_R07B}@0?IrWwis) z@2CR1@rS2dOn=;eneACcDYMfMqQqSpz{*^{yjZ>?b6!1txZUdWw`#t?PZEPGj(y{9 zcbN8Wr%wk4Lo_{Wo*BP6P&G2WV~TJjx%L6H@gF0g!fRgM)(?hQz);G4w7OFEW5=$S z0@_`J5$H$?^ah#>zHW#`V)~pT?)FWUE+9_Sl?vRDd>+6?nbqwvZMxjJk3Stt7V;5P z34N*q@iA1k<1EQBQ2rKJLY-1cz=TndiGPmX-Wz;U9H72E!<$TWsLcrIY!G442D>WJ zLwxJjBInPL93K~fF^f)C+=A6*mjgJstW%e}LUI4(n78uiQKujinB2#2PrfG)b#1IfImb*1gCAjrdb#7aB09#nIc;s^PkN~^Yt@~u3w$L&PaL}B!3I!zW}y;89?99<3khG zi#U{)ybqpZa zNKh+NKxY~FKuA8NWD>8Q?YMDcd2Uh}w^OsAEXu|bT>St=)>};@=os$>04u^a{3Rz~ zSN7iHq1JX97lIQ_zt%1MWc>@rbRQwf$z?dmGIEwOWR(6ja?P@69uw6>S)8Rz_$)<5 z6^M1TPoJL99{gqXvi<5q2bP?zkOz*r%Jc`2?BnaHnQU z%}@C4Ykr18(ljii428>1{dwQ^+Q4g;p#`}IVY-zv2I5b5cX#p=oK@I&tE;P>-h@ZF zi4#VBJW=C#a#5YU-FjB-Cc;+&=fwuhEn2S?6?$KVO=$pIkXgUN@#C69u=mrYH|WKz ze>VU2HqCFO^=0+WxRp;s0D?QDdR(yh4VfyIzfPPV5NH;Uu_Ul*Af8@wao6KpUmk4k zR=)D9FQ-3?V`$Jgvg7QxpM);ENj&~(6*>{^GOO;Y-4y$bgOwHM#8bj#%#M;sA)M-w za=m)fcP}EM>|E~;W>VQtm(FW9ckJyT7~&3OGn~8tCQ_4<9((?<%T);b=EEzHajBE% zW7>D`$_Q_Tm2OhAj86xA$HiUCX@Rd4a6M_=!YNm?=QdM1Wg%Z@V(rcpa_f%Rz>VU* zj&9q*Sh_=nH7P22=LLM?&cz&w9ODvHtg<#*{c$mZ*P}fOu0VV zeYiMORpw+)ZEbB@N65r^#I6VrB`$jE;e7?&;=?}4MnpcTZm)!{R(_qDaxScKezgO! zc2{c?_n#vXm^AmQnA9JZC=|Qh<>p!P`IkoboT!6(oGga`p#j@ARXxPL$Hcq&ME(PQ z!cp^LuO+25XZzr*oHX+2i}XU1XxCH}ko1nbw(1!D2+8~)SU_8eFwcD0H}^l~fh!*k zzEEOr9O_OcwjGgSZW0|U=T6$5dGjmGV%0k(I1Aj5i11Ah>OSzRt;ilL`SYbR+x4a8 ztt7XD>w!3+;a6p&fC_Xz*iU%EfdrQRdb|RM3IDcj;EY}X$N=Y$@$8P45nh472{T=u zf=x$`r|TN~v(GNfj;JasqLbSeZjZ^v$BFTwV%!l&6my>fHAgM%NVpL;yM~apriC@l zYB49i-fnTF1~Ha+Ji|dy%YG*yV7(CO2GfCF#f48T+w3xqXTg=SZRl6dd7{VHpus?& z>gG^u9igj%(o#~d8iW0n8>4DilNx{4N|YTn;Ww34)jdA)GSx$>CTh{^QM|Cw9=!`& zbYVE(07yKOJaGP>yEEKB6JM>S-g&04$aU&vrW0wiS979#)5AxfB^u>V$RvbGP$#|h zr_IVM4&Kpu8c1&!U$8G-JNAwFGhyi$0NxcYJN2!6G04qzYhGf-bD>qEQdcX2@7!UH zFG9ki=lzGK|H-d7#`l-XazzKkfuj2+>tpOL!Il&qIR06=r``UbABy_QbE0+J)RAtU z!Kl>cT`8b)MQoD3^1gnktL3P}wB2rSIhlk(&Xril{N*33JJNmZu&&|Fzi(e{x%2H? zj?7VQ;->=3Y-81g7Jtw4GOj`S0~ZKq0sX4#j(3v8iu7c>KGVhSRlV_<8z&POB2P=M zf11apl~{z+Xtcv!>;u%F4A?y3zy#k;tl0oNWzd2?Ptt2f3J;VX4-WoPHs*SWgR`b2-NR)%>P0G zDXpb{-}vtmBqBtSp^m?6K?NQX#ZUgK%d9v>@?^4f%M_TnqtVzrwBAzW&;gx3<7_dT zx3sRcDu>NJJu?2*2EI@e4U2V#6Z)Jsd9`NCy*GL`2Sg-m;KOTRJ)P{y+g*-2xp9py zsc?2xC)E1Zh8$`tNktl0W+~nZ+@iVq`Ptf(z>9rW%Jgk{y(!lvix7i+T#yW&4;k1= zu~Dvx|LWWw*cHI-t$E{v;KaKSNh1vnmT>18mzckbPfMHEFQhbPzb@>CHDN^q3R-8c z;W#9%9HmyL1lZ5=3paJ_ArrG~^`nI>wk{#uf=04A;OZ&kj!oykubD05vb!bnmi^C} zcHX(#dkxSo>$w?>-frxN2 zTAsCHcmp)Srv53TO+LaV#*Jd$5j8{6U(cFS5*@=@a{J6PeIm86HQWyN?o1=I%?*EU(*>ZqPzGkNt9QlCqN0dHUGk`M#VGu2Yd; z50IGTQo0%57su#vY;kG&pJ}!ps7nkWYbm9!%cI{&jgQtq$-;sSh`$?wGQP}MxvS-X zB-5;w5!XEm4;|AJrL8Nxq^{Dm5M8H7th0t`{wr%K+Q_H|4tfVqc>La*ltEy2GFJT~g?N+#Xv}!D23#X{R zqhod1XOEYrB-iR736~A2iISVE&t>1VHyS~h#<4P2b=4?as3y_gRF`E`=;&a!?T?r9 zw&kR+vRJy;3$Sz_y)j?a`SvRr9bn5qs4Xv_xzqWyeN@~F1jRsE)(7P=1t8@(`D>d5m326T)$K9T_>-JXWoDPvfQ)HwbkZjD$MZEWe=;r z1C*PRr^H_~S(~dJKYD=kkJ{bAw64|-&l(~RG0zqU{^vG!RZg`g(*+BeJgI}!8jf%cj|Eo-Em(CO7KS?BpzaNDt9g#?^Me%1* z?ed(rWFYwkF_d^L{jb*_k{#&Ze(*3*HsXiQgz(D<6nagPsAb;Dv-c^{p=#;p2>Vhu zc#>D0EjrsO*Teo-RL6V0^$Jq!@4pH2EqP(eM{-^UtO!4H7AIaygDk`G|MF{fkno>R zJU9zFk5&R|3ZKz;5)skE`}a3+bGL(Pc8~A4ZEJ4eL!}|G)uY10KmRZG0$vSk)X7f+ zXaVAaJk)m`Z%jl4`#=ryKOn?Jr@JGH8ag^ubf|?z2daP~ zvzSS5SNwjmcuVEi;AL&xxX~nsz!t1jKECPi?}sSkv>Tbw$4o_xPvoPoSPA=Dp!ubR z?z2-7fbO6jVrrE_AE7MC9NQJ7-F`%8dz7sAhK{_1EnCoYIB*?*H;Z&OEFR_;htSP~ z{SMg0e1&gIp+nQ_hPN&D%*coc@Pw{He@RHGA;|5}@Hhv)czjbKtbqwvaZnjl9_>Gz z1E@CcBAUB&Z%?M6U6@vLk~-f^H`Y0z$A-MekBy+rMp>rEAPUp-_wA_O{sjXO#8X;1 zaJwvgou8i%7Ms@BSC`ES+t~PrF!GjCW}h~Z)3%R%15Eb=fx9JEy-LC;)awZ$I367s z+b6P_Nso1iCTI9LH4)|<>rT}Zq@Y*?v8&JX8#itk?s{kS1=xv*))*LHr)&sADCw2g ze@$%loT-mZ+E~@FQ!m?`IUIEtm~>xaY#;sKOTytWiLg{XsYJ z77;#ar3@T8-*SLIBECRyn-6;J;L&BpNTcsugl;xCIQUr+INO9$SXOos!>%AwU9Uj$ z(qH1Vkw`nMG`qNAx?0vgqjEPa>{t#^n<(9Ps3on*d9dFhUbnNT?llMvKa{3LmoydW zBErJxeq^Mj6;``hM8sySs}xy%&qJ3lbR;#Q->*`l%c<8o>2C{`JB! z5^E4Uc^&PV4Ub>79f@6v4BLO&z3OEbM8CiFhQk$kp`HY(;B@HRYaFlgi`;w7nL6&k z)h)(d8UlxB7GXpbSns!e9V~!xm;pG1c6j+nWKJ4sS&h4b8jXNgVvEmE^wTzXW z9kF28%0ZMs3dshjA6yLlGnym%Z2osRH!8ya4JN{tGj@>GoICdwNiZns_J6#+j%pnI zqP%i?mOL&AXE%fn@Et=`9u^USo)HhLkc0VC-o$=TAdxQrwIm5;eoJ*bBjnmC&-3!8 z5YF_xSOWklU25YmBFY+}o~1uM$p1|nyk#)i$v#gVhud7>wWwd!teVBtdX`|2Z>YLb zA4nqg{ocK_UVk{oIxyv@}jVah=` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vBucket - - - - - - - - - - - bucket - - - - - - - - - - - - Actor - - - - - - - - - - - - - - - Buffer data - - - - - - - - - - - - Buffer data -to PFS file - - - - - - - - - - - user data - - - - - - - - - - - vBucket Manager -<<controller>> - - - - - - - - - - - buffer - - - - - - - - - - - Buffer Pool -<<controller>> - - - - - - - - - - - trait - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <<extend>> - - - - - - - - - - - acquire - - - - - - - - - - - - - - (un)link - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - release - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - put, get - - - - - - - - - - - - - - - - - - - - - acquire - - - - - - - - - - - edit - - - - - - - - - - - release - - - - - - - - <?xml version="1.0" encoding="UTF-8" standalone="no"?> -<!-- Created with Inkscape (http://www.inkscape.org/) --> -<svg - xmlns:dc="http://purl.org/dc/elements/1.1/" - xmlns:cc="http://web.resource.org/cc/" - xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" - xmlns:svg="http://www.w3.org/2000/svg" - xmlns="http://www.w3.org/2000/svg" - xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" - xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" - width="41" - height="68.997391" - id="svg2" - sodipodi:version="0.32" - inkscape:version="0.45.1" - sodipodi:docbase="C:\Daten\alberts\projects\yfx" - sodipodi:docname="uml_actor.svg" - inkscape:output_extension="org.inkscape.output.svg.inkscape" - version="1.0"> - <defs - id="defs4" /> - <sodipodi:namedview - id="base" - pagecolor="#ffffff" - bordercolor="#666666" - borderopacity="1.0" - inkscape:pageopacity="0.0" - inkscape:pageshadow="2" - inkscape:zoom="2.934351" - inkscape:cx="144.21983" - inkscape:cy="28.533711" - inkscape:document-units="px" - inkscape:current-layer="layer1" - showgrid="true" - inkscape:window-width="1280" - inkscape:window-height="968" - inkscape:window-x="-4" - inkscape:window-y="-4" - width="48px" - height="48px" - showborder="false" - inkscape:showpageshadow="false" /> - <metadata - id="metadata7"> - <rdf:RDF> - <cc:Work - rdf:about=""> - <dc:format>image/svg+xml</dc:format> - <dc:type - rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> - </cc:Work> - </rdf:RDF> - </metadata> - <g - inkscape:label="Ebene 1" - inkscape:groupmode="layer" - id="layer1" - transform="translate(-29.5,-42.959476)"> - <a - id="a3142" - transform="matrix(1.0873906,0,0,1,-4.4741999,0)"> - <path - transform="translate(11.586889,5.2908993)" - d="M 47.02914 47.36993 A 8.5197716 9.2013531 0 1 1 29.989597,47.36993 A 8.5197716 9.2013531 0 1 1 47.02914 47.36993 z" - sodipodi:ry="9.2013531" - sodipodi:rx="8.5197716" - sodipodi:cy="47.36993" - sodipodi:cx="38.509369" - id="path2160" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - sodipodi:type="arc" /> - </a> - <path - sodipodi:type="arc" - style="fill:none" - id="path3134" - sodipodi:cx="43.962021" - sodipodi:cy="48.392303" - sodipodi:rx="3.7486994" - sodipodi:ry="0" - d="M 47.71072 48.392303 A 3.7486994 0 0 1 1 40.213321,48.392303 A 3.7486994 0 0 1 1 47.71072 48.392303 z" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1.24319649px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - d="M 50,61.33709 C 50,91.363211 50,92.247838 50,92.247838" - id="path3136" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - d="M 69.760668,72.362183 C 69.760668,72.362183 69.760668,72.362183 50.239332,72.362183 C 30.239332,72.362183 30.239332,72.362183 30.239332,72.362183 L 30.239332,72.362183" - id="path3138" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - d="M 30,111.45687 C 30,111.45687 30,111.45687 50,92.013532 C 70,111.45687 70,111.45687 70,111.45687" - id="path3140" /> - </g> -</svg> - - - - diff --git a/doc/design/img/user-core.png b/doc/design/img/user-core.png deleted file mode 100644 index 4fbd94ce8d1a2c003dbde7b6bb5b16cacf57317d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60171 zcmbTeWn5Kz*EPHW6$KGNLO_H=BS=Yi2ncMtq`O5xQMyz_N|cRK(p}P_q=ZO!N=Qf} zjex{6H+sCT_kQj#@53j~`E56A{nv~!#~5=3swm0g;gI7X5C}ZEJGa#kh%=@L1ZMJi zO!z;9HmfoSgfBwwwuFYO;mWw1zQ)e6*gD1;3{?qgX=#EE`WF3e7Ls&^ITGz#4QUZ$ zn50O3@8^1Q+78LP#0~wD?PE;peL0(W3XZaJ^pTEOJVY2$XE8>6*5)3HOwL$6H+ilT zC9vgwJcZRY?{&$lbHBZPzG=k#4g78Rp{AK({QX-LUnAY=Z@%-?3>c@s31f3!IQ`|D zzzezGpX!o8qOTf(aJ^(oc=}7-!^^1CUl8$6qt2ZEwtnsZ^{cpF%E`XGwx24f@zkAI zB|k=Exv3AXC6$qumR3aMs6G0YZ2bQ5uQk5lT%%gUZU23~7#g-jcaw>&`L4cR92}fI zBP`>~I`}5L`E(r}*M|$uer+$# zeE)(4;S2absgcJKyjoE9SaVfAY=3E!mf^c>r?DC zdo=v~QQ>kkpF{T#H$vj#3@RV59{gMrGpRpwAfs1o*)i7|Mq$DKdqIb{7 z$IF9x3XyC*`CrTIzkYnQY)#3a7{l$l{;e}ksCK;`_i#y6O2TfTJL%zEdthK-(q6!~ zGfRD}3??PmDp;;F5X7%5+_vWM#SS?qF2W~2fx-PEse{8r}|O3vdh%5_K?^bu|nq9gxqq3L9e z7g6%wzM{UR<=j+5KuU75q@?5lJnY%oMtckwm!AZ*GP#D;2aSPOq@|?~4~;)V7;j&4 zxCW7uioQr87o}L0fE!XFy|Pno) z?b@|xt0RdLPs=C0J&q1`Y;0@}g{ci7Y}^C*E+G&LgATPvWsXzzk5@-8NV!da|IXv` zV{C8FRo^V7y0^D?ra8>#U@@cG<0r`^eq5P=z1!&V;YDh6VQ>!+TEs&5D$y}Ac0qz3 z@@rdJ&G)3pHa9kYOSeDRf}EffdK^N**EuktIlNgyYHiegeJV%d#_z%rqU-MN?&|9L zYj@S3^J8V@FUayMSFW(!xUpy3O))xJ(qYrqw@bQvc9nEgs3dI9mc!|HHTt};*p~sp zy8N{~pM%C>w8X~RdLc_aV^58Pe-yWb-8$^NN5C%8D*LFNCvz0T@0v6d9uq@ONJt|* z_vOJrk>Ce!bkKHo{3i4LL`9sC1c&-#$Hkij7Zl z{c(SaY#4-qsN3eVd@Ne2UoX&qqin_+zt=mpx@E|3vHMR_?EQk8kg27nR%Fx1FttSM z;qmL+w{H;SAyjqh8B^?8%BNv9l(S3hCL zobg590lIob36Nitm5~|DH%h3v1ljhMRYy%lrH<5ZB(YlA%hw%q2YZRQpYL*1=w(wv z--)SEEQE0lZ0_S9VvwdY+A;K$nlJXtR{I}Z+_|@4M5evaVRWnr@9OFb3JLvO{ptdh-onD-@Z!Nn zD@%TN{zRb}_N1|6{W-`7rN@tXCs|1;C|K^jU#t1`(%~F*X*?0Hg9s6lz#5<0;{&@I zGI9HUpG?x0S$wTz?|&B?Ek4<1s;?6gsJwCn`T4I2I=^r@*L``1W=WwxCDTZCZ8}!sB=rhv!gEW}MBcwNv<^zX^!EePdz@KvD28o^e8cL!xz<*=nzqlM)1iIr?(V*jJLhW$=pr># zrjYwmQoaT$Y2@szc@0^jOeVW(m}!bW9eo1?DA~NVGAF}@bNJVhia&n*@O*mxsveSD z;DI@`Ug+AreSIIE>?{wdOBfp)H$2B9KwYfkKdFIJ5LDIGZT(Eaom}lWg*kaK@$8Z+ zE3+z9O8jTOr=m6_s1Q_86cHI2Gm3?odFrck$FQlIxHzIR$Qio3zF&SmyJA>am@Ja( zBKpx?h~=^fLVC|QnA3G*FKMsHO!fEe@pdzsg~*YSk!fjZ^;2{vh;@}~YLO5T^+TNN zo@^zY6fh5brt|RdxbF_=!1`dM*y_U~{BQ|0(Z)8vq)aiIotiR%*0QoP2E`XnZ=%vg znMs8lbzRJRtk%aT=D1gZef;Aig)!&tg&x%unGcnf#aok}o}Pp64OnXEFQt)|alM18 z9!Qe%>@q4Gg<8aBXJ>a>Uj14QXA{G1qo$>meidcyUw(gF(6sRdq|p#+dAM-qoxnq` zTeqa!_SmQB`?wh4p&16}-@?aXCsBxVy7WO3- z&adZ#@E3C)dvMhY8UgNjiZ5zCQba!h-yg%$cN2Y7S-OR$XT9qCv^6CqF(=O@()L$o zhN&v29DgqNo49I*q=D~Aeb4c54q1~_^Z+VH|Loix!&tqNG?HANC{i8YKq#P3ZN1_`FgzUPik1E6mV!s=l(tq@j1i@_1M>PQ57Cu4JR&N}8dA~3q|p8= zZw;P!#_v0mseb8zbp)+}Y-N=0aiuLODQOex`@zn#tRkS!E9p$1>2L(NW$-UuN?+_x z0GP_e%}L&r>W@oW6u2E69v%+p);c8q-McF5p5zu3Bd&!$Qqbqv6GGgkHHy=$!i+q@ zYnMzh<542L{)Y$tB(6Vy7zFvOIE`=XH8zdDbCwljH~1Lgee83xc5)$)^+Vy%`>UwA zt^{TFB>-VHZd(lYXLJjxd=3ktwrdQQ+G!_+n6O8e*z`Sb{V`rCt7i!fV&w5Y3OdesMlzwD(!&*_Z*3`6=O(`^M-%{D>2je z<>gxLCSU)}(1GZn`m(kaM=PLNY zb%W-J&E0RhZ>uCkj_>|hVuD`Ty%w#RdVk#QNx-&6@=Z~x{YSrkj{NxcW1_*3D^2Ie zB2viZM??!1b)IV{h3sPZ$=(e9zS1kzeu9DHIB&fY>rXMyqEZ)k&6n@l*|o)CWoe)D zKttY7A=d=d7{u%)Hk3ai4#Y)54`Pwtv1~PX+jV!$2%~xJ$&c84e0(-;Kfd?u;|tg1 zps0RmEr0?bon9=n@C3@Ks;Y9i5oYd=7)l~XE1b+I1rGe!vvX4exh$HMuToJ_jcjd* zw?*9^7FvRuzr8i~;o;)Eve;1sPZk;KMuLb{VA*q58jqG~S`Va>(xtBb#JPxvyN-d9LPs@2gX+i_X|??8>;faWtysMN}spe?AsO zio9~bGPpGn)p%b_+=P~eMImZ9U6CTD!ldDxYEn4rM{8^A@!?h%^!J**>EQnUY7dtm zBtc8fbdRMw)~6a0*fpMn)_QC&{DTtHRoXy`J&(ir(#b~u_5sO*6>U_P*ol7TkSIgp zue;rAd6}8k6IB97o`8b~Q@I%^MLk{Hqc}^lM%3Rl0t%pcZ_X~jz#t?1+JtZF8=Iio z!xj{SVlp$WOvqlXK!SkVCS4AxX?XfqRNht$kKHrobH-mz5$)H$3}wAiTWPA5wn+B* zuZZms19K&jg3;PK zwF@P?1*c-bpL!L=q*drwdKop`aJgSBhKQ1Kb$*@^H5nXMIY4pCZPVPAQpBBmQswNb zncDYaA?zYEyJ;+Ds2@(#XFT>LBqZeIw6bJ<-(aqAYz$fCH%<8KV=_*^`M zi3@R*M1WcOQ0m@2)=5j-_xc~WoTeKG^Yp%sjuz!>b9wCEQ?v^S;CN-0N=`SN*O^Im zL$vOk^OsCBH7E{3BoXi4u?n*-LSX1eFF{PL3_|1DxUH?N9lg+^@B}8g!k@%0_Jjl4 z9_H#-ebm2;o$M<|9|onv_~m4=qO3xc-0vB|RaqVTmQ${xoIfq{*UvTFz7J82yACku z`D02zX|m95qziqUJ3UHQCS;zBPF9V=&I7Y=r?JqLt{A&OTMb}n>}E`fgupYeN~a~5 zcg+Id8&oPqahO205}mYDZbQymH1YmT>q6!>|Wf|i7(}TKdwGP(g1ou zwRF{*hAG(gB>PWWJqwC*ULEBjDXCD{9U;CwMk#Eg(s}(Xi~8e?N*L;cgNXy1C%Mh3 z;P{V*dU`fyTTs(ny3`)xQ8BTv4I`1xb4h|kW@<3iS6;KO^TkM(6lLg05Hm`5q+OHW z5B)W4ndJoav_<>9zR8sj%_=I#+w1Gbijv4)XOHZ=R!5w}Xk;lqIjnPCeyvx-OtXnj zsViROgI=5M(<4fq*Zi1S>Sh|w*HJZq@-{I2b1Lm9EB)V0r$|TygorR4ZTBf}Q&8~O zHeSkR>bMcanU$LxBGXlW^BOBFt5TdGeauX)@@Uc&Pqhy~0k{h^#t`*7IPZY9A<k8L;rw)zH_Oe;(`8Ja(fDD zp=ROf3YJ}p+1mLN+D2YG-#%fDio$&BeY7*AQ@FA}&tOvm33UsKvTasY*0zBbunUKW zo;8ZNwx8*u-L)|?X8_tvH(dfkrN-V)#q3@?BZDe_s`s7h*9UZy=LAhxprc3tkk%PW zeebw6r&uHT9q*IHW?wthINbXrlkqvJ-+c}@npdoxkjdRmm|-*7!cooDIK;pCmW0(~L$C%53#d2YC;EH&ox7;nme5 zc%Pm=d7`GKwnsJ#bPN!j$7P-1qEV3wL*obQZF*&Zmf*VEM$5)x8Y?yB+n1+Ynf zBOjBf5FpMFwDz}=+oY%)-?h`(bc&u02JtEY39%@B6Cep!3$QtG4C&dC`dODzH+b|P zq#btAl`I;1ZZrWA)ob^2YfH%S8$yUGA7WD9DQ)ztdXpej%}ie7I>Z6t$j3)4lZkQJ zri2YjnFPQINsQ^<0v?fo{o%>+k*B{q&*#s%ri43E(e*n-iBb%}JDdUhl81EOl2Z3W z4;XZbRaNgqRBj*MmW}IXVi^DMPjR5#6X3&tOUv$&c}yt zk8Gb!hDSz{l934jLuT0#GXg(fzn0td%>mry_deV^$qPFF%Y%OFGlfHU1e{`TOu2PW zeQRq_<@{Tm?FVow^DxTT*`cYn#6)V{mqAgtm)+LCeF6?`=f`-g)|L?Q8M%YyuC7Ylym8}(X}L>+z(a;f z%dz#JXJIBuROR=I;hkdKU-;b?Cr3~03J z*o354iLNQeYz%QC9)lwzI>P=r+8SC~y!XfWO-L~8gDl$K24=i43-3jRH69)wHb%b7 zqC#;S6kSqvNd3-I&2Y9PLkEe;5dsAo{~G@~1ZC~xRdTO-;K<a&~?r1%b{bpLuXgZ3nrdT8G*sS*@rBL=qj=LOae(ieGbkyyv zFvEyUec%YhoC?)&Nmws;B|nkj46OP&;UQUl$t-+jpMLqWhgOE%JA9j<%c#1m7M7Nq zR2&Z;E%g%^$yuB-PITX59co@PzZ*`!veZb2jg8$jFQO$%g{o*yU<#EB5gek-df(F7 zX^CHAevOjS%+wT3ssff~`|Q}YOVCUgeRxh6mz1sQ;k=T!x3{O~VU^ofV^a{oA!YK5 z=ZpEQyQz73%MufP2dy@yo089-C|*OQD<>QSF$t(LtoG#C17<-vxeIx#fXSwpms$Gt zbai_-rE3-Q>`=C4=5fX%0qzWwh>yUH$SnDs{Pd9xyFQ*Ig?&I&R$6L5kUfA+EOn^0W~LG%8j2twR^Jr;X9g&9lP#nFfwRL?}R^k;!b-r`t&Lib&&O? zatI!{8U!`Q)YR0rNsZ%_WQ~NKra^2(WaQ7SF0tYOahwV>`ROQbphKKchGv99HQ`HYH0wE3)9V(jY|J!I55mZyI@a2h3O~`etacZA z8h|IP8gHbbp=m76BtzL2z>$=zr^mPO`LbD)pg2acGBK*0AZzQ4Wl|!5YE9s&Z7KP*;WyzB<%i$m3E)W&_uP zgG0!t>Hn+fJukz6Ym1;?ik$mj*65D!r}v7}ri6HnBcWfSfiw@+YvN?)=f6AV{H+cn zNiKaksvn+=Q;zNQa!?az7R;T=(%I72P~^~4OBom$anXMz=;cciWo?s|aQf-9S`;YT zD)YFy(*N&Y`Qe;#6}7Uu>aP`qoYsOWnq&i=bvm8LZt%P!mqBwXBucQJW^iC2fu)A$ z{csuE$Grw<>=Ve#%L_9^baXTfo&D(fx96H?RMww+5gK14{+BMJ#(3^?pRgg$d46$` z`M&*FMPc{$QcgZjAG5xjn_J*qpA(XBQvTD($Ucmh(^|wZJ&oPGhNgR*mVu&g%#%bC z8cotq$eTmyP8QhXD0bVjbXo>VqA~A{E~^gX`WRicB=X68H}>0=eTD198NTUgy8e~# z=H`RPn|fMWB$Me-rPT}#4Lv=DU2w5~Ox;!OP!_#W~Q%X!CgGlQ2G*LIv1U~}N%Yp7R!jq?+hH}T@yb{(11>gM_^A_63 zEU9wN!0yMHyBkI3Utaw){Q(Zn{P=J`xhaPx#1M@gYVvUOqmr0tlq^h3koIqLMMjTH z?k=G`u@=sohPQ|#xeR0-8NCn)xx1yd#9oXK0%S`vERnWj6&v}r$E_W3a=UY}4x4G4 zUnv~tGZuvzUW1%0P~m0wwgr znH?jv9{}h`ngS^hRYF3Qk%sPzGht%h>)-qg9nP~QDi(Sl9rzq?CEV02OC>HSD@*4e zf;J^4R(lqg1WgI**{_-rUZJ4)3J3~FP&L8L?d?MlzZ@p2++ghMxgo~qo+Jz`+2ZP| zB8v^sT~kw>_s1$GeNIFe8D)ioO{19E*)u}dW@e;n6+E|LsFJ_=HWzPnr61qMD^q#X zK9QC%L=KP_C{x>k#jYEsHF<)g?Kuu=q38)4uuFQF>SMjDq1Lz95eRX-g8UhncQ^46 z=WdIrq&xPV@4hM5l)5~WuU=&{kWH+x3ZQM4B|E*d3OG0l2)2NLfV{lC;^K!eX#fN6 zzBVE9KIHjIPfrg3D5!L=OG_UC6$7ZlZMyNo`;a3Z3;364z$oliR9RUW8XB6D!$NHI z;eoXN{mq`-{Y+(Z zc;wvc8yiVQ)6=H!kvmJ$w{I6j&dkm6g3bmfbhQ6Lv&Yg9fG_!QiOm(SHD{2Rl9Qh} zU~!yZ^61CPqEd61{VMK~dKqCekRB>SM%u4#cH%p zCX~z_PS3jUu4HYmyKCk|zJ6V3-a;s^D0yWdU$=A#3fQziQ(8yW^Jk}!Dd#0 z*c_b^U_?--%ovjqJtHH#Zpn*EcE-vROVC@H{0{f$6QCXgwObE3dt^oIcrQhT`CaT+ zG?U>C4^Vdwc&%7&8yfXHMsIE?f4stO`!8vgT`T{lRT9!)GB~_tO+rrZTk9K^5{VxZ z)s+@)B)c7|19@U^-p7A~dG6DvPo15exx=lkw%v&mLXTHe=4r)a-SGZU%sf()f!kRV zNm2pf;e^0_b!85zPNJAYI>hGzwaGl$l)H211%w906?H_SiiFv8eZ6Fjh6LVR$Cob! zg@sAekV5PBUBIJ~SC)lUE}xfX-z`{z43vxdQvWnKSmE*Gvdppv0KvKTpT7H0Jd_+fMRd?)^YE`QkZfGeJRPQs(hhF8Byi))= z^E+;vUI2(WBGnSHZaX+2{)@{4G1uDJYXoE|(6mGAB-_#jMP%wLGlMfaX zaG-Iexj7x_t!%X!H_%$mH-**hUG%HnOOf8SN=>bx9u`ls2>dAR^jof$~d@9jYqy9=4hpG2m+kw|S?M=en+&Dj=>#rgSWzAwsK#Za#Xy?*T$GP4RdHZ}@tGg?_#NGn2?6DsHOGUVwL zKX>SMd~@TCKTQ)eJZ-Lh1NppJ*6*yd!z2s|KF1$a@=bU8otFBUVYDr~n^r!k0_XSW zJo^0HWnNxmMj5>hR$r^*$jngnIc0Vnutb0e9ylnKo2HfIW?CDs{0UVNTHL;%g{|;f zrfQ1yTcDjb@o+#r%X|Pp!1L*~$_Fy8&V0D|5Mi!0OAf3I234-A!iA$zsm`k8d1`4z zDyjVb_Lm3h8yfuGiE}a}y}ge?>{Y-Bv&V~OkbPDXtxbhiTvHht83PPnQ;?IlmSZen z>~KdPcH;H_1f>G2!!-PL$Zlu6h!D)>QwKB@6tSS4gH)}sQN<(9@0Y2mJ=AhDLi;gk~}vFu5fUb!B}g~vXkE#h$Z?$uaP zm^s0E0&az(c?MO->G;^~36Hgw*`H)aW%owLD_w$vgVoj4j`zEKibk3un}44jSP-m} z%gg!bseW+043gHgE&6kaMb3+AyuUzKfEny9fP8=v+uPfa9#D>40WAh==EpnQg@Q%o z+4JYUA2*0K`z#HmxHi#joAuM02!Lp;s7WGHxeLz0?nFB*O9uVT{ldZ6&eCd}fmxZE zckbK))=jb$+lNS#$3aJjtdd%m;ODiwI2S?XnhTz-SWTx9hI!1Y^Ic&{2@!h%Af8;g zrsNwuI)7?fnx|ApET6Kcoe-z4k5l2&KyXkHpzi)pPqA%ix9kU?B}#DxqOYg_uBs}5 z(Be~>l9rYh&s*Z5Dn_Pcf$pR`oHb%X5%=x$`1rLh zYuB}9bP_~Ln&DrLj*fP8tj4)a2)J)EY5%^wXPe@HozP%^Gdu>UmeqfIH<)H*7eN6k z0~z!29|RV&Kb+k_PxoSGDjFh1Wdj(XN}T^#Ql zy14u?SX{q;{azzXchxRyhBfEup6#-)T9j4V)b>p@``w-i_}8Q0n>;-|4S*2XmZ;G9 z>Z&TR2kCV>-QO^l+l`roO%&Jt&gqlvwweHU$xX z`0&A#GKy&QX$h5$L(#bWugG1lRXebqT>W<#L}b5-h`2Xek_BOTPs859LG1C$Fp5!m z^O5m3kb_U2KDEIwGC2L4t43(Q%HMxzE9kmz3_(A9J{HJeGkD(0@xoKfbzizglSQO| z1+Vhs;s!$!_D#$#*EypDm=b-Fi7&y_0rH=oo*r+frPyU4h++DNpP7$7W>Ox{c5zoJ zM`iC>K?XC?(b4hnykS494vxY^mkN|9PV<|{TK0hBNdIe3<`rQ#!BuKBF)vuAE zwQ>NxnPma>1tm~je7TFciaSbMKaqsKoLzG8@mx4HGA0hre*X@%vXG!)QBqlLt&hAs zjy5VjUWupvjRM#HO$^JC_KRxaqOt)9kGpzXj9O=D7fWA0So+z}N1Aw-! ziiQRe0m0PtbiK@1V1YnC@$A_GZGFBynpW5`u;Sy#moOf}1OVdFbdH2907r@|S8lm) zTU%LKnY8s8CVEMDJj*|qb!w|YaAd-R&&*_kzQy}(cV|cO&Yf?;8V9T8lO$~V!|x4* zO!^Hol;S3;JWddAKFtkDk681XazAv3u1F-~LJz7842E?9!^``@)*iDf`J-sg9GE9t-bxtdwji6ade6;Dft}}fk8{a zYxe~|5yBQ+Z-}~fRT{$1@#NIoZgTnMhQJq?fSCsR2teJ6Bp>L^0r*rkp8K1i>{^+d zuRo0OgGaFPwS3L!}QdF@A`Km*;rC-ABLG(^$0V&JUZVAfcwz~Bqs2o!t|G@PkQ+>htKgLpmem>2q(wqSd!xh zDFwPZ*c*WSvjlr3*n957aS%xnxJ~(!p2H)*%_DZWj^)qfErI9)>g;bFi+YoqN+%{J z#>?d5=at`n$fKu$p@s0hY!S+o{ev_T1!*ud`!$F%RKW5`+$ zw5GmmUwCn6pr<5WRQ=iOkAgcfY*ahhS?@{9F%iCvsp;uzHwp?0UbU<>Q<9KCg=!;> zE`t04LMUi={#-0@ zTJd5&*Im}Jyk2cw(a7G#U(!6{slJh^t(bBk+C^2AS$2E{FY@HVQclW8!!azRdju9|xxhhR-#lgGlesQHGT8Kof7u zoB(0+`3$F=;3?R+NDDM2H+TQw;M*Fvyxd$&EUdoX-aQ(XIp9fGRF$^b?{i~+zWDFQ zw{JpqV`Xjs^htujhulV7T-?_7hS3>Kvw-m7>P1KK3n65t;plz=z(W}*LRZ&{l#~=C z7|q1!>HXO;-wY?w5mHfgdDmgkbLhU1>-9r-??imC?-CFYu(7eRQUh%Prq>PW#B&>( zaut#nLazP&MlB~NCk>4VXd<|GAnPFaJP9&o`6C)%QahcT<>3AMeJo(Q1y%<-HY>H8 zhevZ$6MWx=rUuy{bwmjEzu#w_pPXz8j#bD73k-Ogb|AWX5?J3^ETm1lf)R4&@8<#u z3BJ>iykcTjHB9brB=wui7If4YaQ4*=h@zuJkFN+G5>nFn)zuEn@PGhWXJ=<9mqurJ zlBY9Y2Qm1L`oqV7R7wiDjXr$n^`+0AoiQ>pB6&47HU>2aO4dG2H>S&%7ZioxdUo&{@Q1bU{TR+Lh$xgKBHVaPPc)`4X6Y zpn*LJ7M@IhqG@V7Lu7}rMMEk8sjvI{`+a>8xVX6FkwX0Z&;0$r&&_R0C(_D?5ix(J zOS3?f;=*G_LqUWu4ip1FKR?hoxMj*qOY!jVIyyV&g2gi;t~02G$i30j!0e}Kq+3)u zNCJR_fa(DT1VG)}(sC!gUEJ`b4RV&QzV0PZ&}>4Ko`vs?@LoV)Me$z%aS&8eRD^T! zB2qzNCRki7TTonM;6+-tyRWVv9G9lo>Bp1cfIy)Jf#={sNkPH+^XKm{Z$S5{C$cfjFF~ zZPzwPj~uH1OAH#*aF}AC&{7ft@tK95zG-qY;Vhl3Y%@s9RaL?@=aD#_K7T*J#KXfw zOKwy?4ac>WxQyk3{+ zH*Yk-DypkXZgeKxmjE~F3>;M*`YA;v|0k+QivKIBB!Ax*9pzp(6tF(DHhlg1kA(#$ z)Yi|R%5rj3!Qu}%OkbC^w5=_U^@IUbu{a}Z(y?;QezzjqcxSRHN?3`0{>pWI)Tm%HfeUo$#miJ&Ek zlS1}l|G97LHJCl;yAq5Fj4`|x!K5!FATTmIs;jG8v{sc34$S6Hp8#+e!gfdl_(N-t zcJ+k6pKAm?y3Ap60*nEJCRJ5c08kph&&?4?fct0^s05(6fQ^iD$a%@TH5jtdoC?lo zAQGtroilTDks3K9_HzU0ppr&4!hK)OgK7r@ic-Ar2e6w$!HV!4{`t<|UmBR|%16t* zmhDlmUcK7inr8>QAMDs9B_+WE4w|%y zV^Hd}I!n+QotbO=lCbRxibN1!o^GjW+~c|SXiw>Z{&zot;YyJSec<9!URY?;o#+R9 z8Q|vtPR0KTy~5k>6%fxqdi^T1u4x4x9RRlKl@2vsU3vAIUq6I)EBV2if2Hg0s)M3)NfC|lr(c%gD&p|Np#Zte=u^4phb#v>lh$iVQtW@%X6?ga|f2RI5W@R3kHbPqD{Ycx)0bYt9n;x`75?_st8dH8ZSH^w^Z*x3Pv&8SU`V?h zRCvtCxqvgQ&HWz46&4m2FwaOG|9o#lyt$*5;;u|s8gdUNx%u(IcX*;;7rOsImpQ*P9_I z9S8run1_tZ{6z&nHMK8Vu*@^4vn^?cfUm5$Q-GPpl8xhsh;jofGoB)Gn1l zVc-NDx1B5#a`hj&cDTCUpFL<{;;7*r@0KkZfQB3*jSw6+K=OvF1O>ohj!0HEGS*HuxtpMCZQOYkU_&gwB{`>b0 zX@m@i={~Gp8M!ag4{FQ7;0_$sK-ESF?-nSLuPqlxOIaqV=RtwYg91aKoci-M^7y%_ z)%Jx*I;ZR)xtL=a5)>d`#_R(XD0zqAFoZQUgd< zBYcHvX%;|vh#f4X0MWXhW<0H?t_}-y!*~3jxZJ&a_x^o0qciKsyo3{)F+yL_i0PS` zeakl`(R|=sU=aI!pWceu>YwJfa-t`P4>wqeB%d`DZ|isg73h?cm<>)e2Xjtl28Ov4 zl8uW(?_pt)es>ZAyT?a1$+T4Ge&sk55SGihN=t zub_Z?@uK+*qx?7&Z`)%X%66T+X1Wc6e@{3 zyyD%;Mpx0g5EYVxqo0!yI?7~$HQ&%%pmv&Z4qWyT5fNZaLB?`JV*e)$5{5tx!*x8&bVOt8T*I+KWm%bf!Q@GZ;s%kSBf?+b(L35e^ghZeAL87J(< zMGRpTh{JfoM@q780?M~StuMR@wEF<9%xkp`SI++74uDYk02|%!Q7tJpJZ2& zz&=}BGYOfYWRw7w*MG-BEOCTKHNt%zg8Osq2w~l>h?Q+Cmckw90&R+GpVKpKh?F@w z>eKa4x1*mKq^9oC(a(XoAH^Ju4)Pr{!b;vi(RSjl+bUl6y-;vZ3|qAzS{tZkH+P6JXE2axmKw1^B9aO!=(`+upS6@E427?s#(M4DoJL)=W0U$e#SI7 z7NboOX|F+OZX=3MZLET^y8m{Mf9(532MsZX9*=}-wIA_UO0q#Vuu>d*h_a>xhoSV~ zZP;BXdGNgzpjppYtIhA$+Q2X7jHtTdf=$^*AE*R8L6@_cCsbLwJ)OjOiaey!lH1Vv?E?bkI4+cjYQvKA2>5|#{6iZ zk)$zj2Ey(^c>3_yuRhRgwoR0RMVzLe?U7Z2e{SsHiJ_#mb*2mzHIx=Sh#QVgd`d*7 z0Th2(=tp4J1t%8FuO4WdYdtmEqyVEVy3)>T0;4N-w5^_&k?y+zyihVH%$T?o>3e^M zYeWnhz5J{*Tr40-&EX!_q3+q1jsNTm7@=7(xAtq31HYMRMFQ*rd&aDsnZ)fJ87?}ATtF* z?n2AqbK9JS4UkcgCr3uy+}$DnT)-W-Qb&*m4-Y^3Fr3c2v$Yiv-IoZ1)Z5I-Mz!AD zao_3xVc^X6NfA$p(qKMmBbAoWW%U>zIv`@uj>OhE7{KK!=HA{aG1{XYS(WewX$Ao9I^`xb~y*g>T;!YF-4v;2QegNRk* zMob2bKNlo_1olH11d_V2;kx0(n)K+%4BgxW(n@^}XQ;ULG16YWx?F>4i$zb zwGPk@_-othU}p*(s&~a7{D7a$d<5wc*5p}GVlN!Q zJLXwTPyy8(y>@9g4CF3|LYp0lLcgh%6^z&5@PHN>k-SfXcv`p5jt&~|L&8AqFQQon z9$hdT&&7t;!HyrJ(mN*9B#9^FjpwZe$-i^3(EB^U1=gIe^Zh*rgBce#mDM2mZTNe^ z(l{er3zixy7^F&`0~q%LSiEQ14ZEb!O`z=Ev^N2iT31(BQo;>i^6lSQx)VnKP8jTl zFMj}G8(b+fa2YUk0G1k|p6Kcdb9f%p+R=fBi~G>g5s1@Q1eY(riiv53;3xb&)%W^A zrMe6<-U%S+;id!VpVdw?=Zs9wK$RlAa)lPd(Wp?R9eW^Bz5jpZg0vv^ALhG8DIHXr7D zWMKY9GYtQ^7Vr{{y!Ze-Gn({f2@wbE4}x*CRuQ}y!nMk#1I#tD($YXr6y981UXFeB zs*V)nD&BvtrS2VAk&7^qQ@dd&iSuMD%gu3pImvPZHV4lo4j`hmO87zbEhk% zpwI?$mZpZrI&q>JJa%X25*TBbPbMZNfXRg3^7_+@vq(&I2ufkMM;~woilogi=hu=b zM`Xj^6Q6DW3quEpZ%C>ZJ_CaR@c_L_@oAy)b3B#R6{!F&i||1U~yS$PZ2D>o*< zo&~$M!oPP}!GJI(=Tc8Oa5}(SzK((c<=#Y<4lwQuV#ga{ur#<%T$&A~1pXy3o?QE= zIM9Mxw0-r})L=W7YWY17(?CyG&{HFPYyJVw$CNNC6F@LHopF++b76G`-t9m_IzKn} zc(K<%AJqc81Wn%JNWv6*4{#LWBy4lRv?2F8Fh5fh?Iz^5A1RA|fKl0~c7Qg&vZ!2r3^jDK-=(RUZEY zu!{n}0zNJEm;;AtRWj7oV!_3iP|xk4n%+x+RkL(hk;u=aCz=9rst@Y4E+5W=xnkM= zh?p4p>Tz(bG1S&d|2uoYM%&ZhORPlGpY7K_e+rrFGGO zip$8V{tp9l@-Xl^S_%pZqt4LB1woC3wqV^|3xhTb3yW?c3_cX|eGYcgz&aTlQwx#y z3=Z;gb8`y`#j;~^Sp1G$OyAKGcr8Zs;YaY4bv8E#=)!nje_UxhaO1!M)@X?E1EHFp zIY69}>|(Na=U0-}iPd1>~e-yHQyVBI$R z5mJ4R2GJL=WrVIW7~=LgmoB|+ao{liJyMzC6a2W+@!pJ#)3kA`ZSN~o!n>TN;mP3O z?~yU4*oBgI3^V3F&eOJ*mJ9g!<={Qjge=?WE-nsikY+LUq|u>&nat?JwX^*#V6TR5Vgoz<|MI5D9(Xv5 zZQMEY|<0 zEFx8*+reu5z{*G12)hTS!+t+zR#r%00Xtx#s$o9heoYwZnk8T5}Nr=*Ao3*VJZyM9*>dmFTy|05s5fukRO|r(m|wMtV?>AY z+U}8E39vQmg^C1xRnu0P-H?(#wMj|V!-o$+XI`6~r9+AJ4oyy;{G9ZeU*7%(%09d^ zgow~u7Q7r_hyDd`atLBnmM63a5L#Yt|Ml$-pZqbj*QsDc`-T6yir(J{+?-tge**BJ zp`T)~rV6u!h7Ax&4PLF1P*B_2+s)0oZOsT_#|rH@`E9e>Z%mcnea?XBCMkCf{Yu(YcEex1#H#HUb7hR#WoB9n@j|%Y^HH2!JFUHbgSIX`M$3E zPn;n7RSgZ5vbX>QP^u@FjC^=F4>~(qGfUHj(vy;?fDt9he8dK}bRhp=?ulxkteg4W zvcxw}TNW?M{JhG||3m>oTrna7ATqo!MHk9<0tMhqwBH&yy+Z z!54INbi}S-0dq@xq&)6-@&D5I=uW~hqM+@9`3@}U|1Ij0XMzJJV1n|rCN-YbU~l(; z#WY=A2?>l7JH$oI(U8zzaVadJ-VmJlx%T^C|cpK}a#abLV_s19)+Phl2Nf$R`LKE@4{Z{kcMQ z?iaOSl7PAl!x3Orjt1bSzkU@G7Cr=u-LsT@b88FzuBxXQu&4;*Zrd#5sOrJNkIJ>8 z;$nJFoj)QbMVEk}VDGt9BEXVSPoUt!2HRsmK+x{s#T+>FC!q8f)VgTo!0i)7FWj7* zkA6)ndoIXxZz>>x3B59q!~m2^$JZ2DeBGG!D(nFur_jQ}yh@PJ)AUmrIv9pK}iUnG3Cbf-vM4oxeJWHOrpQ8CXx^n5rHep7$O&N-t_Zh2yEg@sZlKj)6}%u2!leN~*MRN;U1;QLGDQ3d zSe%*U<#yTyCH~d)X+5TZl>6vaN94=9U;uDV0M@xSRE@}gz>+El4b3!2NEJF|2Mc{G zP*edbU*j@A|10)<-+x}1#_xVl3(d?7F<|)Cz|pcomk*_Yb!_T={A)M?+}%WMeNd=i zRc$&LVcYQk`Zl#*KcR8>1JTOJ8-IZY`{2JogQEiMUXZjesOFboNCB4$Mf$ z%>rpW{v>tszxREW9301RSbr)|(N6!Dm-)NL*PsjbF|DnwfC45nOfIilzipk}G|t~^ zT+$7rHno3SdJCk_@h1asJ8t!`iH_zI{(NUzJs2&G6UUF=XxJrrL3k7bL>%3#&=GH*z2>;H0ci&?t9jfd{vd##^m z8^lQj2K9~sH9J^#wT#gzPncO)9_tmKdde$@uyyW{ZUks!-G0g`GD?dPv!wTVB|bL7 zU)axO63iE%I&%FyTcZAD!Kq&UUjWp`)TjToeB+>if`rNy{V?!;1$*z6oCz4+Z3Abv z1`Rtcf)qobW)38dsDJxJyqv3iin;4D9kIyPv5h9Zh7dki9pJVbQ!9ILx_&w6{hy_^ ztPtVn;i-du{oMQi)?N8j2cvO+fKONOTLV`uV(%Lr-;-_CnKEx(WMdlklqx zP;JXTrk?*Pxb_Ia)dTs2rl~vjQWr4%e1Ai~M+OO6geppntYbpmv!C;tAJD3S|IoW$z>#z8Ap7=}T zwb=?VcbuJ^yx?#jw@zQR*S&nI;*&Cd3)G`*9VfDrz-oVpzp!x}+BDI5tqNW@X>NCo zvBJq|(fPv%y|CnWgF-^g*E3=V!GH_4YCUO2@?Pk>zE-LVaJbnEdb5c&3gOH%FTf_B zV$_ru4mIMcB3G{=bAoTZFwtLg47%o7Z+6%iA+*lZQ%Ms1bzPkl*%LCi@_l;=hRf5# zP1@6=-FIZy16OU5P=e=`yB{8bcI6Rl(cM^as7r*Lgn0$VxrG1+_=a%p0+ur7rDoak zAkQXr(qzbP15Cr3b=X8k_AMt>ErtaT%q>z?`#Kf=-egLBh=nILQ0U3KQ0Q?4--TBF zDL4t%L)~qW!n!TzJ9Lka6<+w~-TATYJLy%J`6-|0zkO@6O#Bk|@N(Ir+j2x)PeR{& z(Xlr}F0Qfq40(r3ciA5E6!|Ju$1^{-mb*{yI)JpEhmCVq3cU7drUbX0AhKhZPFqou z<*5x{ilptDGVV5d-quzKe@{F+!Ab`0(bAH1ENd$W4ABk|E0I~XE* zH|9x}a@s23cMjlpWj~H zya8taoQf&|8h)cZIvi4e=){oQ9vb*oT9ktit*pmp4uu^@#2c*n{4bjnj`v zb}wWq6{u!&M6f3Q+$J@LJGc>PcmDVe%x zt)Zt#JSTq8(=Xm!dgnSswERkeW2l3x(FInJd`!f~=7a9_JeSBD6crTANx4m>o6BCn z8y>m>_6;R)1nh!rRgOt5fP^cuglM7jaHi-1v#!0p7+#GaF2Zc1j&L_PFa-CFWPZf4 zH1T}(6n_}sBFN6MA4+~>1F12X*opV=?+4c%c%*(;Fx2rhbA4HSGwtp? z2E2tPQEa;|{y!>b|+q@y_iYPgYmdty>j|NG+V?NI<9d4ygC#FxM& zZni31h@NhwrSr1LFuKR^{ERvM8AM-U?n$8&nk526>7KYZRC%n&L8_5hkF&WE5Mc=#clK4Z3yo-a!(b zgK!U3N5wn8%-NT5emz{ICQxDn!hh?&b7!=E?2)q!(4Z07{DDj7|$7~|>%qU1qGcqUvQD{&GDs<+ajbgbXVgtB|LfK_go)_jc z_D)09h9K7SCI+ZJ!(UJWcEkD z@xn!Vl6J~*nu!K`s;D`_>7Y}_?N_uR$?>SObBd-m=Eq z9JL!z;oE=mLxuUvg0^jUR5tg-$jJWKSOJ~W`ujtb3HvXWLcx8a7sfe&;A zih-jCIFvWIBk&NZO=7E#J0Rpj?eMyEbR$)>>_pI~7+!ldM448&$7IY!(bjJyZG*oJ z?zs36$D6fXe>w~Cgaw{#&)4{>58eAi$L3SX5oJb3<8rGizo@cmPbp0%I?xV~9zb!`sXIt!+a zOnu$X2Et3NdGxR2XRDrQ|0dJKZWf{A;^z+x{ZIh;M)hm;UNa`w_C(YLB_&9mOtDy@ zb;Hb$VJXVWvDXUBTx|j zPonENExrHFTiFa_Au=-bP6_s~bH@<%O+MYxQt z3G9PyrOW!Iwpa40MQ>&jIlU`1-j_D4pBPk%K2%5Ok`W_;6^tMJPTVH<5dI@Z#Z{?qkO2@!9(`v!#+ z`V2F>wRHdU^pDOzL(8JNc0+T%;+o(gSEfEPMM2QXE&5_6jm4`${NfG>7 znMy4Ap=?o562FeKsSQW~&C+InRI0^}VsmZ{2Ih{(#(vl*2(p?VHG6ewD((zt>13{D zfMSr|6>mpR3uZY&qi4carx0cA4xkJNC4>{r(5QT)bM}vYLtm2RRQyRq7yU=NA~xdA zNM(>Dk4fMD^8I`EaGnZ)@tz4@1nWyIGY7>0Oxw_G4OV_he5kcK9GZ6NhnwR30$1WZ z@d%CG?p1qKuxVHz{$xJD$ES4gg|O*eQzmDBLqo&9zCLNE??1t#x?1KDn4LwmI=Vp) z##N0A8zli$p$gN8dX^)M9R}@6qFV5xqR6OVCx-y)88n4~e)+}-vkC`fAd?sKqjCTU197s;Yc zilffFp=+5&jjl&K|9PYDKsL2fqWXf1I@H04%I$_ecWDzail6^|gnc2;;$HLU@hW}` zR(6+(3wWs9QVev+UPA97N+S9x1Vrg}VD?(XTcVBIgyU>)$*T!|Eru{cXIu%Ec zyh@62V-mbE^OA?^%jX`Rf-1ujbzJasT4c-GGG?4RKl@ zOUvDtKsXvF9Ho_~FJAZ-Y2pbj2y6+zc`(Iv9_hCI^G^!sUyr6?R0jtb z*c}j^1;ex0j{C^*KgW8+uF!2nKs>;iC(YsNPdv;ze17(i_sk8!M3cq3FHoEu9hqu| zU*6$Nn)%T!aKbo(QeMKW$MsBNf& zjnq0s$oc^D1WA&`@D@L(5KWW$Em3k!qN1Wep>rmY9sjZreLnF;)5^~7JdVS{i)FC) zQ1B^xoM>Q|gc<55OCFppTnwoW4>dDX0ETN8@EcYf@?ueA2X6>r2(Nt6dw1SSsie49 zvYyBGR!L+Bsq+p^JOoRgpL)_iWPLG{)VHqrrFK*g`2EeBH*qhjsAiE9n6_*wz{Wwn z0SH9l)G6dwZ{EMZH`1MVUPr&(^Td&Zk{{Kr$Pd;9!;_xAJNK-B?GhGqh~f41(`{od zp=}lgyS@y^yFknb8%gupkP#6Z0Hp}ygQHbyUI)f;yUZzMH}aD-q|;*%bykmE11sC` z0oP&i#(SjRVtQ#k=nlXe^A&FA+P^<2uM=!^h_W3(MmQot&SZzYKSa*S0hU^pjH9-85XW;InI@35r(It@LV^# zdLA<)gXo_Q4D5Z-m4KFBP^aS_QlU-*MfNiTIeOne$I-Mdh#CTXJHJ+}AUnS9k#6B} zef_M{9ED1YAw4R}$`M8?caQnV8pHJ$c%iIzzC<=9N4aS40^>R>A(23RuDm^^rx3Q* zuY@hf1ZDT--r}Gm6b|ELQP>ntYCpaN8kOlAgWJmbk4b8vi3EMG@m!<3r;f)r$$KHN z*9Yk|+(hR^+lOJNb4|7-7QSpHe}RL5Uj_>#L#Q@@$zzBFWDTz0Tj9;^9sZFui(%Ym zAM#XveQf`ghjm|?D|CuQtL~@x^r4j;s}r6mBC2*Q6wSK2`g)r6>(Pyb`$bVnlA(bC z*x%@aaqiyzR!4j%agEzB=7?iarT$%1vZHXq?4wj(QlKmIMh+I zCkat5K-6&;oqi0fh-+%XQ>(t*@(6yf_}c5bjjE zck_Vb#m|Uxr<_x~ct(9ue|;3>ajVaB1joFL<(X__}aR1XKax>#xD) zeJnA^tQ!P2BZK%e|8VnXxqa)@#B86}^UNkYJi%BK{+poV{Xpp)`!+IdSCSefGT_j5 znSYDM{r{_IG&eCxK4N%qIrMVJ$huBd=BwD^xO9okX+rF8UJi2D)UL0NmO ztxE2NncMcBckj^J9|8$wde9ZZA1SS1hxTR>lFhPppG`j<@x8fG&J*6}(>j;};|V9o z>eo(=A_JRy61t`h#`>)MYxklEbL2Jz_JA zHBQd}Ws0JyhG&Fg@q_fALxwjgG#OkAn(dCN_h2##g5g}xp@WKK+n%C~BUMTfd#8m2 z6Zi#qcuorPqWMYD)Jh>Lz1TDVt@P^Bk6u_(vp}OiB}9J6fJGlEs9tWvZCJLzrRVIc zJ;+V96BhDe$p$EPo6~J&J^f?)uKK%MDUr#wDky($e{b=V(Gi2UX!YxpSW*;zrZ(vE zlv9AcH+?vc!%|M)Lt&13$B_2@p8x4Za2^d#t0!sX$JYTl@We`jvE=ZsAZ0pC-Vzt< z(#9R}5Z1XmhkYMCTci|~5=+WHb?MXNM!$sB6!wAo+!mVNTZ`@gK7kU9u{++r4V$_i z_Pcmkpdr3)_n;M?WaLh$~&vRLkoCLG;NSA%JFli zbYFYeKVr6#v9T{7ytv@$=(c@(#+J=s`k4Y{oCS*&6=wyz+iCP*VN} zuvaj2`E{QA9qa~vzC{dQZq|k`7881jm5lP@_f3 zPNRTJjWv1w<<&^Gf<`5i_MKREF+m|VNSJGNgQ_aYi&yoCV=9cWY_GBQf^b^e?r{RG zG(Dl9sQB=Xd95HshMk`uAF*iu`0+212YK}DV2^bv+L#zopT0{Y%>CCl#?@!Iu{S%m zaPFj(EOjP>t*Rim#wbN((4UStmJifkop`nDp<^@`Fb*!i1VOQI~khY+~l+ZZ8u`d58CW047r+TCVP zvyguLep1^e^B?U)_x)Q|?y(5-%o|M9AYM%zUqlE&6|LXXJ{%yd!`6Pgv3)pr zAy~za{rxp2|M*i>*#VdtReQ2yx(H-Pt2v;Qq-2cH`PhZgL;2Nb+Spd!8se%!c(G%S z7udb3xQ+S5(PDcC)Ogn2&20v}XH-$0zb(an)f0;Q8bH|~)0{=7j6#kOjAg?yDj=b@ zR<74};-%6H^~DRUOgZ>A1HpTAR?pc=WmW1>gw1-65u3FV zC}p>m?C{RsK3N}Lxq^A}JMfPm78AN-W}~cbGLFi?iO|ad2BFXOdoJPd@ii-Tc8be$ z+P|5xC&q6H-!y>sp7XY{o2tfj7HiHAXNYTy@ZhP0{**9@`%^VN{+^Zfo%}Z2ogK$@ zwNePkKYxwezlGX;0<)RQ)vm774 zq0eBv(QP$Vn!4|3l>O>S3Gfu-Y<3tKX0D?Vg%sMv7i5e1#l_-d_mPqwBL)tw2|6j{ z*Gp~3)^R*L@uxV~8SKCYRy?v=vp7q_h3+>c>c`J%6C=;}a5QXD+}N05cX&xN|6vybRsBWY5Jw zM3`&S-?w@cGVvQ%6U>`7$#XlT2An<&e=oL_8((9#M;~P=nl&O#YxyorugwS@Uav{~ zSJDtApp*ZuKccMcHz;89JGiu*9u40vK5^!v9NlKI^Vq&b?`53|8^d`UhudrU@!V03 zkWWDyp`F9DXa^NDYyt)a{6bnMGtypflHhQaT6h|6jS}eV^#y_Gh*rC%^F=ehxjUYU2OG zKPSDU^%u}gzrA$nzIvm&cQOsOOF2ApvXGX-@?dQ9o0b-b!B;Va9feH6L8p-KN^O&- z>%+phI25Js;{V!BF9(0 z5~r#q&WZ%a8BPYMrURS@tp}PoGYqIO-bv=cHN(S`j@AxB?)KkDSKni|uPa8b3PuO06lxn^l zbvJeyA@)SaB72IT4M z81rYm`r3vBEPSbqfsXETvs@-PH84)#H*CFfK;wf`^psH3Zc(U z*9NWzCFPeaJKv*m4;}!6lZ7`VM%sY;Sr~k#?+-M6txZFqhTaJa`(~x}43v1BMsJKx z0W@BUF^0c)=A3=lg%TI+fW-Lt#ui$7bgdD8yE2fAI6doyi3O`KS>yH3nVId{ZKi+( z&rL60!gNLCq*AUEuf5L;X=T2~IVE_qdFLgbNa87A{;mX9g`su6N2ic&;7AjdHgKCS z6@PJ{MkyK1SSH{j^Iw~`d7q|v3giM=IC_=u;Y>VSe9C{sLJ>c(8$@jo6Vb)_r-hCl zoOL|dIdkCr{v$5Z7TyZ{t++>YUtV@~J?6c1#Mu5HK5n@9B}_dlE-p^qVK55KPH<>w zM!YMoQVhmW&-ahiMBf@)FZ9CYHG=AzniO~&_kk{ z7N+h=d7;LB-HY7^Y=1%;!pIQ&;DI+bEG&hKh9dT#&4sv8rEn+gf?TjNu&GqnBq-by z-6(Lc*wOn21c!utKv4+kD5Aa&(DCJg=QZ@qd=SXL$AbZ-21sCvGlJzTT-s@u1a8{I zhjL;B6fDhg_Lsj#oxw-Q+8v}6$iyqV^z%e+F>G?_HJ1FT{=&hA_}Hnamz>hS?|Q@P zI|j>0Al~;kNzi=3%TCqDhPo@@z+a-I2d1TQ)1&~M$k!_-$;v)>con4s$QOW*H8l5Q z8-S5s4ozv(_s!@J74O-zhxvYQnS(Hg?fnzOsQ}gySUC<^Z*e|cz5I6(&;YbnhG-N3 z8Ch~v8%LS<`$vD)o%hJ3kS&w-Zg*Y4)DzJgN9+Bl-?6Uc5lp^ZTby za_l*_ow@jWDll^!&50!b%_*=_F)vg_1*d9~ zM}8+PD~VS1E*c^DXlL%nxm+GIhOl^UuQXurt)t<{WaHnc;<%hpZJrGZmy)7^q7i6^ zBm6y}rhgBKaQ(6S!L@8S{aaaBPG#*}5H_Y_;oiSL7oXGVuyK{UxG}A$BXT7yc)K5@ zU7%Zh3l=hhFJ7WDAPNTuhcitL!0izk^(Qh_&zz~jF#@eX?++8N3goAh3hyEq$%~BI zs69BEAWTgagLDg`kx^)7i7w8J1Yv)L$jLa!4t*uP5~bC26LSKvzVBb$e^1DOTC0Bx zjs#a=p9FD8-3D)yO}IA8htlrIrf!q*%K1sd#fs|$o!FK*VNQ#O($Cq$fZwG_tYrB^M*MH@5eXXc1~S4wejS(dH0?a zC;5vglY4;sMoR^O98=@7jQu_}Be*$3T7G7UjX@Qp$pO3X=)_Ug_M$TWdJ~->m&!f_ z;W^Rp0LUTy`s9p(E`j7XS>FX0%1K!GfyyjJ!xKa}J;58tJ#;cLB7zt(g!UzD5t8;Q z;w;NT%FD29=-ZXR7(s4|?Q`$O8Ly7~i%f_L>1(9t4PfdFr#OU1#BgN<2~&--Zr|Qn z?d9Rpd=A>yyy?=N$WCf7J}74YSw3zMv$dB0Vhhkd6N%MqK==Y^hXbLcoFnJA7Mrx} z>mFCQaWsVWA-Z})ZoF^{SkGjB%tNCt zpwt=rU7bK$04Tlz(`!> z1NA65F!ZLNMGnjJG9Dha+yQcQ@6dK5&5>M zGw>E5AZhnsbZ=P)RWBMaft_q3BxE9TQb!zt3Ov0FH#bmaiJT{a0GzTb+JM9shx%58iM9-Zh32qX=MQ`dYXf@C-Y)ZjFHRIXk->%tt`C zV%&XHS6^>Y%@yBVdNJDdquUSyS#qP=1!qJV-0c1%Gr~xa!n>?QauE5a+n7VG)tq+e zz)ou;Ot^5DuzW{-x}7+NsFj4NM^A1wMUrXXNOW$_f7;H}rxBtD9JrqCl@hZVG6+xn zYShMCZ`3tY*`8p|+U)V|uhB2x~vU+`EJr!C))Vxb<+zp78^Q z6AwC>qL56!vn$0(d@0j3J3XxBvU!M5>L$K5jZEHQ^xR!clFHY41-j)?{k1fIwd;!> zrdt}*b`d5g9X=Um&~r&&892%PgWP;x*y!0D;bkCX4mD(w=M>$=L^?616J85o2*F^c|_oxIW8G;U$e{)67_MIeIs`qNjidGUx4d>=Fx?hf58qpx1S z=aO6vVneV6)3Dfa6jdphfqNJdkI0TrB~|hQrdF&wO9{J!>OfoLzSjcwU|paf>Fi?z zJ)6<+?KaES;lU;Ww{dNv1CzHf(q*HZZWr#m|h42rjMwLl9^}elA zP-cSua##Fy`*VJ#C3tzjF?NeK_a&$feqtwm8XZi9m?D-5o|~`YEG#-78{P{d1}mIS zks)Op$qVFUgvPFz&hJk>GsK{cG+0-a0W)Cs6~%YJJW^Q!s|GA-rOd(CGAVmc5LEDB z_t21CnM!Qv^6r_Nn6drE2XOR&XC%Tw8%vuHKTj)ju_cej@#D{t>*4ONkAwuYA6e&| zO1T99L%~pMOmbaB>ToW}VrNucI?@pa+IpVDT=2*Jx~VRWsU?z%%J!x5NqKFa`EU#J z!l&z-t#k9lv>{m)(1m{0y^Zjd09Dzs zI0oZ9EfKwu4{y`{(%g1zYQ#c|S@K;(=LDunna?012Ry4DMh~BgOJW6djtnrV(FBTM zv3OUhfCZP6Weeqqq@+g8VTs}!3B;gTb?OpK2whKF+rfl6Y!FWi3pIz`-AjbNy+WKZmYf16q5 z4uElTE7_eQC9&vfavT-SMRJ_w z>}F+P?hff#d3W;mxx9v-VjQfC2-@jDzS?K`lTka%E}X^!X)u<-rO5F{#*V z(PiY-*Onw??~B;O*9jAiQK9YowZj{PH!&Z@dQwmiK||U573`v2gyoBF+I1q*na%A zy9F2Q#QKDa?R!M4Hwu14bj-QetZ#xi_QIO`ccv1{(`v@|p@^q8C>6H`FOzTg5Q**c#3t%?%^(R>;MD7z?kA!r%nH+TL}CO)$h^}q`qrW@ds7x|!{&{k3Smh;QR z#9I}>EuVp>?^)=aS|IOZfjcNm zwgQfcfBfKiCWZZ#AJ@&tZ}{}GUtgX|Tfp&p?9(?3!I&oQdjKc*M1xgS=Km%NXmACh zuzYgy9?;?$4S1>v63W+;&=TWk$X6@tQDKOs~ zzmEi;y_`Fm3VT8gO;MO~>NpL)+TLZMoz$!ZO_)R_OT?G7O(q)IThBc_r=~8$$ON{+ zy=Gtd1>q{5{Pn9Cz3|e~JdL!Mc(9Mwq5k#$hB)J5ZQPM3Fr3GRE}WV*~zd zLzZoYI;C(uh!sA#$-_Pw)mhQ7RkM-1`vRJdd5o5J3As2vblQ+?%a3B*Mf8i7bX5*#8vBLE>H{ zHMTVqSl2UgE0=e_S^>NbGVpM#Byf{7XroS>zhl%M>*DC@*?V}u{bzHGcV4nTWkS73 zIfcDxC$AVpV`;k+hq-z37L~wpA88ctP-QfsAT0HvSB0+R*7m-tQ-`S$T%@L-hbw;6P+4*i;C+c=jj|uyN^-)i^-e(h;$6m8o z%$C8VD&&$cS}e1XpTf3^Qt}$cdx#3e4BoyEAHIjj2pj*!JR#3QaaA&T2`O~0PX6gN zfu^TW3Bv5f-g!@WBn5>Ka@fYlH)HrTOo4^D{f*rR!5WN9=UYKAh;zLOM{t}0n0p~v zIafEaYnvGb4LzD)25P21GBoZe18cbu6B?+DzO8mgvDKOC+N1dQCpf#|Vfo>b)jWY% z(LNwCX9<1nlg}jDG&b!dR4>SR4?;)FwtM%bj4$;Xnn94r!&J>|8qDR8w>nO9bMEk7 z{RSr;PEd`{5e!IA`Ex5}tTNfOg|uDR=xBp+T2HYua&alydncRI`Ej{NYA-dwm|V{S zyR9=l6&D-Eq6nBn^n+7Zi7$e>=K|vOi^q$&T&!{YJPTuBk9oe*!X=1PUeg@B2w>@=9F43X-W(?3f!*WLpL;?QO>R%jiyC5j%NpXufoxX;O z1*Hs#jW+EBYzGXku~CUQc)o5xc5fXJAsnv!ZEsu$-Az3;Sfc=f zNk9-a_Wb~C+TpVu!=a)Y@ke?bacer`snFKfPr}eV$U`Zy3y`Xav85yNpWaF|RTsn$ z73}IT0H5ZxVvy)EMm>IYE#Q0HZoHn?Wg!hH&!z6ieF!O6KWo}fB!5vTw?>7G#J|K3 zv=G#BWlR1CsHk+&T_X0XMp+utspW4G%g}b0I`kV|lO_><6N+9PSGv=nZ?Ds-R)Iap z%|9UD^OY4mc*s2C2l@D`zZcEP@KnQ4sYoYlX?{>ly=nh|Bdn!f|D5nmu`prJoTAD=Xb2H`5o6x-tSg~!}wk^(#?t8`&6%UED*HqI!1W-5+ z@fkgPirgFlj~H-uRu$7|Ah;2V&E0;$Ab#Kta`tSW7tiT_KRsHXi?02-Y<0h`6rg5E zcQ6-5qDjd$m2eMvnKqL$kp?IjYD1WXd}~QM^GB&V_ks0!OiDT-@DtGi=Ykz6NIyp4 z%5SgXe>gPfnJ5lfLS~J@!mpG!uTeb%{Ng{__`^{zc0YgHONw$b*cVQ6!;d;~!=axz zIfFw%CfJcE_{){G$VIk-XR0ysv}i2yl^j$P=rw<5(|B?WBXW%e8K&cAYL@@pF{Qpp zx+qxM1b@iJr}CDTmeo+WW0zxhXQh9|JYQIzpwqGX$B70dbjN($Z9en<&$w$=W-!Y; zSN1{RIWO91hy0S$YkI4lBP!NF1LG&ZkR3v@%e-G(NuBY71XzraWHfs#x%bv0Y=VQ< zi8F8Na~*jeObJDeH`i|Hcu9<)UJ%YSlgiwA)bKP6YC_CeqO1j8#R{q3L*Ci{(;~bZ zA-v(zO+k!_K&d!qT7C^pF9chgoi?h$%>1J0G%g54d7Uz~78`cB46>8`1~u@+fY{!b zb`8s4Z^9>4X613S!HL}9)R)yv2eX}s*QOM!B(GENDfj~H{e9-4U=@gA$jD)9MFb4a zqt|M?!)fU0K=1*wlLUkKi#CO(^`tWoF#YvSca(T99fbeL7qWE=QXkv?N&O>?7?_3l}YV0&SZUZRC!=jlI@# za|yg@eAm6kCk|MvB3@oik;(gAlWFiKfRW|5;Ph0ivMp zdejuD2~W)L`7!zHlD9vG5Sz4$ZEYGDcbS%X6(|2V?J*e!UF)_l+uTqpF5zgS`h-Xx~>M4kwuW^|>jVaxS8DN#e~w zsK%;?<|73mW!w-bTzynWfLz6eg;yLMpO3q^3GnmVKw1On3Jo-rF#h>_giES$n?6RJ zhFK;!kjH>KDqQf*U%#9xb1H`dzjWRVpx_WbB@ll)GwsIE!sAH|5rwfzb4| znTr)qIW1HS;?8GntYE<7D}3Fc5ScqTx%9a@ITLVb5RT;@$q>SYfNW1N z$p>~IKwnQ|w7{tRX+HixaDGGQ#!|WI{d*#?DXAm}{_lOAvu@qG)(4(&b7^mHhp43( z|FJjj8UI%>wq)M6?L~JuCS7JD%|d2%|4A`d| zhI+U9WFQlIa{Yk$FJC@r3|9#`FZWK(Ig`*w~wR`1lh6v>*io{Py6;6;p(_xgFRge7pvM z3goqOE#Z4kWZ#<@8oITZ=-3W=2HGI&I*+_X$2;Bl4Zw)Ii#CA{ic%qs{m9)^J%jka)=sxepd7;hMHsXLs!y8TZ=eB$iZf>Z(PwRH$J84`0#vmJiPK&kd;x|xJ>xD|fx+h54!($3VdDq0 zPe7AHF)?piKzpRq_x>kaCMs+H)t|6@S8$=EA7nsckc#NRM9_7CV!&yi@^SX}?;3n* zW9moiPK#si$-32FSObHY*O$sF9S7-+X)~|lf`=*1B|Qx&1zinV)ep+Q2$w7bXG1`CMJM~ zl?`E)|10?3!rNtjhe^QYTolas4tD<4!A3|UR+D{LcL-N+BwxTKi}(gM+W5z+d&0C2 z1GF&5gl`RQo#Y>>l^}Wo>FzZ|Bk5antu?M%{EjJW1VY?2uylN8vHP}ir=x!cUqctB zhfdEN`UjK?xFYj_UGZ!Vepz213ho060dy}*me9bpO}cbx?<9NMoZ=~{HM zt!Q&Vb71pPbRu+j&CFlIGl#!rT14>|{6Rx_X`>|w4e&cAzIYhy{Ga1{5LplHDNqy= zVAHd6iWs&mm2x=5#I5_uIqp7@N1@T`YZAkn+GNQ7r)A%YY>~Y`DqUUP=3T4ha z=C{Ofu``-IMHazN2|pCL+^69^r!x$a<<@|-0%HPxep8y8eOU!KSCH_cg$h--yf~k5gEkLkc3aXaY{nawt(WX$6YWpC>EvYJDVHN5 zc=+e}n^Sjhns?@rMX^tk;{ zA`^{T)|mt}Sde(4MB?V=2HXhko!;Hkb^>{?O_?UAr#CV(W;k%#6EATR#)H`?=ars2|cJmK4LSwFUL8bc1WOmuKY?unB?-7Ov^OyHtU zw`elbd<6(&X7&qH!>;TNMz(UA5-3S2YUvt%|m`BWPpyg!fOsaV3Ue((+NA7Dvdx zR()L6l21Lk7VG!4LFLVks1?}k<9WVBzNz28vnZ&_g0ytWI{h4({IV*|Y9?NQMa>khV3&&#Al4KKw zsr#^(($Na(*y98OpKs1KJwMPSmRJQ$NhHJ$?NJJ#ilv)j*P9me^Y^GRtt63H?wpS33cenGDv}-E zx^E6#S87oD`5$0Pd&%Lb#6M5tr-2~@^Y3ff1Eo7^70RJ&hJ-bzJK?1g=zjLCrX2Zs zCr<(Cd!gM#cQ;@*GmQ5Xt z@KH?Hef81q3{cI3Td9KOgSP>>pdT&$w<3RtkT^GID+c-7|af>%X%lvM}6Vpx!kqXMX_8>xnR22uG7t~a1P#Gr`LHMn z&d8_l9#WPnNo3BuQ&LoJ#&pphS;Kscy2i2Q;j`@jz2w>;*H<3uK}1J<4HKq*gR<}( zz-jpztsaosoS$9U6-~SHlShpsE)x2A=w)`m(SSg+*^7!$Z{xz(gJS{|QV9Fb-&-+7 z;sXzNUm%VG#s#WFx1zrdV^&}dfDGdeTq}SvQfwjI(#u`LhCrY^pN0NSNfna5iLN|w z2^;3z_gCngB1d8j%e-8FKuWtScr?~J;2=#|B^4F|ft5oVM_ZSx^x_jw0y5M}x=+vwk!v4U#YyT1ck2tK8^;CDkN}{4d(XAVb z?^gJJ95%q<=AJ|tg?kdhC=41t#2=@Ugj|G*YUKjpJ$4WJMuW6dj3X6VJx> zTcV2xu;+nOlzc`f;9Ikt1@0KM2h$8sE@$}ss;6?-{I;^H>Q^Jd3Cgt$~T7S)PIjIMFqKQpTAM`WnLD%4D&lz0K;U|@o?UQ zf}%c-|0BqtcozZ|nj=wd)BJ?rdYytY+DR0esqMxlnmwdcd*V+Jg;1m?U)~S2b3n3D zyLS{6l4^3#Ds30*jfI^-`mk_-S}p3Svm2CtvOl$JsGG6ya#*%!|?xI{cke@B8}xU>|sj z`113OXj2zi3Dw-Tvb2P+gcFh^a1lQ>JDR?JX35(ff3m+G25=uLT^-!rrHo^@nSG;K zkq-_|8^W&$k=NP`V(QvB{!V22AQ%Z`dxEdZHnsD$qGZ+%^|j4D)dRR|m>u9w)5Gpf zBuI>JoDzQH=RrYztepeUMGHgE0pT3;n6nRi$oQx+QABf%ADkfS#arHN8-F7P*jPOq3E zwUa%kdC=MzMS*vIe6>%k#WCW1xw9nyJ&|aBi#}io_30!rQS1FxJG0l%OdOFar`)er zC9J6v23({`7y?y&!#b%|;sfYu^QKf$ns0Mxqiap`NvLpa5nusKnI~Fmg0YEBp;>c3 zKlfCFo0zC5a=`;#5w_ir6Lp1TWx?e5=E)dCbU%xp`4lY4xrK<*>Lg?mG|02tiCvX_6U4xS=IC41`w@T|-=-5Q6%z8*jw=4{aop2xdxeB9VqIV< ziqRJ4LoN~$Wi7IcV`yAO+s^emw7Y)9_b9Z*7ej98WNNy1P+%c4QP#^{PR>_14{4+J zt!g)ZzoVd1OG?h86a&6&^$S{ZwW7x5md$mTB@_feUZoj)WDl?%ZAF3a|{9s*D8ky$>wmyE}j);2wdI^zNLF!31G_m3V(!R^@M9PcCV8Xs{o)*TsI( zvlyu$9A1gQGpJ~UZK(xN=ne)$jy!GknP1C};vomAN1`zLgDr9VE!h7(n@@wO3DN^L zJ{mo1;;9FturT-gc5r2jQU%bDu++Hyx|yH*($_8NsM`Q~$}wjA$jZxTRT$cEL}0E~3~?{tPqpbQ0b;h@OC6WF-s^4JzlT933&@Y-F#9IH zt=Z>s3nVfo;?U3G&T)X}`H*}ySsAtX*&1&zd1pFrwA&FN9O1R$YZG{8JdR)M8=x@H z!gI7O(jfNUB^;|f|9KCpWwt(ukn?LhH@i${kTaxR#tCkCpku1h+-YfLYo3S~gI~IHdN$5l>5D%(p99{h| zU6pL=aiS7e$n9b~eTi{EH}Lp5vF6S=y?m0u@t3a|XbCxh+p&AZTp}rcFgO$n%icq6 z9oUkEMoQdH=k;9Ou#Ug4`&M`W{X`CN+vqvZi0ju+pvp$xkn3%iAVvL+=+>J&fhotg z^;-2jUOLKg@pZAt6VKRi2pz(#GwVEpUCm%6e3)7L`dU=P+|Ev)Qf!fb@YoB0P< zV%kk*-|519@*j}4RHOOi8_G~N&&_O@iVZm#2ZoPKiIu6=!qK@51;RvohU$ zH#C`}AC;WsCrA)qr@2~@)zt;+PhEE>yF>rJ?vGjI$||J#Y{=Ewt|1t4uKi?t#T9I3 zaTwKzR8w|aw`aaVWK3<|T#e<(Bv3TR@yb@Wq{KVGDPP3#%N;+yb}>d({m->oM*Kz z9Mee74l_;UXq|C_loGy`r8?YqIOcs62^L5zFu?^_f{_F8 zS&b|&%{w1!ZV(F84x2fwD>OePLFAWz6Dr7Cf{O=`$D5bE=fC)L?ZWe9&&HceIPXuK zthu;zd?cv8E5v5)pkjziNO1ew7y+i3bgasaEyPpanNM zRd4k>%E0qeez*>=eR%dEDb`jm+UQ~Qc-e!6lfA0&t&ZGuai4;KaKicb6&5*{tl_fP zfO^n+^DhfY5zCJTmW4W|Y8$W*zjnK#DUfl4EBg->$piEWHy}Pl?6ih?%EyWE=W5Xla3v`cnujSVbI6AQ?1>V+BfX0D*>r`eYQnJ zPnar|KaS;aa`_K%?zW~P}6CTwgoyqI~(aVXi#5-+B<`oww zW;T{c0=FAzHn#=WifFB?q1GX&6Z59+^2Yv_p#sWp^GDw}(Mci$Zc##PN6Fxsg`1{Vs{!Q=>=T z0flXGu~@TqU0+{f3?&3y36?BXg8t?o`t-j2Vq8Vb={ofWevQ<15_@6BL6A4*-S^{v zw>=!584e#3{Hz^yQH=H=QS_{I8j0Wh0+L@R2;+y%vAI1Ho~37HwP0>3I1F2F?_5&k z2v+vDB?l9XwWbV2rpO@J_W&1#I&%iE1JUa$Yb`btu*vyH2^W_&fA=(j>4HlPj($D> zBfw$IEr%&2?!;W>hST~4rhf>^>wv$6v?Ds2Id>OCc;I58zZTp%y|B7GO;7OYsp7XF z&Oq{{(GUQ(s_yn#Z-Phe)_;j`r$-ZS0~!NJ!;_oyECdOr8sViQ05XDQTs+Zv->$LX z2urQP!mUG^Ka5AjD@jmuBE3!tKzkka7g{NMvev&QPYs9P3kg(y>dH(dw|k}7p%ZKb z2OYpj@oYgBp;~zt-rvzf@lw@WCgez97W^DpAs;iD=ps(I9D%GL%wjR3S+dO_WAd8bv#4(x6dArFqavBO)cuX%HzC z8Z^Dv6AtJ5{k`v6?;r0U?^$Op=U991=XpNQ=W`F&ecji6!hQ@4L~QW;F-MRJd}SZM z{Y{J_tXIiueu8lC{s2fvw5+tPw5o#IQSZ||@ohfIC}$fO5sBX>ftWyE?}Ip6rdiTu z0h%1XqX5$IVKB%D`1m_%%~IE010wxq*F7$(P?0h?unPvbUe%!!kf(ujWNztdt5#K-uc5AGX3QQM*Dm4hhh_*C(5m*)gAujJvCLh$j>7C2#;;O*~O+Gn!k) zKP;@N0n38PObFhsiy%=D~&^LK91?B-7csK2jt&%m>~HI%kJhB_9Q2>cS!+cciqF=sg##pt)nQ;Scg5^4!xOQ0I4<9zRibMUJ>qwZ_tyWAD<5 zt>OkwjaClKN9R|hn%QM0L4?T4-=O+fqI*_at?>|@n#q$?WwmXuK1`^6JDJfsqK7Ry z54|x`+%g;twwF$Id%;UQk>3FxkFQt`q3UAh)Q;%}LF;$G3Es-Ovr%NOA;X$_}_ENY2Hw+iX5J65OzD+SX*n+ zl(@`zFC!+-5FL>vUFBA-nVjf=!okebJuvy-t7Em=_R}Vk$D^NcMZo*wft$!Sm!)-# zLTh(w>xZwvN#T2DUlz0aAc!rw@XbH#_$HA#wHK%m!D6Kyy*G?c0^I;_ne!3yo1PMv zBN++ZFN~`YQAG!0Kpf)J2$rAgvY)1ziOdu?w-JDwcdbDm5l%k8?$==>HoF5R2 zfbyaiKDlt7F!=w&;shlAU^)<+C#Jx0lpX0Rkq+E)h?hHx-AJGtgx>5mD)UDXN6_PO zBdKweGE5dyi*2(bo--5$lng(z8>JJC%qUZ`10QTBuZFzc04gUb?U+rOf`0d=ZTqILo)OVCTcoH!~VE9-_f zP^(%SZthqlHe-?`^BR97-8(fazpNpdumN7o4li4-bPZygl%tXuRU($N8TBih<`gJM zR~|=Ep?cNy3X?S%gIY9k%Yg)u4iIPU*mqITHR8TyK*Gp)+eSv_c+Hgw3eW1in>;&E zJZu^55NX~aaHq-8zF6RVh<%FBz*X|TXG1RHEB_9;HElSWe)LRc+r{<o#$#omCK_ESk&YD99+g zVBeF1NV&>D6%{LQ>Y6euVAYZVGsIU6_mr38)6me@0B$?RXK?PRB-kiA#h&y~f5pcq zzx%4vM^lV0sFoz4R{i%_za!>jv0?OZNyjIUGba(lr1E*x5}(HAppSr4n6@hhIU>a^ zC?D5Ai+%TC+2AA`biv>b-gW4gVZ>O#@3!MkVTzd+uQG0Jx`%p4^2E;DQsY1l8uTxr z4apYWx5M;2m5gC=t8L1elA|5l-E3OdkX5-4W*S!At{*HDGB|niZu^i4_+UhLRfAi@ zC+s|9NC|UNDRONTll0K;E;HUEJ(@eyow>#Ix*UgoF-!uQvHZ;R%UlcbRYHC$e>M}P z>Vn-%S?PIdL?tV`7CVx71LK_f%4Ey1MNQ1*M;&%3-_(_jwm!*i~etiIB1_wA{fs7FF0xdVVO; zu-M}zBYwG}{RE;(yfNh6M`ogiU?{-3pybb5w>=nyziDA2aS-3oOmthl^dLmbuD-OS zL`6j*?7X&j_+@f%`;ZI@4Au~ykZK1KPqYu~U!v)jI-5!rw=Do76qW&~jQB znRaADEpi)~{4C%Cz(wq{fI8lH^gE{Vks^q%ka*DK{M}Y-^Z_W1Q&Y}%GB`onGg0P% zQjj|J7=F~hv%RxYx`v9eiC9}`VV(A_>B z-h)CT2onjrs684{PFC`nU;mOwVqx<>n)$ zzM^1UJn;7IL(7)^B^H*JY3;1X`u2hMj4^d~cH)S-Ixwz$7qz^H&!to<##=wuy-DTW z8497wsFMM-Mp^kvJF5cPek{6*@*2*9?4YNohnMTQ`g_t+dGkI?;#D=N_@2RJ>uIG+ zoz}oJEsg0#bhG8V$HDliC*E`W3ZCZ*CC!K)F^oV?L65kp`sGppX%5dQ2!J^vDXDFw zI1D`upv=uoBrl#D#GZ59Ao!l|Mqb02tkSpk?w!vvhnRqKq@gm`#|HQhdKETlT_i8T z-yb&5Yrw%)ySZ7w=nNPfmd2kkMC^Ut#@HJ;_2|zALYfgoVpQLtk44R9g0;JoYKS>` zp9=a{jwt{~)S2rN54kVwkQ`2? z&sykDS->Yu@CrWC!MxThvjv8=)}_(6ZcU@qQ-5tAyznG4Fc><8AT%%f(!yZTek)c5 ze-0$F+>}5<1|v^Q3asZ3V2BoU=FhHOb^r%}7^%dAdA(nmn%k4WnJ^{9Q_9h7lP2YWe>N3L` zukUun!E)bIFqu$v^6Xc1*Dqk}MDerX&RMF?g)N>IYNKK(PWDEg$z+-_$Tsn8XXOFv z;{s%UP$?01lvCPK9k+9wx~!Sxp<-~r3-9zhz4K@j3IX^3p7~gOXp}w#kuVU@BPA&9 z3mCc2IpnSP;9KGfTVCNweA;SsD@sF@nSRGd1(ihFd@9D}E^x>`0f;9)I5%|AftHla zFajgtu?@S8siJ)ll?!YdqHCgkC?Ca$9_Ri*!fe(AuCEa(_x#LULF~;MUK|#Wyaq$3 zE&)e}M%l>}&Px|z<1hoT>KTT|eCcxPN$nzWhah4P`*XxfYqxb}?rad!D z=r=)p_~jtrUv#fNgYWY3mKOn+PYM-oZg2pYFJGI=YCrxM=oI`MpJg2E;Gxkkf5U4j zePk{^uy3DkiSvCNc-KERer+-=a6AbI{93zR-m@54LPp4iNp(afTPKC&!$s`l&z<~u zYd*W564VW%A$Pe}1h&!vocJoa{{DWviBv}#`n@1o-Feiq_}LypXaFV$-q14E0b3LC ze{RX(q)2|}^_ZAU9Q2rfEPiayfnhYDZE{1eeTJhh$kaiY3aB@7pehkf0=5k{@`R+I zJm#go|2ww*1R@HoIojPcW`rYA8*+6md99CR5gMb(xQlc1!dMfZ6!gu|r-N$y8VS^WPp1Oj`QJD@l1{xAhBU>CuB0SA(sB zRKXByRNwr>wf@4|KQ32ov z9OR7()?faX>B1tj+`O^&lalgZ!Hg1b_03))RWQD5F7qZ&73fU^s z(|=w10(h-h63hQujkl-&>#v9f5=w;c8LSuEwW|n620Ifx)>m-f%`3bTz}4$ZVinen zbL1(njY^92^9g(Br*aDn#wI4|>713D}%zHds*^9p9s;+qVRFl3CPUvKr=3Ef+`NgPQs@QxAT<-l#LQ5~FDbka>l<%}2EVt;#{pN0=A$WVv*DRn z8BAH4=-+^j1!@pGPB)nYcE=$n2agqGf;g^xsXHI;7@+aCc3?1FIIchg(Eo@kFa(aR zP^v__v2`gLqKvr$;(-TLM~Ejq5?11mQtao94i6XXo<~!}noqBQvVTMN2Gd<{a&!%y zzmFdKGTv!RO%iJ?0;RUEcNbpnT_u~iy^t7f`?hp$aGZCYN*NLZwE4bD(byZ1btE}x ztNLyxDYdyh>f>qz0pt)Fx&(QAVqmJd#HWMH`gW#u4RaV+$oxe>B(1H#+8qc5M$CoZ z?4bnCLO{-kGKNx9D=xqz>~x(XNgaSytb!DdL z?migdn}L1Wa5k|wZwl-WBkDLnqd>d8d#}aoBvMpjHAT=P^R}lzW?}YAyatE&>%GU0 zMZSK6TvTZm4k^-gjhT7OUGCr)X5q>O*w^EnBEFM%Q`q2UF2alh@emLc^sfeF{YBwX zTA%#l(diYD>kIre{`Ji+n^Bjw6xROQ$9pitNow1+m!0Y4OhG*bEIQf{3+)H1R`IS~ za&88>L_&PLVzF~VWXJNt4bF(gcF&A#`_e13J2Ygd2ZRiIsfo z&R9B^NtyU;8O*rnnCo~+%?6v%qZC$`0ThxY0;#pC_}!trr7_DBFDP|sw%|p?jT`*z z2wzsbtmt8xeclDrxspp}>Jc5cfm4@5d^(yqi4gI{hndK&m?<~b3Lm;-LB4Td*cQv- zzd$IPOeew@>Y(c3h+S9LD>&^Psk4avf0<`A3drGg$pJ$;6 z1emc_u&KPb9YkONktT~mS9pti!MNLj_fXiFIs-!K_^Pk!nJbB0ykRc!EufLItpa{h(ywS2(^46b^IH}eY)&N1bK8Wp&9dV1dLjDM}xxU5Zv z5ElEq-fSrWgCpKXf)Hc_b5S3GvZik)4#DQpZVSEV?1lj(KjPyDNKUJd5Td!gEvcmM9 z1w>kv5})%o-m2ZWJ+hUhKSm1?(T-IUwI#cJmndbPAo%bAmIBG9S3#_roa6B0u0Qd8 z;0~9>?W9m4=7aIBAi1glByZ^~Lvx)=j?lTLpLh0LGh}V7%qP2A{oTa65rXgx#3JicsWp9zC|rJru%K+7;p#w7&o)#5JYa+H~K9&lGXkt<+c zIKEMiY!`tl8~4^5=zJ4Y?zV3bL5K3!F{ho+lG}s>)kAHW2c_Ew=QW>$VdnbmEJV8q z(V;+e5Hh8Ek|VSqd&03Qs6EnmEqRP{3adx)4OBHa@W3a6g{?34GaO3YWU}ur1SuLS z1#9g0+EQl59$*5yze|a87-|Q`>jWougI6(q>PsXqGbKZffrzM2XT|7TC8ia3HZqD^ zYuvnjdj>;4ZhvAn6AXG)G7aG{Mo!MNT!*uaWWRXWCjN7)7f^B=r=*^QGw8}a9p)mQ z?oab0P|BZjzJ}BAMOajfiy|z)N1Y)8?alT1ruOy^+9&CP$wFY9!A+aq)QEW|h|!!f z6R*vXc8LLGMXL9)Y~~{hm={V!0w5|`kJ#Jr;%FWaIdo~{cVGB`x-A&~rd<7J9oV3> zV!^r|wdUs*nEc7D6s-k*kX;l=$M^+n^x31DiI%wox963$6hJf+MXfo*N@A16*V{=< zo3Mk%g1R4`)eq*9MHN~&{2G#hi4U!x0QnDF99=Wy=c5PPZ8J8u7N&c>tASC8h@J-k z+TS7Z|D(8dI6?Yj!QRl5BbY$N&Es!?Tb$ zd?RR`9}BmTHZnb{5pJRB`A(96xN*q?yYLv0evnflDLR9~v08y7w5@=&5WQEszOKUN zB?f8%$TBlC1Iwrf%MNgdYo~pcj3C0?St-+497tL@ciA1Ith+!087@UV;E;{ z(*Au1kmsA;pmSMBT$p`s2!fVUKT4$N8fifbA&etpGZ^?9nUg@WfF)rvAB!P8pEV%w z*LC=hCuXze{i?LCRk-m&-~H=ZU7ZVpu$DX4$nPbtpfEUG zz1~JkrN97v9tE>|#WA=t8-Ae2xNZ>dH+#86U8K5>m}s<49M&xlC8vxG zVJ2rx{gJ}I_g9+T%ue{u*Tgz22~P9H&bcy>?vPDO$Ifxyh*BX^a|YwqnaiWMq&(RUt69#ZpF_0TzZI|+3RAd^v2{7jw! z)d(RTS5^5|!?;9Lr#XJ?T(@S8z!gHA=0-A7&%dw*!Kz(lh&jXfg`eu^fksOp>q-XsCPZn z2mrq`)}^7YPNN8nf5EUr;-*bFFi!)UK<6-s5xu60O|!Zb!QT<-On;g~ym2PK($A6T zB%q8ZEG;b;`MM?Qf=lg8s>P>8?h?s%qloN#R9o0yfr3bc$)HBr=RfjQ<@LU`K54_= zE54cNapkpsVXa|pKU9fwN%PZ8|5CiUB{@Gg`iXvFPDpbxu`yhDd+Ss{C^i+N2sZb& zv!kA~rcEeax2!xezD!vi<6S%pHLlkf6d+_ty=}N={UP37zLP(*()vt2OM~upGR~m3 zegIh(36F5`V&=#l2F~g`hRT*iB0iLPoCpN1;n93m;xAvF#<|C&6J502V`}M$;H@ z)>0;)^MRkb%q)D^^P(>m!IL*?lO#6iVEsZZ-)XQ3w#G-8UAIdv23~l(It^{~gAeQP zF~Kz2j^?%2i=I<2`u0j+sbTdYnd9s;9W;>`+0I8dvpTGb45n4*n(GVsj?s!0PGA`U zk%Xq@`V)%e8vZxj3sYCEK6MpO!mBBc zhc+HyepENxnMlXwEFa((92|>n39|ug9LS3k7-$V=4X4enyJm*cZ{wZiki8&*PBk2V z#H-aN?xxqpK|n8z#bgE~=1~yc;%NNgJJ*keZAWigSFdH812a_*S&hnPX{(IL{hflS zJ{=-O?SFDepD6xtOgH(1gMxO1I&Gdss^g`OWS%bqshC6AADVR%Gps_~#cTLO^&veQ z>)vZqcNR48@y1Kli5Q@a&c1fnGtxP{O8zP94el)i920;%1qbS7{16>QQL?X(-d9Ej$h)YmghhNq)T4b0UxI3K{^ z+XV#$H*9$GO5Y}0;&cnZ!!7w8<_Q}$J!?86@G48%ojsheW! zsF}R7UR4nFyu-!4!;E1j%hV1WCQ1iTBIJNYuurdBZ&PB2@-=P{IiY{iia4R46~M#A zBzmgw$dEQ<2t+Lw1`fi*Y%ZzjmBsOn1$hT;qb&ekJyB!C19&i+q%JE3AwjHcst-{| z7Fr-eDL#NBUBKK02JjTNF`tSwMbz>vSxFlzG|jSL_XhJ?jXuBz0pJLwK#>F$qDH8- zaRHU?B9Uyh0WZfnQZZ5sORls-^hdZIz`0PUl^nlLu+@N25nOpSbyDvtlwnc~Nj@aC zKpIH>6O(C?fKjHIkKo~uVGKRJ?bKJb%z=hfVT>XSR#2WkyIHCY9vj2x$X2hjkxui8 zEz9q6>;ONmY;K|hpn}8tiqs$FzJDH*6^lXywMc5w>vC!!32DRRaLXA$w%{;p?I3Yn z`BCq{!Y?jk_BjN`;F0V`Ooz~wJg%vwrG>$zk6mU_6vir^0%k{)oPI6j8Ds>|Y&wQN zU{nHYZ-j(=e!h2ADd2cW&J8amF{b`rmOB0t%Bo!FU5zSvjRYG79}ZoBnc{@PJSqQS03s+SkWE4#9&AW&Xr<93IH1H zbq}N8_Z<2b3^K@;V!+Q~1QY=!T7Xd2iX0>G5D@h>caW%kq`MZ2(LdL4UwC-<1ZuEg zChQUvtoBuh7eGeVh!RW4ty`(qq$$sTEjl|r&VxR>v*3o&k7GTMg#Hn%()eXoBLZbB zqB7ys`B5VIv1wMY;C;q!J+UA7%M(fjs^3JGO0+NlrPtU&^057JREeGLxr6xeTnh~x zJ;0uUC4`U`Gy_Q=+@<%kG7+>(@20)A+Um0lEjRejt)E7bi0C zhNS-bXCaq{`zrF=hwfA1Q;${9( z&HHuO=rceKE&p7(9K~Y^W79OF((hPDIK?LdclTBQEG_+9H!nzke8KqdcX-XPkO3tS z-*|~WL}b}JMMMsCk&62NwY&GwTcK(Rbq^cZ(okNdfLbl^WRd%bz{+D=5Zl}0_v-mH z;WYz;evVUzbZ(tXDyB{VA-=WC!Scd|A}k&)2tq%1;ZnyobPuDw7#s&Kq-ctFfo21a zaRyI`sPW>(K$Y&NnOI>(j6o%upE2nUjDz@43QikQ-oyDAfnXNaJcI8aOHrW#OmCs{ zTqDlH6C54H?5MuIgL-E{6>V=C3R`s>H%Iuo245Tu5o&(*S`m;Yxa@2BykSAk+TgtrCfBx~ z$s!aT@P5v#{Qlu?;)gR0(N^*rj-(CV5Cl#x=nlfk@JdVTcP;e@`fXEuIp8Rxx1BnO z(4^spkl8>ITo!_!_lOmHik-5GsHpw_WmDJ{s+HOko{dKCS#k|{N9QrF4SoV z>03kxA8_oTjHvOna{c}h^`%<&~h^BwV0yNHwR17G|9aH${?NiCb zlP>>AJb(FW5|@StQK&-|4M&Ih6#_h{gyv!5MIIkL7!QEiIev_+&Jz>+6eRtTaWygT z-M!7i&spjIRjBNxaMJG(mi+gP`(z!Gn^ue}#=%MF+>>Ko$2{hb{i!8g!p|EVh->9O z(){=RsYMiFvIin9fagdmTFOi;F+n3j94tSbc3Hvy?YA(zJ5-8j{kBvIK|ZKfVhwM0 z!*OACD4>h8+07Pr!2<2`A%>P%UJdhkyhroT$GvMV?1IRyC5Q+8X90nMB1bcb!FEqZ ztN2-hl9aa{50$U%;}Md7$@+dB&VrToSGF*j?Z+-C1(V-Hhk$7H z9bLgL@tVk=mj^x8Ot_W3kLS2u$e$a+CjlDsDp7g3U>c{+dO~W$sObFZofSOCP&cvW z5-tTXD?T4)>#cIIU#(d}l8!K2n4+Q;c$F6Fs;WW}%x`psB_o$zqh(%8r~1eKP?H+& z$j_1@AR468+ZUk&X#I|1H54aw;g(TpSHdyieCnM(!3DeAL4a-0FE@Yn0ZFx*WJJ12t!dt z@WHLA573RmzF+8Me&d0=ALo>Sa?9)2y3-FOeI>>hOIXd#&4dr zesYdg)V_pXX+LF?qB&*yVK3L}&>Cg6EQ_i{7}GTltKG%V`eZ-SG+W-^cNVY>ykr96 z`WR^^f=PvftLWq!!!KSYr@ztn4z&@C)BEX2*?_A+-b$gQ9w}Wbn1R#rzyydufmNjM zFo+=(gn&11Iw1bXG4bO*oQt5lA-7+i%lz}4n8K(Kdcfxg`o145hAx3*KU>jHn?`X;)b3m@)@PS(s~X-~2T*;eB2r&ppyqZ>^L zh1jH{@pV}d0j}nz-f@_;_;zkRk^Zw4n(daD8~Em2BGe8|I^7{S7FJA zA)n$9nWW>Z&?;DC*lOq${lZjDogfT1sD;d+a54CSU0?()O;yG)p8f?&#Y%;Bp#*Rh zAWJ*Upp@|H~WTv;&rqIuk52?%O_AGc*QW{Bp}d7lE&r zxEm{+S#z}wPbMEOYu>YT&61QrUsXBNpRPzv3RUQshB;epz*L7W1rGUbj1k}Qc3_}( z=zbA3Q<#zSnzscDKG_i(GfPnogLo;kA`o4i@h% zn>?s!s*E4+3UQCq@!VA#7f;G#BG&=Y{NY@esX$ohdh9l#f(H#9@=&ic@L4^!N*ix?}p9J-a z){1;m9CCNit#t}R*;L55Rrg($X`j9O)GJq^V#fdJ7LJS;@4}Hi+Ky-hS)^Ox$MuSF z?4Tme=ERa!2Qm@SBGurMbA8?U@#odK=z`hE3KIT*_f|U{_gOSYES#M8Ng@#H8v^xu(F7k^OsYhpZzy*tQJ&NM@MSY9 z%lBVyfj#d0K8aE_0tDqe=%+x(Am5A0E8(G`$J@W|aG!2Ashe^-j_?{s6)N6W*u<_s z+^Qam68!weYwp}`E|J*2WtS=!5<4YmP|%_p?U%Ew0Fbeb1(G;N@~d4H%wG@B zi(G&PUHo9UzI9vSxn%pZr;use&my{iI@(=3BDv% z0zoAJg0mBuB%2uM=4I1FJd3P)>u5o?_xT`OVR^S;)xHscYXg(3ZyAF-61c+6)iRs^ z)Hh%Tn?J1-Lq9-+jItfFA%wAGjnVUu`uPGbBPAQ;o+<`>#~z&VSwL|5L2Nc-P)@V* zV!Y>*is0EdPipgKOFXEhv3Q;PXXHH6c{H0?)B;7!_w2+}9~Y+T3?O(&e73H_BYww3 zao?bz?ZFahd(+A9QHbL0j&KsF`%tZL^k_I)mvAq0xzn#4Ponva{O9d2pc8kC3*llIYTfQNH%~>KY#E zh|XZ(j;OFS6ZI{!-`%4r)2z#Gk6`^Dno;FsHnt$ z&(P1$Oet3yg_8Fux(&e8-zh6=xbFQZPc$FL^20r51a|+}`09%F;qE4KM`PKAPdQjw zWfb0LL=E*u;rD2#ZgM_Uj*Y6tbvz>mUf_yVip<`~K`bQJa)rr^7yRo<39zYZcmZ!9 zm2*!B9OS$m^FqXEPr30Z%{&9%Xo;JZWJ-+7+jV2{;L~VG3!{-N^L6m}&RugpW>mu$ z)tTG0XSMlT6y&~Id`o>~e-sM#&n42t$L|Iq2H5O;I|Pj#bu<$w^`PI}anSwYj5bDD zADXc%u#HOE{sYt`Ai39-;StTbr(a@8r>AFFG*0!{{=K5F7wVUp6Ym9nJo-IQ7>6oVn&qwZ$(B%qKlK8+lJ!`|G``2 z5izst!T)`6-mhdK)bh-U@TRUFMzUD1d+fYXIbEV6rMe`tC6|%j_1tv)?SQzmRhN z+>7pkQ|9JuewhefMPtpbC(TJ2{A14+VF|0_d2zW7Xps|48vv$T2EPr}N4y#Y)1MY* z+!qdUCMW#3UO^neQoa;tjj+e%RV=%e!_kI+iZXSN_gC!%*`@AU)LZ8xwR{00qcv|T z?ZVrAf9LtmEI~_Bk~T82@+uacXJ zYnTh!((kx3Jciv$QgGQSna>^#b5nPrED(j@a>Nxo0tDe=yNb6Fxfl3JxZQn?#o&7UgW~iZVp4`X;pGICt**yn0A>nB8KJ}3f z6EUx9;3z{sQ}x@38}Lr(x>ymBt%1886+qG6`={Wmr- zA^$+~k7iY})WZCw{{AE0S-gUSh1`PDL}}!l-CsZeOZ$VQK_D{ilv6Bx87+0m&Zx>{ z>?KgR{Uu-luFiSz01&L9csI*9y_wpK-2?IHB%%S~C-(hkKE2@#1ys6IR6G^~lWSuJ zjrDkT|4s#B9BTN12+!YK!ty(c%{Z1_iJv|#Ftf-4k~6Un0Wm!|AhzD diff --git a/doc/design/references.bib b/doc/design/references.bib deleted file mode 100644 index 72357e832..000000000 --- a/doc/design/references.bib +++ /dev/null @@ -1,59 +0,0 @@ - -@Inproceedings{KougkasEtAl2018, - author = {Kougkas, Anthony and Devarajan, Hariharan and Sun, Xian-He}, - title = {Hermes: A Heterogeneous-aware Multi-tiered Distributed I/O Buffering System}, - booktitle = {Proceedings of the 27th International Symposium on High-Performance Parallel and Distributed Computing}, - series = {HPDC '18}, - year = {2018}, - isbn = {978-1-4503-5785-2}, - location = {Tempe, Arizona}, - pages = {219--230}, - numpages = {12}, - url = {http://doi.acm.org/10.1145/3208040.3208059}, - doi = {10.1145/3208040.3208059}, - acmid = {3208059}, - publisher = {ACM}, - address = {New York, NY, USA}, - keywords = {I/O buffering, burst buffers, deep memory hierarchy, heterogeneous buffering, layered buffering}, -} - -@Book{Bach1986, - author = {Bach, Maurice}, - title = {The design of the UNIX operating system}, - publisher = {Prentice-Hall}, - year = {1986}, - address = {Englewood Cliffs, N.J}, - isbn = {978-0132017992} - } - -@Book{Kerrisk2010, - author = {Kerrisk, Michael}, - title = {The Linux programming interface : a Linux and UNIX system programming handbook}, - publisher = {No Starch Press}, - year = {2010}, - address = {San Francisco}, - isbn = {978-1593272203} - } - -@Book{Maier2009, - author = {Maier, Mark}, - title = {The art of systems architecting}, - publisher = {CRC Press}, - year = {2009}, - address = {Boca Raton}, - isbn = {978-1420079135} - } - -@Report{Chaarawi2015, - author = {Chaarawi, Mohamad}, - title = {RFC: Page Buffering}, - institution = {The HDF Group}, - year = {2015} - } - -@Report{ChoiEtAl2017, - author = {Vailin Choi and Quincey Koziol and John Mainzer}, - title = {RFC: HDF5 File Space Management: Paged Aggregation}, - institution = {The HDF Group}, - year = {2017} - } \ No newline at end of file diff --git a/docker/ctest.Dockerfile b/docker/ctest.Dockerfile deleted file mode 100644 index 2e97b729e..000000000 --- a/docker/ctest.Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:20.04 -ARG DEBIAN_FRONTEND=noninteractive -RUN apt update -RUN apt-get install -y --no-install-recommends \ - gcc g++ gfortran \ - autoconf \ - automake \ - libtool \ - libtool-bin \ - mpich \ - lcov \ - zlib1g-dev \ - libsdl2-dev \ - python3 \ - python3-pip \ - git \ - cmake -RUN python3 -m pip install cpplint==1.5.4 - -RUN git clone https://github.com/HDFGroup/hermes -WORKDIR $PWD/hermes -RUN git remote get-url origin -RUN ci/install_deps.sh -RUN ci/install_hermes.sh diff --git a/docker/deps.Dockerfile b/docker/deps.Dockerfile deleted file mode 100644 index 4a171ff9f..000000000 --- a/docker/deps.Dockerfile +++ /dev/null @@ -1,76 +0,0 @@ -FROM ubuntu:20.04 - -ENV USER=hermes -RUN useradd -ms /bin/bash $USER -RUN su - $USER -c "touch me" - -RUN apt-get update -q --fix-missing && \ - apt-get install -yq gcc g++ - -RUN apt-get install -y sudo -RUN echo "${USER} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/$USER && \ - chmod 0440 /etc/sudoers.d/$USER - -RUN cat /etc/sudoers.d/$USER - -RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ - autoconf \ - automake \ - ca-certificates \ - curl \ - environment-modules \ - git \ - build-essential \ - python \ - python-dev \ - python3-dev \ - vim \ - sudo \ - unzip \ - cmake \ - lcov \ - zlib1g-dev \ - libsdl2-dev \ - gfortran \ - graphviz \ - doxygen - -USER $USER - -RUN sudo apt-get update -q - -ENV HOME=/home/$USER - -ENV PROJECT=$HOME/source -ENV INSTALL_DIR=$HOME/install -ENV SPACK_DIR=$HOME/spack -ENV MOCHI_DIR=$HOME/mochi - -RUN echo $INSTALL_DIR && mkdir -p $INSTALL_DIR - -RUN git clone https://github.com/spack/spack ${SPACK_DIR} -RUN git clone https://github.com/mochi-hpc/mochi-spack-packages.git ${MOCHI_DIR} -RUN git clone https://github.com/HDFGroup/hermes ${PROJECT} - -ENV spack=${SPACK_DIR}/bin/spack - -RUN . ${SPACK_DIR}/share/spack/setup-env.sh - -RUN $spack repo add ${MOCHI_DIR} -RUN $spack repo add $PROJECT/ci/hermes - -RUN $spack compiler find - -RUN $spack compiler list - -ENV HERMES_VERSION=master - -ENV HERMES_SPEC="hermes@${HERMES_VERSION}" -RUN $spack install --only dependencies ${HERMES_SPEC} - -RUN echo "export PATH=${SPACK_DIR}/bin:$PATH" >> /home/$USER/.bashrc -RUN echo ". ${SPACK_DIR}/share/spack/setup-env.sh" >> /home/$USER/.bashrc - -ENV PATH=${INSTALL_DIR}/bin:$PATH - -WORKDIR $HOME diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile deleted file mode 100644 index 6c9ee62e3..000000000 --- a/docker/dev.Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM hdfgroup/hermes-deps:latest - -RUN sudo apt-get update -q - -USER $USER - -ENV HOME=/home/$USER - -ENV INSTALL_DIR=$HOME/install - -ENV PROJECT=$HOME/hermes - -RUN $spack install --only package ${HERMES_SPEC} - -RUN echo "spack load --only dependencies ${HERMES_SPEC}" >> /home/$USER/.bashrc -RUN echo "export CC=`which mpicc`" >> /home/$USER/.bashrc -RUN echo "export CXX=`which mpicxx`" >> /home/$USER/.bashrc - -RUN git clone https://github.com/HDFGroup/hermes ${PROJECT} -RUN sh /home/$USER/.bashrc - -RUN mkdir -p $PROJECT/build -WORKDIR $PROJECT/build diff --git a/docker/user.Dockerfile b/docker/user.Dockerfile deleted file mode 100644 index c6cc026d8..000000000 --- a/docker/user.Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM hdfgroup/hermes-deps:latest - -RUN sudo apt-get update -q - -USER $USER - -ENV HOME=/home/$USER - -ENV INSTALL_DIR=$HOME/install - -ENV PROJECT=$HOME/hermes - -RUN $spack install --only package ${HERMES_SPEC} - -RUN echo "spack load ${HERMES_SPEC}" >> /home/$USER/.bashrc -RUN echo "export CC=`which mpicc`" >> /home/$USER/.bashrc -RUN echo "export CXX=`which mpicxx`" >> /home/$USER/.bashrc - -RUN sh /home/$USER/.bashrc - -WORKDIR $HOME diff --git a/hermes_shm/CMake/HermesShmConfig.cmake b/hermes_shm/CMake/HermesShmConfig.cmake deleted file mode 100644 index 579bf9c2d..000000000 --- a/hermes_shm/CMake/HermesShmConfig.cmake +++ /dev/null @@ -1,54 +0,0 @@ -# Find labstor header and library. -# - -# This module defines the following uncached variables: -# HermesShm_FOUND, if false, do not try to use labstor. -# HermesShm_INCLUDE_DIRS, where to find labstor.h. -# HermesShm_LIBRARIES, the libraries to link against to use the labstor library -# HermesShm_LIBRARY_DIRS, the directory where the labstor library is found. - -find_path( - HermesShm_INCLUDE_DIR - hermes_shm/hermes_shm.h -) - -if( HermesShm_INCLUDE_DIR ) - get_filename_component(HermesShm_DIR ${HermesShm_INCLUDE_DIR} PATH) - - #----------------------------------------------------------------------------- - # Find all packages needed by hermes_shm - #----------------------------------------------------------------------------- - find_library( - HermesShm_LIBRARY - NAMES hermes_shm_data_structures - ) - find_library(LIBRT rt) - if(NOT LIBRT) - message(FATAL_ERROR "librt is required for POSIX shared memory") - endif() - - #----------------------------------------------------------------------------- - # Mark hermes as found and set all needed packages - #----------------------------------------------------------------------------- - if( HermesShm_LIBRARY ) - set(HermesShm_LIBRARY_DIR "") - get_filename_component(HermesShm_LIBRARY_DIRS ${HermesShm_LIBRARY} PATH) - # Set uncached variables as per standard. - set(HermesShm_FOUND ON) - set(HermesShm_INCLUDE_DIRS ${HermesShm_INCLUDE_DIR}) - set(HermesShm_LIBRARIES -lrt -ldl ${HermesShm_LIBRARY}) - endif(HermesShm_LIBRARY) - -else(HermesShm_INCLUDE_DIR) - message(STATUS "FindHermesShm: Could not find hermes_shm.h") -endif(HermesShm_INCLUDE_DIR) - -if(HermesShm_FOUND) - if(NOT HermesShm_FIND_QUIETLY) - message(STATUS "FindHermesShm: Found both hermes_shm.h and libhermes_shm.a") - endif(NOT HermesShm_FIND_QUIETLY) -else(HermesShm_FOUND) - if(HermesShm_FIND_REQUIRED) - message(STATUS "FindHermesShm: Could not find hermes_shm.h and/or libhermes_shm.a") - endif(HermesShm_FIND_REQUIRED) -endif(HermesShm_FOUND) diff --git a/hermes_shm/CMake/UseDoxygenDoc.cmake b/hermes_shm/CMake/UseDoxygenDoc.cmake deleted file mode 100644 index b9601910b..000000000 --- a/hermes_shm/CMake/UseDoxygenDoc.cmake +++ /dev/null @@ -1,33 +0,0 @@ -find_package(Perl REQUIRED) -find_package(Doxygen REQUIRED) - -function(add_doxygen_doc) - set(options) - set(oneValueArgs BUILD_DIR DOXY_FILE TARGET_NAME COMMENT) - set(multiValueArgs) - - cmake_parse_arguments(DOXY_DOC - "${options}" - "${oneValueArgs}" - "${multiValueArgs}" - ${ARGN} - ) - - configure_file( - ${DOXY_DOC_DOXY_FILE} - ${DOXY_DOC_BUILD_DIR}/Doxyfile - @ONLY - ) - - add_custom_target(${DOXY_DOC_TARGET_NAME} - COMMAND - ${DOXYGEN_EXECUTABLE} Doxyfile - WORKING_DIRECTORY - ${DOXY_DOC_BUILD_DIR} - COMMENT - "Building ${DOXY_DOC_COMMENT} with Doxygen" - VERBATIM - ) - - message(STATUS "Added ${DOXY_DOC_TARGET_NAME} [Doxygen] target to build documentation") -endfunction() \ No newline at end of file diff --git a/hermes_shm/CMakeLists.txt b/hermes_shm/CMakeLists.txt deleted file mode 100644 index 5f76cd2d2..000000000 --- a/hermes_shm/CMakeLists.txt +++ /dev/null @@ -1,209 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Global variables -#------------------------------------------------------------------------------ -set(HERMES_SHM_ROOT ${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${HERMES_SHM_ROOT}/include) - -#------------------------------------------------------------------------------ -# Options -#------------------------------------------------------------------------------ -option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" ON) -option(BUILD_HSHM_TESTS "Build tests" ON) -option(BUILD_HSHM_BENCHMARKS "Build benchmarks" ON) -option(BUILD_MPI_TESTS "Build tests which depend on MPI" ON) -option(BUILD_OpenMP_TESTS "Build tests which depend on OpenMP" ON) -option(BUILD_Boost_TESTS "Build tests which depend on libboost" ON) -option(HERMES_RPC_THALLIUM "Build tests which depend on thallium" ON) -option(HERMES_ENABLE_COVERAGE "Check how well tests cover code" OFF) -option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" ON) -option(HERMES_CXX_PROFILE "Generate profiling data from benchmarks" OFF) -option(HERMES_PTHREADS_ENABLED "Support spawning pthreads" ON) -option(HERMES_DEBUG_LOCK "Used for debugging locks" OFF) -option(HERMES_ENABLE_PROFILING "View profiling logs" OFF) - -if (HERMES_PTHREADS_ENABLED) - add_compile_definitions(HERMES_PTHREADS_ENABLED) -endif() -if (HERMES_RPC_THALLIUM) - add_compile_definitions(HERMES_RPC_THALLIUM) -endif() -if (HERMES_DEBUG_LOCK) - add_compile_definitions(HERMES_DEBUG_LOCK) -endif() - -#------------------------------------------------------------------------------ -# Setup CMake Environment -#------------------------------------------------------------------------------ -if(NOT HERMES_EXTERNALLY_CONFIGURED) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables.") - set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries") - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries.") -endif() - -#------------------------------------------------------------------------------ -# Setup install and output Directories -#------------------------------------------------------------------------------ -if(NOT HERMES_INSTALL_BIN_DIR) - set(HERMES_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) -endif() -if(NOT HERMES_INSTALL_LIB_DIR) - set(HERMES_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) -endif() -if(NOT HERMES_INSTALL_INCLUDE_DIR) - set(HERMES_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) -endif() -if(NOT HERMES_INSTALL_DATA_DIR) - set(HERMES_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) -endif() - -#------------------------------------------------------------------------------ -# CMake Modules -#------------------------------------------------------------------------------ -if (NOT IS_HERMES_MAIN) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/CMake) - message(${CMAKE_MODULE_PATH}) -endif() - -#------------------------------------------------------------------------------ -# Optimization -#------------------------------------------------------------------------------ -set(CMAKE_CXX_STANDARD 17) -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - message("IN DEBUG MODE") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") -elseif(CMAKE_BUILD_TYPE STREQUAL "Release") - message("IN RELEASE MODE") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O3") -endif() -if(HERMES_CXX_PROFILE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") - message("CXX PROFILER IS ON") -endif() -#function(make_gprof exec_name exec_dir) -# add_custom_target( -# ${exec_name}_gprof -# COMMAND gprof -b -A -p -q ${exec_dir}/${exec_name} gmon.out) -#endfunction() - -#------------------------------------------------------------------------------ -# External libraries -#------------------------------------------------------------------------------ - -# Catch2 -find_package(Catch2 3.0.1 REQUIRED) -message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") - -# MPICH -if(BUILD_MPI_TESTS) - find_package(MPI REQUIRED COMPONENTS C CXX) - message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") -endif() - -# OpenMP -#if(BUILD_OpenMP_TESTS) -find_package(OpenMP REQUIRED COMPONENTS C CXX) -message(STATUS "found omp.h at ${OpenMP_CXX_INCLUDE_DIRS}") -#endif() - -# thallium -if(HERMES_RPC_THALLIUM) - find_package(thallium CONFIG REQUIRED) - if(thallium_FOUND) - message(STATUS "found thallium at ${thallium_DIR}") - endif() -endif() - -# Boost -if(BUILD_Boost_TESTS) - find_package(Boost REQUIRED) - message(STATUS "found boost.h at ${Boost_INCLUDE_DIRS}") -endif() - -#------------------------------------------------------------------------------ -# Code Coverage -#------------------------------------------------------------------------------ -if (NOT IS_HERMES_MAIN) - if(HERMES_ENABLE_COVERAGE) - set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage" CACHE STRING - "Flags to the coverage program to perform coverage inspection") - mark_as_advanced(COVERAGE_FLAGS) - - macro(set_coverage_flags target) - set_target_properties(${target} - PROPERTIES - COMPILE_FLAGS ${COVERAGE_FLAGS} - LINK_FLAGS ${COVERAGE_FLAGS} - ) - endmacro() - endif() -endif() - -#------------------------------------------------------------------------------ -# Documentation -#------------------------------------------------------------------------------ -if (NOT IS_HERMES_MAIN) - if(HERMES_ENABLE_DOXYGEN) - include(UseDoxygenDoc) - add_doxygen_doc( - BUILD_DIR - ${CMAKE_BINARY_DIR}/bin/_build - DOXY_FILE - ${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in - TARGET_NAME - dox - COMMENT - "HTML documentation" - ) - endif() -endif() - -#------------------------------------------------------------------------------ -# Build hermes_shm -#------------------------------------------------------------------------------ -add_subdirectory(src) - -if (NOT IS_HERMES_MAIN) - add_custom_target(lint COMMAND bash ${HERMES_SHM_ROOT}/scripts/lint.sh ${HERMES_SHM_ROOT}) - add_custom_target(preamble COMMAND python3 ${HERMES_SHM_ROOT}/scripts/preamble.py ${HERMES_SHM_ROOT}) -endif() - -#------------------------------------------------------------------------------ -# Build tests + benchmarks -#------------------------------------------------------------------------------ -set(TEST_MAIN ${HERMES_SHM_ROOT}/test/unit) -enable_testing() -if (BUILD_HSHM_TESTS) - message("Building HSHM unit tests") - add_subdirectory(test) -endif() -if (BUILD_HSHM_BENCHMARKS) - message("Building HSHM benchmarks") - add_subdirectory(benchmark) -endif() -#add_subdirectory(example) - -#------------------------------------------------------------------------------ -# Install hshm -#------------------------------------------------------------------------------ -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) - -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/CMake/HermesShmConfig.cmake - ${PROJECT_BINARY_DIR}/CMakeFiles/HermesShmConfig.cmake @ONLY -) - -install( - FILES - ${PROJECT_BINARY_DIR}/CMakeFiles/HermesShmConfig.cmake - DESTINATION - ${CMAKE_INSTALL_PREFIX}/cmake -) diff --git a/hermes_shm/COPYING b/hermes_shm/COPYING deleted file mode 100644 index bb9689a91..000000000 --- a/hermes_shm/COPYING +++ /dev/null @@ -1,52 +0,0 @@ -Copyright Notice and License Terms for -Hermes I/O Buffering Software Library and Utilities - ------------------------------------------------------------------------------- - -Hermes I/O Buffering Software Library and Utilities -Copyright 2018-2021, The HDF Group and Illinois Institute of Technology - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of The HDF Group, Illinois Institute of Technology, nor - the names of Contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -DISCLAIMER: - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -You are under no obligation whatsoever to provide any bug fixes, patches, or -upgrades to the features, functionality or performance of the source code -("Enhancements") to anyone; however, if you choose to make your Enhancements -available either publicly, or directly to the copyright holders, without -imposing a separate written license agreement for such Enhancements, then you -hereby grant the following license: a non-exclusive, royalty-free perpetual -license to install, use, modify, prepare derivative works, incorporate into -other computer software, distribute, and sublicense such enhancements or -derivative works thereof, in binary and source code form. - ------------------------------------------------------------------------------- - -Hermes was developed with support from the National Science Foundation (NSF) -under award NSF OCI-1835764. diff --git a/hermes_shm/CPPLINT.cfg b/hermes_shm/CPPLINT.cfg deleted file mode 100644 index f7d7492eb..000000000 --- a/hermes_shm/CPPLINT.cfg +++ /dev/null @@ -1,2 +0,0 @@ -set noparent -filter=-build/header_guard,-build/include_subdir,-runtime/references,-readability/casting,-runtime/int,-build/include,-build/c++11,-build/c++14 \ No newline at end of file diff --git a/hermes_shm/benchmark/CMakeLists.txt b/hermes_shm/benchmark/CMakeLists.txt deleted file mode 100644 index 9533d420f..000000000 --- a/hermes_shm/benchmark/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -add_subdirectory(data_structure) -add_subdirectory(allocator) -add_subdirectory(lock) \ No newline at end of file diff --git a/hermes_shm/benchmark/allocator/CMakeLists.txt b/hermes_shm/benchmark/allocator/CMakeLists.txt deleted file mode 100644 index 0858bdd58..000000000 --- a/hermes_shm/benchmark/allocator/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -include_directories( ${Boost_INCLUDE_DIRS} ) -include_directories( ${TEST_MAIN} ) -add_executable(benchmark_allocators - ${TEST_MAIN}/main.cc - test_init.cc - allocator.cc -) -add_dependencies(benchmark_allocators hermes_shm_data_structures) -target_link_libraries(benchmark_allocators - hermes_shm_data_structures - Catch2::Catch2 - MPI::MPI_CXX - OpenMP::OpenMP_CXX - ${Boost_LIBRARIES}) diff --git a/hermes_shm/benchmark/allocator/allocator.cc b/hermes_shm/benchmark/allocator/allocator.cc deleted file mode 100644 index 99acbce9c..000000000 --- a/hermes_shm/benchmark/allocator/allocator.cc +++ /dev/null @@ -1,272 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "omp.h" - -#include -#include "hermes_shm/data_structures/ipc/string.h" - -/** Test cases for the allocator */ -class AllocatorTestSuite { - public: - std::string alloc_type_; - Allocator *alloc_; - Timer timer_; - - /**==================================== - * Test Runner - * ===================================*/ - - /** Constructor */ - AllocatorTestSuite(AllocatorType alloc_type, Allocator *alloc) - : alloc_(alloc) { - switch (alloc_type) { - case AllocatorType::kStackAllocator: { - alloc_type_ = "hipc::StackAllocator"; - break; - } - case AllocatorType::kMallocAllocator: { - alloc_type_ = "hipc::MallocAllocator"; - break; - } - case AllocatorType::kFixedPageAllocator: { - alloc_type_ = "hipc::FixedPageAllocator"; - break; - } - case AllocatorType::kScalablePageAllocator: { - alloc_type_ = "hipc::ScalablePageAllocator"; - break; - } - default: { - HELOG(kFatal, "Could not find this allocator type"); - break; - } - } - } - - /**==================================== - * Test Cases - * ===================================*/ - - /** Allocate and Free a single size in a single loop */ - void AllocateAndFreeFixedSize(size_t count, size_t size) { - StartTimer(); - for (size_t i = 0; i < count; ++i) { - Pointer p = alloc_->Allocate(size); - alloc_->Free(p); - } - StopTimer(); - - TestOutput("AllocateAndFreeFixedSize", size, count, timer_); - } - - /** Allocate a fixed size in a loop, and then free in another loop */ - void AllocateThenFreeFixedSize(size_t count, size_t size) { - StartTimer(); - std::vector cache(count); - for (size_t i = 0; i < count; ++i) { - cache[i] = alloc_->Allocate(size); - } - for (size_t i = 0; i < count; ++i) { - alloc_->Free(cache[i]); - } - StopTimer(); - - TestOutput("AllocateThenFreeFixedSize", count, size, timer_); - } - - void seq(std::vector &vec, size_t rep, size_t count) { - for (size_t i = 0; i < count; ++i) { - vec.emplace_back(rep); - } - } - - /** Allocate a window of pages, free the window. Random page sizes. */ - void AllocateAndFreeRandomWindow(size_t count) { - size_t min_page = 64; - size_t max_page = MEGABYTES(1); - std::mt19937 rng(23522523); - std::vector sizes_; - - seq(sizes_, 64, MEGABYTES(1) / 64); - seq(sizes_, 190, MEGABYTES(1) / 190); - seq(sizes_, KILOBYTES(1), MEGABYTES(1) / KILOBYTES(1)); - seq(sizes_, KILOBYTES(4), MEGABYTES(8) / KILOBYTES(4)); - seq(sizes_, KILOBYTES(32), MEGABYTES(4) / KILOBYTES(4)); - seq(sizes_, MEGABYTES(1), MEGABYTES(64) / MEGABYTES(1)); - std::shuffle(std::begin(sizes_), std::end(sizes_), rng); - std::vector window(sizes_.size()); - size_t num_windows = 500; - - StartTimer(); - for (size_t w = 0; w < num_windows; ++w) { - for (size_t i = 0; i < sizes_.size(); ++i) { - auto &size = sizes_[i]; - window[i] = alloc_->Allocate(size); - } - for (size_t i = 0; i < sizes_.size(); ++i) { - alloc_->Free(window[i]); - } - } - StopTimer(); - - TestOutput("AllocateAndFreeRandomWindow", 0, count, timer_); - } - - /**==================================== - * Test Helpers - * ===================================*/ - - void StartTimer() { - int rank = omp_get_thread_num(); - if (rank == 0) { - timer_.Reset(); - timer_.Resume(); - } -#pragma omp barrier - } - - void StopTimer() { -#pragma omp barrier - int rank = omp_get_thread_num(); - if (rank == 0) { - timer_.Pause(); - } - } - - /** The CSV test case */ - void TestOutput(const std::string &test_name, size_t obj_size, - size_t count_per_rank, Timer &t) { - int rank = omp_get_thread_num(); - if (rank != 0) { return; } - int nthreads = omp_get_num_threads(); - double count = (double) count_per_rank * nthreads; - HILOG(kInfo, "{},{},{},{},{},{},{}", - test_name, - alloc_type_, - obj_size, - t.GetMsec(), - nthreads, - count, - count / t.GetMsec()); - } - - /** Print the CSV output */ - static void PrintTestHeader() { - HILOG(kInfo, "test_name,alloc_type,obj_size,msec,nthreads,count,KOps"); - } -}; - -/** The minor number to use for allocators */ -static int minor = 1; -const std::string shm_url = "test_allocators"; - -/** Create the allocator + backend for the test */ -template -Allocator* Pretest(MemoryBackendType backend_type, - Args&& ...args) { - int rank = omp_get_thread_num(); - allocator_id_t alloc_id(0, minor); - Allocator *alloc; - auto mem_mngr = HERMES_MEMORY_MANAGER; - - if (rank == 0) { - // Create the allocator + backend - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator( - shm_url, alloc_id, 0, std::forward(args)...); - } -#pragma omp barrier - - alloc = mem_mngr->GetAllocator(alloc_id); - if (alloc == nullptr) { - HELOG(kFatal, "Failed to find the memory allocator?") - } - return alloc; -} - -/** Destroy the allocator + backend from the test */ -void Posttest() { - int rank = omp_get_thread_num(); -#pragma omp barrier - if (rank == 0) { - allocator_id_t alloc_id(0, minor); - HERMES_MEMORY_MANAGER->UnregisterAllocator( - alloc_id); - HERMES_MEMORY_MANAGER->DestroyBackend(shm_url); - minor += 1; - } -# pragma omp barrier -} - -/** A series of allocator benchmarks for a particular thread */ -template -void AllocatorTest(AllocatorType alloc_type, - MemoryBackendType backend_type, - Args&& ...args) { - Allocator *alloc = Pretest( - backend_type, std::forward(args)...); - size_t count = (1 << 20); - // Allocate many and then free many - /*AllocatorTestSuite(alloc_type, alloc).AllocateThenFreeFixedSize( - count, KILOBYTES(1));*/ - // Allocate and free immediately - /*AllocatorTestSuite(alloc_type, alloc).AllocateAndFreeFixedSize( - count, KILOBYTES(1));*/ - if (alloc_type != AllocatorType::kStackAllocator) { - // Allocate and free randomly - AllocatorTestSuite(alloc_type, alloc).AllocateAndFreeRandomWindow( - count); - } - Posttest(); -} - -/** Test different allocators on a particular thread */ -void FullAllocatorTestPerThread() { - // Scalable page allocator - AllocatorTest( - AllocatorType::kScalablePageAllocator, - MemoryBackendType::kPosixShmMmap); - // Malloc allocator - AllocatorTest( - AllocatorType::kMallocAllocator, - MemoryBackendType::kNullBackend); - // Stack allocator - AllocatorTest( - AllocatorType::kStackAllocator, - MemoryBackendType::kPosixShmMmap); -} - -/** Spawn multiple threads and run allocator tests */ -void FullAllocatorTestThreaded(int nthreads) { - omp_set_dynamic(0); -#pragma omp parallel num_threads(nthreads) - { -#pragma omp barrier - FullAllocatorTestPerThread(); -#pragma omp barrier - } -} - -TEST_CASE("AllocatorBenchmark") { - AllocatorTestSuite::PrintTestHeader(); - FullAllocatorTestThreaded(8); - /*FullAllocatorTestThreaded(2); - FullAllocatorTestThreaded(4); - FullAllocatorTestThreaded(8); - FullAllocatorTestThreaded(16);*/ -} diff --git a/hermes_shm/benchmark/allocator/test_init.h b/hermes_shm/benchmark/allocator/test_init.h deleted file mode 100644 index f18ca220f..000000000 --- a/hermes_shm/benchmark/allocator/test_init.h +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ -#define HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ - -#include -#include -#include -#include -#include - -#include - -#include "hermes_shm/data_structures/data_structure.h" -#include - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::Pointer; - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -using Timer = hshm::HighResMonotonicTimer; - -extern const std::string shm_url; - -#endif //HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ diff --git a/hermes_shm/benchmark/data_structure/CMakeLists.txt b/hermes_shm/benchmark/data_structure/CMakeLists.txt deleted file mode 100644 index 078326c0c..000000000 --- a/hermes_shm/benchmark/data_structure/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -include_directories( ${Boost_INCLUDE_DIRS} ) -include_directories( ${TEST_MAIN} ) -add_executable(benchmark_data_structures - ${TEST_MAIN}/main.cc - test_init.cc - atomic.cc - ref.cc - string.cc - list.cc - vector.cc - unordered_map.cc - queue.cc - lock.cc -) -add_dependencies(benchmark_data_structures hermes_shm_data_structures) -target_link_libraries(benchmark_data_structures - hermes_shm_data_structures - Catch2::Catch2 - MPI::MPI_CXX - OpenMP::OpenMP_CXX - ${Boost_LIBRARIES}) - -# make_gprof(benchmark_data_structures ${CMAKE_CURRENT_BINARY_DIR}) \ No newline at end of file diff --git a/hermes_shm/benchmark/data_structure/atomic.cc b/hermes_shm/benchmark/data_structure/atomic.cc deleted file mode 100644 index 6595124c4..000000000 --- a/hermes_shm/benchmark/data_structure/atomic.cc +++ /dev/null @@ -1,248 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -#include -#include "hermes_shm/data_structures/ipc/string.h" -#include "omp.h" - -/** Stringstream for storing test output */ -std::stringstream ss; - -/** Print the CSV output */ -static void PrintTestOutput() { - std::cout << ss.str() << std::endl; -} - -/** Number of tests executed */ -int test_count = 0; - -/** Global atomic counter */ -template -struct GlobalAtomicCounter { - static AtomicT count_; -}; -#define GLOBAL_COUNTER(Atomic, T)\ - template<>\ - Atomic GlobalAtomicCounter>::count_ = Atomic(0); -GLOBAL_COUNTER(std::atomic, uint16_t) -GLOBAL_COUNTER(std::atomic, uint32_t) -GLOBAL_COUNTER(std::atomic, uint64_t) -GLOBAL_COUNTER(hipc::nonatomic, uint16_t) -GLOBAL_COUNTER(hipc::nonatomic, uint32_t) -GLOBAL_COUNTER(hipc::nonatomic, uint64_t) - -/** Atomic Test Suite */ -template -class AtomicInstructionTestSuite { - public: - std::string memory_order_; - std::string atomic_type_; - void *ptr_; - - public: - ///////////////// - /// Test Cases - ///////////////// - - /** Constructor */ - AtomicInstructionTestSuite() { - if constexpr(MemoryOrder == std::memory_order_relaxed) { - memory_order_ = "std::memory_order_relaxed"; - } else if constexpr(MemoryOrder == std::memory_order_consume) { - memory_order_ = "std::memory_order_consume"; - } else if constexpr(MemoryOrder == std::memory_order_acquire) { - memory_order_ = "std::memory_order_acquire"; - } else if constexpr(MemoryOrder == std::memory_order_release) { - memory_order_ = "std::memory_order_release"; - } else if constexpr(MemoryOrder == std::memory_order_acq_rel) { - memory_order_ = "std::memory_order_acq_rel"; - } else if constexpr(MemoryOrder == std::memory_order_seq_cst) { - memory_order_ = "std::memory_order_seq_cst"; - } - - if constexpr(std::is_same_v, AtomicT>) { - atomic_type_ = "std::atomic"; - } else if constexpr(std::is_same_v, AtomicT>) { - atomic_type_ = "non-atomic"; - } - } - - /** Atomic Increment */ - void AtomicIncrement(size_t count) { - Timer t; -#pragma omp barrier - t.Resume(); - for (size_t i = 0; i < count; ++i) { - GlobalAtomicCounter::count_ += 1; - } -#pragma omp barrier - t.Pause(); - TestOutput("AtomicIncrement", count, t); - } - - /** Atomic Fetch Add */ - void AtomicFetchAdd(size_t count) { - Timer t; -#pragma omp barrier - t.Resume(); - for (size_t i = 0; i < count; ++i) { - GlobalAtomicCounter::count_.fetch_add(1, MemoryOrder); - } -#pragma omp barrier - t.Pause(); - TestOutput("AtomicFetchAdd", count, t); - } - - /** Atomic Assign */ - void AtomicAssign(size_t count) { - Timer t; -#pragma omp barrier - t.Resume(); - for (size_t i = 0; i < count; ++i) { - GlobalAtomicCounter::count_ = i; - } -#pragma omp barrier - t.Pause(); - TestOutput("AtomicAssign", count, t); - } - - /** Atomic Exchange */ - void AtomicExchange(size_t count) { - Timer t; -#pragma omp barrier - t.Resume(); - for (size_t i = 0; i < count; ++i) { - GlobalAtomicCounter::count_.exchange(i, MemoryOrder); - } -#pragma omp barrier - t.Pause(); - TestOutput("AtomicExchange", count, t); - } - - /** Atomic Fetch */ - void AtomicFetch(size_t count) { - Timer t; -#pragma omp barrier - t.Resume(); - for (size_t i = 0; i < count; ++i) { - size_t x = GlobalAtomicCounter::count_.load(MemoryOrder); - USE(x); - } -#pragma omp barrier - t.Pause(); - TestOutput("AtomicFetch", count, t); - } - - ///////////////// - /// Test Output - ///////////////// - - /** The CSV header */ - void TestOutputHeader() { - ss << "test_name" << "," - << "atomic_type" << "," - << "type_size" << "," - << "atomic_size" << "," - << "memory_order" << "," - << "count" << "," - << "nthreads" << "," - << "MOps" << "," - << "time" << std::endl; - } - - /** The CSV test case */ - void TestOutput(const std::string &test_name, size_t count, Timer &t) { - int rank = omp_get_thread_num(); - if (rank != 0) { return; } - if (test_count == 0) { - TestOutputHeader(); - } - size_t nthreads = omp_get_num_threads(); - ss << test_name << "," - << atomic_type_ << "," - << sizeof(T) << "," - << sizeof(AtomicT) << "," - << memory_order_ << "," - << count << "," - << omp_get_num_threads() << "," - << count * nthreads / t.GetUsec() << "," - << t.GetUsec() << std::endl; - ++test_count; - } -}; - -/** - * Compare different atomic instructions on a particular thread. - * Given the atomic type and memory order. - * */ -template -void TestAtomicInstructionsPerThread() { - size_t count = (1<<20); - AtomicInstructionTestSuite().AtomicIncrement(count); - AtomicInstructionTestSuite().AtomicFetchAdd(count); - AtomicInstructionTestSuite().AtomicAssign(count); - AtomicInstructionTestSuite().AtomicExchange(count); - AtomicInstructionTestSuite().AtomicFetch(count); -} - -/** - * Compare the differences between different memory orders on a particular - * thread. Given the atomic type. - * */ -template -void TestMemoryOrdersPerThread() { - TestAtomicInstructionsPerThread(); - TestAtomicInstructionsPerThread(); - TestAtomicInstructionsPerThread(); - TestAtomicInstructionsPerThread(); - TestAtomicInstructionsPerThread(); - TestAtomicInstructionsPerThread(); -} - -/** - * Compare the differences between different atomic types on a particular - * thread. - * */ -void TestAtomicTypes() { - TestMemoryOrdersPerThread, uint16_t>(); - TestMemoryOrdersPerThread, uint32_t>(); - TestMemoryOrdersPerThread, uint64_t>(); - - TestMemoryOrdersPerThread, uint16_t>(); - TestMemoryOrdersPerThread, uint32_t>(); - TestMemoryOrdersPerThread, uint64_t>(); -} - -/** - * Compare the differences between different memory orders and atomic - * instructions with multiple threads. Given number of threads. - * */ -void TestAtomicInstructions(int nthreads) { - omp_set_dynamic(0); -#pragma omp parallel num_threads(nthreads) - { -#pragma omp barrier - TestAtomicTypes(); -#pragma omp barrier - } -} - -TEST_CASE("AtomicInstructionTest") { - TestAtomicInstructions(1); - TestAtomicInstructions(2); - TestAtomicInstructions(4); - TestAtomicInstructions(8); - PrintTestOutput(); -} diff --git a/hermes_shm/benchmark/data_structure/list.cc b/hermes_shm/benchmark/data_structure/list.cc deleted file mode 100644 index ee384fd2e..000000000 --- a/hermes_shm/benchmark/data_structure/list.cc +++ /dev/null @@ -1,269 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -// Boost interprocess -#include - -// Std -#include -#include - -// hermes -#include "hermes_shm/data_structures/ipc/string.h" -#include -#include - -template -using bipc_list = bipc::list::alloc_t>; - -/** - * A series of performance tests for vectors - * OUTPUT: - * [test_name] [vec_type] [internal_type] [time_ms] - * */ -template, ListT*)> -class ListTest { - public: - std::string list_type_; - std::string internal_type_; - ListT *lp_; - ListTPtr list_ptr_; - void *ptr_; - - /**==================================== - * Test Runner - * ===================================*/ - - /** Test case constructor */ - ListTest() { - if constexpr(std::is_same_v, ListT>) { - list_type_ = "std::list"; - } else if constexpr(std::is_same_v, ListT>) { - list_type_ = "hipc::list"; - } else if constexpr(std::is_same_v, ListT>) { - list_type_ = "bipc_list"; - } else if constexpr(std::is_same_v, ListT>) { - list_type_ = "hipc::slist"; - } else { - HELOG(kFatal, "none of the list tests matched") - } - internal_type_ = InternalTypeName::Get(); - } - - /** Run the tests */ - void Test() { - size_t count = 10000; - AllocateTest(count); - EmplaceTest(count); - ForwardIteratorTest(count); - CopyTest(count); - MoveTest(count); - } - - /**==================================== - * Tests - * ===================================*/ - - /** Test performance of list allocation */ - void AllocateTest(size_t count) { - Timer t; - t.Resume(); - for (size_t i = 0; i < count; ++i) { - Allocate(); - Destroy(); - } - t.Pause(); - - TestOutput("Allocate", t); - } - - /** Emplace after reserving enough space */ - void EmplaceTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - t.Resume(); - Emplace(count); - t.Pause(); - - TestOutput("FixedEmplace", t); - Destroy(); - } - - /** Iterator performance */ - void ForwardIteratorTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - Emplace(count); - - t.Resume(); - int i = 0; - for (auto &x : *lp_) { - USE(x); - ++i; - } - t.Pause(); - - TestOutput("ForwardIterator", t); - Destroy(); - } - - /** Copy performance */ - void CopyTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(ListT)) { - auto vec2 = hipc::make_uptr(*lp_); - USE(vec2); - } else { - ListT vec2(*lp_); - USE(vec2); - } - t.Pause(); - - TestOutput("Copy", t); - Destroy(); - } - - /** Move performance */ - void MoveTest(size_t count) { - Timer t; - - Allocate(); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(ListT)) { - auto vec2 = hipc::make_uptr(std::move(*lp_)); - USE(vec2) - } else { - ListT vec2(*lp_); - USE(vec2) - } - t.Pause(); - - TestOutput("Move", t); - Destroy(); - } - - private: - /**==================================== - * Helpers - * ===================================*/ - - /** Output as CSV */ - void TestOutput(const std::string &test_name, Timer &t) { - HIPRINT("{},{},{},{}\n", - test_name, list_type_, internal_type_, t.GetMsec()) - } - - /** Get element at position i */ - void Get(size_t i) { - if constexpr(std::is_same_v>) { - T &x = (*lp_)[i]; - USE(x); - } else if constexpr(std::is_same_v>) { - T &x = (*lp_)[i]; - USE(x); - } else if constexpr(std::is_same_v>) { - T &x = (*lp_)[i]; - USE(x); - } else if constexpr(std::is_same_v>) { - T &x = (*lp_)[i]; - USE(x); - } - } - - /** Emplace elements into the list */ - void Emplace(size_t count) { - StringOrInt var(124); - for (size_t i = 0; i < count; ++i) { - if constexpr(std::is_same_v>) { - lp_->emplace_back(var.Get()); - } else if constexpr(std::is_same_v>) { - lp_->emplace_back(var.Get()); - } else if constexpr(std::is_same_v>) { - lp_->emplace_back(var.Get()); - } else if constexpr(std::is_same_v>) { - lp_->emplace_back(var.Get()); - } - } - } - - /** Allocate an arbitrary list for the test cases */ - void Allocate() { - if constexpr(std::is_same_v>) { - list_ptr_ = hipc::make_mptr(); - lp_ = list_ptr_.get(); - } if constexpr(std::is_same_v>) { - list_ptr_ = hipc::make_mptr(); - lp_ = list_ptr_.get(); - } else if constexpr (std::is_same_v>) { - list_ptr_ = BOOST_SEGMENT->construct("BoostList")( - BOOST_ALLOCATOR((std::pair))); - lp_ = list_ptr_; - } else if constexpr(std::is_same_v>) { - list_ptr_ = new std::list(); - lp_ = list_ptr_; - } - } - - /** Destroy the list */ - void Destroy() { - if constexpr(std::is_same_v>) { - list_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v>) { - list_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v>) { - delete list_ptr_; - } else if constexpr (std::is_same_v>) { - BOOST_SEGMENT->destroy("BoostList"); - } - } -}; - -void FullListTest() { - // std::list tests - ListTest>().Test(); - ListTest>().Test(); - - // boost::ipc::list tests - ListTest>().Test(); - ListTest>().Test(); - ListTest>().Test(); - - // hipc::list tests - ListTest>().Test(); - ListTest>().Test(); - ListTest>().Test(); - - // hipc::slist tests - ListTest>().Test(); - ListTest>().Test(); - ListTest>().Test(); -} - -TEST_CASE("ListBenchmark") { - FullListTest(); -} diff --git a/hermes_shm/benchmark/data_structure/lock.cc b/hermes_shm/benchmark/data_structure/lock.cc deleted file mode 100644 index b6646003c..000000000 --- a/hermes_shm/benchmark/data_structure/lock.cc +++ /dev/null @@ -1,162 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -// Std -#include -#include - -// hermes -#include "hermes_shm/thread/lock.h" - -/** - * A series of performance tests for vectors - * OUTPUT: - * [test_name] [vec_type] [internal_type] [time_ms] - * */ -template -class LockTest { - public: - std::string lock_type_; - std::queue queue_; - LockT lock_; - - /**==================================== - * Test Runner - * ===================================*/ - - /** Test case constructor */ - LockTest() { - if constexpr(std::is_same_v) { - lock_type_ = "std::mutex"; - } else if constexpr(std::is_same_v) { - lock_type_ = "hshm::RwLock"; - } else if constexpr(std::is_same_v) { - lock_type_ = "hshm::Mutex"; - } else { - HELOG(kFatal, "none of the queue tests matched") - } - } - - /** Run the tests */ - void Test(size_t count_per_rank = 100000, int nthreads = 1) { - // AllocateTest(count); - EmplaceTest(count_per_rank, nthreads); - GatherTest(count_per_rank, nthreads); - } - - /**==================================== - * Tests - * ===================================*/ - - /** Emplace after reserving enough space */ - void EmplaceTest(size_t count_per_rank, int nthreads) { - Timer t; - - size_t count = count_per_rank * nthreads; - t.Resume(); - Emplace(count_per_rank, nthreads); - t.Pause(); - - TestOutput("Enqueue", t, count, nthreads); - } - - /** Reader lock scalability */ - void GatherTest(size_t count_per_rank, int nthreads) { - Timer t; - - size_t count = count_per_rank * nthreads; - t.Resume(); - Gather(count_per_rank, nthreads); - t.Pause(); - - TestOutput("Gather", t, count, nthreads); - } - - private: - /**==================================== - * Helpers - * ===================================*/ - - /** Output as CSV */ - void TestOutput(const std::string &test_name, Timer &t, - size_t count, int nthreads) { - HIPRINT("{},{},{},{},{}\n", - test_name, lock_type_, nthreads, t.GetMsec(), - (float)count / t.GetUsec()) - } - - /** Emplace elements into the queue */ - void Emplace(size_t count_per_rank, int nthreads) { - omp_set_dynamic(0); -#pragma omp parallel num_threads(nthreads) - { - for (size_t i = 0; i < count_per_rank; ++i) { - if constexpr(std::is_same_v) { - lock_.lock(); - queue_.emplace(1); - lock_.unlock(); - } else if constexpr(std::is_same_v) { - lock_.Lock(0); - queue_.emplace(1); - lock_.Unlock(); - } else if constexpr(std::is_same_v) { - lock_.WriteLock(0); - queue_.emplace(1); - lock_.WriteUnlock(); - } - } - } - } - - /** Emplace elements into the queue */ - void Gather(size_t count_per_rank, int nthreads) { - std::vector data(count_per_rank * nthreads); - omp_set_dynamic(0); -#pragma omp parallel shared(data) num_threads(nthreads) - { - size_t sum = 0; - for (size_t i = 0; i < count_per_rank; ++i) { - if constexpr(std::is_same_v) { - lock_.lock(); - sum += data[i]; - lock_.unlock(); - } else if constexpr(std::is_same_v) { - lock_.Lock(0); - sum += data[i]; - lock_.Unlock(); - } else if constexpr(std::is_same_v) { - lock_.ReadLock(0); - sum += data[i]; - lock_.ReadUnlock(); - } - } - } - } -}; - -TEST_CASE("LockBenchmark") { - size_t count_per_rank = 1000000; - LockTest().Test(count_per_rank, 1); - LockTest().Test(count_per_rank, 8); - LockTest().Test(count_per_rank, 16); - - LockTest().Test(count_per_rank, 1); - LockTest().Test(count_per_rank, 8); - LockTest().Test(count_per_rank, 16); - - LockTest().Test(count_per_rank, 1); - LockTest().Test(count_per_rank, 8); - LockTest().Test(count_per_rank, 16); -} \ No newline at end of file diff --git a/hermes_shm/benchmark/data_structure/queue.cc b/hermes_shm/benchmark/data_structure/queue.cc deleted file mode 100644 index 0d69ceb61..000000000 --- a/hermes_shm/benchmark/data_structure/queue.cc +++ /dev/null @@ -1,261 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -// Std -#include -#include - -// hermes -#include "hermes_shm/data_structures/ipc/string.h" -#include "hermes_shm/data_structures/ipc/split_ticket_queue.h" -#include -#include -#include - -/** - * A series of performance tests for vectors - * OUTPUT: - * [test_name] [vec_type] [internal_type] [time_ms] - * */ -template, QueueT*)> -class QueueTest { - public: - std::string queue_type_; - std::string internal_type_; - QueueT *queue_; - ListTPtr queue_ptr_; - void *ptr_; - hipc::uptr x_; - std::mutex lock_; - int cpu_; - - /**==================================== - * Test Runner - * ===================================*/ - - /** Test case constructor */ - QueueTest() { - if constexpr(std::is_same_v, QueueT>) { - queue_type_ = "std::queue"; - } else if constexpr(std::is_same_v, QueueT>) { - queue_type_ = "hipc::mpsc_queue"; - } else if constexpr(std::is_same_v, QueueT>) { - queue_type_ = "hipc::spsc_queue"; - } else if constexpr(std::is_same_v, QueueT>) { - queue_type_ = "hipc::ticket_queue"; - } else if constexpr(std::is_same_v, QueueT>) { - queue_type_ = "hipc::split_ticket_queue"; - } else { - HELOG(kFatal, "none of the queue tests matched") - } - internal_type_ = InternalTypeName::Get(); - x_ = hipc::make_uptr(); - } - - /** Run the tests */ - void Test(size_t count_per_rank=100000, int nthreads = 1) { - // AllocateTest(count); - EmplaceTest(count_per_rank, nthreads); - DequeueTest(count_per_rank, nthreads); - } - - /**==================================== - * Tests - * ===================================*/ - - /** Test performance of queue allocation */ - void AllocateTest(size_t count) { - Timer t; - t.Resume(); - for (size_t i = 0; i < count; ++i) { - Allocate(count, count, 1); - Destroy(); - } - t.Pause(); - - TestOutput("Allocate", t); - } - - /** Emplace after reserving enough space */ - void EmplaceTest(size_t count_per_rank, int nthreads) { - Timer t; - - size_t count = count_per_rank * nthreads; - Allocate(count, count_per_rank, nthreads); - t.Resume(); - Emplace(count_per_rank, nthreads); - t.Pause(); - - TestOutput("Enqueue", t, count, nthreads); - Destroy(); - } - - /** Emplace after reserving enough space */ - void DequeueTest(size_t count_per_rank, int nthreads) { - Timer t; - StringOrInt var(124); - - size_t count = count_per_rank * nthreads; - Allocate(count, count_per_rank, nthreads); - Emplace(count, 1); - - t.Resume(); - Dequeue(count_per_rank, nthreads); - t.Pause(); - - TestOutput("Dequeue", t, count, nthreads); - Destroy(); - } - - private: - /**==================================== - * Helpers - * ===================================*/ - - /** Output as CSV */ - void TestOutput(const std::string &test_name, Timer &t, - size_t count, int nthreads) { - HIPRINT("{},{},{},{},{},{}\n", - test_name, queue_type_, internal_type_, nthreads, t.GetMsec(), - (float)count / t.GetUsec()) - } - - /** Emplace elements into the queue */ - void Emplace(size_t count_per_rank, int nthreads) { - omp_set_dynamic(0); -#pragma omp parallel num_threads(nthreads) - { - StringOrInt var(124); - for (size_t i = 0; i < count_per_rank; ++i) { - if constexpr(std::is_same_v>) { - lock_.lock(); - queue_->emplace(var.Get()); - lock_.unlock(); - } else if constexpr(std::is_same_v>) { - queue_->emplace(var.Get()); - } else if constexpr(std::is_same_v>) { - queue_->emplace(var.Get()); - } else if constexpr(std::is_same_v>) { - queue_->emplace(var.Get()); - } else if constexpr( - std::is_same_v>) { - queue_->emplace(var.Get()); - } - } - } - } - - /** Get element at position i */ - void Dequeue(size_t count_per_rank, int nthreads) { - omp_set_dynamic(0); -#pragma omp parallel num_threads(nthreads) - { - for (size_t i = 0; i < count_per_rank; ++i) { - if constexpr(std::is_same_v>) { - lock_.lock(); - T &x = queue_->front(); - USE(x); - queue_->pop(); - lock_.unlock(); - } else if constexpr(std::is_same_v>) { - queue_->pop(*x_); - USE(*x_); - } else if constexpr(std::is_same_v>) { - queue_->pop(*x_); - USE(*x_); - } else if constexpr(std::is_same_v>) { - while (queue_->pop(*x_).IsNull()); - } else if constexpr( - std::is_same_v>) { - while (queue_->pop(*x_).IsNull()); - } - } - } - } - - /** Allocate an arbitrary queue for the test cases */ - void Allocate(size_t count, size_t count_per_rank, int nthreads) { - (void) count_per_rank; (void) nthreads; - if constexpr(std::is_same_v>) { - queue_ptr_ = new std::queue(); - queue_ = queue_ptr_; - } else if constexpr(std::is_same_v>) { - queue_ptr_ = hipc::make_mptr(count); - queue_ = queue_ptr_.get(); - } else if constexpr(std::is_same_v>) { - queue_ptr_ = hipc::make_mptr(count); - queue_ = queue_ptr_.get(); - } else if constexpr(std::is_same_v>) { - queue_ptr_ = hipc::make_mptr(count); - queue_ = queue_ptr_.get(); - } else if constexpr(std::is_same_v>) { - queue_ptr_ = hipc::make_mptr(count_per_rank, nthreads); - queue_ = queue_ptr_.get(); - } - } - - /** Destroy the queue */ - void Destroy() { - if constexpr(std::is_same_v>) { - delete queue_ptr_; - } else if constexpr(std::is_same_v>) { - queue_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v>) { - queue_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v>) { - queue_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v>) { - queue_ptr_.shm_destroy(); - } - } -}; - -void FullQueueTest() { - const size_t count_per_rank = 1000000; - // std::queue tests - QueueTest>().Test(count_per_rank, 1); - QueueTest>().Test(count_per_rank, 4); - QueueTest>().Test(count_per_rank, 8); - QueueTest>().Test(count_per_rank, 16); - QueueTest>().Test(); - - // hipc::ticket_queue tests - QueueTest>().Test(count_per_rank, 1); - QueueTest>().Test(count_per_rank, 4); - QueueTest>().Test(count_per_rank, 8); - QueueTest>().Test(count_per_rank, 16); - - // hipc::ticket_queue tests - QueueTest>().Test(count_per_rank, 1); - QueueTest>().Test(count_per_rank, 4); - QueueTest>().Test(count_per_rank, 8); - QueueTest>().Test(count_per_rank, 16); - - // hipc::mpsc_queue tests - QueueTest>().Test(count_per_rank, 1); - QueueTest>().Test(); - QueueTest>().Test(); - - // hipc::spsc_queue tests - QueueTest>().Test(count_per_rank, 1); - QueueTest>().Test(); - QueueTest>().Test(); - -} - -TEST_CASE("QueueBenchmark") { - FullQueueTest(); -} diff --git a/hermes_shm/benchmark/data_structure/ref.cc b/hermes_shm/benchmark/data_structure/ref.cc deleted file mode 100644 index 9548fd628..000000000 --- a/hermes_shm/benchmark/data_structure/ref.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -#include -#include "hermes_shm/data_structures/ipc/string.h" - -template -void TestCase() { - std::string str_type = InternalTypeName::Get(); - /*size_t count = 1000000; - auto test2 = hipc::make_uptr(); - auto alloc = test2->GetAllocator(); - size_t x; - hipc::ShmArchive ar; - void *ptr_; */ - - Timer t; - t.Resume(); - t.Pause(); - - HIPRINT("{},{},{}\n", - str_type, SHM, t.GetMsec()) -} - -TEST_CASE("RefBenchmark") { - TestCase(); - // TestCase(); - // TestCase(); -} diff --git a/hermes_shm/benchmark/data_structure/string.cc b/hermes_shm/benchmark/data_structure/string.cc deleted file mode 100644 index 8e86abdb2..000000000 --- a/hermes_shm/benchmark/data_structure/string.cc +++ /dev/null @@ -1,120 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -#include -#include "hermes_shm/data_structures/ipc/string.h" - -template -class StringTestSuite { - public: - std::string str_type_; - void *ptr_; - - /**==================================== - * Test Cases - * ===================================*/ - - /** Constructor */ - StringTestSuite() { - if constexpr(std::is_same_v) { - str_type_ = "std::string"; - } else if constexpr(std::is_same_v) { - str_type_ = "hipc::string"; - } else if constexpr(std::is_same_v) { - str_type_ = "bipc::string"; - } - } - - /** Dereference performance */ - void ConstructDestructTest(size_t count, int length) { - std::string data(length, 1); - - Timer t; - t.Resume(); - for (size_t i = 0; i < count; ++i) { - if constexpr(std::is_same_v) { - T hello(data); - USE(hello); - } else if constexpr(std::is_same_v) { - auto hello = hipc::make_uptr(data); - USE(hello); - } else if constexpr(std::is_same_v) { - auto hello = BOOST_SEGMENT->find_or_construct("MyString")( - BOOST_ALLOCATOR(bipc_string)); - BOOST_SEGMENT->destroy("MyString"); - USE(hello); - } - } - t.Pause(); - - TestOutput("ConstructDestructTest", t, length); - } - - /** Deserialize a string in a loop */ - void DeserializeTest(size_t count, int length) { - std::string data(length, 1); - std::string *test1 = &data; - auto test2 = hipc::make_uptr(data); - bipc_string *test3 = BOOST_SEGMENT->find_or_construct("MyString")( - BOOST_ALLOCATOR(bipc_string)); - test3->assign(data); - - Timer t; - t.Resume(); - for (size_t i = 0; i < count; ++i) { - if constexpr(std::is_same_v) { - auto info = test1->data(); - USE(info); - } else if constexpr(std::is_same_v) { - auto info = test2->data(); - USE(info); - } else if constexpr(std::is_same_v) { - auto info = test3->c_str(); - USE(info); - } - } - t.Pause(); - - TestOutput("DeserializeTest", t, length); - } - - /**==================================== - * Test Output - * ===================================*/ - - /** Output test results */ - void TestOutput(const std::string &test_name, Timer &t, size_t length) { - HIPRINT("{},{},{},{}\n", - test_name, str_type_, length, t.GetMsec()) - } -}; - -template -void StringTest() { - size_t count = 100000; - StringTestSuite().ConstructDestructTest(count, 16); - StringTestSuite().ConstructDestructTest(count, 256); - StringTestSuite().DeserializeTest(count, 16); -} - -void FullStringTest() { - StringTest(); - StringTest(); - StringTest(); -} - -TEST_CASE("StringBenchmark") { - FullStringTest(); -} diff --git a/hermes_shm/benchmark/data_structure/test_init.cc b/hermes_shm/benchmark/data_structure/test_init.cc deleted file mode 100644 index 3582fbec7..000000000 --- a/hermes_shm/benchmark/data_structure/test_init.cc +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/vector.h" -#include "hermes_shm/memory/allocator/stack_allocator.h" - -void MainPretest() { - // hermes shared memory - std::string shm_url = "HermesBench"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - auto backend = mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - memset(backend->data_, 0, MEGABYTES(16)); - // TODO(llogan): back to good allocator - mem_mngr->CreateAllocator( - shm_url, alloc_id, 0); - - - // Boost shared memory - BOOST_SEGMENT; - BOOST_ALLOCATOR(char); - BOOST_ALLOCATOR(size_t); - BOOST_ALLOCATOR(std::string); - BOOST_ALLOCATOR(bipc_string); - BOOST_ALLOCATOR((std::pair)); - BOOST_ALLOCATOR((std::pair)); - BOOST_ALLOCATOR((std::pair)); - BOOST_ALLOCATOR((std::pair)); -} - -void MainPosttest() { - bipc::shared_memory_object::remove("LabstorBoostBench"); -} diff --git a/hermes_shm/benchmark/data_structure/test_init.h b/hermes_shm/benchmark/data_structure/test_init.h deleted file mode 100644 index dd3cd082e..000000000 --- a/hermes_shm/benchmark/data_structure/test_init.h +++ /dev/null @@ -1,183 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ -#define HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ - -#include -#include -#include -#include -#include - -#include - -#include "hermes_shm/data_structures/data_structure.h" -#include -#include - -#include - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::Pointer; - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -namespace bipc = boost::interprocess; - -/** Shared memory segment (a large contiguous region) */ -typedef bipc::managed_shared_memory::segment_manager segment_manager_t; - -/** Create boost segment singleton */ -struct BoostSegment { - std::unique_ptr segment_; - - BoostSegment() { - bipc::shared_memory_object::remove("HermesBoostBench"); - segment_ = std::make_unique( - bipc::create_only, "HermesBoostBench", GIGABYTES(4)); - } -}; -#define BOOST_SEGMENT \ - hshm::EasySingleton::GetInstance()->segment_.get() - -/** Create boost allocator singleton */ -template -struct BoostAllocator { - typedef bipc::allocator alloc_t; - alloc_t alloc_; - - BoostAllocator() - : alloc_(BOOST_SEGMENT->get_segment_manager()) {} -}; -#define BOOST_ALLOCATOR(T) \ - hshm::EasySingleton>::GetInstance()->alloc_ - -/** A generic string using allocator */ -typedef boost::interprocess::basic_string< - char, std::char_traits, - typename BoostAllocator::alloc_t> bipc_string; - -/** Instance of the segment */ -extern std::unique_ptr segment_g; - -/** - * Converts an arbitrary type to std::string or int - * */ -template -struct StringOrInt { - typedef typename hshm::type_switch, - bipc_string, bipc_string*>::type - internal_t; - internal_t internal_; - - /** Convert from int to internal_t */ - StringOrInt(size_t num) { - if constexpr(std::is_same_v) { - internal_ = num; - } else if constexpr(std::is_same_v) { - internal_ = std::to_string(num); - } else if constexpr(std::is_same_v) { - internal_ = hipc::make_uptr(std::to_string(num)); - } else if constexpr(std::is_same_v) { - internal_ = BOOST_SEGMENT->find_or_construct("MyString")( - BOOST_ALLOCATOR(bipc_string)); - } - } - - /** Get the internal type */ - T& Get() { - if constexpr(std::is_same_v) { - return internal_; - } else if constexpr(std::is_same_v) { - return internal_; - } else if constexpr(std::is_same_v) { - return *internal_; - } else if constexpr(std::is_same_v) { - return *internal_; - } - } - - /** Convert internal_t to int */ - size_t ToInt() { - if constexpr(std::is_same_v) { - return internal_; - } else { - size_t num; - std::stringstream(ToString(internal_)) >> num; - return num; - } - } - - /** Convert internal_t to std::string */ - std::string ToString() { - if constexpr(std::is_same_v) { - return std::to_string(internal_); - } else if constexpr(std::is_same_v) { - return internal_; - } else if constexpr(std::is_same_v) { - return internal_->str(); - } else if constexpr(std::is_same_v) { - return std::string(internal_->c_str(), internal_->length()); - } - } - - /** Destructor */ - ~StringOrInt() { - if constexpr(std::is_same_v) { - } else if constexpr(std::is_same_v) { - } else if constexpr(std::is_same_v) { - } else if constexpr(std::is_same_v) { - BOOST_SEGMENT->destroy("MyString"); - } - } -}; - -/** - * Gets the semantic name of a type - * */ -template -struct InternalTypeName { - static std::string Get() { - if constexpr(std::is_same_v) { - return "hipc::string"; - } else if constexpr(std::is_same_v) { - return "std::string"; - } else if constexpr(std::is_same_v) { - return "bipc::string"; - } else if constexpr(std::is_same_v) { - return "int"; - } - } -}; - - -/** Timer */ -using Timer = hshm::HighResMonotonicTimer; - -/** Avoid compiler warning */ -#define USE(var) ptr_ = (void*)&(var); - -#endif //HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ diff --git a/hermes_shm/benchmark/data_structure/unordered_map.cc b/hermes_shm/benchmark/data_structure/unordered_map.cc deleted file mode 100644 index 139117b0d..000000000 --- a/hermes_shm/benchmark/data_structure/unordered_map.cc +++ /dev/null @@ -1,270 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -// Boost interprocess -#include - -// Std -#include -#include - -// hermes -#include "hermes_shm/data_structures/ipc/string.h" -#include - -template -using bipc_unordered_map = boost::unordered_map< - Key, T, - std::hash, std::equal_to, - typename BoostAllocator::alloc_t>; - -/** - * A series of performance tests for unordered_maps - * OUTPUT: - * [test_name] [map_type] [internal_type] [time_ms] - * */ -template, MapT*)> -class UnorderedMapTest { - public: - std::string map_type_; - std::string internal_type_; - MapT *map_; - MapTPtr map_ptr_; - void *ptr_; - - /**==================================== - * Test Runner - * ===================================*/ - - /** Test case constructor */ - UnorderedMapTest() { - if constexpr(std::is_same_v, MapT>) { - map_type_ = "std::unordered_map"; - } else if constexpr(std::is_same_v, MapT>) { - map_type_ = "hipc::unordered_map"; - } else if constexpr(std::is_same_v, MapT>) { - map_type_ = "bipc::unordered_map"; - } else { - std::cout << "INVALID: none of the unordered_map tests matched" - << std::endl; - return; - } - internal_type_ = InternalTypeName::Get(); - } - - /** Run the tests */ - void Test() { - size_t count = 100000; - // AllocateTest(count); - EmplaceTest(count); - GetTest(count); - ForwardIteratorTest(count); - // CopyTest(count); - // MoveTest(count); - } - - /**==================================== - * Tests - * ===================================*/ - - /** Test performance of unordered_map allocation */ - void AllocateTest(size_t count) { - Timer t; - t.Resume(); - for (size_t i = 0; i < count; ++i) { - Allocate(); - Destroy(); - } - t.Pause(); - - TestOutput("Allocate", t); - } - - /** Emplace performance */ - void EmplaceTest(size_t count) { - Timer t; - Allocate(); - - t.Resume(); - Emplace(count); - t.Pause(); - - TestOutput("Emplace", t); - Destroy(); - } - - /** Get performance */ - void GetTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - Emplace(count); - - t.Resume(); - for (size_t i = 0; i < count; ++i) { - Get(i); - } - t.Pause(); - - TestOutput("FixedGet", t); - Destroy(); - } - - /** Iterator performance */ - void ForwardIteratorTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - Emplace(count); - - t.Resume(); - for (auto &x : *map_) { - USE(x); - } - t.Pause(); - - TestOutput("ForwardIterator", t); - Destroy(); - } - - /** Copy performance */ - void CopyTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(MapT)) { - auto vec2 = hipc::make_uptr(*map_); - USE(vec2) - } else { - MapT vec2(*map_); - USE(vec2) - } - t.Pause(); - - TestOutput("Copy", t); - Destroy(); - } - - /** Move performance */ - void MoveTest(size_t count) { - Timer t; - - Allocate(); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(MapT)) { - volatile auto vec2 = hipc::make_uptr(std::move(*map_)); - } else { - volatile MapT vec2(*map_); - } - t.Pause(); - - TestOutput("Move", t); - Destroy(); - } - - private: - /**==================================== - * Helpers - * ===================================*/ - - /** Output as CSV */ - void TestOutput(const std::string &test_name, Timer &t) { - HIPRINT("{},{},{},{}\n", - test_name, map_type_, internal_type_, t.GetMsec()) - } - - /** Get element at position i */ - void Get(size_t i) { - if constexpr(std::is_same_v>) { - T &x = (*map_)[i]; - USE(x); - } else if constexpr(std::is_same_v>) { - auto iter = (*map_).find(i); - USE(*iter); - } else if constexpr(std::is_same_v>) { - T &x = (*map_)[i]; - USE(x); - } - } - - /** Emplace elements into the unordered_map */ - void Emplace(size_t count) { - StringOrInt var(124); - for (size_t i = 0; i < count; ++i) { - if constexpr(std::is_same_v>) { - map_->emplace(i, var.Get()); - } else if constexpr(std::is_same_v>) { - map_->emplace(i, var.Get()); - } else if constexpr(std::is_same_v>) { - map_->emplace(i, var.Get()); - } - } - } - - /** Allocate an arbitrary unordered_map for the test cases */ - void Allocate() { - if constexpr(std::is_same_v>) { - map_ptr_ = hipc::make_mptr(5000); - map_ = map_ptr_.get(); - } else if constexpr(std::is_same_v>) { - map_ptr_ = new std::unordered_map(); - map_ = map_ptr_; - } else if constexpr (std::is_same_v>) { - map_ptr_ = BOOST_SEGMENT->construct("BoostMap")( - BOOST_ALLOCATOR((std::pair))); - map_ = map_ptr_; - } - } - - /** Destroy the unordered_map */ - void Destroy() { - if constexpr(std::is_same_v>) { - map_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v>) { - delete map_ptr_; - } else if constexpr (std::is_same_v>) { - BOOST_SEGMENT->destroy("BoostMap"); - } - } -}; - -void FullUnorderedMapTest() { - // std::unordered_map tests - UnorderedMapTest>().Test(); - UnorderedMapTest>().Test(); - - // boost::unordered_map tests - UnorderedMapTest>().Test(); - UnorderedMapTest>().Test(); - UnorderedMapTest>().Test(); - - // hipc::unordered_map tests - UnorderedMapTest>().Test(); - UnorderedMapTest>().Test(); - UnorderedMapTest>().Test(); -} - -TEST_CASE("UnorderedMapBenchmark") { - FullUnorderedMapTest(); -} diff --git a/hermes_shm/benchmark/data_structure/vector.cc b/hermes_shm/benchmark/data_structure/vector.cc deleted file mode 100644 index 49a7ded13..000000000 --- a/hermes_shm/benchmark/data_structure/vector.cc +++ /dev/null @@ -1,344 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" - -// Boost interprocess -#include - -// Std -#include -#include - -// hermes -#include "hermes_shm/data_structures/ipc/string.h" -#include - -template -using bipc_vector = bipc::vector::alloc_t>; - -/** - * A series of performance tests for vectors - * OUTPUT: - * [test_name] [vec_type] [internal_type] [time_ms] - * */ -template, VecT*)> -class VectorTest { - public: - std::string vec_type_; - std::string internal_type_; - VecT *vec_; - VecTPtr vec_ptr_; - void *ptr_; - - /**==================================== - * Test Runner - * ===================================*/ - - /** Test case constructor */ - VectorTest() { - if constexpr(std::is_same_v, VecT>) { - vec_type_ = "std::vector"; - } else if constexpr(std::is_same_v, VecT>) { - vec_type_ = "hipc::vector"; - } else if constexpr(std::is_same_v, VecT>) { - vec_type_ = "bipc_vector"; - } else { - std::cout << "INVALID: none of the vector tests matched" << std::endl; - return; - } - internal_type_ = InternalTypeName::Get(); - } - - /** Run the tests */ - void Test() { - size_t count = 1000000; - // AllocateTest(count); - // ResizeTest(count); - ReserveEmplaceTest(count); - GetTest(count); - BeginIteratorTest(count); - EndIteratorTest(count); - ForwardIteratorTest(count); - // CopyTest(count); - // MoveTest(count); - } - - /**==================================== - * Tests - * ===================================*/ - - /** Test performance of vector allocation */ - void AllocateTest(size_t count) { - Timer t; - t.Resume(); - for (size_t i = 0; i < count; ++i) { - Allocate(); - Destroy(); - } - t.Pause(); - - TestOutput("Allocate", t); - } - - /** Test the performance of a resize */ - void ResizeTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - t.Resume(); - vec_->resize(count); - t.Pause(); - - TestOutput("FixedResize", t); - Destroy(); - } - - /** Emplace after reserving enough space */ - void ReserveEmplaceTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - t.Resume(); - vec_->reserve(count); - Emplace(count); - t.Pause(); - - TestOutput("FixedEmplace", t); - Destroy(); - } - - /** Get performance */ - void GetTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - vec_->reserve(count); - Emplace(count); - - t.Resume(); - for (size_t i = 0; i < count; ++i) { - Get(i); - } - t.Pause(); - - TestOutput("FixedGet", t); - Destroy(); - } - - /** Begin iterator performance */ - void BeginIteratorTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - vec_->reserve(count); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(VecT)) { - auto iter = vec_->begin(); - USE(iter); - } else { - auto iter = vec_->begin(); - USE(iter); - } - t.Pause(); - - TestOutput("BeginIterator", t); - Destroy(); - } - - /** End iterator performance */ - void EndIteratorTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - vec_->reserve(count); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(VecT)) { - auto iter = vec_->end(); - USE(iter); - } else { - auto iter = vec_->end(); - USE(iter); - } - t.Pause(); - - TestOutput("EndIterator", t); - Destroy(); - } - - /** Iterator performance */ - void ForwardIteratorTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - vec_->reserve(count); - Emplace(count); - - t.Resume(); - int i = 0; - for (auto &x : *vec_) { - USE(x); - ++i; - } - t.Pause(); - - if (ptr_ == nullptr) { - std::cout << "hm" << std::endl; - } - - TestOutput("ForwardIterator", t); - Destroy(); - } - - /** Copy performance */ - void CopyTest(size_t count) { - Timer t; - StringOrInt var(124); - - Allocate(); - vec_->reserve(count); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(VecT)) { - auto vec2 = hipc::make_uptr(*vec_); - USE(vec2); - } else { - VecT vec2(*vec_); - USE(vec2); - } - t.Pause(); - - TestOutput("Copy", t); - Destroy(); - } - - /** Move performance */ - void MoveTest(size_t count) { - Timer t; - - Allocate(); - vec_->reserve(count); - Emplace(count); - - t.Resume(); - if constexpr(IS_SHM_ARCHIVEABLE(VecT)) { - auto vec2 = hipc::make_uptr(std::move(*vec_)); - USE(vec2) - } else { - VecT vec2(*vec_); - USE(vec2) - } - t.Pause(); - - TestOutput("Move", t); - Destroy(); - } - - private: - /**==================================== - * Helpers - * ===================================*/ - - /** Output as CSV */ - void TestOutput(const std::string &test_name, Timer &t) { - HIPRINT("{},{},{},{}\n", - test_name, vec_type_, internal_type_, t.GetMsec()) - } - - /** Get element at position i */ - void Get(size_t i) { - if constexpr(std::is_same_v, VecT>) { - T &x = (*vec_)[i]; - USE(x); - } else if constexpr(std::is_same_v, VecT>) { - T &x = (*vec_)[i]; - USE(x); - } else if constexpr(std::is_same_v, VecT>) { - T &x = (*vec_)[i]; - USE(x); - if (!ptr_) { std::cout << "h" << std::endl; } - } - } - - /** Emplace elements into the vector */ - void Emplace(size_t count) { - StringOrInt var(124); - for (size_t i = 0; i < count; ++i) { - if constexpr(std::is_same_v, VecT>) { - vec_->emplace_back(var.Get()); - } else if constexpr(std::is_same_v, VecT>) { - vec_->emplace_back(var.Get()); - } else if constexpr(std::is_same_v, VecT>) { - vec_->emplace_back(var.Get()); - } - } - } - - /** Allocate an arbitrary vector for the test cases */ - void Allocate() { - if constexpr(std::is_same_v, VecT>) { - vec_ptr_ = new std::vector(); - vec_ = vec_ptr_; - } else if constexpr(std::is_same_v, VecT>) { - vec_ptr_ = hipc::make_mptr(); - vec_ = vec_ptr_.get(); - } else if constexpr(std::is_same_v, VecT>) { - vec_ptr_ = BOOST_SEGMENT->construct("BoostVector")( - BOOST_ALLOCATOR((std::pair))); - vec_ = vec_ptr_; - } - } - - /** Destroy the vector */ - void Destroy() { - if constexpr(std::is_same_v, VecT>) { - delete vec_ptr_; - } else if constexpr(std::is_same_v, VecT>) { - vec_ptr_.shm_destroy(); - } else if constexpr(std::is_same_v, VecT>) { - BOOST_SEGMENT->destroy("BoostVector"); - } - } -}; - -void FullVectorTest() { - // std::vector tests - VectorTest>().Test(); - VectorTest>().Test(); - - // boost::ipc::vector tests - VectorTest>().Test(); - VectorTest>().Test(); - VectorTest>().Test(); - - // hipc::vector tests - VectorTest>().Test(); - VectorTest>().Test(); - VectorTest>().Test(); -} - -TEST_CASE("VectorBenchmark") { - FullVectorTest(); -} diff --git a/hermes_shm/benchmark/lock/CMakeLists.txt b/hermes_shm/benchmark/lock/CMakeLists.txt deleted file mode 100644 index 86a2d1d88..000000000 --- a/hermes_shm/benchmark/lock/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -include_directories( ${Boost_INCLUDE_DIRS} ) -include_directories( ${TEST_MAIN} ) -add_executable(benchmark_lock - ${TEST_MAIN}/main.cc - test_init.cc - benchmark_mutex.cc -) -add_dependencies(benchmark_lock hermes_shm_data_structures) -target_link_libraries(benchmark_lock - hermes_shm_data_structures - Catch2::Catch2 - MPI::MPI_CXX - OpenMP::OpenMP_CXX) diff --git a/hermes_shm/benchmark/lock/benchmark_mutex.cc b/hermes_shm/benchmark/lock/benchmark_mutex.cc deleted file mode 100644 index 80b5bafb4..000000000 --- a/hermes_shm/benchmark/lock/benchmark_mutex.cc +++ /dev/null @@ -1,66 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_shm/util/timer.h" -#include "hermes_shm/thread/lock.h" -#include "test_init.h" -#include "basic_test.h" -#include - -TEST_CASE("TestMutexTryLock") { - Timer t; - t.Resume(); - size_t ops = (1ull << 26); - hshm::Mutex lock; - for (size_t i = 0; i < ops; ++i) { - lock.TryLock(0); - lock.Unlock(); - } - t.Pause(); - std::cout << ops / t.GetUsec() << std::endl; -} - -TEST_CASE("TestMutex") { - Timer t; - t.Resume(); - size_t ops = (1ull << 26); - hshm::Mutex lock; - for (size_t i = 0; i < ops; ++i) { - hshm::ScopedMutex scoped_lock(lock, 0); - } - t.Pause(); - std::cout << ops / t.GetUsec() << std::endl; -} - -TEST_CASE("TestRwReadLock") { - Timer t; - t.Resume(); - size_t ops = (1ull << 26); - hshm::RwLock lock_; - for (size_t i = 0; i < ops; ++i) { - hshm::ScopedRwReadLock read_lock(lock_, 0); - } - t.Pause(); - std::cout << ops / t.GetUsec() << std::endl; -} - -TEST_CASE("TestRwWriteLock") { - Timer t; - t.Resume(); - size_t ops = (1ull << 26); - hshm::RwLock lock_; - for (size_t i = 0; i < ops; ++i) { - hshm::ScopedRwReadLock write_lock(lock_, 0); - } - t.Pause(); - std::cout << ops / t.GetUsec() << std::endl; -} \ No newline at end of file diff --git a/hermes_shm/benchmark/lock/test_init.h b/hermes_shm/benchmark/lock/test_init.h deleted file mode 100644 index f8264df31..000000000 --- a/hermes_shm/benchmark/lock/test_init.h +++ /dev/null @@ -1,20 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ -#define HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ - -#include - -using Timer = hshm::HighResMonotonicTimer; - -#endif //HERMES_BENCHMARK_DATA_STRUCTURE_TEST_INIT_H_ diff --git a/hermes_shm/code_generators/cpp_macro_generator.py b/hermes_shm/code_generators/cpp_macro_generator.py deleted file mode 100644 index 344f42688..000000000 --- a/hermes_shm/code_generators/cpp_macro_generator.py +++ /dev/null @@ -1,17 +0,0 @@ -from code_generators.c_macro.macro_generator import CppMacroGenerator -import pathlib, os - -PROJECT_ROOT=pathlib.Path(__file__).parent.parent.resolve() - -DATA_STRUCTURE_TEMPLATES='include/hermes_shm/data_structures/ipc/internal/template' -DATA_STRUCTURE_INTERNAL='include/hermes_shm/data_structures/ipc/internal' - -CppMacroGenerator().generate( - os.path.join(PROJECT_ROOT, - f"{DATA_STRUCTURE_TEMPLATES}/shm_container_base_template.h"), - os.path.join(PROJECT_ROOT, - f"{DATA_STRUCTURE_INTERNAL}/shm_container_macro.h"), - "SHM_CONTAINER_TEMPLATE", - ["CLASS_NAME", "TYPED_CLASS"], - ["TYPE_UNWRAP", "TYPE_UNWRAP"], - "HERMES_DATA_STRUCTURES_INTERNAL_SHM_CONTAINER_MACRO_H_") diff --git a/hermes_shm/doc/Doxyfile.in b/hermes_shm/doc/Doxyfile.in deleted file mode 100644 index e351f448e..000000000 --- a/hermes_shm/doc/Doxyfile.in +++ /dev/null @@ -1,2685 +0,0 @@ -# Doxyfile 1.9.5 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). -# -# Note: -# -# Use doxygen to compare the used configuration file with the template -# configuration file: -# doxygen -x [configFile] -# Use doxygen to compare the used configuration file with the template -# configuration file without replacing the environment variables or CMake type -# replacement variables: -# doxygen -x_noenv [configFile] - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the configuration -# file that follow. The default is UTF-8 which is also the encoding used for all -# text before the first occurrence of this tag. Doxygen uses libiconv (or the -# iconv built into libc) for the transcoding. See -# https://www.gnu.org/software/libiconv/ for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = Hermes - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = 0.8.0-beta - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = "Hierarchical Distributed I/O Buffering System" - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 -# sub-directories (in 2 levels) under the output directory of each output format -# and will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to -# control the number of sub-directories. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# Controls the number of sub-directories that will be created when -# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every -# level increment doubles the number of directories, resulting in 4096 -# directories at level 8 which is the default and also the maximum value. The -# sub-directories are organized in 2 levels, the first level always has a fixed -# numer of 16 directories. -# Minimum value: 0, maximum value: 8, default value: 8. -# This tag requires that the tag CREATE_SUBDIRS is set to YES. - -CREATE_SUBDIRS_LEVEL = 8 - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, -# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English -# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, -# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with -# English messages), Korean, Korean-en (Korean with English messages), Latvian, -# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, -# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, -# Swedish, Turkish, Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = NO - -# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line -# such as -# /*************** -# as being the beginning of a Javadoc-style comment "banner". If set to NO, the -# Javadoc-style will behave just like regular comments and it will not be -# interpreted by doxygen. -# The default value is: NO. - -JAVADOC_BANNER = NO - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = NO - -# By default Python docstrings are displayed as preformatted text and doxygen's -# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the -# doxygen's special commands can be used and the contents of the docstring -# documentation blocks is shown as doxygen documentation. -# The default value is: YES. - -PYTHON_DOCSTRING = YES - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:^^" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". Note that you cannot put \n's in the value part of an alias -# to insert newlines (in the resulting output). You can put ^^ in the value part -# of an alias to insert a newline as if a physical newline was in the original -# file. When you need a literal { or } or , in the value part of an alias you -# have to escape them by means of a backslash (\), this can lead to conflicts -# with the commands \{ and \} for these it is advised to use the version @{ and -# @} or use a double escape (\\{ and \\}) - -ALIASES = "status=A Status object." \ - "ctx{1}=ctx The Context for this \1." \ - "bool{1}=true if \1, otherwise false." - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice -# sources only. Doxygen will then generate output that is more tailored for that -# language. For instance, namespaces will be presented as modules, types will be -# separated into more groups, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_SLICE = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, -# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: -# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser -# tries to guess whether the code is fixed or free formatted code, this is the -# default for Fortran type files). For instance to make doxygen treat .inc files -# as Fortran files (default is PHP), and .f files as C (default is Fortran), -# use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. When specifying no_extension you should add -# * to the FILE_PATTERNS. -# -# Note see also the list of default file extension mappings. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See https://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 5. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 5 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = NO - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use -# during processing. When set to 0 doxygen will based this on the number of -# cores available in the system. You can set it explicitly to a value larger -# than 0 to get more control over the balance between CPU load and processing -# speed. At this moment only the input processing can be done using multiple -# threads. Since this is still an experimental feature the default is set to 1, -# which effectively disables parallel processing. Please report any issues you -# encounter. Generating dot graphs in parallel is controlled by the -# DOT_NUM_THREADS setting. -# Minimum value: 0, maximum value: 32, default value: 1. - -NUM_PROC_THREADS = 1 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual -# methods of a class will be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIV_VIRTUAL = NO - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = NO - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = NO - -# If this flag is set to YES, the name of an unnamed parameter in a declaration -# will be determined by the corresponding definition. By default unnamed -# parameters remain unnamed in the output. -# The default value is: YES. - -RESOLVE_UNNAMED_PARAMS = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# declarations. If set to NO, these declarations will be included in the -# documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = NO - -# With the correct setting of option CASE_SENSE_NAMES doxygen will better be -# able to match the capabilities of the underlying filesystem. In case the -# filesystem is case sensitive (i.e. it supports files in the same directory -# whose names only differ in casing), the option must be set to YES to properly -# deal with such files in case they appear in the input. For filesystems that -# are not case sensitive the option should be set to NO to properly deal with -# output files written for symbols that only differ in casing, such as for two -# classes, one named CLASS and the other named Class, and to also support -# references to files without having to specify the exact matching casing. On -# Windows (including Cygwin) and MacOS, users should typically set this option -# to NO, whereas on Linux or other Unix flavors it should typically be set to -# YES. -# Possible values are: SYSTEM, NO and YES. -# The default value is: SYSTEM. - -CASE_SENSE_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class -# will show which file needs to be included to use the class. -# The default value is: YES. - -SHOW_HEADERFILE = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = NO - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = NO - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. See also section "Changing the -# layout of pages" for information. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = YES - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as documenting some parameters in -# a documented function twice, or documenting parameters that don't exist or -# using markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete -# function parameter documentation. If set to NO, doxygen will accept that some -# parameters have no documentation without warning. -# The default value is: YES. - -WARN_IF_INCOMPLETE_DOC = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong parameter -# documentation, but not about the absence of documentation. If EXTRACT_ALL is -# set to YES then this flag will automatically be disabled. See also -# WARN_IF_INCOMPLETE_DOC -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS -# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but -# at the end of the doxygen process doxygen will return with a non-zero status. -# Possible values are: NO, YES and FAIL_ON_WARNINGS. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# See also: WARN_LINE_FORMAT -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# In the $text part of the WARN_FORMAT command it is possible that a reference -# to a more specific place is given. To make it easier to jump to this place -# (outside of doxygen) the user can define a custom "cut" / "paste" string. -# Example: -# WARN_LINE_FORMAT = "'vi $file +$line'" -# See also: WARN_FORMAT -# The default value is: at line $line of file $file. - -WARN_LINE_FORMAT = "at line $line of file $file" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). In case the file specified cannot be opened for writing the -# warning and error messages are written to standard error. When as file - is -# specified the warning and error messages are written to standard output -# (stdout). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = @PROJECT_SOURCE_DIR@/adapter \ - @PROJECT_SOURCE_DIR@/gotcha_intercept \ - @PROJECT_SOURCE_DIR@/src - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: -# https://www.gnu.org/software/libiconv/) for the list of possible encodings. -# See also: INPUT_FILE_ENCODING -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify -# character encoding on a per file pattern basis. Doxygen will compare the file -# name with each pattern and apply the encoding instead of the default -# INPUT_ENCODING) if there is a match. The character encodings are a list of the -# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding -# "INPUT_ENCODING" for further information on supported encodings. - -INPUT_FILE_ENCODING = - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# Note the list of default checked file patterns might differ from the list of -# default file extension mappings. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, -# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C -# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, -# *.vhdl, *.ucf, *.qsf and *.ice. - -FILE_PATTERNS = *.h \ - *.cc - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = @PROJECT_SOURCE_DIR@/src/stb_ds.h - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# ANamespace::AClass, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that doxygen will use the data processed and written to standard output -# for further processing, therefore nothing else, like debug statements or used -# commands (so in case of a Windows batch file always use @echo OFF), should be -# written to standard output. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = front_page.md - -# The Fortran standard specifies that for fixed formatted Fortran code all -# characters from position 72 are to be considered as comment. A common -# extension is to allow longer lines before the automatic comment starts. The -# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can -# be processed before the automatic comment starts. -# Minimum value: 7, maximum value: 10000, default value: 72. - -FORTRAN_COMMENT_AFTER = 72 - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# entity all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see https://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot of -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output -# should be rendered with a dark or light theme. Default setting AUTO_LIGHT -# enables light output unless the user preference is dark output. Other options -# are DARK to always use dark mode, LIGHT to always use light mode, AUTO_DARK to -# default to dark mode unless the user prefers light mode, and TOGGLE to let the -# user toggle between dark and light mode via a button. -# Possible values are: LIGHT Always generate light output., DARK Always generate -# dark output., AUTO_LIGHT Automatically set the mode according to the user -# preference, use light mode if no preference is set (the default)., AUTO_DARK -# Automatically set the mode according to the user preference, use dark mode if -# no preference is set. and TOGGLE Allow to user to switch between light and -# dark mode via a button.. -# The default value is: AUTO_LIGHT. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE = AUTO_LIGHT - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a color-wheel, see -# https://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use gray-scales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - -# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML -# documentation will contain a main index with vertical navigation menus that -# are dynamically created via JavaScript. If disabled, the navigation index will -# consists of multiple levels of tabs that are statically embedded in every HTML -# page. Disable this option to support browsers that do not have JavaScript, -# like the Qt help browser. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_MENUS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = NO - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 100 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: -# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To -# create a documentation set, doxygen will generate a Makefile in the HTML -# output directory. Running make will produce the docset in that directory and -# running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy -# genXcode/_index.html for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag determines the URL of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDURL = - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# on Windows. In the beginning of 2021 Microsoft took the original page, with -# a.o. the download links, offline the HTML help workshop was already many years -# in maintenance mode). You can download the HTML help workshop from the web -# archives at Installation executable (see: -# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo -# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the main .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location (absolute path -# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to -# run qhelpgenerator on the generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine tune the look of the index (see "Fine-tuning the output"). As an -# example, the default style sheet generated by doxygen has an example that -# shows how to put an image at the root of the tree instead of the PROJECT_NAME. -# Since the tree basically has the same information as the tab index, you could -# consider setting DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = YES - -# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the -# FULL_SIDEBAR option determines if the side bar is limited to only the treeview -# area (value NO) or if it should extend to the full height of the window (value -# YES). Setting this to YES gives a layout similar to -# https://docs.readthedocs.io with more room for contents, but less room for the -# project logo, title, and description. If either GENERATE_TREEVIEW or -# DISABLE_INDEX is set to NO, this option has no effect. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FULL_SIDEBAR = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email -# addresses. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -OBFUSCATE_EMAILS = YES - -# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg -# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see -# https://inkscape.org) to generate formulas as SVG images instead of PNGs for -# the HTML output. These images will generally look nicer at scaled resolutions. -# Possible values are: png (the default) and svg (looks nicer but requires the -# pdf2svg or inkscape tool). -# The default value is: png. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FORMULA_FORMAT = png - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands -# to create new LaTeX commands to be used in formulas as building blocks. See -# the section "Including formulas" for details. - -FORMULA_MACROFILE = - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# https://www.mathjax.org) which uses client side JavaScript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. -# Note that the different versions of MathJax have different requirements with -# regards to the different settings, so it is possible that also other MathJax -# settings have to be changed when switching between the different MathJax -# versions. -# Possible values are: MathJax_2 and MathJax_3. -# The default value is: MathJax_2. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_VERSION = MathJax_2 - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. For more details about the output format see MathJax -# version 2 (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 -# (see: -# http://docs.mathjax.org/en/latest/web/components/output.html). -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility. This is the name for Mathjax version 2, for MathJax version 3 -# this will be translated into chtml), NativeMML (i.e. MathML. Only supported -# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This -# is the name for Mathjax version 3, for MathJax version 2 this will be -# translated into HTML-CSS) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. The default value is: -# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 -# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/ - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# for MathJax version 2 (see -# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# For example for MathJax version 3 (see -# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): -# MATHJAX_EXTENSIONS = ams -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /Node, -# Edge and Graph Attributes specification You need to make sure dot is able -# to find the font, which can be done by putting it in a standard location or by -# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the -# directory containing the font. Default graphviz fontsize is 14. -# The default value is: fontname=Helvetica,fontsize=10. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" - -# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can -# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about -# arrows shapes. -# The default value is: labelfontname=Helvetica,labelfontsize=10. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" - -# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes -# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification -# The default value is: shape=box,height=0.2,width=0.4. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" - -# You can set the path where dot can find font specified with fontname in -# DOT_COMMON_ATTR and others dot attributes. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_FONTPATH = - -# If the CLASS_GRAPH tag is set to YES (or GRAPH) then doxygen will generate a -# graph for each documented class showing the direct and indirect inheritance -# relations. In case HAVE_DOT is set as well dot will be used to draw the graph, -# otherwise the built-in generator will be used. If the CLASS_GRAPH tag is set -# to TEXT the direct and indirect inheritance relations will be shown as texts / -# links. -# Possible values are: NO, YES, TEXT and GRAPH. -# The default value is: YES. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a -# graph for each documented class showing the direct and indirect implementation -# dependencies (inheritance, containment, and class references variables) of the -# class with other documented classes. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. See also the chapter Grouping -# in the manual. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -UML_LOOK = NO - -# If the UML_LOOK tag is enabled, the fields and methods are shown inside the -# class node. If there are many fields or methods and many nodes the graph may -# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the -# number of items for each type to make the size more manageable. Set this to 0 -# for no limit. Note that the threshold may be exceeded by 50% before the limit -# is enforced. So when you set the threshold to 10, up to 15 fields may appear, -# but if the number exceeds 15, the total amount of fields shown is limited to -# 10. -# Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag UML_LOOK is set to YES. - -UML_LIMIT_NUM_FIELDS = 10 - -# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and -# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS -# tag is set to YES, doxygen will add type and arguments for attributes and -# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen -# will not generate fields with class member information in the UML graphs. The -# class diagrams will look similar to the default class diagrams but using UML -# notation for the relationships. -# Possible values are: NO, YES and NONE. -# The default value is: NO. -# This tag requires that the tag UML_LOOK is set to YES. - -DOT_UML_DETAILS = NO - -# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters -# to display on a single line. If the actual line length exceeds this threshold -# significantly it will wrapped across multiple lines. Some heuristics are apply -# to avoid ugly line breaks. -# Minimum value: 0, maximum value: 1000, default value: 17. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_WRAP_THRESHOLD = 17 - -# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and -# collaboration graphs will show the relations between templates and their -# instances. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -TEMPLATE_RELATIONS = NO - -# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to -# YES then doxygen will generate a graph for each documented file showing the -# direct and indirect include dependencies of the file with other documented -# files. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -INCLUDE_GRAPH = YES - -# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are -# set to YES then doxygen will generate a graph for each documented file showing -# the direct and indirect include dependencies of the file with other documented -# files. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH tag is set to YES then doxygen will generate a call -# dependency graph for every global function or class method. -# -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. Disabling a call graph can be -# accomplished by means of the command \hidecallgraph. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -CALL_GRAPH = YES - -# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller -# dependency graph for every global function or class method. -# -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable caller graphs for selected -# functions only using the \callergraph command. Disabling a caller graph can be -# accomplished by means of the command \hidecallergraph. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -CALLER_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical -# hierarchy of all classes instead of a textual one. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the -# dependencies a directory has on other directories in a graphical way. The -# dependency relations are determined by the #include relations between the -# files in the directories. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -DIRECTORY_GRAPH = YES - -# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels -# of child directories generated in directory dependency graphs by dot. -# Minimum value: 1, maximum value: 25, default value: 1. -# This tag requires that the tag DIRECTORY_GRAPH is set to YES. - -DIR_GRAPH_MAX_DEPTH = 1 - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. For an explanation of the image formats see the section -# output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). -# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order -# to make the SVG files visible in IE 9+ (other browsers do not have this -# requirement). -# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, -# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and -# png:gdiplus:gdiplus. -# The default value is: png. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_IMAGE_FORMAT = png - -# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to -# enable generation of interactive SVG images that allow zooming and panning. -# -# Note that this requires a modern browser other than Internet Explorer. Tested -# and working are Firefox, Chrome, Safari, and Opera. -# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make -# the SVG files visible. Older versions of IE do not have SVG support. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -INTERACTIVE_SVG = NO - -# The DOT_PATH tag can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_PATH = @DOXYGEN_DOT_PATH@ - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the \dotfile -# command). -# This tag requires that the tag HAVE_DOT is set to YES. - -DOTFILE_DIRS = - -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). - -MSCFILE_DIRS = - -# The DIAFILE_DIRS tag can be used to specify one or more directories that -# contain dia files that are included in the documentation (see the \diafile -# command). - -DIAFILE_DIRS = - -# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file or to the filename of jar file -# to be used. If left blank, it is assumed PlantUML is not used or called during -# a preprocessing step. Doxygen will generate a warning when it encounters a -# \startuml command in this case and will not generate output for the diagram. - -PLANTUML_JAR_PATH = - -# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a -# configuration file for plantuml. - -PLANTUML_CFG_FILE = - -# When using plantuml, the specified paths are searched for files specified by -# the !include statement in a plantuml block. - -PLANTUML_INCLUDE_PATH = - -# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes -# that will be shown in the graph. If the number of nodes in a graph becomes -# larger than this value, doxygen will truncate the graph, which is visualized -# by representing a node as a red box. Note that doxygen if the number of direct -# children of the root node in a graph is already larger than -# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that -# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. -# Minimum value: 0, maximum value: 10000, default value: 50. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_GRAPH_MAX_NODES = 100 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs -# generated by dot. A depth value of 3 means that only nodes reachable from the -# root by following a path via at most 3 edges will be shown. Nodes that lay -# further from the root node will be omitted. Note that setting this option to 1 -# or 2 may greatly reduce the computation time needed for large code bases. Also -# note that the size of a graph can be further restricted by -# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. -# Minimum value: 0, maximum value: 1000, default value: 0. -# This tag requires that the tag HAVE_DOT is set to YES. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) support -# this, this feature is disabled by default. -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_MULTI_TARGETS = NO - -# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page -# explaining the meaning of the various boxes and arrows in the dot generated -# graphs. -# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal -# graphical representation for inheritance and collaboration diagrams is used. -# The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate -# files that are used to generate the various graphs. -# -# Note: This setting is not only used for dot files but also for msc temporary -# files. -# The default value is: YES. - -DOT_CLEANUP = YES diff --git a/hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h b/hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h deleted file mode 100644 index 38b07173b..000000000 --- a/hermes_shm/include/hermes_shm/constants/data_structure_singleton_macros.h +++ /dev/null @@ -1,26 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_CONSTANTS_DATA_STRUCTURE_SINGLETON_MACROS_H_H -#define HERMES_INCLUDE_HERMES_CONSTANTS_DATA_STRUCTURE_SINGLETON_MACROS_H_H - -#include - -#define HERMES_MEMORY_REGISTRY hshm::GlobalSingleton::GetInstance() -#define HERMES_MEMORY_REGISTRY_REF hshm::GlobalSingleton::GetRef() -#define HERMES_MEMORY_REGISTRY_T hshm::ipc::MemoryRegistry* - -#define HERMES_MEMORY_MANAGER hshm::GlobalSingleton::GetInstance() -#define HERMES_MEMORY_MANAGER_REF hshm::GlobalSingleton::GetRef() -#define HERMES_MEMORY_MANAGER_T hshm::ipc::MemoryManager* - -#endif // HERMES_INCLUDE_HERMES_CONSTANTS_DATA_STRUCTURE_SINGLETON_MACROS_H_H diff --git a/hermes_shm/include/hermes_shm/constants/macros.h b/hermes_shm/include/hermes_shm/constants/macros.h deleted file mode 100644 index 28a0e26fb..000000000 --- a/hermes_shm/include/hermes_shm/constants/macros.h +++ /dev/null @@ -1,67 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MACROS_H -#define HERMES_MACROS_H - -/** Bytes -> Bytes */ -#ifndef BYTES -#define BYTES(n) (size_t)((n) * (((size_t)1) << 0)) -#endif - -/** KILOBYTES -> Bytes */ -#ifndef KILOBYTES -#define KILOBYTES(n) (size_t)((n) * (((size_t)1) << 10)) -#endif - -/** MEGABYTES -> Bytes */ -#ifndef MEGABYTES -#define MEGABYTES(n) (size_t)((n) * (((size_t)1) << 20)) -#endif - -/** GIGABYTES -> Bytes */ -#ifndef GIGABYTES -#define GIGABYTES(n) (size_t)((n) * (((size_t)1) << 30)) -#endif - -/** TERABYTES -> Bytes */ -#ifndef TERABYTES -#define TERABYTES(n) (size_t)((n) * (((size_t)1) << 40)) -#endif - -/** PETABYTES -> Bytes */ -#ifndef PETABYTES -#define PETABYTES(n) (size_t)((n) * (((size_t)1) << 50)) -#endif - -/** - * Remove parenthesis surrounding "X" if it has parenthesis - * Used for helper macros which take templated types as parameters - * E.g., let's say we have: - * - * #define HELPER_MACRO(T) TYPE_UNWRAP(T) - * HELPER_MACRO( (std::vector>) ) - * will return std::vector> without the parenthesis - * */ -#define TYPE_UNWRAP(X) ESC(ISH X) -#define ISH(...) ISH __VA_ARGS__ -#define ESC(...) ESC_(__VA_ARGS__) -#define ESC_(...) VAN ## __VA_ARGS__ -#define VANISH - -/** - * Ensure that the compiler ALWAYS inlines a particular function. - * */ -#define HSHM_ALWAYS_INLINE \ - __attribute__((always_inline)) - -#endif // HERMES_MACROS_H diff --git a/hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h b/hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h deleted file mode 100644 index ea9c9b004..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/containers/charbuf.h +++ /dev/null @@ -1,267 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_TYPES_CHARBUF_H_ -#define HERMES_INCLUDE_HERMES_TYPES_CHARBUF_H_ - -#include "hermes_shm/types/real_number.h" -#include "hermes_shm/memory/memory_registry.h" -#include - -namespace hshm { - -/** An uninterpreted array of bytes */ -struct charbuf { - /**==================================== - * Variables & Types - * ===================================*/ - - hipc::Allocator *alloc_; /**< The allocator used to allocate data */ - char *data_; /**< The pointer to data */ - size_t size_; /**< The size of data */ - size_t total_size_; /**< The true size of data buffer */ - bool destructable_; /**< Whether or not this container owns data */ - - /**==================================== - * Default Constructor - * ===================================*/ - - /** Default constructor */ - HSHM_ALWAYS_INLINE charbuf() - : alloc_(nullptr), data_(nullptr), size_(0), - total_size_(0), destructable_(false) {} - - /**==================================== - * Destructor - * ===================================*/ - - /** Destructor */ - HSHM_ALWAYS_INLINE ~charbuf() { Free(); } - - /**==================================== - * Emplace Constructors - * ===================================*/ - - /** Size-based constructor */ - HSHM_ALWAYS_INLINE explicit charbuf(size_t size) { - Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), size); - } - - /** Allocator + Size-based constructor */ - HSHM_ALWAYS_INLINE explicit charbuf(hipc::Allocator *alloc, size_t size) { - Allocate(alloc, size); - } - - /**==================================== - * Reference Constructors - * ===================================*/ - - /** Reference constructor. From char* + size */ - HSHM_ALWAYS_INLINE explicit charbuf(char *data, size_t size) - : alloc_(nullptr), data_(data), size_(size), - total_size_(size), destructable_(false) {} - - /** - * Reference constructor. From const char* - * We assume that the data will not be modified by the user, but - * we must cast away the const anyway. - * */ - HSHM_ALWAYS_INLINE explicit charbuf(const char *data, size_t size) - : alloc_(nullptr), data_(const_cast(data)), - size_(size), total_size_(size), destructable_(false) {} - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** Copy constructor. From std::string. */ - HSHM_ALWAYS_INLINE explicit charbuf(const std::string &data) { - Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), data.size()); - memcpy(data_, data.data(), data.size()); - } - - /** Copy constructor. From charbuf. */ - HSHM_ALWAYS_INLINE charbuf(const charbuf &other) { - if (!Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), - other.size())) { - return; - } - memcpy(data_, other.data(), size()); - } - - /** Copy assignment operator */ - HSHM_ALWAYS_INLINE charbuf& operator=(const charbuf &other) { - if (this != &other) { - Free(); - if (!Allocate(HERMES_MEMORY_REGISTRY->GetDefaultAllocator(), - other.size())) { - return *this; - } - memcpy(data_, other.data(), size()); - } - return *this; - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** Move constructor */ - charbuf(charbuf &&other) { - alloc_ = other.alloc_; - data_ = other.data_; - size_ = other.size_; - total_size_ = other.total_size_; - destructable_ = other.destructable_; - other.size_ = 0; - other.total_size_ = 0; - other.destructable_ = false; - } - - /** Move assignment operator */ - charbuf& operator=(charbuf &&other) noexcept { - if (this != &other) { - Free(); - alloc_ = other.alloc_; - data_ = other.data_; - size_ = other.size_; - total_size_ = other.total_size_; - destructable_ = other.destructable_; - other.size_ = 0; - other.total_size_ = 0; - other.destructable_ = false; - } - return *this; - } - - /**==================================== - * Methods - * ===================================*/ - - /** Destroy and resize */ - void resize(size_t new_size) { - if (new_size <= total_size_) { - size_ = new_size; - return; - } - if (alloc_ == nullptr) { - alloc_ = HERMES_MEMORY_REGISTRY->GetDefaultAllocator(); - } - if (destructable_) { - data_ = alloc_->ReallocatePtr(data_, new_size); - } else { - data_ = alloc_->AllocatePtr(new_size); - } - destructable_ = true; - size_ = new_size; - total_size_ = new_size; - } - - /** Reference data */ - HSHM_ALWAYS_INLINE char* data() { - return data_; - } - - /** Reference data */ - HSHM_ALWAYS_INLINE char* data() const { - return data_; - } - - /** Reference size */ - HSHM_ALWAYS_INLINE size_t size() const { - return size_; - } - - /** Get allocator */ - HSHM_ALWAYS_INLINE hipc::Allocator* GetAllocator() { - return alloc_; - } - - /** Convert to std::string */ - HSHM_ALWAYS_INLINE std::string str() { - return std::string(data(), size()); - } - - /**==================================== - * Comparison Operators - * ===================================*/ - - HSHM_ALWAYS_INLINE int _strncmp(const char *a, size_t len_a, - const char *b, size_t len_b) const { - if (len_a != len_b) { - return int((int64_t)len_a - (int64_t)len_b); - } - for (size_t i = 0; i < len_a; ++i) { - if (a[i] != b[i]) { - return a[i] - b[i]; - } - } - return 0; - } - -#define HERMES_STR_CMP_OPERATOR(op) \ - bool operator TYPE_UNWRAP(op)(const char *other) const { \ - return _strncmp(data(), size(), other, strlen(other)) op 0; \ - } \ - bool operator op(const std::string &other) const { \ - return _strncmp(data(), size(), other.data(), other.size()) op 0; \ - } \ - bool operator op(const charbuf &other) const { \ - return _strncmp(data(), size(), other.data(), other.size()) op 0; \ - } - - HERMES_STR_CMP_OPERATOR(==) // NOLINT - HERMES_STR_CMP_OPERATOR(!=) // NOLINT - HERMES_STR_CMP_OPERATOR(<) // NOLINT - HERMES_STR_CMP_OPERATOR(>) // NOLINT - HERMES_STR_CMP_OPERATOR(<=) // NOLINT - HERMES_STR_CMP_OPERATOR(>=) // NOLINT - -#undef HERMES_STR_CMP_OPERATOR - - private: - /**==================================== - * Internal functions - * ===================================*/ - - /** Allocate charbuf */ - bool Allocate(hipc::Allocator *alloc, size_t size) { - hipc::OffsetPointer p; - if (size == 0) { - alloc_ = nullptr; - data_ = nullptr; - size_ = 0; - total_size_ = 0; - destructable_ = false; - return false; - } - alloc_ = alloc; - data_ = alloc->AllocatePtr(size, p); - size_ = size; - total_size_ = size; - destructable_ = true; - return true; - } - - /** Explicitly free the charbuf */ - void Free() { - if (destructable_ && data_ && total_size_) { - alloc_->FreePtr(data_); - } - } -}; - -typedef charbuf string; - -} // namespace hshm - -#endif // HERMES_INCLUDE_HERMES_TYPES_CHARBUF_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/containers/converters.h b/hermes_shm/include/hermes_shm/data_structures/containers/converters.h deleted file mode 100644 index 47023a35b..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/containers/converters.h +++ /dev/null @@ -1,57 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_CONVERTERS_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_CONVERTERS_H_ - -#include "hermes_shm/data_structures/data_structure.h" -#include -#include - -namespace hshm { - -/** Convert an iterable object into a vector */ -template -std::vector to_stl_vector(const SharedT &other) { - std::vector vec; - vec.reserve(other.size()); - auto end = other.cend(); - for (auto iter = other.cbegin(); iter != end; ++iter) { - T &obj = (*iter); - vec.emplace_back(obj); - } - return vec; -} - -/** Convert an iterable object into a list */ -template -std::list to_stl_list(const SharedT &other) { - std::list vec; - auto end = other.cend(); - for (auto iter = other.cbegin(); iter != end; ++iter) { - T& obj = (*iter); - vec.emplace_back(obj); - } - return vec; -} - -/** Convert a string to an hshm::charbuf */ -template -hshm::charbuf to_charbuf(StringT &other) { - hshm::charbuf text(other.size()); - memcpy(text.data(), other.data(), other.size()); - return text; -} - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_CONVERTERS_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/containers/functional.h b/hermes_shm/include/hermes_shm/data_structures/containers/functional.h deleted file mode 100644 index 435668c16..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/containers/functional.h +++ /dev/null @@ -1,31 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_SHM_DATA_STRUCTURES_CONTAINERS_FUNCTIONAL_H_ -#define HERMES_SHM_SHM_DATA_STRUCTURES_CONTAINERS_FUNCTIONAL_H_ - -namespace hshm { - -template -IteratorT find(IteratorT start, const IteratorT &end, T &val) { - for (; start != end; ++start) { - T &ref = *start; - if (ref == val) { - return start; - } - } - return end; -} - -} // namespace hshm - -#endif // HERMES_SHM_SHM_DATA_STRUCTURES_CONTAINERS_FUNCTIONAL_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/containers/tuple_base.h b/hermes_shm/include/hermes_shm/data_structures/containers/tuple_base.h deleted file mode 100644 index 35da88aa4..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/containers/tuple_base.h +++ /dev/null @@ -1,238 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_DATA_STRUCTURES_TupleBase_H_ -#define HERMES_INCLUDE_HERMES_DATA_STRUCTURES_TupleBase_H_ - -#include -#include "hermes_shm/types/real_number.h" -#include "hermes_shm/types/argpack.h" - -namespace hshm { - -/** The null container wrapper */ -template -using NullWrap = T; - -/** Recurrence used to create argument pack */ -template< - template typename Wrap, - size_t idx, - typename T = EndTemplateRecurrence, - typename ...Args> -struct TupleBaseRecur { - Wrap arg_; /**< The element stored */ - TupleBaseRecur - recur_; /**< Remaining args */ - - /** Default constructor */ - TupleBaseRecur() = default; - - /** Constructor. Const reference. */ - explicit TupleBaseRecur(const T &arg, Args&& ...args) - : arg_(std::forward(arg)), recur_(std::forward(args)...) {} - - /** Constructor. Lvalue reference. */ - explicit TupleBaseRecur(T& arg, Args&& ...args) - : arg_(std::forward(arg)), recur_(std::forward(args)...) {} - - /** Constructor. Rvalue reference. */ - explicit TupleBaseRecur(T&& arg, Args&& ...args) - : arg_(std::forward(arg)), recur_(std::forward(args)...) {} - - /** Move constructor */ - TupleBaseRecur(TupleBaseRecur &&other) noexcept - : arg_(std::move(other.arg_)), recur_(std::move(other.recur_)) {} - - /** Move assignment operator */ - TupleBaseRecur& operator=(TupleBaseRecur &&other) { - if (this != &other) { - arg_ = std::move(other.arg_); - recur_ = std::move(other.recur_); - } - return *this; - } - - /** Copy constructor */ - TupleBaseRecur(const TupleBaseRecur &other) - : arg_(other.arg_), recur_(other.recur_) {} - - /** Copy assignment operator */ - TupleBaseRecur& operator=(const TupleBaseRecur &other) { - if (this != &other) { - arg_ = other.arg_; - recur_ = other.recur_; - } - return *this; - } - - /** Solidification constructor */ - template - explicit TupleBaseRecur(ArgPack &&other) - : arg_(other.template Forward()), - recur_(std::forward>(other)) {} - - /** Get reference to internal variable (only if tuple) */ - template - constexpr auto& Get() { - if constexpr(i == idx) { - return arg_; - } else { - return recur_.template - Get(); - } - } - - /** Get reference to internal variable (only if tuple, const) */ - template - constexpr auto& Get() const { - if constexpr(i == idx) { - return arg_; - } else { - return recur_.template - Get(); - } - } -}; - -/** Terminator of the TupleBase recurrence */ -template< - template typename Wrap, - size_t idx> -struct TupleBaseRecur { - /** Default constructor */ - TupleBaseRecur() = default; - - /** Solidification constructor */ - template - explicit TupleBaseRecur(ArgPack &&other) {} - - /** Getter */ - template - void Get() { - throw std::logic_error("(Get) TupleBase index outside of range"); - } - - /** Getter */ - template - void Get() const { - throw std::logic_error("(Get) TupleBase index outside of range"); - } -}; - -/** Used to semantically pack arguments */ -template< - bool is_argpack, - template typename Wrap, - typename ...Args> -struct TupleBase { - /** Variable argument pack */ - TupleBaseRecur recur_; - - /** Default constructor */ - TupleBase() = default; - - /** General Constructor. */ - template - explicit TupleBase(Args&& ...args) - : recur_(std::forward(args)...) {} - - /** Move constructor */ - TupleBase(TupleBase &&other) noexcept - : recur_(std::move(other.recur_)) {} - - /** Move assignment operator */ - TupleBase& operator=(TupleBase &&other) noexcept { - if (this != &other) { - recur_ = std::move(other.recur_); - } - return *this; - } - - /** Copy constructor */ - TupleBase(const TupleBase &other) - : recur_(other.recur_) {} - - /** Copy assignment operator */ - TupleBase& operator=(const TupleBase &other) { - if (this != &other) { - recur_ = other.recur_; - } - return *this; - } - - /** Solidification constructor */ - template - explicit TupleBase(ArgPack &&other) - : recur_(std::forward>(other)) {} - - /** Getter */ - template - constexpr auto& Get() { - return recur_.template Get(); - } - - /** Getter (const) */ - template - constexpr auto& Get() const { - return recur_.template Get(); - } - - /** Size */ - constexpr static size_t Size() { - return sizeof...(Args); - } -}; - -/** Tuple definition */ -template -using tuple = TupleBase; - -/** Tuple Wrapper Definition */ -template typename Wrap, typename ...Containers> -using tuple_wrap = TupleBase; - -/** Apply a function over an entire TupleBase / tuple */ -template -class IterateTuple { - public: - /** Apply a function to every element of a tuple */ - template - constexpr static void Apply(TupleT &pack, F &&f) { - _Apply<0, TupleT, F>(pack, std::forward(f)); - } - - private: - /** Apply the function recursively */ - template - constexpr static void _Apply(TupleT &pack, F &&f) { - if constexpr(i < TupleT::Size()) { - if constexpr(reverse) { - _Apply(pack, std::forward(f)); - f(MakeConstexpr(), pack.template Get()); - } else { - f(MakeConstexpr(), pack.template Get()); - _Apply(pack, std::forward(f)); - } - } - } -}; - -/** Forward iterate over tuple and apply function */ -using ForwardIterateTuple = IterateTuple; - -/** Reverse iterate over tuple and apply function */ -using ReverseIterateTuple = IterateTuple; - -} // namespace hshm - -#endif // HERMES_INCLUDE_HERMES_DATA_STRUCTURES_TupleBase_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/data_structure.h b/hermes_shm/include/hermes_shm/data_structures/data_structure.h deleted file mode 100644 index 422238fc8..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/data_structure.h +++ /dev/null @@ -1,38 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_DATA_STRUCTURE_H_ -#define HERMES_DATA_STRUCTURES_DATA_STRUCTURE_H_ - -#include "ipc/internal/shm_internal.h" -#include "hermes_shm/memory/memory_manager.h" - -#include "containers/charbuf.h" -#include "containers/converters.h" -#include "containers/functional.h" -#include "containers/tuple_base.h" - -#include "ipc/pair.h" -#include "ipc/string.h" -#include "ipc/list.h" -#include "ipc/vector.h" -#include "ipc/mpsc_queue.h" -#include "ipc/slist.h" -#include "ipc/split_ticket_queue.h" -#include "ipc/spsc_queue.h" -#include "ipc/ticket_queue.h" -#include "ipc/unordered_map.h" - -namespace hipc = hshm::ipc; - -#endif // HERMES_DATA_STRUCTURES_DATA_STRUCTURE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h deleted file mode 100644 index 311d7e1ab..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/_queue.h +++ /dev/null @@ -1,75 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_QUEUE_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_QUEUE_H_ - -namespace hshm { - -/** Represents the internal qtok type */ -typedef uint64_t _qtok_t; - -/** Represents a ticket in the queue */ -struct qtok_t { - _qtok_t id_; - - /** Default constructor */ - HSHM_ALWAYS_INLINE qtok_t() = default; - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit qtok_t(_qtok_t id) : id_(id) {} - - /** Copy constructor */ - HSHM_ALWAYS_INLINE qtok_t(const qtok_t &other) : id_(other.id_) {} - - /** Copy assign */ - HSHM_ALWAYS_INLINE qtok_t& operator=(const qtok_t &other) { - if (this != &other) { - id_ = other.id_; - } - return *this; - } - - /** Move constructor */ - HSHM_ALWAYS_INLINE qtok_t(qtok_t &&other) : id_(other.id_) { - other.SetNull(); - } - - /** Move assign */ - HSHM_ALWAYS_INLINE qtok_t& operator=(qtok_t &&other) { - if (this != &other) { - id_ = other.id_; - other.SetNull(); - } - return *this; - } - - /** Set to the null qtok */ - HSHM_ALWAYS_INLINE void SetNull() { - id_ = qtok_t::GetNull().id_; - } - - /** Get the null qtok */ - HSHM_ALWAYS_INLINE static qtok_t GetNull() { - static qtok_t other(std::numeric_limits<_qtok_t>::max()); - return other; - } - - /** Check if null */ - HSHM_ALWAYS_INLINE bool IsNull() const { - return id_ == GetNull().id_; - } -}; - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_QUEUE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h deleted file mode 100644 index a4184a106..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_archive.h +++ /dev/null @@ -1,143 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_SHM_ARCHIVE_H_ -#define HERMES_DATA_STRUCTURES_SHM_ARCHIVE_H_ - -#include "shm_macros.h" -#include "hermes_shm/memory/memory.h" -#include "hermes_shm/types/argpack.h" -#include "hermes_shm/memory/allocator/allocator.h" - -namespace hshm::ipc { - -/** - * Represents the layout of a data structure in shared memory. - * */ -template -class ShmArchive { - public: - typedef T internal_t; - char obj_[sizeof(T)]; - - /** Default constructor */ - ShmArchive() = default; - - /** Destructor */ - ~ShmArchive() = default; - - /** Pointer to internal object */ - HSHM_ALWAYS_INLINE T* get() { - return reinterpret_cast(obj_); - } - - /** Pointer to internal object (const) */ - HSHM_ALWAYS_INLINE const T* get() const { - return reinterpret_cast(obj_); - } - - /** Reference to internal object */ - HSHM_ALWAYS_INLINE T& get_ref() { - return reinterpret_cast(obj_); - } - - /** Reference to internal object (const) */ - HSHM_ALWAYS_INLINE const T& get_ref() const { - return reinterpret_cast(obj_); - } - - /** Dereference operator */ - HSHM_ALWAYS_INLINE T& operator*() { - return get_ref(); - } - - /** Dereference operator */ - HSHM_ALWAYS_INLINE const T& operator*() const { - return get_ref(); - } - - /** Arrow operator */ - HSHM_ALWAYS_INLINE T* operator->() { - return get(); - } - - /** Arrow operator */ - HSHM_ALWAYS_INLINE const T* operator->() const { - return get(); - } - - /** Copy constructor */ - ShmArchive(const ShmArchive &other) = delete; - - /** Copy assignment operator */ - ShmArchive& operator=(const ShmArchive &other) = delete; - - /** Move constructor */ - ShmArchive(ShmArchive &&other) = delete; - - /** Move assignment operator */ - ShmArchive& operator=(ShmArchive &&other) = delete; - - /** Initialize */ - template - HSHM_ALWAYS_INLINE void shm_init(Args&& ...args) { - Allocator::ConstructObj(get_ref(), std::forward(args)...); - } - - /** Initialize piecewise */ - template - HSHM_ALWAYS_INLINE void shm_init_piecewise(ArgPackT_1 &&args1, - ArgPackT_2 &&args2) { - return hshm::PassArgPack::Call( - MergeArgPacks::Merge( - make_argpack(get_ref()), - std::forward(args1), - std::forward(args2)), - [](auto&& ...args) constexpr { - Allocator::ConstructObj(std::forward(args)...); - }); - } -}; - -#define HSHM_AR_GET_TYPE(AR) \ - (typename std::remove_reference::type::internal_t) - -#define HSHM_MAKE_AR0(AR, ALLOC) \ - if constexpr(IS_SHM_ARCHIVEABLE(HSHM_AR_GET_TYPE(AR))) { \ - (AR).shm_init(ALLOC); \ - } else { \ - (AR).shm_init(); \ - } - -#define HSHM_MAKE_AR(AR, ALLOC, ...) \ - if constexpr(IS_SHM_ARCHIVEABLE(HSHM_AR_GET_TYPE(AR))) { \ - (AR).shm_init(ALLOC, __VA_ARGS__); \ - } else { \ - (AR).shm_init(__VA_ARGS__); \ - } - -#define HSHM_MAKE_AR_PW(AR, ALLOC, ...) \ - if constexpr(IS_SHM_ARCHIVEABLE(HSHM_AR_GET_TYPE(AR))) { \ - (AR).shm_init_piecewise(make_argpack(ALLOC), __VA_ARGS__); \ - } else { \ - (AR).shm_init_piecewise(make_argpack(), __VA_ARGS__); \ - } - -#define HSHM_DESTROY_AR(AR) \ - if constexpr(IS_SHM_ARCHIVEABLE(HSHM_AR_GET_TYPE(AR))) { \ - (AR).get_ref().shm_destroy(); \ - } - -} // namespace hshm::ipc - -#endif // HERMES_DATA_STRUCTURES_SHM_ARCHIVE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h deleted file mode 100644 index 51dd1bf6a..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container.h +++ /dev/null @@ -1,79 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_SHM_CONTAINER_H_ -#define HERMES_SHM_CONTAINER_H_ - -#include "hermes_shm/memory/memory_registry.h" -#include "hermes_shm/constants/macros.h" -#include "shm_container_macro.h" -#include "shm_macros.h" -#include "shm_archive.h" - -namespace hipc = hshm::ipc; - -namespace hshm::ipc { - -/** The shared-memory header used for data structures */ -template -struct ShmHeader; - -/** The ShmHeader used for base containers */ -#define SHM_CONTAINER_HEADER_TEMPLATE(HEADER_NAME)\ - /** Default constructor */\ - TYPE_UNWRAP(HEADER_NAME)() = default;\ - \ - /** Copy constructor */\ - TYPE_UNWRAP(HEADER_NAME)(const TYPE_UNWRAP(HEADER_NAME) &other) {\ - strong_copy(other);\ - }\ - \ - /** Copy assignment operator */\ - TYPE_UNWRAP(HEADER_NAME)& operator=(const TYPE_UNWRAP(HEADER_NAME) &other) {\ - if (this != &other) {\ - strong_copy(other);\ - }\ - return *this;\ - }\ -\ - /** Move constructor */\ - TYPE_UNWRAP(HEADER_NAME)(TYPE_UNWRAP(HEADER_NAME) &&other) {\ - strong_copy(other);\ - }\ - \ - /** Move operator */\ - TYPE_UNWRAP(HEADER_NAME)& operator=(TYPE_UNWRAP(HEADER_NAME) &&other) {\ - if (this != &other) {\ - strong_copy(other);\ - }\ - return *this;\ - } - -/** The ShmHeader used for wrapper containers */ -struct ShmWrapperHeader {}; - -/** - * ShmContainers all have a header, which is stored in - * shared memory as a TypedPointer. - * */ -class ShmContainer {}; - -/** Typed nullptr */ -template -HSHM_ALWAYS_INLINE static T* typed_nullptr() { - return reinterpret_cast(NULL); -} - -} // namespace hshm::ipc - -#endif // HERMES_SHM_CONTAINER_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container_macro.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container_macro.h deleted file mode 100644 index 13dff7cbe..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_container_macro.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef HERMES_DATA_STRUCTURES_INTERNAL_SHM_CONTAINER_MACRO_H_ -#define HERMES_DATA_STRUCTURES_INTERNAL_SHM_CONTAINER_MACRO_H_ -#define SHM_CONTAINER_TEMPLATE(CLASS_NAME,TYPED_CLASS)\ -public:\ -/**====================================\ - * Variables & Types\ - * ===================================*/\ -hipc::allocator_id_t alloc_id_;\ -\ -/**====================================\ - * Constructors\ - * ===================================*/\ -\ -/** Default constructor. Deleted. */\ -TYPE_UNWRAP(CLASS_NAME)() = delete;\ -\ -/** Move constructor. Deleted. */\ -TYPE_UNWRAP(CLASS_NAME)(TYPE_UNWRAP(CLASS_NAME) &&other) = delete;\ -\ -/** Copy constructor. Deleted. */\ -TYPE_UNWRAP(CLASS_NAME)(const TYPE_UNWRAP(CLASS_NAME) &other) = delete;\ -\ -/** Initialize container */\ -void shm_init_container(hipc::Allocator *alloc) {\ - alloc_id_ = alloc->GetId();\ -}\ -\ -/**====================================\ - * Destructor\ - * ===================================*/\ -\ -/** Destructor. */\ -HSHM_ALWAYS_INLINE ~TYPE_UNWRAP(CLASS_NAME)() = default;\ -\ -/** Destruction operation */\ -HSHM_ALWAYS_INLINE void shm_destroy() {\ - if (IsNull()) { return; }\ - shm_destroy_main();\ - SetNull();\ -}\ -\ -/**====================================\ - * Header Operations\ - * ===================================*/\ -\ -/** Get a typed pointer to the object */\ -template\ -HSHM_ALWAYS_INLINE POINTER_T GetShmPointer() const {\ - return GetAllocator()->template Convert(this);\ -}\ -\ -/**====================================\ - * Query Operations\ - * ===================================*/\ -\ -/** Get the allocator for this container */\ -HSHM_ALWAYS_INLINE hipc::Allocator* GetAllocator() const {\ - return HERMES_MEMORY_REGISTRY_REF.GetAllocator(alloc_id_);\ -}\ -\ -/** Get the shared-memory allocator id */\ -HSHM_ALWAYS_INLINE hipc::allocator_id_t& GetAllocatorId() const {\ - return GetAllocator()->GetId();\ -}\ - -#endif // HERMES_DATA_STRUCTURES_INTERNAL_SHM_CONTAINER_MACRO_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_internal.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_internal.h deleted file mode 100644 index 76538b00a..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_internal.h +++ /dev/null @@ -1,24 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* a * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_HERMES_SHM_DATA_STRUCTURES_INTERNAL_SHM_INTERNAL_H_ -#define HERMES_HERMES_SHM_DATA_STRUCTURES_INTERNAL_SHM_INTERNAL_H_ - -#include "shm_archive.h" -#include "shm_container.h" -#include "shm_macros.h" -#include "shm_container.h" - -namespace hipc = hshm::ipc; - -#endif // HERMES_HERMES_SHM_DATA_STRUCTURES_INTERNAL_SHM_INTERNAL_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_macros.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_macros.h deleted file mode 100644 index 820b70806..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_macros.h +++ /dev/null @@ -1,33 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_MEMORY_SHM_MACROS_H_ -#define HERMES_MEMORY_SHM_MACROS_H_ - -#include "hermes_shm/constants/macros.h" - -/** - * Determine whether or not \a T type is designed for shared memory - * */ -#define IS_SHM_ARCHIVEABLE(T) \ - std::is_base_of::value - -/** - * SHM_X_OR_Y: X if T is SHM_SERIALIZEABLE, Y otherwise - * */ -#define SHM_X_OR_Y(T, X, Y) \ - typename std::conditional< \ - IS_SHM_ARCHIVEABLE(T), \ - TYPE_UNWRAP(X), TYPE_UNWRAP(Y)>::type - -#endif // HERMES_MEMORY_SHM_MACROS_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_null_container.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_null_container.h deleted file mode 100644 index b415d3702..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_null_container.h +++ /dev/null @@ -1,127 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_DATA_STRUCTURES_INTERNAL_SHM_NULL_CONTAINER_H_ -#define HERMES_INCLUDE_HERMES_DATA_STRUCTURES_INTERNAL_SHM_NULL_CONTAINER_H_ - -#include "shm_container.h" -#include - -namespace hshm::ipc { - -/** forward declaration for string */ -class NullContainer; - -/** - * MACROS used to simplify the string namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME NullContainer -#define TYPED_CLASS NullContainer -#define TYPED_HEADER ShmHeader - -/** string shared-memory header */ -template<> -struct ShmHeader { - /** Default constructor */ - ShmHeader() = default; - - /** Copy constructor */ - ShmHeader(const ShmHeader &other) { - strong_copy(other); - } - - /** Copy assignment operator */ - ShmHeader& operator=(const ShmHeader &other) { - if (this != &other) { - strong_copy(other); - } - return *this; - } - - /** Strong copy operation */ - void strong_copy(const ShmHeader &other) {} - - /** Move constructor */ - ShmHeader(ShmHeader &&other) { - weak_move(other); - } - - /** Move operator */ - ShmHeader& operator=(ShmHeader &&other) { - if (this != &other) { - weak_move(other); - } - return *this; - } - - /** Move operation */ - void weak_move(ShmHeader &other) { - strong_copy(other); - } -}; - -/** - * A string of characters. - * */ -class NullContainer : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS), (TYPED_HEADER)) - - public: - //////////////////////////// - /// SHM Overrides - //////////////////////////// - - /** Default constructor */ - NullContainer() = default; - - /** Default shm constructor */ - void shm_init_main(TYPED_HEADER *header, - Allocator *alloc) { - shm_init_allocator(alloc); - shm_make_header(header); - } - - /** Move constructor */ - void shm_strong_move_main(TYPED_HEADER *header, - Allocator *alloc, - NullContainer &other) {} - - /** Copy constructor */ - void shm_strong_copy_main(TYPED_HEADER *header, - Allocator *alloc, - const NullContainer &other) {} - - /** - * Destroy the shared-memory data. - * */ - void shm_destroy_main() {} - - /** Store into shared memory */ - void shm_serialize_main() const {} - - /** Load from shared memory */ - void shm_deserialize_main() {} -}; - -} // namespace hshm::ipc - -namespace std { - -} // namespace std - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_INCLUDE_HERMES_DATA_STRUCTURES_INTERNAL_SHM_NULL_CONTAINER_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_smart_ptr.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_smart_ptr.h deleted file mode 100644 index 0ff93fd37..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/shm_smart_ptr.h +++ /dev/null @@ -1,57 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_INTERNAL_SHM_DATA_STRUCTURE_POINTER_H_ -#define HERMES_DATA_STRUCTURES_INTERNAL_SHM_DATA_STRUCTURE_POINTER_H_ - -namespace hshm::ipc { - -/** - * Indicates a data structure represents a memory paradigm for Shm. - * */ -class ShmSmartPointer {}; - -} // namespace hshm::ipc - -/** - * Enables a specific TypedPointer type to be serialized - * */ -#define SHM_SERIALIZE_OPS(TYPED_CLASS)\ - void operator>>(hipc::TypedPointer &ar) const {\ - shm_serialize(ar);\ - }\ - void operator>>(\ - hipc::TypedAtomicPointer &ar) const {\ - shm_serialize(ar);\ - } - -/** - * Enables a specific TypedPointer type to be deserialized - * */ -#define SHM_DESERIALIZE_OPS(TYPED_CLASS)\ - void operator<<(const hipc::TypedPointer &ar) {\ - shm_deserialize(ar);\ - }\ - void operator<<(\ - const hipc::TypedAtomicPointer &ar) {\ - shm_deserialize(ar);\ - } - -/** - * A macro for defining shared memory (de)serializations - * */ -#define SHM_SERIALIZE_DESERIALIZE_OPS(AR_TYPE)\ - SHM_SERIALIZE_OPS(AR_TYPE)\ - SHM_DESERIALIZE_OPS(AR_TYPE) - -#endif // HERMES_DATA_STRUCTURES_INTERNAL_SHM_DATA_STRUCTURE_POINTER_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_example.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_example.h deleted file mode 100644 index e25ecb254..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_example.h +++ /dev/null @@ -1,133 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_DATA_STRUCTURES_INTERNAL_SHM_CONTAINER_EXAMPLE_H_ -#define HERMES_INCLUDE_HERMES_DATA_STRUCTURES_INTERNAL_SHM_CONTAINER_EXAMPLE_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_container.h" -#include "hermes_shm/memory/memory_manager.h" - -namespace honey { - -class ShmContainerExample; - -#define CLASS_NAME ShmContainerExample -#define TYPED_CLASS ShmContainerExample - -class ShmContainerExample : public hipc::ShmContainer { - public: - /**==================================== - * Shm Overrides - * ===================================*/ - - /** Constructor. Empty. */ - explicit CLASS_NAME(hipc::Allocator *alloc) { - alloc_id_ = alloc->GetId(); - } - - /** Default initialization */ - void shm_init() { - SetNull(); - } - - /** Load from shared memory */ - void shm_deserialize_main() {} - - /** Destroy object */ - void shm_destroy_main() {} - - /** Internal copy operation */ - void shm_strong_copy_main(const CLASS_NAME &other) { - } - - /** Internal move operation */ - void shm_strong_move_main(CLASS_NAME &&other) { - } - - /** Check if header is NULL */ - bool IsNull() { - } - - /** Nullify object header */ - void SetNull() { - } - - public: - /**==================================== - * Variables & Types - * ===================================*/ - hipc::allocator_id_t alloc_id_; - - /**==================================== - * Constructors - * ===================================*/ - - /** Default constructor. Deleted. */ - CLASS_NAME() = delete; - - /** Move constructor. Deleted. */ - CLASS_NAME(CLASS_NAME &&other) = delete; - - /** Copy constructor. Deleted. */ - CLASS_NAME(const CLASS_NAME &other) = delete; - - /** Initialize container */ - void shm_init_container(hipc::Allocator *alloc) { - alloc_id_ = alloc->GetId(); - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Destructor. */ - HSHM_ALWAYS_INLINE ~CLASS_NAME() = default; - - /** Destruction operation */ - HSHM_ALWAYS_INLINE void shm_destroy() { - if (IsNull()) { return; } - shm_destroy_main(); - SetNull(); - } - - /**==================================== - * Header Operations - * ===================================*/ - - /** Get a typed pointer to the object */ - template - HSHM_ALWAYS_INLINE POINTER_T GetShmPointer() const { - return GetAllocator()->template Convert(this); - } - - /**==================================== - * Query Operations - * ===================================*/ - - /** Get the allocator for this container */ - HSHM_ALWAYS_INLINE hipc::Allocator* GetAllocator() const { - return HERMES_MEMORY_REGISTRY_REF.GetAllocator(alloc_id_); - } - - /** Get the shared-memory allocator id */ - HSHM_ALWAYS_INLINE hipc::allocator_id_t& GetAllocatorId() const { - return GetAllocator()->GetId(); - } -}; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif //HERMES_INCLUDE_HERMES_DATA_STRUCTURES_INTERNAL_SHM_CONTAINER_EXAMPLE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_template.h b/hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_template.h deleted file mode 100644 index d62d26170..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/internal/template/shm_container_base_template.h +++ /dev/null @@ -1,61 +0,0 @@ -public: -/**==================================== - * Variables & Types - * ===================================*/ -hipc::allocator_id_t alloc_id_; - -/**==================================== - * Constructors - * ===================================*/ - -/** Default constructor. Deleted. */ -CLASS_NAME() = delete; - -/** Move constructor. Deleted. */ -CLASS_NAME(CLASS_NAME &&other) = delete; - -/** Copy constructor. Deleted. */ -CLASS_NAME(const CLASS_NAME &other) = delete; - -/** Initialize container */ -void shm_init_container(hipc::Allocator *alloc) { - alloc_id_ = alloc->GetId(); -} - -/**==================================== - * Destructor - * ===================================*/ - -/** Destructor. */ -HSHM_ALWAYS_INLINE ~CLASS_NAME() = default; - -/** Destruction operation */ -HSHM_ALWAYS_INLINE void shm_destroy() { - if (IsNull()) { return; } - shm_destroy_main(); - SetNull(); -} - -/**==================================== - * Header Operations - * ===================================*/ - -/** Get a typed pointer to the object */ -template -HSHM_ALWAYS_INLINE POINTER_T GetShmPointer() const { - return GetAllocator()->template Convert(this); -} - -/**==================================== - * Query Operations - * ===================================*/ - -/** Get the allocator for this container */ -HSHM_ALWAYS_INLINE hipc::Allocator* GetAllocator() const { - return HERMES_MEMORY_REGISTRY_REF.GetAllocator(alloc_id_); -} - -/** Get the shared-memory allocator id */ -HSHM_ALWAYS_INLINE hipc::allocator_id_t& GetAllocatorId() const { - return GetAllocator()->GetId(); -} \ No newline at end of file diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/iqueue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/iqueue.h deleted file mode 100644 index a0a799a99..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/iqueue.h +++ /dev/null @@ -1,352 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_THREAD_UNSAFE_IQUEUE_H -#define HERMES_DATA_STRUCTURES_THREAD_UNSAFE_IQUEUE_H - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" - -namespace hshm::ipc { - -/** forward pointer for iqueue */ -template -class iqueue; - -/** represents an object within a iqueue */ -struct iqueue_entry { - OffsetPointer next_ptr_; -}; - -/** - * The iqueue iterator - * */ -template -struct iqueue_iterator_templ { - public: - /**< A shm reference to the containing iqueue object. */ - iqueue *iqueue_; - /**< A pointer to the entry in shared memory */ - iqueue_entry *entry_; - /**< A pointer to the entry prior to this one */ - iqueue_entry *prior_entry_; - - /** Default constructor */ - iqueue_iterator_templ() = default; - - /** Construct begin iterator */ - explicit iqueue_iterator_templ(iqueue &iqueue, - iqueue_entry *entry) - : iqueue_(&iqueue), entry_(entry), prior_entry_(nullptr) {} - - /** Copy constructor */ - iqueue_iterator_templ(const iqueue_iterator_templ &other) - : iqueue_(other.iqueue_) { - iqueue_ = other.iqueue_; - entry_ = other.entry_; - prior_entry_ = other.prior_entry_; - } - - /** Assign this iterator from another iterator */ - iqueue_iterator_templ& operator=(const iqueue_iterator_templ &other) { - if (this != &other) { - iqueue_ = other.iqueue_; - entry_ = other.entry_; - prior_entry_ = other.prior_entry_; - } - return *this; - } - - /** Get the object the iterator points to */ - T* operator*() { - return reinterpret_cast(entry_); - } - - /** Get the object the iterator points to */ - T* operator*() const { - return reinterpret_cast(entry_); - } - - /** Get the next iterator (in place) */ - iqueue_iterator_templ& operator++() { - if (is_end()) { return *this; } - prior_entry_ = entry_; - entry_ = iqueue_->GetAllocator()->template - Convert(entry_->next_ptr_); - return *this; - } - - /** Return the next iterator */ - iqueue_iterator_templ operator++(int) const { - iqueue_iterator_templ next_iter(*this); - ++next_iter; - return next_iter; - } - - /** Return the iterator at count after this one */ - iqueue_iterator_templ operator+(size_t count) const { - iqueue_iterator_templ pos(*this); - for (size_t i = 0; i < count; ++i) { - ++pos; - } - return pos; - } - - /** Get the iterator at count after this one (in-place) */ - void operator+=(size_t count) { - iqueue_iterator_templ pos = (*this) + count; - entry_ = pos.entry_; - prior_entry_ = pos.prior_entry_; - } - - /** Determine if two iterators are equal */ - friend bool operator==(const iqueue_iterator_templ &a, - const iqueue_iterator_templ &b) { - return (a.is_end() && b.is_end()) || (a.entry_ == b.entry_); - } - - /** Determine if two iterators are inequal */ - friend bool operator!=(const iqueue_iterator_templ &a, - const iqueue_iterator_templ &b) { - return !(a.is_end() && b.is_end()) && (a.entry_ != b.entry_); - } - - /** Determine whether this iterator is the end iterator */ - bool is_end() const { - return entry_ == nullptr; - } - - /** Determine whether this iterator is the begin iterator */ - bool is_begin() const { - if (entry_) { - return prior_entry_ == nullptr; - } else { - return false; - } - } -}; - -/** - * MACROS used to simplify the iqueue namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME iqueue -#define TYPED_CLASS iqueue -#define TYPED_HEADER ShmHeader> - -/** - * Doubly linked iqueue implementation - * */ -template -class iqueue : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - OffsetPointer head_ptr_; - size_t length_; - - /**==================================== - * Typedefs - * ===================================*/ - - /** forward iterator typedef */ - typedef iqueue_iterator_templ iterator_t; - /** const forward iterator typedef */ - typedef iqueue_iterator_templ citerator_t; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit iqueue(Allocator *alloc) { - shm_init_container(alloc); - length_ = 0; - head_ptr_.SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit iqueue(Allocator *alloc, - const iqueue &other) { - shm_init_container(alloc); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - iqueue& operator=(const iqueue &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor + operator */ - void shm_strong_copy_construct_and_op(const iqueue &other) { - memcpy((void*)this, (void*)&other, sizeof(*this)); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - iqueue(Allocator *alloc, iqueue &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - memcpy((void*)this, (void*)&other, sizeof(*this)); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - iqueue& operator=(iqueue &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (this != &other) { - memcpy((void *) this, (void *) &other, sizeof(*this)); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Check if the iqueue is null */ - bool IsNull() { - return length_ == 0; - } - - /** Set the iqueue to null */ - void SetNull() { - length_ = 0; - } - - /** SHM destructor. */ - void shm_destroy_main() { - clear(); - } - - /**==================================== - * SHM Deserialization - * ===================================*/ - - /** Load from shared memory */ - void shm_deserialize_main() {} - - /**==================================== - * iqueue Methods - * ===================================*/ - - /** Construct an element at \a pos position in the iqueue */ - void enqueue(T *entry) { - OffsetPointer entry_ptr = GetAllocator()-> - template Convert(entry); - reinterpret_cast(entry)->next_ptr_ = head_ptr_; - head_ptr_ = entry_ptr; - ++length_; - } - - /** Dequeue the first element */ - T* dequeue() { - if (size() == 0) { return nullptr; } - auto entry = GetAllocator()-> - template Convert(head_ptr_); - head_ptr_ = entry->next_ptr_; - --length_; - return reinterpret_cast(entry); - } - - /** Dequeue the element at the iterator position */ - T* dequeue(iqueue_iterator_templ pos) { - if (pos.prior_entry_ == nullptr) { - return dequeue(); - } - auto entry = *pos; - auto prior_cast = reinterpret_cast(pos.prior_entry_); - auto pos_cast = reinterpret_cast(pos.entry_); - prior_cast->next_ptr_ = pos_cast->next_ptr_; - --length_; - return reinterpret_cast(entry); - } - - /** Peek the first element of the queue */ - T* peek() { - if (size() == 0) { return nullptr; } - auto entry = GetAllocator()-> - template Convert(head_ptr_); - return reinterpret_cast(entry); - } - - /** Destroy all elements in the iqueue */ - void clear() { - while (size()) { - dequeue(); - } - } - - /** Get the number of elements in the iqueue */ - size_t size() const { - return length_; - } - - /**==================================== - * Iterators - * ===================================*/ - - /** Forward iterator begin */ - iterator_t begin() { - if (size() == 0) { return end(); } - auto head = GetAllocator()->template - Convert(head_ptr_); - return iterator_t(*this, head); - } - - /** Forward iterator end */ - iterator_t const end() { - return iterator_t(*this, nullptr); - } - - /** Constant forward iterator begin */ - citerator_t cbegin() const { - if (size() == 0) { return cend(); } - auto head = GetAllocator()->template - Convert(head_ptr_); - return citerator_t(const_cast(*this), head); - } - - /** Constant forward iterator end */ - citerator_t const cend() const { - return citerator_t(const_cast(*this), nullptr); - } -}; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_DATA_STRUCTURES_THREAD_UNSAFE_IQUEUE_H diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/list.h b/hermes_shm/include/hermes_shm/data_structures/ipc/list.h deleted file mode 100644 index 6f00c2398..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/list.h +++ /dev/null @@ -1,489 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_THREAD_UNSAFE_LIST_H_ -#define HERMES_DATA_STRUCTURES_THREAD_UNSAFE_LIST_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/data_structures/containers/functional.h" - -#include - -namespace hshm::ipc { - -/** forward pointer for list */ -template -class list; - -/** represents an object within a list */ -template -struct list_entry { - public: - OffsetPointer next_ptr_, prior_ptr_; - ShmArchive data_; -}; - -/** - * The list iterator - * */ -template -struct list_iterator_templ { - public: - /**< A shm reference to the containing list object. */ - list *list_; - /**< A pointer to the entry in shared memory */ - list_entry *entry_; - /**< The offset of the entry in the shared-memory allocator */ - OffsetPointer entry_ptr_; - - /** Default constructor */ - list_iterator_templ() = default; - - /** Construct an iterator */ - explicit list_iterator_templ(list &list, - list_entry *entry, - OffsetPointer entry_ptr) - : list_(&list), entry_(entry), entry_ptr_(entry_ptr) {} - - /** Copy constructor */ - list_iterator_templ(const list_iterator_templ &other) { - list_ = other.list_; - entry_ = other.entry_; - entry_ptr_ = other.entry_ptr_; - } - - /** Assign this iterator from another iterator */ - list_iterator_templ& operator=(const list_iterator_templ &other) { - if (this != &other) { - list_ = other.list_; - entry_ = other.entry_; - entry_ptr_ = other.entry_ptr_; - } - return *this; - } - - /** Get the object the iterator points to */ - T& operator*() { - return entry_->data_.get_ref(); - } - - /** Get the object the iterator points to */ - const T& operator*() const { - return entry_->data_.get_ref(); - } - - /** Get the next iterator (in place) */ - list_iterator_templ& operator++() { - if (is_end()) { return *this; } - entry_ptr_ = entry_->next_ptr_; - entry_ = list_->GetAllocator()->template - Convert>(entry_->next_ptr_); - return *this; - } - - /** Get the prior iterator (in place) */ - list_iterator_templ& operator--() { - if (is_end() || is_begin()) { return *this; } - entry_ptr_ = entry_->prior_ptr_; - entry_ = list_->GetAllocator()->template - Convert>(entry_->prior_ptr_); - return *this; - } - - /** Return the next iterator */ - list_iterator_templ operator++(int) const { - list_iterator_templ next_iter(*this); - ++next_iter; - return next_iter; - } - - /** Return the prior iterator */ - list_iterator_templ operator--(int) const { - list_iterator_templ prior_iter(*this); - --prior_iter; - return prior_iter; - } - - /** Return the iterator at count after this one */ - list_iterator_templ operator+(size_t count) const { - list_iterator_templ pos(*this); - for (size_t i = 0; i < count; ++i) { - ++pos; - } - return pos; - } - - /** Return the iterator at count before this one */ - list_iterator_templ operator-(size_t count) const { - list_iterator_templ pos(*this); - for (size_t i = 0; i < count; ++i) { - --pos; - } - return pos; - } - - /** Get the iterator at count after this one (in-place) */ - void operator+=(size_t count) { - list_iterator_templ pos = (*this) + count; - entry_ = pos.entry_; - entry_ptr_ = pos.entry_ptr_; - } - - /** Get the iterator at count before this one (in-place) */ - void operator-=(size_t count) { - list_iterator_templ pos = (*this) - count; - entry_ = pos.entry_; - entry_ptr_ = pos.entry_ptr_; - } - - /** Determine if two iterators are equal */ - friend bool operator==(const list_iterator_templ &a, - const list_iterator_templ &b) { - return (a.is_end() && b.is_end()) || (a.entry_ == b.entry_); - } - - /** Determine if two iterators are inequal */ - friend bool operator!=(const list_iterator_templ &a, - const list_iterator_templ &b) { - return !(a.is_end() && b.is_end()) && (a.entry_ != b.entry_); - } - - /** Determine whether this iterator is the end iterator */ - bool is_end() const { - return entry_ == nullptr; - } - - /** Determine whether this iterator is the begin iterator */ - bool is_begin() const { - if (entry_) { - return entry_->prior_ptr_.IsNull(); - } else { - return false; - } - } -}; - -/** - * MACROS used to simplify the list namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME list -#define TYPED_CLASS list -#define TYPED_HEADER ShmHeader> - -/** - * Doubly linked list implementation - * */ -template -class list : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - OffsetPointer head_ptr_, tail_ptr_; - size_t length_; - - public: - /**==================================== - * Typedefs - * ===================================*/ - - /** forward iterator typedef */ - typedef list_iterator_templ iterator_t; - /** const forward iterator typedef */ - typedef list_iterator_templ citerator_t; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit list(Allocator *alloc) { - shm_init_container(alloc); - SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit list(Allocator *alloc, - const list &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - list& operator=(const list &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor. From std::list */ - explicit list(Allocator *alloc, - std::list &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op>(other); - } - - /** SHM copy assignment operator. From std::list. */ - list& operator=(const std::list &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op>(other); - } - return *this; - } - - /** SHM copy constructor + operator main */ - template - void shm_strong_copy_construct_and_op(const ListT &other) { - for (auto iter = other.cbegin(); iter != other.cend(); ++iter) { - emplace_back(*iter); - } - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - list(Allocator *alloc, list &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - memcpy((void*) this, (void *) &other, sizeof(*this)); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - list& operator=(list &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - memcpy((void *) this, (void *) &other, sizeof(*this)); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** SHM destructor. */ - void shm_destroy_main() { - clear(); - } - - /** Check if the list is empty */ - bool IsNull() const { - return length_ == 0; - } - - /** Sets this list as empty */ - void SetNull() { - length_ = 0; - head_ptr_.SetNull(); - tail_ptr_.SetNull(); - } - - /**==================================== - * list Methods - * ===================================*/ - - /** Construct an element at the back of the list */ - template - void emplace_back(Args&&... args) { - emplace(end(), std::forward(args)...); - } - - /** Construct an element at the beginning of the list */ - template - void emplace_front(Args&&... args) { - emplace(begin(), std::forward(args)...); - } - - /** Construct an element at \a pos position in the list */ - template - void emplace(iterator_t pos, Args&&... args) { - OffsetPointer entry_ptr; - auto entry = _create_entry(entry_ptr, std::forward(args)...); - if (size() == 0) { - entry->prior_ptr_.SetNull(); - entry->next_ptr_.SetNull(); - head_ptr_ = entry_ptr; - tail_ptr_ = entry_ptr; - } else if (pos.is_begin()) { - entry->prior_ptr_.SetNull(); - entry->next_ptr_ = head_ptr_; - auto head = GetAllocator()->template - Convert>(tail_ptr_); - head->prior_ptr_ = entry_ptr; - head_ptr_ = entry_ptr; - } else if (pos.is_end()) { - entry->prior_ptr_ = tail_ptr_; - entry->next_ptr_.SetNull(); - auto tail = GetAllocator()->template - Convert>(tail_ptr_); - tail->next_ptr_ = entry_ptr; - tail_ptr_ = entry_ptr; - } else { - auto next = GetAllocator()->template - Convert>(pos.entry_->next_ptr_); - auto prior = GetAllocator()->template - Convert>(pos.entry_->prior_ptr_); - entry->next_ptr_ = pos.entry_->next_ptr_; - entry->prior_ptr_ = pos.entry_->prior_ptr_; - next->prior_ptr_ = entry_ptr; - prior->next_ptr_ = entry_ptr; - } - ++length_; - } - - /** Erase element with ID */ - void erase(const T &entry) { - auto iter = find(entry); - erase(iter); - } - - /** Erase the element at pos */ - void erase(iterator_t pos) { - erase(pos, pos+1); - } - - /** Erase all elements between first and last */ - void erase(iterator_t first, - iterator_t last) { - if (first.is_end()) { return; } - auto first_prior_ptr = first.entry_->prior_ptr_; - auto pos = first; - while (pos != last) { - auto next = pos + 1; - HSHM_DESTROY_AR(pos.entry_->data_) - GetAllocator()->Free(pos.entry_ptr_); - --length_; - pos = next; - } - - if (first_prior_ptr.IsNull()) { - head_ptr_ = last.entry_ptr_; - } else { - auto first_prior = GetAllocator()->template - Convert>(first_prior_ptr); - first_prior->next_ptr_ = last.entry_ptr_; - } - - if (last.entry_ptr_.IsNull()) { - tail_ptr_ = first_prior_ptr; - } else { - last.entry_->prior_ptr_ = first_prior_ptr; - } - } - - /** Destroy all elements in the list */ - void clear() { - erase(begin(), end()); - } - - /** Get the object at the front of the list */ - T& front() { - return *begin(); - } - - /** Get the object at the back of the list */ - T& back() { - return *last(); - } - - /** Get the number of elements in the list */ - size_t size() const { - return length_; - } - - /** Find an element in this list */ - iterator_t find(const T &entry) { - return hshm::find(begin(), end(), entry); - } - - /**==================================== - * Iterators - * ===================================*/ - - /** Forward iterator begin */ - iterator_t begin() { - if (size() == 0) { return end(); } - auto head = GetAllocator()->template - Convert>(head_ptr_); - return iterator_t(*this, - head, head_ptr_); - } - - /** Last iterator begin */ - iterator_t last() { - if (size() == 0) { return end(); } - auto tail = GetAllocator()->template - Convert>(tail_ptr_); - return iterator_t(*this, tail, tail_ptr_); - } - - /** Forward iterator end */ - iterator_t end() { - return iterator_t(*this, nullptr, OffsetPointer::GetNull()); - } - - /** Constant forward iterator begin */ - citerator_t cbegin() const { - if (size() == 0) { return cend(); } - auto head = GetAllocator()->template - Convert>(head_ptr_); - return citerator_t(const_cast(*this), - head, head_ptr_); - } - - /** Constant forward iterator end */ - citerator_t cend() const { - return iterator_t(const_cast(*this), - nullptr, OffsetPointer::GetNull()); - } - - private: - template - HSHM_ALWAYS_INLINE list_entry* _create_entry( - OffsetPointer &p, Args&& ...args) { - auto entry = GetAllocator()->template - AllocateObjs>(1, p); - HSHM_MAKE_AR(entry->data_, GetAllocator(), std::forward(args)...) - return entry; - } -}; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_DATA_STRUCTURES_THREAD_UNSAFE_LIST_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h deleted file mode 100644 index b04cb436d..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/mpsc_queue.h +++ /dev/null @@ -1,215 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/thread/lock.h" -#include "vector.h" -#include "pair.h" -#include "_queue.h" - -namespace hshm::ipc { - -/** Forward declaration of mpsc_queue */ -template -class mpsc_queue; - -/** - * MACROS used to simplify the mpsc_queue namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME mpsc_queue -#define TYPED_CLASS mpsc_queue -#define TYPED_HEADER ShmHeader> - -/** - * A queue optimized for multiple producers (emplace) with a single - * consumer (pop). - * */ -template -class mpsc_queue : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - ShmArchive>> queue_; - std::atomic<_qtok_t> tail_; - std::atomic<_qtok_t> head_; - RwLock lock_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit mpsc_queue(Allocator *alloc, - size_t depth = 1024) { - shm_init_container(alloc); - HSHM_MAKE_AR(queue_, GetAllocator(), depth); - SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit mpsc_queue(Allocator *alloc, - const mpsc_queue &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - mpsc_queue& operator=(const mpsc_queue &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor + operator main */ - void shm_strong_copy_construct_and_op(const mpsc_queue &other) { - head_ = other.head_.load(); - tail_ = other.tail_.load(); - (*queue_) = (*other.queue_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - mpsc_queue(Allocator *alloc, - mpsc_queue &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - head_ = other.head_.load(); - tail_ = other.tail_.load(); - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - mpsc_queue& operator=(mpsc_queue &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - head_ = other.head_.load(); - tail_ = other.tail_.load(); - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** SHM destructor. */ - void shm_destroy_main() { - (*queue_).shm_destroy(); - } - - /** Check if the list is empty */ - bool IsNull() const { - return (*queue_).IsNull(); - } - - /** Sets this list as empty */ - void SetNull() { - head_ = 0; - tail_ = 0; - } - - /**==================================== - * MPSC Queue Methods - * ===================================*/ - - /** Construct an element at \a pos position in the list */ - template - qtok_t emplace(Args&&... args) { - // Allocate a slot in the queue - // The slot is marked NULL, so pop won't do anything if context switch - _qtok_t head = head_.load(); - _qtok_t tail = tail_.fetch_add(1); - size_t size = tail - head + 1; - - // Check if there's space in the queue. - if (size > (*queue_).size()) { - while (true) { - head = head_.load(); - size = tail - head + 1; - if (size <= (*queue_).size()) { - break; - } - HERMES_THREAD_MODEL->Yield(); - } - } - - // Emplace into queue at our slot - uint32_t idx = tail % (*queue_).size(); - auto iter = (*queue_).begin() + idx; - (*queue_).replace(iter, - hshm::PiecewiseConstruct(), - make_argpack(), - make_argpack(std::forward(args)...)); - - // Let pop know that the data is fully prepared - pair &entry = (*iter); - entry.GetFirst().SetBits(1); - return qtok_t(tail); - } - - public: - /** Consumer pops the head object */ - qtok_t pop(T &val) { - // Don't pop if there's no entries - _qtok_t head = head_.load(); - _qtok_t tail = tail_.load(); - if (head >= tail) { - return qtok_t::GetNull(); - } - - // Pop the element, but only if it's marked valid - _qtok_t idx = head % (*queue_).size(); - hipc::pair &entry = (*queue_)[idx]; - if (entry.GetFirst().Any(1)) { - val = std::move(entry.GetSecond()); - entry.GetFirst().Clear(); - head_.fetch_add(1); - return qtok_t(head); - } else { - return qtok_t::GetNull(); - } - } -}; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_mpsc_queue_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/pair.h b/hermes_shm/include/hermes_shm/data_structures/ipc/pair.h deleted file mode 100644 index be1b0bbb5..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/pair.h +++ /dev/null @@ -1,211 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_DATA_STRUCTURES_PAIR_H_ -#define HERMES_INCLUDE_HERMES_DATA_STRUCTURES_PAIR_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/data_structures/smart_ptr/smart_ptr_base.h" - -namespace hshm::ipc { - -/** forward declaration for string */ -template -class pair; - -/** -* MACROS used to simplify the string namespace -* Used as inputs to the SHM_CONTAINER_TEMPLATE -* */ -#define CLASS_NAME pair -#define TYPED_CLASS pair -#define TYPED_HEADER ShmHeader - -/** -* A pair of two objects. -* */ -template -class pair : public ShmContainer { - public: - /**==================================== - * Variables - * ===================================*/ - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - ShmArchive first_; - ShmArchive second_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit pair(Allocator *alloc) { - shm_init_container(alloc); - HSHM_MAKE_AR0(first_, GetAllocator()) - HSHM_MAKE_AR0(second_, GetAllocator()) - } - - /**==================================== - * Emplace Constructors - * ===================================*/ - - /** SHM constructor. Move parameters. */ - explicit pair(Allocator *alloc, - FirstT &&first, SecondT &&second) { - shm_init_container(alloc); - HSHM_MAKE_AR(first_, GetAllocator(), - std::forward(first)) - HSHM_MAKE_AR(second_, GetAllocator(), - std::forward(second)) - } - - /** SHM constructor. Copy parameters. */ - explicit pair(Allocator *alloc, - const FirstT &first, const SecondT &second) { - shm_init_container(alloc); - HSHM_MAKE_AR(first_, GetAllocator(), first) - HSHM_MAKE_AR(second_, GetAllocator(), second) - } - - /** SHM constructor. Piecewise emplace. */ - template - explicit pair(Allocator *alloc, - PiecewiseConstruct &&hint, - FirstArgPackT &&first, - SecondArgPackT &&second) { - shm_init_container(alloc); - HSHM_MAKE_AR_PW(first_, GetAllocator(), - std::forward(first)) - HSHM_MAKE_AR_PW(second_, GetAllocator(), - std::forward(second)) - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor. From pair. */ - explicit pair(Allocator *alloc, const pair &other) { - shm_init_container(alloc); - shm_strong_copy_construct(other); - } - - /** SHM copy constructor main */ - void shm_strong_copy_construct(const pair &other) { - HSHM_MAKE_AR(first_, GetAllocator(), *other.first_) - HSHM_MAKE_AR(second_, GetAllocator(), *other.second_) - } - - /** SHM copy assignment operator. From pair. */ - pair& operator=(const pair &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_assign_op(other); - } - return *this; - } - - /** SHM copy assignment operator main */ - void shm_strong_copy_assign_op(const pair &other) { - (first_.get_ref()) = (*other.first_); - (second_.get_ref()) = (*other.second_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. From pair. */ - explicit pair(Allocator *alloc, pair &&other) { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - HSHM_MAKE_AR(first_, GetAllocator(), - std::forward(*other.first_)) - HSHM_MAKE_AR(second_, GetAllocator(), - std::forward(*other.second_)) - } else { - shm_strong_copy_construct(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. From pair. */ - pair& operator=(pair &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - (*first_) = std::move(*other.first_); - (*second_) = std::move(*other.second_); - other.SetNull(); - } else { - shm_strong_copy_assign_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Check if the pair is empty */ - HSHM_ALWAYS_INLINE bool IsNull() const { - return false; - } - - /** Sets this pair as empty */ - HSHM_ALWAYS_INLINE void SetNull() {} - - /** Destroy the shared-memory data */ - HSHM_ALWAYS_INLINE void shm_destroy_main() { - HSHM_DESTROY_AR(first_) - HSHM_DESTROY_AR(second_) - } - - /**==================================== - * pair Methods - * ===================================*/ - - /** Get the first object */ - HSHM_ALWAYS_INLINE FirstT& GetFirst() { return first_.get_ref(); } - - /** Get the first object (const) */ - HSHM_ALWAYS_INLINE FirstT& GetFirst() const { return first_.get_ref(); } - - /** Get the second object */ - HSHM_ALWAYS_INLINE SecondT& GetSecond() { return second_.get_ref(); } - - /** Get the second object (const) */ - HSHM_ALWAYS_INLINE SecondT& GetSecond() const { return second_.get_ref(); } - - /** Get the first object (treated as key) */ - HSHM_ALWAYS_INLINE FirstT& GetKey() { return first_.get_ref(); } - - /** Get the first object (treated as key) (const) */ - HSHM_ALWAYS_INLINE FirstT& GetKey() const { return first_.get_ref(); } - - /** Get the second object (treated as value) */ - HSHM_ALWAYS_INLINE SecondT& GetVal() { return second_.get_ref(); } - - /** Get the second object (treated as value) (const) */ - HSHM_ALWAYS_INLINE SecondT& GetVal() const { return second_.get_ref(); } -}; - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -} // namespace hshm::ipc - -#endif // HERMES_INCLUDE_HERMES_DATA_STRUCTURES_PAIR_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/slist.h b/hermes_shm/include/hermes_shm/data_structures/ipc/slist.h deleted file mode 100644 index 1f79d018c..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/slist.h +++ /dev/null @@ -1,506 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_THREAD_UNSAFE_Sslist_H -#define HERMES_DATA_STRUCTURES_THREAD_UNSAFE_Sslist_H - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/data_structures/containers/functional.h" - -namespace hshm::ipc { - -/** forward pointer for slist */ -template -class slist; - -/** represents an object within a slist */ -template -struct slist_entry { - public: - OffsetPointer next_ptr_; - ShmArchive data_; - - /** Constructor */ - template - explicit slist_entry(Allocator *alloc, Args&& ...args) { - HSHM_MAKE_AR(data_, alloc, std::forward(args)...) - } -}; - -/** - * The slist iterator - * */ -template -struct slist_iterator_templ { - public: - /**< A shm reference to the containing slist object. */ - slist *slist_; - /**< A pointer to the entry in shared memory */ - slist_entry *entry_; - /**< The offset of the entry in the shared-memory allocator */ - OffsetPointer entry_ptr_; - - /** Default constructor */ - slist_iterator_templ() = default; - - /** Construct an iterator */ - explicit slist_iterator_templ(slist& slist, - slist_entry *entry, - OffsetPointer entry_ptr) - : slist_(&slist), entry_(entry), entry_ptr_(entry_ptr) {} - - /** Copy constructor */ - slist_iterator_templ(const slist_iterator_templ &other) { - slist_ = other.slist_; - entry_ = other.entry_; - entry_ptr_ = other.entry_ptr_; - } - - /** Assign this iterator from another iterator */ - slist_iterator_templ& operator=(const slist_iterator_templ &other) { - if (this != &other) { - slist_ = other.slist_; - entry_ = other.entry_; - entry_ptr_ = other.entry_ptr_; - } - return *this; - } - - /** Get the object the iterator points to */ - T& operator*() { - return entry_->data_.get_ref(); - } - - /** Get the object the iterator points to */ - const T& operator*() const { - return entry_->data_.get_ref(); - } - - /** Get the next iterator (in place) */ - slist_iterator_templ& operator++() { - if (is_end()) { return *this; } - entry_ptr_ = entry_->next_ptr_; - entry_ = slist_->GetAllocator()->template - Convert>(entry_->next_ptr_); - return *this; - } - - /** Get the prior iterator (in place) */ - slist_iterator_templ& operator--() { - if (is_end() || is_begin()) { return *this; } - entry_ptr_ = entry_->prior_ptr_; - entry_ = slist_->GetAllocator()->template - Convert>(entry_->prior_ptr_); - return *this; - } - - /** Return the next iterator */ - slist_iterator_templ operator++(int) const { - slist_iterator_templ next_iter(*this); - ++next_iter; - return next_iter; - } - - /** Return the prior iterator */ - slist_iterator_templ operator--(int) const { - slist_iterator_templ prior_iter(*this); - --prior_iter; - return prior_iter; - } - - /** Return the iterator at count after this one */ - slist_iterator_templ operator+(size_t count) const { - slist_iterator_templ pos(*this); - for (size_t i = 0; i < count; ++i) { - ++pos; - } - return pos; - } - - /** Return the iterator at count before this one */ - slist_iterator_templ operator-(size_t count) const { - slist_iterator_templ pos(*this); - for (size_t i = 0; i < count; ++i) { - --pos; - } - return pos; - } - - /** Get the iterator at count after this one (in-place) */ - void operator+=(size_t count) { - slist_iterator_templ pos = (*this) + count; - entry_ = pos.entry_; - entry_ptr_ = pos.entry_ptr_; - } - - /** Get the iterator at count before this one (in-place) */ - void operator-=(size_t count) { - slist_iterator_templ pos = (*this) - count; - entry_ = pos.entry_; - entry_ptr_ = pos.entry_ptr_; - } - - /** Determine if two iterators are equal */ - friend bool operator==(const slist_iterator_templ &a, - const slist_iterator_templ &b) { - return (a.is_end() && b.is_end()) || (a.entry_ == b.entry_); - } - - /** Determine if two iterators are inequal */ - friend bool operator!=(const slist_iterator_templ &a, - const slist_iterator_templ &b) { - return !(a.is_end() && b.is_end()) && (a.entry_ != b.entry_); - } - - /** Determine whether this iterator is the end iterator */ - bool is_end() const { - return entry_ == nullptr; - } - - /** Determine whether this iterator is the begin iterator */ - bool is_begin() const { - if (entry_) { - return entry_ptr_ == slist_->head_ptr_; - } else { - return false; - } - } -}; - -/** - * MACROS used to simplify the slist namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME slist -#define TYPED_CLASS slist -#define TYPED_HEADER ShmHeader> - -/** - * Doubly linked slist implementation - * */ -template -class slist : public ShmContainer { - public: - /**==================================== - * Variables - * ===================================*/ - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - OffsetPointer head_ptr_, tail_ptr_; - size_t length_; - - /**==================================== - * Iterator Typedefs - * ===================================*/ - /** forward iterator typedef */ - typedef slist_iterator_templ iterator_t; - /** const forward iterator typedef */ - typedef slist_iterator_templ citerator_t; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit slist(Allocator *alloc) { - shm_init_container(alloc); - SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** Strong copy operation */ - void strong_copy(const slist &other) { - head_ptr_ = other.head_ptr_; - tail_ptr_ = other.tail_ptr_; - length_ = other.length_; - } - - /** SHM copy constructor. From slist. */ - explicit slist(Allocator *alloc, - const slist &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator. From slist. */ - slist& operator=(const slist &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor. From std::list */ - explicit slist(Allocator *alloc, - std::list &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op>(other); - } - - /** SHM copy assignment operator. From std::list. */ - slist& operator=(const std::list &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op>(other); - } - return *this; - } - - /** SHM copy constructor + operator main */ - template - void shm_strong_copy_construct_and_op(const ListT &other) { - for (auto iter = other.cbegin(); iter != other.cend(); ++iter) { - emplace_back(*iter); - } - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. From slist. */ - slist(Allocator *alloc, slist &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. From slist. */ - slist& operator=(slist &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Check if the list is empty */ - bool IsNull() const { - return length_ == 0; - } - - /** Sets this list as empty */ - void SetNull() { - length_ = 0; - head_ptr_.SetNull(); - tail_ptr_.SetNull(); - } - - /** Destroy all shared memory allocated by the slist */ - void shm_destroy_main() { - clear(); - } - - /**==================================== - * slist Methods - * ===================================*/ - - /** Construct an element at the back of the slist */ - template - void emplace_back(Args&&... args) { - emplace(end(), std::forward(args)...); - } - - /** Construct an element at the beginning of the slist */ - template - void emplace_front(Args&&... args) { - emplace(begin(), std::forward(args)...); - } - - /** Construct an element at \a pos position in the slist */ - template - void emplace(iterator_t pos, Args&&... args) { - OffsetPointer entry_ptr; - auto entry = _create_entry(entry_ptr, std::forward(args)...); - if (size() == 0) { - entry->next_ptr_.SetNull(); - head_ptr_ = entry_ptr; - tail_ptr_ = entry_ptr; - } else if (pos.is_begin()) { - entry->next_ptr_ = head_ptr_; - head_ptr_ = entry_ptr; - } else if (pos.is_end()) { - entry->next_ptr_.SetNull(); - auto tail = GetAllocator()->template - Convert>(tail_ptr_); - tail->next_ptr_ = entry_ptr; - tail_ptr_ = entry_ptr; - } else { - auto prior_iter = find_prior(pos); - slist_entry *prior = prior_iter.entry_; - entry->next_ptr_ = pos.entry_->next_ptr_; - prior->next_ptr_ = entry_ptr; - } - ++length_; - } - - /** Find the element prior to an slist_entry */ - iterator_t find_prior(iterator_t pos) { - if (pos.is_end()) { - return last(); - } else if (pos.is_begin()) { - return end(); - } else { - iterator_t prior_iter = end(); - for (auto iter = begin(); !iter.is_end(); ++iter) { - if (iter == pos) { - return prior_iter; - } - prior_iter = iter; - } - return prior_iter; - } - } - - /** Erase element with ID */ - void erase(const T &entry) { - auto iter = find(entry); - erase(iter); - } - - /** Erase the element at pos */ - void erase(iterator_t pos) { - erase(pos, pos+1); - } - - /** Erase all elements between first and last */ - void erase(iterator_t first, - iterator_t last) { - if (first.is_end()) { return; } - auto first_prior = find_prior(first); - auto pos = first; - while (pos != last) { - auto next = pos + 1; - HSHM_DESTROY_AR(pos.entry_->data_) - GetAllocator()->Free(pos.entry_ptr_); - --length_; - pos = next; - } - - if (first_prior.is_end()) { - head_ptr_ = last.entry_ptr_; - } else { - first_prior.entry_->next_ptr_ = last.entry_ptr_; - } - - if (last.entry_ptr_.IsNull()) { - tail_ptr_ = first_prior.entry_ptr_; - } - } - - /** Destroy all elements in the slist */ - void clear() { - erase(begin(), end()); - } - - /** Get the object at the front of the slist */ - T& front() { - return *begin(); - } - - /** Get the object at the back of the slist */ - T& back() { - return *last(); - } - - /** Get the number of elements in the slist */ - size_t size() const { - if (!IsNull()) { - return length_; - } - return 0; - } - - /** Find an element in this slist */ - iterator_t find(const T &entry) { - return hshm::find(begin(), end(), entry); - } - - /**==================================== - * Iterators - * ===================================*/ - - /** Forward iterator begin */ - iterator_t begin() { - if (size() == 0) { return end(); } - auto head = GetAllocator()->template - Convert>(head_ptr_); - return iterator_t(*this, head, head_ptr_); - } - - /** Forward iterator end */ - iterator_t end() { - return iterator_t(*this, - nullptr, OffsetPointer::GetNull()); - } - - /** Forward iterator to last entry of list */ - iterator_t last() { - if (size() == 0) { return end(); } - auto tail = GetAllocator()->template - Convert>(tail_ptr_); - return iterator_t(*this, tail, tail_ptr_); - } - - /** Constant forward iterator begin */ - citerator_t cbegin() const { - if (size() == 0) { return cend(); } - auto head = GetAllocator()->template - Convert>(head_ptr_); - return citerator_t(const_cast(*this), head, head_ptr_); - } - - /** Constant forward iterator end */ - citerator_t cend() const { - return iterator_t(const_cast(*this), - nullptr, OffsetPointer::GetNull()); - } - - private: - template - slist_entry* _create_entry(OffsetPointer &p, Args&& ...args) { - auto entry = GetAllocator()->template - AllocateObjs>(1, p); - HSHM_MAKE_AR(entry->data_, GetAllocator(), std::forward(args)...) - return entry; - } -}; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_DATA_STRUCTURES_THREAD_UNSAFE_Sslist_H diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h deleted file mode 100644 index c33d9ed5c..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/split_ticket_queue.h +++ /dev/null @@ -1,188 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM__DATA_STRUCTURES_IPC_SPLIT_TICKET_QUEUE_H_ -#define HERMES_SHM__DATA_STRUCTURES_IPC_SPLIT_TICKET_QUEUE_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/thread/lock.h" -#include "vector.h" -#include "ticket_queue.h" - -namespace hshm::ipc { - -/** Forward declaration of split_ticket_queue */ -template -class split_ticket_queue; - -/** - * MACROS used to simplify the split_ticket_queue namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME split_ticket_queue -#define TYPED_CLASS split_ticket_queue -#define TYPED_HEADER ShmHeader> - -/** - * A MPMC queue for allocating tickets. Handles concurrency - * without blocking. - * */ -template -class split_ticket_queue : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - ShmArchive>> splits_; - std::atomic rr_tail_, rr_head_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit split_ticket_queue(Allocator *alloc, - size_t depth_per_split = 1024, - size_t split = 0) { - shm_init_container(alloc); - if (split == 0) { - split = HERMES_SYSTEM_INFO->ncpu_; - } - HSHM_MAKE_AR(splits_, GetAllocator(), split, depth_per_split); - SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit split_ticket_queue(Allocator *alloc, - const split_ticket_queue &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - split_ticket_queue& operator=(const split_ticket_queue &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor + operator main */ - void shm_strong_copy_construct_and_op(const split_ticket_queue &other) { - (*splits_) = (*other.splits_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - split_ticket_queue(Allocator *alloc, - split_ticket_queue &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - (*splits_) = std::move(*other.splits_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - split_ticket_queue& operator=(split_ticket_queue &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - (*splits_) = std::move(*other.splits_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** SHM destructor. */ - void shm_destroy_main() { - (*splits_).shm_destroy(); - } - - /** Check if the list is empty */ - bool IsNull() const { - return (*splits_).IsNull(); - } - - /** Sets this list as empty */ - void SetNull() { - rr_tail_ = 0; - rr_head_ = 0; - } - - /**==================================== - * ticket Queue Methods - * ===================================*/ - - /** Construct an element at \a pos position in the queue */ - template - qtok_t emplace(T &tkt) { - uint16_t rr = rr_tail_.fetch_add(1); - auto &splits = (*splits_); - size_t num_splits = splits.size(); - uint16_t qid_start = rr % num_splits; - for (size_t i = 0; i < num_splits; ++i) { - uint32_t qid = (qid_start + i) % num_splits; - ticket_queue &queue = (*splits_)[qid]; - qtok_t qtok = queue.emplace(tkt); - if (!qtok.IsNull()) { - return qtok; - } - } - return qtok_t::GetNull(); - } - - public: - /** Pop an element from the queue */ - qtok_t pop(T &tkt) { - uint16_t rr = rr_head_.fetch_add(1); - auto &splits = (*splits_); - size_t num_splits = splits.size(); - uint16_t qid_start = rr % num_splits; - for (size_t i = 0; i < num_splits; ++i) { - uint32_t qid = (qid_start + i) % num_splits; - ticket_queue &queue = (*splits_)[qid]; - qtok_t qtok = queue.pop(tkt); - if (!qtok.IsNull()) { - return qtok; - } - } - return qtok_t::GetNull(); - } -}; - -} // namespace hshm::ipc - -#undef TYPED_HEADER -#undef TYPED_CLASS -#undef CLASS_NAME - -#endif // HERMES_SHM__DATA_STRUCTURES_IPC_SPLIT_TICKET_QUEUE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h deleted file mode 100644 index bb5a4fbe1..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/spsc_queue.h +++ /dev/null @@ -1,201 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_spsc_queue_templ_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_spsc_queue_templ_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/util/auto_trace.h" -#include "hermes_shm/thread/lock.h" -#include "vector.h" -#include "pair.h" -#include "_queue.h" - -namespace hshm::ipc { - -/** Forward declaration of spsc_queue_templ */ -template -class spsc_queue_templ; - -/** - * MACROS used to simplify the spsc_queue_templ namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME spsc_queue_templ -#define TYPED_CLASS spsc_queue_templ -#define TYPED_HEADER ShmHeader> - -/** - * A queue optimized for multiple producers (emplace) with a single - * consumer (pop). - * */ -template -class spsc_queue_templ : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - ShmArchive> queue_; - _qtok_t tail_; - _qtok_t head_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit spsc_queue_templ(Allocator *alloc, - size_t depth = 1024) { - shm_init_container(alloc); - HSHM_MAKE_AR(queue_, GetAllocator(), depth) - SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit spsc_queue_templ(Allocator *alloc, - const spsc_queue_templ &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - spsc_queue_templ& operator=(const spsc_queue_templ &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor + operator main */ - void shm_strong_copy_construct_and_op(const spsc_queue_templ &other) { - head_ = other.head_; - tail_ = other.tail_; - (*queue_) = (*other.queue_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - spsc_queue_templ(Allocator *alloc, - spsc_queue_templ &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - head_ = other.head_; - tail_ = other.tail_; - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - spsc_queue_templ& operator=(spsc_queue_templ &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - head_ = other.head_; - tail_ = other.tail_; - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** SHM destructor. */ - void shm_destroy_main() { - (*queue_).shm_destroy(); - } - - /** Check if the list is empty */ - bool IsNull() const { - return (*queue_).IsNull(); - } - - /** Sets this list as empty */ - void SetNull() { - head_ = 0; - tail_ = 0; - } - - /**==================================== - * spsc Queue Methods - * ===================================*/ - - /** Construct an element at \a pos position in the list */ - template - qtok_t emplace(Args&&... args) { - // Don't emplace if there is no space - _qtok_t entry_tok = tail_; - size_t size = tail_ - head_; - auto &queue = (*queue_); - if (size >= queue.size()) { - return qtok_t::GetNull(); - } - - // Do the emplace - _qtok_t idx = entry_tok % queue.size(); - auto iter = queue.begin() + idx; - queue.replace(iter, std::forward(args)...); - tail_ += 1; - return qtok_t(entry_tok); - } - - public: - /** Consumer pops the head object */ - qtok_t pop(T &val) { - // Don't pop if there's no entries - _qtok_t head = head_; - _qtok_t tail = tail_; - if (head >= tail) { - return qtok_t::GetNull(); - } - - // Pop the element - auto &queue = (*queue_); - _qtok_t idx = head % queue.size(); - T &entry = queue[idx]; - (val) = std::move(entry); - head_ += 1; - return qtok_t(head); - } -}; - -template -using spsc_queue_ext = spsc_queue_templ; - -template -using spsc_queue = spsc_queue_templ; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_spsc_queue_templ_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/string.h b/hermes_shm/include/hermes_shm/data_structures/ipc/string.h deleted file mode 100644 index 694e8c671..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/string.h +++ /dev/null @@ -1,357 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_LOCKLESS_STRING_H_ -#define HERMES_DATA_STRUCTURES_LOCKLESS_STRING_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/data_structures/containers/charbuf.h" -#include - -namespace hshm::ipc { - -/** forward declaration for string */ -template -class string_templ; - -/** - * MACROS used to simplify the string namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME string_templ -#define TYPED_CLASS string_templ -#define TYPED_HEADER ShmHeader> - -/** string shared-memory header */ -template -struct ShmHeader> { - SHM_CONTAINER_HEADER_TEMPLATE(ShmHeader) - size_t length_; - char sso_[SSO]; - Pointer text_; - - /** Strong copy operation */ - void strong_copy(const ShmHeader &other) { - length_ = other.length_; - text_ = other.text_; - if (length_ < SSO) { - memcpy(sso_, other.sso_, other.length_ + 1); - } - } -}; - -/** - * A string of characters. - * */ -template -class string_templ : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - - public: - size_t length_; - Pointer text_; - char sso_[SSO]; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM Constructor. Default. */ - explicit string_templ(Allocator *alloc) { - shm_init_container(alloc); - SetNull(); - } - - /**==================================== - * Emplace Constructors - * ===================================*/ - - /** SHM Constructor. Just allocate space. */ - explicit string_templ(Allocator *alloc, - size_t length) { - shm_init_container(alloc); - _create_str(length); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM Constructor. From const char* */ - explicit string_templ(Allocator *alloc, - const char *text) { - shm_init_container(alloc); - size_t length = strlen(text); - _create_str(text, length); - } - - /** SHM Constructor. From const char* and size */ - explicit string_templ(Allocator *alloc, - const char *text, size_t length) { - shm_init_container(alloc); - _create_str(text, length); - } - - /** SHM Constructor. From std::string */ - explicit string_templ(Allocator *alloc, - const std::string &text) { - shm_init_container(alloc); - _create_str(text.data(), text.size()); - } - - /** SHM copy assignment operator. From std::string. */ - string_templ& operator=(const std::string &other) { - shm_destroy(); - _create_str(other.data(), other.size()); - return *this; - } - - /** SHM Constructor. From std::string */ - explicit string_templ(Allocator *alloc, - const hshm::charbuf &text) { - shm_init_container(alloc); - _create_str(text.data(), text.size()); - } - - /** SHM copy assignment operator. From std::string. */ - string_templ& operator=(const hshm::charbuf &other) { - shm_destroy(); - _create_str(other.data(), other.size()); - return *this; - } - - /** SHM copy constructor. From string. */ - explicit string_templ(Allocator *alloc, - const string_templ &other) { - shm_init_container(alloc); - _create_str(other.data(), other.size()); - } - - /** SHM copy assignment operator. From string. */ - string_templ& operator=(const string_templ &other) { - if (this != &other) { - shm_destroy(); - _create_str(other.data(), other.size()); - } - return *this; - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** Strong copy operation */ - HSHM_ALWAYS_INLINE void strong_copy(const string_templ &other) { - length_ = other.length_; - text_ = other.text_; - if (length_ < SSO) { - memcpy(sso_, other.sso_, other.length_ + 1); - } - } - - /** SHM move constructor. */ - string_templ(Allocator *alloc, string_templ &&other) { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - other.SetNull(); - } else { - _create_str(other.data(), other.size()); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - string_templ& operator=(string_templ &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - other.SetNull(); - } else { - _create_str(other.data(), other.size()); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructors - * ===================================*/ - - /** Check if this string is NULL */ - HSHM_ALWAYS_INLINE bool IsNull() const { - return length_ == 0; - } - - /** Set this string to NULL */ - HSHM_ALWAYS_INLINE void SetNull() { - text_.SetNull(); - length_ = 0; - } - - /** Destroy the shared-memory data. */ - HSHM_ALWAYS_INLINE void shm_destroy_main() { - if (size() >= SSO) { - GetAllocator()->Free(text_); - } - } - - /**==================================== - * String Operations - * ===================================*/ - - /** Get character at index i in the string */ - HSHM_ALWAYS_INLINE char& operator[](size_t i) { - return data()[i]; - } - - /** Get character at index i in the string */ - HSHM_ALWAYS_INLINE const char& operator[](size_t i) const { - return data()[i]; - } - - /** Convert into a std::string */ - HSHM_ALWAYS_INLINE std::string str() const { - return {c_str(), length_}; - } - - /** Get the size of the current string */ - HSHM_ALWAYS_INLINE size_t size() const { - return length_; - } - - /** Get a constant reference to the C-style string */ - HSHM_ALWAYS_INLINE const char* c_str() const { - return data(); - } - - /** Get a constant reference to the C-style string */ - HSHM_ALWAYS_INLINE const char* data() const { - if (length_ < SSO) { - return sso_; - } else { - return GetAllocator()->template Convert(text_); - } - } - - /** Get a mutable reference to the C-style string */ - HSHM_ALWAYS_INLINE char* data() { - if (length_ < SSO) { - return sso_; - } else { - return GetAllocator()->template Convert(text_); - } - } - - /** Resize this string */ - void resize(size_t new_size) { - if (IsNull()) { - _create_str(new_size); - } else if (new_size > size()) { - GetAllocator()->template Reallocate(text_, new_size); - length_ = new_size; - } else { - length_ = new_size; - } - } - - /**==================================== - * Comparison Operations - * ===================================*/ - - int _strncmp(const char *a, size_t len_a, - const char *b, size_t len_b) const { - if (len_a != len_b) { - return int((int64_t)len_a - (int64_t)len_b); - } - for (size_t i = 0; i < len_a; ++i) { - if (a[i] != b[i]) { - return a[i] - b[i]; - } - } - return 0; - } - -#define HERMES_STR_CMP_OPERATOR(op) \ - bool operator op(const char *other) const { \ - return _strncmp(data(), size(), other, strlen(other)) op 0; \ - } \ - bool operator op(const std::string &other) const { \ - return _strncmp(data(), size(), other.data(), other.size()) op 0; \ - } \ - bool operator op(const string_templ &other) const { \ - return _strncmp(data(), size(), other.data(), other.size()) op 0; \ - } - - HERMES_STR_CMP_OPERATOR(==) // NOLINT - HERMES_STR_CMP_OPERATOR(!=) // NOLINT - HERMES_STR_CMP_OPERATOR(<) // NOLINT - HERMES_STR_CMP_OPERATOR(>) // NOLINT - HERMES_STR_CMP_OPERATOR(<=) // NOLINT - HERMES_STR_CMP_OPERATOR(>=) // NOLINT - -#undef HERMES_STR_CMP_OPERATOR - - private: - HSHM_ALWAYS_INLINE void _create_str(size_t length) { - if (length < SSO) { - // NOTE(llogan): less than and not equal because length doesn't - // account for trailing 0. - } else { - text_ = GetAllocator()->Allocate(length + 1); - } - length_ = length; - } - HSHM_ALWAYS_INLINE void _create_str(const char *text, size_t length) { - _create_str(length); - char *str = data(); - memcpy(str, text, length); - str[length] = 0; - } -}; - -/** Our default SSO value */ -typedef string_templ<32> string; - -/** Consider the string as an uniterpreted set of bytes */ -typedef string charbuf; - -} // namespace hshm::ipc - -namespace std { - -/** Hash function for string */ -template -struct hash> { - size_t operator()(const hshm::ipc::string_templ &text) const { - size_t sum = 0; - for (size_t i = 0; i < text.size(); ++i) { - auto shift = static_cast(i % sizeof(size_t)); - auto c = static_cast((unsigned char)text[i]); - sum = 31*sum + (c << shift); - } - return sum; - } -}; - -} // namespace std - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_DATA_STRUCTURES_LOCKLESS_STRING_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h b/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h deleted file mode 100644 index 0893bf261..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_queue.h +++ /dev/null @@ -1,164 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_QUEUE_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_QUEUE_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/thread/lock.h" -#include "spsc_queue.h" -#include "_queue.h" - -namespace hshm::ipc { - -/** Forward declaration of ticket_queue */ -template -class ticket_queue; - -/** - * MACROS used to simplify the ticket_queue namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME ticket_queue -#define TYPED_CLASS ticket_queue -#define TYPED_HEADER ShmHeader> - -/** - * A MPMC queue for allocating tickets. Handles concurrency - * without blocking. - * */ -template -class ticket_queue : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - ShmArchive> queue_; - hshm::Mutex lock_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit ticket_queue(Allocator *alloc, - size_t depth = 1024) { - shm_init_container(alloc); - HSHM_MAKE_AR(queue_, GetAllocator(), depth); - lock_.Init(); - SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit ticket_queue(Allocator *alloc, - const ticket_queue &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - ticket_queue& operator=(const ticket_queue &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor + operator main */ - void shm_strong_copy_construct_and_op(const ticket_queue &other) { - (*queue_) = (*other.queue_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - ticket_queue(Allocator *alloc, - ticket_queue &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - ticket_queue& operator=(ticket_queue &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** SHM destructor. */ - void shm_destroy_main() { - (*queue_).shm_destroy(); - } - - /** Check if the list is empty */ - bool IsNull() const { - return (*queue_).IsNull(); - } - - /** Sets this list as empty */ - void SetNull() {} - - /**==================================== - * ticket Queue Methods - * ===================================*/ - - /** Construct an element at \a pos position in the queue */ - template - HSHM_ALWAYS_INLINE qtok_t emplace(T &tkt) { - lock_.Lock(0); - auto qtok = queue_->emplace(tkt); - lock_.Unlock(); - return qtok; - } - - public: - /** Pop an element from the queue */ - HSHM_ALWAYS_INLINE qtok_t pop(T &tkt) { - lock_.Lock(0); - auto qtok = queue_->pop(tkt); - lock_.Unlock(); - return qtok; - } -}; - -} // namespace hshm::ipc - -#undef TYPED_HEADER -#undef TYPED_CLASS -#undef CLASS_NAME - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_QUEUE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h b/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h deleted file mode 100644 index 7d419c3a0..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/ticket_stack.h +++ /dev/null @@ -1,166 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_STACK_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_STACK_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/thread/lock.h" -#include "vector.h" -#include "_queue.h" -#include "spsc_queue.h" - -namespace hshm::ipc { - -/** Forward declaration of ticket_stack */ -template -class ticket_stack; - -/** - * MACROS used to simplify the ticket_stack namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME ticket_stack -#define TYPED_CLASS ticket_stack -#define TYPED_HEADER ShmHeader> - -/** - * A MPMC queue for allocating tickets. Handles concurrency - * without blocking. - * */ -template -class ticket_stack : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - ShmArchive> queue_; - hshm::Mutex lock_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit ticket_stack(Allocator *alloc, - size_t depth = 1024) { - shm_init_container(alloc); - HSHM_MAKE_AR(queue_, GetAllocator(), depth); - lock_.Init(); - SetNull(); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit ticket_stack(Allocator *alloc, - const ticket_stack &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - ticket_stack& operator=(const ticket_stack &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_construct_and_op(other); - } - return *this; - } - - /** SHM copy constructor + operator main */ - void shm_strong_copy_construct_and_op(const ticket_stack &other) { - (*queue_) = (*other.queue_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - ticket_stack(Allocator *alloc, - ticket_stack &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - ticket_stack& operator=(ticket_stack &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - (*queue_) = std::move(*other.queue_); - other.SetNull(); - } else { - shm_strong_copy_construct_and_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** SHM destructor. */ - void shm_destroy_main() { - (*queue_).shm_destroy(); - } - - /** Check if the list is empty */ - bool IsNull() const { - return (*queue_).IsNull(); - } - - /** Sets this list as empty */ - void SetNull() { - } - - /**==================================== - * ticket Queue Methods - * ===================================*/ - - /** Construct an element at \a pos position in the queue */ - template - HSHM_ALWAYS_INLINE qtok_t emplace(T &tkt) { - lock_.Lock(0); - auto qtok = queue_->emplace(tkt); - lock_.Unlock(); - return qtok; - } - - public: - /** Pop an element from the queue */ - HSHM_ALWAYS_INLINE qtok_t pop(T &tkt) { - lock_.Lock(0); - auto qtok = queue_->pop(tkt); - lock_.Unlock(); - return qtok; - } -}; - -} // namespace hshm::ipc - -#undef TYPED_HEADER -#undef TYPED_CLASS -#undef CLASS_NAME - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_IPC_TICKET_STACK_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/unordered_map.h b/hermes_shm/include/hermes_shm/data_structures/ipc/unordered_map.h deleted file mode 100644 index 5d4629e99..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/unordered_map.h +++ /dev/null @@ -1,537 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_UNORDERED_MAP_H_ -#define HERMES_DATA_STRUCTURES_UNORDERED_MAP_H_ - -#include "hermes_shm/thread/thread_model_manager.h" -#include "hermes_shm/data_structures/ipc/vector.h" -#include "hermes_shm/data_structures/ipc/slist.h" -#include "pair.h" -#include "hermes_shm/types/atomic.h" -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" - -namespace hshm::ipc { - -/** forward pointer for unordered_map */ -template> -class unordered_map; - -/** - * The unordered map iterator (bucket_iter, slist_iter) - * */ -template -struct unordered_map_iterator { - public: - using COLLISION_T = hipc::pair; - using BUCKET_T = hipc::slist; - - public: - unordered_map *map_; - typename vector::iterator_t bucket_; - typename slist::iterator_t collision_; - - /** Default constructor */ - unordered_map_iterator() = default; - - /** Construct the iterator */ - HSHM_ALWAYS_INLINE explicit unordered_map_iterator( - unordered_map &map) - : map_(&map) {} - - /** Copy constructor */ - HSHM_ALWAYS_INLINE unordered_map_iterator( - const unordered_map_iterator &other) { - shm_strong_copy(other); - } - - /** Assign one iterator into another */ - HSHM_ALWAYS_INLINE unordered_map_iterator& - operator=(const unordered_map_iterator &other) { - if (this != &other) { - shm_strong_copy(other); - } - return *this; - } - - /** Copy an iterator */ - void shm_strong_copy(const unordered_map_iterator &other) { - map_ = other.map_; - bucket_ = other.bucket_; - collision_ = other.collision_; - } - - /** Get the pointed object */ - HSHM_ALWAYS_INLINE COLLISION_T& operator*() { - return *collision_; - } - - /** Get the pointed object */ - HSHM_ALWAYS_INLINE const COLLISION_T& operator*() const { - return *collision_; - } - - /** Go to the next object */ - HSHM_ALWAYS_INLINE unordered_map_iterator& operator++() { - ++collision_; - make_correct(); - return *this; - } - - /** Return the next iterator */ - HSHM_ALWAYS_INLINE unordered_map_iterator operator++(int) const { - unordered_map_iterator next(*this); - ++next; - return next; - } - - /** - * Shifts bucket and collision iterator until there is a valid element. - * Returns true if such an element is found, and false otherwise. - * */ - HSHM_ALWAYS_INLINE bool make_correct() { - do { - if (bucket_.is_end()) { - return false; - } - if (!collision_.is_end()) { - return true; - } else { - ++bucket_; - if (bucket_.is_end()) { - return false; - } - BUCKET_T &bkt = *bucket_; - collision_ = bkt.begin(); - } - } while (true); - } - - /** Check if two iterators are equal */ - HSHM_ALWAYS_INLINE friend bool operator==( - const unordered_map_iterator &a, const unordered_map_iterator &b) { - if (a.is_end() && b.is_end()) { - return true; - } - return (a.bucket_ == b.bucket_) && (a.collision_ == b.collision_); - } - - /** Check if two iterators are inequal */ - HSHM_ALWAYS_INLINE friend bool operator!=( - const unordered_map_iterator &a, const unordered_map_iterator &b) { - if (a.is_end() && b.is_end()) { - return false; - } - return (a.bucket_ != b.bucket_) || (a.collision_ != b.collision_); - } - - /** Determine whether this iterator is the end iterator */ - HSHM_ALWAYS_INLINE bool is_end() const { - return bucket_.is_end(); - } - - /** Set this iterator to the end iterator */ - HSHM_ALWAYS_INLINE void set_end() { - bucket_.set_end(); - } -}; - -/** - * MACROS to simplify the unordered_map namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ - -#define CLASS_NAME unordered_map -#define TYPED_CLASS unordered_map -#define TYPED_HEADER ShmHeader> - -/** - * The unordered map implementation - * */ -template -class unordered_map : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - - /**==================================== - * Typedefs - * ===================================*/ - typedef unordered_map_iterator iterator_t; - friend iterator_t; - using COLLISION_T = hipc::pair; - using BUCKET_T = hipc::slist; - - /**==================================== - * Variables - * ===================================*/ - ShmArchive> buckets_; - RealNumber max_capacity_; - RealNumber growth_; - hipc::atomic length_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** - * SHM constructor. Initialize the map. - * - * @param alloc the shared-memory allocator - * @param num_buckets the number of buckets to create - * @param max_capacity the maximum number of elements before a growth is - * triggered - * @param growth the multiplier to grow the bucket vector size - * */ - explicit unordered_map(Allocator *alloc, - int num_buckets = 20, - RealNumber max_capacity = RealNumber(4, 5), - RealNumber growth = RealNumber(5, 4)) { - shm_init_container(alloc); - HSHM_MAKE_AR(buckets_, GetAllocator(), num_buckets) - max_capacity_ = max_capacity; - growth_ = growth; - length_ = 0; - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor */ - explicit unordered_map(Allocator *alloc, - const unordered_map &other) { - shm_init_container(alloc); - shm_strong_copy_construct(other); - } - - /** SHM copy constructor main */ - void shm_strong_copy_construct(const unordered_map &other) { - SetNull(); - HSHM_MAKE_AR(buckets_, GetAllocator(), other.GetBuckets()) - shm_strong_copy_construct_and_op(other); - } - - /** SHM copy assignment operator */ - unordered_map& operator=(const unordered_map &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_op(other); - } - return *this; - } - - /** SHM copy assignment main */ - void shm_strong_copy_op(const unordered_map &other) { - int num_buckets = other.get_num_buckets(); - GetBuckets().resize(num_buckets); - shm_strong_copy_construct_and_op(other); - } - - /** Internal copy operation */ - void shm_strong_copy_construct_and_op(const unordered_map &other) { - max_capacity_ = other.max_capacity_; - growth_ = other.growth_; - for (hipc::pair &entry : other) { - emplace_templ( - entry.GetKey(), entry.GetVal()); - } - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - HSHM_ALWAYS_INLINE unordered_map(Allocator *alloc, - unordered_map &&other) noexcept { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - HSHM_MAKE_AR(buckets_, GetAllocator(), std::move(other.GetBuckets())) - other.SetNull(); - } else { - shm_strong_copy_construct(other); - other.shm_destroy(); - } - } - - /** Copy */ - HSHM_ALWAYS_INLINE void strong_copy(const unordered_map &other) { - max_capacity_ = other.max_capacity_; - growth_ = other.growth_; - length_ = other.length_.load(); - } - - /** SHM move assignment operator. */ - unordered_map& operator=(unordered_map &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - GetBuckets() = std::move(other.GetBuckets()); - strong_copy(other); - other.SetNull(); - } else { - shm_strong_copy_op(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Check if the pair is empty */ - HSHM_ALWAYS_INLINE bool IsNull() { - return length_.load() == 0; - } - - /** Sets this pair as empty */ - HSHM_ALWAYS_INLINE void SetNull() { - length_ = 0; - } - - /** Destroy the unordered_map buckets */ - HSHM_ALWAYS_INLINE void shm_destroy_main() { - vector& buckets = GetBuckets(); - buckets.shm_destroy(); - } - - /**==================================== - * Emplace Methods - * ===================================*/ - - /** - * Construct an object directly in the map. Overrides the object if - * key already exists. - * - * @param key the key to future index the map - * @param args the arguments to construct the object - * @return None - * */ - template - bool emplace(const Key &key, Args&&... args) { - return emplace_templ(key, std::forward(args)...); - } - - /** - * Construct an object directly in the map. Does not modify the key - * if it already exists. - * - * @param key the key to future index the map - * @param args the arguments to construct the object - * @return None - * */ - template - bool try_emplace(const Key &key, Args&&... args) { - return emplace_templ(key, std::forward(args)...); - } - - private: - /** - * Insert a serialized (key, value) pair in the map - * - * @param growth whether or not to grow the unordered map on collision - * @param modify_existing whether or not to override an existing entry - * @param entry the (key,value) pair shared-memory serialized - * @return None - * */ - template - HSHM_ALWAYS_INLINE bool emplace_templ(const Key &key, Args&& ...args) { - // Hash the key to a bucket - vector& buckets = GetBuckets(); - size_t bkt_id = Hash{}(key) % buckets.size(); - BUCKET_T& bkt = (buckets)[bkt_id]; - - // Insert into the map - auto has_key_iter = find_collision(key, bkt); - if (!has_key_iter.is_end()) { - if constexpr(!modify_existing) { - return false; - } else { - bkt.erase(has_key_iter); - --length_; - } - } - bkt.emplace_back(PiecewiseConstruct(), - make_argpack(key), - make_argpack(std::forward(args)...)); - - // Increment the size of the map - ++length_; - return true; - } - - public: - /**==================================== - * Erase Methods - * ===================================*/ - - /** - * Erase an object indexable by \a key key - * */ - void erase(const Key &key) { - // Get the bucket the key belongs to - vector& buckets = GetBuckets(); - size_t bkt_id = Hash{}(key) % buckets.size(); - BUCKET_T& bkt = (buckets)[bkt_id]; - - // Find and remove key from collision slist - auto iter = find_collision(key, bkt); - if (iter.is_end()) { - return; - } - bkt.erase(iter); - - // Decrement the size of the map - --length_; - } - - /** - * Erase an object at the iterator - * */ - void erase(iterator_t &iter) { - if (iter == end()) return; - // Acquire the bucket lock for a write (modifying collisions) - BUCKET_T& bkt = *iter.bucket_; - - // Erase the element from the collision slist - bkt.erase(iter.collision_); - - // Decrement the size of the map - --length_; - } - - /** - * Erase the entire map - * */ - void clear() { - vector& buckets = GetBuckets(); - size_t num_buckets = buckets.size(); - buckets.clear(); - buckets.resize(num_buckets); - length_ = 0; - } - - /**==================================== - * Index Methods - * ===================================*/ - - /** - * Locate an entry in the unordered_map - * - * @return the object pointed by key - * @exception UNORDERED_MAP_CANT_FIND the key was not in the map - * */ - HSHM_ALWAYS_INLINE T& operator[](const Key &key) { - auto iter = find(key); - if (!iter.is_end()) { - return (*iter).second_.get_ref(); - } - throw UNORDERED_MAP_CANT_FIND.format(); - } - - /** Find an object in the unordered_map */ - iterator_t find(const Key &key) { - iterator_t iter(*this); - - // Determine the bucket corresponding to the key - vector& buckets = GetBuckets(); - size_t bkt_id = Hash{}(key) % buckets.size(); - iter.bucket_ = buckets.begin() + bkt_id; - BUCKET_T& bkt = (*iter.bucket_); - - // Get the specific collision iterator - iter.collision_ = find_collision(key, bkt); - if (iter.collision_.is_end()) { - iter.set_end(); - } - return iter; - } - - /** Find a key in the collision slist */ - typename BUCKET_T::iterator_t - HSHM_ALWAYS_INLINE find_collision(const Key &key, BUCKET_T &bkt) { - auto iter = bkt.begin(); - auto iter_end = bkt.end(); - for (; iter != iter_end; ++iter) { - COLLISION_T &collision = *iter; - if (collision.GetKey() == key) { - return iter; - } - } - return iter_end; - } - - /**==================================== - * Query Methods - * ===================================*/ - - /** The number of entries in the map */ - HSHM_ALWAYS_INLINE size_t size() const { - return length_.load(); - } - - /** The number of buckets in the map */ - HSHM_ALWAYS_INLINE size_t get_num_buckets() const { - vector& buckets = GetBuckets(); - return buckets.size(); - } - - public: - /**==================================== - * Iterators - * ===================================*/ - - /** Forward iterator begin */ - HSHM_ALWAYS_INLINE iterator_t begin() const { - iterator_t iter(const_cast(*this)); - vector& buckets(GetBuckets()); - if (buckets.size() == 0) { - return iter; - } - BUCKET_T& bkt = buckets[0]; - iter.bucket_ = buckets.cbegin(); - iter.collision_ = bkt.begin(); - iter.make_correct(); - return iter; - } - - /** Forward iterator end */ - HSHM_ALWAYS_INLINE iterator_t end() const { - iterator_t iter(const_cast(*this)); - vector& buckets(GetBuckets()); - iter.bucket_ = buckets.cend(); - return iter; - } - - /** Get the buckets */ - HSHM_ALWAYS_INLINE vector& GetBuckets() { - return *buckets_; - } - - /** Get the buckets (const) */ - HSHM_ALWAYS_INLINE vector& GetBuckets() const { - return const_cast&>(*buckets_); - } -}; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_DATA_STRUCTURES_UNORDERED_MAP_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/ipc/vector.h b/hermes_shm/include/hermes_shm/data_structures/ipc/vector.h deleted file mode 100644 index 4a3da9816..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/ipc/vector.h +++ /dev/null @@ -1,688 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_LOCKLESS_VECTOR_H_ -#define HERMES_DATA_STRUCTURES_LOCKLESS_VECTOR_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" - -#include - -namespace hshm::ipc { - -/** forward pointer for vector */ -template -class vector; - -/** - * The vector iterator implementation - * */ -template -struct vector_iterator_templ { - public: - vector *vec_; - off64_t i_; - - /** Default constructor */ - HSHM_ALWAYS_INLINE vector_iterator_templ() = default; - - /** Construct an iterator (called from vector class) */ - template - HSHM_ALWAYS_INLINE explicit vector_iterator_templ(vector *vec, SizeT i) - : vec_(vec), i_(static_cast(i)) {} - - /** Construct an iterator (called from iterator) */ - HSHM_ALWAYS_INLINE explicit vector_iterator_templ(vector *vec, off64_t i) - : vec_(vec), i_(i) {} - - /** Copy constructor */ - HSHM_ALWAYS_INLINE vector_iterator_templ(const vector_iterator_templ &other) - : vec_(other.vec_), i_(other.i_) {} - - /** Copy assignment operator */ - HSHM_ALWAYS_INLINE vector_iterator_templ& - operator=(const vector_iterator_templ &other) { - if (this != &other) { - vec_ = other.vec_; - i_ = other.i_; - } - return *this; - } - - /** Move constructor */ - HSHM_ALWAYS_INLINE vector_iterator_templ( - vector_iterator_templ &&other) noexcept { - vec_ = other.vec_; - i_ = other.i_; - } - - /** Move assignment operator */ - HSHM_ALWAYS_INLINE vector_iterator_templ& - operator=(vector_iterator_templ &&other) noexcept { - if (this != &other) { - vec_ = other.vec_; - i_ = other.i_; - } - return *this; - } - - /** Dereference the iterator */ - HSHM_ALWAYS_INLINE T& operator*() { - return vec_->data_ar()[i_].get_ref(); - } - - /** Dereference the iterator */ - HSHM_ALWAYS_INLINE const T& operator*() const { - return vec_->data_ar()[i_].get_ref(); - } - - /** Increment iterator in-place */ - HSHM_ALWAYS_INLINE vector_iterator_templ& operator++() { - if constexpr(FORWARD_ITER) { - ++i_; - } else { - --i_; - } - return *this; - } - - /** Decrement iterator in-place */ - HSHM_ALWAYS_INLINE vector_iterator_templ& operator--() { - if constexpr(FORWARD_ITER) { - --i_; - } else { - ++i_; - } - return *this; - } - - /** Create the next iterator */ - HSHM_ALWAYS_INLINE vector_iterator_templ operator++(int) const { - vector_iterator_templ next_iter(*this); - ++next_iter; - return next_iter; - } - - /** Create the prior iterator */ - HSHM_ALWAYS_INLINE vector_iterator_templ operator--(int) const { - vector_iterator_templ prior_iter(*this); - --prior_iter; - return prior_iter; - } - - /** Increment iterator by \a count and return */ - HSHM_ALWAYS_INLINE vector_iterator_templ operator+(size_t count) const { - if constexpr(FORWARD_ITER) { - return vector_iterator_templ(vec_, i_ + count); - } else { - return vector_iterator_templ(vec_, i_ - count); - } - } - - /** Decrement iterator by \a count and return */ - HSHM_ALWAYS_INLINE vector_iterator_templ operator-(size_t count) const { - if constexpr(FORWARD_ITER) { - return vector_iterator_templ(vec_, i_ - count); - } else { - return vector_iterator_templ(vec_, i_ + count); - } - } - - /** Increment iterator by \a count in-place */ - HSHM_ALWAYS_INLINE void operator+=(size_t count) { - if constexpr(FORWARD_ITER) { - i_ += count; - } else { - i_ -= count; - } - } - - /** Decrement iterator by \a count in-place */ - HSHM_ALWAYS_INLINE void operator-=(size_t count) { - if constexpr(FORWARD_ITER) { - i_ -= count; - } else { - i_ += count; - } - } - - /** Check if two iterators are equal */ - HSHM_ALWAYS_INLINE friend bool operator==(const vector_iterator_templ &a, - const vector_iterator_templ &b) { - return (a.i_ == b.i_); - } - - /** Check if two iterators are inequal */ - HSHM_ALWAYS_INLINE friend bool operator!=(const vector_iterator_templ &a, - const vector_iterator_templ &b) { - return (a.i_ != b.i_); - } - - /** Set this iterator to end */ - HSHM_ALWAYS_INLINE void set_end() { - if constexpr(FORWARD_ITER) { - i_ = vec_->size(); - } else { - i_ = -1; - } - } - - /** Set this iterator to begin */ - HSHM_ALWAYS_INLINE void set_begin() { - if constexpr(FORWARD_ITER) { - i_ = 0; - } else { - i_ = vec_->size() - 1; - } - } - - /** Determine whether this iterator is the begin iterator */ - HSHM_ALWAYS_INLINE bool is_begin() const { - if constexpr(FORWARD_ITER) { - return (i_ == 0); - } else { - return (i_ == vec_->template size() - 1); - } - } - - /** Determine whether this iterator is the end iterator */ - HSHM_ALWAYS_INLINE bool is_end() const { - if constexpr(FORWARD_ITER) { - return i_ >= vec_->template size(); - } else { - return i_ == -1; - } - } -}; - -/** - * MACROS used to simplify the vector namespace - * Used as inputs to the SHM_CONTAINER_TEMPLATE - * */ -#define CLASS_NAME vector -#define TYPED_CLASS vector -#define TYPED_HEADER ShmHeader> - -/** - * The vector class - * */ -template -class vector : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS)) - - public: - /**==================================== - * Typedefs - * ===================================*/ - - /** forwrard iterator */ - typedef vector_iterator_templ iterator_t; - /** reverse iterator */ - typedef vector_iterator_templ riterator_t; - /** const iterator */ - typedef vector_iterator_templ citerator_t; - /** const reverse iterator */ - typedef vector_iterator_templ criterator_t; - - public: - /**==================================== - * Variables - * ===================================*/ - AtomicPointer vec_ptr_; - size_t max_length_, length_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit vector(Allocator *alloc) { - shm_init_container(alloc); - SetNull(); - } - - /** SHM constructor. Resize + construct. */ - template - explicit vector(Allocator *alloc, size_t length, Args&& ...args) { - shm_init_container(alloc); - SetNull(); - resize(length, std::forward(args)...); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor. From vector. */ - explicit vector(Allocator *alloc, const vector &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_main>(other); - } - - /** SHM copy assignment operator. From vector. */ - vector& operator=(const vector &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_main(other); - } - return *this; - } - - /** SHM copy constructor. From std::vector */ - explicit vector(Allocator *alloc, const std::vector &other) { - shm_init_container(alloc); - SetNull(); - shm_strong_copy_main>(other); - } - - /** SHM copy assignment operator. From std::vector */ - vector& operator=(const std::vector &other) { - shm_destroy(); - shm_strong_copy_main>(other); - return *this; - } - - /** The main copy operation */ - template - void shm_strong_copy_main(const VectorT &other) { - reserve(other.size()); - if constexpr(std::is_pod() && !IS_SHM_ARCHIVEABLE(T)) { - memcpy(data(), other.data(), other.size() * sizeof(T)); - length_ = other.size(); - } else { - for (auto iter = other.cbegin(); iter != other.cend(); ++iter) { - if constexpr(IS_SHM_ARCHIVEABLE(VectorT)) { - emplace_back((*iter)); - } else { - emplace_back((*iter)); - } - } - } - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - vector(Allocator *alloc, vector &&other) { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - memcpy((void *) this, (void *) &other, sizeof(*this)); - other.SetNull(); - } else { - shm_strong_copy_main(other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - vector& operator=(vector &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - memcpy((void *) this, (void *) &other, sizeof(*this)); - other.SetNull(); - } else { - shm_strong_copy_main(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Check if null */ - HSHM_ALWAYS_INLINE bool IsNull() const { - return vec_ptr_.IsNull(); - } - - /** Make null */ - HSHM_ALWAYS_INLINE void SetNull() { - length_ = 0; - max_length_ = 0; - vec_ptr_.SetNull(); - } - - /** Destroy all shared memory allocated by the vector */ - HSHM_ALWAYS_INLINE void shm_destroy_main() { - erase(begin(), end()); - GetAllocator()->Free(vec_ptr_); - } - - /**==================================== - * Vector Operations - * ===================================*/ - - /** - * Convert to std::vector - * */ - HSHM_ALWAYS_INLINE std::vector vec() { - std::vector v; - v.reserve(size()); - for (T& entry : *this) { - v.emplace_back(entry); - } - return v; - } - - /** - * Reserve space in the vector to emplace elements. Does not - * change the size of the list. - * - * @param length the maximum size the vector can get before a growth occurs - * @param args the arguments to construct - * */ - template - HSHM_ALWAYS_INLINE void reserve(size_t length, Args&& ...args) { - if (length == 0) { return; } - grow_vector(data_ar(), length, false, std::forward(args)...); - } - - /** - * Reserve space in the vector to emplace elements. Changes the - * size of the list. - * - * @param length the maximum size the vector can get before a growth occurs - * @param args the arguments used to construct the vector elements - * */ - template - void resize(size_t length, Args&& ...args) { - if (length == 0) { - length_ = 0; - return; - } - grow_vector(data_ar(), length, true, - std::forward(args)...); - length_ = length; - } - - /** Index the vector at position i */ - HSHM_ALWAYS_INLINE T& operator[](const size_t i) { - return data_ar()[i].get_ref(); - } - - /** Index the vector at position i */ - HSHM_ALWAYS_INLINE const T& operator[](const size_t i) const { - return data_ar()[i].get_ref(); - } - - /** Get first element of vector */ - HSHM_ALWAYS_INLINE T& front() { - return (*this)[0]; - } - - /** Get last element of vector */ - HSHM_ALWAYS_INLINE T& back() { - return (*this)[size() - 1]; - } - - /** Construct an element at the back of the vector */ - template - void emplace_back(Args&& ...args) { - ShmArchive *vec = data_ar(); - if (length_ == max_length_) { - vec = grow_vector(vec, 0, false); - } - HSHM_MAKE_AR(vec[length_], GetAllocator(), - std::forward(args)...) - ++length_; - } - - /** Construct an element in the front of the vector */ - template - HSHM_ALWAYS_INLINE void emplace_front(Args&& ...args) { - emplace(begin(), std::forward(args)...); - } - - /** Construct an element at an arbitrary position in the vector */ - template - void emplace(iterator_t pos, Args&&... args) { - if (pos.is_end()) { - emplace_back(std::forward(args)...); - return; - } - ShmArchive *vec = data_ar(); - if (length_ == max_length_) { - vec = grow_vector(vec, 0, false); - } - shift_right(pos); - HSHM_MAKE_AR(vec[pos.i_], GetAllocator(), - std::forward(args)...) - ++length_; - } - - /** Replace an element at a position */ - template - HSHM_ALWAYS_INLINE void replace(iterator_t pos, Args&&... args) { - if (pos.is_end()) { - return; - } - ShmArchive *vec = data_ar(); - hipc::Allocator::DestructObj((*this)[pos.i_]); - HSHM_MAKE_AR(vec[pos.i_], GetAllocator(), - std::forward(args)...) - } - - /** Delete the element at \a pos position */ - HSHM_ALWAYS_INLINE void erase(iterator_t pos) { - if (pos.is_end()) return; - shift_left(pos, 1); - length_ -= 1; - } - - /** Delete elements between first and last */ - HSHM_ALWAYS_INLINE void erase(iterator_t first, iterator_t last) { - size_t last_i; - if (first.is_end()) return; - if (last.is_end()) { - last_i = size(); - } else { - last_i = last.i_; - } - size_t count = last_i - first.i_; - if (count == 0) return; - shift_left(first, count); - length_ -= count; - } - - /** Delete all elements from the vector */ - HSHM_ALWAYS_INLINE void clear() { - erase(begin(), end()); - } - - /** Get the size of the vector */ - template - HSHM_ALWAYS_INLINE SizeT size() const { - return static_cast(length_); - } - - /** Get the data in the vector */ - HSHM_ALWAYS_INLINE void* data() { - return reinterpret_cast(data_ar()); - } - - /** Get constant pointer to the data */ - HSHM_ALWAYS_INLINE void* data() const { - return reinterpret_cast(data_ar()); - } - - /** Retreives a pointer to the internal array */ - HSHM_ALWAYS_INLINE ShmArchive* data_ar() { - return GetAllocator()->template Convert>(vec_ptr_); - } - - /** Retreives a pointer to the array */ - HSHM_ALWAYS_INLINE ShmArchive* data_ar() const { - return GetAllocator()->template Convert>(vec_ptr_); - } - - /**==================================== - * Internal Operations - * ===================================*/ - private: - /** - * Grow a vector to a new size. - * - * @param vec the C-style array of elements to grow - * @param max_length the new length of the vector. If 0, the current size - * of the vector will be multiplied by a constant. - * @param args the arguments used to construct the elements of the vector - * */ - template - ShmArchive* grow_vector(ShmArchive *vec, size_t max_length, - bool resize, Args&& ...args) { - // Grow vector by 25% - if (max_length == 0) { - max_length = 5 * max_length_ / 4; - if (max_length <= max_length_ + 10) { - max_length += 10; - } - } - if (max_length < max_length_) { - return nullptr; - } - - // Allocate new shared-memory vec - ShmArchive *new_vec; - if constexpr(std::is_pod() && !IS_SHM_ARCHIVEABLE(T)) { - // Use reallocate for well-behaved objects - new_vec = GetAllocator()->template - ReallocateObjs>(vec_ptr_, max_length); - } else { - // Use std::move for unpredictable objects - Pointer new_p; - new_vec = GetAllocator()->template - AllocateObjs>(max_length, new_p); - for (size_t i = 0; i < length_; ++i) { - T& old_entry = (*this)[i]; - HSHM_MAKE_AR(new_vec[i], GetAllocator(), - std::move(old_entry)) - } - if (!vec_ptr_.IsNull()) { - GetAllocator()->Free(vec_ptr_); - } - vec_ptr_ = new_p; - } - if (new_vec == nullptr) { - throw OUT_OF_MEMORY.format("vector::emplace_back", - max_length*sizeof(ShmArchive)); - } - if (resize) { - for (size_t i = length_; i < max_length; ++i) { - HSHM_MAKE_AR(new_vec[i], GetAllocator(), - std::forward(args)...) - } - } - - // Update vector header - max_length_ = max_length; - return new_vec; - } - - /** - * Shift every element starting at "pos" to the left by count. Any element - * who would be shifted before "pos" will be deleted. - * - * @param pos the starting position - * @param count the amount to shift left by - * */ - void shift_left(const iterator_t pos, size_t count = 1) { - ShmArchive *vec = data_ar(); - for (size_t i = 0; i < count; ++i) { - HSHM_DESTROY_AR(vec[pos.i_ + i]) - } - auto dst = vec + pos.i_; - auto src = dst + count; - for (auto i = pos.i_ + count; i < size(); ++i) { - memcpy((void*)dst, (void*)src, sizeof(ShmArchive)); - dst += 1; src += 1; - } - } - - /** - * Shift every element starting at "pos" to the right by count. Increases - * the total number of elements of the vector by "count". Does not modify - * the size parameter of the vector, this is done elsewhere. - * - * @param pos the starting position - * @param count the amount to shift right by - * */ - void shift_right(const iterator_t pos, size_t count = 1) { - auto src = data_ar() + size() - 1; - auto dst = src + count; - auto sz = static_cast(size()); - for (auto i = sz - 1; i >= pos.i_; --i) { - memcpy((void*)dst, (void*)src, sizeof(ShmArchive)); - dst -= 1; src -= 1; - } - } - - /**==================================== - * Iterators - * ===================================*/ - public: - /** Beginning of the forward iterator */ - iterator_t begin() { - return iterator_t(this, 0); - } - - /** End of the forward iterator */ - iterator_t end() { - return iterator_t(this, size()); - } - - /** Beginning of the constant forward iterator */ - citerator_t cbegin() const { - return citerator_t(const_cast(this), 0); - } - - /** End of the forward iterator */ - citerator_t cend() const { - return citerator_t(const_cast(this), size()); - } - - /** Beginning of the reverse iterator */ - riterator_t rbegin() { - return riterator_t(this, size() - 1); - } - - /** End of the reverse iterator */ - riterator_t rend() { - return citerator_t(this, (off64_t)-1); - } - - /** Beginning of the constant reverse iterator */ - criterator_t crbegin() const { - return criterator_t(const_cast(this), size() - 1); - } - - /** End of the constant reverse iterator */ - criterator_t crend() const { - return criterator_t(const_cast(this), (off64_t)-1); - } -}; - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS -#undef TYPED_HEADER - -#endif // HERMES_DATA_STRUCTURES_LOCKLESS_VECTOR_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_list.h b/hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_list.h deleted file mode 100644 index 8c9cea91c..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_list.h +++ /dev/null @@ -1,169 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_NUMA_AWARE_LIST_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_NUMA_AWARE_LIST_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/data_structures/ipc/list.h" -#include "hermes_shm/data_structures/ipc/vector.h" - -namespace hshm::ipc { - -template -class numa_list; - -#define CLASS_NAME numa_list -#define TYPED_CLASS numa_list -#define TYPED_HEADER ShmHeader - -template -class ShmHeader; - -template -class ShmHeader> : public hipc::ShmBaseHeader { - ShmArchive>> numa_lists_; - - explicit ShmHeader(Allocator *alloc, int num_numa) { - numa_lists_.shm_init(alloc, num_numa); - } -}; - -template -class numa_list : public ShmContainer { - public: - SHM_CONTAINER_TEMPLATE((CLASS_NAME), (TYPED_CLASS), (TYPED_HEADER)) - - public: - /**==================================== - * Shm Overrides - * ===================================*/ - - /** Default constructor. A default is required. */ - CLASS_NAME() = default; - - /** Default shm constructor */ - void shm_init_main(TYPED_HEADER *header, - hipc::Allocator *alloc) { - shm_init_allocator(alloc); - shm_make_header(header, alloc_); - } - - /** Move constructor */ - void shm_strong_move_main(TYPED_HEADER *header, - hipc::Allocator *alloc, - CLASS_NAME &other) { - shm_init_main(header, alloc); - } - - /** Copy constructor */ - void shm_strong_copy_main(TYPED_HEADER *header, - hipc::Allocator *alloc, - const CLASS_NAME &other) { - shm_init_main(header, alloc); - } - - /** Destroy the shared-memory data. */ - void shm_destroy_main() {} - - /** Store into shared memory */ - void shm_serialize_main() const {} - - /** Load from shared memory */ - void shm_deserialize_main() {} - - /**==================================== - * Custom methods - * ===================================*/ - - /** Construct an element at the back of the numa_list */ - template - void emplace_back(Args&&... args) { - } - - /** Construct an element at the beginning of the numa_list */ - template - void emplace_front(Args&&... args) { - } - - /** Construct an element at \a pos position in the numa_list */ - template - void emplace(list_iterator pos, Args&&... args) { - } - - /** Erase element with ID */ - void erase(const T &entry) { - } - - /** Erase the element at pos */ - void erase(list_iterator pos) { - } - - /** Erase all elements between first and last */ - void erase(list_iterator first, - list_iterator last) { - } - - /** Destroy all elements in the numa_list */ - void clear() { - } - - /** Get the object at the front of the numa_list */ - Ref front() { - } - - /** Get the object at the back of the numa_list */ - Ref back() { - } - - /** Get the number of elements in the numa_list */ - size_t size() const { - if (!IsNull()) { - return header_->length_; - } - return 0; - } - - /** Find an element in this numa_list */ - list_iterator find(const T &entry) { - for (auto iter = begin(); iter != end(); ++iter) { - hipc::Ref ref = *iter; - if (*ref == entry) { - return iter; - } - } - return end(); - } - - /** - * ITERATORS - * */ - - /** Forward iterator begin */ - list_iterator begin() { - } - - /** Forward iterator end */ - static list_iterator const end() { - } - - /** Constant forward iterator begin */ - list_citerator cbegin() const { - } - - /** Constant forward iterator end */ - static list_citerator const cend() { - } -}; -} // namespace hshm::ipc - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_NUMA_AWARE_LIST_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_vector.h b/hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_vector.h deleted file mode 100644 index 93d500909..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/numa_aware/numa_vector.h +++ /dev/null @@ -1,18 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_NUMA_AWARE_VECTOR_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_NUMA_AWARE_VECTOR_H_ - -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_DATA_STRUCTURES_NUMA_AWARE_VECTOR_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h b/hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h deleted file mode 100644 index 49c51e37e..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/serialization/shm_serialize.h +++ /dev/null @@ -1,111 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ -#define HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ - -#define NOREF typename std::remove_reference::type - -namespace hshm::ipc { - -class ShmSerializer { - public: - size_t off_; - - /** Default constructor */ - ShmSerializer() : off_(0) {} - - /** Get the SHM serialized size of an argument pack */ - template - HSHM_ALWAYS_INLINE static size_t shm_buf_size(Args&& ...args) { - size_t size = 0; - auto lambda = [&size](auto i, auto &&arg) { - if constexpr(IS_SHM_ARCHIVEABLE(NOREF)) { - size += sizeof(hipc::OffsetPointer); - } else if constexpr(std::is_pod()) { - size += sizeof(arg); - } else { - throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); - } - }; - ForwardIterateArgpack::Apply(make_argpack( - std::forward(args)...), lambda); - return size; - } - - /** Serialize a set of arguments into shared memory */ - template - HSHM_ALWAYS_INLINE char* serialize(Allocator *alloc, Args&& ...args) { - size_t buf_size = sizeof(allocator_id_t) + shm_buf_size( - std::forward(args)...); - Pointer p; - char *buf = alloc->AllocatePtr(buf_size, p); - memcpy(buf, &p.allocator_id_, sizeof(allocator_id_t)); - off_ = sizeof(allocator_id_t); - auto lambda = [buf, this](auto i, auto &&arg) { - if constexpr(IS_SHM_ARCHIVEABLE(NOREF)) { - OffsetPointer p = arg.template GetShmPointer(); - memcpy(buf + this->off_, (void*)&p, sizeof(p)); - this->off_ += sizeof(p); - } else if constexpr(std::is_pod()) { - memcpy(buf + this->off_, &arg, sizeof(arg)); - this->off_ += sizeof(arg); - } else { - throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); - } - }; - ForwardIterateArgpack::Apply(make_argpack( - std::forward(args)...), lambda); - return buf; - } - - /** Deserialize an allocator from the SHM buffer */ - HSHM_ALWAYS_INLINE Allocator* deserialize(char *buf) { - allocator_id_t alloc_id; - memcpy((void*)&alloc_id, buf + off_, sizeof(allocator_id_t)); - off_ += sizeof(allocator_id_t); - return HERMES_MEMORY_MANAGER->GetAllocator(alloc_id); - } - - /** Deserialize an argument from the SHM buffer */ - template - HSHM_ALWAYS_INLINE T deserialize(Allocator *alloc, char *buf) { - if constexpr(std::is_pod()) { - T arg; - memcpy(&arg, buf + off_, sizeof(arg)); - off_ += sizeof(arg); - return arg; - } else { - throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); - } - } - - /** Deserialize an argument from the SHM buffer */ - template - HSHM_ALWAYS_INLINE void deserialize(Allocator *alloc, - char *buf, hipc::mptr &arg) { - if constexpr(IS_SHM_ARCHIVEABLE(T)) { - OffsetPointer p; - memcpy((void*)&p, buf + off_, sizeof(p)); - arg.shm_deserialize(alloc, p); - off_ += sizeof(p); - } else { - throw IPC_ARGS_NOT_SHM_COMPATIBLE.format(); - } - } -}; - -} // namespace hshm::ipc - -#undef NOREF - -#endif // HERMES_SHM_DATA_STRUCTURES_SERIALIZATION_SHM_SERIALIZE_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/serialization/thallium.h b/hermes_shm/include/hermes_shm/data_structures/serialization/thallium.h deleted file mode 100644 index 219c0a2f8..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/serialization/thallium.h +++ /dev/null @@ -1,222 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_SERIALIZATION_THALLIUM_H_ -#define HERMES_DATA_STRUCTURES_SERIALIZATION_THALLIUM_H_ - -#include -#include "hermes_shm/data_structures/ipc/string.h" -#include -#include -#include -#include "hermes_shm/data_structures/containers/charbuf.h" -#include "hermes_shm/data_structures/ipc/slist.h" - -namespace thallium { - -/**==================================== - * Identifier Serialization - * ===================================*/ - -/** Lets Thallium know how to serialize an hipc::allocator_id */ -template -void serialize(A &ar, hipc::allocator_id_t &alloc_id) { - ar &alloc_id.int_; -} - -/**==================================== - * Vector Serialization - * ===================================*/ - -/** Lets Thallium know how to serialize a generic vector */ -template -void save_vec(A &ar, ContainerT &obj) { - ar << obj.size(); - for (auto iter = obj.cbegin(); iter != obj.cend(); ++iter) { - ar << (*iter); - } -} - -/** Lets Thallium know how to deserialize a generic vector */ -template -void load_vec(A &ar, ContainerT &obj) { - size_t size; - ar >> size; - obj.resize(size); - for (size_t i = 0; i < size; ++i) { - ar >> (obj[i]); - } -} - -/** Lets Thallium know how to serialize an hipc::vector. */ -template -void save(A &ar, hipc::vector &vec) { - save_vec, T>(ar, vec); -} -template -void save(A &ar, hipc::uptr> &vec) { - save_vec, T>(ar, *vec); -} - -/** Lets Thallium know how to deserialize an hipc::vector. */ -template -void load(A &ar, hipc::vector &vec) { - load_vec, T>(ar, vec); -} -template -void load(A &ar, hipc::uptr> &vec) { - vec = hipc::make_uptr>(); - load_vec, T>(ar, *vec); -} - -/**==================================== - * List Serialization - * ===================================*/ - -/** Lets Thallium know how to serialize a generic list */ -template -void save_list(A &ar, ContainerT &obj) { - ar << obj.size(); - for (auto iter = obj.cbegin(); iter != obj.cend(); ++iter) { - ar << *(*iter); - } -} - -/** Lets Thallium know how to deserialize a generic list */ -template -void load_list(A &ar, ContainerT &obj) { - size_t size; - ar >> size; - for (int i = 0; i < size; ++i) { - obj->emplace_back(); - auto last = obj->back(); - ar >> (*last); - } -} - -/** Lets Thallium know how to serialize an hipc::slist. */ -template -void save(A &ar, hipc::slist &lp) { - save_list, T>(ar, lp); -} -template -void save(A &ar, hipc::uptr> &lp) { - save_list, T>(ar, *lp); -} - -/** Lets Thallium know how to deserialize an hipc::slist. */ -template -void load(A &ar, hipc::slist &lp) { - load_list, T>(ar, lp); -} -template -void load(A &ar, hipc::uptr> &lp) { - lp = hipc::make_uptr>(); - load_list, T>(ar, *lp); -} - -/** Lets Thallium know how to serialize an hipc::list. */ -template -void save(A &ar, const hipc::list &lp) { - save_list, T>(ar, *lp); -} -template -void save(A &ar, hipc::uptr> &lp) { - save_list, T>(ar, *lp); -} - -/** Lets Thallium know how to deserialize an hipc::list. */ -template -void load(A &ar, hipc::list &lp) { - load_list, T>(ar, lp); -} -template -void load(A &ar, hipc::uptr> &lp) { - lp = hipc::make_uptr>(); - load_list, T>(ar, *lp); -} - -/**==================================== - * String Serialization - * ===================================*/ - -/** Lets Thallium know how to serialize a generic string. */ -template -void save_string(A &ar, StringT &text) { - ar << text.size(); - ar.write(text.data(), text.size()); -} -/** Lets Thallium know how to serialize a generic string. */ -template -void load_string(A &ar, StringT &text) { - size_t size; - ar >> size; - text.resize(size); - ar.read(text.data(), text.size()); -} - -/** Lets Thallium know how to serialize an hipc::string. */ -template -void save(A &ar, hipc::string &text) { - save_string(ar, text); -} -template -void save(A &ar, hipc::uptr &text) { - save_string(ar, *text); -} - -/** Lets Thallium know how to deserialize an hipc::string. */ -template -void load(A &ar, hipc::string &text) { - load_string(ar, text); -} -template -void load(A &ar, hipc::uptr &text) { - text = hipc::make_uptr(); - load_string(ar, *text); -} - -/** Lets Thallium know how to serialize an hshm::charbuf. */ -template -void save(A &ar, hshm::charbuf &text) { - save_string(ar, text); -} - -/** Lets Thallium know how to deserialize an hshm::charbuf. */ -template -void load(A &ar, hshm::charbuf &text) { - load_string(ar, text); -} - -/**==================================== - * Bitfield Serialization - * ===================================*/ - -/** Lets Thallium know how to serialize a bitfield. */ -template -void save(A &ar, hshm::bitfield32_t &field) { - uint32_t bits = field.bits_; - ar << bits; -} - -/** Lets Thallium know how to deserialize a bitfield. */ -template -void load(A &ar, hshm::bitfield32_t &field) { - uint32_t bits; - ar >> bits; - field.bits_ = bits; -} - -} // namespace thallium - -#endif // HERMES_DATA_STRUCTURES_SERIALIZATION_THALLIUM_H_ diff --git a/hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h b/hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h deleted file mode 100644 index f861269c8..000000000 --- a/hermes_shm/include/hermes_shm/data_structures/smart_ptr/smart_ptr_base.h +++ /dev/null @@ -1,307 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_DATA_STRUCTURES_PTR_H_ -#define HERMES_DATA_STRUCTURES_PTR_H_ - -#include "hermes_shm/constants/macros.h" -#include "hermes_shm/memory/memory.h" -#include "hermes_shm/memory/allocator/allocator.h" -#include "hermes_shm/memory/memory_registry.h" -#include "hermes_shm/data_structures/ipc/internal/shm_internal.h" -#include "hermes_shm/data_structures/ipc/internal/shm_smart_ptr.h" - -namespace hshm::ipc { - -/** - * MACROS to simplify the ptr namespace - * */ -#define CLASS_NAME smart_ptr_base -#define TYPED_CLASS smart_ptr_base - -/** - * Flags used for the smart pointer - * */ -#define POINTER_IS_OWNED BIT_OPT(uint32_t, 0) - -/** - * Creates a unique instance of a shared-memory data structure - * and deletes eventually. - * */ -template -class smart_ptr_base { - public: - T *obj_; - Allocator *alloc_; - bitfield32_t flags_; - - public: - /**==================================== - * Initialization + Destruction - * ===================================*/ - - /** Default constructor. */ - HSHM_ALWAYS_INLINE smart_ptr_base() = default; - - /** Create the mptr contents */ - template - void shm_init(Allocator *alloc, Args&& ...args) { - alloc_ = alloc; - OffsetPointer p; - if constexpr(IS_SHM_ARCHIVEABLE(T)) { - obj_ = alloc_->template AllocateConstructObjs( - 1, p, alloc, std::forward(args)...); - } else { - obj_ = alloc_->template AllocateConstructObjs( - 1, p, std::forward(args)...); - } - if constexpr(unique) { - flags_.SetBits(POINTER_IS_OWNED); - } - } - - /** Destructor. Does not free data. */ - HSHM_ALWAYS_INLINE ~smart_ptr_base() { - if constexpr(unique) { - if (flags_.Any(POINTER_IS_OWNED)) { - shm_destroy(); - } - } - } - - /** Explicit destructor */ - HSHM_ALWAYS_INLINE void shm_destroy() { - if constexpr(IS_SHM_ARCHIVEABLE(T)) { - obj_->shm_destroy(); - } - alloc_->template FreeDestructObjs(obj_, 1); - } - - /**==================================== - * Dereference Operations - * ===================================*/ - - /** Gets a pointer to the internal object */ - HSHM_ALWAYS_INLINE T* get() { - return obj_; - } - - /** Gets a pointer to the internal object */ - HSHM_ALWAYS_INLINE const T* get() const { - return obj_; - } - - /** Dereference operator */ - HSHM_ALWAYS_INLINE T& operator*() { - return *get(); - } - - /** Constant Dereference operator */ - HSHM_ALWAYS_INLINE const T& operator*() const { - return *get(); - } - - /** Pointer operator */ - HSHM_ALWAYS_INLINE T* operator->() { - return get(); - } - - /** Constant pointer operator */ - HSHM_ALWAYS_INLINE const T* operator->() const { - return get(); - } - - /**==================================== - * Move + Copy Operations - * ===================================*/ - - /** Copy constructor (equivalent to move) */ - HSHM_ALWAYS_INLINE smart_ptr_base(const smart_ptr_base &other) { - shm_strong_copy(other); - if constexpr(unique) { - flags_.UnsetBits(POINTER_IS_OWNED); - } - } - - /** Copy assignment operator (equivalent to move) */ - HSHM_ALWAYS_INLINE smart_ptr_base& operator=(const smart_ptr_base &other) { - if (this != &other) { - shm_strong_copy(other); - if constexpr(unique) { - flags_.UnsetBits(POINTER_IS_OWNED); - } - } - return *this; - } - - /** Move constructor */ - HSHM_ALWAYS_INLINE smart_ptr_base(smart_ptr_base&& other) noexcept { - shm_strong_move(std::forward(other)); - } - - /** Move assignment operator */ - HSHM_ALWAYS_INLINE smart_ptr_base& - operator=(smart_ptr_base&& other) noexcept { - if (this != &other) { - shm_strong_move(std::forward(other)); - } - return *this; - } - - /** Internal copy operation */ - HSHM_ALWAYS_INLINE void shm_strong_copy(const smart_ptr_base &other) { - obj_ = other.obj_; - alloc_ = other.alloc_; - flags_ = other.flags_; - } - - /** Internal move operation */ - HSHM_ALWAYS_INLINE void shm_strong_move(smart_ptr_base &&other) { - shm_strong_copy(other); - if constexpr(unique) { - flags_ = other.flags_; - other.flags_.UnsetBits(POINTER_IS_OWNED); - } - } - - /**==================================== - * Deserialization - * ===================================*/ - - /** Constructor. Deserialize from a TypedPointer */ - HSHM_ALWAYS_INLINE explicit smart_ptr_base(const TypedPointer &ar) { - shm_deserialize(ar); - } - - /** Constructor. Deserialize from a TypedAtomicPointer */ - HSHM_ALWAYS_INLINE explicit smart_ptr_base(const TypedAtomicPointer &ar) { - shm_deserialize(ar); - } - - /** Deserialize from a process-independent pointer */ - template - HSHM_ALWAYS_INLINE void shm_deserialize(const PointerT &ar) { - auto alloc = HERMES_MEMORY_REGISTRY_REF.GetAllocator(ar.allocator_id_); - obj_ = alloc->template Convert(ar); - if constexpr(unique) { - flags_.UnsetBits(POINTER_IS_OWNED); - } - } - - /** Deserialize from an offset pointer */ - HSHM_ALWAYS_INLINE void shm_deserialize(Allocator *alloc, - const OffsetPointer &ar) { - obj_ = alloc->template Convert(ar); - if constexpr(unique) { - flags_.UnsetBits(POINTER_IS_OWNED); - } - } - - /**==================================== - * Serialization - * ===================================*/ - - /** Serialize to a process-independent pointer */ - template - HSHM_ALWAYS_INLINE void shm_serialize(PointerT &ar) const { - ar = alloc_->template Convert(obj_); - } - - /**==================================== - * Serialization + Deserialization Ops - * ===================================*/ - SHM_SERIALIZE_DESERIALIZE_OPS((T)) - - /**==================================== - * Hash function - * ===================================*/ - HSHM_ALWAYS_INLINE size_t hash() const { - return std::hash{}(*obj_); - } -}; - -/**==================================== -* Helper Functions -* ===================================*/ - -/** Mptr is non-unique and requires explicit destruction */ -template -using mptr = smart_ptr_base; - -/** Uptr has a specific container owning the object */ -template -using uptr = smart_ptr_base; - -/** Construct an mptr with default allocator */ -template -static PointerT make_ptr_base(Args&& ...args) { - PointerT ptr; - ptr.shm_init(std::forward(args)...); - return ptr; -} - -/** Create a manual pointer with default allocator */ -template -mptr make_mptr(Args&& ...args) { - auto alloc = HERMES_MEMORY_REGISTRY->GetDefaultAllocator(); - return make_ptr_base>(alloc, std::forward(args)...); -} - -/** Create a manual pointer with non-default allocator */ -template -mptr make_mptr(Allocator *alloc, Args&& ...args) { - return make_ptr_base>(alloc, std::forward(args)...); -} - -/** Create a unique pointer with default allocator */ -template -uptr make_uptr(Args&& ...args) { - auto alloc = HERMES_MEMORY_REGISTRY->GetDefaultAllocator(); - return make_ptr_base>(alloc, std::forward(args)...); -} - -/** Create a unique pointer with non-default allocator */ -template -uptr make_uptr(Allocator *alloc, Args&& ...args) { - return make_ptr_base>(alloc, std::forward(args)...); -} - -} // namespace hshm::ipc - -#undef CLASS_NAME -#undef TYPED_CLASS - -/**==================================== -* Hash Functions -* ===================================*/ -namespace std { - -/** Hash function for mptr */ -template -struct hash> { - size_t operator()(const hshm::ipc::mptr &obj) const { - return obj.hash(); - } -}; - -/** Hash function for uptr */ -template -struct hash> { - size_t operator()(const hshm::ipc::uptr &obj) const { - return obj.hash(); - } -}; - -} // namespace std - -#endif // HERMES_DATA_STRUCTURES_PTR_H_ diff --git a/hermes_shm/include/hermes_shm/hermes_shm.h b/hermes_shm/include/hermes_shm/hermes_shm.h deleted file mode 100644 index 787de1f85..000000000 --- a/hermes_shm/include/hermes_shm/hermes_shm.h +++ /dev/null @@ -1,38 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_HERMES_SHM_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_HERMES_SHM_H_ - -#include "thread/thread_model_manager.h" -#include "thread/lock.h" - -#include "util/singleton.h" -#include "util/auto_trace.h" -#include "util/config_parse.h" -#include "util/errors.h" -#include "util/formatter.h" -#include "util/logging.h" -#include "util/partitioner.h" -#include "util/timer.h" -#include "util/type_switch.h" - -#include "types/argpack.h" -#include "types/atomic.h" -#include "types/bitfield.h" -#include "types/real_number.h" - -#include "memory/memory_manager.h" - -#include "data_structures/data_structure.h" - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_HERMES_SHM_H_ diff --git a/hermes_shm/include/hermes_shm/introspect/system_info.h b/hermes_shm/include/hermes_shm/introspect/system_info.h deleted file mode 100644 index 626d35db8..000000000 --- a/hermes_shm/include/hermes_shm/introspect/system_info.h +++ /dev/null @@ -1,48 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SYSINFO_INFO_H_ -#define HERMES_SYSINFO_INFO_H_ - -#include -#include -#include "hermes_shm/util/singleton/_global_singleton.h" - -#define HERMES_SYSTEM_INFO \ - hshm::GlobalSingleton::GetInstance() -#define HERMES_SYSTEM_INFO_T hshm::SystemInfo* - -namespace hshm { - -struct SystemInfo { - int pid_; - int ncpu_; - int page_size_; - int uid_; - int gid_; - size_t ram_size_; - - SystemInfo() { - pid_ = getpid(); - ncpu_ = get_nprocs_conf(); - page_size_ = getpagesize(); - struct sysinfo info; - sysinfo(&info); - uid_ = getuid(); - gid_ = getgid(); - ram_size_ = info.totalram; - } -}; - -} // namespace hshm - -#endif // HERMES_SYSINFO_INFO_H_ diff --git a/hermes_shm/include/hermes_shm/memory/allocator/allocator.h b/hermes_shm/include/hermes_shm/memory/allocator/allocator.h deleted file mode 100644 index c6978c469..000000000 --- a/hermes_shm/include/hermes_shm/memory/allocator/allocator.h +++ /dev/null @@ -1,523 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MEMORY_ALLOCATOR_ALLOCATOR_H_ -#define HERMES_MEMORY_ALLOCATOR_ALLOCATOR_H_ - -#include -#include -#include - -namespace hshm::ipc { - -/** - * The allocator type. - * Used to reconstruct allocator from shared memory - * */ -enum class AllocatorType { - kStackAllocator, - kMallocAllocator, - kFixedPageAllocator, - kScalablePageAllocator, -}; - -/** - * The basic shared-memory allocator header. - * Allocators inherit from this. - * */ -struct AllocatorHeader { - int allocator_type_; - allocator_id_t allocator_id_; - size_t custom_header_size_; - - AllocatorHeader() = default; - - void Configure(allocator_id_t allocator_id, - AllocatorType type, - size_t custom_header_size) { - allocator_type_ = static_cast(type); - allocator_id_ = allocator_id; - custom_header_size_ = custom_header_size; - } -}; - -/** - * The allocator base class. - * */ -class Allocator { - protected: - char *buffer_; - size_t buffer_size_; - char *custom_header_; - - public: - /** - * Constructor - * */ - Allocator() : custom_header_(nullptr) {} - - /** - * Destructor - * */ - virtual ~Allocator() = default; - - /** - * Create the shared-memory allocator with \a id unique allocator id over - * the particular slot of a memory backend. - * - * The shm_init function is required, but cannot be marked virtual as - * each allocator has its own arguments to this method. Though each - * allocator must have "id" as its first argument. - * */ - // virtual void shm_init(allocator_id_t id, Args ...args) = 0; - - /** - * Deserialize allocator from a buffer. - * */ - virtual void shm_deserialize(char *buffer, - size_t buffer_size) = 0; - - /** - * Allocate a region of memory of \a size size - * */ - virtual OffsetPointer AllocateOffset(size_t size) = 0; - - /** - * Allocate a region of memory to a specific pointer type - * */ - template - HSHM_ALWAYS_INLINE POINTER_T Allocate(size_t size) { - return POINTER_T(GetId(), AllocateOffset(size).load()); - } - - /** - * Allocate a region of memory of \a size size - * and \a alignment alignment. Assumes that - * alignment is not 0. - * */ - virtual OffsetPointer AlignedAllocateOffset(size_t size, - size_t alignment) = 0; - - /** - * Allocate a region of memory to a specific pointer type - * */ - template - HSHM_ALWAYS_INLINE POINTER_T AlignedAllocate(size_t size, size_t alignment) { - return POINTER_T(GetId(), AlignedAllocateOffset(size, alignment).load()); - } - - /** - * Allocate a region of \a size size and \a alignment - * alignment. Will fall back to regular Allocate if - * alignmnet is 0. - * */ - template - HSHM_ALWAYS_INLINE POINTER_T Allocate(size_t size, size_t alignment) { - if (alignment == 0) { - return Allocate(size); - } else { - return AlignedAllocate(size, alignment); - } - } - - /** - * Reallocate \a pointer to \a new_size new size - * If p is kNullPointer, will internally call Allocate. - * - * @return true if p was modified. - * */ - template - HSHM_ALWAYS_INLINE bool Reallocate(POINTER_T &p, size_t new_size) { - if (p.IsNull()) { - p = Allocate(new_size); - return true; - } - auto new_p = ReallocateOffsetNoNullCheck(p.ToOffsetPointer(), - new_size); - bool ret = new_p == p.ToOffsetPointer(); - p.off_ = new_p.load(); - return ret; - } - - /** - * Reallocate \a pointer to \a new_size new size. - * Assumes that p is not kNullPointer. - * - * @return true if p was modified. - * */ - virtual OffsetPointer ReallocateOffsetNoNullCheck(OffsetPointer p, - size_t new_size) = 0; - - /** - * Free the memory pointed to by \a ptr Pointer - * */ - template - HSHM_ALWAYS_INLINE void FreePtr(T *ptr) { - if (ptr == nullptr) { - throw INVALID_FREE.format(); - } - FreeOffsetNoNullCheck(Convert(ptr)); - } - - /** - * Free the memory pointed to by \a p Pointer - * */ - template - HSHM_ALWAYS_INLINE void Free(POINTER_T &p) { - if (p.IsNull()) { - throw INVALID_FREE.format(); - } - FreeOffsetNoNullCheck(OffsetPointer(p.off_.load())); - } - - /** - * Free the memory pointed to by \a ptr Pointer - * */ - virtual void FreeOffsetNoNullCheck(OffsetPointer p) = 0; - - /** - * Get the allocator identifier - * */ - virtual allocator_id_t &GetId() = 0; - - /** - * Get the amount of memory that was allocated, but not yet freed. - * Useful for memory leak checks. - * */ - virtual size_t GetCurrentlyAllocatedSize() = 0; - - /**==================================== - * Pointer Allocators - * ===================================*/ - - /** - * Allocate a pointer of \a size size and return \a p process-independent - * pointer and a process-specific pointer. - * */ - template - HSHM_ALWAYS_INLINE T* AllocatePtr(size_t size, - POINTER_T &p, size_t alignment = 0) { - p = Allocate(size, alignment); - if (p.IsNull()) { return nullptr; } - return reinterpret_cast(buffer_ + p.off_.load()); - } - - /** - * Allocate a pointer of \a size size - * */ - template - HSHM_ALWAYS_INLINE T* AllocatePtr(size_t size, size_t alignment = 0) { - POINTER_T p; - return AllocatePtr(size, p, alignment); - } - - /** - * Allocate a pointer of \a size size - * */ - template - HSHM_ALWAYS_INLINE T* ClearAllocatePtr(size_t size, size_t alignment = 0) { - POINTER_T p; - return ClearAllocatePtr(size, p, alignment); - } - - /** - * Allocate a pointer of \a size size and return \a p process-independent - * pointer and a process-specific pointer. - * */ - template - HSHM_ALWAYS_INLINE T* ClearAllocatePtr(size_t size, - POINTER_T &p, size_t alignment = 0) { - p = Allocate(size, alignment); - if (p.IsNull()) { return nullptr; } - auto ptr = reinterpret_cast(buffer_ + p.off_.load()); - if (ptr) { - memset(ptr, 0, size); - } - return ptr; - } - - /** - * Reallocate a pointer to a new size - * - * @param p process-independent pointer (input & output) - * @param new_size the new size to allocate - * @param modified whether or not p was modified (output) - * @return A process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* ReallocatePtr(POINTER_T &p, size_t new_size, - bool &modified) { - modified = Reallocate(p, new_size); - return Convert(p); - } - - /** - * Reallocate a pointer to a new size - * - * @param p process-independent pointer (input & output) - * @param new_size the new size to allocate - * @return A process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* ReallocatePtr(POINTER_T &p, size_t new_size) { - Reallocate(p, new_size); - return Convert(p); - } - - /** - * Reallocate a pointer to a new size - * - * @param old_ptr process-specific pointer to reallocate - * @param new_size the new size to allocate - * @return A process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* ReallocatePtr(T *old_ptr, size_t new_size) { - OffsetPointer p = Convert(old_ptr); - return ReallocatePtr(p, new_size); - } - - /**==================================== - * Object Allocators - * ===================================*/ - - /** - * Allocate an array of objects (but don't construct). - * - * @return A process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* AllocateObjs(size_t count) { - OffsetPointer p; - return AllocateObjs(count, p); - } - - /** - * Allocate an array of objects (but don't construct). - * - * @param count the number of objects to allocate - * @param p process-independent pointer (output) - * @return A process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* AllocateObjs(size_t count, POINTER_T &p) { - return AllocatePtr(count * sizeof(T), p); - } - - /** - * Allocate an array of objects and memset to 0. - * - * @param count the number of objects to allocate - * @param p process-independent pointer (output) - * @return A process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* ClearAllocateObjs(size_t count, POINTER_T &p) { - return ClearAllocatePtr(count * sizeof(T), p); - } - - /** - * Allocate and construct an array of objects - * - * @param count the number of objects to allocate - * @param p process-independent pointer (output) - * @param args parameters to construct object of type T - * @return A process-specific pointer - * */ - template< - typename T, - typename POINTER_T = Pointer, - typename ...Args> - HSHM_ALWAYS_INLINE T* AllocateConstructObjs(size_t count, - POINTER_T &p, Args&& ...args) { - T *ptr = AllocateObjs(count, p); - ConstructObjs(ptr, 0, count, std::forward(args)...); - return ptr; - } - - /** - * Reallocate a pointer of objects to a new size. - * - * @param p process-independent pointer (input & output) - * @param old_count the original number of objects (avoids reconstruction) - * @param new_count the new number of objects - * - * @return A process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* ReallocateObjs(POINTER_T &p, size_t new_count) { - T *ptr = ReallocatePtr(p, new_count*sizeof(T)); - return ptr; - } - - /** - * Reallocate a pointer of objects to a new size and construct the - * new elements in-place. - * - * @param p process-independent pointer (input & output) - * @param old_count the original number of objects (avoids reconstruction) - * @param new_count the new number of objects - * @param args parameters to construct object of type T - * - * @return A process-specific pointer - * */ - template< - typename T, - typename POINTER_T = Pointer, - typename ...Args> - HSHM_ALWAYS_INLINE T* ReallocateConstructObjs(POINTER_T &p, - size_t old_count, - size_t new_count, - Args&& ...args) { - T *ptr = ReallocatePtr(p, new_count*sizeof(T)); - ConstructObjs(ptr, old_count, new_count, std::forward(args)...); - return ptr; - } - - /**==================================== - * Object Deallocators - * ===================================*/ - - /** - * Free + destruct objects - * */ - template - HSHM_ALWAYS_INLINE void FreeDestructObjs(T *ptr, size_t count) { - DestructObjs(ptr, count); - auto p = Convert(ptr); - Free(p); - } - - - /**==================================== - * Object Constructors - * ===================================*/ - - /** - * Construct each object in an array of objects. - * - * @param ptr the array of objects (potentially archived) - * @param old_count the original size of the ptr - * @param new_count the new size of the ptr - * @param args parameters to construct object of type T - * @return None - * */ - template< - typename T, - typename ...Args> - HSHM_ALWAYS_INLINE static void ConstructObjs(T *ptr, - size_t old_count, - size_t new_count, Args&& ...args) { - if (ptr == nullptr) { return; } - for (size_t i = old_count; i < new_count; ++i) { - ConstructObj(*(ptr + i), std::forward(args)...); - } - } - - /** - * Construct an object. - * - * @param ptr the object to construct (potentially archived) - * @param args parameters to construct object of type T - * @return None - * */ - template< - typename T, - typename ...Args> - HSHM_ALWAYS_INLINE static void ConstructObj(T &obj, Args&& ...args) { - new (&obj) T(std::forward(args)...); - } - - /** - * Destruct an array of objects - * - * @param ptr the object to destruct (potentially archived) - * @param count the length of the object array - * @return None - * */ - template - HSHM_ALWAYS_INLINE static void DestructObjs(T *ptr, size_t count) { - if (ptr == nullptr) { return; } - for (size_t i = 0; i < count; ++i) { - DestructObj(*(ptr + i)); - } - } - - /** - * Destruct an object - * - * @param ptr the object to destruct (potentially archived) - * @param count the length of the object array - * @return None - * */ - template - HSHM_ALWAYS_INLINE static void DestructObj(T &obj) { - obj.~T(); - } - - /**==================================== - * Helpers - * ===================================*/ - - /** - * Get the custom header of the shared-memory allocator - * - * @return Custom header pointer - * */ - template - HSHM_ALWAYS_INLINE HEADER_T* GetCustomHeader() { - return reinterpret_cast(custom_header_); - } - - /** - * Convert a process-independent pointer into a process-specific pointer - * - * @param p process-independent pointer - * @return a process-specific pointer - * */ - template - HSHM_ALWAYS_INLINE T* Convert(const POINTER_T &p) { - if (p.IsNull()) { return nullptr; } - return reinterpret_cast(buffer_ + p.off_.load()); - } - - /** - * Convert a process-specific pointer into a process-independent pointer - * - * @param ptr process-specific pointer - * @return a process-independent pointer - * */ - template - HSHM_ALWAYS_INLINE POINTER_T Convert(const T *ptr) { - if (ptr == nullptr) { return POINTER_T::GetNull(); } - return POINTER_T(GetId(), - reinterpret_cast(ptr) - - reinterpret_cast(buffer_)); - } - - /** - * Determine whether or not this allocator contains a process-specific - * pointer - * - * @param ptr process-specific pointer - * @return True or false - * */ - template - HSHM_ALWAYS_INLINE bool ContainsPtr(T *ptr) { - return reinterpret_cast(ptr) >= - reinterpret_cast(buffer_); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_ALLOCATOR_ALLOCATOR_H_ diff --git a/hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h b/hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h deleted file mode 100644 index 9971f96fe..000000000 --- a/hermes_shm/include/hermes_shm/memory/allocator/allocator_factory.h +++ /dev/null @@ -1,100 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_MEMORY_ALLOCATOR_ALLOCATOR_FACTORY_H_ -#define HERMES_MEMORY_ALLOCATOR_ALLOCATOR_FACTORY_H_ - -#include "allocator.h" -#include "stack_allocator.h" -#include "malloc_allocator.h" -#include "scalable_page_allocator.h" - -namespace hshm::ipc { - -class AllocatorFactory { - public: - /** - * Create a new memory allocator - * */ - template - static std::unique_ptr shm_init(allocator_id_t alloc_id, - size_t custom_header_size, - MemoryBackend *backend, - Args&& ...args) { - if constexpr(std::is_same_v) { - // StackAllocator - auto alloc = std::make_unique(); - alloc->shm_init(alloc_id, - custom_header_size, - backend->data_, - backend->data_size_, - std::forward(args)...); - return alloc; - } else if constexpr(std::is_same_v) { - // Malloc Allocator - auto alloc = std::make_unique(); - alloc->shm_init(alloc_id, - custom_header_size, - backend->data_size_, - std::forward(args)...); - return alloc; - } else if constexpr(std::is_same_v) { - // Scalable Page Allocator - auto alloc = std::make_unique(); - alloc->shm_init(alloc_id, - custom_header_size, - backend->data_, - backend->data_size_, - std::forward(args)...); - return alloc; - } else { - // Default - throw std::logic_error("Not a valid allocator"); - } - } - - /** - * Deserialize the allocator managing this backend. - * */ - static std::unique_ptr shm_deserialize(MemoryBackend *backend) { - auto header_ = reinterpret_cast(backend->data_); - switch (static_cast(header_->allocator_type_)) { - // Stack Allocator - case AllocatorType::kStackAllocator: { - auto alloc = std::make_unique(); - alloc->shm_deserialize(backend->data_, - backend->data_size_); - return alloc; - } - // Malloc Allocator - case AllocatorType::kMallocAllocator: { - auto alloc = std::make_unique(); - alloc->shm_deserialize(backend->data_, - backend->data_size_); - return alloc; - } - // Scalable Page Allocator - case AllocatorType::kScalablePageAllocator: { - auto alloc = std::make_unique(); - alloc->shm_deserialize(backend->data_, - backend->data_size_); - return alloc; - } - default: return nullptr; - } - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_ALLOCATOR_ALLOCATOR_FACTORY_H_ diff --git a/hermes_shm/include/hermes_shm/memory/allocator/heap.h b/hermes_shm/include/hermes_shm/memory/allocator/heap.h deleted file mode 100644 index 2b1d03911..000000000 --- a/hermes_shm/include/hermes_shm/memory/allocator/heap.h +++ /dev/null @@ -1,50 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_ALLOCATOR_HEAP_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_ALLOCATOR_HEAP_H_ - -#include "allocator.h" -#include "hermes_shm/thread/lock.h" - -namespace hshm::ipc { - -struct HeapAllocator { - std::atomic heap_off_; - size_t heap_size_; - - /** Default constructor */ - HeapAllocator() : heap_off_(0), heap_size_(0) {} - - /** Emplace constructor */ - explicit HeapAllocator(size_t heap_off, size_t heap_size) - : heap_off_(heap_off), heap_size_(heap_size) {} - - /** Explicit initialization */ - void shm_init(size_t heap_off, size_t heap_size) { - heap_off_ = heap_off; - heap_size_ = heap_size; - } - - /** Allocate off heap */ - HSHM_ALWAYS_INLINE OffsetPointer AllocateOffset(size_t size) { - size_t off = heap_off_.fetch_add(size); - if (off + size > heap_size_) { - throw OUT_OF_MEMORY.format(); - } - return OffsetPointer(off); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_ALLOCATOR_HEAP_H_ diff --git a/hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h b/hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h deleted file mode 100644 index ec777f7bc..000000000 --- a/hermes_shm/include/hermes_shm/memory/allocator/malloc_allocator.h +++ /dev/null @@ -1,100 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_MEMORY_ALLOCATOR_MALLOC_ALLOCATOR_H_ -#define HERMES_MEMORY_ALLOCATOR_MALLOC_ALLOCATOR_H_ - -#include "allocator.h" -#include "hermes_shm/thread/lock.h" - -namespace hshm::ipc { - -struct MallocAllocatorHeader : public AllocatorHeader { - std::atomic total_alloc_size_; - - MallocAllocatorHeader() = default; - - void Configure(allocator_id_t alloc_id, - size_t custom_header_size) { - AllocatorHeader::Configure(alloc_id, AllocatorType::kStackAllocator, - custom_header_size); - total_alloc_size_ = 0; - } -}; - -class MallocAllocator : public Allocator { - private: - MallocAllocatorHeader *header_; - - public: - /** - * Allocator constructor - * */ - MallocAllocator() - : header_(nullptr) {} - - /** - * Get the ID of this allocator from shared memory - * */ - allocator_id_t &GetId() override { - return header_->allocator_id_; - } - - /** - * Initialize the allocator in shared memory - * */ - void shm_init(allocator_id_t id, - size_t custom_header_size, - size_t buffer_size); - - /** - * Attach an existing allocator from shared memory - * */ - void shm_deserialize(char *buffer, - size_t buffer_size) override; - - /** - * Allocate a memory of \a size size. The page allocator cannot allocate - * memory larger than the page size. - * */ - OffsetPointer AllocateOffset(size_t size) override; - - /** - * Allocate a memory of \a size size, which is aligned to \a - * alignment. - * */ - OffsetPointer AlignedAllocateOffset(size_t size, size_t alignment) override; - - /** - * Reallocate \a p pointer to \a new_size new size. - * - * @return whether or not the pointer p was changed - * */ - OffsetPointer ReallocateOffsetNoNullCheck(OffsetPointer p, - size_t new_size) override; - - /** - * Free \a ptr pointer. Null check is performed elsewhere. - * */ - void FreeOffsetNoNullCheck(OffsetPointer p) override; - - /** - * Get the current amount of data allocated. Can be used for leak - * checking. - * */ - size_t GetCurrentlyAllocatedSize() override; -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_ALLOCATOR_MALLOC_ALLOCATOR_H_ diff --git a/hermes_shm/include/hermes_shm/memory/allocator/mp_page.h b/hermes_shm/include/hermes_shm/memory/allocator/mp_page.h deleted file mode 100644 index 306c65d34..000000000 --- a/hermes_shm/include/hermes_shm/memory/allocator/mp_page.h +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_MEMORY_ALLOCATOR_MP_PAGE_H_ -#define HERMES_INCLUDE_HERMES_MEMORY_ALLOCATOR_MP_PAGE_H_ - -#include "hermes_shm/data_structures/ipc/iqueue.h" - -namespace hshm::ipc { - -struct MpPage { - bitfield32_t flags_; /**< Flags of the page (e.g., free/alloc) */ - /** Offset from the start of the page to the beginning of this header */ - uint32_t off_; - size_t page_size_; /**< The total size of the page allocated */ - - HSHM_ALWAYS_INLINE void SetAllocated() { - flags_.SetBits(0x1); - } - - HSHM_ALWAYS_INLINE void UnsetAllocated() { - flags_.Clear(); - } - - HSHM_ALWAYS_INLINE bool IsAllocated() const { - return flags_.All(0x1); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_INCLUDE_HERMES_MEMORY_ALLOCATOR_MP_PAGE_H_ diff --git a/hermes_shm/include/hermes_shm/memory/allocator/scalable_page_allocator.h b/hermes_shm/include/hermes_shm/memory/allocator/scalable_page_allocator.h deleted file mode 100644 index 1829d55da..000000000 --- a/hermes_shm/include/hermes_shm/memory/allocator/scalable_page_allocator.h +++ /dev/null @@ -1,218 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_MEMORY_ALLOCATOR_SCALABLE_PAGE_ALLOCATOR_H -#define HERMES_MEMORY_ALLOCATOR_SCALABLE_PAGE_ALLOCATOR_H - -#include "allocator.h" -#include "hermes_shm/thread/lock.h" -#include "hermes_shm/data_structures/ipc/pair.h" -#include "hermes_shm/data_structures/ipc/vector.h" -#include "hermes_shm/data_structures/ipc/list.h" -#include "hermes_shm/data_structures/ipc/pair.h" -#include -#include "mp_page.h" - -namespace hshm::ipc { - -struct FreeListStats { - size_t page_size_; /**< Page size stored in this free list */ - size_t cur_alloc_; /**< Number of pages currently allocated */ - size_t max_alloc_; /**< Maximum number of pages allocated at a time */ - Mutex lock_; /**< The enqueue / dequeue lock */ - - /** Default constructor */ - FreeListStats() = default; - - /** Copy constructor */ - FreeListStats(const FreeListStats &other) { - strong_copy(other); - } - - /** Copy assignment operator */ - FreeListStats& operator=(const FreeListStats &other) { - strong_copy(other); - return *this; - } - - /** Move constructor */ - FreeListStats(FreeListStats &&other) { - strong_copy(other); - } - - /** Move assignment operator */ - FreeListStats& operator=(FreeListStats &&other) { - strong_copy(other); - return *this; - } - - /** Internal copy */ - void strong_copy(const FreeListStats &other) { - page_size_ = other.page_size_; - cur_alloc_ = other.cur_alloc_; - max_alloc_ = other.max_alloc_; - } - - /** Increment allocation count */ - void AddAlloc() { - cur_alloc_ += 1; - if (cur_alloc_ > max_alloc_) { - max_alloc_ += 1; - } - } - - /** Decrement allocation count */ - void AddFree() { - cur_alloc_ -= 1; - } -}; - -struct ScalablePageAllocatorHeader : public AllocatorHeader { - ShmArchive>>> free_lists_; - std::atomic total_alloc_; - size_t coalesce_trigger_; - size_t coalesce_window_; - RwLock coalesce_lock_; - - ScalablePageAllocatorHeader() = default; - - void Configure(allocator_id_t alloc_id, - size_t custom_header_size, - Allocator *alloc, - size_t buffer_size, - RealNumber coalesce_trigger, - size_t coalesce_window) { - AllocatorHeader::Configure(alloc_id, - AllocatorType::kScalablePageAllocator, - custom_header_size); - HSHM_MAKE_AR0(free_lists_, alloc) - total_alloc_ = 0; - coalesce_trigger_ = (coalesce_trigger * buffer_size).as_int(); - coalesce_window_ = coalesce_window; - coalesce_lock_.Init(); - } -}; - -class ScalablePageAllocator : public Allocator { - private: - ScalablePageAllocatorHeader *header_; - vector>> *free_lists_; - StackAllocator alloc_; - /** The power-of-two exponent of the minimum size that can be cached */ - static const size_t min_cached_size_exp_ = 6; - /** The minimum size that can be cached directly (64 bytes) */ - static const size_t min_cached_size_ = (1 << min_cached_size_exp_); - /** The power-of-two exponent of the minimum size that can be cached */ - static const size_t max_cached_size_exp_ = 24; - /** The maximum size that can be cached directly (16MB) */ - static const size_t max_cached_size_ = (1 << max_cached_size_exp_); - /** Cache every size between 64 (2^6) BYTES and 16MB (2^24): (19 entries) */ - static const size_t num_caches_ = 24 - 6 + 1; - /** - * The last free list stores sizes larger than 16KB or sizes which are - * not exactly powers-of-two. - * */ - static const size_t num_free_lists_ = num_caches_ + 1; - - public: - /** - * Allocator constructor - * */ - ScalablePageAllocator() - : header_(nullptr) {} - - /** - * Get the ID of this allocator from shared memory - * */ - allocator_id_t &GetId() override { - return header_->allocator_id_; - } - - /** - * Initialize the allocator in shared memory - * */ - void shm_init(allocator_id_t id, - size_t custom_header_size, - char *buffer, - size_t buffer_size, - RealNumber coalesce_trigger = RealNumber(1, 5), - size_t coalesce_window = MEGABYTES(1)); - - /** - * Attach an existing allocator from shared memory - * */ - void shm_deserialize(char *buffer, - size_t buffer_size) override; - - /** - * Allocate a memory of \a size size. The page allocator cannot allocate - * memory larger than the page size. - * */ - OffsetPointer AllocateOffset(size_t size) override; - - private: - /** Check if a cached page on this core can be re-used */ - MpPage* CheckLocalCaches(size_t size_mp, uint32_t cpu); - - /** - * Find the first fit of an element in a free list - * */ - MpPage* FindFirstFit(size_t size_mp, - FreeListStats &stats, - iqueue &free_list); - - /** - * Divide a page into smaller pages and cache them - * */ - void DividePage(FreeListStats &stats, - iqueue &free_list, - MpPage *fit_page, - MpPage *&rem_page, - size_t size_mp, - size_t max_divide); - - - public: - /** - * Allocate a memory of \a size size, which is aligned to \a - * alignment. - * */ - OffsetPointer AlignedAllocateOffset(size_t size, size_t alignment) override; - - /** - * Reallocate \a p pointer to \a new_size new size. - * - * @return whether or not the pointer p was changed - * */ - OffsetPointer ReallocateOffsetNoNullCheck( - OffsetPointer p, size_t new_size) override; - - /** - * Free \a ptr pointer. Null check is performed elsewhere. - * */ - void FreeOffsetNoNullCheck(OffsetPointer p) override; - - /** - * Get the current amount of data allocated. Can be used for leak - * checking. - * */ - size_t GetCurrentlyAllocatedSize() override; - - private: - /** Round a number up to the nearest page size. */ - size_t RoundUp(size_t num, size_t &exp); -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_ALLOCATOR_SCALABLE_PAGE_ALLOCATOR_H diff --git a/hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h b/hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h deleted file mode 100644 index 313a71c6c..000000000 --- a/hermes_shm/include/hermes_shm/memory/allocator/stack_allocator.h +++ /dev/null @@ -1,107 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_MEMORY_ALLOCATOR_STACK_ALLOCATOR_H_ -#define HERMES_MEMORY_ALLOCATOR_STACK_ALLOCATOR_H_ - -#include "allocator.h" -#include "heap.h" -#include "hermes_shm/thread/lock.h" - -namespace hshm::ipc { - -struct StackAllocatorHeader : public AllocatorHeader { - HeapAllocator heap_; - std::atomic total_alloc_; - - StackAllocatorHeader() = default; - - void Configure(allocator_id_t alloc_id, - size_t custom_header_size, - size_t region_off, - size_t region_size) { - AllocatorHeader::Configure(alloc_id, AllocatorType::kStackAllocator, - custom_header_size); - heap_.shm_init(region_off, region_size); - total_alloc_ = 0; - } -}; - -class StackAllocator : public Allocator { - public: - StackAllocatorHeader *header_; - HeapAllocator *heap_; - - public: - /** - * Allocator constructor - * */ - StackAllocator() - : header_(nullptr) {} - - /** - * Get the ID of this allocator from shared memory - * */ - allocator_id_t &GetId() override { - return header_->allocator_id_; - } - - /** - * Initialize the allocator in shared memory - * */ - void shm_init(allocator_id_t id, - size_t custom_header_size, - char *buffer, - size_t buffer_size); - - /** - * Attach an existing allocator from shared memory - * */ - void shm_deserialize(char *buffer, - size_t buffer_size) override; - - /** - * Allocate a memory of \a size size. The page allocator cannot allocate - * memory larger than the page size. - * */ - OffsetPointer AllocateOffset(size_t size) override; - - /** - * Allocate a memory of \a size size, which is aligned to \a - * alignment. - * */ - OffsetPointer AlignedAllocateOffset(size_t size, size_t alignment) override; - - /** - * Reallocate \a p pointer to \a new_size new size. - * - * @return whether or not the pointer p was changed - * */ - OffsetPointer ReallocateOffsetNoNullCheck( - OffsetPointer p, size_t new_size) override; - - /** - * Free \a ptr pointer. Null check is performed elsewhere. - * */ - void FreeOffsetNoNullCheck(OffsetPointer p) override; - - /** - * Get the current amount of data allocated. Can be used for leak - * checking. - * */ - size_t GetCurrentlyAllocatedSize() override; -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_ALLOCATOR_STACK_ALLOCATOR_H_ diff --git a/hermes_shm/include/hermes_shm/memory/backend/array_backend.h b/hermes_shm/include/hermes_shm/memory/backend/array_backend.h deleted file mode 100644 index 6d7ae3d52..000000000 --- a/hermes_shm/include/hermes_shm/memory/backend/array_backend.h +++ /dev/null @@ -1,65 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_MEMORY_BACKEND_ARRAY_BACKEND_H_ -#define HERMES_INCLUDE_HERMES_MEMORY_BACKEND_ARRAY_BACKEND_H_ - -#include "memory_backend.h" -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace hshm::ipc { - -class ArrayBackend : public MemoryBackend { - public: - ArrayBackend() = default; - - ~ArrayBackend() override {} - - bool shm_init(size_t size, char *region) { - if (size < sizeof(MemoryBackendHeader)) { - throw SHMEM_CREATE_FAILED.format(); - } - SetInitialized(); - Own(); - header_ = reinterpret_cast(region); - header_->data_size_ = size - sizeof(MemoryBackendHeader); - data_size_ = header_->data_size_; - data_ = region + sizeof(MemoryBackendHeader); - return true; - } - - bool shm_deserialize(std::string url) override { - (void) url; - throw SHMEM_NOT_SUPPORTED.format(); - } - - void shm_detach() override {} - - void shm_destroy() override {} -}; - -} // namespace hshm::ipc - -#endif // HERMES_INCLUDE_HERMES_MEMORY_BACKEND_ARRAY_BACKEND_H_ diff --git a/hermes_shm/include/hermes_shm/memory/backend/memory_backend.h b/hermes_shm/include/hermes_shm/memory/backend/memory_backend.h deleted file mode 100644 index 01039fbdb..000000000 --- a/hermes_shm/include/hermes_shm/memory/backend/memory_backend.h +++ /dev/null @@ -1,90 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MEMORY_H -#define HERMES_MEMORY_H - -#include -#include -#include -#include -#include "hermes_shm/constants/macros.h" -#include - -namespace hshm::ipc { - -struct MemoryBackendHeader { - size_t data_size_; -}; - -enum class MemoryBackendType { - kPosixShmMmap, - kNullBackend, - kArrayBackend, - kPosixMmap, -}; - -#define MEMORY_BACKEND_INITIALIZED 0x1 -#define MEMORY_BACKEND_OWNED 0x2 - -class MemoryBackend { - public: - MemoryBackendHeader *header_; - char *data_; - size_t data_size_; - bitfield32_t flags_; - - public: - MemoryBackend() = default; - - virtual ~MemoryBackend() = default; - - /** Mark data as valid */ - void SetInitialized() { - flags_.SetBits(MEMORY_BACKEND_INITIALIZED); - } - - /** Check if data is valid */ - bool IsInitialized() { - return flags_.Any(MEMORY_BACKEND_INITIALIZED); - } - - /** Mark data as invalid */ - void UnsetInitialized() { - flags_.UnsetBits(MEMORY_BACKEND_INITIALIZED); - } - - /** This is the process which destroys the backend */ - void Own() { - flags_.SetBits(MEMORY_BACKEND_OWNED); - } - - /** This is owned */ - bool IsOwned() { - return flags_.Any(MEMORY_BACKEND_OWNED); - } - - /** This is not the process which destroys the backend */ - void Disown() { - flags_.UnsetBits(MEMORY_BACKEND_OWNED); - } - - /// Each allocator must define its own shm_init. - // virtual bool shm_init(size_t size, ...) = 0; - virtual bool shm_deserialize(std::string url) = 0; - virtual void shm_detach() = 0; - virtual void shm_destroy() = 0; -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_H diff --git a/hermes_shm/include/hermes_shm/memory/backend/memory_backend_factory.h b/hermes_shm/include/hermes_shm/memory/backend/memory_backend_factory.h deleted file mode 100644 index cb0f35ff1..000000000 --- a/hermes_shm/include/hermes_shm/memory/backend/memory_backend_factory.h +++ /dev/null @@ -1,104 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_MEMORY_BACKEND_MEMORY_BACKEND_FACTORY_H_ -#define HERMES_MEMORY_BACKEND_MEMORY_BACKEND_FACTORY_H_ - -#include "memory_backend.h" -#include "posix_mmap.h" -#include "posix_shm_mmap.h" -#include "null_backend.h" -#include "array_backend.h" - -namespace hshm::ipc { - -class MemoryBackendFactory { - public: - /** Initialize a new backend */ - template - static std::unique_ptr shm_init( - size_t size, const std::string &url, Args ...args) { - if constexpr(std::is_same_v) { - // PosixShmMmap - auto backend = std::make_unique(); - backend->shm_init(size, url, std::forward(args)...); - return backend; - } else if constexpr(std::is_same_v) { - // PosixMmap - auto backend = std::make_unique(); - backend->shm_init(size, std::forward(args)...); - return backend; - } else if constexpr(std::is_same_v) { - // NullBackend - auto backend = std::make_unique(); - backend->shm_init(size, url, std::forward(args)...); - return backend; - } else if constexpr(std::is_same_v) { - // ArrayBackend - auto backend = std::make_unique(); - backend->shm_init(size, url, std::forward(args)...); - return backend; - } else { - throw MEMORY_BACKEND_NOT_FOUND.format(); - } - } - - /** Deserialize an existing backend */ - static std::unique_ptr shm_deserialize( - MemoryBackendType type, const std::string &url) { - switch (type) { - // PosixShmMmap - case MemoryBackendType::kPosixShmMmap: { - auto backend = std::make_unique(); - if (!backend->shm_deserialize(url)) { - throw MEMORY_BACKEND_NOT_FOUND.format(); - } - return backend; - } - - // PosixMmap - case MemoryBackendType::kPosixMmap: { - auto backend = std::make_unique(); - if (!backend->shm_deserialize(url)) { - throw MEMORY_BACKEND_NOT_FOUND.format(); - } - return backend; - } - - // NullBackend - case MemoryBackendType::kNullBackend: { - auto backend = std::make_unique(); - if (!backend->shm_deserialize(url)) { - throw MEMORY_BACKEND_NOT_FOUND.format(); - } - return backend; - } - - // ArrayBackend - case MemoryBackendType::kArrayBackend: { - auto backend = std::make_unique(); - if (!backend->shm_deserialize(url)) { - throw MEMORY_BACKEND_NOT_FOUND.format(); - } - return backend; - } - - // Default - default: return nullptr; - } - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_BACKEND_MEMORY_BACKEND_FACTORY_H_ diff --git a/hermes_shm/include/hermes_shm/memory/backend/null_backend.h b/hermes_shm/include/hermes_shm/memory/backend/null_backend.h deleted file mode 100644 index f58b76999..000000000 --- a/hermes_shm/include/hermes_shm/memory/backend/null_backend.h +++ /dev/null @@ -1,81 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_MEMORY_BACKEND_NULL_H_ -#define HERMES_INCLUDE_HERMES_MEMORY_BACKEND_NULL_H_ - -#include "memory_backend.h" -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace hshm::ipc { - -class NullBackend : public MemoryBackend { - private: - size_t total_size_; - - public: - NullBackend() = default; - - ~NullBackend() override {} - - bool shm_init(size_t size, const std::string &url) { - (void) url; - SetInitialized(); - Own(); - total_size_ = sizeof(MemoryBackendHeader) + size; - char *ptr = (char*)malloc(sizeof(MemoryBackendHeader)); - header_ = reinterpret_cast(ptr); - header_->data_size_ = size; - data_size_ = size; - data_ = nullptr; - return true; - } - - bool shm_deserialize(std::string url) override { - (void) url; - throw SHMEM_NOT_SUPPORTED.format(); - } - - void shm_detach() override { - _Detach(); - } - - void shm_destroy() override { - _Destroy(); - } - - protected: - void _Detach() { - free(header_); - } - - void _Destroy() { - free(header_); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_INCLUDE_HERMES_MEMORY_BACKEND_NULL_H_ diff --git a/hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h b/hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h deleted file mode 100644 index 8b8ba62b5..000000000 --- a/hermes_shm/include/hermes_shm/memory/backend/posix_mmap.h +++ /dev/null @@ -1,112 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_MEMORY_BACKEND_POSIX_MMAP_H -#define HERMES_INCLUDE_MEMORY_BACKEND_POSIX_MMAP_H - -#include "memory_backend.h" -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace hshm::ipc { - -class PosixMmap : public MemoryBackend { - private: - size_t total_size_; - - public: - /** Constructor */ - PosixMmap() = default; - - /** Destructor */ - ~PosixMmap() override { - if (IsOwned()) { - _Destroy(); - } else { - _Detach(); - } - } - - /** Initialize backend */ - bool shm_init(size_t size) { - SetInitialized(); - Own(); - total_size_ = sizeof(MemoryBackendHeader) + size; - char *ptr = _Map(total_size_); - header_ = reinterpret_cast(ptr); - header_->data_size_ = size; - data_size_ = size; - data_ = reinterpret_cast(header_ + 1); - return true; - } - - /** Deserialize the backend */ - bool shm_deserialize(std::string url) override { - (void) url; - throw SHMEM_NOT_SUPPORTED.format(); - } - - /** Detach the mapped memory */ - void shm_detach() override { - _Detach(); - } - - /** Destroy the mapped memory */ - void shm_destroy() override { - _Destroy(); - } - - protected: - /** Map shared memory */ - template - T* _Map(size_t size) { - T *ptr = reinterpret_cast( - mmap64(nullptr, MemoryAlignment::AlignToPageSize(size), - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - if (ptr == MAP_FAILED) { - perror("map failed"); - throw SHMEM_CREATE_FAILED.format(); - } - return ptr; - } - - /** Unmap shared memory */ - void _Detach() { - if (!IsInitialized()) { return; } - munmap(reinterpret_cast(header_), total_size_); - UnsetInitialized(); - } - - /** Destroy shared memory */ - void _Destroy() { - if (!IsInitialized()) { return; } - _Detach(); - UnsetInitialized(); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_INCLUDE_MEMORY_BACKEND_POSIX_MMAP_H diff --git a/hermes_shm/include/hermes_shm/memory/backend/posix_shm_mmap.h b/hermes_shm/include/hermes_shm/memory/backend/posix_shm_mmap.h deleted file mode 100644 index acdebb619..000000000 --- a/hermes_shm/include/hermes_shm/memory/backend/posix_shm_mmap.h +++ /dev/null @@ -1,136 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_MEMORY_BACKEND_POSIX_SHM_MMAP_H -#define HERMES_INCLUDE_MEMORY_BACKEND_POSIX_SHM_MMAP_H - -#include "memory_backend.h" -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace hshm::ipc { - -class PosixShmMmap : public MemoryBackend { - private: - std::string url_; - int fd_; - - public: - /** Constructor */ - PosixShmMmap() : fd_(-1) {} - - /** Destructor */ - ~PosixShmMmap() override { - if (IsOwned()) { - _Destroy(); - } else { - _Detach(); - } - } - - /** Initialize backend */ - bool shm_init(size_t size, std::string url) { - SetInitialized(); - Own(); - url_ = std::move(url); - shm_unlink(url_.c_str()); - fd_ = shm_open(url_.c_str(), O_CREAT | O_RDWR, 0666); - if (fd_ < 0) { - return false; - } - _Reserve(size + HERMES_SYSTEM_INFO->page_size_); - header_ = _Map(HERMES_SYSTEM_INFO->page_size_, 0); - header_->data_size_ = size; - data_size_ = size; - data_ = _Map(size, HERMES_SYSTEM_INFO->page_size_); - return true; - } - - /** Deserialize the backend */ - bool shm_deserialize(std::string url) override { - SetInitialized(); - Disown(); - url_ = std::move(url); - fd_ = shm_open(url_.c_str(), O_RDWR, 0666); - if (fd_ < 0) { - return false; - } - header_ = _Map(HERMES_SYSTEM_INFO->page_size_, 0); - data_size_ = header_->data_size_; - data_ = _Map(data_size_, HERMES_SYSTEM_INFO->page_size_); - return true; - } - - /** Detach the mapped memory */ - void shm_detach() override { - _Detach(); - } - - /** Destroy the mapped memory */ - void shm_destroy() override { - _Destroy(); - } - - protected: - /** Reserve shared memory */ - void _Reserve(size_t size) { - int ret = ftruncate64(fd_, static_cast(size)); - if (ret < 0) { - throw SHMEM_RESERVE_FAILED.format(); - } - } - - /** Map shared memory */ - template - T* _Map(size_t size, off64_t off) { - T *ptr = reinterpret_cast( - mmap64(nullptr, size, PROT_READ | PROT_WRITE, - MAP_SHARED, fd_, off)); - if (ptr == MAP_FAILED) { - throw SHMEM_CREATE_FAILED.format(); - } - return ptr; - } - - /** Unmap shared memory */ - void _Detach() { - if (!IsInitialized()) { return; } - munmap(data_, data_size_); - munmap(header_, HERMES_SYSTEM_INFO->page_size_); - close(fd_); - UnsetInitialized(); - } - - /** Destroy shared memory */ - void _Destroy() { - if (!IsInitialized()) { return; } - _Detach(); - shm_unlink(url_.c_str()); - UnsetInitialized(); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_INCLUDE_MEMORY_BACKEND_POSIX_SHM_MMAP_H diff --git a/hermes_shm/include/hermes_shm/memory/memory.h b/hermes_shm/include/hermes_shm/memory/memory.h deleted file mode 100644 index 6e3ad861b..000000000 --- a/hermes_shm/include/hermes_shm/memory/memory.h +++ /dev/null @@ -1,404 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_MEMORY_MEMORY_H_ -#define HERMES_MEMORY_MEMORY_H_ - -#include -#include -#include -#include -#include -#include - -namespace hshm::ipc { - -/** - * The identifier for an allocator - * */ -union allocator_id_t { - struct { - uint32_t major_; // Typically some sort of process id - uint32_t minor_; // Typically a process-local id - } bits_; - uint64_t int_; - - HSHM_ALWAYS_INLINE allocator_id_t() = default; - - /** - * Constructor which sets major & minor - * */ - HSHM_ALWAYS_INLINE explicit allocator_id_t(uint32_t major, uint32_t minor) { - bits_.major_ = major; - bits_.minor_ = minor; - } - - /** - * Set this allocator to null - * */ - HSHM_ALWAYS_INLINE void SetNull() { - int_ = 0; - } - - /** - * Check if this is the null allocator - * */ - HSHM_ALWAYS_INLINE bool IsNull() const { return int_ == 0; } - - /** Equality check */ - HSHM_ALWAYS_INLINE bool operator==(const allocator_id_t &other) const { - return other.int_ == int_; - } - - /** Inequality check */ - HSHM_ALWAYS_INLINE bool operator!=(const allocator_id_t &other) const { - return other.int_ != int_; - } - - /** Get the null allocator */ - HSHM_ALWAYS_INLINE static allocator_id_t GetNull() { - static allocator_id_t alloc(0, 0); - return alloc; - } - - /** To index */ - HSHM_ALWAYS_INLINE uint32_t ToIndex() { - return bits_.major_ * 4 + bits_.minor_; - } -}; - -typedef uint32_t slot_id_t; // Uniquely ids a MemoryBackend slot - -/** - * Stores an offset into a memory region. Assumes the developer knows - * which allocator the pointer comes from. - * */ -template -struct OffsetPointerBase { - typedef typename std::conditional, nonatomic>::type atomic_t; - atomic_t off_; /**< Offset within the allocator's slot */ - - /** Default constructor */ - HSHM_ALWAYS_INLINE OffsetPointerBase() = default; - - /** Full constructor */ - HSHM_ALWAYS_INLINE explicit OffsetPointerBase(size_t off) : off_(off) {} - - /** Full constructor */ - HSHM_ALWAYS_INLINE explicit OffsetPointerBase(atomic_t off) - : off_(off.load()) {} - - /** Pointer constructor */ - HSHM_ALWAYS_INLINE explicit OffsetPointerBase(allocator_id_t alloc_id, - size_t off) - : off_(off) { - (void) alloc_id; - } - - /** Copy constructor */ - HSHM_ALWAYS_INLINE OffsetPointerBase(const OffsetPointerBase &other) - : off_(other.off_.load()) {} - - /** Other copy constructor */ - HSHM_ALWAYS_INLINE OffsetPointerBase(const OffsetPointerBase &other) - : off_(other.off_.load()) {} - - /** Move constructor */ - HSHM_ALWAYS_INLINE OffsetPointerBase(OffsetPointerBase &&other) noexcept - : off_(other.off_.load()) { - other.SetNull(); - } - - /** Get the offset pointer */ - HSHM_ALWAYS_INLINE OffsetPointerBase ToOffsetPointer() { - return OffsetPointerBase(off_.load()); - } - - /** Set to null */ - HSHM_ALWAYS_INLINE void SetNull() { - off_ = (size_t)-1; - } - - /** Check if null */ - HSHM_ALWAYS_INLINE bool IsNull() const { - return off_.load() == (size_t)-1; - } - - /** Get the null pointer */ - HSHM_ALWAYS_INLINE static OffsetPointerBase GetNull() { - static const OffsetPointerBase p(-1); - return p; - } - - /** Atomic load wrapper */ - HSHM_ALWAYS_INLINE size_t load( - std::memory_order order = std::memory_order_seq_cst) const { - return off_.load(order); - } - - /** Atomic exchange wrapper */ - HSHM_ALWAYS_INLINE void exchange( - size_t count, std::memory_order order = std::memory_order_seq_cst) { - off_.exchange(count, order); - } - - /** Atomic compare exchange weak wrapper */ - HSHM_ALWAYS_INLINE bool compare_exchange_weak( - size_t& expected, size_t desired, - std::memory_order order = std::memory_order_seq_cst) { - return off_.compare_exchange_weak(expected, desired, order); - } - - /** Atomic compare exchange strong wrapper */ - HSHM_ALWAYS_INLINE bool compare_exchange_strong( - size_t& expected, size_t desired, - std::memory_order order = std::memory_order_seq_cst) { - return off_.compare_exchange_weak(expected, desired, order); - } - - /** Atomic add operator */ - HSHM_ALWAYS_INLINE OffsetPointerBase operator+(size_t count) const { - return OffsetPointerBase(off_ + count); - } - - /** Atomic subtract operator */ - HSHM_ALWAYS_INLINE OffsetPointerBase operator-(size_t count) const { - return OffsetPointerBase(off_ - count); - } - - /** Atomic add assign operator */ - HSHM_ALWAYS_INLINE OffsetPointerBase& operator+=(size_t count) { - off_ += count; - return *this; - } - - /** Atomic subtract assign operator */ - HSHM_ALWAYS_INLINE OffsetPointerBase& operator-=(size_t count) { - off_ -= count; - return *this; - } - - /** Atomic assign operator */ - HSHM_ALWAYS_INLINE OffsetPointerBase& operator=(size_t count) { - off_ = count; - return *this; - } - - /** Atomic copy assign operator */ - HSHM_ALWAYS_INLINE OffsetPointerBase& operator=( - const OffsetPointerBase &count) { - off_ = count.load(); - return *this; - } - - /** Equality check */ - HSHM_ALWAYS_INLINE bool operator==(const OffsetPointerBase &other) const { - return off_ == other.off_; - } - - /** Inequality check */ - HSHM_ALWAYS_INLINE bool operator!=(const OffsetPointerBase &other) const { - return off_ != other.off_; - } -}; - -/** Non-atomic offset */ -typedef OffsetPointerBase OffsetPointer; - -/** Atomic offset */ -typedef OffsetPointerBase AtomicOffsetPointer; - -/** Typed offset pointer */ -template -using TypedOffsetPointer = OffsetPointer; - -/** Typed atomic pointer */ -template -using TypedAtomicOffsetPointer = AtomicOffsetPointer; - -/** - * A process-independent pointer, which stores both the allocator's - * information and the offset within the allocator's region - * */ -template -struct PointerBase { - allocator_id_t allocator_id_; /// Allocator the pointer comes from - OffsetPointerBase off_; /// Offset within the allocator's slot - - /** Default constructor */ - PointerBase() = default; - - /** Full constructor */ - HSHM_ALWAYS_INLINE explicit PointerBase(allocator_id_t id, size_t off) - : allocator_id_(id), off_(off) {} - - /** Full constructor using offset pointer */ - HSHM_ALWAYS_INLINE explicit PointerBase(allocator_id_t id, OffsetPointer off) - : allocator_id_(id), off_(off) {} - - /** Copy constructor */ - HSHM_ALWAYS_INLINE PointerBase(const PointerBase &other) - : allocator_id_(other.allocator_id_), off_(other.off_) {} - - /** Other copy constructor */ - HSHM_ALWAYS_INLINE PointerBase(const PointerBase &other) - : allocator_id_(other.allocator_id_), off_(other.off_.load()) {} - - /** Move constructor */ - HSHM_ALWAYS_INLINE PointerBase(PointerBase &&other) noexcept - : allocator_id_(other.allocator_id_), off_(other.off_) { - other.SetNull(); - } - - /** Get the offset pointer */ - HSHM_ALWAYS_INLINE OffsetPointerBase ToOffsetPointer() const { - return OffsetPointerBase(off_.load()); - } - - /** Set to null */ - HSHM_ALWAYS_INLINE void SetNull() { - allocator_id_.SetNull(); - } - - /** Check if null */ - HSHM_ALWAYS_INLINE bool IsNull() const { - return allocator_id_.IsNull(); - } - - /** Get the null pointer */ - HSHM_ALWAYS_INLINE static PointerBase GetNull() { - static const PointerBase p(allocator_id_t::GetNull(), - OffsetPointer::GetNull()); - return p; - } - - /** Copy assignment operator */ - HSHM_ALWAYS_INLINE PointerBase& operator=(const PointerBase &other) { - if (this != &other) { - allocator_id_ = other.allocator_id_; - off_ = other.off_; - } - return *this; - } - - /** Move assignment operator */ - HSHM_ALWAYS_INLINE PointerBase& operator=(PointerBase &&other) { - if (this != &other) { - allocator_id_ = other.allocator_id_; - off_.exchange(other.off_.load()); - other.SetNull(); - } - return *this; - } - - /** Addition operator */ - HSHM_ALWAYS_INLINE PointerBase operator+(size_t size) const { - PointerBase p; - p.allocator_id_ = allocator_id_; - p.off_ = off_ + size; - return p; - } - - /** Subtraction operator */ - HSHM_ALWAYS_INLINE PointerBase operator-(size_t size) const { - PointerBase p; - p.allocator_id_ = allocator_id_; - p.off_ = off_ - size; - return p; - } - - /** Addition assignment operator */ - HSHM_ALWAYS_INLINE PointerBase& operator+=(size_t size) { - off_ += size; - return *this; - } - - /** Subtraction assignment operator */ - HSHM_ALWAYS_INLINE PointerBase& operator-=(size_t size) { - off_ -= size; - return *this; - } - - /** Equality check */ - HSHM_ALWAYS_INLINE bool operator==(const PointerBase &other) const { - return (other.allocator_id_ == allocator_id_ && other.off_ == off_); - } - - /** Inequality check */ - HSHM_ALWAYS_INLINE bool operator!=(const PointerBase &other) const { - return (other.allocator_id_ != allocator_id_ || other.off_ != off_); - } -}; - -/** Non-atomic pointer */ -typedef PointerBase Pointer; - -/** Atomic pointer */ -typedef PointerBase AtomicPointer; - -/** Typed pointer */ -template -using TypedPointer = Pointer; - -/** Typed atomic pointer */ -template -using TypedAtomicPointer = AtomicPointer; - -class MemoryAlignment { - public: - /** - * Round up to the nearest multiple of the alignment - * @param alignment the alignment value (e.g., 4096) - * @param size the size to make a multiple of alignment (e.g., 4097) - * @return the new size (e.g., 8192) - * */ - HSHM_ALWAYS_INLINE static size_t AlignTo(size_t alignment, - size_t size) { - auto page_size = HERMES_SYSTEM_INFO->page_size_; - size_t new_size = size; - size_t page_off = size % alignment; - if (page_off) { - new_size = size + page_size - page_off; - } - return new_size; - } - - /** - * Round up to the nearest multiple of page size - * @param size the size to align to the PAGE_SIZE - * */ - HSHM_ALWAYS_INLINE static size_t AlignToPageSize(size_t size) { - auto page_size = HERMES_SYSTEM_INFO->page_size_; - size_t new_size = AlignTo(page_size, size); - return new_size; - } -}; - -} // namespace hshm::ipc - -namespace std { - -/** Allocator ID hash */ -template <> -struct hash { - HSHM_ALWAYS_INLINE std::size_t operator()( - const hshm::ipc::allocator_id_t &key) const { - return std::hash{}(key.int_); - } -}; - -} // namespace std - - -#endif // HERMES_MEMORY_MEMORY_H_ diff --git a/hermes_shm/include/hermes_shm/memory/memory_manager.h b/hermes_shm/include/hermes_shm/memory/memory_manager.h deleted file mode 100644 index eb412358c..000000000 --- a/hermes_shm/include/hermes_shm/memory/memory_manager.h +++ /dev/null @@ -1,209 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MEMORY_MEMORY_MANAGER_H_ -#define HERMES_MEMORY_MEMORY_MANAGER_H_ - -#include "hermes_shm/memory/backend/memory_backend_factory.h" -#include "hermes_shm/memory/allocator/allocator_factory.h" -#include "hermes_shm/memory/memory_registry.h" -#include "hermes_shm/constants/macros.h" -#include "hermes_shm/util/logging.h" -#include - -namespace hipc = hshm::ipc; - -namespace hshm::ipc { - -class MemoryManager { - public: - /** Default constructor. */ - MemoryManager() = default; - - /** Default backend size */ - static size_t GetDefaultBackendSize() { - return HERMES_SYSTEM_INFO->ram_size_; - } - - /** - * Create a memory backend. Memory backends are divided into slots. - * Each slot corresponds directly with a single allocator. - * There can be multiple slots per-backend, enabling multiple allocation - * policies over a single memory region. - * */ - template - MemoryBackend* CreateBackend(size_t size, - const std::string &url, - Args&& ...args) { - auto backend_u = MemoryBackendFactory::shm_init( - size, url, std::forward(args)...); - auto backend = HERMES_MEMORY_REGISTRY_REF.RegisterBackend(url, backend_u); - backend->Own(); - return backend; - } - - /** - * Attaches to an existing memory backend located at \a url url. - * */ - MemoryBackend* AttachBackend(MemoryBackendType type, - const std::string &url) { - auto backend_u = MemoryBackendFactory::shm_deserialize(type, url); - auto backend = HERMES_MEMORY_REGISTRY_REF.RegisterBackend(url, backend_u); - ScanBackends(); - backend->Disown(); - return backend; - } - - /** - * Returns a pointer to a backend that has already been attached. - * */ - MemoryBackend* GetBackend(const std::string &url) { - return HERMES_MEMORY_REGISTRY_REF.GetBackend(url); - } - - /** - * Unregister backend - * */ - void UnregisterBackend(const std::string &url) { - HERMES_MEMORY_REGISTRY_REF.UnregisterBackend(url); - } - - /** - * Destroy backend - * */ - void DestroyBackend(const std::string &url) { - auto backend = GetBackend(url); - backend->Own(); - UnregisterBackend(url); - } - - /** - * Scans all attached backends for new memory allocators. - * */ - void ScanBackends(); - - /** - * Registers an allocator. Used internally by ScanBackends, but may - * also be used externally. - * */ - void RegisterAllocator(std::unique_ptr &alloc) { - HERMES_MEMORY_REGISTRY_REF.RegisterAllocator(alloc); - } - - /** - * Registers an allocator. Used internally by ScanBackends, but may - * also be used externally. - * */ - void RegisterAllocator(Allocator *alloc) { - HERMES_MEMORY_REGISTRY_REF.RegisterAllocator(alloc); - } - - /** - * Destroys an allocator - * */ - void UnregisterAllocator(allocator_id_t alloc_id) { - HERMES_MEMORY_REGISTRY_REF.UnregisterAllocator(alloc_id); - } - - /** - * Create and register a memory allocator for a particular backend. - * */ - template - Allocator* CreateAllocator(const std::string &url, - allocator_id_t alloc_id, - size_t custom_header_size, - Args&& ...args) { - auto backend = GetBackend(url); - if (alloc_id.IsNull()) { - HELOG(kFatal, "Allocator cannot be created with a NIL ID"); - } - auto alloc = AllocatorFactory::shm_init( - alloc_id, custom_header_size, backend, std::forward(args)...); - RegisterAllocator(alloc); - return GetAllocator(alloc_id); - } - - /** - * Locates an allocator of a particular id - * */ - HSHM_ALWAYS_INLINE Allocator* GetAllocator(allocator_id_t alloc_id) { - return HERMES_MEMORY_REGISTRY_REF.GetAllocator(alloc_id); - } - - /** - * Gets the allocator used for initializing other allocators. - * */ - HSHM_ALWAYS_INLINE Allocator* GetRootAllocator() { - return HERMES_MEMORY_REGISTRY_REF.GetRootAllocator(); - } - - /** - * Gets the allocator used by default when no allocator is - * used to construct an object. - * */ - HSHM_ALWAYS_INLINE Allocator* GetDefaultAllocator() { - return HERMES_MEMORY_REGISTRY_REF.GetDefaultAllocator(); - } - - /** - * Sets the allocator used by default when no allocator is - * used to construct an object. - * */ - HSHM_ALWAYS_INLINE void SetDefaultAllocator(Allocator *alloc) { - HERMES_MEMORY_REGISTRY_REF.SetDefaultAllocator(alloc); - } - - /** - * Convert a process-independent pointer into a process-specific pointer. - * */ - template - HSHM_ALWAYS_INLINE T* Convert(const POINTER_T &p) { - if (p.IsNull()) { - return nullptr; - } - return GetAllocator(p.allocator_id_)->template - Convert(p); - } - - /** - * Convert a process-specific pointer into a process-independent pointer - * - * @param allocator_id the allocator the pointer belongs to - * @param ptr the pointer to convert - * */ - template - HSHM_ALWAYS_INLINE POINTER_T Convert(allocator_id_t allocator_id, T *ptr) { - return GetAllocator(allocator_id)->template - Convert(ptr); - } - - /** - * Convert a process-specific pointer into a process-independent pointer when - * the allocator is unkown. - * - * @param ptr the pointer to convert - * */ - template - HSHM_ALWAYS_INLINE POINTER_T Convert(T *ptr) { - for (auto &alloc : HERMES_MEMORY_REGISTRY_REF.allocators_) { - if (alloc && alloc->ContainsPtr(ptr)) { - return alloc->template - Convert(ptr); - } - } - return Pointer::GetNull(); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_MEMORY_MEMORY_MANAGER_H_ diff --git a/hermes_shm/include/hermes_shm/memory/memory_registry.h b/hermes_shm/include/hermes_shm/memory/memory_registry.h deleted file mode 100644 index 9ff144cb9..000000000 --- a/hermes_shm/include/hermes_shm/memory/memory_registry.h +++ /dev/null @@ -1,144 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_MEMORY_REGISTRY_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_MEMORY_REGISTRY_H_ - -#include "hermes_shm/memory/allocator/allocator.h" -#include "backend/memory_backend.h" -#include "hermes_shm/memory/allocator/stack_allocator.h" -#include "hermes_shm/memory/backend/posix_mmap.h" -#include "hermes_shm/util/errors.h" - -namespace hipc = hshm::ipc; - -namespace hshm::ipc { - -#define MAX_ALLOCATORS 64 - -class MemoryRegistry { - public: - allocator_id_t root_allocator_id_; - PosixMmap root_backend_; - StackAllocator root_allocator_; - std::unordered_map> backends_; - std::unique_ptr allocators_made_[MAX_ALLOCATORS]; - Allocator *allocators_[MAX_ALLOCATORS]; - Allocator *default_allocator_; - - public: - /** - * Constructor. Create the root allocator and backend, which is used - * until the user specifies a new default. The root allocator stores - * only private memory. - * */ - MemoryRegistry(); - - /** - * Register a unique memory backend. Throws an exception if the backend - * already exists. This is because unregistering a backend can cause - * ramifications across allocators. - * - * @param url the backend's unique identifier - * @param backend the backend to register - * */ - HSHM_ALWAYS_INLINE MemoryBackend* RegisterBackend( - const std::string &url, - std::unique_ptr &backend) { - auto ptr = backend.get(); - if (GetBackend(url)) { - throw MEMORY_BACKEND_REPEATED.format(); - } - backends_.emplace(url, std::move(backend)); - return ptr; - } - - /** Unregister memory backend */ - HSHM_ALWAYS_INLINE void UnregisterBackend(const std::string &url) { - backends_.erase(url); - } - - /** - * Returns a pointer to a backend that has already been attached. - * */ - HSHM_ALWAYS_INLINE MemoryBackend* GetBackend(const std::string &url) { - auto iter = backends_.find(url); - if (iter == backends_.end()) { - return nullptr; - } - return (*iter).second.get(); - } - - /** Registers an allocator. */ - HSHM_ALWAYS_INLINE Allocator* RegisterAllocator( - std::unique_ptr &alloc) { - if (default_allocator_ == nullptr || - default_allocator_ == &root_allocator_ || - default_allocator_->GetId() == alloc->GetId()) { - default_allocator_ = alloc.get(); - } - RegisterAllocator(alloc.get()); - auto idx = alloc->GetId().ToIndex(); - auto &alloc_made = allocators_made_[idx]; - alloc_made = std::move(alloc); - return alloc_made.get(); - } - - /** Registers an allocator. */ - HSHM_ALWAYS_INLINE void RegisterAllocator(Allocator *alloc) { - auto idx = alloc->GetId().ToIndex(); - allocators_[idx] = alloc; - } - - /** Unregisters an allocator */ - HSHM_ALWAYS_INLINE void UnregisterAllocator(allocator_id_t alloc_id) { - if (alloc_id == default_allocator_->GetId()) { - default_allocator_ = &root_allocator_; - } - allocators_made_[alloc_id.ToIndex()] = nullptr; - allocators_[alloc_id.ToIndex()] = nullptr; - } - - /** - * Locates an allocator of a particular id - * */ - HSHM_ALWAYS_INLINE Allocator* GetAllocator(allocator_id_t alloc_id) { - return allocators_[alloc_id.ToIndex()]; - } - - /** - * Gets the allocator used for initializing other allocators. - * */ - HSHM_ALWAYS_INLINE Allocator* GetRootAllocator() { - return &root_allocator_; - } - - /** - * Gets the allocator used by default when no allocator is - * used to construct an object. - * */ - HSHM_ALWAYS_INLINE Allocator* GetDefaultAllocator() { - return reinterpret_cast(default_allocator_); - } - - /** - * Sets the allocator used by default when no allocator is - * used to construct an object. - * */ - HSHM_ALWAYS_INLINE void SetDefaultAllocator(Allocator *alloc) { - default_allocator_ = alloc; - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_MEMORY_MEMORY_REGISTRY_H_ diff --git a/hermes_shm/include/hermes_shm/thread/lock.h b/hermes_shm/include/hermes_shm/thread/lock.h deleted file mode 100644 index 08876f437..000000000 --- a/hermes_shm/include/hermes_shm/thread/lock.h +++ /dev/null @@ -1,21 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_THREAD_LOCK_H_ -#define HERMES_THREAD_LOCK_H_ - -#include "lock/mutex.h" -#include "lock/rwlock.h" -#include "thread_model_manager.h" - -#endif // HERMES_THREAD_LOCK_H_ diff --git a/hermes_shm/include/hermes_shm/thread/lock/mutex.h b/hermes_shm/include/hermes_shm/thread/lock/mutex.h deleted file mode 100644 index c2a97fcdf..000000000 --- a/hermes_shm/include/hermes_shm/thread/lock/mutex.h +++ /dev/null @@ -1,114 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_THREAD_MUTEX_H_ -#define HERMES_THREAD_MUTEX_H_ - -#include -#include "hermes_shm/thread/lock.h" -#include "hermes_shm/thread/thread_model_manager.h" - -namespace hshm { - -struct Mutex { - std::atomic lock_; -#ifdef HERMES_DEBUG_LOCK - uint32_t owner_; -#endif - - /** Default constructor */ - HSHM_ALWAYS_INLINE Mutex() : lock_(0) {} - - /** Explicit initialization */ - HSHM_ALWAYS_INLINE void Init() { - lock_ = 0; - } - - /** Acquire lock */ - HSHM_ALWAYS_INLINE void Lock(uint32_t owner) { - do { - for (int i = 0; i < 1; ++i) { - if (TryLock(owner)) { return; } - } - HERMES_THREAD_MODEL->Yield(); - } while (true); - } - - /** Try to acquire the lock */ - HSHM_ALWAYS_INLINE bool TryLock(uint32_t owner) { - if (lock_.load() != 0) { - return false; - } - uint32_t tkt = lock_.fetch_add(1); - if (tkt != 0) { - lock_.fetch_sub(1); - return false; - } -#ifdef HERMES_DEBUG_LOCK - owner_ = owner; -#endif - return true; - } - - /** Unlock */ - HSHM_ALWAYS_INLINE void Unlock() { -#ifdef HERMES_DEBUG_LOCK - owner_ = 0; -#endif - lock_.fetch_sub(1); - } -}; - -struct ScopedMutex { - Mutex &lock_; - bool is_locked_; - - /** Acquire the mutex */ - HSHM_ALWAYS_INLINE explicit ScopedMutex(Mutex &lock, uint32_t owner) - : lock_(lock), is_locked_(false) { - Lock(owner); - } - - /** Release the mutex */ - HSHM_ALWAYS_INLINE ~ScopedMutex() { - Unlock(); - } - - /** Explicitly acquire the mutex */ - HSHM_ALWAYS_INLINE void Lock(uint32_t owner) { - if (!is_locked_) { - lock_.Lock(owner); - is_locked_ = true; - } - } - - /** Explicitly try to lock the mutex */ - HSHM_ALWAYS_INLINE bool TryLock(uint32_t owner) { - if (!is_locked_) { - is_locked_ = lock_.TryLock(owner); - } - return is_locked_; - } - - /** Explicitly unlock the mutex */ - HSHM_ALWAYS_INLINE void Unlock() { - if (is_locked_) { - lock_.Unlock(); - is_locked_ = false; - } - } -}; - -} // namespace hshm - -#endif // HERMES_THREAD_MUTEX_H_ diff --git a/hermes_shm/include/hermes_shm/thread/lock/rwlock.h b/hermes_shm/include/hermes_shm/thread/lock/rwlock.h deleted file mode 100644 index f01c18b0c..000000000 --- a/hermes_shm/include/hermes_shm/thread/lock/rwlock.h +++ /dev/null @@ -1,230 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_THREAD_RWLOCK_H_ -#define HERMES_THREAD_RWLOCK_H_ - -#include -#include -#include "hermes_shm/thread/lock.h" -#include "hermes_shm/thread/thread_model_manager.h" - -namespace hshm { - -enum class RwLockMode { - kNone, - kWrite, - kRead, -}; - -/** A reader-writer lock implementation */ -struct RwLock { - std::atomic readers_; - std::atomic writers_; - std::atomic ticket_; - std::atomic mode_; - std::atomic cur_writer_; -#ifdef HERMES_DEBUG_LOCK - uint32_t owner_; -#endif - - /** Default constructor */ - RwLock() - : readers_(0), - writers_(0), - ticket_(0), - mode_(RwLockMode::kNone), - cur_writer_(0) {} - - /** Explicit constructor */ - void Init() { - readers_ = 0; - writers_ = 0; - ticket_ = 0; - mode_ = RwLockMode::kNone; - cur_writer_ = 0; - } - - /** Delete copy constructor */ - RwLock(const RwLock &other) = delete; - - /** Move constructor */ - RwLock(RwLock &&other) noexcept - : readers_(other.readers_.load()), - writers_(other.writers_.load()), - ticket_(other.ticket_.load()), - mode_(other.mode_.load()), - cur_writer_(other.cur_writer_.load()) {} - - /** Move assignment operator */ - RwLock& operator=(RwLock &&other) noexcept { - if (this != &other) { - readers_ = other.readers_.load(); - writers_ = other.writers_.load(); - ticket_ = other.ticket_.load(); - mode_ = other.mode_.load(); - cur_writer_ = other.cur_writer_.load(); - } - return *this; - } - - /** Acquire read lock */ - void ReadLock(uint32_t owner) { - RwLockMode mode; - - // Increment # readers. Check if in read mode. - readers_.fetch_add(1); - - // Wait until we are in read mode - do { - UpdateMode(mode); - if (mode == RwLockMode::kRead) { - return; - } - if (mode == RwLockMode::kNone) { - bool ret = mode_.compare_exchange_weak(mode, RwLockMode::kRead); - if (ret) { -#ifdef HERMES_DEBUG_LOCK - owner_ = owner; - HILOG(kDebug, "Acquired read lock for {}", owner); -#endif - return; - } - } - HERMES_THREAD_MODEL->Yield(); - } while (true); - } - - /** Release read lock */ - void ReadUnlock() { - readers_.fetch_sub(1); - } - - /** Acquire write lock */ - void WriteLock(uint32_t owner) { - RwLockMode mode; - uint32_t cur_writer; - - // Increment # writers & get ticket - writers_.fetch_add(1); - uint64_t tkt = ticket_.fetch_add(1); - - // Wait until we are in read mode - do { - UpdateMode(mode); - if (mode == RwLockMode::kNone) { - mode_.compare_exchange_weak(mode, RwLockMode::kWrite); - mode = mode_.load(); - } - if (mode == RwLockMode::kWrite) { - cur_writer = cur_writer_.load(); - if (cur_writer == tkt) { -#ifdef HERMES_DEBUG_LOCK - owner_ = owner; - HILOG(kDebug, "Acquired write lock for {}", owner); -#endif - return; - } - } - HERMES_THREAD_MODEL->Yield(); - } while (true); - } - - /** Release write lock */ - void WriteUnlock() { - writers_.fetch_sub(1); - cur_writer_.fetch_add(1); - } - - private: - /** Update the mode of the lock */ - HSHM_ALWAYS_INLINE void UpdateMode(RwLockMode &mode) { - // When # readers is 0, there is a lag to when the mode is updated - // When # writers is 0, there is a lag to when the mode is updated - mode = mode_.load(); - if ((readers_.load() == 0 && mode == RwLockMode::kRead) || - (writers_.load() == 0 && mode == RwLockMode::kWrite)) { - mode_.compare_exchange_weak(mode, RwLockMode::kNone); - } - } -}; - -/** Acquire the read lock in a scope */ -struct ScopedRwReadLock { - RwLock &lock_; - bool is_locked_; - - /** Acquire the read lock */ - explicit ScopedRwReadLock(RwLock &lock, uint32_t owner) - : lock_(lock), is_locked_(false) { - Lock(owner); - } - - /** Release the read lock */ - ~ScopedRwReadLock() { - Unlock(); - } - - /** Explicitly acquire read lock */ - void Lock(uint32_t owner) { - if (!is_locked_) { - lock_.ReadLock(owner); - is_locked_ = true; - } - } - - /** Explicitly release read lock */ - void Unlock() { - if (is_locked_) { - lock_.ReadUnlock(); - is_locked_ = false; - } - } -}; - -/** Acquire scoped write lock */ -struct ScopedRwWriteLock { - RwLock &lock_; - bool is_locked_; - - /** Acquire the write lock */ - explicit ScopedRwWriteLock(RwLock &lock, uint32_t owner) - : lock_(lock), is_locked_(false) { - Lock(owner); - } - - /** Release the write lock */ - ~ScopedRwWriteLock() { - Unlock(); - } - - /** Explicity acquire the write lock */ - void Lock(uint32_t owner) { - if (!is_locked_) { - lock_.WriteLock(owner); - is_locked_ = true; - } - } - - /** Explicitly release the write lock */ - void Unlock() { - if (is_locked_) { - lock_.WriteUnlock(); - is_locked_ = false; - } - } -}; - -} // namespace hshm - -#endif // HERMES_THREAD_RWLOCK_H_ diff --git a/hermes_shm/include/hermes_shm/thread/thread_model/argobots.h b/hermes_shm/include/hermes_shm/thread/thread_model/argobots.h deleted file mode 100644 index 6f9eb1ecd..000000000 --- a/hermes_shm/include/hermes_shm/thread/thread_model/argobots.h +++ /dev/null @@ -1,55 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_THREAD_THALLIUM_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_THREAD_THALLIUM_H_ - -#include "thread_model.h" -#include -#include -#include "hermes_shm/util/errors.h" -#include -#include "hermes_shm/introspect/system_info.h" - -namespace hshm::thread_model { - -class Argobots : public ThreadModel { - public: - /** Default constructor */ - Argobots() = default; - - /** Virtual destructor */ - virtual ~Argobots() = default; - - /** Yield the current thread for a period of time */ - void SleepForUs(size_t us) override { - usleep(us); - } - - /** Yield thread time slice */ - void Yield() override { - ABT_thread_yield(); - } - - /** Get the TID of the current thread */ - tid_t GetTid() override { - ABT_thread thread; - ABT_thread_id tid; - ABT_thread_self(&thread); - ABT_thread_get_id(thread, &tid); - return (tid_t)tid; - } -}; - -} // namespace hshm::thread_model - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_THREAD_THALLIUM_H_ diff --git a/hermes_shm/include/hermes_shm/thread/thread_model/pthread.h b/hermes_shm/include/hermes_shm/thread/thread_model/pthread.h deleted file mode 100644 index 491772f13..000000000 --- a/hermes_shm/include/hermes_shm/thread/thread_model/pthread.h +++ /dev/null @@ -1,51 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_THREAD_PTHREAD_H_ -#define HERMES_THREAD_PTHREAD_H_ - -#include "thread_model.h" -#include -#include "hermes_shm/util/errors.h" -#include -#include "hermes_shm/introspect/system_info.h" - -namespace hshm::thread_model { - -class Pthread : public ThreadModel { - public: - /** Default constructor */ - Pthread() = default; - - /** Virtual destructor */ - virtual ~Pthread() = default; - - /** Yield the thread for a period of time */ - void SleepForUs(size_t us) override { - usleep(us); - } - - /** Yield thread time slice */ - void Yield() override { - sched_yield(); - } - - /** Get the TID of the current thread */ - tid_t GetTid() override { - return (tid_t)omp_get_thread_num(); - // return static_cast(pthread_self()); - } -}; - -} // namespace hshm::thread_model - -#endif // HERMES_THREAD_PTHREAD_H_ diff --git a/hermes_shm/include/hermes_shm/thread/thread_model/thread_model.h b/hermes_shm/include/hermes_shm/thread/thread_model/thread_model.h deleted file mode 100644 index 85ec7a2b6..000000000 --- a/hermes_shm/include/hermes_shm/thread/thread_model/thread_model.h +++ /dev/null @@ -1,52 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_THREAD_THREAD_H_ -#define HERMES_THREAD_THREAD_H_ - -#include -#include -#include -#include -#include "hermes_shm/types/bitfield.h" - -namespace hshm { - -/** Available threads that are mapped */ -enum class ThreadType { - kNone, - kPthread, - kArgobots -}; - -/** Used to represent tid */ -typedef uint64_t tid_t; - -} // namespace hshm - -namespace hshm::thread_model { -/** Represents the generic operations of a thread */ -class ThreadModel { - public: - /** Sleep thread for a period of time */ - virtual void SleepForUs(size_t us) = 0; - - /** Yield thread time slice */ - virtual void Yield() = 0; - - /** Get the TID of the current thread */ - virtual tid_t GetTid() = 0; -}; - -} // namespace hshm::thread_model - -#endif // HERMES_THREAD_THREAD_H_ diff --git a/hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h b/hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h deleted file mode 100644 index ff66752a3..000000000 --- a/hermes_shm/include/hermes_shm/thread/thread_model/thread_model_factory.h +++ /dev/null @@ -1,28 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_THREAD_THREAD_FACTORY_H_ -#define HERMES_THREAD_THREAD_FACTORY_H_ - -#include "thread_model.h" - -namespace hshm::thread_model { - -class ThreadFactory { - public: - /** Get a thread instance */ - static std::unique_ptr Get(ThreadType type); -}; - -} // namespace hshm::thread_model - -#endif // HERMES_THREAD_THREAD_FACTORY_H_ diff --git a/hermes_shm/include/hermes_shm/thread/thread_model_manager.h b/hermes_shm/include/hermes_shm/thread/thread_model_manager.h deleted file mode 100644 index abb07a7d4..000000000 --- a/hermes_shm/include/hermes_shm/thread/thread_model_manager.h +++ /dev/null @@ -1,76 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_THREAD_THREAD_MANAGER_H_ -#define HERMES_THREAD_THREAD_MANAGER_H_ - -#include "hermes_shm/thread/thread_model/thread_model.h" -#include "hermes_shm/thread/thread_model/thread_model_factory.h" -#include -#include - -#include "hermes_shm/util/singleton/_global_singleton.h" -#define HERMES_THREAD_MODEL \ - hshm::GlobalSingleton::GetInstance() -#define HERMES_THREAD_MODEL_T \ - hshm::ThreadModelManager* - -namespace hshm { - -class ThreadModelManager { - public: - ThreadType type_; /**< The type of threads used in this program */ - std::unique_ptr - thread_static_; /**< Functions static to all threads */ - - /** Default constructor */ - ThreadModelManager() { - SetThreadModel(ThreadType::kPthread); - } - - /** Set the threading model of this application */ - void SetThreadModel(ThreadType type); - - /** Sleep for a period of time (microseconds) */ - void SleepForUs(size_t us); - - /** Call Yield */ - void Yield(); - - /** Call GetTid */ - tid_t GetTid(); -}; - -/** A unique identifier of this thread across processes */ -union NodeThreadId { - struct { - uint32_t tid_; - uint32_t pid_; - } bits_; - uint64_t as_int_; - - /** Default constructor */ - NodeThreadId() { - bits_.tid_ = HERMES_THREAD_MODEL->GetTid(); - bits_.pid_ = HERMES_SYSTEM_INFO->pid_; - } - - /** Hash function */ - uint32_t hash() { - return as_int_; - } -}; - -} // namespace hshm - -#endif // HERMES_THREAD_THREAD_MANAGER_H_ diff --git a/hermes_shm/include/hermes_shm/types/argpack.h b/hermes_shm/include/hermes_shm/types/argpack.h deleted file mode 100644 index bd902f1a3..000000000 --- a/hermes_shm/include/hermes_shm/types/argpack.h +++ /dev/null @@ -1,279 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_TYPES_ARGPACK_H_ -#define HERMES_INCLUDE_HERMES_TYPES_ARGPACK_H_ - -#include "real_number.h" -#include "hermes_shm/constants/macros.h" -#include - -namespace hshm { - -/** Type which indicates that a constructor takes ArgPacks as input */ -struct PiecewiseConstruct {}; - -/** Type which ends template recurrence */ -struct EndTemplateRecurrence {}; - -/** Recurrence used to create argument pack */ -template< - size_t idx, - typename T = EndTemplateRecurrence, - typename ...Args> -struct ArgPackRecur { - constexpr static bool is_rval = std::is_rvalue_reference(); - typedef typename std::conditional::type ElementT; - - ElementT arg_; /**< The element stored */ - ArgPackRecur - recur_; /**< Remaining args */ - - /** Default constructor */ - HSHM_ALWAYS_INLINE ArgPackRecur() = default; - - /** Constructor. Rvalue reference. */ - HSHM_ALWAYS_INLINE explicit ArgPackRecur(T arg, Args&& ...args) - : arg_(std::forward(arg)), recur_(std::forward(args)...) {} - - /** Forward an rvalue reference (only if argpack) */ - template - HSHM_ALWAYS_INLINE constexpr decltype(auto) Forward() const { - if constexpr(i == idx) { - if constexpr(is_rval) { - return std::forward(arg_); - } else { - return arg_; - } - } else { - return recur_.template - Forward(); - } - } -}; - -/** Terminator of the ArgPack recurrence */ -template -struct ArgPackRecur { - /** Default constructor */ - HSHM_ALWAYS_INLINE ArgPackRecur() = default; - - /** Forward an rvalue reference (only if argpack) */ - template - HSHM_ALWAYS_INLINE constexpr void Forward() const { - throw std::logic_error("(Forward) ArgPack index outside of range"); - } -}; - -/** Used to semantically pack arguments */ -template -struct ArgPack { - /** Variable argument pack */ - ArgPackRecur<0, Args...> recur_; - /** Size of the argpack */ - static constexpr const size_t size_ = sizeof...(Args); - - /** General Constructor. */ - HSHM_ALWAYS_INLINE ArgPack(Args&& ...args) // NOLINT - : recur_(std::forward(args)...) {} - - /** Get forward reference */ - template - HSHM_ALWAYS_INLINE constexpr decltype(auto) Forward() const { - return recur_.template Forward(); - } - - /** Size */ - HSHM_ALWAYS_INLINE constexpr static size_t Size() { - return size_; - } -}; - -/** Make an argpack */ -template -ArgPack make_argpack(Args&& ...args) { - return ArgPack(std::forward(args)...); -} - -/** Get the type + reference of the forward for \a pack pack at \a index i */ -#define FORWARD_ARGPACK_FULL_TYPE(pack, i)\ - decltype(pack.template Forward()) - -/** Forward the param for \a pack pack at \a index i */ -#define FORWARD_ARGPACK_PARAM(pack, i)\ - std::forward(\ - pack.template Forward()) - -/** Forward an argpack */ -#define FORWARD_ARGPACK(pack) \ - std::forward(pack) - -/** Used to pass an argument pack to a function or class method */ -class PassArgPack { - public: - /** Call function with ArgPack */ - template - HSHM_ALWAYS_INLINE constexpr static decltype(auto) Call(ArgPackT &&pack, - F &&f) { - return _CallRecur<0, ArgPackT, F>( - std::forward(pack), std::forward(f)); - } - - private: - /** Unpacks the ArgPack and passes it to the function */ - template - HSHM_ALWAYS_INLINE constexpr static decltype(auto) - _CallRecur(ArgPackT &&pack, - F &&f, - CurArgs&& ...args) { - if constexpr(i < ArgPackT::Size()) { - return _CallRecur( - std::forward(pack), - std::forward(f), - std::forward(args)..., - FORWARD_ARGPACK_PARAM(pack, i)); - } else { - return std::__invoke(f, std::forward(args)...); - } - } -}; - -/** Combine multiple argpacks into a single argpack */ -class MergeArgPacks { - public: - /** Call function with ArgPack */ - template - HSHM_ALWAYS_INLINE constexpr static decltype(auto) - Merge(ArgPacks&& ...packs) { - return _MergePacksRecur<0>(make_argpack(std::forward(packs)...)); - } - - private: - /** Unpacks the C++ parameter pack of ArgPacks */ - template - HSHM_ALWAYS_INLINE constexpr static decltype(auto) _MergePacksRecur( - ArgPacksT &&packs, CurArgs&& ...args) { - if constexpr(cur_pack < ArgPacksT::Size()) { - return _MergeRecur< - cur_pack, ArgPacksT, 0>( - // End template parameters - std::forward(packs), - FORWARD_ARGPACK_PARAM(packs, cur_pack), - std::forward(args)...); - } else { - return make_argpack(std::forward(args)...); - } - } - - /** Unpacks the C++ parameter pack of ArgPacks */ - template< - size_t cur_pack, typename ArgPacksT, - size_t i, typename ArgPackT, - typename ...CurArgs> - HSHM_ALWAYS_INLINE constexpr static decltype(auto) _MergeRecur( - ArgPacksT &&packs, ArgPackT &&pack, CurArgs&& ...args) { - if constexpr(i < ArgPackT::Size()) { - return _MergeRecur( - std::forward(packs), - std::forward(pack), - std::forward(args)..., FORWARD_ARGPACK_PARAM(pack, i)); - } else { - return _MergePacksRecur( - std::forward(packs), - std::forward(args)...); - } - } -}; - -/** Insert an argpack at the head of each pack in a set of ArgPacks */ -class ProductArgPacks { - public: - /** The product function */ - template - HSHM_ALWAYS_INLINE constexpr static decltype(auto) Product( - ProductPackT &&prod_pack, ArgPacks&& ...packs) { - return _ProductPacksRecur<0>( - std::forward(prod_pack), - make_argpack(std::forward(packs)...)); - } - - private: - /** Prepend \a ArgPack prod_pack to every ArgPack in orig_packs */ - template< - size_t cur_pack, - typename ProductPackT, - typename OrigPacksT, - typename ...NewPacks> - HSHM_ALWAYS_INLINE constexpr static decltype(auto) _ProductPacksRecur( - ProductPackT &&prod_pack, - OrigPacksT &&orig_packs, - NewPacks&& ...packs) { - if constexpr(cur_pack < OrigPacksT::Size()) { - return _ProductPacksRecur( - std::forward(prod_pack), - std::forward(orig_packs), - std::forward(packs)..., - std::forward(prod_pack), - FORWARD_ARGPACK_PARAM(orig_packs, cur_pack)); - } else { - return make_argpack(std::forward(packs)...); - } - } -}; - -/** Used to emulate constexpr to lambda */ -template -struct MakeConstexpr { - constexpr static T val_ = Val; - HSHM_ALWAYS_INLINE constexpr static T Get() { - return val_; - } -}; - -/** Apply a function over an entire TupleBase / tuple */ -template -class IterateArgpack { - public: - /** Apply a function to every element of a tuple */ - template - HSHM_ALWAYS_INLINE constexpr static void Apply(TupleT &&pack, F &&f) { - _Apply<0, TupleT, F>(std::forward(pack), std::forward(f)); - } - - private: - /** Apply the function recursively */ - template - HSHM_ALWAYS_INLINE constexpr static void _Apply(TupleT &&pack, F &&f) { - if constexpr(i < TupleT::Size()) { - if constexpr(reverse) { - _Apply(std::forward(pack), - std::forward(f)); - f(MakeConstexpr(), pack.template Forward()); - } else { - f(MakeConstexpr(), pack.template Forward()); - _Apply(std::forward(pack), - std::forward(f)); - } - } - } -}; - -/** Forward iterate over tuple and apply function */ -using ForwardIterateArgpack = IterateArgpack; - -/** Reverse iterate over tuple and apply function */ -using ReverseIterateArgpack = IterateArgpack; - -} // namespace hshm - -#endif // HERMES_INCLUDE_HERMES_TYPES_ARGPACK_H_ diff --git a/hermes_shm/include/hermes_shm/types/atomic.h b/hermes_shm/include/hermes_shm/types/atomic.h deleted file mode 100644 index 9aef4972d..000000000 --- a/hermes_shm/include/hermes_shm/types/atomic.h +++ /dev/null @@ -1,253 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_TYPES_ATOMIC_H_ -#define HERMES_INCLUDE_HERMES_TYPES_ATOMIC_H_ - -#include -#include - -namespace hshm::ipc { - -/** Provides the API of an atomic, without being atomic */ -template -struct nonatomic { - T x; - - /** Constructor */ - HSHM_ALWAYS_INLINE nonatomic() = default; - - /** Full constructor */ - HSHM_ALWAYS_INLINE explicit nonatomic(T def) : x(def) {} - - /** Atomic fetch_add wrapper*/ - HSHM_ALWAYS_INLINE T fetch_add( - T count, std::memory_order order = std::memory_order_seq_cst) { - (void) order; - x += count; - return x; - } - - /** Atomic fetch_sub wrapper*/ - HSHM_ALWAYS_INLINE T fetch_sub( - T count, std::memory_order order = std::memory_order_seq_cst) { - (void) order; - x -= count; - return x; - } - - /** Atomic load wrapper */ - HSHM_ALWAYS_INLINE T load( - std::memory_order order = std::memory_order_seq_cst) const { - (void) order; - return x; - } - - /** Atomic exchange wrapper */ - HSHM_ALWAYS_INLINE void exchange( - T count, std::memory_order order = std::memory_order_seq_cst) { - (void) order; - x = count; - } - - /** Atomic compare exchange weak wrapper */ - HSHM_ALWAYS_INLINE bool compare_exchange_weak(T& expected, T desired, - std::memory_order order = - std::memory_order_seq_cst) { - (void) expected; (void) order; - x = desired; - return true; - } - - /** Atomic compare exchange strong wrapper */ - HSHM_ALWAYS_INLINE bool compare_exchange_strong(T& expected, T desired, - std::memory_order order = - std::memory_order_seq_cst) { - (void) expected; (void) order; - x = desired; - return true; - } - - /** Atomic pre-increment operator */ - HSHM_ALWAYS_INLINE nonatomic& operator++() { - ++x; - return *this; - } - - /** Atomic post-increment operator */ - HSHM_ALWAYS_INLINE nonatomic operator++(int) { - return atomic(x+1); - } - - /** Atomic pre-decrement operator */ - HSHM_ALWAYS_INLINE nonatomic& operator--() { - --x; - return *this; - } - - /** Atomic post-decrement operator */ - HSHM_ALWAYS_INLINE nonatomic operator--(int) { - return atomic(x-1); - } - - /** Atomic add operator */ - HSHM_ALWAYS_INLINE nonatomic operator+(T count) const { - return nonatomic(x + count); - } - - /** Atomic subtract operator */ - HSHM_ALWAYS_INLINE nonatomic operator-(T count) const { - return nonatomic(x - count); - } - - /** Atomic add assign operator */ - HSHM_ALWAYS_INLINE nonatomic& operator+=(T count) { - x += count; - return *this; - } - - /** Atomic subtract assign operator */ - HSHM_ALWAYS_INLINE nonatomic& operator-=(T count) { - x -= count; - return *this; - } - - /** Atomic assign operator */ - HSHM_ALWAYS_INLINE nonatomic& operator=(T count) { - x = count; - return *this; - } - - /** Equality check */ - HSHM_ALWAYS_INLINE bool operator==(const nonatomic &other) const { - return (other.x == x); - } - - /** Inequality check */ - HSHM_ALWAYS_INLINE bool operator!=(const nonatomic &other) const { - return (other.x != x); - } -}; - -/** A wrapper around std::atomic */ -template -struct atomic { - std::atomic x; - - /** Constructor */ - HSHM_ALWAYS_INLINE atomic() = default; - - /** Full constructor */ - HSHM_ALWAYS_INLINE explicit atomic(T def) : x(def) {} - - /** Atomic fetch_add wrapper*/ - HSHM_ALWAYS_INLINE T fetch_add( - T count, std::memory_order order = std::memory_order_seq_cst) { - return x.fetch_add(count, order); - } - - /** Atomic fetch_sub wrapper*/ - HSHM_ALWAYS_INLINE T fetch_sub( - T count, std::memory_order order = std::memory_order_seq_cst) { - return x.fetch_sub(count, order); - } - - /** Atomic load wrapper */ - HSHM_ALWAYS_INLINE T load( - std::memory_order order = std::memory_order_seq_cst) const { - return x.load(order); - } - - /** Atomic exchange wrapper */ - HSHM_ALWAYS_INLINE void exchange( - T count, std::memory_order order = std::memory_order_seq_cst) { - x.exchange(count, order); - } - - /** Atomic compare exchange weak wrapper */ - HSHM_ALWAYS_INLINE bool compare_exchange_weak(T& expected, T desired, - std::memory_order order = - std::memory_order_seq_cst) { - return x.compare_exchange_weak(expected, desired, order); - } - - /** Atomic compare exchange strong wrapper */ - HSHM_ALWAYS_INLINE bool compare_exchange_strong(T& expected, T desired, - std::memory_order order = - std::memory_order_seq_cst) { - return x.compare_exchange_strong(expected, desired, order); - } - - /** Atomic pre-increment operator */ - HSHM_ALWAYS_INLINE atomic& operator++() { - ++x; - return *this; - } - - /** Atomic post-increment operator */ - HSHM_ALWAYS_INLINE atomic operator++(int) { - return atomic(x + 1); - } - - /** Atomic pre-decrement operator */ - HSHM_ALWAYS_INLINE atomic& operator--() { - --x; - return *this; - } - - /** Atomic post-decrement operator */ - HSHM_ALWAYS_INLINE atomic operator--(int) { - return atomic(x - 1); - } - - /** Atomic add operator */ - HSHM_ALWAYS_INLINE atomic operator+(T count) const { - return x + count; - } - - /** Atomic subtract operator */ - HSHM_ALWAYS_INLINE atomic operator-(T count) const { - return x - count; - } - - /** Atomic add assign operator */ - HSHM_ALWAYS_INLINE atomic& operator+=(T count) { - x += count; - return *this; - } - - /** Atomic subtract assign operator */ - HSHM_ALWAYS_INLINE atomic& operator-=(T count) { - x -= count; - return *this; - } - - /** Atomic assign operator */ - HSHM_ALWAYS_INLINE atomic& operator=(T count) { - x.exchange(count); - return *this; - } - - /** Equality check */ - HSHM_ALWAYS_INLINE bool operator==(const atomic &other) const { - return (other.x == x); - } - - /** Inequality check */ - HSHM_ALWAYS_INLINE bool operator!=(const atomic &other) const { - return (other.x != x); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_INCLUDE_HERMES_TYPES_ATOMIC_H_ diff --git a/hermes_shm/include/hermes_shm/types/bitfield.h b/hermes_shm/include/hermes_shm/types/bitfield.h deleted file mode 100644 index 13a987444..000000000 --- a/hermes_shm/include/hermes_shm/types/bitfield.h +++ /dev/null @@ -1,169 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_TYPES_BITFIELD_H_ -#define HERMES_INCLUDE_HERMES_TYPES_BITFIELD_H_ - -#include -#include - -namespace hshm { - -#define BIT_OPT(T, n) (((T)1) << n) -#define ALL_BITS(T) (~((T)0)) - -/** - * A generic bitfield template - * */ -template -struct bitfield { - T bits_; - - HSHM_ALWAYS_INLINE bitfield() : bits_(0) {} - - HSHM_ALWAYS_INLINE explicit bitfield(T mask) : bits_(mask) {} - - HSHM_ALWAYS_INLINE void SetBits(T mask) { - bits_ |= mask; - } - - HSHM_ALWAYS_INLINE void UnsetBits(T mask) { - bits_ &= ~mask; - } - - HSHM_ALWAYS_INLINE bool Any(T mask) const { - return bits_ & mask; - } - - HSHM_ALWAYS_INLINE bool All(T mask) const { - return Any(mask) == mask; - } - - HSHM_ALWAYS_INLINE void CopyBits(bitfield field, T mask) { - bits_ &= (field.bits_ & mask); - } - - HSHM_ALWAYS_INLINE void Clear() { - bits_ = 0; - } - - HSHM_ALWAYS_INLINE static T MakeMask(int start, int length) { - return ((((T)1) << length) - 1) << start; - } -} __attribute__((packed)); -typedef bitfield bitfield8_t; -typedef bitfield bitfield16_t; -typedef bitfield bitfield32_t; - -/** - * A helper type needed for std::conditional - * */ -template -struct len_bits { - static constexpr size_t value = LEN; -}; - -/** - * A generic bitfield template - * */ -template 0)), - len_bits<(NUM_BITS / 32)>, - len_bits<(NUM_BITS / 32) + 1>>::type> -struct big_bitfield { - bitfield32_t bits_[LEN::value]; - - HSHM_ALWAYS_INLINE big_bitfield() : bits_() {} - - HSHM_ALWAYS_INLINE size_t size() const { - return LEN::value; - } - - HSHM_ALWAYS_INLINE void SetBits(int start, int length) { - int bf_idx = start / 32; - int bf_idx_count = 32 - bf_idx; - int rem = length; - while (rem) { - bits_[bf_idx].SetBits(bitfield32_t::MakeMask(start, bf_idx_count)); - rem -= bf_idx_count; - bf_idx += 1; - if (rem >= 32) { - bf_idx_count = 32; - } else { - bf_idx_count = rem; - } - } - } - - HSHM_ALWAYS_INLINE void UnsetBits(int start, int length) { - int bf_idx = start / 32; - int bf_idx_count = 32 - bf_idx; - int rem = length; - while (rem) { - bits_[bf_idx].SetBits(bitfield32_t::MakeMask(start, bf_idx_count)); - rem -= bf_idx_count; - bf_idx += 1; - if (rem >= 32) { - bf_idx_count = 32; - } else { - bf_idx_count = rem; - } - } - } - - HSHM_ALWAYS_INLINE bool Any(int start, int length) const { - int bf_idx = start / 32; - int bf_idx_count = 32 - bf_idx; - int rem = length; - while (rem) { - if (bits_[bf_idx].Any(bitfield32_t::MakeMask(start, bf_idx_count))) { - return true; - } - rem -= bf_idx_count; - bf_idx += 1; - if (rem >= 32) { - bf_idx_count = 32; - } else { - bf_idx_count = rem; - } - } - return false; - } - - HSHM_ALWAYS_INLINE bool All(int start, int length) const { - int bf_idx = start / 32; - int bf_idx_count = 32 - bf_idx; - int rem = length; - while (rem) { - if (!bits_[bf_idx].All(bitfield32_t::MakeMask(start, bf_idx_count))) { - return false; - } - rem -= bf_idx_count; - bf_idx += 1; - if (rem >= 32) { - bf_idx_count = 32; - } else { - bf_idx_count = rem; - } - } - return true; - } - - HSHM_ALWAYS_INLINE void Clear() { - memset((void*)bits_, 0, sizeof(bitfield32_t) * LEN::value); - } -} __attribute__((packed)); - -} // namespace hshm - -#endif // HERMES_INCLUDE_HERMES_TYPES_BITFIELD_H_ diff --git a/hermes_shm/include/hermes_shm/types/real_number.h b/hermes_shm/include/hermes_shm/types/real_number.h deleted file mode 100644 index e3db4535a..000000000 --- a/hermes_shm/include/hermes_shm/types/real_number.h +++ /dev/null @@ -1,98 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_BASICS_H -#define HERMES_BASICS_H - -#include -#include - -#ifdef __cplusplus - -namespace hshm { - -/** - * decimal + (numerator/65536) - * */ -struct RealNumber { - uint64_t decimal_; - uint32_t numerator_; - static const uint32_t precision = 65536; - - RealNumber() = default; - - /** - * Converts numerator / denomintor -> - * decimal + (numerator / 65536); - * - * For example, - * 4/5 = (4 * 65536 / 5) / 65536 - * */ - explicit RealNumber(uint64_t numerator, uint64_t denominator) { - decimal_ = numerator / denominator; - uint64_t rem = numerator % denominator; - numerator_ = rem * precision / denominator; - } - - /** - * (d1 + n1/p) * d2 = - * d1 * d2 + d2 * n1 / p - * */ - RealNumber operator*(size_t other) const { - RealNumber res; - res.decimal_ = other * decimal_; - uint64_t frac = other * numerator_; - res.decimal_ += frac / precision; - res.numerator_ = frac % precision; - return res; - } - - /** - * (d1 + n1/p) * (d2 + n2/p) = - * (d1 * d2) + (d1 * n2)/p + (d2 * n1) / p + (n1 * n2 / p) / p = - * (d1 * d2) + [(d1 * n2) + (d2 * n1) + (n1 * n2)/p] / p - * */ - RealNumber operator*(const RealNumber &other) const { - RealNumber res; - // d1 * d2 - res.decimal_ = other.decimal_ * decimal_; - uint64_t frac = - (decimal_ * other.numerator_) + // d1 * n2 - (other.decimal_ * numerator_) + // d2 * n1 - (numerator_ * other.numerator_) / precision; // n1 * n2 / p - res.decimal_ += frac / precision; - res.numerator_ = frac % precision; - return res; - } - - RealNumber operator*=(size_t other) { - (*this) = (*this) * other; - return *this; - } - - RealNumber operator*=(const RealNumber &other) { - (*this) = (*this) * other; - return *this; - } - - size_t as_int() const { - return decimal_ + numerator_ / precision; - } -}; - -} // namespace hshm - -#endif - - - -#endif // HERMES_BASICS_H diff --git a/hermes_shm/include/hermes_shm/util/auto_trace.h b/hermes_shm/include/hermes_shm/util/auto_trace.h deleted file mode 100644 index c2cd1f2a6..000000000 --- a/hermes_shm/include/hermes_shm/util/auto_trace.h +++ /dev/null @@ -1,94 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_INCLUDE_HERMES_UTIL_AUTO_TRACE_H_ -#define HERMES_INCLUDE_HERMES_UTIL_AUTO_TRACE_H_ - -#include "formatter.h" -#include "timer.h" -#include "logging.h" -#include - -namespace hshm { - -#ifdef HERMES_ENABLE_PROFILING -#define AUTO_TRACE(LOG_LEVEL) \ - hshm::AutoTrace hshm_tracer_(__func__); -#define TIMER_START(NAME) \ - hshm_tracer_.StartTimer(NAME); -#define TIMER_END() \ - hshm_tracer_.EndTimer(); -#else -#define AUTO_TRACE(LOG_LEVEL) -#define TIMER_START(NAME) -#define TIMER_END() -#endif - -/** Trace function execution times */ -template -class AutoTrace { - private: - HighResMonotonicTimer timer_; - HighResMonotonicTimer timer2_; - std::string fname_; - std::string internal_name_; - - public: - template - explicit AutoTrace(const char *fname) : fname_(fname) { - _StartTimer(timer_); - } - - ~AutoTrace() { - _EndTimer(timer_); - } - - void StartTimer(const char *internal_name) { - internal_name_ = "/" + std::string(internal_name); - _StartTimer(timer2_); - } - - void EndTimer() { - _EndTimer(timer2_); - } - - private: - template - void _StartTimer(HighResMonotonicTimer &timer) { -#ifdef HERMES_ENABLE_PROFILING - if constexpr(LOG_LEVEL <= HERMES_LOG_VERBOSITY) { - timer.Resume(); - HILOG(LOG_LEVEL, "{}{}", - fname_, - internal_name_) - } -#endif - } - - void _EndTimer(HighResMonotonicTimer &timer) { -#ifdef HERMES_ENABLE_PROFILING - if constexpr(LOG_LEVEL <= HERMES_LOG_VERBOSITY) { - timer.Pause(); - HILOG(LOG_LEVEL, "{}{} {}ns", - fname_, - internal_name_, - timer.GetNsec()) - timer.Reset(); - internal_name_.clear(); - } -#endif - } -}; - -} // namespace hshm - -#endif // HERMES_INCLUDE_HERMES_UTIL_AUTO_TRACE_H_ diff --git a/hermes_shm/include/hermes_shm/util/config_parse.h b/hermes_shm/include/hermes_shm/util/config_parse.h deleted file mode 100644 index 085634d9c..000000000 --- a/hermes_shm/include/hermes_shm/util/config_parse.h +++ /dev/null @@ -1,241 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_CONFIG_PARSE_PARSER_H -#define HERMES_CONFIG_PARSE_PARSER_H - -#include -#include -#include -#include -#include "formatter.h" -#include "logging.h" -#include "hermes_shm/constants/macros.h" - -#include -#include -#include -#include -#include - -namespace hshm { - -class ConfigParse { - public: - /** - * parse a hostfile string - * [] represents a range to generate - * ; represents a new host name completely - * - * Example: hello[00-09,10]-40g;hello2[11-13]-40g - * */ - static void ParseHostNameString(std::string hostname_set_str, - std::vector &list) { - // Remove all whitespace characters from host name - std::remove(hostname_set_str.begin(), hostname_set_str.end(), ' '); - std::remove(hostname_set_str.begin(), hostname_set_str.end(), '\n'); - std::remove(hostname_set_str.begin(), hostname_set_str.end(), '\r'); - std::remove(hostname_set_str.begin(), hostname_set_str.end(), '\t'); - if (hostname_set_str.size() == 0) { - return; - } - // Expand hostnames - std::stringstream ss(hostname_set_str); - while (ss.good()) { - // Get the current host - std::string hostname; - std::getline(ss, hostname, ';'); - - // Divide the hostname string into prefix, ranges, and suffix - auto lbracket = hostname.find_first_of('['); - auto rbracket = hostname.find_last_of(']'); - std::string prefix, ranges_str, suffix; - if (lbracket != std::string::npos && rbracket != std::string::npos) { - /* - * For example, hello[00-09]-40g - * lbracket = 5 - * rbracket = 11 - * prefix = hello (length: 5) - * range = 00-09 (length: 5) - * suffix = -40g (length: 4) - * */ - prefix = hostname.substr(0, lbracket); - ranges_str = hostname.substr(lbracket + 1, rbracket - lbracket - 1); - suffix = hostname.substr(rbracket + 1); - } else { - list.emplace_back(hostname); - continue; - } - - // Parse the range list into a tuple of (min, max, num_width) - std::stringstream ss_ranges(ranges_str); - std::vector> ranges; - while (ss_ranges.good()) { - // Parse ',' and remove spaces - std::string range_str; - std::getline(ss_ranges, range_str, ','); - std::remove(range_str.begin(), range_str.end(), ' '); - - // Divide the range by '-' - auto dash = range_str.find_first_of('-'); - if (dash != std::string::npos) { - int min, max; - // Get the minimum and maximum value - std::string min_str = range_str.substr(0, dash); - std::string max_str = range_str.substr(dash + 1); - std::stringstream(min_str) >> min; - std::stringstream(max_str) >> max; - - // Check for leading 0s - int num_width = 0; - if (min_str.size() == max_str.size()) { - num_width = min_str.size(); - } - - // Place the range with width - ranges.emplace_back(min, max, num_width); - } else if (range_str.size()) { - int val; - std::stringstream(range_str) >> val; - ranges.emplace_back(val, val, range_str.size()); - } - } - - // Expand the host names by each range - for (auto &range : ranges) { - int min = std::get<0>(range); - int max = std::get<1>(range); - int num_width = std::get<2>(range); - - for (int i = min; i <= max; ++i) { - std::stringstream host_ss; - host_ss << prefix; - host_ss << std::setw(num_width) << std::setfill('0') << i; - host_ss << suffix; - list.emplace_back(host_ss.str()); - } - } - } - } - - /** parse the suffix of \a num_text NUMBER text */ - static std::string ParseNumberSuffix(const std::string &num_text) { - size_t i; - for (i = 0; i < num_text.size(); ++i) { - char c = num_text[i]; - // Skip numbers - if ('0' <= c && c <= '9') { - continue; - } - // Skip whitespace - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - continue; - } - // Skip period (for floats) - if (c == '.') { - continue; - } - break; - } - return std::string(num_text.begin() + i, num_text.end());; - } - - /** parse the number of \a num_text NUMBER text */ - template - static T ParseNumber(const std::string &num_text) { - T size; - if (num_text == "inf") { - return std::numeric_limits::max(); - } - std::stringstream(num_text) >> size; - return size; - } - - /** Converts \a size_text SIZE text into a size_t */ - static size_t ParseSize(const std::string &size_text) { - auto size = ParseNumber(size_text); - if (size_text == "inf") { - return std::numeric_limits::max(); - } - std::string suffix = ParseNumberSuffix(size_text); - if (suffix.empty()) { - return BYTES(size); - } else if (suffix[0] == 'k' || suffix[0] == 'K') { - return KILOBYTES(size); - } else if (suffix[0] == 'm' || suffix[0] == 'M') { - return MEGABYTES(size); - } else if (suffix[0] == 'g' || suffix[0] == 'G') { - return GIGABYTES(size); - } else if (suffix[0] == 't' || suffix[0] == 'T') { - return TERABYTES(size); - } else if (suffix[0] == 'p' || suffix[0] == 'P') { - return PETABYTES(size); - } else { - HELOG(kFatal, "Could not parse the size: {}", size_text) - exit(1); - } - } - - /** Returns bandwidth (bytes / second) */ - static size_t ParseBandwidth(const std::string &size_text) { - return ParseSize(size_text); - } - - /** Returns latency (nanoseconds) */ - static size_t ParseLatency(const std::string &latency_text) { - auto size = ParseNumber(latency_text); - std::string suffix = ParseNumberSuffix(latency_text); - if (suffix.empty()) { - return BYTES(size); - } else if (suffix[0] == 'n' || suffix[0] == 'N') { - return BYTES(size); - } else if (suffix[0] == 'u' || suffix[0] == 'U') { - return KILOBYTES(size); - } else if (suffix[0] == 'm' || suffix[0] == 'M') { - return MEGABYTES(size); - } else if (suffix[0] == 's' || suffix[0] == 'S') { - return GIGABYTES(size); - } - HELOG(kFatal, "Could not parse the latency: {}", latency_text) - return 0; - } - - /** Expands all environment variables in a path string */ - static std::string ExpandPath(std::string path) { - std::smatch env_names; - std::regex expr("\\$\\{[^\\}]+\\}"); - if (!std::regex_search(path, env_names, expr)) { - return path; - } - for (auto &env_name_re : env_names) { - std::string to_replace = std::string(env_name_re); - std::string env_name = to_replace.substr(2, to_replace.size()-3); - std::string env_val = env_name; - try { - char *ret = getenv(env_name.c_str()); - if (ret) { - env_val = ret; - } else { - continue; - } - } catch(...) { - } - std::regex replace_expr("\\$\\{" + env_name + "\\}"); - path = std::regex_replace(path, replace_expr, env_val); - } - return path; - } -}; - -} // namespace hshm - -#endif // HERMES_CONFIG_PARSE_PARSER_H diff --git a/hermes_shm/include/hermes_shm/util/error.h b/hermes_shm/include/hermes_shm/util/error.h deleted file mode 100644 index 113df1392..000000000 --- a/hermes_shm/include/hermes_shm/util/error.h +++ /dev/null @@ -1,64 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ERROR_H -#define HERMES_ERROR_H - -// #ifdef __cplusplus - -#include -#include -#include -#include -#include - -#define HERMES_ERROR_TYPE std::shared_ptr -#define HERMES_ERROR_HANDLE_START() try { -#define HERMES_ERROR_HANDLE_END() \ - } catch(HERMES_ERROR_TYPE &err) { err->print(); exit(-1024); } -#define HERMES_ERROR_HANDLE_TRY try -#define HERMES_ERROR_PTR err -#define HERMES_ERROR_HANDLE_CATCH catch(HERMES_ERROR_TYPE &HERMES_ERROR_PTR) -#define HERMES_ERROR_IS(err, check) (err->get_code() == check.get_code()) - -namespace hshm { - -class Error : std::exception { - private: - std::string fmt_; - std::string msg_; - public: - Error() : fmt_() {} - explicit Error(std::string fmt) : fmt_(std::move(fmt)) {} - ~Error() override = default; - - template - Error format(Args&& ...args) const { - Error err = Error(fmt_); - err.msg_ = Formatter::format(fmt_, std::forward(args)...); - return err; - } - - const char* what() const throw() override { - return msg_.c_str(); - } - - void print() { - std::cout << what() << std::endl; - } -}; - -} // namespace hshm - -// #endif - -#endif // HERMES_ERROR_H diff --git a/hermes_shm/include/hermes_shm/util/errors.h b/hermes_shm/include/hermes_shm/util/errors.h deleted file mode 100644 index 88616de58..000000000 --- a/hermes_shm/include/hermes_shm/util/errors.h +++ /dev/null @@ -1,67 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ERRORS_H -#define HERMES_ERRORS_H - -#ifdef __cplusplus - -#include - -namespace hshm { - const Error FILE_NOT_FOUND("File not found at {}"); - const Error INVALID_STORAGE_TYPE("{} is not a valid storage method"); - const Error INVALID_SERIALIZER_TYPE("{} is not a valid serializer type"); - const Error INVALID_TRANSPORT_TYPE("{} is not a valid transport type"); - const Error INVALID_AFFINITY("Could not set CPU affinity of thread: {}"); - const Error MMAP_FAILED("Could not mmap file: {}"); - const Error LAZY_ERROR("Error in function {}"); - const Error PTHREAD_CREATE_FAILED("Failed to create a pthread"); - const Error MEMORY_BACKEND_REPEATED("Attempted to register two backends " - "with the same id"); - const Error NOT_IMPLEMENTED("{} not implemented"); - - const Error DLSYM_MODULE_NOT_FOUND("Module {} was not loaded; error {}"); - const Error DLSYM_MODULE_NO_CONSTRUCTOR("Module {} has no constructor"); - - const Error UNIX_SOCKET_FAILED("Failed to create socket: {}"); - const Error UNIX_BIND_FAILED("Failed to bind socket: {}"); - const Error UNIX_CONNECT_FAILED("Failed to connect over socket: {}"); - const Error UNIX_SENDMSG_FAILED("Failed to send message over socket: {}"); - const Error UNIX_RECVMSG_FAILED("Failed to receive message over socket: {}"); - const Error UNIX_SETSOCKOPT_FAILED("Failed to set socket options: {}"); - const Error UNIX_GETSOCKOPT_FAILED("Failed to acquire user credentials: {}"); - const Error UNIX_LISTEN_FAILED("Failed to listen for connections: {}"); - const Error UNIX_ACCEPT_FAILED("Failed accept connections: {}"); - - const Error ARRAY_OUT_OF_BOUNDS("Exceeded the bounds of array in {}"); - - const Error SHMEM_CREATE_FAILED("Failed to allocate SHMEM"); - const Error SHMEM_RESERVE_FAILED("Failed to reserve SHMEM"); - const Error SHMEM_NOT_SUPPORTED("Attempting to deserialize a non-shm backend"); - const Error MEMORY_BACKEND_NOT_FOUND("Failed to find the memory backend"); - const Error NOT_ENOUGH_CONCURRENT_SPACE("{}: Failed to divide memory slot {} among {} devices"); - const Error ALIGNED_ALLOC_NOT_SUPPORTED("Allocator does not support aligned allocations"); - const Error PAGE_SIZE_UNSUPPORTED("Allocator does not support size: {}"); - const Error OUT_OF_MEMORY("{}: could not allocate memory of size {}"); - const Error OUT_OF_CACHE("{}: could not cache a page. Allocator overloaded."); - const Error INVALID_FREE("{}: could not free memory of size {}"); - const Error DOUBLE_FREE("Freeing the same memory twice!"); - - const Error IPC_ARGS_NOT_SHM_COMPATIBLE("Args are not compatible with SHM"); - - const Error UNORDERED_MAP_CANT_FIND("Could not find key in unordered_map"); -} // namespace hshm - -#endif - -#endif // HERMES_ERRORS_H diff --git a/hermes_shm/include/hermes_shm/util/formatter.h b/hermes_shm/include/hermes_shm/util/formatter.h deleted file mode 100644 index e236279f7..000000000 --- a/hermes_shm/include/hermes_shm/util/formatter.h +++ /dev/null @@ -1,85 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ERROR_SERIALIZER_H -#define HERMES_ERROR_SERIALIZER_H - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace hshm { - -class Formatter { - public: - template - static std::string format(std::string fmt, Args&& ...args) { - std::stringstream ss; - std::vector> offsets = tokenize(fmt); - size_t packlen = - make_argpack(std::forward(args)...).Size(); - if (offsets.size() != packlen + 1) { - return fmt; - } - auto lambda = [&ss, &fmt, &offsets](auto i, auto &&arg) { - if (i.Get() >= offsets.size()) { return; } - auto &sub = offsets[i.Get()]; - ss << fmt.substr(sub.first, sub.second); - ss << arg; - }; - ForwardIterateArgpack::Apply( - make_argpack(std::forward(args)...), lambda); - if (offsets.back().second > 0) { - auto &sub = offsets.back(); - ss << fmt.substr(sub.first, sub.second); - } - return ss.str(); - } - - static std::vector> - tokenize(const std::string &fmt) { - std::vector> offsets; - size_t i = 0; - offsets.emplace_back(std::pair(0, fmt.size())); - while (i < fmt.size()) { - if (fmt[i] == '{') { - // Set the size of the prior substring - // E.g., "hello{}there". - // prior.first is 0 - // i = 5 for the '{' - // The substring's length is i - 0 = 5. - auto &prior = offsets.back(); - prior.second = i - prior.first; - - // The token after the closing '}' - // i = 5 for the '{' - // i = 7 for the 't' - // The total size is 12 - // The remaining size is: 12 - 7 = 5 (length of "there"). - i += 2; - offsets.emplace_back(std::pair(i, fmt.size() - i)); - continue; - } - ++i; - } - return offsets; - } -}; - -} // namespace hshm - -#endif //HERMES_ERROR_SERIALIZER_H diff --git a/hermes_shm/include/hermes_shm/util/logging.h b/hermes_shm/include/hermes_shm/util/logging.h deleted file mode 100644 index 8389558a7..000000000 --- a/hermes_shm/include/hermes_shm/util/logging.h +++ /dev/null @@ -1,221 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_LOGGING_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_LOGGING_H_ - -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include "formatter.h" -#include "singleton.h" - -namespace hshm { - -/** - * A macro to indicate the verbosity of the logger. - * Verbosity indicates how much data will be printed. - * The higher the verbosity, the more data will be printed. - * */ -#ifndef HERMES_LOG_VERBOSITY -#define HERMES_LOG_VERBOSITY 10 -#endif -#if HERMES_LOG_VERBOSITY < 0 -#error "HERMES_LOG_VERBOSITY cannot be less than 0" -#endif - -/** Prints log verbosity at compile time */ -#define XSTR(s) STR(s) -#define STR(s) #s -// #pragma message XSTR(HERMES_LOG_VERBOSITY) - -/** Simplify access to Logger singleton */ -#define HERMES_LOG hshm::EasySingleton::GetInstance() - -/** Information Logging levels */ -#define kDebug 10 /**< Low-priority debugging information*/ -// ... may want to add more levels here -#define kInfo 1 /**< Useful information the user should know */ - -/** Error Logging Levels */ -#define kFatal 0 /**< A fatal error has occurred */ -#define kError 1 /**< A non-fatal error has occurred */ -#define kWarning 2 /**< Something might be wrong */ - -/** - * Hermes Print. Like printf, except types are inferred - * */ -#define HIPRINT(...) \ - HERMES_LOG->Print(__VA_ARGS__); - -/** - * Hermes Info (HI) Log - * LOG_LEVEL indicates the priority of the log. - * LOG_LEVEL 0 is considered required - * LOG_LEVEL 10 is considered debugging priority. - * */ -#define HILOG(LOG_LEVEL, ...) \ - if constexpr(LOG_LEVEL <= HERMES_LOG_VERBOSITY) { \ - HERMES_LOG->InfoLog(LOG_LEVEL, __FILE__, \ - __func__, __LINE__, __VA_ARGS__); \ - } - -/** - * Hermes Error (HE) Log - * LOG_LEVEL indicates the priority of the log. - * LOG_LEVEL 0 is considered required - * LOG_LEVEL 10 is considered debugging priority. - * */ -#define HELOG(LOG_LEVEL, ...) \ - if constexpr(LOG_LEVEL <= HERMES_LOG_VERBOSITY) { \ - HERMES_LOG->ErrorLog(LOG_LEVEL, __FILE__, \ - __func__, __LINE__, __VA_ARGS__); \ - } - -class Logger { - public: - int verbosity_; - FILE *fout_; - - public: - Logger() { - // exe_name_ = std::filesystem::path(exe_path_).filename().string(); - int verbosity = kDebug; - auto verbosity_env = getenv("HERMES_LOG_VERBOSITY"); - if (verbosity_env && strlen(verbosity_env)) { - try { - verbosity = std::stoi(verbosity_env); - } catch (...) { - verbosity = kDebug; - } - } - SetVerbosity(verbosity); - - auto env = getenv("HERMES_LOG_OUT"); - if (env == nullptr) { - fout_ = nullptr; - } else { - fout_ = fopen(env, "w"); - } - } - - void SetVerbosity(int LOG_LEVEL) { - verbosity_ = LOG_LEVEL; - if (verbosity_ < 0) { - verbosity_ = 0; - } - } - - template - void Print(const char *fmt, - Args&& ...args) { - std::string out = - hshm::Formatter::format(fmt, std::forward(args)...); - std::cout << out; - if (fout_) { - fwrite(out.data(), 1, out.size(), fout_); - } - } - - template - void InfoLog(int LOG_LEVEL, - const char *path, - const char *func, - int line, - const char *fmt, - Args&& ...args) { - if (LOG_LEVEL > verbosity_) { return; } - std::string msg = - hshm::Formatter::format(fmt, std::forward(args)...); - int tid = GetTid(); - std::string out = hshm::Formatter::format( - "{}:{} {} {} {}\n", - path, line, tid, func, msg); - std::cerr << out; - if (fout_) { - fwrite(out.data(), 1, out.size(), fout_); - } - } - - template - void ErrorLog(int LOG_LEVEL, - const char *path, - const char *func, - int line, - const char *fmt, - Args&& ...args) { - if (LOG_LEVEL > verbosity_) { return; } - std::string level; - switch (LOG_LEVEL) { - case kWarning: { - level = "Warning"; - break; - } - case kError: { - level = "ERROR"; - break; - } - case kFatal: { - level = "FATAL"; - break; - } - default: { - level = "WARNING"; - break; - } - } - - std::string msg = - hshm::Formatter::format(fmt, std::forward(args)...); - int tid = GetTid(); - std::string out = hshm::Formatter::format( - "{}:{} {} {} {} {}\n", - path, line, level, tid, func, msg); - std::cerr << out; - if (fout_) { - fwrite(out.data(), 1, out.size(), fout_); - } - if (LOG_LEVEL == kFatal) { - exit(1); - } - } - - int GetTid() { -#ifdef SYS_gettid - return (pid_t)syscall(SYS_gettid); -#else - #warning "GetTid is not defined" - return GetPid(); -#endif - } - - int GetPid() { -#ifdef SYS_getpid - return (pid_t)syscall(SYS_getpid); -#else - #warning "GetPid is not defined" - return 0; -#endif - } -}; - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_LOGGING_H_ diff --git a/hermes_shm/include/hermes_shm/util/partitioner.h b/hermes_shm/include/hermes_shm/util/partitioner.h deleted file mode 100644 index 9919ef0d3..000000000 --- a/hermes_shm/include/hermes_shm/util/partitioner.h +++ /dev/null @@ -1,156 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_PARTITIONER_H -#define HERMES_PARTITIONER_H - -// Reference: https://stackoverflow.com/questions/63372288/getting-list-of-pids-from-proc-in-linux - -#include -#include -#include -#include -#include - -namespace hshm { - -class ProcessAffiner { - private: - int n_cpu_; - cpu_set_t *cpus_; - - public: - ProcessAffiner() { - n_cpu_ = get_nprocs_conf(); - cpus_ = new cpu_set_t[n_cpu_]; - CPU_ZERO(cpus_); - } - - ~ProcessAffiner() { - delete cpus_; - } - - inline bool isdigit(char digit) { - return ('0' <= digit && digit <= '9'); - } - - inline int GetNumCPU() { - return n_cpu_; - } - - inline void SetCpu(int cpu) { - CPU_SET(cpu, cpus_); - } - - inline void SetCpus(int off, int len) { - for (int i = 0; i < len; ++i) { - SetCpu(off + i); - } - } - - inline void ClearCpu(int cpu) { - CPU_CLR(cpu, cpus_); - } - - inline void ClearCpus(int off, int len) { - for (int i = 0; i < len; ++i) { - ClearCpu(off + i); - } - } - - inline void Clear() { - CPU_ZERO(cpus_); - } - - int AffineAll(void) { - DIR *procdir; - struct dirent *entry; - size_t count = 0; - - // Open /proc directory. - procdir = opendir("/proc"); - if (!procdir) { - perror("opendir failed"); - return 0; - } - - // Iterate through all files and folders of /proc. - while ((entry = readdir(procdir))) { - // Skip anything that is not a PID folder. - if (!is_pid_folder(entry)) - continue; - // Get the PID of the running process - int proc_pid = atoi(entry->d_name); - // Set the affinity of all running process to this mask - count += Affine(proc_pid); - } - closedir(procdir); - return count; - } - int Affine(std::vector &&pids) { - return Affine(pids); - } - int Affine(std::vector &pids) { - // Set the affinity of all running process to this mask - size_t count = 0; - for (pid_t &pid : pids) { - count += Affine(pid); - } - return count; - } - int Affine(int pid) { - return SetAffinitySafe(pid, n_cpu_, cpus_); - } - - void PrintAffinity(int pid) { - PrintAffinity("", pid); - } - void PrintAffinity(std::string prefix, int pid) { - std::vector cpus(n_cpu_); - sched_getaffinity(pid, n_cpu_, cpus.data()); - PrintAffinity(prefix, pid, cpus.data()); - } - - void PrintAffinity(std::string prefix, int pid, cpu_set_t *cpus) { - std::string affinity = ""; - for (int i = 0; i < n_cpu_; ++i) { - if (CPU_ISSET(i, cpus)) { - affinity += std::to_string(i) + ", "; - } - } - printf("%s: CPU affinity[pid=%d]: %s\n", prefix.c_str(), - pid, affinity.c_str()); - } - - private: - int SetAffinitySafe(int pid, int n_cpu, cpu_set_t *cpus) { - int ret = sched_setaffinity(pid, n_cpu, cpus); - if (ret == -1) { - return 0; - } - return 1; - } - - // Helper function to check if a struct dirent from /proc is a PID folder. - int is_pid_folder(const struct dirent *entry) { - const char *p; - for (p = entry->d_name; *p; p++) { - if (!isdigit(*p)) - return false; - } - return true; - } -}; - -} // namespace hshm - -#endif // HERMES_PARTITIONER_H diff --git a/hermes_shm/include/hermes_shm/util/singleton.h b/hermes_shm/include/hermes_shm/util/singleton.h deleted file mode 100644 index a87a0c479..000000000 --- a/hermes_shm/include/hermes_shm/util/singleton.h +++ /dev/null @@ -1,21 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_SINGLETON_H -#define HERMES_SHM_SINGLETON_H - -#include "singleton/_easy_global_singleton.h" -#include "singleton/_easy_singleton.h" -#include "singleton/_global_singleton.h" -#include "singleton/_singleton.h" - -#endif // HERMES_SHM_SINGLETON_H diff --git a/hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h deleted file mode 100644 index b396c5067..000000000 --- a/hermes_shm/include/hermes_shm/util/singleton/_easy_global_singleton.h +++ /dev/null @@ -1,40 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_GLOBAL_SINGLETON_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_GLOBAL_SINGLETON_H_ - -#include -#include "hermes_shm/constants/macros.h" - -namespace hshm { - -/** - * Makes a singleton. Constructs during initialization of program. - * Does not require specific initialization of the static variable. - * */ -template -class EasyGlobalSingleton { - private: - static T obj_; - public: - EasyGlobalSingleton() = default; - static T* GetInstance() { - return &obj_; - } -}; -template -T EasyGlobalSingleton::obj_; - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_GLOBAL_SINGLETON_H_ diff --git a/hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h deleted file mode 100644 index b896fc9c0..000000000 --- a/hermes_shm/include/hermes_shm/util/singleton/_easy_singleton.h +++ /dev/null @@ -1,57 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_SINGLETON_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_SINGLETON_H_ - -#include -#include "hermes_shm/thread/lock/mutex.h" -#include "hermes_shm/constants/macros.h" - -namespace hshm { - -/** - * A class to represent singleton pattern - * Does not require specific initialization of the static variable - * */ -template -class EasySingleton { - protected: - /** static instance. */ - static T* obj_; - static hshm::Mutex lock_; - - public: - /** - * Uses unique pointer to build a static global instance of variable. - * @tparam T - * @return instance of T - */ - template - static T* GetInstance(Args&& ...args) { - if (obj_ == nullptr) { - hshm::ScopedMutex lock(lock_, 0); - if (obj_ == nullptr) { - obj_ = new T(std::forward(args)...); - } - } - return obj_; - } -}; -template -T* EasySingleton::obj_ = nullptr; -template -hshm::Mutex EasySingleton::lock_ = hshm::Mutex(); - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_EASY_SINGLETON_H_ diff --git a/hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h deleted file mode 100644 index e3be7ec01..000000000 --- a/hermes_shm/include/hermes_shm/util/singleton/_global_singleton.h +++ /dev/null @@ -1,56 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON__GLOBAL_SINGLETON_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON__GLOBAL_SINGLETON_H_ - -#include -#include "hermes_shm/constants/macros.h" - -namespace hshm { - -/** - * Makes a singleton. Constructs during initialization of program. - * Requires user to define the static storage of obj_ in separate file. - * */ -template -class GlobalSingleton { - public: - static T *obj_; - - public: - GlobalSingleton() = default; - - /** Get instance of type T */ - HSHM_ALWAYS_INLINE static T* GetInstance() { - return obj_; - } - - /** Get ref of type T */ - HSHM_ALWAYS_INLINE static T& GetRef() { - return *obj_; - } - - /** Static initialization method for obj */ - static T& _GetObj(); -}; -template -T* GlobalSingleton::obj_ = &GlobalSingleton::_GetObj(); -#define DEFINE_GLOBAL_SINGLETON_CC(T)\ - template<> T& hshm::GlobalSingleton::_GetObj() {\ - static T obj; \ - return obj;\ - } - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON__GLOBAL_SINGLETON_H_ diff --git a/hermes_shm/include/hermes_shm/util/singleton/_singleton.h b/hermes_shm/include/hermes_shm/util/singleton/_singleton.h deleted file mode 100644 index d7977eea9..000000000 --- a/hermes_shm/include/hermes_shm/util/singleton/_singleton.h +++ /dev/null @@ -1,69 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_SINGLETON_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_SINGLETON_H_ - -#include -#include "hermes_shm/thread/lock/mutex.h" -#include "hermes_shm/constants/macros.h" - -namespace hshm { - -/** - * Makes a singleton. Constructs the first time GetInstance is called. - * Requires user to define the static storage of obj_ in separate file. - * @tparam T - */ -template -class Singleton { - private: - static T *obj_; - static hshm::Mutex *lock_; - - public: - Singleton() = default; - - /** Get or create an instance of type T */ - inline static T *GetInstance() { - if (!obj_) { - hshm::ScopedMutex lock(*lock_, 0); - if (obj_ == nullptr) { - obj_ = new T(); - } - } - return obj_; - } - - /** Static initialization method for obj */ - static T *_GetObj(); - - /** Static initialization method for lock */ - static hshm::Mutex *_GetLock(); -}; - -template -T* Singleton::obj_ = Singleton::_GetObj(); -template -hshm::Mutex* Singleton::lock_ = Singleton::_GetLock(); -#define DEFINE_SINGLETON_CC(T)\ - template<> T* hshm::Singleton::_GetObj() {\ - return nullptr;\ - }\ - template<> hshm::Mutex* hshm::Singleton::_GetLock() {\ - static hshm::Mutex lock;\ - return &lock;\ - } - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_SINGLETON_SINGLETON_H_ diff --git a/hermes_shm/include/hermes_shm/util/timer.h b/hermes_shm/include/hermes_shm/util/timer.h deleted file mode 100644 index 2dde003ae..000000000 --- a/hermes_shm/include/hermes_shm/util/timer.h +++ /dev/null @@ -1,96 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TIMER_H -#define HERMES_TIMER_H - -#include -#include -#include - -namespace hshm { - -template -class TimerBase { - private: - std::chrono::time_point start_, end_; - double time_ns_; - - public: - TimerBase() : time_ns_(0) {} - - void Resume() { - start_ = T::now(); - } - double Pause() { - time_ns_ += GetNsecFromStart(); - return time_ns_; - } - double Pause(double &dt) { - dt = GetNsecFromStart(); - time_ns_ += dt; - return time_ns_; - } - void Reset() { - time_ns_ = 0; - } - - double GetNsecFromStart() { - end_ = T::now(); - double elapsed = std::chrono::duration_cast( - end_ - start_).count(); - return elapsed; - } - double GetUsecFromStart() { - end_ = T::now(); - return std::chrono::duration_cast( - end_ - start_).count(); - } - double GetMsecFromStart() { - end_ = T::now(); - return std::chrono::duration_cast( - end_ - start_).count(); - } - double GetSecFromStart() { - end_ = T::now(); - return std::chrono::duration_cast( - end_ - start_).count(); - } - - double GetNsec() const { - return time_ns_; - } - double GetUsec() const { - return time_ns_/1000; - } - double GetMsec() const { - return time_ns_/1000000; - } - double GetSec() const { - return time_ns_/1000000000; - } - - double GetUsFromEpoch() const { - std::chrono::time_point point = - std::chrono::system_clock::now(); - return std::chrono::duration_cast( - point.time_since_epoch()).count(); - } -}; - -typedef TimerBase HighResCpuTimer; -typedef TimerBase HighResMonotonicTimer; -typedef HighResMonotonicTimer Timer; - -} // namespace hshm - -#endif // HERMES_TIMER_H diff --git a/hermes_shm/include/hermes_shm/util/type_switch.h b/hermes_shm/include/hermes_shm/util/type_switch.h deleted file mode 100644 index 4a26f9033..000000000 --- a/hermes_shm/include/hermes_shm/util/type_switch.h +++ /dev/null @@ -1,51 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_STATIC_SWITCH_H_ -#define HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_STATIC_SWITCH_H_ - -#include - -namespace hshm { - -/** Ends the recurrence of the switch-case */ -class EndTypeSwitch {}; - -/** - * A compile-time switch-case statement used for choosing - * a type based on another type - * - * @param T the type being checked (i.e., switch (T) - * @param Default the default case - * @param Case a case of the switch (i.e., case Case:) - * @param Val the body of the case - * */ -template -struct type_switch { - typedef typename std::conditional< - std::is_same_v, - Val, - typename type_switch::type>::type type; -}; - -/** The default case */ -template -struct type_switch { - typedef Default type; -}; - -} // namespace hshm - -#endif // HERMES_SHM_INCLUDE_HERMES_SHM_UTIL_STATIC_SWITCH_H_ diff --git a/hermes_shm/readme.md b/hermes_shm/readme.md deleted file mode 100644 index 8f65871a8..000000000 --- a/hermes_shm/readme.md +++ /dev/null @@ -1,7 +0,0 @@ - -# hermes_shmn - -This library contains a variety of data structures and synchronization -primitives which are compatible with shared memory. - -[![Coverage Status](https://coveralls.io/repos/github/lukemartinlogan/hermes_shm/badge.svg?branch=master)](https://coveralls.io/github/lukemartinlogan/hermes_shm?branch=master) \ No newline at end of file diff --git a/hermes_shm/scripts/ci/packages.yaml b/hermes_shm/scripts/ci/packages.yaml deleted file mode 100644 index 7be206aad..000000000 --- a/hermes_shm/scripts/ci/packages.yaml +++ /dev/null @@ -1,38 +0,0 @@ -packages: - all: - target: [x86_64] - mpich: - externals: - - spec: mpich@3.3.2 - prefix: /usr - buildable: False - cmake: - externals: - - spec: cmake@3.23.2 - prefix: /usr/local - buildable: False - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - buildable: False - automake: - externals: - - spec: automake@1.16 - prefix: /usr - buildable: False - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - buildable: False - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - buildable: False - pkg-config: - externals: - - spec: pkg-config@0.29.1 - prefix: /usr - buildable: False diff --git a/hermes_shm/scripts/hermes_shm/packages/hermes_shm/__init__.py b/hermes_shm/scripts/hermes_shm/packages/hermes_shm/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/hermes_shm/scripts/hermes_shm/repo.yaml b/hermes_shm/scripts/hermes_shm/repo.yaml deleted file mode 100644 index f38c31d94..000000000 --- a/hermes_shm/scripts/hermes_shm/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: 'hermes_shm' diff --git a/hermes_shm/scripts/lint.sh b/hermes_shm/scripts/lint.sh deleted file mode 100644 index ff11eda53..000000000 --- a/hermes_shm/scripts/lint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -HERMES_ROOT=$1 - -cpplint --recursive \ ---exclude="${HERMES_ROOT}/include/hermes_shm/constants/singleton_macros.h" \ ---exclude="${HERMES_ROOT}/include/hermes_shm/constants/data_structure_singleton_macros.h" \ ---exclude="${HERMES_ROOT}/include/hermes_shm/data_structures/ipc/internal/template" \ ---exclude="${HERMES_ROOT}/include/hermes_shm/data_structures/ipc/internal/shm_container_macro.h" \ ---exclude="${HERMES_ROOT}/src/singleton.cc" \ ---exclude="${HERMES_ROOT}/src/data_structure_singleton.cc" \ ---exclude="${HERMES_ROOT}/include/hermes_shm/util/formatter.h" \ ---exclude="${HERMES_ROOT}/include/hermes_shm/util/errors.h" \ -"${HERMES_ROOT}/src" "${HERMES_ROOT}/include" "${HERMES_ROOT}/test" \ No newline at end of file diff --git a/hermes_shm/scripts/preamble.py b/hermes_shm/scripts/preamble.py deleted file mode 100644 index f4906b8ab..000000000 --- a/hermes_shm/scripts/preamble.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Prepends the preabmle to all files in the repo - -USAGE: - python3 scripts/preamble.py ${HERMES_ROOT} -""" - -import sys,os,re -import pathlib - -hermes_preamble = """ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -""".strip() - -preamble2 = """ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -""".strip() - -labstor_preamble = """ -/* - * Copyright (C) 2022 SCS Lab , - * Luke Logan , - * Jaime Cernuda Garcia - * Jay Lofstead , - * Anthony Kougkas , - * Xian-He Sun - * - * This file is part of HermesShm - * - * HermesShm is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - */ -""".strip() - -def PrependPreamble(path): - text = "" - with open(path, 'r') as fp: - text = fp.read() - text = text.replace(hermes_preamble, "") - text = text.replace(preamble2, "") - text = text.replace(labstor_preamble, "") - text = re.sub("//\n// Created by [^\n]*\n//\n", "", text) - text = text.strip() - text = hermes_preamble + "\n\n" + text + "\n" - - with open(path, 'w') as fp: - fp.write(text) - - -def LocateCppFiles(root): - for entry in os.listdir(root): - full_path = os.path.join(root, entry) - if os.path.isdir(full_path): - LocateCppFiles(full_path) - elif os.path.isfile(full_path): - file_ext = pathlib.Path(full_path).suffix - if file_ext == '.cc' or file_ext == '.h': - PrependPreamble(full_path) - - -PROJECT_ROOT=pathlib.Path(__file__).parent.parent.resolve() -LocateCppFiles(PROJECT_ROOT) \ No newline at end of file diff --git a/hermes_shm/scripts/singleton_generator.py b/hermes_shm/scripts/singleton_generator.py deleted file mode 100644 index b0da22b0b..000000000 --- a/hermes_shm/scripts/singleton_generator.py +++ /dev/null @@ -1,98 +0,0 @@ - -""" -Create the .h and .cc files for defining hermes singletons -USAGE: - cd /path/to/hermes - python3 scripts/singleton_generator.py -""" - -import re - -def ToCamelCase(string): - if string is None: - return - words = re.sub(r"(_|-)+", " ", string).split() - words = [word.capitalize() for word in words] - return "".join(words) - -def ToSnakeCase(string): - if string is None: - return - string = re.sub('(\.|-)+', '_', string) - words = re.split('([A-Z][^A-Z]*)', string) - words = [word for word in words if len(word)] - string = "_".join(words) - return string.lower() - -class SingletonDefinition: - def __init__(self, namespace, class_name, include): - snake = ToSnakeCase(class_name).upper() - self.macro_name = f"HERMES_{snake}" - self.type_name = f"{self.macro_name}_T" - if namespace is not None: - self.class_name = f"{namespace}::{class_name}" - else: - self.class_name = class_name - self.include = include - -class SingletonGenerator: - def __init__(self): - self.defs = [] - - def Add(self, namespace, class_name, include): - self.defs.append(SingletonDefinition( - namespace, class_name, include - )) - - def Generate(self, cc_file, h_file): - self._GenerateCC(cc_file, h_file) - self._GenerateH(h_file) - - def _GenerateCC(self, path, h_file): - lines = [] - lines.append("#include ") - lines.append("#include ") - lines.append("#include ") - lines.append(f"#include <{h_file.replace('include/', '')}>") - lines.append("") - for defn in self.defs: - lines.append(f"#include <{defn.include}>") - lines.append(f"template<> {defn.class_name} hshm::GlobalSingleton<{defn.class_name}>::obj_ = {defn.class_name}();") - #lines.append(f"template<> hshm::Mutex hshm::GlobalSingleton<{defn.class_name}>::lock_ = hshm::Mutex();") - self._SaveLines(lines, path) - - def _GenerateH(self, path): - lines = [] - guard = ToSnakeCase(path).replace('/', '_') - lines.append(f"#ifndef HERMES_{guard.upper()}_H") - lines.append(f"#define HERMES_{guard.upper()}_H") - lines.append("") - lines.append("#include ") - lines.append("") - for defn in self.defs: - lines.append(f"#define {defn.macro_name} hshm::GlobalSingleton<{defn.class_name}>::GetInstance()") - lines.append(f"#define {defn.type_name} {defn.class_name}*") - lines.append("") - lines.append(f"#endif // {guard}") - self._SaveLines(lines, path) - - def _SaveLines(self, lines, path): - text = "\n".join(lines) + "\n" - if path is None: - print(text) - return - with open(path, 'w') as fp: - fp.write(text) - -gen = SingletonGenerator() -gen.Add("hermes", "IpcManager", "hermes_shm/ipc_manager/ipc_manager.h") -gen.Add("hermes", "ConfigurationManager", "hermes_shm/runtime/configuration_manager.h") -gen.Generate("src/singleton.cc", - "include/hermes_shm/constants/singleton_macros.h") - -gen = SingletonGenerator() -gen.Add("hermes", "SystemInfo", "hermes_shm/introspect/system_info.h") -gen.Add("hshm::ipc", "MemoryManager", "hermes_shm/memory/memory_manager.h") -gen.Add("hermes", "ThreadManager", "hermes_shm/thread/thread_manager.h") -gen.Generate("src/data_structure_singleton.cc", - "include/hermes_shm/constants/data_structure_singleton_macros.h") \ No newline at end of file diff --git a/hermes_shm/src/CMakeLists.txt b/hermes_shm/src/CMakeLists.txt deleted file mode 100644 index 987ec1d7c..000000000 --- a/hermes_shm/src/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#----------------------------------------------------------------------------- -# Build HSHM -#----------------------------------------------------------------------------- -add_library(hermes_shm_data_structures - memory/malloc_allocator.cc - memory/stack_allocator.cc - memory/scalable_page_allocator.cc - memory/memory_registry.cc - memory/memory_manager.cc - thread_model_manager.cc - thread_factory.cc - data_structure_singleton.cc) -target_link_libraries(hermes_shm_data_structures - pthread -lrt -ldl OpenMP::OpenMP_CXX - $<$:thallium>) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install(TARGETS - hermes_shm_data_structures - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_shm_data_structures - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_shm_data_structures) -endif() diff --git a/hermes_shm/src/data_structure_singleton.cc b/hermes_shm/src/data_structure_singleton.cc deleted file mode 100644 index b923b28d1..000000000 --- a/hermes_shm/src/data_structure_singleton.cc +++ /dev/null @@ -1,25 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include -#include -#include - -#include -DEFINE_GLOBAL_SINGLETON_CC(hshm::SystemInfo) -#include -DEFINE_GLOBAL_SINGLETON_CC(hshm::ipc::MemoryRegistry) -#include -DEFINE_GLOBAL_SINGLETON_CC(hshm::ipc::MemoryManager) -#include -DEFINE_GLOBAL_SINGLETON_CC(hshm::ThreadModelManager) diff --git a/hermes_shm/src/memory/malloc_allocator.cc b/hermes_shm/src/memory/malloc_allocator.cc deleted file mode 100644 index 46c48554d..000000000 --- a/hermes_shm/src/memory/malloc_allocator.cc +++ /dev/null @@ -1,82 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include - -namespace hshm::ipc { - -struct MallocPage { - size_t page_size_; -}; - -void MallocAllocator::shm_init(allocator_id_t id, - size_t custom_header_size, - size_t buffer_size) { - buffer_ = nullptr; - buffer_size_ = buffer_size; - header_ = reinterpret_cast( - malloc(sizeof(MallocAllocatorHeader) + custom_header_size)); - custom_header_ = reinterpret_cast(header_ + 1); - header_->Configure(id, custom_header_size); -} - -void MallocAllocator::shm_deserialize(char *buffer, - size_t buffer_size) { - throw NOT_IMPLEMENTED.format("MallocAllocator::shm_deserialize"); -} - -size_t MallocAllocator::GetCurrentlyAllocatedSize() { - return header_->total_alloc_size_; -} - -OffsetPointer MallocAllocator::AllocateOffset(size_t size) { - auto page = reinterpret_cast( - malloc(sizeof(MallocPage) + size)); - page->page_size_ = size; - header_->total_alloc_size_ += size; - return OffsetPointer((size_t)(page + 1)); -} - -OffsetPointer MallocAllocator::AlignedAllocateOffset(size_t size, - size_t alignment) { - auto page = reinterpret_cast( - aligned_alloc(alignment, sizeof(MallocPage) + size)); - page->page_size_ = size; - header_->total_alloc_size_ += size; - return OffsetPointer(size_t(page + 1)); -} - -OffsetPointer MallocAllocator::ReallocateOffsetNoNullCheck(OffsetPointer p, - size_t new_size) { - // Get the input page - auto page = reinterpret_cast( - p.off_.load() - sizeof(MallocPage)); - header_->total_alloc_size_ += new_size - page->page_size_; - - // Reallocate the input page - auto new_page = reinterpret_cast( - realloc(page, sizeof(MallocPage) + new_size)); - new_page->page_size_ = new_size; - - // Create the pointer - return OffsetPointer(size_t(new_page + 1)); -} - -void MallocAllocator::FreeOffsetNoNullCheck(OffsetPointer p) { - auto page = reinterpret_cast( - p.off_.load() - sizeof(MallocPage)); - header_->total_alloc_size_ -= page->page_size_; - free(page); -} - -} // namespace hshm::ipc diff --git a/hermes_shm/src/memory/memory_intercept.cc b/hermes_shm/src/memory/memory_intercept.cc deleted file mode 100644 index b6a7dead6..000000000 --- a/hermes_shm/src/memory/memory_intercept.cc +++ /dev/null @@ -1,94 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include -#include "hermes_shm/memory/memory_manager.h" - -using hshm::ipc::Pointer; -using hshm::ipc::Allocator; - -/** Allocate SIZE bytes of memory. */ -void* malloc(size_t size) { - auto alloc = HERMES_MEMORY_MANAGER->GetDefaultAllocator(); - return alloc->AllocatePtr(size); -} - -/** Allocate NMEMB elements of SIZE bytes each, all initialized to 0. */ -void* calloc(size_t nmemb, size_t size) { - auto alloc = HERMES_MEMORY_MANAGER->GetDefaultAllocator(); - return alloc->ClearAllocatePtr(nmemb * size); -} - -/** - * Re-allocate the previously allocated block in ptr, making the new - * block SIZE bytes long. - * */ -void* realloc(void *ptr, size_t size) { - Pointer p = HERMES_MEMORY_MANAGER->Convert(ptr); - auto alloc = HERMES_MEMORY_MANAGER->GetAllocator(p.allocator_id_); - return alloc->AllocatePtr(size); -} - -/** - * Re-allocate the previously allocated block in PTR, making the new - * block large enough for NMEMB elements of SIZE bytes each. - * */ -void* reallocarray(void *ptr, size_t nmemb, size_t size) { - return realloc(ptr, nmemb * size); -} - -/** Free a block allocated by `malloc', `realloc' or `calloc'. */ -void free(void *ptr) { - Pointer p = HERMES_MEMORY_MANAGER->Convert(ptr); - auto alloc = HERMES_MEMORY_MANAGER->GetAllocator(p.allocator_id_); - alloc->Free(p); -} - -/** Allocate SIZE bytes allocated to ALIGNMENT bytes. */ -void* memalign(size_t alignment, size_t size) { - // TODO(llogan): need to add an aligned allocator - return malloc(size); -} - -/** Allocate SIZE bytes on a page boundary. */ -void* valloc(size_t size) { - return memalign(HERMES_SYSTEM_INFO->page_size_, size); -} - -/** - * Equivalent to valloc(minimum-page-that-holds(n)), - * that is, round up size to nearest pagesize. - * */ -void* pvalloc(size_t size) { - size_t new_size = hipc::MemoryAlignment::AlignToPageSize(size); - return valloc(new_size); -} - -/** - * Allocates size bytes and places the address of the - * allocated memory in *memptr. The address of the allocated memory - * will be a multiple of alignment, which must be a power of two and a multiple - * of sizeof(void*). Returns NULL if size is 0. */ -int posix_memalign(void **memptr, size_t alignment, size_t size) { - (*memptr) = memalign(alignment, size); - return 0; -} - -/** - * Aligned to an alignment with a size that is a multiple of the - * alignment - * */ -void *aligned_alloc(size_t alignment, size_t size) { - return memalign(alignment, - hshm::ipc::NextAlignmentMultiple(alignment, size)); -} diff --git a/hermes_shm/src/memory/memory_intercept.h b/hermes_shm/src/memory/memory_intercept.h deleted file mode 100644 index ce3989570..000000000 --- a/hermes_shm/src/memory/memory_intercept.h +++ /dev/null @@ -1,16 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_MEMORY_MEMORY_INTERCEPT_H_ -#define HERMES_SRC_MEMORY_MEMORY_INTERCEPT_H_ - -#endif // HERMES_SRC_MEMORY_MEMORY_INTERCEPT_H_ diff --git a/hermes_shm/src/memory/memory_manager.cc b/hermes_shm/src/memory/memory_manager.cc deleted file mode 100644 index 0be31f6ed..000000000 --- a/hermes_shm/src/memory/memory_manager.cc +++ /dev/null @@ -1,29 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include -#include "hermes_shm/memory/backend/memory_backend_factory.h" -#include "hermes_shm/memory/allocator/allocator_factory.h" -#include -#include "hermes_shm/constants/data_structure_singleton_macros.h" - -namespace hshm::ipc { - -void MemoryManager::ScanBackends() { - for (auto &[url, backend] : HERMES_MEMORY_REGISTRY->backends_) { - auto alloc = AllocatorFactory::shm_deserialize(backend.get()); - RegisterAllocator(alloc); - } -} - -} // namespace hshm::ipc diff --git a/hermes_shm/src/memory/memory_registry.cc b/hermes_shm/src/memory/memory_registry.cc deleted file mode 100644 index 9300217bd..000000000 --- a/hermes_shm/src/memory/memory_registry.cc +++ /dev/null @@ -1,30 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_shm/memory/memory_registry.h" - -namespace hshm::ipc { - -MemoryRegistry::MemoryRegistry() { - root_allocator_id_.bits_.major_ = 3; - root_allocator_id_.bits_.minor_ = 3; - root_backend_.shm_init(MEGABYTES(128)); - root_backend_.Own(); - root_allocator_.shm_init(root_allocator_id_, 0, - root_backend_.data_, - root_backend_.data_size_); - default_allocator_ = &root_allocator_; - memset(allocators_, 0, sizeof(allocators_)); - RegisterAllocator(&root_allocator_); -} - -} // namespace hshm::ipc diff --git a/hermes_shm/src/memory/scalable_page_allocator.cc b/hermes_shm/src/memory/scalable_page_allocator.cc deleted file mode 100644 index 3c028a125..000000000 --- a/hermes_shm/src/memory/scalable_page_allocator.cc +++ /dev/null @@ -1,294 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include -#include - -namespace hshm::ipc { - -void ScalablePageAllocator::shm_init(allocator_id_t id, - size_t custom_header_size, - char *buffer, - size_t buffer_size, - RealNumber coalesce_trigger, - size_t coalesce_window) { - buffer_ = buffer; - buffer_size_ = buffer_size; - header_ = reinterpret_cast(buffer_); - custom_header_ = reinterpret_cast(header_ + 1); - size_t region_off = (custom_header_ - buffer_) + custom_header_size; - size_t region_size = buffer_size_ - region_off; - allocator_id_t sub_id(id.bits_.major_, id.bits_.minor_ + 1); - alloc_.shm_init(sub_id, 0, buffer + region_off, region_size); - HERMES_MEMORY_REGISTRY_REF.RegisterAllocator(&alloc_); - header_->Configure(id, custom_header_size, &alloc_, - buffer_size, coalesce_trigger, coalesce_window); - free_lists_ = header_->free_lists_.get(); - // Cache every power-of-two between 32B and 16KB - size_t ncpu = HERMES_SYSTEM_INFO->ncpu_; - free_lists_->resize(ncpu * num_free_lists_); - for (int i = 0; i < HERMES_SYSTEM_INFO->ncpu_; ++i) { - for (size_t j = 0; j < num_free_lists_; ++j) { - pair>& - free_list = (*free_lists_)[i * num_free_lists_ + j]; - free_list.GetFirst().page_size_ = sizeof(MpPage) + - min_cached_size_ * (1 << j); - free_list.GetFirst().lock_.Init(); - free_list.GetFirst().cur_alloc_ = 0; - free_list.GetFirst().max_alloc_ = - max_cached_size_ / free_list.GetFirst().page_size_; - if (free_list.GetFirst().max_alloc_ > 16) { - free_list.GetFirst().max_alloc_ = 16; - } - } - } -} - -void ScalablePageAllocator::shm_deserialize(char *buffer, - size_t buffer_size) { - buffer_ = buffer; - buffer_size_ = buffer_size; - header_ = reinterpret_cast(buffer_); - custom_header_ = reinterpret_cast(header_ + 1); - size_t region_off = (custom_header_ - buffer_) + header_->custom_header_size_; - size_t region_size = buffer_size_ - region_off; - alloc_.shm_deserialize(buffer + region_off, region_size); - HERMES_MEMORY_REGISTRY_REF.RegisterAllocator(&alloc_); - free_lists_ = header_->free_lists_.get(); -} - -size_t ScalablePageAllocator::GetCurrentlyAllocatedSize() { - return header_->total_alloc_; -} - -/** Round a number up to the nearest page size. */ -size_t ScalablePageAllocator::RoundUp(size_t num, size_t &exp) { - size_t round; - for (exp = 0; exp < num_caches_; ++exp) { - round = 1 << (exp + min_cached_size_exp_); - round += sizeof(MpPage); - if (num <= round) { - return round; - } - } - return num; -} - -OffsetPointer ScalablePageAllocator::AllocateOffset(size_t size) { - MpPage *page = nullptr; - size_t exp; - size_t size_mp = RoundUp(size + sizeof(MpPage), exp); - uint32_t cpu = NodeThreadId().hash() % HERMES_SYSTEM_INFO->ncpu_; - - // Case 1: Can we re-use an existing page? - page = CheckLocalCaches(size_mp, cpu); - - // Case 2: Coalesce if enough space is being wasted - // if (page == nullptr) {} - - // Case 3: Allocate from stack if no page found - if (page == nullptr) { - auto off = alloc_.AllocateOffset(size_mp); - if (!off.IsNull()) { - page = alloc_.Convert(off - sizeof(MpPage)); - page->page_size_ = size_mp; - } - } - - // Case 4: Completely out of memory - if (page == nullptr) { - throw OUT_OF_MEMORY; - } - - // Mark as allocated - header_->total_alloc_.fetch_add(page->page_size_); - auto p = Convert(page); - page->SetAllocated(); - return p + sizeof(MpPage); -} - -MpPage *ScalablePageAllocator::CheckLocalCaches(size_t size_mp, uint32_t cpu) { - MpPage *page; - // ScopedRwReadLock coalesce_lock(header_->coalesce_lock_, 0); - uint32_t cpu_start = cpu * num_free_lists_; - pair> &first_free_list = - (*free_lists_)[cpu_start]; - ScopedMutex first_list_lock(first_free_list.GetFirst().lock_, 0); - - // Check the small buffer caches - if (size_mp <= max_cached_size_) { - size_t exp; - // Check the nearest buffer cache - size_mp = RoundUp(size_mp, exp); - pair> &free_list = - (*free_lists_)[cpu_start + exp]; - if (free_list.GetSecond().size()) { - MpPage *rem_page; - page = free_list.GetSecond().dequeue(); - DividePage(free_list.GetFirst(), - free_list.GetSecond(), - page, rem_page, size_mp, - 16); - return page; - } else { - return nullptr; - } - - // Check all upper buffer caches - /* for (size_t i = exp + 1; i < num_caches_; ++i) { - pair> &high_free_list = - (*free_lists_)[cpu_start + i]; - if (high_free_list.GetSecond().size()) { - MpPage *rem_page; - page = high_free_list.GetSecond().dequeue(); - DividePage(free_list.GetFirst(), - free_list.GetSecond(), - page, rem_page, size_mp, - 16); - if (rem_page) { - pair> &last_free_list = - (*free_lists_)[cpu_start + num_caches_]; - last_free_list.GetSecond().enqueue(rem_page); - } - return page; - } - } */ - } else { - // Check the arbitrary buffer cache - pair> &last_free_list = - (*free_lists_)[cpu_start + num_caches_]; - page = FindFirstFit(size_mp, - last_free_list.GetFirst(), - last_free_list.GetSecond()); - return page; - } -} - -MpPage* ScalablePageAllocator::FindFirstFit( - size_t size_mp, - FreeListStats &stats, - iqueue &free_list) { - for (auto iter = free_list.begin(); iter != free_list.end(); ++iter) { - MpPage *fit_page = *iter; - MpPage *rem_page; - if (fit_page->page_size_ >= size_mp) { - DividePage(stats, free_list, fit_page, rem_page, size_mp, 0); - free_list.dequeue(iter); - if (rem_page) { - free_list.enqueue(rem_page); - } - return fit_page; - } - } - return nullptr; -} - -void ScalablePageAllocator::DividePage(FreeListStats &stats, - iqueue &free_list, - MpPage *fit_page, - MpPage *&rem_page, - size_t size_mp, - size_t max_divide) { - // Get space remaining after size_mp is allocated - size_t rem_size; - rem_size = fit_page->page_size_ - size_mp; - stats.AddAlloc(); - - // Case 1: The remaining size can't be cached - rem_page = nullptr; - if (rem_size < min_cached_size_) { - return; - } - - // Case 2: Divide the remaining space into units of size_mp - fit_page->page_size_ = size_mp; - rem_page = (MpPage *) ((char *) fit_page + size_mp); - if (max_divide > 0 && rem_size >= size_mp) { - size_t num_divisions = (rem_size - size_mp) / size_mp; - if (num_divisions > max_divide) { num_divisions = max_divide; } - for (size_t i = 0; i < num_divisions; ++i) { - rem_page->page_size_ = size_mp; - rem_page->flags_.Clear(); - rem_page->off_ = 0; - stats.AddAlloc(); - free_list.enqueue(rem_page); - rem_page = (MpPage *) ((char *) rem_page + size_mp); - rem_size -= size_mp; - } - } - - // Case 3: There is still remaining space after the divisions - if (rem_size > 0) { - rem_page->page_size_ = rem_size; - rem_page->flags_.Clear(); - rem_page->off_ = 0; - } else { - rem_page = nullptr; - } -} - -OffsetPointer ScalablePageAllocator::AlignedAllocateOffset(size_t size, - size_t alignment) { - throw ALIGNED_ALLOC_NOT_SUPPORTED.format(); -} - -OffsetPointer ScalablePageAllocator::ReallocateOffsetNoNullCheck( - OffsetPointer p, size_t new_size) { - OffsetPointer new_p; - void *ptr = AllocatePtr(new_size, new_p); - MpPage *hdr = Convert(p - sizeof(MpPage)); - void *old = (void*)(hdr + 1); - memcpy(ptr, old, hdr->page_size_ - sizeof(MpPage)); - FreeOffsetNoNullCheck(p); - return new_p; -} - -void ScalablePageAllocator::FreeOffsetNoNullCheck(OffsetPointer p) { - // Mark as free - auto hdr_offset = p - sizeof(MpPage); - auto hdr = Convert(hdr_offset); - if (!hdr->IsAllocated()) { - throw DOUBLE_FREE.format(); - } - hdr->UnsetAllocated(); - header_->total_alloc_.fetch_sub(hdr->page_size_); - - // Get the free list to start from - uint32_t cpu = NodeThreadId().hash() % HERMES_SYSTEM_INFO->ncpu_; - // hdr->cpu_; - // NodeThreadId().hash() % HERMES_SYSTEM_INFO->ncpu_; - uint32_t cpu_start = cpu * num_free_lists_; - pair> &first_free_list = - (*free_lists_)[cpu_start]; - ScopedMutex first_list_lock(first_free_list.GetFirst().lock_, 0); - - // Append to small buffer cache free list - if (hdr->page_size_ <= max_cached_size_) { - for (size_t i = 0; i < num_caches_; ++i) { - pair> &free_list = - (*free_lists_)[cpu_start + i]; - size_t page_size = free_list.GetFirst().page_size_; - if (page_size == hdr->page_size_) { - free_list.GetSecond().enqueue(hdr); - return; - } - } - } - - // Append to arbitrary free list - pair> &last_free_list = - (*free_lists_)[cpu_start + num_caches_]; - last_free_list.GetSecond().enqueue(hdr); -} - -} // namespace hshm::ipc diff --git a/hermes_shm/src/memory/stack_allocator.cc b/hermes_shm/src/memory/stack_allocator.cc deleted file mode 100644 index d6dd7d970..000000000 --- a/hermes_shm/src/memory/stack_allocator.cc +++ /dev/null @@ -1,83 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include -#include - -namespace hshm::ipc { - -void StackAllocator::shm_init(allocator_id_t id, - size_t custom_header_size, - char *buffer, - size_t buffer_size) { - buffer_ = buffer; - buffer_size_ = buffer_size; - header_ = reinterpret_cast(buffer_); - custom_header_ = reinterpret_cast(header_ + 1); - size_t region_off = (custom_header_ - buffer_) + custom_header_size; - size_t region_size = buffer_size_ - region_off; - header_->Configure(id, custom_header_size, region_off, region_size); - heap_ = &header_->heap_; -} - -void StackAllocator::shm_deserialize(char *buffer, - size_t buffer_size) { - buffer_ = buffer; - buffer_size_ = buffer_size; - header_ = reinterpret_cast(buffer_); - custom_header_ = reinterpret_cast(header_ + 1); - heap_ = &header_->heap_; -} - -size_t StackAllocator::GetCurrentlyAllocatedSize() { - return header_->total_alloc_; -} - -OffsetPointer StackAllocator::AllocateOffset(size_t size) { - size += sizeof(MpPage); - OffsetPointer p = heap_->AllocateOffset(size); - auto hdr = Convert(p); - hdr->SetAllocated(); - hdr->page_size_ = size; - hdr->off_ = 0; - header_->total_alloc_.fetch_add(hdr->page_size_); - return p + sizeof(MpPage); -} - -OffsetPointer StackAllocator::AlignedAllocateOffset(size_t size, - size_t alignment) { - throw ALIGNED_ALLOC_NOT_SUPPORTED.format(); -} - -OffsetPointer StackAllocator::ReallocateOffsetNoNullCheck(OffsetPointer p, - size_t new_size) { - OffsetPointer new_p; - void *src = Convert(p); - auto hdr = Convert(p - sizeof(MpPage)); - size_t old_size = hdr->page_size_ - sizeof(MpPage); - void *dst = AllocatePtr(new_size, new_p); - memcpy((void*)dst, (void*)src, old_size); - Free(p); - return new_p; -} - -void StackAllocator::FreeOffsetNoNullCheck(OffsetPointer p) { - auto hdr = Convert(p - sizeof(MpPage)); - if (!hdr->IsAllocated()) { - throw DOUBLE_FREE.format(); - } - hdr->UnsetAllocated(); - header_->total_alloc_.fetch_sub(hdr->page_size_); -} - -} // namespace hshm::ipc diff --git a/hermes_shm/src/thread_factory.cc b/hermes_shm/src/thread_factory.cc deleted file mode 100644 index 15b1efd69..000000000 --- a/hermes_shm/src/thread_factory.cc +++ /dev/null @@ -1,44 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_shm/thread/thread_model/thread_model_factory.h" -#include "hermes_shm/thread/thread_model/thread_model.h" -#include "hermes_shm/thread/thread_model/pthread.h" -#include "hermes_shm/thread/thread_model/argobots.h" -#include "hermes_shm/util/logging.h" - -namespace hshm::thread_model { - -std::unique_ptr ThreadFactory::Get(ThreadType type) { - switch (type) { - case ThreadType::kPthread: { -#ifdef HERMES_PTHREADS_ENABLED - return std::make_unique(); -#else - return nullptr; -#endif - } - case ThreadType::kArgobots: { -#ifdef HERMES_RPC_THALLIUM - return std::make_unique(); -#else - return nullptr; -#endif - } - default: { - HELOG(kWarning, "No such thread type"); - return nullptr; - } - } -} - -} // namespace hshm::thread_model diff --git a/hermes_shm/src/thread_model_manager.cc b/hermes_shm/src/thread_model_manager.cc deleted file mode 100644 index ff4f1c252..000000000 --- a/hermes_shm/src/thread_model_manager.cc +++ /dev/null @@ -1,49 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_shm/thread/thread_model_manager.h" -#include "hermes_shm/util/logging.h" - -namespace hshm { - -/** Set the threading model of this application */ -void ThreadModelManager::SetThreadModel(ThreadType type) { - static std::mutex lock_; - lock_.lock(); - if (type_ == type) { - lock_.unlock(); - return; - } - type_ = type; - thread_static_ = thread_model::ThreadFactory::Get(type); - if (thread_static_ == nullptr) { - HELOG(kFatal, "Could not load the threading model"); - } - lock_.unlock(); -} - -/** Sleep for a period of time (microseconds) */ -void ThreadModelManager::SleepForUs(size_t us) { - thread_static_->SleepForUs(us); -} - -/** Call Yield */ -void ThreadModelManager::Yield() { - thread_static_->Yield(); -} - -/** Call GetTid */ -tid_t ThreadModelManager::GetTid() { - return thread_static_->GetTid(); -} - -} // namespace hshm diff --git a/hermes_shm/test/CMakeLists.txt b/hermes_shm/test/CMakeLists.txt deleted file mode 100644 index 81aad4158..000000000 --- a/hermes_shm/test/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -add_subdirectory(unit) \ No newline at end of file diff --git a/hermes_shm/test/unit/CMakeLists.txt b/hermes_shm/test/unit/CMakeLists.txt deleted file mode 100644 index cadd80c1b..000000000 --- a/hermes_shm/test/unit/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -set(TEST_MAIN ${CMAKE_CURRENT_SOURCE_DIR}) - -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -add_subdirectory(data_structures) -add_subdirectory(allocators) -add_subdirectory(allocators_mpi) -add_subdirectory(backend) -add_subdirectory(types) -add_subdirectory(thread) \ No newline at end of file diff --git a/hermes_shm/test/unit/allocators/CMakeLists.txt b/hermes_shm/test/unit/allocators/CMakeLists.txt deleted file mode 100644 index 28eca3f22..000000000 --- a/hermes_shm/test/unit/allocators/CMakeLists.txt +++ /dev/null @@ -1,58 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Build Tests -#------------------------------------------------------------------------------ - -add_executable(test_allocator_exec - ${TEST_MAIN}/main.cc - test_init.cc - allocator.cc - allocator_thread.cc) -add_dependencies(test_allocator_exec hermes_shm_data_structures) -target_link_libraries(test_allocator_exec - hermes_shm_data_structures Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) - -#------------------------------------------------------------------------------ -# Test Cases -#------------------------------------------------------------------------------ - -# ALLOCATOR tests -set(ALLOCATORS - StackAllocator - MallocAllocator - ScalablePageAllocator) -foreach(ALLOCATOR ${ALLOCATORS}) - add_test(NAME test_${ALLOCATOR} COMMAND - ${CMAKE_BINARY_DIR}/bin/test_allocator_exec "${ALLOCATOR}") -endforeach() - -# Multi-Thread ALLOCATOR tests -set(MT_ALLOCATORS - StackAllocator - ScalablePageAllocator) -foreach(ALLOCATOR ${MT_ALLOCATORS}) - add_test(NAME test_${ALLOCATOR}_4t COMMAND - ${CMAKE_BINARY_DIR}/bin/test_allocator_exec "${ALLOCATOR}Multithreaded") -endforeach() - -#------------------------------------------------------------------------------ -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_allocator_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_allocator_exec) -endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/allocators/allocator.cc b/hermes_shm/test/unit/allocators/allocator.cc deleted file mode 100644 index 1bf0fd253..000000000 --- a/hermes_shm/test/unit/allocators/allocator.cc +++ /dev/null @@ -1,157 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "test_init.h" - -void PageAllocationTest(Allocator *alloc) { - size_t count = 1024; - size_t page_size = KILOBYTES(4); - auto mem_mngr = HERMES_MEMORY_MANAGER; - - // Allocate pages - std::vector ps(count); - void *ptrs[count]; - for (size_t i = 0; i < count; ++i) { - ptrs[i] = alloc->AllocatePtr(page_size, ps[i]); - memset(ptrs[i], i, page_size); - REQUIRE(ps[i].off_.load() != 0); - REQUIRE(!ps[i].IsNull()); - REQUIRE(ptrs[i] != nullptr); - } - - // Convert process pointers into independent pointers - for (size_t i = 0; i < count; ++i) { - Pointer p = mem_mngr->Convert(ptrs[i]); - REQUIRE(p == ps[i]); - REQUIRE(VerifyBuffer((char*)ptrs[i], page_size, i)); - } - - // Check the custom header - auto hdr = alloc->GetCustomHeader(); - REQUIRE(hdr->checksum_ == HEADER_CHECKSUM); - - // Free pages - for (size_t i = 0; i < count; ++i) { - alloc->Free(ps[i]); - } - - // Reallocate pages - for (size_t i = 0; i < count; ++i) { - ptrs[i] = alloc->AllocatePtr(page_size, ps[i]); - REQUIRE(ps[i].off_.load() != 0); - REQUIRE(!ps[i].IsNull()); - } - - // Free again - for (size_t i = 0; i < count; ++i) { - alloc->Free(ps[i]); - } -} - -void MultiPageAllocationTest(Allocator *alloc) { - std::vector alloc_sizes = { - 64, 128, 256, - KILOBYTES(1), KILOBYTES(4), KILOBYTES(64), - MEGABYTES(1) - }; - - // Allocate and free pages between 64 bytes and 32MB - { - for (size_t r = 0; r < 4; ++r) { - for (size_t i = 0; i < alloc_sizes.size(); ++i) { - Pointer ps[16]; - for (size_t j = 0; j < 16; ++j) { - ps[j] = alloc->Allocate(alloc_sizes[i]); - } - for (size_t j = 0; j < 16; ++j) { - alloc->Free(ps[j]); - } - } - } - } -} - -void ReallocationTest(Allocator *alloc) { - std::vector> sizes = { - {KILOBYTES(3), KILOBYTES(4)}, - {KILOBYTES(4), MEGABYTES(1)} - }; - - // Reallocate a small page to a larger page - for (auto &[small_size, large_size] : sizes) { - Pointer p; - char *ptr = alloc->AllocatePtr(small_size, p); - memset(ptr, 10, small_size); - char *new_ptr = alloc->ReallocatePtr(p, large_size); - for (size_t i = 0; i < small_size; ++i) { - REQUIRE(ptr[i] == 10); - } - memset(new_ptr, 0, large_size); - alloc->Free(p); - } -} - -void AlignedAllocationTest(Allocator *alloc) { - std::vector> sizes = { - {KILOBYTES(4), KILOBYTES(4)}, - }; - - // Aligned allocate pages - for (auto &[size, alignment] : sizes) { - for (size_t i = 0; i < 1024; ++i) { - Pointer p; - char *ptr = alloc->AllocatePtr(size, p, alignment); - REQUIRE(((size_t)ptr % alignment) == 0); - memset(alloc->Convert(p), 0, size); - alloc->Free(p); - } - } -} - -TEST_CASE("StackAllocator") { - auto alloc = Pretest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - PageAllocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - Posttest(); -} - -TEST_CASE("MallocAllocator") { - auto alloc = Pretest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - PageAllocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - MultiPageAllocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - - Posttest(); -} - -TEST_CASE("ScalablePageAllocator") { - auto alloc = Pretest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - PageAllocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - MultiPageAllocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ReallocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - - Posttest(); -} diff --git a/hermes_shm/test/unit/allocators/allocator_thread.cc b/hermes_shm/test/unit/allocators/allocator_thread.cc deleted file mode 100644 index 85efb0155..000000000 --- a/hermes_shm/test/unit/allocators/allocator_thread.cc +++ /dev/null @@ -1,48 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "test_init.h" - -void MultiThreadedPageAllocationTest(Allocator *alloc) { - size_t nthreads = 8; - omp_set_dynamic(0); -#pragma omp parallel shared(alloc) num_threads(nthreads) - { -#pragma omp barrier - PageAllocationTest(alloc); -#pragma omp barrier - try { - MultiPageAllocationTest(alloc); - } catch (std::shared_ptr &err) { - err->print(); - exit(1); - } -#pragma omp barrier - } -} - -TEST_CASE("StackAllocatorMultithreaded") { - auto alloc = Pretest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - MultiThreadedPageAllocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - Posttest(); -} - -TEST_CASE("ScalablePageAllocatorMultithreaded") { - auto alloc = Pretest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - MultiThreadedPageAllocationTest(alloc); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - Posttest(); -} diff --git a/hermes_shm/test/unit/allocators/test_init.h b/hermes_shm/test/unit/allocators/test_init.h deleted file mode 100644 index b59c6d89e..000000000 --- a/hermes_shm/test/unit/allocators/test_init.h +++ /dev/null @@ -1,56 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_TEST_UNIT_ALLOCATORS_TEST_INIT_H_ -#define HERMES_TEST_UNIT_ALLOCATORS_TEST_INIT_H_ - -#include "basic_test.h" -#include "omp.h" -#include "hermes_shm/memory/memory_manager.h" - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -#define HEADER_CHECKSUM 8482942 - -struct SimpleAllocatorHeader { - int checksum_; -}; - -template -Allocator* Pretest() { - std::string shm_url = "test_allocators"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator( - shm_url, alloc_id, sizeof(SimpleAllocatorHeader)); - auto alloc = mem_mngr->GetAllocator(alloc_id); - auto hdr = alloc->GetCustomHeader(); - hdr->checksum_ = HEADER_CHECKSUM; - return alloc; -} - -void Posttest(); -void PageAllocationTest(Allocator *alloc); -void MultiPageAllocationTest(Allocator *alloc); - -#endif // HERMES_TEST_UNIT_ALLOCATORS_TEST_INIT_H_ diff --git a/hermes_shm/test/unit/allocators_mpi/CMakeLists.txt b/hermes_shm/test/unit/allocators_mpi/CMakeLists.txt deleted file mode 100644 index 185a8c8cb..000000000 --- a/hermes_shm/test/unit/allocators_mpi/CMakeLists.txt +++ /dev/null @@ -1,47 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Build Tests -#------------------------------------------------------------------------------ - -add_executable(test_allocator_mpi_exec - ${TEST_MAIN}/main_mpi.cc - test_init.cc - allocator_mpi.cc) -add_dependencies(test_allocator_mpi_exec hermes_shm_data_structures) -target_link_libraries(test_allocator_mpi_exec - hermes_shm_data_structures Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) - -#------------------------------------------------------------------------------ -# Test Cases -#------------------------------------------------------------------------------ - -# Multi-Thread ALLOCATOR tests -set(MP_ALLOCATORS - StackAllocator - ScalablePageAllocator) -foreach(ALLOCATOR ${MP_ALLOCATORS}) - add_test(NAME test_${ALLOCATOR}_mpi COMMAND - mpirun -n 4 ${CMAKE_BINARY_DIR}/bin/test_allocator_mpi_exec "${ALLOCATOR}Mpi") -endforeach() - -#------------------------------------------------------------------------------ -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_allocator_mpi_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_allocator_mpi_exec) -endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc b/hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc deleted file mode 100644 index 33d16a892..000000000 --- a/hermes_shm/test/unit/allocators_mpi/allocator_mpi.cc +++ /dev/null @@ -1,75 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "test_init.h" -#include "basic_test.h" - -struct Record { - char *data; - size_t size; - Pointer ptr; -}; - -void MpiPageAllocationTest(Allocator *alloc, size_t count) { - size_t window_length = 32; - size_t min_page = 64; - size_t max_page = MEGABYTES(1); - std::mt19937 rng(23522523); - std::uniform_int_distribution uni(min_page, max_page); - - MPI_Barrier(MPI_COMM_WORLD); - size_t num_windows = count / window_length; - std::vector window(window_length); - for (size_t w = 0; w < num_windows; ++w) { - for (size_t i = 0; i < window_length; ++i) { - window[i].size = uni(rng); - window[i].data = alloc->AllocatePtr(window[i].size, - window[i].ptr); - memset(window[i].data, (char)i, window[i].size); - } - for (size_t i = 0; i < window_length; ++i) { - VerifyBuffer(window[i].data, window[i].size, (char)i); - alloc->Free(window[i].ptr); - } - } - MPI_Barrier(MPI_COMM_WORLD); -} - -template -Allocator* TestAllocatorMpi() { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (rank == 0) { - PretestRank0(); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank != 0) { - PretestRankN(); - } - HILOG(kInfo, "Allocator: {}", (size_t)alloc_g); - return alloc_g; -} - -TEST_CASE("StackAllocatorMpi") { - auto alloc = TestAllocatorMpi(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - MpiPageAllocationTest(alloc, 100); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("ScalablePageAllocatorMpi") { - auto alloc = TestAllocatorMpi(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - MpiPageAllocationTest(alloc, 1000); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/allocators_mpi/test_init.cc b/hermes_shm/test/unit/allocators_mpi/test_init.cc deleted file mode 100644 index e248a58ea..000000000 --- a/hermes_shm/test/unit/allocators_mpi/test_init.cc +++ /dev/null @@ -1,33 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "basic_test.h" -#include "test_init.h" - -Allocator *alloc_g = nullptr; - -void PretestRankN() { - std::string shm_url = "test_allocators"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->AttachBackend(MemoryBackendType::kPosixShmMmap, shm_url); - alloc_g = mem_mngr->GetAllocator(alloc_id); -} - -void MainPretest() { -} - -void MainPosttest() { -} diff --git a/hermes_shm/test/unit/allocators_mpi/test_init.h b/hermes_shm/test/unit/allocators_mpi/test_init.h deleted file mode 100644 index ef4601edc..000000000 --- a/hermes_shm/test/unit/allocators_mpi/test_init.h +++ /dev/null @@ -1,53 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ -#define HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ - -#include "hermes_shm/data_structures/data_structure.h" -#include - -using hshm::ipc::PosixShmMmap; -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::Pointer; - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -extern Allocator *alloc_g; - -template -void PretestRank0() { - std::string shm_url = "test_allocators"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator(shm_url, alloc_id, 0); - alloc_g = mem_mngr->GetAllocator(alloc_id); -} - -void PretestRankN(); - -#endif // HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ diff --git a/hermes_shm/test/unit/backend/CMakeLists.txt b/hermes_shm/test/unit/backend/CMakeLists.txt deleted file mode 100644 index 5ddb37ade..000000000 --- a/hermes_shm/test/unit/backend/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Build Tests -#------------------------------------------------------------------------------ - -add_executable(test_memory_exec - ${TEST_MAIN}/main_mpi.cc - test_init.cc - backend.cc - memory_slots.cc - memory_manager.cc) -add_dependencies(test_memory_exec hermes_shm_data_structures) -target_link_libraries(test_memory_exec - hermes_shm_data_structures Catch2::Catch2 MPI::MPI_CXX) -#------------------------------------------------------------------------------ -# Test Cases -#------------------------------------------------------------------------------ - -add_test(NAME test_memory_slots COMMAND - mpirun -n 2 ${CMAKE_BINARY_DIR}/bin/test_memory_exec "MemorySlot") -add_test(NAME test_reserve COMMAND - ${CMAKE_BINARY_DIR}/bin/test_memory_exec "BackendReserve") -add_test(NAME test_memory_manager COMMAND - mpirun -n 2 ${CMAKE_BINARY_DIR}/bin/test_memory_exec "MemoryManager") - -#------------------------------------------------------------------------------ -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_memory_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_memory_exec) -endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/backend/backend.cc b/hermes_shm/test/unit/backend/backend.cc deleted file mode 100644 index d5159c7aa..000000000 --- a/hermes_shm/test/unit/backend/backend.cc +++ /dev/null @@ -1,30 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" - -#include "hermes_shm/memory/backend/posix_shm_mmap.h" - -using hshm::ipc::PosixShmMmap; - -TEST_CASE("BackendReserve") { - PosixShmMmap b1; - - // Reserve + Map 8GB of memory - b1.shm_init(GIGABYTES(8), "shmem_test"); - - // Set 2GB of SHMEM - memset(b1.data_, 0, GIGABYTES(2)); - - // Destroy SHMEM - b1.shm_destroy(); -} diff --git a/hermes_shm/test/unit/backend/memory_manager.cc b/hermes_shm/test/unit/backend/memory_manager.cc deleted file mode 100644 index 9ec598339..000000000 --- a/hermes_shm/test/unit/backend/memory_manager.cc +++ /dev/null @@ -1,80 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "basic_test.h" - -#include -#include "hermes_shm/memory/memory_manager.h" - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::MemoryManager; - -struct SimpleHeader { - hshm::ipc::Pointer p_; -}; - -TEST_CASE("MemoryManager") { - int rank; - char nonce = 8; - size_t page_size = KILOBYTES(4); - std::string shm_url = "test_mem_backend"; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - allocator_id_t alloc_id(0, 1); - - HERMES_ERROR_HANDLE_START() - auto mem_mngr = HERMES_MEMORY_MANAGER; - - if (rank == 0) { - std::cout << "Creating SHMEM (rank 0): " << shm_url << std::endl; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator( - shm_url, alloc_id, 0); - mem_mngr->ScanBackends(); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank != 0) { - std::cout << "Attaching SHMEM (rank 1): " << shm_url << std::endl; - mem_mngr->AttachBackend(MemoryBackendType::kPosixShmMmap, shm_url); - mem_mngr->ScanBackends(); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank == 0) { - std::cout << "Allocating pages (rank 0)" << std::endl; - hipc::Allocator *alloc = mem_mngr->GetAllocator(alloc_id); - char *page = alloc->AllocatePtr(page_size); - memset(page, nonce, page_size); - auto header = alloc->GetCustomHeader(); - hipc::Pointer p1 = mem_mngr->Convert(alloc_id, page); - hipc::Pointer p2 = mem_mngr->Convert(page); - header->p_ = p1; - REQUIRE(p1 == p2); - REQUIRE(VerifyBuffer(page, page_size, nonce)); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank != 0) { - std::cout << "Finding and checking pages (rank 1)" << std::endl; - hipc::Allocator *alloc = mem_mngr->GetAllocator(alloc_id); - SimpleHeader *header = alloc->GetCustomHeader(); - char *page = alloc->Convert(header->p_); - REQUIRE(VerifyBuffer(page, page_size, nonce)); - } - MPI_Barrier(MPI_COMM_WORLD); - - HERMES_ERROR_HANDLE_END() -} diff --git a/hermes_shm/test/unit/backend/memory_slots.cc b/hermes_shm/test/unit/backend/memory_slots.cc deleted file mode 100644 index 8eeafd708..000000000 --- a/hermes_shm/test/unit/backend/memory_slots.cc +++ /dev/null @@ -1,60 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" - -#include -#include -#include "hermes_shm/memory/backend/posix_shm_mmap.h" - -using hshm::ipc::PosixShmMmap; - -TEST_CASE("MemorySlot") { - int rank; - char nonce = 8; - std::string shm_url = "test_mem_backend"; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - HERMES_ERROR_HANDLE_START() - - PosixShmMmap backend; - if (rank == 0) { - { - std::cout << "Creating SHMEM (rank 0)" << std::endl; - if (!backend.shm_init(MEGABYTES(1), shm_url)) { - throw std::runtime_error("Couldn't create backend"); - } - std::cout << "Backend data: " << (void*)backend.data_ << std::endl; - std::cout << "Backend sz: " << backend.data_size_ << std::endl; - memset(backend.data_, nonce, backend.data_size_); - std::cout << "Wrote backend data" << std::endl; - } - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank != 0) { - { - std::cout << "Attaching SHMEM (rank 1)" << std::endl; - backend.shm_deserialize(shm_url); - char *ptr = backend.data_; - REQUIRE(VerifyBuffer(ptr, backend.data_size_, nonce)); - } - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank == 0) { - { - std::cout << "Destroying shmem (rank 1)" << std::endl; - backend.shm_destroy(); - } - } - - HERMES_ERROR_HANDLE_END() -} diff --git a/hermes_shm/test/unit/basic_test.h b/hermes_shm/test/unit/basic_test.h deleted file mode 100644 index 814b1e510..000000000 --- a/hermes_shm/test/unit/basic_test.h +++ /dev/null @@ -1,72 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TEST_UNIT_BASIC_TEST_H_ -#define HERMES_TEST_UNIT_BASIC_TEST_H_ - -#define CATCH_CONFIG_RUNNER -#include - -namespace cl = Catch::Clara; -cl::Parser define_options(); - -#include -#include - -static inline bool VerifyBuffer(char *ptr, size_t size, char nonce) { - for (size_t i = 0; i < size; ++i) { - if (ptr[i] != nonce) { - std::cout << (int)ptr[i] << std::endl; - return false; - } - } - return true; -} - -/** var = TYPE(val) */ -#define _CREATE_SET_VAR_TO_INT_OR_STRING(TYPE, VAR, TMP_VAR, VAL)\ - if constexpr(std::is_same_v) {\ - TMP_VAR = hipc::make_uptr(std::to_string(VAL));\ - } else if constexpr(std::is_same_v) {\ - TMP_VAR = hipc::make_uptr(std::to_string(VAL));\ - } else {\ - TMP_VAR = hipc::make_uptr(VAL);\ - }\ - TYPE &VAR = *TMP_VAR;\ - (void)VAR; - -/** TYPE VAR = TYPE(VAL) */ -#define CREATE_SET_VAR_TO_INT_OR_STRING(TYPE, VAR, VAL)\ - hipc::uptr VAR##_tmp;\ - _CREATE_SET_VAR_TO_INT_OR_STRING(TYPE, VAR, VAR##_tmp, VAL); - -/** RET = int(TYPE(VAR)); */ -#define GET_INT_FROM_VAR(TYPE, RET, VAR)\ - if constexpr(std::is_same_v) {\ - RET = atoi((VAR).str().c_str());\ - } else if constexpr(std::is_same_v) {\ - RET = atoi((VAR).c_str());\ - } else {\ - RET = VAR;\ - } - -/** int RET = int(TYPE(VAR)); */ -#define CREATE_GET_INT_FROM_VAR(TYPE, RET, VAR)\ - int RET;\ - GET_INT_FROM_VAR(TYPE, RET, VAR) - -void MainPretest(); -void MainPosttest(); - -#define PAGE_DIVIDE(TEXT) - -#endif // HERMES_TEST_UNIT_BASIC_TEST_H_ diff --git a/hermes_shm/test/unit/data_structures/CMakeLists.txt b/hermes_shm/test/unit/data_structures/CMakeLists.txt deleted file mode 100644 index f606e5239..000000000 --- a/hermes_shm/test/unit/data_structures/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -include_directories(${HERMES_SHM_ROOT}/test/unit) -add_subdirectory(containers) -#add_subdirectory(containers_mpi) -add_subdirectory(serialize) diff --git a/hermes_shm/test/unit/data_structures/containers/CMakeLists.txt b/hermes_shm/test/unit/data_structures/containers/CMakeLists.txt deleted file mode 100644 index 44a8af076..000000000 --- a/hermes_shm/test/unit/data_structures/containers/CMakeLists.txt +++ /dev/null @@ -1,102 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Build Tests -#------------------------------------------------------------------------------ - -add_executable(test_data_structure_exec - ${TEST_MAIN}/main.cc - test_init.cc - string.cc - pair.cc - #tuple.cc - list.cc - slist.cc - vector.cc - iqueue.cc - manual_ptr.cc - unique_ptr.cc - unordered_map.cc - mpsc_queue.cc - spsc_queue.cc - charbuf.cc - ticket_queue.cc -) - -add_dependencies(test_data_structure_exec hermes_shm_data_structures) -target_link_libraries(test_data_structure_exec - hermes_shm_data_structures Catch2::Catch2 - MPI::MPI_CXX OpenMP::OpenMP_CXX) - -#------------------------------------------------------------------------------ -# Test Cases -#------------------------------------------------------------------------------ - -# STRING TESTS -add_test(NAME test_string COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "String") - -# VECTOR TESTS -add_test(NAME test_vector COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "Vector*") - -# LIST TESTS -add_test(NAME test_list COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "List*") - -# SLIST TESTS -add_test(NAME test_slist COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "Slist*") - -# MANUAL PTR TESTS -add_test(NAME test_manual_ptr COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "ManualPtr*") - -# UNIQUE PTR TESTS -add_test(NAME test_unique_ptr COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UniquePtr*") - -# UNORDERED_MAP TESTS -add_test(NAME test_unordered_map COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "UnorderedMap*") - -# PAIR TESTS -add_test(NAME test_pair COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "Pair*") - -# IQUEUE TESTS -add_test(NAME test_iqueue COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "IqueueOfMpPage") - -# SPSC TESTS -add_test(NAME test_spsc COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "TestSpsc*") - -# MPSC TESTS -add_test(NAME test_mpsc COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "TestMpsc*") - -# TicketQueue TESTS -add_test(NAME test_tkt_queue COMMAND - ${CMAKE_BINARY_DIR}/bin/test_data_structure_exec "TestTicket*") - -#------------------------------------------------------------------------------ -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_data_structure_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_data_structure_exec) -endif() diff --git a/hermes_shm/test/unit/data_structures/containers/charbuf.cc b/hermes_shm/test/unit/data_structures/containers/charbuf.cc deleted file mode 100644 index 21c35eef6..000000000 --- a/hermes_shm/test/unit/data_structures/containers/charbuf.cc +++ /dev/null @@ -1,222 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/containers/charbuf.h" - -using hshm::ipc::string; - -void TestCharbuf() { - Allocator *alloc = alloc_g; - - PAGE_DIVIDE("Construct from allocator") { - hshm::charbuf data(256); - memset(data.data(), 0, 256); - REQUIRE(data.size() == 256); - REQUIRE(data.GetAllocator() == alloc); - } - - PAGE_DIVIDE("Construct from malloc") { - char *ptr = (char*)malloc(256); - hshm::charbuf data(ptr, 256); - memset(data.data(), 0, 256); - REQUIRE(data.size() == 256); - REQUIRE(data.GetAllocator() == nullptr); - free(ptr); - } - - PAGE_DIVIDE("Resize null charbuf to higher value") { - hshm::charbuf data; - data.resize(256); - REQUIRE(data.size() == 256); - REQUIRE(data.GetAllocator() == alloc); - } - - PAGE_DIVIDE("Resize null charbuf to 0 value") { - hshm::charbuf data; - data.resize(0); - REQUIRE(data.size() == 0); - REQUIRE(data.GetAllocator() == nullptr); - } - - PAGE_DIVIDE("Resize destructable charbuf to 0 value") { - hshm::charbuf data(8192); - data.resize(0); - REQUIRE(data.size() == 0); - REQUIRE(data.GetAllocator() == alloc); - } - - PAGE_DIVIDE("Resize destructable charbuf to lower value") { - hshm::charbuf data(8192); - data.resize(256); - REQUIRE(data.size() == 256); - REQUIRE(data.GetAllocator() == alloc); - } - - PAGE_DIVIDE("Resize destructable charbuf to higher value") { - hshm::charbuf data(256); - data.resize(8192); - REQUIRE(data.size() == 8192); - REQUIRE(data.GetAllocator() == alloc); - } - - PAGE_DIVIDE("Resize indestructable charbuf to higher value") { - char *ptr = (char*)malloc(256); - hshm::charbuf data(ptr, 256); - data.resize(8192); - REQUIRE(data.size() == 8192); - free(ptr); - } - - PAGE_DIVIDE("Resize indestructable charbuf to lower value") { - char *ptr = (char*)malloc(8192); - hshm::charbuf data(ptr, 8192); - data.resize(256); - REQUIRE(data.size() == 256); - free(ptr); - } - - PAGE_DIVIDE("Move construct from destructable") { - hshm::charbuf data1(8192); - hshm::charbuf data2(std::move(data1)); - REQUIRE(data2.size() == 8192); - } - - PAGE_DIVIDE("Move construct from indestructable") { - char *ptr1 = (char*)malloc(8192); - hshm::charbuf data1(ptr1, 8192); - hshm::charbuf data2(std::move(data1)); - REQUIRE(data2.size() == 8192); - free(ptr1); - } - - PAGE_DIVIDE("Move assign between two destructables") { - hshm::charbuf data1(8192); - hshm::charbuf data2(512); - data1 = std::move(data2); - REQUIRE(data1.size() == 512); - } - - PAGE_DIVIDE("Move assign between two indestructables") { - char *ptr1 = (char*)malloc(8192); - hshm::charbuf data1(ptr1, 8192); - char *ptr2 = (char*)malloc(512); - hshm::charbuf data2(ptr2, 512); - data1 = std::move(data2); - REQUIRE(data1.size() == 512); - free(ptr1); - free(ptr2); - } - - PAGE_DIVIDE("Move assign indestructable -> destructable") { - hshm::charbuf data1(8192); - char *ptr2 = (char*)malloc(512); - hshm::charbuf data2(ptr2, 512); - data1 = std::move(data2); - REQUIRE(data1.size() == 512); - free(ptr2); - } - - PAGE_DIVIDE("Move assign destructable -> indestructable") { - char *ptr1 = (char*)malloc(8192); - hshm::charbuf data1(ptr1, 8192); - hshm::charbuf data2(512); - data1 = std::move(data2); - REQUIRE(data1.size() == 512); - free(ptr1); - } - - PAGE_DIVIDE("Move assign to null") { - hshm::charbuf data1; - hshm::charbuf data2(512); - data1 = std::move(data2); - REQUIRE(data1.size() == 512); - } - - PAGE_DIVIDE("Copy construct from destructable") { - hshm::charbuf data1(8192); - hshm::charbuf data2(data1); - REQUIRE(data1.size() == 8192); - REQUIRE(data2.size() == 8192); - REQUIRE(data1 == data2); - } - - PAGE_DIVIDE("Copy construct from indestructable") { - char *ptr1 = (char*)malloc(8192); - hshm::charbuf data1(ptr1, 8192); - hshm::charbuf data2(data1); - REQUIRE(data1.size() == 8192); - REQUIRE(data2.size() == 8192); - REQUIRE(data1 == data2); - free(ptr1); - } - - PAGE_DIVIDE("Copy assign between two destructables") { - hshm::charbuf data1(8192); - hshm::charbuf data2(512); - data1 = data2; - REQUIRE(data2.size() == 512); - REQUIRE(data1.size() == 512); - REQUIRE(data1 == data2); - } - - PAGE_DIVIDE("Copy assign between two indestructables") { - char *ptr1 = (char*)malloc(8192); - hshm::charbuf data1(ptr1, 8192); - char *ptr2 = (char*)malloc(512); - hshm::charbuf data2(ptr2, 512); - data1 = data2; - REQUIRE(data2.size() == 512); - REQUIRE(data1.size() == 512); - REQUIRE(data1 == data2); - free(ptr1); - free(ptr2); - } - - PAGE_DIVIDE("Copy assign indestructable -> destructable") { - hshm::charbuf data1(8192); - char *ptr2 = (char*)malloc(512); - hshm::charbuf data2(ptr2, 512); - data1 = data2; - REQUIRE(data2.size() == 512); - REQUIRE(data1.size() == 512); - free(ptr2); - } - - PAGE_DIVIDE("Copy assign destructable -> indestructable") { - char *ptr1 = (char*)malloc(8192); - hshm::charbuf data1(ptr1, 8192); - hshm::charbuf data2(512); - data1 = data2; - REQUIRE(data2.size() == 512); - REQUIRE(data1.size() == 512); - free(ptr1); - } - - PAGE_DIVIDE("Copy assign to null") { - char *ptr1 = (char*)malloc(8192); - hshm::charbuf data1; - hshm::charbuf data2(512); - data1 = data2; - REQUIRE(data2.size() == 512); - REQUIRE(data1.size() == 512); - free(ptr1); - } -} - -TEST_CASE("Charbuf") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - TestCharbuf(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/iqueue.cc b/hermes_shm/test/unit/data_structures/containers/iqueue.cc deleted file mode 100644 index a98672b02..000000000 --- a/hermes_shm/test/unit/data_structures/containers/iqueue.cc +++ /dev/null @@ -1,42 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "iqueue.h" -#include "hermes_shm/data_structures/ipc/iqueue.h" -#include "hermes_shm/data_structures/smart_ptr/smart_ptr_base.h" - -using hshm::ipc::mptr; -using hshm::ipc::make_mptr; -using hshm::ipc::iqueue; - -template -void IqueueTest() { - Allocator *alloc = alloc_g; - auto lp = hipc::make_uptr>(alloc); - IqueueTestSuite> test(*lp, alloc); - - test.EnqueueTest(30); - test.ForwardIteratorTest(); - test.ConstForwardIteratorTest(); - test.DequeueTest(30); - test.DequeueMiddleTest(); - test.EraseTest(); -} - -TEST_CASE("IqueueOfMpPage") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - IqueueTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/iqueue.h b/hermes_shm/test/unit/data_structures/containers/iqueue.h deleted file mode 100644 index 25e71a395..000000000 --- a/hermes_shm/test/unit/data_structures/containers/iqueue.h +++ /dev/null @@ -1,108 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_IQUEUE_H_ -#define HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_IQUEUE_H_ - -#include "basic_test.h" -#include "test_init.h" -#include - -using hipc::MpPage; - -template -class IqueueTestSuite { - public: - Container &obj_; - Allocator *alloc_; - - /// Constructor - IqueueTestSuite(Container &obj, Allocator *alloc) - : obj_(obj), alloc_(alloc) {} - - /// Enqueue elements - void EnqueueTest(size_t count = 30) { - for (size_t i = 0; i < count; ++i) { - hipc::OffsetPointer p; - auto page = alloc_->template - AllocateConstructObjs(1, p); - page->page_size_ = count - i - 1; - obj_.enqueue(page); - } - REQUIRE(obj_.size() == count); - } - - /// Dequeue and then re-enqueue - void DequeueTest(size_t count = 30) { - std::vector tmp(count); - for (size_t i = 0; i < count; ++i) { - tmp[i] = obj_.dequeue(); - } - for (int i = count - 1; i >= 0; --i) { - obj_.enqueue(tmp[i]); - } - REQUIRE(obj_.size() == count); - ForwardIteratorTest(count); - } - - /// Forward iterator - void ForwardIteratorTest(size_t count = 30) { - size_t fcur = 0; - for (T *page : obj_) { - REQUIRE(page->page_size_ == fcur); - ++fcur; - } - } - - /// Constant Forward iterator - void ConstForwardIteratorTest(size_t count = 30) { - const Container &obj = obj_; - size_t fcur = 0; - for (auto iter = obj.cbegin(); iter != obj.cend(); ++iter) { - T *page = *iter; - REQUIRE(page->page_size_ == fcur); - ++fcur; - } - } - - /// Dequeue an element in the middle of the queue - void DequeueMiddleTest() { - size_t count = obj_.size(); - size_t mid = obj_.size() / 2; - auto iter = obj_.begin(); - for (size_t i = 0; i < mid; ++i) { - ++iter; - } - auto page = obj_.dequeue(iter); - REQUIRE(page->page_size_ == mid); - REQUIRE(obj_.size() == count - 1); - for (T *page : obj_) { - REQUIRE(page->page_size_ != mid); - } - obj_.enqueue(page); - } - - /// Verify erase - void EraseTest() { - std::vector tmp; - for (T *page : obj_) { - tmp.emplace_back(page); - } - obj_.clear(); - for (T *page : tmp) { - alloc_->FreePtr(page); - } - REQUIRE(obj_.size() == 0); - } -}; - -#endif // HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_IQUEUE_H_ diff --git a/hermes_shm/test/unit/data_structures/containers/list.cc b/hermes_shm/test/unit/data_structures/containers/list.cc deleted file mode 100644 index fde1283d8..000000000 --- a/hermes_shm/test/unit/data_structures/containers/list.cc +++ /dev/null @@ -1,63 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "list.h" -#include "hermes_shm/data_structures/ipc/list.h" -#include "hermes_shm/data_structures/ipc/string.h" - -using hshm::ipc::list; - -template -void ListTestRunner(ListTestSuite> &test) { - test.EmplaceTest(15); - test.ForwardIteratorTest(); - test.ConstForwardIteratorTest(); - test.CopyConstructorTest(); - test.CopyAssignmentTest(); - test.MoveConstructorTest(); - test.MoveAssignmentTest(); - test.EmplaceFrontTest(); - test.ModifyEntryCopyIntoTest(); - test.ModifyEntryMoveIntoTest(); - test.EraseTest(); -} - -template -void ListTest() { - Allocator *alloc = alloc_g; - auto lp = hipc::make_uptr>(alloc); - ListTestSuite> test(*lp, alloc); - ListTestRunner(test); -} - -TEST_CASE("ListOfInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ListTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("ListOfString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ListTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("ListOfStdString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ListTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/list.h b/hermes_shm/test/unit/data_structures/containers/list.h deleted file mode 100644 index 13dd93ad1..000000000 --- a/hermes_shm/test/unit/data_structures/containers/list.h +++ /dev/null @@ -1,192 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_LIST_H_ -#define HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_LIST_H_ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" - -template -class ListTestSuite { - public: - Container &obj_; - Allocator *alloc_; - - /// Constructor - ListTestSuite(Container &obj, Allocator *alloc) - : obj_(obj), alloc_(alloc) {} - - /// Emplace elements - void EmplaceTest(size_t count = 30) { - for (size_t i = 0; i < count; ++i) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, var, i); - obj_.emplace_back(var); - } - REQUIRE(obj_.size() == count); - } - - /// Forward iterator - void ForwardIteratorTest(size_t count = 30) { - size_t fcur = 0; - for (auto &num : obj_) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, fcur_conv, fcur); - REQUIRE(num == fcur_conv); - ++fcur; - } - } - - /// Constant Forward iterator - void ConstForwardIteratorTest(size_t count = 30) { - const Container &obj = obj_; - size_t fcur = 0; - for (auto iter = obj.cbegin(); iter != obj.cend(); ++iter) { - T& num = *iter; - CREATE_SET_VAR_TO_INT_OR_STRING(T, fcur_conv, fcur); - REQUIRE(num == fcur_conv); - ++fcur; - } - } - - /// Copy constructor - void CopyConstructorTest() { - size_t count = obj_.size(); - auto cpy = hipc::make_uptr(obj_); - VerifyCopy(obj_, *cpy, count); - } - - /// Copy assignment - void CopyAssignmentTest() { - size_t count = obj_.size(); - auto cpy = hipc::make_uptr(); - *cpy = obj_; - VerifyCopy(obj_, *cpy, count); - } - - /// Move constructor - void MoveConstructorTest() { - size_t count = obj_.size(); - auto cpy = hipc::make_uptr(std::move(obj_)); - VerifyMove(obj_, *cpy, count); - obj_ = std::move(*cpy); - VerifyMove(*cpy, obj_, count); - } - - /// Move assignment - void MoveAssignmentTest() { - size_t count = obj_.size(); - auto cpy = hipc::make_uptr(); - (*cpy) = std::move(obj_); - VerifyMove(obj_, *cpy, count); - obj_ = std::move(*cpy); - VerifyMove(*cpy, obj_, count); - } - - /// Emplace and erase front - void EmplaceFrontTest() { - CREATE_SET_VAR_TO_INT_OR_STRING(T, i0, 100); - size_t old_size = obj_.size(); - obj_.emplace_front(i0); - REQUIRE(obj_.front() == i0); - REQUIRE(obj_.size() == old_size + 1); - obj_.erase(obj_.begin(), obj_.begin() + 1); - } - - /// Copy an object into the container - void ModifyEntryCopyIntoTest() { - // Modify the fourth list entry - { - CREATE_SET_VAR_TO_INT_OR_STRING(T, i4, 25); - auto iter = obj_.begin() + 4; - (*iter) = i4; - } - - // Verify the modification took place - { - CREATE_SET_VAR_TO_INT_OR_STRING(T, i4, 25); - auto iter = obj_.begin() + 4; - REQUIRE((*iter) == i4); - } - } - - /// Move an object into the container - void ModifyEntryMoveIntoTest() { - // Modify the fourth list entry - { - CREATE_SET_VAR_TO_INT_OR_STRING(T, i4, 25); - auto iter = obj_.begin() + 4; - (*iter) = std::move(i4); - } - - // Verify the modification took place - { - CREATE_SET_VAR_TO_INT_OR_STRING(T, i4, 25); - auto iter = obj_.begin() + 4; - REQUIRE((*iter) == i4); - } - } - - /// Verify erase - void EraseTest() { - obj_.clear(); - REQUIRE(obj_.size() == 0); - } - - private: - /// Verify copy construct/assign worked - void VerifyCopy(Container &obj, - Container &cpy, - size_t count) { - REQUIRE(obj_.size() == count); - REQUIRE(cpy.size() == count); - - // Verify obj - { - size_t fcur = 0; - for (auto &num : obj_) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, fcur_conv, fcur); - REQUIRE(num == fcur_conv); - ++fcur; - } - } - - // Verify copy - { - size_t fcur = 0; - for (auto &num : cpy) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, fcur_conv, fcur); - REQUIRE(num == fcur_conv); - ++fcur; - } - } - } - - /// Verify move worked - void VerifyMove(Container &orig_obj, - Container &new_obj, - size_t count) { - // Verify move into cpy worked - { - size_t fcur = 0; - REQUIRE(orig_obj.IsNull()); - REQUIRE(new_obj.size() == count); - for (auto &num : new_obj) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, fcur_conv, fcur); - REQUIRE(num == fcur_conv); - ++fcur; - } - } - } -}; - -#endif // HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_LIST_H_ diff --git a/hermes_shm/test/unit/data_structures/containers/manual_ptr.cc b/hermes_shm/test/unit/data_structures/containers/manual_ptr.cc deleted file mode 100644 index f1b956df2..000000000 --- a/hermes_shm/test/unit/data_structures/containers/manual_ptr.cc +++ /dev/null @@ -1,55 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "hermes_shm/data_structures/smart_ptr/smart_ptr_base.h" -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "smart_ptr.h" - -using hshm::ipc::string; -using hshm::ipc::mptr; -using hshm::ipc::uptr; -using hshm::ipc::mptr; -using hshm::ipc::make_mptr; -using hshm::ipc::TypedPointer; - -template -void ManualPtrTest() { - CREATE_SET_VAR_TO_INT_OR_STRING(T, num, 25); - auto ptr = hipc::make_mptr(num); - hipc::mptr ptr2; - hipc::SmartPtrTestSuite> test(ptr, ptr2); - test.DereferenceTest(num); - test.MoveConstructorTest(num); - test.MoveAssignmentTest(num); - test.CopyConstructorTest(num); - test.CopyAssignmentTest(num); - test.SerializeationConstructorTest(num); - test.SerializeationOperatorTest(num); - ptr.shm_destroy(); -} - -TEST_CASE("ManualPtrOfInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ManualPtrTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("ManualPtrOfString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ManualPtrTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc b/hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc deleted file mode 100644 index 30d9c8d66..000000000 --- a/hermes_shm/test/unit/data_structures/containers/mpsc_queue.cc +++ /dev/null @@ -1,50 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/mpsc_queue.h" -#include "queue.h" - -/** - * TEST MPSC QUEUE - * */ - -TEST_CASE("TestMpscQueueInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(1, 1, 32, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestMpscQueueString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, hipc::string>( - 1, 1, 32, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestMpscQueueIntMultiThreaded") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceAndConsume, int>(8, 1, 8192, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestMpscQueueStringMultiThreaded") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceAndConsume, hipc::string>( - 8, 1, 8192, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/pair.cc b/hermes_shm/test/unit/data_structures/containers/pair.cc deleted file mode 100644 index f6be33246..000000000 --- a/hermes_shm/test/unit/data_structures/containers/pair.cc +++ /dev/null @@ -1,95 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/pair.h" -#include "hermes_shm/data_structures/ipc/string.h" - -template -void PairTest() { - Allocator *alloc = alloc_g; - - // Construct test - PAGE_DIVIDE("Construct") { - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - auto data = hipc::make_uptr>( - alloc, first, second); - REQUIRE(*data->first_ == first); - REQUIRE(*data->second_ == second); - } - - // Copy constructor test - PAGE_DIVIDE("Copy constructor") { - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - auto data = hipc::make_uptr>( - alloc, first, second); - auto cpy = hipc::make_uptr>( - alloc, *data); - REQUIRE(*cpy->first_ == first); - REQUIRE(*cpy->second_ == second); - } - - // Copy assignment test - PAGE_DIVIDE("Copy assignment operator") { - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - auto data = hipc::make_uptr>( - alloc, first, second); - auto cpy = hipc::make_uptr>( - alloc); - *cpy = *data; - REQUIRE(*cpy->first_ == first); - REQUIRE(*cpy->second_ == second); - } - - // Move constructor test - PAGE_DIVIDE("Move constructor") { - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - auto data = hipc::make_uptr>( - alloc, first, second); - auto cpy = hipc::make_uptr>( - alloc, std::move(*data)); - REQUIRE(*cpy->first_ == first); - REQUIRE(*cpy->second_ == second); - } - - // Move assignment test - PAGE_DIVIDE("Move assignment operator") { - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - auto data = hipc::make_uptr>( - alloc, first, second); - auto cpy = hipc::make_uptr>( - alloc); - *cpy = std::move(*data); - REQUIRE(*cpy->first_ == first); - REQUIRE(*cpy->second_ == second); - } -} - -TEST_CASE("PairOfIntInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - PairTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("PairOfIntString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - PairTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/queue.h b/hermes_shm/test/unit/data_structures/containers/queue.h deleted file mode 100644 index 6eafd8f15..000000000 --- a/hermes_shm/test/unit/data_structures/containers/queue.h +++ /dev/null @@ -1,149 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_QUEUE_H_ -#define HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_QUEUE_H_ - -#include "hermes_shm/data_structures/data_structure.h" -#include "omp.h" -#include "hermes_shm/data_structures/ipc/ticket_stack.h" - -template -class QueueTestSuite { - public: - hipc::uptr &queue_; - - public: - /** Constructor */ - explicit QueueTestSuite(hipc::uptr &queue) - : queue_(queue) {} - - /** Producer method */ - void Produce(size_t count_per_rank) { - std::vector idxs; - int rank = omp_get_thread_num(); - try { - for (size_t i = 0; i < count_per_rank; ++i) { - size_t idx = rank * count_per_rank + i; - CREATE_SET_VAR_TO_INT_OR_STRING(T, var, idx); - CREATE_GET_INT_FROM_VAR(T, entry_int, var) - idxs.emplace_back(entry_int); - while (queue_->emplace(var).IsNull()) {} - } - } catch (hshm::Error &e) { - HELOG(kFatal, e.what()) - } - REQUIRE(idxs.size() == count_per_rank); - std::sort(idxs.begin(), idxs.end()); - for (size_t i = 0; i < count_per_rank; ++i) { - size_t idx = rank * count_per_rank + i; - REQUIRE(idxs[i] == idx); - } - } - - /** Consumer method */ - void Consume(std::atomic &count, - size_t total_count, - std::vector &entries) { - auto entry = hipc::make_uptr(); - auto &entry_ref = *entry; - - // Consume everything - while (count < total_count) { - auto qtok = queue_->pop(entry_ref); - if (qtok.IsNull()) { - continue; - } - CREATE_GET_INT_FROM_VAR(T, entry_int, entry_ref) - size_t off = count.fetch_add(1); - if (off >= total_count) { - break; - } - entries[off] = entry_int; - } - - int rank = omp_get_thread_num(); - if (rank == 0) { - // Ensure there's no data left in the queue - REQUIRE(queue_->pop(entry_ref).IsNull()); - - // Ensure the data is all correct - REQUIRE(entries.size() == total_count); - std::sort(entries.begin(), entries.end()); - REQUIRE(entries.size() == total_count); - for (size_t i = 0; i < total_count; ++i) { - REQUIRE(entries[i] == i); - } - } - } -}; - -template -void ProduceThenConsume(size_t nproducers, - size_t nconsumers, - size_t count_per_rank, - size_t depth) { - auto queue = hipc::make_uptr(depth); - QueueTestSuite q(queue); - std::atomic count = 0; - std::vector entries; - entries.resize(count_per_rank * nproducers); - - // Produce all the data - omp_set_dynamic(0); -#pragma omp parallel shared(nproducers, count_per_rank, q, count, entries) num_threads(nproducers) // NOLINT - { // NOLINT -#pragma omp barrier - q.Produce(count_per_rank); -#pragma omp barrier - } - - omp_set_dynamic(0); -#pragma omp parallel shared(nproducers, count_per_rank, q) num_threads(nconsumers) // NOLINT - { // NOLINT -#pragma omp barrier - // Consume all the data - q.Consume(count, count_per_rank * nproducers, entries); -#pragma omp barrier - } -} - -template -void ProduceAndConsume(size_t nproducers, - size_t nconsumers, - size_t count_per_rank, - size_t depth) { - auto queue = hipc::make_uptr(depth); - size_t nthreads = nproducers + nconsumers; - QueueTestSuite q(queue); - std::atomic count = 0; - std::vector entries; - entries.resize(count_per_rank * nproducers); - - // Produce all the data - omp_set_dynamic(0); -#pragma omp parallel shared(nproducers, count_per_rank, q, count) num_threads(nthreads) // NOLINT - { // NOLINT -#pragma omp barrier - size_t rank = omp_get_thread_num(); - if (rank < nproducers) { - // Producer - q.Produce(count_per_rank); - } else { - // Consumer - q.Consume(count, count_per_rank * nproducers, entries); - } -#pragma omp barrier - } -} - -#endif // HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_QUEUE_H_ diff --git a/hermes_shm/test/unit/data_structures/containers/slist.cc b/hermes_shm/test/unit/data_structures/containers/slist.cc deleted file mode 100644 index ac6322964..000000000 --- a/hermes_shm/test/unit/data_structures/containers/slist.cc +++ /dev/null @@ -1,59 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "list.h" -#include "hermes_shm/data_structures/ipc/slist.h" -#include "hermes_shm/data_structures/ipc/string.h" - -using hshm::ipc::slist; - -template -void SlistTest() { - Allocator *alloc = alloc_g; - auto lp = hipc::make_uptr>(alloc); - ListTestSuite> test(*lp, alloc); - - test.EmplaceTest(30); - test.ForwardIteratorTest(); - test.ConstForwardIteratorTest(); - test.CopyConstructorTest(); - test.CopyAssignmentTest(); - test.MoveConstructorTest(); - test.MoveAssignmentTest(); - test.EmplaceFrontTest(); - test.ModifyEntryCopyIntoTest(); - test.ModifyEntryMoveIntoTest(); - test.EraseTest(); -} - -TEST_CASE("SlistOfInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - SlistTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("SlistOfString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - SlistTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("SlistOfStdString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - SlistTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/smart_ptr.h b/hermes_shm/test/unit/data_structures/containers/smart_ptr.h deleted file mode 100644 index cc1253ab0..000000000 --- a/hermes_shm/test/unit/data_structures/containers/smart_ptr.h +++ /dev/null @@ -1,86 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TEST_UNIT_ptr__STRUCTURES_CONTAINERS_SMART_PTR_H_ -#define HERMES_TEST_UNIT_ptr__STRUCTURES_CONTAINERS_SMART_PTR_H_ - -#include "basic_test.h" -#include "test_init.h" - -namespace hshm::ipc { - -template -class SmartPtrTestSuite { - public: - PointerT &ptr_; /**< Allocated. Used for all tests. */ - PointerT &ptr2_; /**< Unallocated. Used for move tests. */ - - explicit SmartPtrTestSuite(PointerT &ptr, - PointerT &ptr2) : ptr_(ptr), ptr2_(ptr2) {} - - public: - // Test dereference - void DereferenceTest(T &num) { - REQUIRE(*ptr_ == num); - } - - // Test move constructor - void MoveConstructorTest(T &num) { - PointerT ptr2(std::move(ptr_)); - // REQUIRE(ptr_.IsNull()); - REQUIRE(std::hash{}(ptr2) == std::hash{}(num)); - ptr_ = std::move(ptr2); - } - - // Test move assignment operator - void MoveAssignmentTest(T &num) { - ptr2_ = std::move(ptr_); - // REQUIRE(ptr_.IsNull()); - REQUIRE(std::hash{}(ptr2_) == std::hash{}(num)); - ptr_ = std::move(ptr2_); - } - - // Test copy constructor - void CopyConstructorTest(T &num) { - PointerT ptr2(ptr_); - REQUIRE(*ptr_ == num); - REQUIRE(*ptr2 == num); - } - - // Test copy assignment - void CopyAssignmentTest(T &num) { - PointerT ptr2 = ptr_; - REQUIRE(*ptr_ == num); - REQUIRE(*ptr2 == num); - } - - // Test serialization + deserialization (constructor) - void SerializeationConstructorTest(T &num) { - TypedPointer ar; - ptr_ >> ar; - PointerT from_ar(ar); - REQUIRE(*from_ar == num); - } - - // Test serialization + deserialization (operator) - void SerializeationOperatorTest(T &num) { - TypedPointer ar; - ptr_ >> ar; - PointerT from_ar; - from_ar << ar; - REQUIRE(*from_ar == num); - } -}; - -} // namespace hshm::ipc - -#endif // HERMES_TEST_UNIT_ptr__STRUCTURES_CONTAINERS_SMART_PTR_H_ diff --git a/hermes_shm/test/unit/data_structures/containers/spsc_queue.cc b/hermes_shm/test/unit/data_structures/containers/spsc_queue.cc deleted file mode 100644 index cc665bd0c..000000000 --- a/hermes_shm/test/unit/data_structures/containers/spsc_queue.cc +++ /dev/null @@ -1,35 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/spsc_queue.h" -#include "queue.h" - -/** - * TEST SPSC QUEUE - * */ - -TEST_CASE("TestSpscQueueInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(1, 1, 32, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestSpscQueueString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, hipc::string>( - 1, 1, 32, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/string.cc b/hermes_shm/test/unit/data_structures/containers/string.cc deleted file mode 100644 index e82dab5c5..000000000 --- a/hermes_shm/test/unit/data_structures/containers/string.cc +++ /dev/null @@ -1,78 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" - -using hshm::ipc::string; - -void TestString() { - Allocator *alloc = alloc_g; - - PAGE_DIVIDE("Test construction from const char*") { - auto text = hipc::make_uptr(alloc, "hello1"); - REQUIRE(*text == "hello1"); - REQUIRE(*text != "h"); - REQUIRE(*text != "asdfklaf"); - } - - PAGE_DIVIDE("Test construction from std::string") { - auto text = hipc::make_uptr(alloc, std::string("hello1")); - REQUIRE(*text == "hello1"); - REQUIRE(*text != "h"); - REQUIRE(*text != "asdfklaf"); - } - - PAGE_DIVIDE("Test the mutability of the string") { - auto text = hipc::make_uptr(alloc, 6); - memcpy(text->data(), "hello4", strlen("hello4")); - REQUIRE(*text == "hello4"); - } - - PAGE_DIVIDE("Test copy assign from hipc::string") { - auto text1 = hipc::make_uptr(alloc, "hello"); - auto text2 = hipc::make_uptr(alloc); - (*text2) = (*text1); - REQUIRE(*text1 == "hello"); - } - - PAGE_DIVIDE("Test copy assign from std::string") { - auto text1 = hipc::make_uptr(alloc, "hello"); - (*text1) = std::string("hello2"); - REQUIRE(*text1 == "hello2"); - } - - PAGE_DIVIDE("Test move assign from hipc::string") { - auto text1 = hipc::make_uptr(alloc, "hello"); - auto text2 = hipc::make_uptr(alloc); - (*text2) = std::move(*text1); - REQUIRE(*text2 == "hello"); - } - - PAGE_DIVIDE("Move from a string. Re-assign moved string.") { - auto text1 = hipc::make_uptr(alloc, "hello"); - auto text2 = hipc::make_uptr(alloc); - (*text2) = std::move(*text1); - (*text1) = "hello2"; - REQUIRE(*text2 == "hello"); - REQUIRE(*text1 == "hello2"); - } -} - -TEST_CASE("String") { - Allocator *alloc = alloc_g; - REQUIRE(IS_SHM_ARCHIVEABLE(string)); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - TestString(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/test_init.cc b/hermes_shm/test/unit/data_structures/containers/test_init.cc deleted file mode 100644 index be6a85f13..000000000 --- a/hermes_shm/test/unit/data_structures/containers/test_init.cc +++ /dev/null @@ -1,31 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "basic_test.h" -#include "test_init.h" -#include - -Allocator *alloc_g = nullptr; - -void Posttest() { - std::string shm_url = "test_allocators"; - alloc_g = nullptr; -} - -void MainPretest() { - Pretest(); -} - -void MainPosttest() { - Posttest(); -} diff --git a/hermes_shm/test/unit/data_structures/containers/test_init.h b/hermes_shm/test/unit/data_structures/containers/test_init.h deleted file mode 100644 index 155cf9375..000000000 --- a/hermes_shm/test/unit/data_structures/containers/test_init.h +++ /dev/null @@ -1,52 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ -#define HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ - -#include "hermes_shm/data_structures/data_structure.h" - -using hshm::ipc::PosixShmMmap; -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::Pointer; - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -extern Allocator *alloc_g; - -template -void Pretest() { - std::string shm_url = "test_allocators"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator(shm_url, alloc_id, 0); - alloc_g = mem_mngr->GetAllocator(alloc_id); -} - -void Posttest(); - -#endif // HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ diff --git a/hermes_shm/test/unit/data_structures/containers/ticket_queue.cc b/hermes_shm/test/unit/data_structures/containers/ticket_queue.cc deleted file mode 100644 index b5c3e96c6..000000000 --- a/hermes_shm/test/unit/data_structures/containers/ticket_queue.cc +++ /dev/null @@ -1,68 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/ticket_queue.h" -#include "hermes_shm/data_structures/ipc/ticket_stack.h" -#include "hermes_shm/data_structures/ipc/split_ticket_queue.h" -#include "queue.h" - -/** - * TEST TICKET QUEUE - * */ - -TEST_CASE("TestTicketStackInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(1, 1, 32, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestTicketStackIntMultiThreaded") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(8, 1, 8192, 8192 * 8); - ProduceAndConsume, int>(8, 1, 8192, 64); - ProduceAndConsume, int>(8, 8, 8192, 64); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestTicketQueueInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(1, 1, 32, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestTicketQueueIntMultiThreaded") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceAndConsume, int>(8, 1, 8192, 64); - ProduceAndConsume, int>(8, 8, 8192, 64); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestSplitTicketQueueInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceThenConsume, int>(1, 1, 32, 32); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TestSplitTicketQueueIntMultiThreaded") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - ProduceAndConsume, int>(8, 1, 8192, 64); - ProduceAndConsume, int>(8, 8, 8192, 64); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/tuple.cc b/hermes_shm/test/unit/data_structures/containers/tuple.cc deleted file mode 100644 index 864583288..000000000 --- a/hermes_shm/test/unit/data_structures/containers/tuple.cc +++ /dev/null @@ -1,82 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" - -template -void TupleTest() { - Allocator *alloc = alloc_g; - - // Construct test - /*{ - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - hipc::ShmHeader> hdr; - hipc::ShmStruct - data(hdr, alloc, - hshm::make_argpack(first), - hshm::make_argpack(second)); - REQUIRE(data.template Get<0>() == first); - REQUIRE(data.template Get<1>() == second); - } - - // Copy test - { - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - hipc::tuple data(alloc, first, second); - hipc::tuple cpy(data); - REQUIRE(cpy.template Get<0>() == first); - REQUIRE(cpy.template Get<1>() == second); - } - - // Move test - { - CREATE_SET_VAR_TO_INT_OR_STRING(FirstT, first, 124); - CREATE_SET_VAR_TO_INT_OR_STRING(SecondT, second, 130); - hipc::tuple data(alloc, first, second); - hipc::tuple cpy(std::move(data)); - REQUIRE(cpy.template Get<0>() == first); - REQUIRE(cpy.template Get<1>() == second); - }*/ -} - -#include - -int y() { - return 0; -} - - -class Y { - public: - Y() = default; - explicit Y(int x) {} - explicit Y(int x, int w) {} -}; - -TEST_CASE("TupleOfIntInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - - TupleTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("TupleOfIntString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - // TupleTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/unique_ptr.cc b/hermes_shm/test/unit/data_structures/containers/unique_ptr.cc deleted file mode 100644 index 5515c0b59..000000000 --- a/hermes_shm/test/unit/data_structures/containers/unique_ptr.cc +++ /dev/null @@ -1,50 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "hermes_shm/data_structures/smart_ptr/smart_ptr_base.h" -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "smart_ptr.h" - -using hshm::ipc::string; -using hshm::ipc::uptr; -using hshm::ipc::mptr; -using hshm::ipc::TypedPointer; - -template -void UniquePtrTest() { - CREATE_SET_VAR_TO_INT_OR_STRING(T, num, 25); - auto ptr = hipc::make_uptr(num); - hipc::uptr ptr2; - hipc::SmartPtrTestSuite> test(ptr, ptr2); - test.DereferenceTest(num); - test.MoveConstructorTest(num); - test.MoveAssignmentTest(num); - test.SerializeationConstructorTest(num); - test.SerializeationOperatorTest(num); -} - -TEST_CASE("UniquePtrOfInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - UniquePtrTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("UniquePtrOfString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - UniquePtrTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/unordered_map.cc b/hermes_shm/test/unit/data_structures/containers/unordered_map.cc deleted file mode 100644 index 32f6ff7e1..000000000 --- a/hermes_shm/test/unit/data_structures/containers/unordered_map.cc +++ /dev/null @@ -1,249 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/unordered_map.h" -#include "hermes_shm/data_structures/ipc/string.h" - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; -using hshm::ipc::unordered_map; -using hshm::ipc::string; - -#define GET_INT_FROM_KEY(VAR) CREATE_GET_INT_FROM_VAR(Key, key_ret, VAR) -#define GET_INT_FROM_VAL(VAR) CREATE_GET_INT_FROM_VAR(Val, val_ret, VAR) - -#define CREATE_KV_PAIR(KEY_NAME, KEY, VAL_NAME, VAL)\ - CREATE_SET_VAR_TO_INT_OR_STRING(Key, KEY_NAME, KEY); \ - CREATE_SET_VAR_TO_INT_OR_STRING(Val, VAL_NAME, VAL); - -template -void UnorderedMapOpTest() { - Allocator *alloc = alloc_g; - auto map_p = hipc::make_uptr>(alloc, 5); - auto &map = *map_p; - - // Insert 20 entries into the map (no growth trigger) - PAGE_DIVIDE("Insert entries") { - for (int i = 0; i < 20; ++i) { - CREATE_KV_PAIR(key, i, val, i); - map.emplace(key, val); - } - } - - // Iterate over the map - PAGE_DIVIDE("Forward iterate") { - std::vector keys, vals; - for (auto &entry : map) { - GET_INT_FROM_KEY(entry.GetKey()); - GET_INT_FROM_VAL(entry.GetVal()); - keys.emplace_back(key_ret); - vals.emplace_back(val_ret); - } - REQUIRE(keys.size() == 20); - REQUIRE(vals.size() == 20); - std::sort(keys.begin(), keys.end()); - std::sort(vals.begin(), vals.end()); - for (int i = 0; i < 20; ++i) { - REQUIRE(keys[i] == i); - REQUIRE(vals[i] == i); - } - } - - // Check if the 20 entries are indexable - PAGE_DIVIDE("Check if entries are indexable") { - for (int i = 0; i < 20; ++i) { - CREATE_KV_PAIR(key, i, val, i); - REQUIRE((map[key]) == val); - } - } - - // Check if 20 entries are findable - PAGE_DIVIDE("Check if entries are findable") { - for (int i = 0; i < 20; ++i) { - CREATE_KV_PAIR(key, i, val, i); - auto iter = map.find(key); - hipc::pair &pair = *iter; - REQUIRE(pair.GetVal() == val); - } - } - - // Re-emplace elements (adding 100 to i) - PAGE_DIVIDE("Re-emplace elements") { - for (int i = 0; i < 20; ++i) { - CREATE_KV_PAIR(key, i, val, i + 100); - map.emplace(key, val); - REQUIRE((map[key]) == val); - } - } - - // Modify the fourth map entry (move assignment) - PAGE_DIVIDE("Modify the fourth map entry") { - CREATE_KV_PAIR(key, 4, val, 25); - auto iter = map.find(key); - hipc::pair& pair = *iter; - pair.GetVal() = val; - REQUIRE(pair.GetVal() == val); - } - - // Verify the modification took place - PAGE_DIVIDE("Verify the modification took place") { - CREATE_KV_PAIR(key, 4, val, 25); - REQUIRE((map[key]) == val); - } - - // Modify the fourth map entry (copy assignment) - PAGE_DIVIDE("Copy assignment test") { - CREATE_KV_PAIR(key, 4, val, 50); - auto iter = map.find(key); - hipc::pair& pair = *iter; - pair.GetVal() = val; - REQUIRE(pair.GetVal() == val); - } - - // Verify the modification took place - PAGE_DIVIDE("Verify the copy assignment held") { - CREATE_KV_PAIR(key, 4, val, 50); - REQUIRE((map[key]) == val); - } - - // Modify the fourth map entry (copy assignment) - PAGE_DIVIDE("Modify the fourth map entry (copy assignment)") { - CREATE_KV_PAIR(key, 4, val, 100); - auto &x = map[key]; - x = val; - } - - // Verify the modification took place - PAGE_DIVIDE("Verify the modification took place") { - CREATE_KV_PAIR(key, 4, val, 100); - REQUIRE(map[key] == val); - } - - // Remove 15 entries from the map - PAGE_DIVIDE("Remove 15 entries from the map") { - for (int i = 0; i < 15; ++i) { - CREATE_KV_PAIR(key, i, val, i); - map.erase(key); - } - REQUIRE(map.size() == 5); - for (int i = 0; i < 15; ++i) { - CREATE_KV_PAIR(key, i, val, i); - REQUIRE(map.find(key) == map.end()); - } - } - - // Attempt to replace an existing key - PAGE_DIVIDE("Try emplace on an existing key") { - for (int i = 15; i < 20; ++i) { - CREATE_KV_PAIR(key, i, val, 100); - REQUIRE(map.try_emplace(key, val) == false); - } - for (int i = 15; i < 20; ++i) { - CREATE_KV_PAIR(key, i, val, 100); - GET_INT_FROM_VAL(map[key]) - REQUIRE(val_ret == i + 100); - } - } - - // Erase the entire map - PAGE_DIVIDE("Erase the entire map") { - map.clear(); - REQUIRE(map.size() == 0); - } - - // Add 100 entries to the map (should force a growth) - PAGE_DIVIDE("Add 100 entries to the map") { - for (int i = 0; i < 100; ++i) { - CREATE_KV_PAIR(key, i, val, i); - map.emplace(key, val); - } - for (int i = 0; i < 100; ++i) { - CREATE_KV_PAIR(key, i, val, i); - auto iter = map.find(key); - REQUIRE(iter != map.end()); - hipc::pair& pair = *iter; - REQUIRE(pair.GetKey() == key); - REQUIRE(pair.GetVal() == val); - } - } - - // Copy assignment operator - PAGE_DIVIDE("Copy the map") { - auto cpy = hipc::make_uptr>(alloc); - (*cpy) = map; - for (int i = 0; i < 100; ++i) { - CREATE_KV_PAIR(key, i, val, i); - auto iter1 = map.find(key); - auto iter2 = cpy->find(key); - REQUIRE(!iter1.is_end()); - hipc::pair& pair1 = *iter1; - REQUIRE(pair1.GetKey() == key); - REQUIRE(pair1.GetVal() == val); - - REQUIRE(!iter2.is_end()); - hipc::pair& pair2 = *iter2; - REQUIRE(pair2.GetKey() == key); - REQUIRE(pair2.GetVal() == val); - } - } - - // Move assignment operator - PAGE_DIVIDE("Move the map") { - auto cpy = hipc::make_uptr>(alloc); - (*cpy) = std::move(map); - for (int i = 0; i < 100; ++i) { - CREATE_KV_PAIR(key, i, val, i); - auto iter = cpy->find(key); - REQUIRE(!iter.is_end()); - hipc::pair& pair = *iter; - REQUIRE(pair.GetKey() == key); - REQUIRE(pair.GetVal() == val); - } - map = std::move(*cpy); - } -} - -TEST_CASE("UnorderedMapOfIntInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - UnorderedMapOpTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("UnorderedMapOfIntString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - UnorderedMapOpTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - - -TEST_CASE("UnorderedMapOfStringInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - UnorderedMapOpTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("UnorderedMapOfStringString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - UnorderedMapOpTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/vector.cc b/hermes_shm/test/unit/data_structures/containers/vector.cc deleted file mode 100644 index bf9f52c12..000000000 --- a/hermes_shm/test/unit/data_structures/containers/vector.cc +++ /dev/null @@ -1,127 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/vector.h" -#include "hermes_shm/data_structures/ipc/list.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "vector.h" - -using hshm::ipc::vector; -using hshm::ipc::list; -using hshm::ipc::string; - -template -void VectorTestRunner(VectorTestSuite> &test) { - test.EmplaceTest(15); - test.IndexTest(); - test.ForwardIteratorTest(); - test.ConstForwardIteratorTest(); - test.CopyConstructorTest(); - test.CopyAssignmentTest(); - test.MoveConstructorTest(); - test.MoveAssignmentTest(); - test.EmplaceFrontTest(); - test.ModifyEntryCopyIntoTest(); - test.ModifyEntryMoveIntoTest(); - test.EraseTest(); -} - -template -void VectorTest() { - Allocator *alloc = alloc_g; - auto vec = hipc::make_uptr>(alloc); - VectorTestSuite> test(*vec, alloc); - VectorTestRunner(test); -} - -void VectorOfVectorOfStringTest() { - Allocator *alloc = alloc_g; - auto vec = hipc::make_uptr< - vector>>(alloc); - - vec->resize(10); - for (vector &bkt : *vec) { - bkt.emplace_back("hello"); - } - vec->clear(); -} - -void VectorOfListOfStringTest() { - Allocator *alloc = alloc_g; - auto vec = hipc::make_uptr>>(alloc); - - vec->resize(10); - - PAGE_DIVIDE("Emplace an element into each bucket") { - size_t count = 0; - for (list &bkt : *vec) { - bkt.emplace_back(std::to_string(count)); - count += 1; - } - REQUIRE(count == 10); - } - - PAGE_DIVIDE("Get string from each bucket") { - size_t count = 0; - for (list &bkt : *vec) { - for (string &val : bkt) { - REQUIRE(val == std::to_string(count)); - } - count += 1; - } - REQUIRE(count == 10); - } - - vec->clear(); -} - -TEST_CASE("VectorOfInt") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - VectorTest(); - VectorTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("VectorOfString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - VectorTest(); - VectorTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("VectorOfStdString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - VectorTest(); - VectorTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("VectorOfVectorOfString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - VectorOfVectorOfStringTest(); - VectorOfVectorOfStringTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} - -TEST_CASE("VectorOfListOfString") { - Allocator *alloc = alloc_g; - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); - VectorOfListOfStringTest(); - VectorOfListOfStringTest(); - REQUIRE(alloc->GetCurrentlyAllocatedSize() == 0); -} diff --git a/hermes_shm/test/unit/data_structures/containers/vector.h b/hermes_shm/test/unit/data_structures/containers/vector.h deleted file mode 100644 index ae9cdad98..000000000 --- a/hermes_shm/test/unit/data_structures/containers/vector.h +++ /dev/null @@ -1,37 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_VECTOR_H_ -#define HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_VECTOR_H_ - -#include "list.h" - -template -class VectorTestSuite : public ListTestSuite { - public: - using ListTestSuite::obj_; - - public: - /// Constructor - VectorTestSuite(Container &obj, Allocator *alloc) - : ListTestSuite(obj, alloc) {} - - /// Test vector index operator - void IndexTest() { - for (size_t i = 0; i < obj_.size(); ++i) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, var, i); - REQUIRE(obj_[i] == var); - } - } -}; - -#endif // HERMES_TEST_UNIT_DATA_STRUCTURES_CONTAINERS_VECTOR_H_ diff --git a/hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt b/hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt deleted file mode 100644 index a18256d28..000000000 --- a/hermes_shm/test/unit/data_structures/containers_mpi/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Test Cases -#------------------------------------------------------------------------------ - -add_executable(test_data_structure_mpi_exec - ${TEST_MAIN}/main_mpi.cc - test_init.cc - list_vec_mpi.cc -) - -add_dependencies(test_data_structure_mpi_exec hermes_shm_data_structures) -target_link_libraries(test_data_structure_mpi_exec - hermes_shm_data_structures Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) - -#------------------------------------------------------------------------------ -# Build Tests -#------------------------------------------------------------------------------ - -# VECTOR TESTS -add_test(NAME test_vector_of_int_mpi COMMAND - mpirun -n 4 ${CMAKE_BINARY_DIR}/bin/test_data_structure_mpi_exec "VectorOfIntMpi") -add_test(NAME test_vector_of_string_mpi COMMAND - mpirun -n 1 ${CMAKE_BINARY_DIR}/bin/test_data_structure_mpi_exec "VectorOfStringMpi") - -# LIST TESTS -add_test(NAME test_list_of_int_mpi COMMAND - mpirun -n 4 ${CMAKE_BINARY_DIR}/bin/test_data_structure_mpi_exec "ListOfIntMpi") -add_test(NAME test_list_of_string_mpi COMMAND - mpirun -n 2 ${CMAKE_BINARY_DIR}/bin/test_data_structure_mpi_exec "ListOfStringMpi") - -message("mpirun -n 1 ${CMAKE_BINARY_DIR}/bin/test_data_structure_mpi_exec ListOfStringMpi") - -#------------------------------------------------------------------------------ -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_data_structure_mpi_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_data_structure_mpi_exec) -endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/containers_mpi/list_vec_mpi.cc b/hermes_shm/test/unit/data_structures/containers_mpi/list_vec_mpi.cc deleted file mode 100644 index e672e63e1..000000000 --- a/hermes_shm/test/unit/data_structures/containers_mpi/list_vec_mpi.cc +++ /dev/null @@ -1,109 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "hermes_shm/data_structures/ipc/list.h" -#include "hermes_shm/data_structures/ipc/vector.h" -#include "hermes_shm/util/error.h" - -template -void ListVecTest(size_t count) { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - Allocator *alloc = alloc_g; - Pointer *header = alloc->GetCustomHeader(); - ContainerT obj; - - try { - if (rank == 0) { - obj.shm_init(alloc); - obj >> (*header); - } - MPI_Barrier(MPI_COMM_WORLD); - obj.shm_deserialize(*header); - MPI_Barrier(MPI_COMM_WORLD); - - // Write 100 objects from rank 0 - { - if (rank == 0) { - for (size_t i = 0; i < count; ++i) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, var, i); - obj.emplace_back(var); - } - } - MPI_Barrier(MPI_COMM_WORLD); - } - - // Read 100 objects from every rank - { - REQUIRE(obj.size() == count); - int i = 0; - for (hipc::Ref var : obj) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, orig, i); - REQUIRE(*var == orig); - ++i; - } - MPI_Barrier(MPI_COMM_WORLD); - } - - // Modify an object in rank 0 - { - if (rank == 0) { - CREATE_SET_VAR_TO_INT_OR_STRING(T, update, count); - hipc::Ref first = *obj.begin(); - (*first) = update; - } - MPI_Barrier(MPI_COMM_WORLD); - } - - // Check if modification received - { - CREATE_SET_VAR_TO_INT_OR_STRING(T, update, count); - hipc::Ref first = *obj.begin(); - REQUIRE((*first) == update); - MPI_Barrier(MPI_COMM_WORLD); - MPI_Barrier(MPI_COMM_WORLD); - } - } catch(HERMES_ERROR_TYPE &HERMES_ERROR_PTR) { - std::cout << "HERE0" << std::endl; - err->print(); - } catch(hshm::Error &err) { - std::cout << "HERE1" << std::endl; - err.print(); - } catch(int err) { - std::cout << "HERE2" << std::endl; - } catch(std::runtime_error &err) { - std::cout << "HERE3" << std::endl; - } catch(std::logic_error &err) { - std::cout << "HERE4" << std::endl; - } catch(...) { - std::cout << "HERE5" << std::endl; - } -} - -TEST_CASE("ListOfIntMpi") { - ListVecTest>(100); -} - -TEST_CASE("ListOfStringMpi") { - ListVecTest>(100); -} - -TEST_CASE("VectorOfIntMpi") { - ListVecTest>(100); -} - -TEST_CASE("VectorOfStringMpi") { - ListVecTest>(100); -} diff --git a/hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc b/hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc deleted file mode 100644 index c4517f13a..000000000 --- a/hermes_shm/test/unit/data_structures/containers_mpi/test_init.cc +++ /dev/null @@ -1,53 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "basic_test.h" -#include "test_init.h" - -Allocator *alloc_g = nullptr; - -template -void PretestRank0() { - std::string shm_url = "test_allocators"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator(shm_url, alloc_id, sizeof(Pointer)); - alloc_g = mem_mngr->GetAllocator(alloc_id); -} - -void PretestRankN() { - std::string shm_url = "test_allocators"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->AttachBackend(MemoryBackendType::kPosixShmMmap, shm_url); - alloc_g = mem_mngr->GetAllocator(alloc_id); -} - -void MainPretest() { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (rank == 0) { - PretestRank0(); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank != 0) { - PretestRankN(); - } -} - -void MainPosttest() { -} diff --git a/hermes_shm/test/unit/data_structures/containers_mpi/test_init.h b/hermes_shm/test/unit/data_structures/containers_mpi/test_init.h deleted file mode 100644 index 5f1b52cd5..000000000 --- a/hermes_shm/test/unit/data_structures/containers_mpi/test_init.h +++ /dev/null @@ -1,40 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#ifndef HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ -#define HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ - -#include "hermes_shm/data_structures/data_structure.h" -#include - -using hshm::ipc::PosixShmMmap; -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::Pointer; - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -extern Allocator *alloc_g; - -void Posttest(); - -#endif // HERMES_TEST_UNIT_DATA_STRUCTURES_TEST_INIT_H_ diff --git a/hermes_shm/test/unit/data_structures/serialize/CMakeLists.txt b/hermes_shm/test/unit/data_structures/serialize/CMakeLists.txt deleted file mode 100644 index 666c25541..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -add_subdirectory(shm) -add_subdirectory(thallium) \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt b/hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt deleted file mode 100644 index 46adc3080..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/shm/CMakeLists.txt +++ /dev/null @@ -1,40 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Test Cases -#------------------------------------------------------------------------------ -set (LIBS - hermes_shm_data_structures - Catch2::Catch2 - MPI::MPI_CXX - OpenMP::OpenMP_CXX) -add_executable(test_shm_exec - ${TEST_MAIN}/main.cc - test_init.cc - test_shm.cc) -add_dependencies(test_shm_exec - hermes_shm_data_structures) -target_link_libraries(test_shm_exec ${LIBS}) - -add_test(NAME test_shm COMMAND ${CMAKE_BINARY_DIR}/bin/test_shm_exec) - -#------------------------------------------------------------------------------ -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_shm_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_shm_exec) -endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc b/hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc deleted file mode 100644 index a0dd32ebc..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/shm/test_init.cc +++ /dev/null @@ -1,32 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "hermes_shm/data_structures/serialization/thallium.h" -#include "hermes_shm/data_structures/containers/charbuf.h" -#include - -void MainPretest() { - std::string shm_url = "test_serializers"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator(shm_url, alloc_id, 0); -} - -void MainPosttest() { -} diff --git a/hermes_shm/test/unit/data_structures/serialize/shm/test_init.h b/hermes_shm/test/unit/data_structures/serialize/shm/test_init.h deleted file mode 100644 index d19ebf96a..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/shm/test_init.h +++ /dev/null @@ -1,35 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_SERIALIZE_THALLIUM_TEST_INIT_H_ -#define HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_SERIALIZE_THALLIUM_TEST_INIT_H_ - -#include "hermes_shm/data_structures/data_structure.h" -#include "hermes_shm/data_structures/serialization/shm_serialize.h" - -using hshm::ipc::PosixShmMmap; -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::Pointer; - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -#endif // HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_SERIALIZE_THALLIUM_TEST_INIT_H_ diff --git a/hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc b/hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc deleted file mode 100644 index aee4add30..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/shm/test_shm.cc +++ /dev/null @@ -1,64 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "hermes_shm/data_structures/data_structure.h" - -TEST_CASE("SerializePod") { - hipc::ShmSerializer istream; - Allocator *alloc = HERMES_MEMORY_MANAGER->GetDefaultAllocator(); - int a = 1; - double b = 2; - float c = 3; - size_t size = sizeof(int) + sizeof(double) + - sizeof(float) + sizeof(allocator_id_t); - REQUIRE(istream.shm_buf_size(alloc->GetId(), a, b, c) == size); - char *buf = istream.serialize(alloc, a, b, c); - - hipc::ShmSerializer ostream; - Allocator *alloc2 = ostream.deserialize(buf); - REQUIRE(alloc == alloc2); - auto a2 = ostream.deserialize(alloc2, buf); - REQUIRE(a2 == a); - auto b2 = ostream.deserialize(alloc2, buf); - REQUIRE(b2 == b); - auto c2 = ostream.deserialize(alloc2, buf); - REQUIRE(c2 == c); -} - -TEST_CASE("SerializeString") { - hipc::ShmSerializer istream; - Allocator *alloc = HERMES_MEMORY_MANAGER->GetDefaultAllocator(); - - auto a = hipc::make_uptr(alloc, "h1"); - auto b = hipc::make_uptr(alloc, "h2"); - int c; - size_t size = 2 * sizeof(hipc::OffsetPointer) + - sizeof(int) + sizeof(allocator_id_t); - REQUIRE(istream.shm_buf_size(alloc->GetId(), *a, *b, c) == size); - char *buf = istream.serialize(alloc, *a, *b, c); - - hipc::ShmSerializer ostream; - Allocator *alloc2 = ostream.deserialize(buf); - REQUIRE(alloc == alloc2); - hipc::mptr a2; - ostream.deserialize(alloc2, buf, a2); - REQUIRE(*a2 == *a); - hipc::mptr b2; - ostream.deserialize(alloc2, buf, b2); - REQUIRE(*b2 == *b); - int c2 = ostream.deserialize(alloc2, buf); - REQUIRE(c2 == c); -} - diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt b/hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt deleted file mode 100644 index e4f893903..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Build Tests -#------------------------------------------------------------------------------ -set (LIBS - hermes_shm_data_structures - Catch2::Catch2 - MPI::MPI_CXX - OpenMP::OpenMP_CXX - thallium) - -add_executable(test_thallium_server server.cc) -add_dependencies(test_thallium_server - hermes_shm_data_structures) -target_link_libraries(test_thallium_server ${LIBS}) - -#------------------------------------------------------------------------------ -# Test Cases -#------------------------------------------------------------------------------ -add_executable(test_thallium_exec - ${TEST_MAIN}/main.cc - test_init.cc - test_thallium.cc) -add_dependencies(test_thallium_exec - hermes_shm_data_structures) -target_link_libraries(test_thallium_exec ${LIBS}) - -add_test(NAME test_thallium COMMAND - bash ${CMAKE_CURRENT_SOURCE_DIR}/test_thallium.sh - "${CMAKE_BINARY_DIR}") - -#------------------------------------------------------------------------------ -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_thallium_server - test_thallium_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_thallium_server) - set_coverage_flags(test_thallium_exec) -endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/server.cc b/hermes_shm/test/unit/data_structures/serialize/thallium/server.cc deleted file mode 100644 index c7c9d271e..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/server.cc +++ /dev/null @@ -1,123 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "hermes_shm/data_structures/serialization/thallium.h" -#include "hermes_shm/data_structures/containers/charbuf.h" -#include - -std::unique_ptr client_; -std::unique_ptr server_; - -template -bool VerifyVector(hipc::vector &vec) { - for (int i = 0; i < 20; ++i) { - if constexpr(std::is_same_v) { - if (vec[i] != i) { - return false; - } - } else { - if (vec[i] != std::to_string(i)) { - return false; - } - } - } - return true; -} - -int main() { - // Pretest - ServerPretest(); - - // Create thallium server - server_ = std::make_unique( - tcnst::kServerName, - THALLIUM_SERVER_MODE, - true, 1); - std::cout << "Server running at address " << server_->self() << std::endl; - - // Test transfer of 0-length string - auto string_test0 = [](const request &req, - hipc::uptr &text) { - bool ret = (*text == ""); - req.respond(ret); - }; - server_->define(tcnst::kStringTest0, string_test0); - - // Test transfer of long string - auto string_test1 = [](const request &req, - hipc::uptr &text) { - bool ret = (*text == tcnst::kTestString); - req.respond(ret); - }; - server_->define(tcnst::kStringTestLarge, string_test1); - - // Test transfer of 0-length charbuf - auto charbuf_test0 = [](const request &req, - hshm::charbuf &text) { - bool ret = (text == ""); - req.respond(ret); - }; - server_->define(tcnst::kCharbufTest0, charbuf_test0); - - // Test transfer of long charbuf - auto charbuf_test1 = [](const request &req, - hshm::charbuf &text) { - bool ret = (text == tcnst::kTestString); - req.respond(ret); - }; - server_->define(tcnst::kCharbufTestLarge, charbuf_test1); - - // Test transfer of empty vector - auto vec_of_int0_test = [](const request &req, - hipc::uptr> &vec) { - bool ret = vec->size() == 0; - req.respond(ret); - }; - server_->define(tcnst::kVecOfInt0Test, vec_of_int0_test); - - // Test transfer of large vector - auto vec_of_int_large_test = [](const request &req, - hipc::uptr> &vec) { - bool ret = VerifyVector(*vec); - req.respond(ret); - }; - server_->define(tcnst::kVecOfIntLargeTest, vec_of_int_large_test); - - // Test transfer of empty string vector - auto vec_of_string0_test = [](const request &req, - hipc::uptr> &vec) { - bool ret = vec->size() == 0; - req.respond(ret); - }; - server_->define(tcnst::kVecOfString0Test, vec_of_string0_test); - - // Test transfer of large string vector - auto vec_of_string_large_test = []( - const request &req, hipc::uptr> &vec) { - bool ret = VerifyVector(*vec); - req.respond(ret); - }; - server_->define(tcnst::kVecOfStringLargeTest, vec_of_string_large_test); - - // Test transfer of bitfield - auto bitfield_test = [](const request &req, hshm::bitfield32_t &field) { - bool ret = field.Any(0x8); - req.respond(ret); - }; - server_->define(tcnst::kBitfieldTest, bitfield_test); - - // Start daemon - server_->enable_remote_shutdown(); - server_->wait_for_finalize(); -} diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.cc b/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.cc deleted file mode 100644 index a51194f45..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.cc +++ /dev/null @@ -1,33 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" -#include "hermes_shm/data_structures/serialization/thallium.h" -#include "hermes_shm/data_structures/containers/charbuf.h" -#include - -std::unique_ptr client_; -std::unique_ptr server_; - -void MainPretest() { - ClientPretest(); - client_ = std::make_unique( - "ofi+sockets", - THALLIUM_CLIENT_MODE); -} - -void MainPosttest() { - tl::endpoint server = client_->lookup(tcnst::kServerName); - client_->shutdown_remote_engine(server); -} diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h b/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h deleted file mode 100644 index 3bb5f47eb..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/test_init.h +++ /dev/null @@ -1,85 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_SERIALIZE_THALLIUM_TEST_INIT_H_ -#define HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_SERIALIZE_THALLIUM_TEST_INIT_H_ - -#include -#include "hermes_shm/thread/thread_model_manager.h" -#include "hermes_shm/thread/thread_model/thread_model_factory.h" -#include "hermes_shm/data_structures/data_structure.h" -#include "hermes_shm/data_structures/serialization/thallium.h" - -using hshm::ipc::PosixShmMmap; -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::Pointer; - -using hshm::ipc::MemoryBackendType; -using hshm::ipc::MemoryBackend; -using hshm::ipc::allocator_id_t; -using hshm::ipc::AllocatorType; -using hshm::ipc::Allocator; -using hshm::ipc::MemoryManager; -using hshm::ipc::Pointer; - -namespace thallium { -class Constants { - public: - static inline const char *kServerName = "ofi+sockets://127.0.0.1:8080"; - static inline const char *kTestString = "012344823723642364723874623"; - - /** Test cases */ - static inline const char *kStringTest0 = "kStringTest0"; - static inline const char *kStringTestLarge = "kStringTestLarge"; - static inline const char *kCharbufTest0 = "kCharbufTest0"; - static inline const char *kCharbufTestLarge = "kCharbufTestLarge"; - static inline const char *kVecOfInt0Test = "kVecOfInt0Test"; - static inline const char *kVecOfIntLargeTest = "kVecOfIntLargeTest"; - static inline const char *kVecOfString0Test = "kVecOfString0Test"; - static inline const char *kVecOfStringLargeTest = "kVecOfStringLargeTest"; - static inline const char *kBitfieldTest = "kBitfieldTest"; -}; -} // namespace thallium -using tcnst = thallium::Constants; - -namespace tl = thallium; -using thallium::request; - -/** Test init */ -template -void ServerPretest() { - std::string shm_url = "test_serializers"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator(shm_url, alloc_id, 0); -} - -template -void ClientPretest() { - std::string shm_url = "test_serializers"; - allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->AttachBackend(MemoryBackendType::kPosixShmMmap, shm_url); -} - -extern std::unique_ptr client_; -extern std::unique_ptr server_; - -#endif // HERMES_SHM_TEST_UNIT_DATA_STRUCTURES_SERIALIZE_THALLIUM_TEST_INIT_H_ diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.cc b/hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.cc deleted file mode 100644 index 46eea911d..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.cc +++ /dev/null @@ -1,91 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "test_init.h" -#include "hermes_shm/data_structures/ipc/string.h" - -TEST_CASE("SerializeString") { - tl::endpoint server = client_->lookup(tcnst::kServerName); - tl::remote_procedure string0_proc = client_->define(tcnst::kStringTest0); - tl::remote_procedure string_large_proc = client_->define( - tcnst::kStringTestLarge); - - auto empty_str = hipc::make_uptr(""); - auto large_str = hipc::make_uptr(tcnst::kTestString); - - REQUIRE(string0_proc.on(server)(empty_str)); - REQUIRE(string_large_proc.on(server)(large_str)); -} - -TEST_CASE("SerializeCharBuf") { - tl::endpoint server = client_->lookup(tcnst::kServerName); - tl::remote_procedure string0_proc = client_->define( - tcnst::kCharbufTest0); - tl::remote_procedure string_large_proc = client_->define( - tcnst::kCharbufTestLarge); - - hshm::charbuf empty_str(""); - hshm::charbuf large_str(tcnst::kTestString); - - REQUIRE(string0_proc.on(server)(empty_str)); - REQUIRE(string_large_proc.on(server)(large_str)); -} - -TEST_CASE("SerializeVectorOfInt") { - tl::endpoint server = client_->lookup(tcnst::kServerName); - tl::remote_procedure vec_int0_proc = client_->define( - tcnst::kVecOfInt0Test); - tl::remote_procedure vec_int_proc = client_->define( - tcnst::kVecOfIntLargeTest); - - // Send empty vector - auto vec_int = hipc::make_uptr>(); - REQUIRE(vec_int0_proc.on(server)(vec_int)); - - // Send initialized vector - for (int i = 0; i < 20; ++i) { - vec_int->emplace_back(i); - } - REQUIRE(vec_int_proc.on(server)(vec_int)); -} - -TEST_CASE("SerializeVectorOfString") { - tl::endpoint server = client_->lookup(tcnst::kServerName); - tl::remote_procedure vec_string0_proc = client_->define( - tcnst::kVecOfString0Test); - tl::remote_procedure vec_string_proc = client_->define( - tcnst::kVecOfStringLargeTest); - - // Send empty vector - auto vec_string = hipc::make_uptr>(); - REQUIRE(vec_string0_proc.on(server)(vec_string)); - - // Send initialized vector - for (int i = 0; i < 20; ++i) { - vec_string->emplace_back(std::to_string(i)); - } - REQUIRE(vec_string_proc.on(server)(vec_string)); -} - -TEST_CASE("SerializeBitfield") { - tl::endpoint server = client_->lookup( - tcnst::kServerName); - tl::remote_procedure bitfield_proc = client_->define( - tcnst::kBitfieldTest); - - // Send bitfield - hshm::bitfield32_t field; - field.SetBits(0x8); - REQUIRE(bitfield_proc.on(server)(field)); -} - diff --git a/hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.sh b/hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.sh deleted file mode 100644 index b5e914cc3..000000000 --- a/hermes_shm/test/unit/data_structures/serialize/thallium/test_thallium.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -CMAKE_BINARY_DIR=$1 - -# Start thallium server -echo "Starting thallium server: ${CMAKE_BINARY_DIR}/bin/test_thallium_server" -${CMAKE_BINARY_DIR}/bin/test_thallium_server & - -# Wait for it to start -sleep 1 - -# Run test -echo "Thallium test: ${CMAKE_BINARY_DIR}/bin/test_thallium_exec" -${CMAKE_BINARY_DIR}/bin/test_thallium_exec - diff --git a/hermes_shm/test/unit/main.cc b/hermes_shm/test/unit/main.cc deleted file mode 100644 index bd480daba..000000000 --- a/hermes_shm/test/unit/main.cc +++ /dev/null @@ -1,29 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "basic_test.h" -#include - -int main(int argc, char **argv) { - int rc; - Catch::Session session; - auto cli = session.cli(); - session.cli(cli); - rc = session.applyCommandLine(argc, argv); - if (rc != 0) return rc; - MainPretest(); - rc = session.run(); - MainPosttest(); - if (rc != 0) return rc; - return rc; -} diff --git a/hermes_shm/test/unit/main_mpi.cc b/hermes_shm/test/unit/main_mpi.cc deleted file mode 100644 index c2a3249a6..000000000 --- a/hermes_shm/test/unit/main_mpi.cc +++ /dev/null @@ -1,31 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "basic_test.h" -#include - -int main(int argc, char **argv) { - int rc; - MPI_Init(&argc, &argv); - Catch::Session session; - auto cli = session.cli(); - session.cli(cli); - rc = session.applyCommandLine(argc, argv); - if (rc != 0) return rc; - MainPretest(); - rc = session.run(); - MainPosttest(); - if (rc != 0) return rc; - MPI_Finalize(); - return rc; -} diff --git a/hermes_shm/test/unit/thread/test_lock.cc b/hermes_shm/test/unit/thread/test_lock.cc deleted file mode 100644 index bac5d51ee..000000000 --- a/hermes_shm/test/unit/thread/test_lock.cc +++ /dev/null @@ -1,86 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - - -#include "basic_test.h" -#include "omp.h" -#include "hermes_shm/thread/lock.h" - -using hshm::Mutex; -using hshm::RwLock; - -void MutexTest() { - size_t nthreads = 8; - size_t loop_count = 10000; - size_t count = 0; - Mutex lock; - - omp_set_dynamic(0); -#pragma omp parallel shared(lock) num_threads(nthreads) - { - // Support parallel write -#pragma omp barrier - for (size_t i = 0; i < loop_count; ++i) { - lock.Lock(i); - count += 1; - lock.Unlock(); - } -#pragma omp barrier - REQUIRE(count == loop_count * nthreads); -#pragma omp barrier - } -} - -void RwLockTest(int producers, int consumers, size_t loop_count) { - size_t nthreads = producers + consumers; - size_t count = 0; - RwLock lock; - - omp_set_dynamic(0); -#pragma omp parallel \ - shared(lock, nthreads, producers, consumers, loop_count, count) \ - num_threads(nthreads) - { // NOLINT - int tid = omp_get_thread_num(); - -#pragma omp barrier - size_t total_size = producers * loop_count; - if (tid < consumers) { - // The left 2 threads will be readers - lock.ReadLock(tid); - for (size_t i = 0; i < loop_count; ++i) { - REQUIRE(count <= total_size); - } - lock.ReadUnlock(); - } else { - // The right 4 threads will be writers - lock.WriteLock(tid); - for (size_t i = 0; i < loop_count; ++i) { - count += 1; - } - lock.WriteUnlock(); - } - -#pragma omp barrier - REQUIRE(count == total_size); - } -} - -TEST_CASE("Mutex") { - MutexTest(); -} - -TEST_CASE("RwLock") { - RwLockTest(8, 0, 1000000); - RwLockTest(7, 1, 1000000); - RwLockTest(4, 4, 1000000); -} diff --git a/hermes_shm/test/unit/thread/test_thread.cc b/hermes_shm/test/unit/thread/test_thread.cc deleted file mode 100644 index 7e39052cf..000000000 --- a/hermes_shm/test/unit/thread/test_thread.cc +++ /dev/null @@ -1,23 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "omp.h" -#include "hermes_shm/thread/thread_model_manager.h" - -TEST_CASE("TestPthread") { - HERMES_THREAD_MODEL->SetThreadModel(hshm::ThreadType::kPthread); -} - -TEST_CASE("TestArgobots") { - HERMES_THREAD_MODEL->SetThreadModel(hshm::ThreadType::kArgobots); -} diff --git a/hermes_shm/test/unit/types/CMakeLists.txt b/hermes_shm/test/unit/types/CMakeLists.txt deleted file mode 100644 index f4cdeb082..000000000 --- a/hermes_shm/test/unit/types/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# Build Tests -#------------------------------------------------------------------------------ -add_executable(test_types_exec - ${TEST_MAIN}/main.cc - test_init.cc - test_argpack.cc - test_util.cc) -add_dependencies(test_types_exec hermes_shm_data_structures) -target_link_libraries(test_types_exec - hermes_shm_data_structures Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) - -add_test(NAME test_types COMMAND - ${CMAKE_BINARY_DIR}/bin/test_types_exec "~[error=FatalError]") -add_test(NAME test_fatal_logger - COMMAND ${CMAKE_BINARY_DIR}/bin/test_types_exec "[error=FatalError]") -set_tests_properties( - test_fatal_logger PROPERTIES - WILL_FAIL TRUE) - -#----------------------------------------------------------------------------- -# Install Targets -#------------------------------------------------------------------------------ -install(TARGETS - test_types_exec - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) - -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_types_exec) -endif() \ No newline at end of file diff --git a/hermes_shm/test/unit/types/test_argpack.cc b/hermes_shm/test/unit/types/test_argpack.cc deleted file mode 100644 index 2f75dbe3b..000000000 --- a/hermes_shm/test/unit/types/test_argpack.cc +++ /dev/null @@ -1,180 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "basic_test.h" -#include "hermes_shm/data_structures/containers/tuple_base.h" -#include - -void test_argpack0_pass() { - std::cout << "HERE0" << std::endl; -} - -void test_argpack0() { - hshm::PassArgPack::Call(hshm::ArgPack<>(), test_argpack0_pass); -} - -template -void test_argpack3_pass(T1 x, T2 y, T3 z) { - REQUIRE(x == 0); - REQUIRE(y == 1); - REQUIRE(z == 0); - std::cout << "HERE3" << std::endl; -} - -void test_product1(int b, int c) { - REQUIRE(b == 1); - REQUIRE(c == 2); -} - -void test_product2(double d, double e) { - REQUIRE(d == 3); - REQUIRE(e == 4); -} - -template -void test_product(int a, Pack1 &&pack1, int a2, Pack2 &&pack2) { - REQUIRE(a == 0); - REQUIRE(a2 == 0); - hshm::PassArgPack::Call( - std::forward(pack1), - test_product1); - hshm::PassArgPack::Call( - std::forward(pack2), - test_product2); -} - -template -void verify_tuple3(hshm::tuple &x) { - REQUIRE(x.Size() == 3); - REQUIRE(x.template Get<0>() == 0); - REQUIRE(x.template Get<1>() == 1); - REQUIRE(x.template Get<2>() == 0); -#ifdef TEST_COMPILER_ERROR - std::cout << x.Get<3>() << std::endl; -#endif -} - -template -void test_argpack3() { - // Pass an argpack to a function - PAGE_DIVIDE("") { - hshm::PassArgPack::Call( - hshm::make_argpack(T1(0), T2(1), T3(0)), - test_argpack3_pass); - } - - // Pass an argpack containing references to a function - PAGE_DIVIDE("") { - T2 y = 1; - hshm::PassArgPack::Call( - hshm::make_argpack(0, y, 0), - test_argpack3_pass); - } - - // Create a 3-tuple - PAGE_DIVIDE("") { - hshm::tuple x(0, 1, 0); - verify_tuple3(x); - } - - // Copy a tuple - PAGE_DIVIDE("") { - hshm::tuple y(0, 1, 0); - hshm::tuple x(y); - verify_tuple3(x); - } - - // Copy assign tuple - PAGE_DIVIDE("") { - hshm::tuple y(0, 1, 0); - hshm::tuple x; - x = y; - verify_tuple3(x); - } - - // Move tuple - PAGE_DIVIDE("") { - hshm::tuple y(0, 1, 0); - hshm::tuple x(std::move(y)); - verify_tuple3(x); - } - - // Move assign tuple - PAGE_DIVIDE("") { - hshm::tuple y(0, 1, 0); - hshm::tuple x; - x = std::move(y); - verify_tuple3(x); - } - - // Iterate over a tuple - PAGE_DIVIDE("") { - hshm::tuple x(0, 1, 0); - hshm::ForwardIterateTuple::Apply( - x, - [](auto i, auto &arg) constexpr { - std::cout << "lambda: " << i.Get() << std::endl; - }); - } - - // Merge two argpacks into a single pack - PAGE_DIVIDE("") { - size_t y = hshm::MergeArgPacks::Merge( - hshm::make_argpack(T1(0)), - hshm::make_argpack(T2(1), T2(0))).Size(); - REQUIRE(y == 3); - } - - // Pass a merged argpack to a function - PAGE_DIVIDE("") { - hshm::PassArgPack::Call( - hshm::MergeArgPacks::Merge( - hshm::make_argpack(0), - hshm::make_argpack(1, 0)), - test_argpack3_pass); - } - - // Construct tuple from argpack - PAGE_DIVIDE("") { - hshm::tuple x( - hshm::make_argpack(10, 11, 12)); - REQUIRE(x.Get<0>() == 10); - REQUIRE(x.Get<1>() == 11); - REQUIRE(x.Get<2>() == 12); - } - - // Product an argpack - PAGE_DIVIDE("") { - auto&& pack = hshm::ProductArgPacks::Product( - 0, - hshm::make_argpack(1, 2), - hshm::make_argpack(3, 4)); - REQUIRE(pack.Size() == 4); - } - - // Product an argpack - PAGE_DIVIDE("") { - hshm::PassArgPack::Call( - hshm::ProductArgPacks::Product( - 0, - hshm::make_argpack(1, 2), - hshm::make_argpack(3.0, 4.0)), - test_product< - hshm::ArgPack, - hshm::ArgPack>); - } -} - -TEST_CASE("TestArgpack") { - test_argpack0(); - test_argpack3(); -} diff --git a/hermes_shm/test/unit/types/test_util.cc b/hermes_shm/test/unit/types/test_util.cc deleted file mode 100644 index aec093256..000000000 --- a/hermes_shm/test/unit/types/test_util.cc +++ /dev/null @@ -1,170 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include -#include -#include -#include "basic_test.h" -#include "hermes_shm/util/singleton.h" -#include "hermes_shm/util/type_switch.h" -#include - -TEST_CASE("TypeSwitch") { - typedef hshm::type_switch::type internal_t; - REQUIRE(std::is_same_v); - - typedef hshm::type_switch::type internal2_t; - REQUIRE(std::is_same_v); - - typedef hshm::type_switch::type internal3_t; - REQUIRE(std::is_same_v); - - typedef hshm::type_switch, int, - std::string, std::string, - size_t, size_t>::type internal4_t; - REQUIRE(std::is_same_v); -} - -TEST_CASE("TestPathParser") { - setenv("PATH_PARSER_TEST", "HOME", true); - auto x = hshm::ConfigParse::ExpandPath("${PATH_PARSER_TEST}/hello"); - unsetenv("PATH_PARSER_TEST"); - auto y = hshm::ConfigParse::ExpandPath("${PATH_PARSER_TEST}/hello"); - auto z = hshm::ConfigParse::ExpandPath("${HOME}/hello"); - REQUIRE(x == "HOME/hello"); - REQUIRE(y == "${PATH_PARSER_TEST}/hello"); - REQUIRE(z != "${HOME}/hello"); -} - -TEST_CASE("TestNumberParser") { - REQUIRE(KILOBYTES(1.5) == 1536); - REQUIRE(MEGABYTES(1.5) == 1572864); - REQUIRE(GIGABYTES(1.5) == 1610612736); - REQUIRE(TERABYTES(1.5) == 1649267441664); - REQUIRE(PETABYTES(1.5) == 1688849860263936); - - std::pair sizes[] = { - {"1", 1}, - {"1.5", 1}, - {"1KB", KILOBYTES(1)}, - {"1.5MB", MEGABYTES(1.5)}, - {"1.5GB", GIGABYTES(1.5)}, - {"2TB", TERABYTES(2)}, - {"1.5PB", PETABYTES(1.5)}, - }; - - for (auto &[text, val] : sizes) { - REQUIRE(hshm::ConfigParse::ParseSize(text) == val); - } - REQUIRE(hshm::ConfigParse::ParseSize("inf")); -} - -TEST_CASE("TestTerminal") { - std::cout << "\033[1m" << "Bold text" << "\033[0m" << std::endl; - std::cout << "\033[4m" << "Underlined text" << "\033[0m" << std::endl; - std::cout << "\033[31m" << "Red text" << "\033[0m" << std::endl; - std::cout << "\033[32m" << "Green text" << "\033[0m" << std::endl; - std::cout << "\033[33m" << "Yellow text" << "\033[0m" << std::endl; - std::cout << "\033[34m" << "Blue text" << "\033[0m" << std::endl; - std::cout << "\033[35m" << "Magenta text" << "\033[0m" << std::endl; - std::cout << "\033[36m" << "Cyan text" << "\033[0m" << std::endl; -} - -TEST_CASE("TestAutoTrace") { - AUTO_TRACE(0) - - TIMER_START("Example") - sleep(1); - TIMER_END() -} - -TEST_CASE("TestLogger") { - HILOG(kInfo, "I'm more likely to be printed: {}", 0) - HILOG(kDebug, "I'm not likely to be printed: {}", 10) - - HERMES_LOG->SetVerbosity(kInfo); - HILOG(kInfo, "I'm more likely to be printed (2): {}", 0) - HILOG(kDebug, "I won't be printed: {}", 10) - - HELOG(kWarning, "I am a WARNING! Will NOT cause an EXIT!") - HELOG(kError, "I am an ERROR! I will NOT cause an EXIT!") -} - -TEST_CASE("TestFatalLogger", "[error=FatalError]") { - HELOG(kFatal, "I will cause an EXIT!") -} - -TEST_CASE("TestFormatter") { - int rank = 0; - int i = 0; - - PAGE_DIVIDE("Test with equivalent parameters") { - std::string name = hshm::Formatter::format("bucket{}_{}", - rank, i); - REQUIRE(name == "bucket0_0"); - } - - PAGE_DIVIDE("Test with equivalent parameters at start") { - std::string name = hshm::Formatter::format("{}bucket{}", - rank, i); - REQUIRE(name == "0bucket0"); - } - - PAGE_DIVIDE("Test with too many parameters") { - std::string name = hshm::Formatter::format("bucket", - rank, i); - REQUIRE(name == "bucket"); - } - - PAGE_DIVIDE("Test with fewer parameters") { - std::string name = hshm::Formatter::format("bucket{}_{}", - rank); - REQUIRE(name == "bucket{}_{}"); - } - - PAGE_DIVIDE("Test with parameters next to each other") { - std::string name = hshm::Formatter::format("bucket{}{}", - rank, i); - REQUIRE(name == "bucket00"); - } -} - -struct SimpleClass { - int a_; - - SimpleClass() { - a_ = 100; - } -}; - -DEFINE_SINGLETON_CC(SimpleClass) -DEFINE_GLOBAL_SINGLETON_CC(SimpleClass) - -TEST_CASE("Singleton") { - SimpleClass *cls[4]; - - cls[0] = hshm::Singleton::GetInstance(); - cls[1] = hshm::GlobalSingleton::GetInstance(); - cls[2] = hshm::EasySingleton::GetInstance(); - cls[3] = hshm::EasyGlobalSingleton::GetInstance(); - - for (int i = 0; i < 4; ++i) { - REQUIRE(cls[i]->a_ == 100); - } -} diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h new file mode 100644 index 000000000..68c53535a --- /dev/null +++ b/include/labstor/api/labstor_client.h @@ -0,0 +1,291 @@ +// +// Created by lukemartinlogan on 6/23/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_CLIENT_H_ +#define LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_CLIENT_H_ + +#include +#include "manager.h" +#include "labstor/queue_manager/queue_manager_client.h" + +// Singleton macros +#define LABSTOR_CLIENT hshm::Singleton::GetInstance() +#define LABSTOR_CLIENT_T labstor::Client* + +namespace labstor { + +class Client : public ConfigurationManager { + public: + int data_; + QueueManagerClient queue_manager_; + std::atomic *unique_; + u32 node_id_; + + public: + /** Default constructor */ + Client() {} + + /** Initialize the client */ + Client* Create(std::string server_config_path = "", + std::string client_config_path = "", + bool server = false) { + hshm::ScopedMutex lock(lock_, 1); + if (is_initialized_) { + return this; + } + mode_ = LabstorMode::kClient; + is_being_initialized_ = true; + ClientInit(std::move(server_config_path), + std::move(client_config_path), + server); + is_initialized_ = true; + is_being_initialized_ = false; + return this; + } + + private: + /** Initialize client */ + void ClientInit(std::string server_config_path, + std::string client_config_path, + bool server) { + LoadServerConfig(server_config_path); + LoadClientConfig(client_config_path); + LoadSharedMemory(server); + queue_manager_.ClientInit(main_alloc_, + header_->queue_manager_, + header_->node_id_); + if (!server) { + HERMES_THREAD_MODEL->SetThreadModel(hshm::ThreadType::kPthread); + } + } + + public: + /** Connect to a Daemon's shared memory */ + void LoadSharedMemory(bool server) { + // Load shared-memory allocator + auto mem_mngr = HERMES_MEMORY_MANAGER; + if (!server) { + mem_mngr->AttachBackend(hipc::MemoryBackendType::kPosixShmMmap, + server_config_.queue_manager_.shm_name_); + } + main_alloc_ = mem_mngr->GetAllocator(main_alloc_id_); + header_ = main_alloc_->GetCustomHeader(); + unique_ = &header_->unique_; + node_id_ = header_->node_id_; + } + + /** Finalize Hermes explicitly */ + void Finalize() {} + + /** Create task node id */ + TaskNode MakeTaskNodeId() { + return TaskId(header_->node_id_, unique_->fetch_add(1));; + } + + /** Create a unique ID */ + TaskStateId MakeTaskStateId() { + return TaskStateId(header_->node_id_, unique_->fetch_add(1)); + } + + /** Create a default-constructed task */ + template + HSHM_ALWAYS_INLINE + TaskT* NewEmptyTask(hipc::Pointer &p) { + return main_alloc_->NewObj(p, main_alloc_); + } + + /** Allocate task */ + template + HSHM_ALWAYS_INLINE + hipc::LPointer AllocateTask() { + return main_alloc_->AllocateLocalPtr(sizeof(TaskT)); + } + + /** Construct task */ + template + HSHM_ALWAYS_INLINE + void ConstructTask(TaskT *task, Args&& ...args) { + return hipc::Allocator::ConstructObj( + *task, main_alloc_, std::forward(args)...); + } + + /** Create a task */ + template + HSHM_ALWAYS_INLINE + LPointer NewTask(const TaskNode &task_node, Args&& ...args) { + LPointer ptr = main_alloc_->NewObjLocal( + main_alloc_, task_node, std::forward(args)...); + if (ptr.ptr_ == nullptr) { + throw std::runtime_error("Could not allocate buffer"); + } + return ptr; + } + + /** Create a root task */ + template + HSHM_ALWAYS_INLINE + LPointer NewTaskRoot(Args&& ...args) { + TaskNode task_node = MakeTaskNodeId(); + LPointer ptr = main_alloc_->NewObjLocal( + main_alloc_, task_node, std::forward(args)...); + if (ptr.ptr_ == nullptr) { + throw std::runtime_error("Could not allocate buffer"); + } + return ptr; + } + + /** Destroy a task */ + template + HSHM_ALWAYS_INLINE + void DelTask(TaskT *task) { + main_alloc_->DelObj(task); + } + + /** Destroy a task */ + template + HSHM_ALWAYS_INLINE + void DelTask(LPointer &task) { + main_alloc_->DelObjLocal(task); + } + + /** Get a queue by its ID */ + HSHM_ALWAYS_INLINE + MultiQueue* GetQueue(const QueueId &queue_id) { + return queue_manager_.GetQueue(queue_id); + } + + /** Detect if a task is local or remote */ + HSHM_ALWAYS_INLINE + bool IsRemote(Task *task) { + if (task->domain_id_.IsNode()) { + return task->domain_id_.GetId() != header_->node_id_; + } else if (task->domain_id_.IsGlobal()) { + return true; + } else { + return false; + } + } + + /** Allocate a buffer */ + HSHM_ALWAYS_INLINE + LPointer AllocateBuffer(size_t size) { + LPointer p = main_alloc_->AllocateLocalPtr(size); + if (p.ptr_ == nullptr) { + throw std::runtime_error("Could not allocate buffer"); + } + return p; + } + + /** Convert pointer to char* */ + template + HSHM_ALWAYS_INLINE + T* GetPrivatePointer(const hipc::Pointer &p) { + return main_alloc_->Convert(p); + } + + /** Free a buffer */ + HSHM_ALWAYS_INLINE + void FreeBuffer(hipc::Pointer &p) { + main_alloc_->Free(p); + } + + /** Free a buffer */ + HSHM_ALWAYS_INLINE + void FreeBuffer(LPointer &p) { + main_alloc_->FreeLocalPtr(p); + } +}; + +/** A function which creates a new TaskNode value */ +#define LABSTOR_TASK_NODE_ROOT(CUSTOM)\ + template\ + auto CUSTOM##Root(Args&& ...args) {\ + TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId();\ + return CUSTOM(task_node, std::forward(args)...);\ + } + +/** Fill in common default parameters for task client wrapper function */ +#define LABSTOR_TASK_NODE_ADMIN_ROOT(CUSTOM)\ + template\ + hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node,\ + Args&& ...args) {\ + hipc::LPointer task = LABSTOR_CLIENT->AllocateTask();\ + Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...);\ + return task;\ + }\ + template\ + hipc::LPointer Async##CUSTOM(const TaskNode &task_node, \ + Args&& ...args) {\ + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...);\ + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_);\ + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_);\ + return task;\ + }\ + template\ + hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue,\ + const TaskNode &task_node,\ + Args&& ...args) {\ + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...);\ + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_);\ + return task;\ + }\ + template\ + hipc::LPointer Async##CUSTOM##Root(Args&& ...args) {\ + TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId();\ + hipc::LPointer task = Async##CUSTOM(task_node + 1, std::forward(args)...);\ + return task;\ + } + +/** The default asynchronous method behavior */ +#define LABSTOR_TASK_NODE_PUSH_ROOT(CUSTOM)\ + template\ + hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node,\ + Args&& ...args) {\ + hipc::LPointer task = LABSTOR_CLIENT->AllocateTask();\ + Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...);\ + return task;\ + }\ + template\ + hipc::LPointer Async##CUSTOM(const TaskNode &task_node, \ + Args&& ...args) {\ + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...);\ + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_);\ + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_);\ + return task;\ + }\ + template\ + hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue,\ + const TaskNode &task_node,\ + Args&& ...args) {\ + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...);\ + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_);\ + return task;\ + }\ + template\ + hipc::LPointer> Async##CUSTOM##Root(Args&& ...args) {\ + TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId();\ + hipc::LPointer task = Async##CUSTOM##Alloc(task_node + 1, std::forward(args)...);\ + hipc::LPointer> push_task =\ + LABSTOR_PROCESS_QUEUE->AsyncPush(task_node,\ + DomainId::GetLocal(),\ + task.shm_);\ + return push_task;\ + } + +} // namespace labstor + +static inline bool TRANSPARENT_LABSTOR() { + if (!LABSTOR_CLIENT->IsInitialized() && + !LABSTOR_CLIENT->IsBeingInitialized() && + !LABSTOR_CLIENT->IsTerminated()) { + LABSTOR_CLIENT->Create(); + LABSTOR_CLIENT->is_transparent_ = true; + return true; + } + return false; +} + +#define HASH_TO_NODE_ID(hash) (1 + ((hash) % LABSTOR_CLIENT->GetNumNodes())) + +#endif // LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_CLIENT_H_ diff --git a/include/labstor/api/labstor_runtime.h b/include/labstor/api/labstor_runtime.h new file mode 100644 index 000000000..76f4525e6 --- /dev/null +++ b/include/labstor/api/labstor_runtime.h @@ -0,0 +1,186 @@ +// +// Created by lukemartinlogan on 6/27/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_SERVER_H_ +#define LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_SERVER_H_ + +#include "labstor/task_registry/task_registry.h" +#include "labstor/work_orchestrator/work_orchestrator.h" +#include "labstor/queue_manager/queue_manager_runtime.h" +#include "labstor_admin/labstor_admin.h" +#include "remote_queue/remote_queue.h" +#include "labstor_client.h" +#include "manager.h" +#include "labstor/network/rpc.h" +#include "labstor/network/rpc_thallium.h" + +// Singleton macros +#define LABSTOR_RUNTIME hshm::Singleton::GetInstance() +#define LABSTOR_RUNTIME_T labstor::Runtime* +#define LABSTOR_REMOTE_QUEUE (&LABSTOR_RUNTIME->remote_queue_) +#define LABSTOR_THALLIUM (&LABSTOR_RUNTIME->thallium_) +#define LABSTOR_RPC (&LABSTOR_RUNTIME->rpc_) + +namespace labstor { + +class Runtime : public ConfigurationManager { + public: + int data_; + TaskRegistry task_registry_; + WorkOrchestrator work_orchestrator_; + QueueManagerRuntime queue_manager_; + remote_queue::Client remote_queue_; + RpcContext rpc_; + ThalliumRpc thallium_; + + public: + /** Default constructor */ + Runtime() = default; + + /** Create the server-side API */ + Runtime* Create(std::string server_config_path = "") { + hshm::ScopedMutex lock(lock_, 1); + if (is_initialized_) { + return this; + } + mode_ = LabstorMode::kServer; + is_being_initialized_ = true; + ServerInit(std::move(server_config_path)); + is_initialized_ = true; + is_being_initialized_ = false; + return this; + } + + private: + /** Initialize */ + void ServerInit(std::string server_config_path) { + LoadServerConfig(server_config_path); + InitSharedMemory(); + rpc_.ServerInit(&server_config_); + thallium_.ServerInit(&rpc_); + header_->node_id_ = rpc_.node_id_; + header_->unique_ = 0; + header_->num_nodes_ = server_config_.rpc_.host_names_.size(); + task_registry_.ServerInit(&server_config_, rpc_.node_id_, header_->unique_); + // Queue manager + client must be initialized before Work Orchestrator + queue_manager_.ServerInit(main_alloc_, + rpc_.node_id_, + &server_config_, + header_->queue_manager_); + LABSTOR_CLIENT->Create(server_config_path, "", true); + HERMES_THREAD_MODEL->SetThreadModel(hshm::ThreadType::kPthread); + work_orchestrator_.ServerInit(&server_config_, queue_manager_); + hipc::mptr admin_task; + + // Create the admin library + LABSTOR_CLIENT->MakeTaskStateId(); + admin_task = hipc::make_mptr(); + task_registry_.RegisterTaskLib("labstor_admin"); + task_registry_.CreateTaskState( + "labstor_admin", + "labstor_admin", + LABSTOR_QM_CLIENT->admin_task_state_, + admin_task.get()); + + // Create the process queue + LABSTOR_CLIENT->MakeTaskStateId(); + admin_task = hipc::make_mptr(); + task_registry_.RegisterTaskLib("proc_queue"); + task_registry_.CreateTaskState( + "proc_queue", + "proc_queue", + LABSTOR_QM_CLIENT->process_queue_, + admin_task.get()); + + // Create the work orchestrator queue scheduling library + TaskStateId queue_sched_id = LABSTOR_CLIENT->MakeTaskStateId(); + admin_task = hipc::make_mptr(); + task_registry_.RegisterTaskLib("worch_queue_round_robin"); + task_registry_.CreateTaskState( + "worch_queue_round_robin", + "worch_queue_round_robin", + queue_sched_id, + admin_task.get()); + + // Create the work orchestrator process scheduling library + TaskStateId proc_sched_id = LABSTOR_CLIENT->MakeTaskStateId(); + admin_task = hipc::make_mptr(); + task_registry_.RegisterTaskLib("worch_proc_round_robin"); + task_registry_.CreateTaskState( + "worch_proc_round_robin", + "worch_proc_round_robin", + proc_sched_id, + admin_task.get()); + + // Set the work orchestrator queue scheduler + LABSTOR_ADMIN->SetWorkOrchQueuePolicyRoot(labstor::DomainId::GetLocal(), queue_sched_id); + LABSTOR_ADMIN->SetWorkOrchProcPolicyRoot(labstor::DomainId::GetLocal(), proc_sched_id); + + // Create the remote queue library + task_registry_.RegisterTaskLib("remote_queue"); + remote_queue_.CreateRoot(DomainId::GetLocal(), "remote_queue", + LABSTOR_CLIENT->MakeTaskStateId()); + } + + public: + /** Initialize shared-memory between daemon and client */ + void InitSharedMemory() { + // Create shared-memory allocator + auto mem_mngr = HERMES_MEMORY_MANAGER; + if (server_config_.queue_manager_.shm_size_ == 0) { + server_config_.queue_manager_.shm_size_ = + hipc::MemoryManager::GetDefaultBackendSize(); + } + mem_mngr->CreateBackend( + server_config_.queue_manager_.shm_size_, + server_config_.queue_manager_.shm_name_); + main_alloc_ = + mem_mngr->CreateAllocator( + server_config_.queue_manager_.shm_name_, + main_alloc_id_, + sizeof(LabstorShm)); + header_ = main_alloc_->GetCustomHeader(); + } + + /** Finalize Hermes explicitly */ + void Finalize() {} + + /** Run the Hermes core Daemon */ + void RunDaemon() { + thallium_.RunDaemon(); + HILOG(kInfo, "Daemon is running") +// while (LABSTOR_WORK_ORCHESTRATOR->IsRuntimeAlive()) { +// // Scheduler callbacks? +// HERMES_THREAD_MODEL->SleepForUs(1000); +// } + HILOG(kInfo, "Finishing up last requests") + LABSTOR_WORK_ORCHESTRATOR->Join(); + HILOG(kInfo, "Daemon is exiting") + } + + /** Stop the Hermes core Daemon */ + void StopDaemon() { + LABSTOR_WORK_ORCHESTRATOR->FinalizeRuntime(); + } + + /** Get the set of DomainIds */ + std::vector ResolveDomainId(const DomainId &domain_id) { + std::vector ids; + if (domain_id.IsGlobal()) { + ids.reserve(rpc_.hosts_.size()); + for (HostInfo &host_info : rpc_.hosts_) { + ids.push_back(DomainId::GetNode(host_info.node_id_)); + } + } else if (domain_id.IsNode()) { + ids.reserve(1); + ids.push_back(domain_id); + } + // TODO(llogan): handle named domain ID sets + return ids; + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_SERVER_H_ diff --git a/include/labstor/api/manager.h b/include/labstor/api/manager.h new file mode 100644 index 000000000..d4a27f119 --- /dev/null +++ b/include/labstor/api/manager.h @@ -0,0 +1,85 @@ +// +// Created by lukemartinlogan on 6/27/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_MANAGER_MANAGER_H_ +#define LABSTOR_INCLUDE_LABSTOR_MANAGER_MANAGER_H_ + +#include "labstor/labstor_types.h" +#include "labstor/labstor_constants.h" +#include "labstor/config/config_client.h" +#include "labstor/config/config_server.h" +#include "labstor/queue_manager/queue_manager.h" + +namespace labstor { + +/** Shared-memory header for LabStor */ +struct LabstorShm { + u32 node_id_; + QueueManagerShm queue_manager_; + std::atomic unique_; + u64 num_nodes_; +}; + +/** The configuration used inherited by runtime + client */ +class ConfigurationManager { + public: + LabstorMode mode_; + LabstorShm *header_; + ClientConfig client_config_; + ServerConfig server_config_; + static inline const hipc::allocator_id_t main_alloc_id_ = + hipc::allocator_id_t(0, 1); + hipc::Allocator *main_alloc_; + bool is_being_initialized_; + bool is_initialized_; + bool is_terminated_; + bool is_transparent_; + hshm::Mutex lock_; + hshm::ThreadType thread_type_; + + /** Default constructor */ + ConfigurationManager() : is_being_initialized_(false), + is_initialized_(false), + is_terminated_(false), + is_transparent_(false) {} + + /** Destructor */ + ~ConfigurationManager() {} + + /** Whether or not Labstor is currently being initialized */ + bool IsBeingInitialized() { return is_being_initialized_; } + + /** Whether or not Labstor is initialized */ + bool IsInitialized() { return is_initialized_; } + + /** Whether or not Labstor is finalized */ + bool IsTerminated() { return is_terminated_; } + + /** Load the server-side configuration */ + void LoadServerConfig(std::string config_path) { + if (config_path.empty()) { + config_path = Constants::GetEnvSafe(Constants::kServerConfEnv); + } + HILOG(kInfo, "Loading server configuration: {}", config_path); + server_config_.LoadFromFile(config_path); + } + + /** Load the client-side configuration */ + void LoadClientConfig(std::string config_path) { + if (config_path.empty()) { + config_path = Constants::GetEnvSafe(Constants::kClientConfEnv); + } + HILOG(kDebug, "Loading client configuration: {}", config_path); + client_config_.LoadFromFile(config_path); + } + + /** Get number of nodes */ + HSHM_ALWAYS_INLINE int GetNumNodes() { + return server_config_.rpc_.host_names_.size(); + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_MANAGER_MANAGER_H_ diff --git a/include/labstor/api/template/labstor_task_node_admin_root.template b/include/labstor/api/template/labstor_task_node_admin_root.template new file mode 100644 index 000000000..acc5327cc --- /dev/null +++ b/include/labstor/api/template/labstor_task_node_admin_root.template @@ -0,0 +1,29 @@ +template +hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = LABSTOR_CLIENT->AllocateTask(); + Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...); + return task; +} +template +hipc::LPointer Async##CUSTOM(const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); + return task; +} +template +hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue, + const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); + return task; +} +template +hipc::LPointer Async##CUSTOM##Root(Args&& ...args) { + TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId(); + hipc::LPointer task = Async##CUSTOM(task_node + 1, std::forward(args)...); + return task; +} \ No newline at end of file diff --git a/include/labstor/api/template/labstor_task_node_push_root.template b/include/labstor/api/template/labstor_task_node_push_root.template new file mode 100644 index 000000000..703048125 --- /dev/null +++ b/include/labstor/api/template/labstor_task_node_push_root.template @@ -0,0 +1,33 @@ +template +hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = LABSTOR_CLIENT->AllocateTask(); + Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...); + return task; +} +template +hipc::LPointer Async##CUSTOM(const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); + return task; +} +template +hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue, + const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); + return task; +} +template +hipc::LPointer> Async##CUSTOM##Root(Args&& ...args) { + TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId(); + hipc::LPointer task = Async##CUSTOM##Alloc(task_node + 1, std::forward(args)...); + hipc::LPointer> push_task = + LABSTOR_PROCESS_QUEUE->AsyncPush(task_node, + DomainId::GetLocal(), + task.shm_); + return push_task; +} \ No newline at end of file diff --git a/include/labstor/config/config.h b/include/labstor/config/config.h new file mode 100644 index 000000000..151c997cc --- /dev/null +++ b/include/labstor/config/config.h @@ -0,0 +1,73 @@ +// +// Created by lukemartinlogan on 6/17/23. +// + +#ifndef LABSTOR_SRC_CONFIG_H_ +#define LABSTOR_SRC_CONFIG_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "hermes_shm/util/config_parse.h" + +namespace labstor::config { + +/** + * Base class for configuration files + * */ +class BaseConfig { + public: + /** load configuration from a string */ + void LoadText(const std::string &config_string, bool with_default = true) { + if (with_default) { + LoadDefault(); + } + if (config_string.size() == 0) { + return; + } + YAML::Node yaml_conf = YAML::Load(config_string); + ParseYAML(yaml_conf); + } + + /** load configuration from file */ + void LoadFromFile(const std::string &path, bool with_default = true) { + if (with_default) { + LoadDefault(); + } + if (path.size() == 0) { + return; + } + auto real_path = hshm::ConfigParse::ExpandPath(path); + HILOG(kDebug, "Start load config {}", real_path) + try { + YAML::Node yaml_conf = YAML::LoadFile(real_path); + HILOG(kDebug, "Complete load config {}", real_path) + ParseYAML(yaml_conf); + } catch (std::exception &e) { + HELOG(kFatal, e.what()) + } + } + + /** load the default configuration */ + virtual void LoadDefault() = 0; + + public: + /** parse \a list_node vector from configuration file in YAML */ + template> + static void ParseVector(YAML::Node list_node, VEC_TYPE &list) { + for (auto val_node : list_node) { + list.emplace_back(val_node.as()); + } + } + + private: + virtual void ParseYAML(YAML::Node &yaml_conf) = 0; +}; + +} // namespace labstor::config + +#endif // LABSTOR_SRC_CONFIG_H_ diff --git a/src/dpe/round_robin.h b/include/labstor/config/config_client.h similarity index 59% rename from src/dpe/round_robin.h rename to include/labstor/config/config_client.h index bfc7976bf..8636eaab5 100644 --- a/src/dpe/round_robin.h +++ b/include/labstor/config/config_client.h @@ -10,31 +10,33 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef HERMES_SRC_DPE_ROUND_ROBIN_H_ -#define HERMES_SRC_DPE_ROUND_ROBIN_H_ +#ifndef LABSTOR_SRC_CONFIG_CLIENT_H_ +#define LABSTOR_SRC_CONFIG_CLIENT_H_ -#include +#include +#include "config.h" -#include "data_placement_engine.h" +namespace stdfs = std::filesystem; -namespace hermes { - -using api::Status; - -/** Represents the state of a Round-Robin data placement strategy */ -class RoundRobin : public DPE { - private: - std::atomic counter_; +namespace labstor::config { +/** + * Configuration used to intialize client + * */ +class ClientConfig : public BaseConfig { public: - RoundRobin() : counter_(0) {} + /** The thread model of the application */ + std::string thread_model_; - Status Placement(const std::vector &blob_sizes, - std::vector &targets, - api::Context &ctx, - std::vector &output); + private: + void ParseYAML(YAML::Node &yaml_conf) override; + void LoadDefault() override; }; -} // namespace hermes +} // namespace labstor::config + +namespace labstor { +using ClientConfig = config::ClientConfig; +} // namespace labstor -#endif // HERMES_SRC_DPE_ROUND_ROBIN_H_ +#endif // LABSTOR_SRC_CONFIG_CLIENT_H_ diff --git a/include/labstor/config/config_client_default.h b/include/labstor/config/config_client_default.h new file mode 100644 index 000000000..94df443ad --- /dev/null +++ b/include/labstor/config/config_client_default.h @@ -0,0 +1,5 @@ +#ifndef LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ +#define LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ +const char* kLabstorClientDefaultConfigStr = +"thread_model: kStd\n"; +#endif // LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/include/labstor/config/config_server.h b/include/labstor/config/config_server.h new file mode 100644 index 000000000..be0af57ed --- /dev/null +++ b/include/labstor/config/config_server.h @@ -0,0 +1,96 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef LABSTOR_SRC_CONFIG_SERVER_H_ +#define LABSTOR_SRC_CONFIG_SERVER_H_ + +#include "config.h" +#include "labstor/labstor_types.h" + +namespace labstor::config { + +/** + * Work orchestrator information defined in server config + * */ +struct WorkOrchestratorInfo { + /** Maximum number of workers to spawn */ + size_t max_workers_; +}; + +/** + * Queue manager information defined in server config + * */ +struct QueueManagerInfo { + /** Maximum depth of IPC queues */ + u32 queue_depth_; + /** Maximum number of lanes per IPC queue */ + u32 max_lanes_; + /** Maximum number of allocatable IPC queues */ + u32 max_queues_; + /** Shared memory allocator */ + std::string shm_allocator_; + /** Shared memory region name */ + std::string shm_name_; + /** Shared memory region size */ + size_t shm_size_; +}; + +/** + * RPC information defined in server config + * */ +struct RpcInfo { + /** The name of a file that contains host names, 1 per line */ + std::string host_file_; + /** The parsed hostnames from the hermes conf */ + std::vector host_names_; + /** The RPC protocol to be used. */ + std::string protocol_; + /** The RPC domain name for verbs transport. */ + std::string domain_; + /** The RPC port number. */ + int port_; + /** Number of RPC threads */ + int num_threads_; +}; + +/** + * System configuration for Hermes + */ +class ServerConfig : public BaseConfig { + public: + /** Work orchestrator info */ + WorkOrchestratorInfo wo_; + /** Queue manager info */ + QueueManagerInfo queue_manager_; + /** The RPC information */ + RpcInfo rpc_; + /** Bootstrap task registry */ + std::vector task_libs_; + + public: + ServerConfig() = default; + void LoadDefault(); + + private: + void ParseYAML(YAML::Node &yaml_conf); + void ParseWorkOrchestrator(YAML::Node yaml_conf); + void ParseQueueManager(YAML::Node yaml_conf); + void ParseRpcInfo(YAML::Node yaml_conf); +}; + +} // namespace labstor::config + +namespace labstor { +using ServerConfig = config::ServerConfig; +} // namespace labstor + +#endif // LABSTOR_SRC_CONFIG_SERVER_H_ diff --git a/include/labstor/config/config_server_default.h b/include/labstor/config/config_server_default.h new file mode 100644 index 000000000..f7f7067b2 --- /dev/null +++ b/include/labstor/config/config_server_default.h @@ -0,0 +1,58 @@ +#ifndef LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ +#define LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ +const char* kLabstorServerDefaultConfigStr = +"### Runtime orchestration settings\n" +"work_orchestrator:\n" +" # The number of worker threads to spawn\n" +" max_workers: 4\n" +"\n" +"### Queue Manager settings\n" +"queue_manager:\n" +" # The default depth of allocated queues\n" +" queue_depth: 256\n" +" # The maximum number of lanes per queue\n" +" max_lanes: 16\n" +" # The maximum number of queues\n" +" max_queues: 1024\n" +" # The shared memory allocator to use\n" +" shm_allocator: kScalablePageAllocator\n" +" # The name of the shared memory region to create\n" +" shm_name: \"labstor_shm\"\n" +" # The size of the shared memory region to allocate\n" +" shm_size: 0g\n" +"\n" +"### Define properties of RPCs\n" +"rpc:\n" +" # A path to a file containing a list of server names, 1 per line. If your\n" +" # servers are named according to a pattern (e.g., server-1, server-2, etc.),\n" +" # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this\n" +" # option is not empty, it will override anything in `rpc_server_base_name`.\n" +" host_file: \"\"\n" +"\n" +" # Host names can be defined using the following syntax:\n" +" # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ...\n" +" # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ...\n" +" host_names: [\"localhost\"]\n" +"\n" +" # The RPC protocol. This must come from the documentation of the specific RPC\n" +" # library in use.\n" +" protocol: \"ofi+sockets\"\n" +"\n" +" # RPC domain name for verbs transport. Blank for tcp.\n" +" domain: \"\"\n" +"\n" +" # Desired RPC port number.\n" +" port: 8080\n" +"\n" +" # The number of handler threads for each RPC server.\n" +" num_threads: 4\n" +"\n" +"### Task Registry\n" +"task_registry: [\n" +" \'hermes_mdm\',\n" +" \'hermes_blob_mdm\',\n" +" \'hermes_bucket_mdm\',\n" +" \'posix_bdev\',\n" +" \'ram_bdev\'\n" +"]\n"; +#endif // LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file diff --git a/include/labstor/labstor_constants.h b/include/labstor/labstor_constants.h new file mode 100644 index 000000000..9cfb242e8 --- /dev/null +++ b/include/labstor/labstor_constants.h @@ -0,0 +1,29 @@ +// +// Created by lukemartinlogan on 6/22/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_LABSTOR_CONSTANTS_H_ +#define LABSTOR_INCLUDE_LABSTOR_LABSTOR_CONSTANTS_H_ + +namespace labstor { + +#include + +class Constants { + public: + inline static const std::string kClientConfEnv = "LABSTOR_CLIENT_CONF"; + inline static const std::string kServerConfEnv = "LABSTOR_SERVER_CONF"; + + static std::string GetEnvSafe(const std::string &env_name) { + char *data = getenv(env_name.c_str()); + if (data == nullptr) { + return ""; + } else { + return data; + } + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_LABSTOR_CONSTANTS_H_ diff --git a/include/labstor/labstor_namespace.h b/include/labstor/labstor_namespace.h new file mode 100644 index 000000000..d80e18f6f --- /dev/null +++ b/include/labstor/labstor_namespace.h @@ -0,0 +1,35 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_LABSTOR_NAMESPACE_H_ +#define LABSTOR_INCLUDE_LABSTOR_LABSTOR_NAMESPACE_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" + +using labstor::TaskMethod; +using labstor::BinaryOutputArchive; +using labstor::BinaryInputArchive; +using labstor::Task; +using labstor::TaskPointer; +using labstor::MultiQueue; +using labstor::PriorityInfo; +using labstor::Task; +using labstor::TaskFlags; +using labstor::DataTransfer; +using labstor::TaskLib; +using labstor::TaskLibClient; +using labstor::config::QueueManagerInfo; +using labstor::TaskPrio; + +using hshm::RwLock; +using hshm::Mutex; +using hshm::bitfield; +using hshm::bitfield32_t; +typedef hshm::bitfield bitfield64_t; +using hshm::ScopedRwReadLock; +using hshm::ScopedRwWriteLock; +using hipc::LPointer; + +#endif // LABSTOR_INCLUDE_LABSTOR_LABSTOR_NAMESPACE_H_ diff --git a/include/labstor/labstor_types.h b/include/labstor/labstor_types.h new file mode 100644 index 000000000..c6594db35 --- /dev/null +++ b/include/labstor/labstor_types.h @@ -0,0 +1,374 @@ +// +// Created by lukemartinlogan on 6/22/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_LABSTOR_TYPES_H_ +#define LABSTOR_INCLUDE_LABSTOR_LABSTOR_TYPES_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "hermes_shm/data_structures/serialization/shm_serialize.h" +#include +#include +#include +#include +#include "hermes_shm/util/singleton.h" +#include "hermes_shm/constants/macros.h" + +typedef uint8_t u8; /**< 8-bit unsigned integer */ +typedef uint16_t u16; /**< 16-bit unsigned integer */ +typedef uint32_t u32; /**< 32-bit unsigned integer */ +typedef uint64_t u64; /**< 64-bit unsigned integer */ +typedef int8_t i8; /**< 8-bit signed integer */ +typedef int16_t i16; /**< 16-bit signed integer */ +typedef int32_t i32; /**< 32-bit signed integer */ +typedef int64_t i64; /**< 64-bit signed integer */ +typedef float f32; /**< 32-bit float */ +typedef double f64; /**< 64-bit float */ + +namespace labstor { + +using hshm::RwLock; +using hshm::Mutex; +using hshm::bitfield; +using hshm::bitfield32_t; +typedef hshm::bitfield bitfield64_t; +using hshm::ScopedRwReadLock; +using hshm::ScopedRwWriteLock; +using hipc::LPointer; + +/** Determine the mode that LabStor is initialized for */ +enum class LabstorMode { + kNone, + kClient, + kServer +}; + +#define DOMAIN_FLAG_T static inline const int + +/** + * Represents a unique ID for a scheduling domain + * There are a few domain types: + * 1. A node domain, which is a single node + * 2. A global domain, which is all nodes + * 3. A specific domain, which is a subset of nodes + * 4. A specific domain + node, temporarily includes this node in a domain + * */ +struct DomainId { + bitfield32_t flags_; /**< Flags indicating how to interpret id */ + u32 id_; /**< The domain id, 0 is NULL */ + DOMAIN_FLAG_T kLocal = BIT_OPT(u32, 0); /**< Include local node in scheduling decision */ + DOMAIN_FLAG_T kGlobal = BIT_OPT(u32, 1); /**< Use all nodes in scheduling decision */ + DOMAIN_FLAG_T kSet = BIT_OPT(u32, 2); /**< ID represents node set ID, not a single node */ + DOMAIN_FLAG_T kNode = BIT_OPT(u32, 3); /**< ID represents a specific node */ + + /** Serialize domain id */ + template + void serialize(Ar &ar) { + ar(flags_, id_); + } + + /** Default constructor. */ + HSHM_ALWAYS_INLINE + DomainId() : id_(0) {} + + /** Domain has the local node */ + HSHM_ALWAYS_INLINE + bool IsRemote(size_t num_hosts, u32 this_node) const { + if (num_hosts == 1) { + return false; + } else { + return (flags_.Any(kGlobal | kSet) || (flags_.Any(kNode) && id_ != this_node)); + } + // return flags_.Any(kGlobal | kSet | kNode); + } + + /** DomainId representing the local node */ + HSHM_ALWAYS_INLINE + static DomainId GetLocal() { + DomainId id; + id.id_ = 0; + id.flags_.SetBits(kLocal); + return id; + } + + /** Get the ID */ + HSHM_ALWAYS_INLINE + u32 GetId() const { + return id_; + } + + /** Domain is a specific node */ + HSHM_ALWAYS_INLINE + bool IsNode() const { + return flags_.Any(kNode); + } + + /** DomainId representing a specific node */ + HSHM_ALWAYS_INLINE + static DomainId GetNode(u32 node_id) { + DomainId id; + id.id_ = node_id; + id.flags_.SetBits(kNode); + return id; + } + + /** DomainId representing a specific node + local node */ + HSHM_ALWAYS_INLINE + static DomainId GetNodeWithLocal(u32 node_id) { + DomainId id; + id.id_ = node_id; + id.flags_.SetBits(kLocal); + return id; + } + + /** Domain represents all nodes */ + HSHM_ALWAYS_INLINE + bool IsGlobal() const { + return flags_.Any(kGlobal); + } + + /** DomainId representing all nodes */ + HSHM_ALWAYS_INLINE + static DomainId GetGlobal() { + DomainId id; + id.id_ = 0; + id.flags_.SetBits(kGlobal); + return id; + } + + /** DomainId represents a named node set */ + HSHM_ALWAYS_INLINE + bool IsSet() const { + return flags_.Any(kSet); + } + + /** DomainId representing a named node set */ + HSHM_ALWAYS_INLINE + static DomainId GetSet(u32 domain_id) { + DomainId id; + id.id_ = domain_id; + id.flags_.SetBits(kSet); + return id; + } + + /** DomainId representing a node set + local node */ + HSHM_ALWAYS_INLINE + static DomainId GetSetWithLocal(u32 domain_id) { + DomainId id; + id.id_ = domain_id; + id.flags_.SetBits(kSet | kLocal); + return id; + } + + /** Copy constructor */ + HSHM_ALWAYS_INLINE + DomainId(const DomainId &other) { + id_ = other.id_; + flags_ = other.flags_; + } + + /** Copy operator */ + HSHM_ALWAYS_INLINE + DomainId& operator=(const DomainId &other) { + if (this != &other) { + id_ = other.id_; + flags_ = other.flags_; + } + return *this; + } + + /** Move constructor */ + HSHM_ALWAYS_INLINE + DomainId(DomainId &&other) noexcept { + id_ = other.id_; + flags_ = other.flags_; + } + + /** Move operator */ + HSHM_ALWAYS_INLINE + DomainId& operator=(DomainId &&other) noexcept { + if (this != &other) { + id_ = other.id_; + flags_ = other.flags_; + } + return *this; + } + + /** Equality operator */ + HSHM_ALWAYS_INLINE + bool operator==(const DomainId &other) const { + return id_ == other.id_ && flags_.bits_ == other.flags_.bits_; + } + + /** Inequality operator */ + HSHM_ALWAYS_INLINE + bool operator!=(const DomainId &other) const { + return id_ != other.id_ || flags_.bits_ != other.flags_.bits_; + } +}; + +/** Represents unique ID for states + queues */ +template +struct UniqueId { + u32 node_id_; /**< The node the content is on */ + u64 unique_; /**< A unique id for the blob */ + + /** Serialization */ + template + void serialize(Ar &ar) { + ar & node_id_; + ar & unique_; + } + + /** Default constructor */ + HSHM_ALWAYS_INLINE + UniqueId() = default; + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE + UniqueId(u32 node_id, u64 unique) : node_id_(node_id), unique_(unique) {} + + /** Copy constructor */ + HSHM_ALWAYS_INLINE + UniqueId(const UniqueId &other) { + node_id_ = other.node_id_; + unique_ = other.unique_; + } + + /** Copy constructor */ + template + HSHM_ALWAYS_INLINE + UniqueId(const UniqueId &other) { + node_id_ = other.node_id_; + unique_ = other.unique_; + } + + /** Copy assignment */ + HSHM_ALWAYS_INLINE + UniqueId& operator=(const UniqueId &other) { + if (this != &other) { + node_id_ = other.node_id_; + unique_ = other.unique_; + } + return *this; + } + + /** Move constructor */ + HSHM_ALWAYS_INLINE + UniqueId(UniqueId &&other) noexcept { + node_id_ = other.node_id_; + unique_ = other.unique_; + } + + /** Move assignment */ + HSHM_ALWAYS_INLINE + UniqueId& operator=(UniqueId &&other) noexcept { + if (this != &other) { + node_id_ = other.node_id_; + unique_ = other.unique_; + } + return *this; + } + + /** Check if null */ + [[nodiscard]] + HSHM_ALWAYS_INLINE bool IsNull() const { + return node_id_ == 0; + } + + /** Get null id */ + HSHM_ALWAYS_INLINE + static UniqueId GetNull() { + static const UniqueId id(0, 0); + return id; + } + + /** Set to null id */ + HSHM_ALWAYS_INLINE + void SetNull() { + node_id_ = 0; + unique_ = 0; + } + + /** Get id of node from this id */ + [[nodiscard]] + HSHM_ALWAYS_INLINE + u32 GetNodeId() const { return node_id_; } + + /** Compare two ids for equality */ + HSHM_ALWAYS_INLINE + bool operator==(const UniqueId &other) const { + return unique_ == other.unique_ && node_id_ == other.node_id_; + } + + /** Compare two ids for inequality */ + HSHM_ALWAYS_INLINE + bool operator!=(const UniqueId &other) const { + return unique_ != other.unique_ || node_id_ != other.node_id_; + } +}; + +/** Uniquely identify a task state */ +using TaskStateId = UniqueId<1>; +/** Uniquely identify a queue */ +using QueueId = UniqueId<2>; +/** Uniquely identify a task */ +using TaskId = UniqueId<3>; + +/** Allow unique ids to be printed as strings */ +template +std::ostream &operator<<(std::ostream &os, UniqueId const &obj) { + return os << (std::to_string(obj.node_id_) + "." + + std::to_string(obj.unique_)); +} + +/** The types of I/O that can be performed (for IoCall RPC) */ +enum class IoType { + kRead, + kWrite, + kNone +}; + +} // namespace labstor + +namespace std { + +/** Hash function for UniqueId */ +template +struct hash> { + HSHM_ALWAYS_INLINE + std::size_t operator()(const labstor::UniqueId &key) const { + return + std::hash{}(key.unique_) + + std::hash{}(key.node_id_); + } +}; + +/** Hash function for DomainId */ +template<> +struct hash { + HSHM_ALWAYS_INLINE + std::size_t operator()(const labstor::DomainId &key) const { + return + std::hash{}(key.id_) + + std::hash{}(key.flags_.bits_); + } +}; + +} // namespace std + +#endif // LABSTOR_INCLUDE_LABSTOR_LABSTOR_TYPES_H_ diff --git a/include/labstor/network/local_serialize.h b/include/labstor/network/local_serialize.h new file mode 100644 index 000000000..09eb83468 --- /dev/null +++ b/include/labstor/network/local_serialize.h @@ -0,0 +1,76 @@ +// +// Created by lukemartinlogan on 9/5/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_NETWORK_LOCAL_SERIALIZE_H_ +#define LABSTOR_INCLUDE_LABSTOR_NETWORK_LOCAL_SERIALIZE_H_ + +#include "hermes_shm/data_structures/data_structure.h" + +namespace labstor { + +/** A class for serializing simple objects into private memory */ +template +class LocalSerialize { + public: + DataT &data_; + public: + LocalSerialize(DataT &data) : data_(data) { + data_.resize(0); + } + + /** left shift operator */ + template + HSHM_ALWAYS_INLINE + LocalSerialize& operator<<(const T &obj) { + if constexpr(std::is_arithmetic::value) { + size_t size = sizeof(T); + size_t off = data_.size(); + data_.resize(off + size); + memcpy(data_.data() + off, &obj, size); + } else if constexpr (std::is_same::value || std::is_same::value) { + size_t size = obj.size(); + size_t off = data_.size(); + data_.resize(off + size); + memcpy(data_.data() + off, obj.data(), size); + } else { + throw std::runtime_error("Cannot serialize object"); + } + return *this; + } +}; + +/** A class for serializing simple objects into private memory */ +template +class LocalDeserialize { + public: + const DataT &data_; + size_t cur_off_ = 0; + public: + LocalDeserialize(const DataT &data) : data_(data) { + cur_off_ = 0; + } + + /** right shift operator */ + template + HSHM_ALWAYS_INLINE + LocalDeserialize& operator>>(T &obj) { + size_t size; + size_t off = cur_off_; + if constexpr(std::is_arithmetic::value) { + size = sizeof(T); + memcpy(&obj, data_.data() + off, size); + } else if constexpr (std::is_same::value || std::is_same::value) { + size = obj.size(); + memcpy(obj.data(), data_.data() + off, size); + } else { + throw std::runtime_error("Cannot serialize object"); + } + cur_off_ += size; + return *this; + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_NETWORK_LOCAL_SERIALIZE_H_ diff --git a/include/labstor/network/rpc.h b/include/labstor/network/rpc.h new file mode 100644 index 000000000..ce4546cef --- /dev/null +++ b/include/labstor/network/rpc.h @@ -0,0 +1,226 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef LABSTOR_RPC_H_ +#define LABSTOR_RPC_H_ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "labstor/labstor_types.h" +#include "labstor/config/config_server.h" + +namespace labstor { + +/** RPC types */ +enum class RpcType { + kThallium +}; + +/** Uniquely identify a host machine */ +struct HostInfo { + u32 node_id_; /**< Hermes-assigned node id */ + std::string hostname_; /**< Host name */ + std::string ip_addr_; /**< Host IP address */ + + HostInfo() = default; + explicit HostInfo(const std::string &hostname, + const std::string &ip_addr, + u32 node_id) + : hostname_(hostname), ip_addr_(ip_addr), node_id_(node_id) {} +}; + +/** Fabric protocols */ + + +/** A structure to represent RPC context. */ +class RpcContext { + public: + ServerConfig *config_; + int port_; /**< port number */ + std::string protocol_; /**< Libfabric provider */ + std::string domain_; /**< Libfabric domain */ + u32 node_id_; /**< the ID of this node */ + int num_threads_; /**< Number of RPC threads */ + std::vector hosts_; /**< Hostname and ip addr per-node */ + + public: + /** Default constructor */ + RpcContext() = default; + + /** Get the nubmer of hosts */ + HSHM_ALWAYS_INLINE + size_t GetNumHosts() { + return hosts_.size(); + } + + /** initialize host info list */ + void ServerInit(ServerConfig *config) { + config_ = config; + port_ = config_->rpc_.port_; + protocol_ = config_->rpc_.protocol_; + domain_ = config_->rpc_.domain_; + num_threads_ = config_->rpc_.num_threads_; + if (hosts_.size()) { return; } + // Uses hosts produced by host_names + auto &hosts = config_->rpc_.host_names_; + + // Get all host info + hosts_.reserve(hosts.size()); + u32 node_id = 1; + for (const auto& name : hosts) { + hosts_.emplace_back(name, _GetIpAddress(name), node_id++); + } + + // Get id of current host + node_id_ = _FindThisHost(); + if (node_id_ == 0 || node_id_ > (u32)hosts_.size()) { + HELOG(kFatal, "Couldn't identify this host.") + } + } + + /** get RPC address */ + std::string GetRpcAddress(const DomainId &domain_id, int port) { + std::string result = config_->rpc_.protocol_ + "://"; + if (!config_->rpc_.domain_.empty()) { + result += config_->rpc_.domain_ + "/"; + } + std::string host_name = GetHostNameFromNodeId(domain_id); + result += host_name + ":" + std::to_string(port); + return result; + } + + /** Get RPC address for this node */ + std::string GetMyRpcAddress() { + return GetRpcAddress(DomainId::GetNode(node_id_), port_); + } + + /** get host name from node ID */ + std::string GetHostNameFromNodeId(const DomainId &domain_id) { + // NOTE(llogan): node_id 0 is reserved as the NULL node + u32 node_id = domain_id.id_; + if (node_id <= 0 || node_id > (i32)hosts_.size()) { + HELOG(kFatal, "Attempted to get from node {}, which is out of " + "the range 1-{}", node_id, hosts_.size() + 1) + } + u32 index = node_id - 1; + return hosts_[index].hostname_; + } + + /** get host name from node ID */ + std::string GetIpAddressFromNodeId(const DomainId &domain_id){ + // NOTE(llogan): node_id 0 is reserved as the NULL node + u32 node_id = domain_id.id_; + if (node_id <= 0 || node_id > (u32)hosts_.size()) { + HELOG(kFatal, "Attempted to get from node {}, which is out of " + "the range 1-{}", node_id, hosts_.size() + 1) + } + u32 index = node_id - 1; + return hosts_[index].ip_addr_; + } + + /** Get RPC protocol */ + std::string GetProtocol() { + return config_->rpc_.protocol_; + } + + private: + /** Get the node ID of this machine according to hostfile */ + int _FindThisHost() { + int node_id = 1; + for (HostInfo &host : hosts_) { + if (_IsAddressLocal(host.ip_addr_)) { + return node_id; + } + ++node_id; + } + HELOG(kFatal, "Could not identify this host"); + return -1; + } + + /** Check if an IP address is local */ + bool _IsAddressLocal(const std::string &addr) { + struct ifaddrs* ifAddrList = nullptr; + bool found = false; + + if (getifaddrs(&ifAddrList) == -1) { + perror("getifaddrs"); + return false; + } + + for (struct ifaddrs* ifAddr = ifAddrList; + ifAddr != nullptr; ifAddr = ifAddr->ifa_next) { + if (ifAddr->ifa_addr == nullptr || + ifAddr->ifa_addr->sa_family != AF_INET) { + continue; + } + + struct sockaddr_in* sin = + reinterpret_cast(ifAddr->ifa_addr); + char ipAddress[INET_ADDRSTRLEN] = {0}; + inet_ntop(AF_INET, &(sin->sin_addr), ipAddress, INET_ADDRSTRLEN); + + if (addr == ipAddress) { + found = true; + break; + } + } + + freeifaddrs(ifAddrList); + return found; + } + + /** Get IPv4 address from the host with "host_name" */ + std::string _GetIpAddress(const std::string &host_name) { + struct hostent hostname_info = {}; + struct hostent *hostname_result; + int hostname_error = 0; + char hostname_buffer[4096] = {}; +#ifdef __APPLE__ + hostname_result = gethostbyname(host_name.c_str()); + in_addr **addr_list = (struct in_addr **)hostname_result->h_addr_list; +#else + int gethostbyname_result = + gethostbyname_r(host_name.c_str(), &hostname_info, hostname_buffer, + 4096, &hostname_result, &hostname_error); + if (gethostbyname_result != 0) { + HELOG(kFatal, hstrerror(h_errno)) + } + in_addr **addr_list = (struct in_addr **)hostname_info.h_addr_list; +#endif + if (!addr_list[0]) { + HELOG(kFatal, hstrerror(h_errno)) + } + + char ip_address[INET_ADDRSTRLEN] = {0}; + const char *inet_result = + inet_ntop(AF_INET, addr_list[0], ip_address, INET_ADDRSTRLEN); + if (!inet_result) { + perror("inet_ntop"); + HELOG(kFatal, "inet_ntop failed"); + } + return ip_address; + } +}; + +} // namespace hermes + +#endif // LABSTOR_RPC_H_ diff --git a/include/labstor/network/rpc_thallium.h b/include/labstor/network/rpc_thallium.h new file mode 100644 index 000000000..3a722e3b3 --- /dev/null +++ b/include/labstor/network/rpc_thallium.h @@ -0,0 +1,315 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_RPC_THALLIUM_H_ +#define HERMES_RPC_THALLIUM_H_ + +#include +#include +#include "hermes_shm/util/singleton.h" + +#include "rpc.h" + +namespace tl = thallium; + +namespace labstor { + +/** + A structure to represent Thallium state +*/ +class ThalliumRpc { + public: + std::atomic kill_requested_; /**< is kill requested? */ + std::unique_ptr client_engine_; /**< pointer to client engine */ + std::unique_ptr server_engine_; /**< pointer to server engine */ + RpcContext *rpc_; + + /** initialize RPC context */ + ThalliumRpc() {} + + /** Initialize server */ + void ServerInit(RpcContext *rpc) { + rpc_ = rpc; + HILOG(kInfo, "Initializing RPC server"); + std::string addr = rpc->GetMyRpcAddress(); + HILOG(kInfo, "Attempting to start server on: {}", addr); + try { + server_engine_ = std::make_unique( + addr, THALLIUM_SERVER_MODE, true, rpc->num_threads_); + } catch (std::exception &e) { + HELOG(kFatal, "RPC init failed for host: {}\n{}", addr, e.what()); + } + std::string rpc_server_name = server_engine_->self(); + HILOG(kInfo, "Serving {} (i.e., {}) with {} RPC threads as node id {}", + rpc_server_name, + addr, + rpc->num_threads_, + rpc->node_id_); + ClientInit(rpc); + } + + /** Initialize client */ + void ClientInit(RpcContext *rpc) { + rpc_ = rpc; + std::string protocol = rpc->GetProtocol(); + client_engine_ = std::make_unique(protocol, + THALLIUM_CLIENT_MODE, + true, 1); + HILOG(kInfo, "This client is on node {} (i.e., {}, proto: {})", + rpc->node_id_, rpc->GetHostNameFromNodeId(DomainId::GetNode(rpc->node_id_)), protocol); + } + + /** Run the daemon */ + void RunDaemon() { + HILOG(kInfo, "Starting the daemon on node: {}", rpc_->node_id_); + server_engine_->enable_remote_shutdown(); + auto prefinalize_callback = [this]() { + HILOG(kInfo, "Finalizing RPCs on node: {}", rpc_->node_id_); + }; + server_engine_->push_prefinalize_callback(prefinalize_callback); + server_engine_->wait_for_finalize(); + HILOG(kInfo, "Daemon has stopped on node: {}", rpc_->node_id_); + } + + /** Stop this daemon */ + void StopThisDaemon() { + StopDaemon(rpc_->node_id_); + } + + /** Stop a thallium daemon */ + void StopDaemon(u32 node_id) { + try { + HILOG(kInfo, "Sending stop signal to: {}", node_id); + std::string server_name = GetServerName(node_id); + tl::endpoint server = client_engine_->lookup(server_name.c_str()); + client_engine_->shutdown_remote_engine(server); + } catch (std::exception &e) { + HELOG(kFatal, e.what()); + } + } + + /** Stop the thallium daemon */ + void StopAllDaemons() { + for (u32 node_id = 1; node_id < (int)rpc_->hosts_.size() + 1; ++node_id) { + StopDaemon(node_id); + } + } + + /** Thallium-compatible server name */ + std::string GetServerName(u32 node_id) { + std::string ip_address = rpc_->GetIpAddressFromNodeId(DomainId::GetNode(node_id)); + return rpc_->protocol_ + "://" + + std::string(ip_address) + + ":" + std::to_string(rpc_->port_); + } + + /** Register an RPC with thallium */ + template + void RegisterRpc(const char *name, RpcLambda &&lambda) { + server_engine_->define(name, std::forward(lambda)); + } + + /** RPC call */ + template + RetT Call(u32 node_id, const char *func_name, Args&&... args) { + HILOG(kDebug, "Calling {} {} -> {}", func_name, rpc_->node_id_, node_id) + try { + std::string server_name = GetServerName(node_id); + tl::remote_procedure remote_proc = client_engine_->define(func_name); + tl::endpoint server = client_engine_->lookup(server_name); + HILOG(kDebug, "Found the server: {}={}", node_id, server_name) + if constexpr(!ASYNC) { + if constexpr (std::is_same::value) { + remote_proc.disable_response(); + remote_proc.on(server)(std::forward(args)...); + } else { + RetT result = remote_proc.on(server)(std::forward(args)...); + return result; + } + } else { + return remote_proc.on(server).async(std::forward(args)...); + } + } catch (tl::margo_exception &err) { + HELOG(kFatal, "Thallium failed on function: {}\n{}", + func_name, err.what()) + exit(1); + } + } + + /** RPC call */ + template + RetT SyncCall(u32 node_id, const char *func_name, Args&&... args) { + return Call( + node_id, func_name, std::forward(args)...); + } + + /** Async RPC call */ + template + thallium::async_response AsyncCall(u32 node_id, const char *func_name, Args&&... args) { + return Call( + node_id, func_name, std::forward(args)...); + } + + /** I/O transfers */ + template + RetT IoCall(i32 node_id, const char *func_name, + IoType type, char *data, size_t size, Args&& ...args) { + HILOG(kDebug, "Calling {} {} -> {}", func_name, rpc_->node_id_, node_id) + std::string server_name = GetServerName(node_id); + tl::bulk_mode flag; + switch (type) { + case IoType::kRead: { + // The "bulk" object will be modified + flag = tl::bulk_mode::write_only; + break; + } + case IoType::kWrite: { + // The "bulk" object will only be read from + flag = tl::bulk_mode::read_only; + break; + } + case IoType::kNone: { + // TODO(llogan) + HELOG(kFatal, "Cannot have none I/O type") + exit(1); + } + } + + tl::remote_procedure remote_proc = client_engine_->define(func_name); + tl::endpoint server = client_engine_->lookup(server_name); + + std::vector> segments(1); + segments[0].first = data; + segments[0].second = size; + + tl::bulk bulk = client_engine_->expose(segments, flag); + if constexpr (!ASYNC) { + if constexpr (std::is_same_v) { + remote_proc.on(server)(bulk, std::forward(args)...); + } else { + return remote_proc.on(server)(bulk, std::forward(args)...); + } + } else { + return remote_proc.on(server).async(bulk, std::forward(args)...); + } + } + + /** Synchronous I/O transfer */ + template + RetT SyncIoCall(i32 node_id, const char *func_name, + IoType type, char *data, size_t size, Args&& ...args) { + return IoCall( + node_id, func_name, type, data, size, std::forward(args)...); + } + + /** I/O transfers */ + template + thallium::async_response AsyncIoCall(u32 node_id, const char *func_name, + IoType type, char *data, size_t size, + Args&& ...args) { + return IoCall( + node_id, func_name, type, data, size, std::forward(args)...); + } + + /** Io transfer at the server */ + size_t IoCallServer(const tl::request &req, const tl::bulk &bulk, + IoType type, char *data, size_t size) { + tl::bulk_mode flag = tl::bulk_mode::write_only; + switch (type) { + case IoType::kRead: { + // The "local_bulk" object will only be read from + flag = tl::bulk_mode::read_only; + // flag = tl::bulk_mode::read_write; + HILOG(kInfo, "(node {}) Reading {} bytes from the server", + rpc_->node_id_, size) + break; + } + case IoType::kWrite: { + // The "local_bulk" object will only be written to + flag = tl::bulk_mode::write_only; + // flag = tl::bulk_mode::read_write; + HILOG(kInfo, "(node {}) Writing {} bytes to the server", + rpc_->node_id_, size) + break; + } + default: { + // NOTE(llogan): Avoids "uninitalized" warning + HELOG(kFatal, "Cannot have none I/O type") + } + } + + tl::endpoint endpoint = req.get_endpoint(); + std::vector> segments(1); + segments[0].first = data; + segments[0].second = size; + tl::bulk local_bulk = server_engine_->expose(segments, flag); + size_t io_bytes = 0; + + try { + switch (type) { + case IoType::kRead: { + // Read from "local_bulk" to "bulk" + io_bytes = bulk.on(endpoint) << local_bulk; + break; + } + case IoType::kWrite: { + // Write to "local_bulk" from "bulk" + io_bytes = bulk.on(endpoint) >> local_bulk; + break; + } + case IoType::kNone: { + HELOG(kFatal, "Cannot have none I/O type") + } + } + } catch (std::exception &e) { + HELOG(kFatal, "(node {}) Failed to perform bulk I/O thallium: {} (type={})", + rpc_->node_id_, e.what(), (type==IoType::kRead)?"read":"write"); + } + if (io_bytes != size) { + HELOG(kFatal, "Failed to perform bulk I/O thallium") + } + return io_bytes; + } + + /** Check if request is complete */ + bool IsDone(thallium::async_response &req) { + return req.received(); + } + + /** Wait for thallium to complete */ + template + RetT Wait(thallium::async_response &req) { + if constexpr(std::is_same_v) { + req.wait().as(); + } else { + return req.wait(); + } + } +}; + +} // namespace hermes + +/** Lets thallium know how to serialize an enum */ +#define SERIALIZE_ENUM(T)\ + template \ + void save(A &ar, T &mode) {\ + int cast = static_cast(mode);\ + ar << cast;\ + }\ + template \ + void load(A &ar, T &mode) {\ + int cast;\ + ar >> cast;\ + mode = static_cast(cast);\ + } + +#endif // HERMES_RPC_THALLIUM_H_ diff --git a/include/labstor/network/serialize.h b/include/labstor/network/serialize.h new file mode 100644 index 000000000..70a44241d --- /dev/null +++ b/include/labstor/network/serialize.h @@ -0,0 +1,261 @@ +// +// Created by lukemartinlogan on 8/7/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_NETWORK_SERIALIZE_H_ +#define LABSTOR_INCLUDE_LABSTOR_NETWORK_SERIALIZE_H_ + +#include "labstor/labstor_types.h" +#include "labstor/task_registry/task.h" +#include +#include +#include +#include +#include +#include + +namespace labstor { + +/** Receiver will read from data_ */ +#define DT_RECEIVER_READ BIT_OPT(u32, 0) + +/** Receiver will write to data_ */ +#define DT_RECEIVER_WRITE BIT_OPT(u32, 1) + +/** Free data_ when the data transfer is complete */ +#define DT_FREE_DATA BIT_OPT(u32, 2) + +/** Indicate how data should be transferred over network */ +template +struct DataTransferBase { + hshm::bitfield32_t flags_; /**< Indicates how data will be accessed */ + void *data_; /**< The virtual address of data on the node */ + size_t data_size_; /**< The amount of data to transfer */ + DomainId node_id_; /**< The node data is located */ + + /** Serialize a data transfer object */ + template + void serialize(Ar &ar) const { + ar(flags_, (size_t)data_, data_size_, node_id_); + } + + /** Default constructor */ + DataTransferBase() = default; + + /** Emplace constructor */ + DataTransferBase(u32 flags, void *data, size_t data_size, + const DomainId &node_id = DomainId::GetLocal()) : + flags_(flags), data_(data), data_size_(data_size), node_id_(node_id) {} + + /** Copy constructor */ + DataTransferBase(const DataTransferBase &xfer) : + flags_(xfer.flags_), data_(xfer.data_), + data_size_(xfer.data_size_), node_id_(xfer.node_id_) {} + + /** Copy assignment */ + DataTransferBase& operator=(const DataTransferBase &xfer) { + flags_ = xfer.flags_; + data_ = xfer.data_; + data_size_ = xfer.data_size_; + node_id_ = xfer.node_id_; + return *this; + } + + /** Move constructor */ + DataTransferBase(DataTransferBase &&xfer) noexcept : + flags_(xfer.flags_), data_(xfer.data_), + data_size_(xfer.data_size_), node_id_(xfer.node_id_) {} + + /** Equality operator */ + bool operator==(const DataTransferBase &other) const { + return flags_.bits_ == other.flags_.bits_ && + data_ == other.data_ && + data_size_ == other.data_size_ && + node_id_ == other.node_id_; + } +}; + +using DataTransfer = DataTransferBase; +using PassDataTransfer = DataTransferBase; + +/** Serialize a data structure */ +template +class BinaryOutputArchive { + public: + std::vector xfer_; + std::stringstream ss_; + cereal::BinaryOutputArchive ar_; + DomainId node_id_; + + public: + /** Default constructor */ + BinaryOutputArchive(const DomainId &node_id) + : node_id_(node_id), ar_(ss_) {} + + /** Serialize using call */ + template + BinaryOutputArchive& operator()(T &var, Args &&...args) { + return Serialize(0, var, std::forward(args)...); + } + + /** Serialize using left shift */ + template + BinaryOutputArchive& operator<<(T &var) { + return Serialize(0, var); + } + + /** Serialize using ampersand */ + template + BinaryOutputArchive& operator&(T &var) { + return Serialize(0, var); + } + + /** Serialize using left shift */ + template + BinaryOutputArchive& operator<<(T &&var) { + return Serialize(0, var); + } + + /** Serialize using ampersand */ + template + BinaryOutputArchive& operator&(T &&var) { + return Serialize(0, var); + } + + /** Serialize an array */ + template + BinaryOutputArchive& write(T *data, size_t count) { + size_t size = count * sizeof(T); + return Serialize(0, cereal::binary_data(data, size)); + } + + /** Serialize a parameter */ + template + BinaryOutputArchive& Serialize(u32 replica, T &var, Args&& ...args) { + if constexpr (IS_TASK(T)) { + if constexpr (IS_SRL(T)) { + if constexpr (is_start) { + if constexpr (USES_SRL_START(T)) { + var.SerializeStart(*this); + } else { + var.SaveStart(*this); + } + } else { + if constexpr (USES_SRL_END(T)) { + var.SerializeEnd(replica, *this); + } else { + var.SaveEnd(*this); + } + } + } + } else if constexpr (std::is_same_v){ + var.node_id_ = node_id_; + xfer_.emplace_back(var); + } else { + ar_ << var; + } + return Serialize(replica, std::forward(args)...); + } + + /** End serialization recursion */ + BinaryOutputArchive& Serialize(u32 replica) { + (void) replica; + return *this; + } + + /** Get serialized data */ + std::vector Get() { + // Serialize metadata parameters + std::string str = ss_.str(); + void *data = nullptr; + if (str.size() > 0) { + data = malloc(str.size()); + memcpy(data, str.data(), str.size()); + ss_.clear(); + } + xfer_.emplace_back(DT_RECEIVER_READ | DT_FREE_DATA, data, str.size(), node_id_); + + // Return transfer buffers + return std::move(xfer_); + } +}; + +/** Desrialize a data structure */ +template +class BinaryInputArchive { + public: + std::vector xfer_; + std::stringstream ss_; + cereal::BinaryInputArchive ar_; + int xfer_off_; + + public: + /** Default constructor */ + BinaryInputArchive(std::vector &xfer) + : xfer_(std::move(xfer)), xfer_off_(0), ss_(), ar_(ss_) { + auto ¶m_xfer = xfer_.back(); + ss_.str(std::string((char*)param_xfer.data_, param_xfer.data_size_)); + } + + /** Deserialize using call */ + template + BinaryInputArchive& operator()(T &var, Args &&...args) { + return Deserialize(0, var, std::forward(args)...); + } + + /** Deserialize using right shift */ + template + BinaryInputArchive& operator>>(T &var) { + return Deserialize(0, var); + } + + /** Deserialize using ampersand */ + template + BinaryInputArchive& operator&(T &var) { + return Deserialize(0, var); + } + + /** Deserialize an array */ + template + BinaryInputArchive& read(T *data, size_t count) { + size_t size = count * sizeof(T); + Deserialize(0, cereal::binary_data(data, size)); + } + + /** Deserialize a parameter */ + template + BinaryInputArchive& Deserialize(u32 replica, T &var, Args&& ...args) { + if constexpr (IS_TASK(T)) { + if constexpr (IS_SRL(T)) { + if constexpr (is_start) { + if constexpr (USES_SRL_START(T)) { + var.SerializeStart(*this); + } else { + var.LoadStart(*this); + } + } else { + if constexpr (USES_SRL_END(T)) { + var.SerializeEnd(replica, *this); + } else { + var.LoadEnd(replica, *this); + } + } + } + } else if constexpr (std::is_same_v) { + var = xfer_[xfer_off_++]; + } else { + ar_ >> var; + } + return Deserialize(replica, std::forward(args)...); + } + + /** End deserialize recursion */ + HSHM_ALWAYS_INLINE + BinaryInputArchive& Deserialize(u32 replica) { + return *this; + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_NETWORK_SERIALIZE_H_ diff --git a/include/labstor/queue_manager/queue.h b/include/labstor/queue_manager/queue.h new file mode 100644 index 000000000..4f284ffd8 --- /dev/null +++ b/include/labstor/queue_manager/queue.h @@ -0,0 +1,105 @@ +// +// Created by lukemartinlogan on 6/27/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_H_ +#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_H_ + +#include "labstor/labstor_types.h" +#include "labstor/task_registry/task.h" +#include + +/** This queue contains only latency-sensitive tasks */ +#define QUEUE_LOW_LATENCY BIT_OPT(u32, 0) +/** This queue is currently being resized */ +#define QUEUE_RESIZE BIT_OPT(u32, 1) +/** This queue is currently processing updates */ +#define QUEUE_UPDATE BIT_OPT(u32, 2) +/** Requests in this queue can be processed in any order */ +#define QUEUE_UNORDERED BIT_OPT(u32, 3) +/** Requests in this queue are long-running */ +#define QUEUE_LONG_RUNNING BIT_OPT(u32, 3) + +namespace labstor { + +/** Prioritization info needed to be set by client */ +struct PriorityInfo { + u32 max_lanes_; /**< Maximum number of lanes in the queue */ + u32 num_lanes_; /**< Current number of lanes in use */ + u32 depth_; /**< The maximum depth of individual lanes */ + bitfield32_t flags_; /**< Scheduling hints for the queue */ + + /** Default constructor */ + PriorityInfo() = default; + + /** Emplace constructor */ + PriorityInfo(u32 num_lanes, u32 max_lanes, u32 depth, u32 flags) { + max_lanes_ = max_lanes; + num_lanes_ = num_lanes; + depth_ = depth; + flags_ = bitfield32_t(flags); + } + + /** Emplace constructor */ + PriorityInfo(u32 num_lanes, u32 max_lanes, u32 depth, bitfield32_t flags) { + max_lanes_ = max_lanes; + num_lanes_ = num_lanes; + depth_ = depth; + flags_ = flags; + } + + /** Copy constructor */ + PriorityInfo(const PriorityInfo &priority) { + max_lanes_ = priority.max_lanes_; + num_lanes_ = priority.num_lanes_; + depth_ = priority.depth_; + flags_ = priority.flags_; + } + + /** Move constructor */ + PriorityInfo(PriorityInfo &&priority) noexcept { + max_lanes_ = priority.max_lanes_; + num_lanes_ = priority.num_lanes_; + depth_ = priority.depth_; + flags_ = priority.flags_; + } + + /** Copy assignment operator */ + PriorityInfo& operator=(const PriorityInfo &priority) { + if (this != &priority) { + max_lanes_ = priority.max_lanes_; + num_lanes_ = priority.num_lanes_; + depth_ = priority.depth_; + flags_ = priority.flags_; + } + return *this; + } + + /** Move assignment operator */ + PriorityInfo& operator=(PriorityInfo &&priority) noexcept { + if (this != &priority) { + max_lanes_ = priority.max_lanes_; + num_lanes_ = priority.num_lanes_; + depth_ = priority.depth_; + flags_ = priority.flags_; + } + return *this; + } + + /** Serialize Priority Info */ + template + void serialize(Ar &ar) { + ar & max_lanes_; + ar & num_lanes_; + ar & depth_; + ar & flags_; + } +}; + +/** Forward declaration of the queue type */ +template +class MultiQueueT; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_H_ diff --git a/include/labstor/queue_manager/queue_factory.h b/include/labstor/queue_manager/queue_factory.h new file mode 100644 index 000000000..7a2f3bf46 --- /dev/null +++ b/include/labstor/queue_manager/queue_factory.h @@ -0,0 +1,19 @@ +// +// Created by llogan on 7/1/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_FACTORY_H_ +#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_FACTORY_H_ + +#include "queues/hshm_queue.h" + +namespace labstor { + +#ifdef QUEUE_TYPE +using MultiQueue = MultiQueueT; +#else +using MultiQueue = MultiQueueT; +#endif +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_FACTORY_H_ diff --git a/include/labstor/queue_manager/queue_manager.h b/include/labstor/queue_manager/queue_manager.h new file mode 100644 index 000000000..52593026a --- /dev/null +++ b/include/labstor/queue_manager/queue_manager.h @@ -0,0 +1,49 @@ +// +// Created by lukemartinlogan on 6/28/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_H_ +#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_H_ + +#include "labstor/config/config_server.h" +#include "labstor/labstor_types.h" +#include "queue_factory.h" + +namespace labstor { + +/** Shared-memory representation of the QueueManager */ +struct QueueManagerShm { + hipc::ShmArchive> queue_map_; + hipc::ShmArchive> tickets_; +}; + +/** A base class inherited by Client & Server QueueManagers */ +class QueueManager { + public: + hipc::vector *queue_map_; /**< Queues which directly interact with tasks states */ + u32 node_id_; /**< The ID of the node this QueueManager is on */ + QueueId admin_queue_; /**< The queue used to submit administrative requests */ + QueueId process_queue_; /**< The queue used to submit tasks from clients */ + TaskStateId admin_task_state_; /**< The ID of the admin queue */ + + public: + void Init(u32 node_id) { + node_id_ = node_id; + admin_queue_ = QueueId(1, 0); + admin_task_state_ = TaskStateId(1, 0); + process_queue_ = QueueId(1, 1); + } + + /** + * Get a queue by ID + * + * TODO(llogan): Maybe make a local hashtable to map id -> ticket? + * */ + HSHM_ALWAYS_INLINE MultiQueue* GetQueue(const QueueId &id) { + return &(*queue_map_)[id.unique_]; + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_H_ diff --git a/include/labstor/queue_manager/queue_manager_client.h b/include/labstor/queue_manager/queue_manager_client.h new file mode 100644 index 000000000..5be5e9001 --- /dev/null +++ b/include/labstor/queue_manager/queue_manager_client.h @@ -0,0 +1,37 @@ +// +// Created by lukemartinlogan on 6/28/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ +#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ + +#include "queue_manager.h" + +namespace labstor { + +#define LABSTOR_QM_CLIENT \ + (&LABSTOR_CLIENT->queue_manager_) + +/** Enable client programs to access queues */ +class QueueManagerClient : public QueueManager { + public: + hipc::Allocator *alloc_; + + public: + /** Default constructor */ + QueueManagerClient() = default; + + /** Destructor*/ + ~QueueManagerClient() = default; + + /** Initialize client */ + void ClientInit(hipc::Allocator *alloc, QueueManagerShm &shm, u32 node_id) { + alloc_ = alloc; + queue_map_ = shm.queue_map_.get(); + Init(node_id); + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ diff --git a/include/labstor/queue_manager/queue_manager_runtime.h b/include/labstor/queue_manager/queue_manager_runtime.h new file mode 100644 index 000000000..8164b1520 --- /dev/null +++ b/include/labstor/queue_manager/queue_manager_runtime.h @@ -0,0 +1,86 @@ +// +// Created by lukemartinlogan on 6/28/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ +#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ + +#include "queue_manager.h" +#include "queue_manager_client.h" + +namespace labstor { + +#define LABSTOR_QM_RUNTIME \ + (&LABSTOR_RUNTIME->queue_manager_) + +/** Administrative queuing actions */ +class QueueManagerRuntime : public QueueManager { + public: + ServerConfig *config_; + size_t max_queues_; + hipc::split_ticket_queue *tickets_; + u32 node_id_; + + public: + /** Default constructor */ + QueueManagerRuntime() = default; + + /** Destructor*/ + ~QueueManagerRuntime() = default; + + /** Create queues in shared memory */ + void ServerInit(hipc::Allocator *alloc, u32 node_id, ServerConfig *config, QueueManagerShm &shm) { + config_ = config; + Init(node_id); + QueueManagerInfo &qm = config_->queue_manager_; + // Initialize ticket queue (ticket 0 is for admin queue) + max_queues_ = qm.max_queues_; + HSHM_MAKE_AR(shm.tickets_, alloc, max_queues_) + for (u64 i = 1; i <= max_queues_; ++i) { + shm.tickets_->emplace(i); + } + // Initialize queue map + HSHM_MAKE_AR0(shm.queue_map_, alloc) + queue_map_ = shm.queue_map_.get(); + queue_map_->resize(max_queues_); + // Create the admin queue + CreateQueue(admin_queue_, { + {1, 1, qm.queue_depth_, QUEUE_UNORDERED} + }); + CreateQueue(process_queue_, { + {1, 1, qm.queue_depth_, QUEUE_UNORDERED}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }); + } + + /** Create a new queue (with pre-allocated ID) in the map */ + void CreateQueue(const QueueId &id, const std::vector &queue_info) { + MultiQueue *queue = GetQueue(id); + if (id.IsNull()) { + HILOG(kDebug, "Cannot create null queue {}", id); + return; + } + if (!queue->id_.IsNull()) { + HILOG(kDebug, "Queue {} already exists", id); + return; + } + HILOG(kDebug, "Creating queue {}", id); + queue_map_->replace(queue_map_->begin() + id.unique_, id, queue_info); + } + + /** + * Remove a queue + * + * For now, this function assumes that the queue is not in use. + * TODO(llogan): don't assume this + * */ + void DestroyQueue(QueueId &id) { + queue_map_->erase(queue_map_->begin() + id.unique_); + tickets_->emplace(id.unique_); + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ diff --git a/include/labstor/queue_manager/queues/hshm_queue.h b/include/labstor/queue_manager/queues/hshm_queue.h new file mode 100644 index 000000000..d9c32ba31 --- /dev/null +++ b/include/labstor/queue_manager/queues/hshm_queue.h @@ -0,0 +1,288 @@ +// +// Created by llogan on 7/1/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ +#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ + +#include "include/labstor/queue_manager/queue.h" + +namespace labstor { + +/** The data stored in a lane */ +struct LaneData { + hipc::Pointer p_; + bool complete_; + + LaneData() = default; + + LaneData(hipc::Pointer &p, bool complete) { + p_ = p; + complete_ = complete; + } +}; + +/** Represents a lane tasks can be stored */ +typedef hipc::mpsc_queue Lane; + +/** Prioritization of different lanes in the queue */ +struct LaneGroup : public PriorityInfo { + u32 prio_; /**< The priority of the lane group */ + u32 num_scheduled_; /**< The number of lanes currently scheduled on workers */ + hipc::ShmArchive> lanes_; /**< The lanes of the queue */ + + /** Default constructor */ + HSHM_ALWAYS_INLINE + LaneGroup() = default; + + /** Set priority info */ + HSHM_ALWAYS_INLINE + LaneGroup(const PriorityInfo &priority) { + max_lanes_ = priority.max_lanes_; + num_lanes_ = priority.num_lanes_; + num_scheduled_ = 0; + depth_ = priority.depth_; + flags_ = priority.flags_; + // prio_ is set externally + } + + /** Copy constructor. Should never actually be called. */ + HSHM_ALWAYS_INLINE + LaneGroup(const LaneGroup &priority) { + max_lanes_ = priority.max_lanes_; + num_lanes_ = priority.num_lanes_; + num_scheduled_ = priority.num_scheduled_; + depth_ = priority.depth_; + flags_ = priority.flags_; + // prio_ is set externally + } + + /** Move constructor. Should never actually be called. */ + HSHM_ALWAYS_INLINE + LaneGroup(LaneGroup &&priority) noexcept { + max_lanes_ = priority.max_lanes_; + num_lanes_ = priority.num_lanes_; + num_scheduled_ = priority.num_scheduled_; + depth_ = priority.depth_; + flags_ = priority.flags_; + // prio_ is set externally + } + + /** Check if this group is long-running or ADMIN */ + HSHM_ALWAYS_INLINE + bool IsLowPriority() { + return flags_.Any(QUEUE_LONG_RUNNING) || prio_ == 0; + } +}; + +/** Represents the HSHM queue type */ +class Hshm {}; + +/** + * The shared-memory representation of a Queue + * */ +template<> +struct MultiQueueT : public hipc::ShmContainer { + SHM_CONTAINER_TEMPLATE((MultiQueueT), (MultiQueueT)) + QueueId id_; /**< Globally unique ID of this queue */ + hipc::ShmArchive> groups_; /**< Divide the lanes into groups */ + bitfield32_t flags_; /**< Flags for the queue */ + + public: + /**==================================== + * Constructor + * ===================================*/ + + /** SHM constructor. Default. */ + explicit MultiQueueT(hipc::Allocator *alloc) { + shm_init_container(alloc); + SetNull(); + } + + /** SHM constructor. */ + explicit MultiQueueT(hipc::Allocator *alloc, const QueueId &id, + const std::vector &prios) { + shm_init_container(alloc); + id_ = id; + HSHM_MAKE_AR0(groups_, GetAllocator()); + groups_->reserve(prios.size()); + for (u32 prio = 0; prio < prios.size(); ++prio) { + const PriorityInfo &prio_info = prios[prio]; + groups_->emplace_back(prio_info); + LaneGroup &lane_group = (*groups_)[prio]; + // Initialize lanes + HSHM_MAKE_AR0(lane_group.lanes_, GetAllocator()); + lane_group.lanes_->reserve(prio_info.max_lanes_); + lane_group.prio_ = prio; + for (u32 lane_id = 0; lane_id < lane_group.num_lanes_; ++lane_id) { + lane_group.lanes_->emplace_back(lane_group.depth_); + Lane &lane = lane_group.lanes_->back(); + lane.flags_ = prio_info.flags_; + } + } + SetNull(); + } + + /**==================================== + * Copy Constructors + * ===================================*/ + + /** SHM copy constructor */ + explicit MultiQueueT(hipc::Allocator *alloc, const MultiQueueT &other) { + shm_init_container(alloc); + SetNull(); + shm_strong_copy_construct_and_op(other); + } + + /** SHM copy assignment operator */ + MultiQueueT& operator=(const MultiQueueT &other) { + if (this != &other) { + shm_destroy(); + shm_strong_copy_construct_and_op(other); + } + return *this; + } + + /** SHM copy constructor + operator main */ + void shm_strong_copy_construct_and_op(const MultiQueueT &other) { + (*groups_) = (*other.groups_); + } + + /**==================================== + * Move Constructors + * ===================================*/ + + /** SHM move constructor. */ + MultiQueueT(hipc::Allocator *alloc, + MultiQueueT &&other) noexcept { + shm_init_container(alloc); + if (GetAllocator() == other.GetAllocator()) { + (*groups_) = std::move(*other.groups_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + + /** SHM move assignment operator. */ + MultiQueueT& operator=(MultiQueueT &&other) noexcept { + if (this != &other) { + shm_destroy(); + if (GetAllocator() == other.GetAllocator()) { + (*groups_) = std::move(*other.groups_); + other.SetNull(); + } else { + shm_strong_copy_construct_and_op(other); + other.shm_destroy(); + } + } + return *this; + } + + /**==================================== + * Destructor + * ===================================*/ + + /** SHM destructor. */ + void shm_destroy_main() { + (*groups_).shm_destroy(); + } + + /** Check if the list is empty */ + HSHM_ALWAYS_INLINE + bool IsNull() const { + return (*groups_).IsNull(); + } + + /** Sets this list as empty */ + HSHM_ALWAYS_INLINE + void SetNull() {} + + /**==================================== + * Helpers + * ===================================*/ + + /** Get the priority struct */ + HSHM_ALWAYS_INLINE LaneGroup& GetGroup(u32 prio) { + return (*groups_)[prio]; + } + + /** Get a lane of the queue */ + HSHM_ALWAYS_INLINE Lane& GetLane(u32 prio, u32 lane_id) { + return (*(*groups_)[prio].lanes_)[lane_id]; + } + + /** Get a lane of the queue */ + HSHM_ALWAYS_INLINE Lane& GetLane(LaneGroup &lane_group, u32 lane_id) { + return (*lane_group.lanes_)[lane_id]; + } + + /** Emplace a SHM pointer to a task */ + HSHM_ALWAYS_INLINE + bool Emplace(u32 prio, u32 lane_hash, hipc::Pointer &p, bool complete = false) { + return Emplace(prio, lane_hash, LaneData(p, complete)); + } + + /** Emplace a SHM pointer to a task */ + bool Emplace(u32 prio, u32 lane_hash, const LaneData &data) { + if (IsEmplacePlugged()) { + WaitForEmplacePlug(); + } + LaneGroup &lane_group = GetGroup(prio); + u32 lane_id = lane_hash % lane_group.num_lanes_; + Lane &lane = GetLane(lane_group, lane_id); + hshm::qtok_t ret = lane.emplace(data); + return !ret.IsNull(); + } + + /** + * Change the number of active lanes + * This assumes that PlugForResize and UnplugForResize are called externally. + * */ + void Resize(u32 num_lanes) { + } + + /** Begin plugging the queue for resize */ + HSHM_ALWAYS_INLINE bool PlugForResize() { + return true; + } + + /** Begin plugging the queue for update tasks */ + HSHM_ALWAYS_INLINE bool PlugForUpdateTask() { + return true; + } + + /** Check if emplace operations are plugged */ + HSHM_ALWAYS_INLINE bool IsEmplacePlugged() { + return flags_.Any(QUEUE_RESIZE); + } + + /** Check if pop operations are plugged */ + HSHM_ALWAYS_INLINE bool IsPopPlugged() { + return flags_.Any(QUEUE_UPDATE | QUEUE_RESIZE); + } + + /** Wait for emplace plug to complete */ + void WaitForEmplacePlug() { + // NOTE(llogan): will this infinite loop due to CPU caching? + while (flags_.Any(QUEUE_UPDATE)) { + HERMES_THREAD_MODEL->Yield(); + } + } + + /** Enable emplace & pop */ + HSHM_ALWAYS_INLINE void UnplugForResize() { + flags_.UnsetBits(QUEUE_RESIZE); + } + + /** Enable pop */ + HSHM_ALWAYS_INLINE void UnplugForUpdateTask() { + flags_.UnsetBits(QUEUE_UPDATE); + } +}; + +} // namespace labstor + + +#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h new file mode 100644 index 000000000..94c4932ff --- /dev/null +++ b/include/labstor/task_registry/task.h @@ -0,0 +1,456 @@ +// +// Created by lukemartinlogan on 6/23/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_REQUEST_H_ +#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_REQUEST_H_ + +#include "labstor/labstor_types.h" +#include "labstor/network/local_serialize.h" +#include + +namespace labstor { + +/** This task reads a state */ +#define TASK_READ BIT_OPT(u32, 0) +/** This task writes to a state */ +#define TASK_WRITE BIT_OPT(u32, 1) +/** This task fundamentally updates a state */ +#define TASK_UPDATE BIT_OPT(u32, 2) +/** This task is paused until a set of tasks complete */ +#define TASK_BLOCKED BIT_OPT(u32, 3) +/** This task is latency-sensitive */ +#define TASK_LOW_LATENCY BIT_OPT(u32, 4) +/** This task makes system calls and may hurt caching */ +#define TASK_SYSCALL BIT_OPT(u32, 5) +/** This task does not depend on state */ +#define TASK_STATELESS BIT_OPT(u32, 6) +/** This task does not depend on its position in the queue */ +#define TASK_UNORDERED BIT_OPT(u32, 7) +/** This task has began execution */ +#define TASK_HAS_STARTED BIT_OPT(u32, 8) +/** This task is completed */ +#define TASK_COMPLETE BIT_OPT(u32, 9) +/** This task was marked completed outside of the worker thread */ +#define TASK_MODULE_COMPLETE BIT_OPT(u32, 10) +/** This task is long-running */ +#define TASK_LONG_RUNNING (BIT_OPT(u32, 11) | TASK_UNORDERED) +/** This task is fire and forget. Free when completed */ +#define TASK_FIRE_AND_FORGET BIT_OPT(u32, 12) +/** This task should not be run at this time */ +#define TASK_DISABLE_RUN BIT_OPT(u32, 13) +/** This task owns the data in the task */ +#define TASK_DATA_OWNER BIT_OPT(u32, 14) +/** This task is marked */ +#define TASK_MARKED BIT_OPT(u32, 15) + +/** Used to define task methods */ +#define TASK_METHOD_T static inline const u32 + +/** The baseline set of tasks */ +struct TaskMethod { + TASK_METHOD_T kConstruct = 0; /**< The constructor of the task */ + TASK_METHOD_T kDestruct = 1; /**< The destructor of the task */ + TASK_METHOD_T kLast = 2; /**< Where the next method should take place */ +}; + +/** + * Let's say we have an I/O request to a device + * I/O requests + MD operations need to be controlled for correctness + * Is there a case where root tasks from different TaskStates need to be ordered? No. + * Tasks spawned from the same root task need to be keyed to the same worker stack + * Tasks apart of the same task group need to be ordered + * */ + +/** An identifier used for representing the location of a task in a task graph */ +struct TaskNode { + TaskId root_; /**< The id of the root task */ + u32 node_depth_; /**< The depth of the task in the task graph */ + + /** Default constructor */ + HSHM_ALWAYS_INLINE + TaskNode() = default; + + /** Emplace constructor for root task */ + HSHM_ALWAYS_INLINE + TaskNode(TaskId root) { + root_ = root; + node_depth_ = 0; + } + + /** Copy constructor */ + HSHM_ALWAYS_INLINE + TaskNode(const TaskNode &other) { + root_ = other.root_; + node_depth_ = other.node_depth_; + } + + /** Copy assignment operator */ + HSHM_ALWAYS_INLINE + TaskNode& operator=(const TaskNode &other) { + root_ = other.root_; + node_depth_ = other.node_depth_; + return *this; + } + + /** Move constructor */ + HSHM_ALWAYS_INLINE + TaskNode(TaskNode &&other) noexcept { + root_ = other.root_; + node_depth_ = other.node_depth_; + } + + /** Move assignment operator */ + HSHM_ALWAYS_INLINE + TaskNode& operator=(TaskNode &&other) noexcept { + root_ = other.root_; + node_depth_ = other.node_depth_; + return *this; + } + + /** Addition operator*/ + HSHM_ALWAYS_INLINE + TaskNode operator+(int i) const { + TaskNode ret; + ret.root_ = root_; + ret.node_depth_ = node_depth_ + i; + return ret; + } + + /** Null task node */ + HSHM_ALWAYS_INLINE + static TaskNode GetNull() { + TaskNode ret; + ret.root_ = TaskId::GetNull(); + ret.node_depth_ = 0; + return ret; + } + + /** Check if null */ + HSHM_ALWAYS_INLINE + bool IsNull() const { + return root_.IsNull(); + } + + /** Check if the root task */ + HSHM_ALWAYS_INLINE + bool IsRoot() const { + return node_depth_ == 0; + } + + /** Serialization*/ + template + void serialize(Ar &ar) { + ar(root_, node_depth_); + } +}; + +/** Allow TaskNode to be printed as strings */ +static inline std::ostream &operator<<(std::ostream &os, const TaskNode &obj) { + return os << obj.root_ << "/" << std::to_string(obj.node_depth_); +} + +/** This task uses SerializeStart */ +#define TF_SRL_SYM_START BIT_OPT(u32, 0) +/** This task uses SaveStart + LoadStart */ +#define TF_SRL_ASYM_START BIT_OPT(u32, 1) +/** This task uses SerializeEnd */ +#define TF_SRL_SYM_END BIT_OPT(u32, 2) +/** This task uses SaveEnd + LoadEnd */ +#define TF_SRL_ASYM_END BIT_OPT(u32, 3) +/** This task uses symmetric serialization */ +#define TF_SRL_SYM (TF_SRL_SYM_START | TF_SRL_SYM_END) +/** This task uses asymmetric serialization */ +#define TF_SRL_ASYM (TF_SRL_ASYM_START | TF_SRL_ASYM_END) +/** This task uses replication */ +#define TF_REPLICA BIT_OPT(u32, 31) +/** This task is intended to be used only locally */ +#define TF_LOCAL BIT_OPT(u32, 5) + +/** All tasks inherit this to easily check if a class is a task using SFINAE */ +class IsTask {}; +/** The type of a compile-time task flag */ +#define TASK_FLAG_T constexpr inline static bool +/** Determine this is a task */ +#define IS_TASK(T) \ + std::is_base_of_v +/** Determine this task supports serialization */ +#define IS_SRL(T) \ + T::SUPPORTS_SRL +/** Determine this task uses SerializeStart */ +#define USES_SRL_START(T) \ + T::SRL_SYM_START +/** Determine this task uses SerializeEnd */ +#define USES_SRL_END(T) \ + T::SRL_SYM_END +/** Call replica start if applicable */ +template +constexpr inline void CALL_REPLICA_START(u32 count, T *task) { + if constexpr(T::REPLICA) { + task->ReplicateStart(count); + } +} +/** Call replica end if applicable */ +template +constexpr inline void CALL_REPLICA_END(T *task) { + if constexpr(T::REPLICA) { + task->ReplicateEnd(); + } +} + +/** Compile-time flags indicating task methods and operation support */ +template +struct TaskFlags : public IsTask { + public: + TASK_FLAG_T SUPPORTS_SRL = FLAGS & (TF_SRL_SYM | TF_SRL_ASYM); + TASK_FLAG_T SRL_SYM_START = FLAGS & TF_SRL_SYM_START; + TASK_FLAG_T SRL_SYM_END = FLAGS & TF_SRL_SYM_END; + TASK_FLAG_T REPLICA = FLAGS & TF_REPLICA; +}; + +/** The type of a compile-time task flag */ +#define TASK_PRIO_T constexpr inline static int + +/** Prioritization of tasks */ +class TaskPrio { + public: + TASK_PRIO_T kAdmin = 0; + TASK_PRIO_T kLongRunning = 1; + TASK_PRIO_T kLowLatency = 2; +}; + +/** A generic task base class */ +struct Task : public hipc::ShmContainer { + SHM_CONTAINER_TEMPLATE((Task), (Task)) + public: + TaskStateId task_state_; /**< The unique name of a task state */ + TaskNode task_node_; /**< The unique ID of this task in the graph */ + DomainId domain_id_; /**< The nodes that the task should run on */ + u32 prio_; /**< An indication of the priority of the request */ + u32 lane_hash_; /**< Determine the lane a task is keyed to */ + u32 method_; /**< The method to call in the state */ + bitfield32_t task_flags_; /**< Properties of the task */ + + /**==================================== + * Task Helpers + * ===================================*/ + + /** Set task as externally complete */ + HSHM_ALWAYS_INLINE void SetModuleComplete() { + task_flags_.SetBits(TASK_MODULE_COMPLETE); + } + + /** Check if a task marked complete externally */ + HSHM_ALWAYS_INLINE bool IsModuleComplete() { + return task_flags_.Any(TASK_MODULE_COMPLETE); + } + + /** Check if a task is fire & forget */ + HSHM_ALWAYS_INLINE bool IsFireAndForget() { + return task_flags_.Any(TASK_FIRE_AND_FORGET); + } + + /** Unset fire & forget */ + HSHM_ALWAYS_INLINE void UnsetFireAndForget() { + task_flags_.UnsetBits(TASK_FIRE_AND_FORGET); + } + + /** Check if task is long running */ + HSHM_ALWAYS_INLINE bool IsLongRunning() { + return task_flags_.Any(TASK_LONG_RUNNING); + } + + /** Check if task is unordered */ + HSHM_ALWAYS_INLINE bool IsUnordered() { + return task_flags_.Any(TASK_UNORDERED); + } + + /** Set task as unordered */ + HSHM_ALWAYS_INLINE bool SetUnordered() { + return task_flags_.Any(TASK_UNORDERED); + } + + /** Set task as complete */ + HSHM_ALWAYS_INLINE void SetComplete() { + task_flags_.SetBits(TASK_MODULE_COMPLETE | TASK_COMPLETE); + } + + /** Check if task is complete */ + HSHM_ALWAYS_INLINE bool IsComplete() { + return task_flags_.Any(TASK_COMPLETE); + } + + /** Disable the running of a task */ + HSHM_ALWAYS_INLINE void DisableRun() { + task_flags_.SetBits(TASK_DISABLE_RUN); + } + + /** Check if running task is disable */ + HSHM_ALWAYS_INLINE bool IsRunDisabled() { + return task_flags_.Any(TASK_DISABLE_RUN); + } + + /** Set task as data owner */ + HSHM_ALWAYS_INLINE void SetDataOwner() { + task_flags_.SetBits(TASK_DATA_OWNER); + } + + /** Check if task is data owner */ + HSHM_ALWAYS_INLINE bool IsDataOwner() { + return task_flags_.Any(TASK_DATA_OWNER); + } + + /** Unset task as data owner */ + HSHM_ALWAYS_INLINE void UnsetDataOwner() { + task_flags_.UnsetBits(TASK_DATA_OWNER); + } + + /** Set this task as started */ + HSHM_ALWAYS_INLINE void SetStarted() { + task_flags_.SetBits(TASK_HAS_STARTED); + } + + /** Set this task as started */ + HSHM_ALWAYS_INLINE void UnsetStarted() { + task_flags_.UnsetBits(TASK_HAS_STARTED); + } + + /** Check if task has started */ + HSHM_ALWAYS_INLINE bool IsStarted() { + return task_flags_.Any(TASK_HAS_STARTED); + } + + /** Set this task as started */ + HSHM_ALWAYS_INLINE void SetMarked() { + task_flags_.SetBits(TASK_MARKED); + } + + /** Set this task as started */ + HSHM_ALWAYS_INLINE void UnsetMarked() { + task_flags_.UnsetBits(TASK_MARKED); + } + + /** Check if task is marked */ + HSHM_ALWAYS_INLINE bool IsMarked() { + return task_flags_.Any(TASK_MARKED); + } + + /** Wait for task to complete */ + template + void Wait() { + while (!IsComplete()) { + for (int i = 0; i < 100000; ++i) { + if (IsComplete()) { + return; + } + } + if constexpr(THREAD_MODEL == 0) { + HERMES_THREAD_MODEL->Yield(); + } else { + ABT_thread_yield(); + } + } + } + + /**==================================== + * Default Constructor + * ===================================*/ + + /** Default SHM constructor */ + HSHM_ALWAYS_INLINE explicit + Task(hipc::Allocator *alloc) { + shm_init_container(alloc); + } + + /** SHM constructor */ + HSHM_ALWAYS_INLINE explicit + Task(hipc::Allocator *alloc, + const TaskNode &task_node) { + shm_init_container(alloc); + task_node_ = task_node; + } + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + Task(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &task_state, + u32 lane_hash, + u32 method, + bitfield32_t task_flags) { + shm_init_container(alloc); + task_node_ = task_node; + lane_hash_ = lane_hash; + prio_ = TaskPrio::kLowLatency; + task_state_ = task_state; + method_ = method; + domain_id_ = domain_id; + task_flags_ = task_flags; + } + + /**==================================== + * Copy Constructors + * ===================================*/ + + /** SHM copy constructor */ + HSHM_ALWAYS_INLINE explicit Task(hipc::Allocator *alloc, const Task &other) {} + + /** SHM copy assignment operator */ + HSHM_ALWAYS_INLINE Task& operator=(const Task &other) { + return *this; + } + + /**==================================== + * Move Constructors + * ===================================*/ + + /** SHM move constructor. */ + HSHM_ALWAYS_INLINE Task(hipc::Allocator *alloc, Task &&other) noexcept {} + + /** SHM move assignment operator. */ + HSHM_ALWAYS_INLINE Task& operator=(Task &&other) noexcept { + return *this; + } + + /**==================================== + * Destructor + * ===================================*/ + + /** SHM destructor. */ + HSHM_ALWAYS_INLINE void shm_destroy_main() {} + + /** Check if the Task is empty */ + HSHM_ALWAYS_INLINE bool IsNull() const { return false; } + + /** Sets this Task as empty */ + HSHM_ALWAYS_INLINE void SetNull() {} + + /**==================================== + * Serialization + * ===================================*/ + template + void task_serialize(Ar &ar) { + ar(task_state_, task_node_, domain_id_, lane_hash_, prio_, method_, task_flags_); + } + + /**==================================== + * Grouping + * ===================================*/ + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** Decorator macros */ +#define IN +#define OUT +#define INOUT +#define TEMP + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_REQUEST_H_ diff --git a/include/labstor/task_registry/task_lib.h b/include/labstor/task_registry/task_lib.h new file mode 100644 index 000000000..e67de5852 --- /dev/null +++ b/include/labstor/task_registry/task_lib.h @@ -0,0 +1,143 @@ +// +// Created by lukemartinlogan on 6/23/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_TASK_TASK_H_ +#define LABSTOR_INCLUDE_LABSTOR_TASK_TASK_H_ + +#include +#include "labstor/labstor_types.h" +#include "labstor/queue_manager/queue_factory.h" +#include "labstor/network/serialize.h" +#include "task.h" + +namespace labstor { + +struct TaskPointer { + Task *task_; + hipc::Pointer p_; + + /** Default constructor */ + TaskPointer() : task_(nullptr) {} + + /** Task-only constructor */ + TaskPointer(Task *task) : task_(task) {} + + /** Emplace constructor */ + TaskPointer(Task *task, hipc::Pointer p) : task_(task), p_(p) {} + + /** Copy constructor */ + TaskPointer(const TaskPointer &other) : task_(other.task_), p_(other.p_) {} + + /** Copy operator */ + TaskPointer &operator=(const TaskPointer &other) { + task_ = other.task_; + p_ = other.p_; + return *this; + } + + /** Move constructor */ + TaskPointer(TaskPointer &&other) noexcept + : task_(other.task_), p_(other.p_) { + other.task_ = nullptr; + other.p_ = hipc::Pointer(); + } + + /** Move operator */ + TaskPointer &operator=(TaskPointer &&other) noexcept { + task_ = other.task_; + p_ = other.p_; + other.task_ = nullptr; + other.p_ = hipc::Pointer(); + return *this; + } +}; + +/** + * Represents a custom operation to perform. + * Tasks are independent of Hermes. + * */ +class TaskLib { + public: + TaskStateId id_; /**< The unique name of a task state */ + QueueId queue_id_; /**< The queue id of a task state */ + std::string name_; /**< The unique semantic name of a task state */ + + /** Default constructor */ + TaskLib() : id_(TaskStateId::GetNull()) {} + + /** Emplace Constructor */ + void Init(const TaskStateId &id, const std::string &name) { + id_ = id; + queue_id_ = QueueId(id); + name_ = name; + } + + /** Virtual destructor */ + virtual ~TaskLib() = default; + + /** Run a method of the task */ + virtual void Run(u32 method, Task *task) = 0; + + /** Allow task to store replicas of completion */ + virtual void ReplicateStart(u32 method, u32 count, Task *task) = 0; + + /** Can be used to summarize the completions */ + virtual void ReplicateEnd(u32 method, Task *task) = 0; + + /** Serialize a task when initially pushing into remote */ + virtual std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) = 0; + + /** Deserialize a task when popping from remote queue */ + virtual TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) = 0; + + /** Serialize a task when returning from remote queue */ + virtual std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) = 0; + + /** Deserialize a task when returning from remote queue */ + virtual void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) = 0; + + /** Deserialize a task when returning from remote queue */ + virtual u32 GetGroup(u32 method, Task *task, hshm::charbuf &buf) = 0; +}; + +/** Represents a TaskLib in action */ +typedef TaskLib TaskState; + +/** Represents the TaskLib client-side */ +class TaskLibClient { + public: + TaskStateId id_; + QueueId queue_id_; + + public: + /** Init from existing ID */ + void Init(const TaskStateId &id) { + id_ = id; + queue_id_ = QueueId(id_); + } +}; + +extern "C" { +/** The two methods provided by all tasks */ +typedef TaskState* (*create_state_t)(Task *task, const char *state_name); +/** Get the name of a task */ +typedef const char* (*get_task_lib_name_t)(void); +} // extern c + +/** Used internally by task source file */ +#define LABSTOR_TASK_CC(TRAIT_CLASS, TASK_NAME) \ + extern "C" { \ + void* create_state(labstor::Admin::CreateTaskStateTask *task, const char *state_name) { \ + labstor::TaskState *exec = reinterpret_cast( \ + new TYPE_UNWRAP(TRAIT_CLASS)()); \ + exec->Init(task->id_, state_name); \ + exec->Run(labstor::TaskMethod::kConstruct, task); \ + return exec; \ + } \ + const char* get_task_lib_name(void) { return TASK_NAME; } \ + bool is_labstor_task_ = true; \ + } +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_TASK_TASK_H_ diff --git a/include/labstor/task_registry/task_registry.h b/include/labstor/task_registry/task_registry.h new file mode 100644 index 000000000..c294faf08 --- /dev/null +++ b/include/labstor/task_registry/task_registry.h @@ -0,0 +1,299 @@ +// +// Created by lukemartinlogan on 6/23/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_TASK_TASK_REGISTRY_H_ +#define LABSTOR_INCLUDE_LABSTOR_TASK_TASK_REGISTRY_H_ + +#include +#include +#include +#include +#include +#include "task_lib.h" +#include "labstor/config/config_server.h" +#include "labstor_admin/labstor_admin.h" + +namespace stdfs = std::filesystem; + +namespace labstor { + +/** All information needed to create a trait */ +struct TaskLibInfo { + void *lib_; /**< The dlfcn library */ + create_state_t create_state_; /**< The create task function */ + get_task_lib_name_t get_task_lib_name; /**< The get task name function */ + + /** Default constructor */ + TaskLibInfo() = default; + + /** Destructor */ + ~TaskLibInfo() { + if (lib_) { + dlclose(lib_); + } + } + + /** Emplace constructor */ + explicit TaskLibInfo(void *lib, + create_state_t create_task, + get_task_lib_name_t get_task_name) + : lib_(lib), create_state_(create_task), get_task_lib_name(get_task_name) {} + + /** Copy constructor */ + TaskLibInfo(const TaskLibInfo &other) + : lib_(other.lib_), + create_state_(other.create_state_), + get_task_lib_name(other.get_task_lib_name) {} + + /** Move constructor */ + TaskLibInfo(TaskLibInfo &&other) noexcept + : lib_(other.lib_), + create_state_(other.create_state_), + get_task_lib_name(other.get_task_lib_name) { + other.lib_ = nullptr; + other.create_state_ = nullptr; + other.get_task_lib_name = nullptr; + } +}; + +/** + * Stores the registered set of TaskLibs and TaskStates + * */ +class TaskRegistry { + public: + /** The node the registry is on */ + u32 node_id_; + /** The dirs to search for task libs */ + std::vector lib_dirs_; + /** Map of a semantic lib name to lib info */ + std::unordered_map libs_; + /** Map of a semantic exec name to exec id */ + std::unordered_map task_state_ids_; + /** Map of a semantic exec id to state */ + std::unordered_map task_states_; + /** A unique identifier counter */ + std::atomic *unique_; + + public: + /** Default constructor */ + TaskRegistry() {} + + /** Initialize the Task Registry */ + void ServerInit(ServerConfig *config, u32 node_id, std::atomic &unique) { + node_id_ = node_id; + unique_ = &unique; + + // Load the LD_LIBRARY_PATH variable + auto ld_lib_path_env = getenv("LD_LIBRARY_PATH"); + std::string ld_lib_path; + if (ld_lib_path_env) { + ld_lib_path = ld_lib_path_env; + } + + // Load the LABSTOR_TASK_PATH variable + std::string hermes_lib_path; + auto hermes_lib_path_env = getenv("LABSTOR_TASK_PATH"); + if (hermes_lib_path_env) { + hermes_lib_path = hermes_lib_path_env; + } + + // Combine LD_LIBRARY_PATH and LABSTOR_TASK_PATH + std::string paths = hermes_lib_path + ":" + ld_lib_path; + std::stringstream ss(paths); + std::string lib_dir; + while (std::getline(ss, lib_dir, ':')) { + lib_dirs_.emplace_back(lib_dir); + } + + // Find each lib in LD_LIBRARY_PATH + for (const std::string &lib_name : config->task_libs_) { + if (!RegisterTaskLib(lib_name)) { + HELOG(kWarning, "Failed to load the lib: {}", lib_name); + } + } + } + + /** Load a task lib */ + bool RegisterTaskLib(const std::string &lib_name) { + std::string lib_dir; + for (const std::string &lib_dir : lib_dirs_) { + // Determine if this directory contains the library + std::string lib_path1 = hshm::Formatter::format("{}/{}.so", + lib_dir, + lib_name); + std::string lib_path2 = hshm::Formatter::format("{}/lib{}.so", + lib_dir, + lib_name); + std::string lib_path; + if (stdfs::exists(lib_path1)) { + lib_path = std::move(lib_path1); + } else if (stdfs::exists(lib_path2)) { + lib_path = std::move(lib_path2); + } else { + continue; + } + + // Load the library + TaskLibInfo info; + info.lib_ = dlopen(lib_path.c_str(), RTLD_GLOBAL | RTLD_NOW); + if (!info.lib_) { + HELOG(kError, "Could not open the lib library: {}. Reason: {}", lib_path, dlerror()); + return false; + } + info.create_state_ = (create_state_t)dlsym( + info.lib_, "create_state"); + if (!info.create_state_) { + HELOG(kError, "The lib {} does not have create_state symbol", + lib_path); + return false; + } + info.get_task_lib_name = (get_task_lib_name_t)dlsym( + info.lib_, "get_task_lib_name"); + if (!info.get_task_lib_name) { + HELOG(kError, "The lib {} does not have get_task_lib_name symbol", + lib_path); + return false; + } + std::string task_lib_name = info.get_task_lib_name(); + HILOG(kInfo, "Finished loading the lib: {}", task_lib_name) + libs_.emplace(task_lib_name, std::move(info)); + return true; + } + HELOG(kError, "Could not find the lib: {}", lib_name); + return false; + } + + /** Destroy a task lib */ + void DestroyTaskLib(const std::string &lib_name) { + auto it = libs_.find(lib_name); + if (it == libs_.end()) { + HELOG(kError, "Could not find the task lib: {}", lib_name); + return; + } + libs_.erase(it); + } + + /** Allocate a task state ID */ + HSHM_ALWAYS_INLINE + TaskStateId CreateTaskStateId() { + return TaskStateId(node_id_, unique_->fetch_add(1)); + } + + /** Check if task state exists by ID */ + HSHM_ALWAYS_INLINE + bool TaskStateExists(const TaskStateId &state_id) { + auto it = task_states_.find(state_id); + return it != task_states_.end(); + } + + /** + * Create a task state + * state_id must not be NULL. + * */ + bool CreateTaskState(const char *lib_name, + const char *state_name, + const TaskStateId &state_id, + Admin::CreateTaskStateTask *task) { + // Ensure state_id is not NULL + if (state_id.IsNull()) { + HILOG(kError, "The task state ID cannot be null"); + task->SetModuleComplete(); + return false; + } + HILOG(kInfo, "(node {}) Creating an instance of {} with name {}", + LABSTOR_CLIENT->node_id_, lib_name, state_name) + + // Find the task library to instantiate + auto it = libs_.find(lib_name); + if (it == libs_.end()) { + HELOG(kError, "Could not find the task lib: {}", lib_name); + task->SetModuleComplete(); + return false; + } + + // Ensure the task state does not already exist + if (TaskStateExists(state_id)) { + HELOG(kError, "The task state already exists: {}", state_name); + task->SetModuleComplete(); + return true; + } + + // Create the state instance + task->id_ = state_id; + TaskLibInfo &info = it->second; + TaskState *task_state = info.create_state_(task, state_name); + if (!task_state) { + HELOG(kError, "Could not create the task state: {}", state_name); + task->SetModuleComplete(); + return false; + } + + // Add the state to the registry + task_state->id_ = state_id; + task_state->name_ = state_name; + task_state_ids_.emplace(state_name, state_id); + task_states_.emplace(state_id, task_state); + HILOG(kInfo, "(node {}) Allocated an instance of {} with name {} and ID {}", + LABSTOR_CLIENT->node_id_, lib_name, state_name, state_id) + return true; + } + + /** Get or create a task state's ID */ + TaskStateId GetOrCreateTaskStateId(const std::string &state_name) { + auto it = task_state_ids_.find(state_name); + if (it == task_state_ids_.end()) { + TaskStateId state_id = CreateTaskStateId(); + task_state_ids_.emplace(state_name, state_id); + return state_id; + } + return it->second; + } + + /** Get a task state's ID */ + TaskStateId GetTaskStateId(const std::string &state_name) { + auto it = task_state_ids_.find(state_name); + if (it == task_state_ids_.end()) { + return TaskStateId::GetNull(); + } + return it->second; + } + + /** Get a task state instance */ + TaskState* GetTaskState(const TaskStateId &task_state_id) { + auto it = task_states_.find(task_state_id); + if (it == task_states_.end()) { + return nullptr; + } + return it->second; + } + + /** Get task state instance by name OR by ID */ + TaskState* GetTaskState(const std::string &task_name, const TaskStateId &task_state_id) { + TaskStateId id = GetTaskStateId(task_name); + if (id.IsNull()) { + id = task_state_id; + } + return GetTaskState(id); + } + + /** Destroy a task state */ + void DestroyTaskState(const TaskStateId &task_state_id) { + auto it = task_states_.find(task_state_id); + if (it == task_states_.end()) { + HELOG(kWarning, "Could not find the task state"); + return; + } + TaskState *task_state = it->second; + task_state_ids_.erase(task_state->name_); + task_states_.erase(it); + delete task_state; + } +}; + +/** Singleton macro for task registry */ +#define LABSTOR_TASK_REGISTRY \ + (&LABSTOR_RUNTIME->task_registry_) +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_TASK_TASK_REGISTRY_H_ diff --git a/include/labstor/work_orchestrator/affinity.h b/include/labstor/work_orchestrator/affinity.h new file mode 100644 index 000000000..903eab9ad --- /dev/null +++ b/include/labstor/work_orchestrator/affinity.h @@ -0,0 +1,27 @@ +// +// Created by lukemartinlogan on 7/5/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_AFFINITY_H_ +#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_AFFINITY_H_ + +#include + +class ProcessAffiner { + public: + /** Set the CPU affinity of a process */ + static void SetCpuAffinity(int pid, int cpu_id) { + // Create a CPU set and set CPU affinity to CPU 0 + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(cpu_id, &cpuset); + + // Set the CPU affinity of the process + int result = sched_setaffinity(pid, sizeof(cpuset), &cpuset); + if (result == -1) { + // HELOG(kError, "Failed to set CPU affinity for process {}", pid); + } + } +}; + +#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_AFFINITY_H_ diff --git a/include/labstor/work_orchestrator/scheduler.h b/include/labstor/work_orchestrator/scheduler.h new file mode 100644 index 000000000..bcd6d9889 --- /dev/null +++ b/include/labstor/work_orchestrator/scheduler.h @@ -0,0 +1,53 @@ +// +// Created by llogan on 7/2/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_SCHEDULER_H_ +#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_SCHEDULER_H_ + +#include "labstor/task_registry/task.h" + +namespace labstor { + +/** The set of methods in the admin task */ +struct SchedulerMethod : public TaskMethod { + TASK_METHOD_T kSchedule = TaskMethod::kLast; +}; + +/** The task type used for scheduling */ +struct ScheduleTask : public Task, TaskFlags { + OUT hipc::pod_array ret_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ScheduleTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ScheduleTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLongRunning; + task_state_ = state_id; + method_ = SchedulerMethod::kSchedule; + task_flags_.SetBits(TASK_LONG_RUNNING); + domain_id_ = domain_id; + + // Custom params + ret_.construct(alloc, 1); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_SCHEDULER_H_ diff --git a/include/labstor/work_orchestrator/work_orchestrator.h b/include/labstor/work_orchestrator/work_orchestrator.h new file mode 100644 index 000000000..aad42852c --- /dev/null +++ b/include/labstor/work_orchestrator/work_orchestrator.h @@ -0,0 +1,126 @@ +// +// Created by lukemartinlogan on 6/23/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ +#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ + +#include "labstor/labstor_types.h" +#include "labstor/api/labstor_runtime.h" +#include "labstor/queue_manager/queue_manager_runtime.h" +#include "worker.h" +#include "labstor/network/rpc_thallium.h" +#include + +namespace labstor { + +class WorkOrchestrator { + public: + ServerConfig *config_; /**< The server configuration */ + std::vector workers_; /**< Workers execute tasks */ + std::atomic stop_runtime_; /**< Begin killing the runtime */ + std::atomic kill_requested_; /**< Kill flushing threads eventually */ + ABT_xstream xstream_; + + public: + /** Default constructor */ + WorkOrchestrator() = default; + + /** Destructor */ + ~WorkOrchestrator() = default; + + /** Create thread pool */ + void ServerInit(ServerConfig *config, QueueManager &qm) { + config_ = config; + + // Initialize argobots + ABT_init(0, nullptr); + + // Create argobots xstream + int ret = ABT_xstream_create(ABT_SCHED_NULL, &xstream_); + if (ret != ABT_SUCCESS) { + HELOG(kFatal, "Could not create argobots xstream"); + } + + // Spawn workers on the stream + size_t num_workers = config_->wo_.max_workers_; + workers_.reserve(num_workers); + for (u32 worker_id = 0; worker_id < num_workers; ++worker_id) { + workers_.emplace_back(worker_id, xstream_); + } + stop_runtime_ = false; + kill_requested_ = false; + + // Schedule admin queue on worker 0 + MultiQueue *admin_queue = qm.GetQueue(qm.admin_queue_); + LaneGroup *admin_group = &admin_queue->GetGroup(0); + for (u32 lane_id = 0; lane_id < admin_group->num_lanes_; ++lane_id) { + Worker &worker = workers_[0]; + worker.PollQueues({WorkEntry(0, lane_id, admin_queue)}); + } + admin_group->num_scheduled_ = admin_group->num_lanes_; + + HILOG(kInfo, "Started {} workers", num_workers); + } + + /** Get worker with this id */ + Worker& GetWorker(u32 worker_id) { + return workers_[worker_id]; + } + + /** Get the number of workers */ + size_t GetNumWorkers() { + return workers_.size(); + } + + /** Begin finalizing the runtime */ + void FinalizeRuntime() { + stop_runtime_.store(true); + } + + /** Finalize thread pool */ + void Join() { + kill_requested_.store(true); + for (Worker &worker : workers_) { + worker.thread_->join(); + ABT_xstream_join(xstream_); + ABT_xstream_free(&xstream_); + } + } + + /** Whether threads should still be executing */ + bool IsAlive() { + return !kill_requested_.load(); + } + + /** Whether runtime should still be executing */ + bool IsRuntimeAlive() { + return !stop_runtime_.load(); + } + + /** Spawn an argobots thread */ + template + ABT_thread SpawnAsyncThread(FUNC &&func, TaskT *data) { + ABT_thread tl_thread; + int ret = ABT_thread_create_on_xstream(xstream_, + func, (void*) data, + ABT_THREAD_ATTR_NULL, + &tl_thread); + if (ret != ABT_SUCCESS) { + HELOG(kFatal, "Couldn't spawn worker"); + } + return tl_thread; + } + + /** Wait for argobots thread */ + void JoinAsyncThread(ABT_thread tl_thread) { + ABT_thread_join(tl_thread); + } +}; + +} // namespace labstor + +#define LABSTOR_WORK_ORCHESTRATOR \ + (&LABSTOR_RUNTIME->work_orchestrator_) + +#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h new file mode 100644 index 000000000..6ab2f7509 --- /dev/null +++ b/include/labstor/work_orchestrator/worker.h @@ -0,0 +1,367 @@ +// +// Created by lukemartinlogan on 6/27/23. +// + +#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORKER_H_ +#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORKER_H_ + +#include "labstor/labstor_types.h" +#include "labstor/queue_manager/queue_manager_runtime.h" +#include +#include +#include "affinity.h" +#include "labstor/network/rpc_thallium.h" + +namespace labstor { + +#define WORKER_CONTINUOUS_POLLING BIT_OPT(u32, 0) + +/** Uniquely identify a queue lane */ +struct WorkEntry { + u32 prio_; + u32 lane_id_; + u32 count_; + Lane *lane_; + LaneGroup *group_; + MultiQueue *queue_; + + /** Default constructor */ + HSHM_ALWAYS_INLINE + WorkEntry() = default; + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE + WorkEntry(u32 prio, u32 lane_id, MultiQueue *queue) + : prio_(prio), lane_id_(lane_id), queue_(queue) { + group_ = &queue->GetGroup(prio); + lane_ = &queue->GetLane(*group_, lane_id); + count_ = 0; + } + + /** Copy constructor */ + HSHM_ALWAYS_INLINE + WorkEntry(const WorkEntry &other) { + prio_ = other.prio_; + lane_id_ = other.lane_id_; + lane_ = other.lane_; + group_ = other.group_; + queue_ = other.queue_; + } + + /** Copy assignment */ + HSHM_ALWAYS_INLINE + WorkEntry& operator=(const WorkEntry &other) { + if (this != &other) { + prio_ = other.prio_; + lane_id_ = other.lane_id_; + lane_ = other.lane_; + group_ = other.group_; + queue_ = other.queue_; + } + return *this; + } + + /** Move constructor */ + HSHM_ALWAYS_INLINE + WorkEntry(WorkEntry &&other) noexcept { + prio_ = other.prio_; + lane_id_ = other.lane_id_; + lane_ = other.lane_; + group_ = other.group_; + queue_ = other.queue_; + } + + /** Move assignment */ + HSHM_ALWAYS_INLINE + WorkEntry& operator=(WorkEntry &&other) noexcept { + if (this != &other) { + prio_ = other.prio_; + lane_id_ = other.lane_id_; + lane_ = other.lane_; + group_ = other.group_; + queue_ = other.queue_; + } + return *this; + } + + /** Check if null */ + [[nodiscard]] + HSHM_ALWAYS_INLINE bool IsNull() const { + return queue_->IsNull(); + } + + /** Equality operator */ + HSHM_ALWAYS_INLINE + bool operator==(const WorkEntry &other) const { + return queue_ == other.queue_ && lane_id_ == other.lane_id_ && + prio_ == other.prio_; + } +}; + +} // namespace labstor + +namespace std { +/** Hash function for WorkEntry */ +template<> +struct hash { + HSHM_ALWAYS_INLINE + std::size_t + operator()(const labstor::WorkEntry &key) const { + return std::hash{}(key.queue_) + + std::hash{}(key.lane_id_) + std::hash{}(key.prio_); + } +}; +} // namespace std + +namespace labstor { + +class Worker { + public: + u32 id_; /**< Unique identifier of this worker */ + std::unique_ptr thread_; /**< The worker thread handle */ + int pthread_id_; /**< The worker pthread handle */ + // ABT_thread tl_thread_; + int pid_; /**< The worker process id */ + u32 numa_node_; // TODO(llogan): track NUMA affinity + std::vector work_queue_; /**< The set of queues to poll */ + /** A set of queues to begin polling in a worker */ + hshm::spsc_queue> poll_queues_; + /** A set of queues to stop polling in a worker */ + hshm::spsc_queue> relinquish_queues_; + size_t sleep_us_; /** Time the worker should sleep after a run */ + u32 retries_; /** The number of times to repeat the internal run loop before sleeping */ + bitfield32_t flags_; /** Worker metadata flags */ + std::unordered_map group_map_; /** Determine if a task can be executed right now */ + hshm::charbuf group_; /** The current group */ + + + public: + /** Constructor */ + Worker(u32 id, ABT_xstream &xstream) { + poll_queues_.Resize(1024); + relinquish_queues_.Resize(1024); + id_ = id; + sleep_us_ = 0; + EnableContinuousPolling(); + retries_ = 1; + pid_ = 0; + thread_ = std::make_unique(&Worker::Loop, this); + pthread_id_ = thread_->native_handle(); + // TODO(llogan): implement reserve for group + group_.resize(512); + group_.resize(0); + /* int ret = ABT_thread_create_on_xstream(xstream, + [](void *args) { ((Worker*)args)->Loop(); }, this, + ABT_THREAD_ATTR_NULL, &tl_thread_); + if (ret != ABT_SUCCESS) { + HELOG(kFatal, "Couldn't spawn worker"); + }*/ + } + + /** Constructor without threading */ + Worker(u32 id) { + poll_queues_.Resize(1024); + relinquish_queues_.Resize(1024); + id_ = id; + sleep_us_ = 0; + EnableContinuousPolling(); + retries_ = 1; + pid_ = 0; + pthread_id_ = gettid(); + // TODO(llogan): implement reserve for group + group_.resize(512); + group_.resize(0); + } + + /** + * Tell worker to poll a set of queues + * */ + void PollQueues(const std::vector &queues) { + poll_queues_.emplace(queues); + } + + /** + * Tell worker to start relinquishing some of its queues + * This function must be called from a single thread (outside of worker) + * */ + void RelinquishingQueues(const std::vector &queues) { + relinquish_queues_.emplace(queues); + } + + /** Check if worker is still stealing queues */ + bool IsRelinquishingQueues() { + return relinquish_queues_.size() > 0; + } + + /** Set the sleep cycle */ + void SetPollingFrequency(size_t sleep_us, u32 num_retries) { + sleep_us_ = sleep_us; + retries_ = num_retries; + flags_.UnsetBits(WORKER_CONTINUOUS_POLLING); + } + + /** Enable continuous polling */ + void EnableContinuousPolling() { + flags_.SetBits(WORKER_CONTINUOUS_POLLING); + } + + /** Disable continuous polling */ + void DisableContinuousPolling() { + flags_.UnsetBits(WORKER_CONTINUOUS_POLLING); + } + + /** Set the CPU affinity of this worker */ + void SetCpuAffinity(int cpu_id) { + ProcessAffiner::SetCpuAffinity(pid_, cpu_id); + } + + /** The work loop */ + void Loop(); + + /** Worker yields for a period of time */ + void Yield() { + if (flags_.Any(WORKER_CONTINUOUS_POLLING)) { + return; + } + if (sleep_us_ > 0) { + HERMES_THREAD_MODEL->SleepForUs(sleep_us_); + } else { + HERMES_THREAD_MODEL->Yield(); + } + } + + /** Worker merges the set of queues to poll */ + void _PollQueues() { + std::vector work_queue; + while (!poll_queues_.pop(work_queue).IsNull()) { + for (const WorkEntry &entry : work_queue) { + // HILOG(kDebug, "Scheduled queue {} (lane {})", entry.queue_->id_, entry.lane_); + work_queue_.emplace_back(entry); + } + } + } + + /** Relinquish queues. Called from wihtin Worker */ + void _RelinquishQueues() { + std::vector work_queue; + while (!poll_queues_.pop(work_queue).IsNull()) { + for (auto &entry : work_queue) { + work_queue_.erase(std::find(work_queue_.begin(), + work_queue_.end(), entry)); + } + } + } + + /** + * Poll work queue + * + * @return true if work was found, false otherwise + * */ + void Run(); + + HSHM_ALWAYS_INLINE + bool CheckTaskGroup(Task *task, TaskState *exec, + u32 lane_id, + TaskNode node, const bool &is_remote) { + if (is_remote || task->IsStarted()) { + return true; + } + int ret = exec->GetGroup(task->method_, task, group_); + if (ret == TASK_UNORDERED || task->IsUnordered()) { + HILOG(kDebug, "(node {}) Task {} is unordered, so count remains 0 worker={}", + LABSTOR_CLIENT->node_id_, task->task_node_, id_); + return true; + } + +#ifdef DEBUG + // TODO(llogan): remove + std::stringstream ss; + for (int i = 0; i < group_.size(); ++i) { + ss << std::to_string((int)group_[i]); + } +#endif + + // Ensure that concurrent requests are not serialized + LocalSerialize srl(group_); + srl << lane_id; + + auto it = group_map_.find(group_); + if (it == group_map_.end()) { + node.node_depth_ = 1; + group_map_.emplace(group_, node); + HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", + LABSTOR_CLIENT->node_id_, node.node_depth_, id_); + return true; + } + TaskNode &node_cmp = it->second; + if (node_cmp.root_ == node.root_) { + node_cmp.node_depth_ += 1; + HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", + LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, id_); + return true; + } + return false; + } + + HSHM_ALWAYS_INLINE + void RemoveTaskGroup(Task *task, TaskState *exec, + u32 lane_id, const bool &is_remote) { + if (is_remote) { + return; + } + int ret = exec->GetGroup(task->method_, task, group_); + if (ret == TASK_UNORDERED || task->IsUnordered() || task->method_ < 2) { + HILOG(kDebug, "(node {}) Decreasing depth of group remains 0 (task_node={} worker={})", + LABSTOR_CLIENT->node_id_, task->task_node_, id_); + return; + } + +#ifdef DEBUG + // TODO(llogan): remove + std::stringstream ss; + for (int i = 0; i < group_.size(); ++i) { + ss << std::to_string((int)group_[i]); + } +#endif + // Ensure that concurrent requests are not serialized + LocalSerialize srl(group_); + srl << lane_id; + + TaskNode &node_cmp = group_map_[group_]; + if (node_cmp.node_depth_ == 0) { + HELOG(kFatal, "(node {}) Group {} depth is already 0 (task_node={} worker={})", + LABSTOR_CLIENT->node_id_, task->task_node_, id_); + } + node_cmp.node_depth_ -= 1; + HILOG(kDebug, "(node {}) Decreasing depth of to {} (task_node={} worker={})", + LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, task->task_node_, id_); + if (node_cmp.node_depth_ == 0) { + group_map_.erase(group_); + } + } + + HSHM_ALWAYS_INLINE + void EndTask(Lane *lane, Task *task, int &off) { + PopTask(lane, off); + if (task->IsFireAndForget()) { + LABSTOR_CLIENT->DelTask(task); + } else { + task->SetComplete(); + } + } + + HSHM_ALWAYS_INLINE + void PopTask(Lane *lane, int &off) { + if (off == 0) { + lane->pop(); + } else { + off += 1; + } + } + + void PollGrouped(WorkEntry &entry); +}; + +} // namespace labstor + +#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORKER_H_ diff --git a/hermes_shm/scripts/ci/coverage.sh b/scripts/ci/coverage.sh similarity index 100% rename from hermes_shm/scripts/ci/coverage.sh rename to scripts/ci/coverage.sh diff --git a/hermes_shm/scripts/ci/external/CMakeLists.txt b/scripts/ci/external/CMakeLists.txt similarity index 100% rename from hermes_shm/scripts/ci/external/CMakeLists.txt rename to scripts/ci/external/CMakeLists.txt diff --git a/hermes_shm/scripts/ci/external/test.cc b/scripts/ci/external/test.cc similarity index 100% rename from hermes_shm/scripts/ci/external/test.cc rename to scripts/ci/external/test.cc diff --git a/hermes_shm/scripts/ci/install_deps.sh b/scripts/ci/install_deps.sh similarity index 100% rename from hermes_shm/scripts/ci/install_deps.sh rename to scripts/ci/install_deps.sh diff --git a/hermes_shm/scripts/ci/install_docs.sh b/scripts/ci/install_docs.sh similarity index 86% rename from hermes_shm/scripts/ci/install_docs.sh rename to scripts/ci/install_docs.sh index 63f912bd8..f7d9189e7 100755 --- a/hermes_shm/scripts/ci/install_docs.sh +++ b/scripts/ci/install_docs.sh @@ -17,8 +17,8 @@ cmake \ -DCMAKE_INSTALL_RPATH=${INSTALL_PREFIX}/lib \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_SHARED_LIBS=ON \ - -DHERMES_ENABLE_DOXYGEN=ON \ - -DHERMES_ENABLE_COVERAGE=ON \ + -DLABSTOR_ENABLE_DOXYGEN=ON \ + -DLABSTOR_ENABLE_COVERAGE=ON \ .. make dox >& out.txt # cat out.txt | grep warning | grep -v "ignoring unsupported tag" diff --git a/hermes_shm/scripts/ci/install_hshm.sh b/scripts/ci/install_hshm.sh similarity index 90% rename from hermes_shm/scripts/ci/install_hshm.sh rename to scripts/ci/install_hshm.sh index 7dec11efb..4f1445d91 100755 --- a/hermes_shm/scripts/ci/install_hshm.sh +++ b/scripts/ci/install_hshm.sh @@ -14,21 +14,19 @@ SPACK_DIR=${INSTALL_DIR}/spack mkdir -p "${HOME}/install" mkdir build -pushd build +cd build spack load --only dependencies hermes_shm cmake ../ \ -DCMAKE_BUILD_TYPE=Debug \ --DHERMES_ENABLE_COVERAGE=ON \ --DHERMES_ENABLE_DOXYGEN=ON \ +-DLABSTOR_ENABLE_COVERAGE=ON \ +-DLABSTOR_ENABLE_DOXYGEN=ON \ -DBUILD_HSHM_BENCHMARKS=ON \ -DBUILD_HSHM_TESTS=ON \ -DCMAKE_INSTALL_PREFIX=${HOME}/install make -j8 -make install export CXXFLAGS=-Wall ctest -VV -popd # Set proper flags for cmake to find Hermes INSTALL_PREFIX="${HOME}/install" @@ -41,7 +39,7 @@ export CMAKE_PREFIX_PATH="${INSTALL_PREFIX}:${CMAKE_PREFIX_PATH}" export CXXFLAGS="-I${INSTALL_PREFIX}/include:${CXXFLAGS}" # Run make install unit test -cd scripts/ci/external +cd ci/external mkdir build cd build cmake ../ diff --git a/ci/packages.yaml b/scripts/ci/packages.yaml similarity index 100% rename from ci/packages.yaml rename to scripts/ci/packages.yaml diff --git a/ci/py_hermes_ci/bin/run_test b/scripts/ci/py_hermes_ci/bin/run_test similarity index 86% rename from ci/py_hermes_ci/bin/run_test rename to scripts/ci/py_hermes_ci/bin/run_test index 0427ac0fd..7d1f04027 100755 --- a/ci/py_hermes_ci/bin/run_test +++ b/scripts/ci/py_hermes_ci/bin/run_test @@ -17,10 +17,8 @@ if __name__ == '__main__': address_sanitizer = False # The root of Hermes HERMES_ROOT = str(pathlib.Path(__file__).parent. - parent.parent.parent.resolve()) - ADAPTER_TEST_ROOT = f"{HERMES_ROOT}/adapter/test" - # Ensure that all calls beneath know how to resolve jarvis_util - sys.path.insert(0, f"{HERMES_ROOT}/ci/jarvis-util") + parent.parent.parent.parent.resolve()) + ADAPTER_TEST_ROOT = f"{HERMES_ROOT}/test/unit/hermes_adapters" # Ensure subsequent classes know how to resolve py_hermes_ci package sys.path.insert(0, f"{HERMES_ROOT}/ci/py_hermes_ci") # Choose which unit test file to load @@ -37,7 +35,7 @@ if __name__ == '__main__': elif test_type == 'data_stager': pkg_dir = f"{HERMES_ROOT}/data_stager/test" elif test_type == 'kvstore': - pkg_dir = f"{HERMES_ROOT}/adapter/kvstore" + pkg_dir = f"{HERMES_ROOT}/hermes_adapters/kvstore" elif test_type == 'java_wrapper': pkg_dir = f"{HERMES_ROOT}/wrapper/java" else: diff --git a/ci/py_hermes_ci/py_hermes_ci/test_manager.py b/scripts/ci/py_hermes_ci/py_hermes_ci/test_manager.py similarity index 100% rename from ci/py_hermes_ci/py_hermes_ci/test_manager.py rename to scripts/ci/py_hermes_ci/py_hermes_ci/test_manager.py diff --git a/hermes_shm/scripts/docs.sh b/scripts/docs.sh similarity index 100% rename from hermes_shm/scripts/docs.sh rename to scripts/docs.sh diff --git a/ci/jarvis-util/jarvis_util/introspect/__init__.py b/scripts/hermes/packages/hermes/__init__.py similarity index 100% rename from ci/jarvis-util/jarvis_util/introspect/__init__.py rename to scripts/hermes/packages/hermes/__init__.py diff --git a/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py b/scripts/hermes/packages/hermes/package.py similarity index 58% rename from hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py rename to scripts/hermes/packages/hermes/package.py index 6beada344..f4f69caba 100644 --- a/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py +++ b/scripts/hermes/packages/hermes/package.py @@ -1,17 +1,26 @@ from spack import * -class HermesShm(CMakePackage): - homepage = "https://github.com/lukemartinlogan/hermes_shm/wiki" - git = "https://github.com/lukemartinlogan/hermes_shm.git" - version('master', branch='master') - depends_on('mochi-thallium~cereal@0.8.3') +class Hermes(CMakePackage): + homepage = "https://github.com/lukemartinlogan/labstor/wiki" + git = "https://github.com/lukemartinlogan/labstor.git" + version('master', branch='dev') + depends_on('mochi-thallium~cereal@0.10.1') + depends_on('cereal') depends_on('catch2@3.0.1') depends_on('mpich@3.3.2:') + depends_on('yaml-cpp') depends_on('boost@1.7:') - depends_on('doxygen@1.9.3') + depends_on('hermes_shm') + depends_on('libzmq', '+zmq') + + variant('debug', default=False, description='Build shared libraries') + variant('zmq', default=False, description='Build ZeroMQ tests') def cmake_args(self): - return [] + args = ['-DCMAKE_INSTALL_PREFIX={}'.format(self.prefix)] + if '+debug' in self.spec: + args.append('-DCMAKE_BUILD_TYPE=Debug') + return args def set_include(self, env, path): env.append_flags('CFLAGS', '-I{}'.format(path)) @@ -29,6 +38,7 @@ def set_flags(self, env): self.set_include(env, '{}/include'.format(self.prefix)) self.set_lib(env, '{}/lib'.format(self.prefix)) self.set_lib(env, '{}/lib64'.format(self.prefix)) + env.prepend_path('CMAKE_PREFIX_PATH', '{}/cmake'.format(self.prefix)) def setup_dependent_environment(self, spack_env, run_env, dependent_spec): self.set_flags(spack_env) diff --git a/scripts/hermes/repo.yaml b/scripts/hermes/repo.yaml new file mode 100644 index 000000000..0fdccdc29 --- /dev/null +++ b/scripts/hermes/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: 'labstor' diff --git a/scripts/lint.sh b/scripts/lint.sh new file mode 100644 index 000000000..1953c341c --- /dev/null +++ b/scripts/lint.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +LABSTOR_ROOT=$1 + +cpplint --recursive \ +"${LABSTOR_ROOT}/src" "${LABSTOR_ROOT}/include" "${LABSTOR_ROOT}/test" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 60b99d42a..04bfaf302 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,118 +1,54 @@ -include_directories(${CMAKE_SOURCE_DIR}) - -configure_file(hermes_version.h.in hermes_version.h) - -# Create rpc factory -set(CMAKE_HERMES_COMMUNICATION_TYPE HERMES_RPC_NONE) -if (HERMES_RPC_THALLIUM) - set(CMAKE_HERMES_RPC_TYPE HERMES_RPC_THALLIUM) - set(CMAKE_HERMES_RPC_TYPE_LIB thallium) -endif() -configure_file(rpc_factory.h.in rpc_factory.h) - #------------------------------------------------------------------------------ # External dependencies #------------------------------------------------------------------------------ # None for now #------------------------------------------------------------------------------ -# Set sources +# Build Labstor Client Library #------------------------------------------------------------------------------ -set(HERMES_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/api/hermes_singleton.cc - ${CMAKE_CURRENT_SOURCE_DIR}/config_client.cc - ${CMAKE_CURRENT_SOURCE_DIR}/config_server.cc - ${CMAKE_CURRENT_SOURCE_DIR}/hermes_types.cc - ${CMAKE_CURRENT_SOURCE_DIR}/rpc.cc - ${CMAKE_CURRENT_SOURCE_DIR}/rpc_thallium.cc - ${CMAKE_CURRENT_SOURCE_DIR}/rpc_thallium_defs.cc - ${CMAKE_CURRENT_SOURCE_DIR}/metadata_types.cc - ${CMAKE_CURRENT_SOURCE_DIR}/metadata_manager.cc - ${CMAKE_CURRENT_SOURCE_DIR}/api/bucket.cc - ${CMAKE_CURRENT_SOURCE_DIR}/api/hermes.cc - ${CMAKE_CURRENT_SOURCE_DIR}/buffer_pool.cc - ${CMAKE_CURRENT_SOURCE_DIR}/buffer_organizer.cc - ${CMAKE_CURRENT_SOURCE_DIR}/prefetcher.cc - ${CMAKE_CURRENT_SOURCE_DIR}/prefetcher/apriori_prefetcher.cc - ${CMAKE_CURRENT_SOURCE_DIR}/trait_manager.cc - ${CMAKE_CURRENT_SOURCE_DIR}/data_placement_engine.cc - ${CMAKE_CURRENT_SOURCE_DIR}/dpe/random.cc - ${CMAKE_CURRENT_SOURCE_DIR}/dpe/round_robin.cc - ${CMAKE_CURRENT_SOURCE_DIR}/dpe/minimize_io_time.cc) - -#------------------------------------------------------------------------------ -# Build Hermes Shared Library -#------------------------------------------------------------------------------ -set(HERMES_BUILD_INCLUDE_DEPENDENCIES - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/api - ${CMAKE_CURRENT_BINARY_DIR} -) - -add_library(hermes SHARED ${HERMES_SRCS}) -include_directories(${CMAKE_SOURCE_DIR}/io_client) -target_include_directories(hermes - PUBLIC "$" - $ -) -target_include_directories(hermes - SYSTEM PUBLIC ${HERMES_EXT_INCLUDE_DEPENDENCIES}) -add_dependencies(hermes - hermes_shm_data_structures) -target_link_libraries(hermes - hermes_shm_data_structures - ${CMAKE_HERMES_RPC_TYPE_LIB} +add_library(labstor_client SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/config_client.cc + ${CMAKE_CURRENT_SOURCE_DIR}/config_server.cc + ${CMAKE_CURRENT_SOURCE_DIR}/labstor_client.cc) +target_link_libraries(labstor_client + ${HermesShm_LIBRARIES} yaml-cpp cereal::cereal - MPI::MPI_CXX) - -if (HERMES_ENABLE_PROFILING) - message("profiling is enabled: ${HERMES_ENABLE_PROFILING}") -endif() - -target_compile_definitions(hermes - PRIVATE $<$:HERMES_ONLY_RPC> - PRIVATE $<$:HERMES_ENABLE_PROFILING> -) - -hermes_set_lib_options(hermes "hermes" ${HERMES_LIBTYPE}) -set(HERMES_EXPORTED_LIBS hermes hermes_daemon finalize_hermes ${HERMES_EXPORTED_LIBS}) + thallium + -ldl -lrt -lc -pthread) #------------------------------------------------------------------------------ -# Build Hermes Daemon +# Build Labstor Runtime Library #------------------------------------------------------------------------------ -find_package(Catch2 REQUIRED) -add_executable(hermes_daemon api/hermes_daemon.cc) -add_dependencies(hermes_daemon hermes) -target_link_libraries(hermes_daemon hermes -ldl -lc MPI::MPI_CXX) -target_compile_definitions(hermes_daemon - PRIVATE $<$:HERMES_ENABLE_PROFILING>) - +add_library(labstor_runtime + worker.cc + labstor_runtime.cc) +add_dependencies(labstor_runtime ${Labstor_CLIENT_DEPS}) +target_link_libraries(labstor_runtime thallium ${Labstor_CLIENT_LIBRARIES}) #------------------------------------------------------------------------------ -# Build Hermes Shared Library +# Build Labstor Runtime Start Function #------------------------------------------------------------------------------ -add_executable(finalize_hermes api/finalize_hermes.cc) -add_dependencies(finalize_hermes hermes) -target_link_libraries(finalize_hermes hermes) +add_executable(labstor_start_runtime + labstor_start_runtime.cc) +add_dependencies(labstor_start_runtime ${Labstor_RUNTIME_DEPS}) +target_link_libraries(labstor_start_runtime ${Labstor_RUNTIME_LIBRARIES}) -#----------------------------------------------------------------------------- -# Specify project header files to be installed -#----------------------------------------------------------------------------- -file(GLOB_RECURSE HERMES_HEADERS "*.h") -set(HERMES_HEADERS - ${CMAKE_BINARY_DIR}/src/rpc_factory.h - ${CMAKE_BINARY_DIR}/src/hermes_version.h - ${HERMES_HEADERS}) +#------------------------------------------------------------------------------ +# Build LabStor Runtime Stop Function +#------------------------------------------------------------------------------ +add_executable(labstor_stop_runtime labstor_stop_runtime.cc) +add_dependencies(labstor_stop_runtime ${Labstor_CLIENT_DEPS}) +target_link_libraries(labstor_stop_runtime ${Labstor_CLIENT_LIBRARIES}) #----------------------------------------------------------------------------- # Add file(s) to CMake Install #----------------------------------------------------------------------------- install( FILES - ${HERMES_HEADERS} + ${LABSTOR_HEADERS} DESTINATION - ${HERMES_INSTALL_INCLUDE_DIR} + ${LABSTOR_INSTALL_INCLUDE_DIR} COMPONENT headers ) @@ -122,61 +58,53 @@ install( #----------------------------------------------------------------------------- install( TARGETS - hermes - hermes_daemon - finalize_hermes + labstor_client + labstor_runtime + labstor_start_runtime + labstor_stop_runtime EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Add Target(s) to CMake Install for import into other projects #----------------------------------------------------------------------------- install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -if(NOT HERMES_EXTERNALLY_CONFIGURED) -EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) +set(LABSTOR_EXPORTED_LIBS + labstor_client + labstor_runtime + labstor_start_runtime + labstor_stop_runtime + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) endif() -#------------------------------------------------------------------------------ -# Set variables for parent scope -#------------------------------------------------------------------------------ -# Used by config.cmake.build.in and Testing -set(HERMES_INCLUDES_BUILD_TIME - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${HERMES_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) - -# Used by config.cmake.install.in -set(HERMES_INCLUDES_INSTALL_TIME - ${HERMES_INSTALL_INCLUDE_DIR} - ${HERMES_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) - #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes) +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(labstor_client) + set_coverage_flags(labstor_runtime) + set_coverage_flags(labstor_start_runtime) + set_coverage_flags(labstor_stop_runtime) endif() \ No newline at end of file diff --git a/src/api/bucket.cc b/src/api/bucket.cc deleted file mode 100644 index 9d297041a..000000000 --- a/src/api/bucket.cc +++ /dev/null @@ -1,260 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "bucket.h" -#include "data_placement_engine_factory.h" - -namespace hermes::api { - -using hermes::adapter::AdapterMode; - -/**==================================== - * Bucket Operations - * ===================================*/ - -/** - * Either initialize or fetch the bucket. - * */ -Bucket::Bucket(const std::string &bkt_name, - Context &ctx, - size_t backend_size) -: mdm_(&HERMES->mdm_), bpm_(&HERMES->bpm_), name_(bkt_name) { - std::vector traits; - auto ret = mdm_->GlobalGetOrCreateTag(bkt_name, true, traits, backend_size); - id_ = ret.first; - did_create_ = ret.second; -} - -/** - * Either initialize or fetch the bucket. - * */ -Bucket::Bucket(TagId tag_id) - : mdm_(&HERMES->mdm_), bpm_(&HERMES->bpm_) { - name_ = mdm_->GlobalGetTagName(tag_id); - id_ = tag_id; - did_create_ = false; -} - -/** - * Attach a trait to the bucket - * */ -void Bucket::AttachTrait(TraitId trait_id) { - HERMES->AttachTrait(id_, trait_id); -} - -/** - * Get the current size of the bucket - * */ -size_t Bucket::GetSize() { - return mdm_->GlobalGetBucketSize(id_); -} - -/** - * Update the size of the bucket - * Needed for the adapters for now. - * */ -void Bucket::SetSize(size_t new_size) { - mdm_->GlobalSetBucketSize(id_, new_size); -} - -/** - * Lock a bucket - * */ -void Bucket::LockBucket(MdLockType type) { - mdm_->GlobalLockBucket(id_, type); -} - -/** - * Unlock a bucket - * */ -void Bucket::UnlockBucket(MdLockType type) { - mdm_->GlobalUnlockBucket(id_, type); -} - -/** - * Rename this bucket - * */ -void Bucket::Rename(const std::string &new_bkt_name) { - mdm_->GlobalRenameTag(id_, new_bkt_name); -} - -/** - * Clears the buckets contents, but doesn't destroy its metadata - * */ -void Bucket::Clear() { - mdm_->GlobalClearBucket(id_); -} - -/** - * Destroys this bucket along with all its contents. - * */ -void Bucket::Destroy() { - mdm_->GlobalDestroyTag(id_); -} - -/**==================================== - * Blob Operations - * ===================================*/ - -/** - * Get the id of a blob from the blob name - * */ -Status Bucket::GetBlobId(const std::string &blob_name, - BlobId &blob_id) { - blob_id = mdm_->GlobalGetBlobId(GetId(), blob_name); - return Status(); -} - -/** - * Get the name of a blob from the blob id - * - * @param blob_id the blob_id - * @return The Status of the operation - * */ -std::string Bucket::GetBlobName(const BlobId &blob_id) { - return mdm_->GlobalGetBlobName(blob_id); -} - -/** - * Get the score of a blob from the blob id - * - * @param blob_id the blob_id - * @return The Status of the operation - * */ -float Bucket::GetBlobScore(const BlobId &blob_id) { - return mdm_->GlobalGetBlobScore(blob_id); -} - - -/** - * Lock the bucket - * */ -bool Bucket::LockBlob(BlobId blob_id, MdLockType lock_type) { - return mdm_->GlobalLockBlob(blob_id, lock_type); -} - -/** - * Unlock the bucket - * */ -bool Bucket::UnlockBlob(BlobId blob_id, MdLockType lock_type) { - return mdm_->GlobalUnlockBlob(blob_id, lock_type); -} - -/** - * Put \a blob_id Blob into the bucket - * */ -Status Bucket::TryCreateBlob(const std::string &blob_name, - BlobId &blob_id, - Context &ctx) { - std::pair ret = mdm_->GlobalTryCreateBlob(id_, blob_name); - blob_id = ret.first; - if (ret.second) { - mdm_->GlobalTagAddBlob(id_, - blob_id); - } - return Status(); -} - -/** - * Label \a blob_id blob with \a tag_name TAG - * */ -Status Bucket::TagBlob(BlobId &blob_id, - TagId &tag_id) { - mdm_->GlobalTagAddBlob(tag_id, blob_id); - return mdm_->GlobalTagBlob(blob_id, tag_id); -} - -/** - * Put \a blob_id Blob into the bucket - * */ -Status Bucket::Put(std::string blob_name, - const Blob &blob, - BlobId &blob_id, - Context &ctx) { - // Calculate placement - auto dpe = DPEFactory::Get(ctx.policy); - std::vector blob_sizes(1, blob.size()); - std::vector schemas; - dpe->CalculatePlacement(blob_sizes, schemas, ctx); - - // Allocate buffers for the blob & enqueue placement - for (auto &schema : schemas) { - auto buffers = bpm_->GlobalAllocateAndSetBuffers(schema, blob); - auto put_ret = mdm_->GlobalPutBlobMetadata(id_, blob_name, - blob.size(), buffers, - ctx.blob_score_); - blob_id = std::get<0>(put_ret); - bool did_create = std::get<1>(put_ret); - if (did_create) { - mdm_->GlobalTagAddBlob(id_, blob_id); - } - } - - // Update the local MDM I/O log - mdm_->AddIoStat(id_, blob_id, blob.size(), IoType::kWrite); - - return Status(); -} - -/** - * Get \a blob_id Blob from the bucket - * */ -Status Bucket::Get(BlobId blob_id, Blob &blob, Context &ctx) { - std::vector buffers = mdm_->GlobalGetBlobBuffers(blob_id); - blob = HERMES->borg_.GlobalReadBlobFromBuffers(buffers); - // Update the local MDM I/O log - mdm_->AddIoStat(id_, blob_id, blob.size(), IoType::kRead); - return Status(); -} - -/** - * Determine if the bucket contains \a blob_id BLOB - * */ -bool Bucket::ContainsBlob(const std::string &blob_name, - BlobId &blob_id) { - GetBlobId(blob_name, blob_id); - return !blob_id.IsNull(); -} - -/** - * Determine if the bucket contains \a blob_id BLOB - * */ -bool Bucket::ContainsBlob(BlobId blob_id) { - return mdm_->GlobalBlobHasTag(blob_id, id_); -} - -/** - * Rename \a blob_id blob to \a new_blob_name new name - * */ -void Bucket::RenameBlob(BlobId blob_id, - std::string new_blob_name, - Context &ctx) { - mdm_->GlobalRenameBlob(id_, blob_id, new_blob_name); -} - -/** - * Delete \a blob_id blob - * */ -void Bucket::DestroyBlob(BlobId blob_id, - Context &ctx) { - mdm_->GlobalTagRemoveBlob(id_, blob_id); - mdm_->GlobalDestroyBlob(id_, blob_id); -} - -/** - * Get the set of blob IDs owned by the bucket - * */ -std::vector Bucket::GetContainedBlobIds() { - return mdm_->GlobalGroupByTag(id_); -} - -} // namespace hermes::api diff --git a/src/api/bucket.h b/src/api/bucket.h deleted file mode 100644 index 3370e8b90..000000000 --- a/src/api/bucket.h +++ /dev/null @@ -1,243 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_API_BUCKET_H_ -#define HERMES_SRC_API_BUCKET_H_ - -#include "hermes_types.h" -#include "status.h" -#include "buffer_pool.h" - -namespace hermes::api { - -class Bucket { - private: - MetadataManager *mdm_; - BufferPool *bpm_; - TagId id_; - std::string name_; - Context ctx_; - bool did_create_; - - public: - /**==================================== - * Bucket Operations - * ===================================*/ - - /** - * Get or create \a bkt_name bucket. - * - * Called from hermes.h in GetBucket(). Should not - * be used directly. - * */ - explicit Bucket(const std::string &bkt_name, - Context &ctx, - size_t backend_size = 0); - - /** - * Get an existing bucket. - * */ - explicit Bucket(TagId tag_id); - - /** - * Check if the bucket was created in the constructor - * */ - bool DidCreate() { - return did_create_; - } - - /** Default constructor */ - Bucket() = default; - - /** Default copy constructor */ - Bucket(const Bucket &other) = default; - - /** Default copy assign */ - Bucket& operator=(const Bucket &other) = default; - - /** Default move constructor */ - Bucket(Bucket &&other) = default; - - /** Default move assign */ - Bucket& operator=(Bucket &&other) = default; - - public: - /** - * Get the name of this bucket. Name is cached instead of - * making an RPC. Not coherent if Rename is called. - * */ - const std::string& GetName() const { - return name_; - } - - /** - * Get the identifier of this bucket - * */ - TagId GetId() const { - return id_; - } - - /** - * Get the context object of this bucket - * */ - Context& GetContext() { - return ctx_; - } - - /** - * Attach a trait to the bucket - * */ - void AttachTrait(TraitId trait_id); - - /** - * Get the current size of the bucket - * */ - size_t GetSize(); - - /** - * Update the size of the bucket - * Needed for the adapters for now. - * */ - void SetSize(size_t new_size); - - /** - * Lock a bucket - * */ - void LockBucket(MdLockType type); - - /** - * Unlock a bucket - * */ - void UnlockBucket(MdLockType type); - - /** - * Rename this bucket - * */ - void Rename(const std::string &new_bkt_name); - - /** - * Clears the buckets contents, but doesn't destroy its metadata - * */ - void Clear(); - - /** - * Destroys this bucket along with all its contents. - * */ - void Destroy(); - - /** - * Check if this bucket is valid - * */ - bool IsNull() { - return id_.IsNull(); - } - - public: - /**==================================== - * Blob Operations - * ===================================*/ - - /** - * Get the id of a blob from the blob name - * - * @param blob_name the name of the blob - * @param blob_id (output) the returned blob_id - * @return The Status of the operation - * */ - Status GetBlobId(const std::string &blob_name, BlobId &blob_id); - - /** - * Get the name of a blob from the blob id - * - * @param blob_id the blob_id - * @param blob_name the name of the blob - * @return The Status of the operation - * */ - std::string GetBlobName(const BlobId &blob_id); - - - /** - * Get the score of a blob from the blob id - * - * @param blob_id the blob_id - * @return The Status of the operation - * */ - float GetBlobScore(const BlobId &blob_id); - - /** - * Lock the blob - * */ - bool LockBlob(BlobId blob_id, MdLockType lock_type); - - /** - * Unlock the blob - * */ - bool UnlockBlob(BlobId blob_id, MdLockType lock_type); - - /** - * Create \a blob_name EMPTY BLOB if it does not already exist. - * */ - Status TryCreateBlob(const std::string &blob_name, - BlobId &blob_id, - Context &ctx); - - /** - * Label \a blob_id blob with \a tag_name TAG - * */ - Status TagBlob(BlobId &blob_id, - TagId &tag_id); - - /** - * Put \a blob_name Blob into the bucket - * */ - Status Put(std::string blob_name, - const Blob &blob, - BlobId &blob_id, - Context &ctx); - - /** - * Get \a blob_id Blob from the bucket - * */ - Status Get(BlobId blob_id, - Blob &blob, - Context &ctx); - - /** - * Determine if the bucket contains \a blob_id BLOB - * */ - bool ContainsBlob(const std::string &blob_name, - BlobId &blob_id); - - /** - * Determine if the bucket contains \a blob_id BLOB - * */ - bool ContainsBlob(BlobId blob_id); - - /** - * Rename \a blob_id blob to \a new_blob_name new name - * */ - void RenameBlob(BlobId blob_id, std::string new_blob_name, Context &ctx); - - /** - * Delete \a blob_id blob - * */ - void DestroyBlob(BlobId blob_id, Context &ctx); - - /** - * Get the set of blob IDs contained in the bucket - * */ - std::vector GetContainedBlobIds(); -}; - -} // namespace hermes::api - -#endif // HERMES_SRC_API_BUCKET_H_ diff --git a/src/api/finalize_hermes.cc b/src/api/finalize_hermes.cc deleted file mode 100644 index fbc0e025c..000000000 --- a/src/api/finalize_hermes.cc +++ /dev/null @@ -1,22 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_shm/util/singleton.h" -#include "hermes.h" - -int main(int argc, char **argv) { - HILOG(kDebug, "Finalize Hermes begins") - HERMES->Create(hermes::HermesType::kClient); - HERMES->client_config_.stop_daemon_ = true; - HERMES->Flush(); - HERMES->Finalize(); -} diff --git a/src/api/hermes.cc b/src/api/hermes.cc deleted file mode 100644 index d5e3ae6a7..000000000 --- a/src/api/hermes.cc +++ /dev/null @@ -1,232 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes.h" -#include "bucket.h" - -namespace hermes::api { - -/**==================================== - * PRIVATE Init + Finalize Operations - * ===================================*/ - -/** Internal initialization of Hermes */ -void Hermes::Init(HermesType mode, - std::string server_config_path, - std::string client_config_path) { - // Initialize hermes - hshm::ScopedMutex lock(lock_, 1); - if (is_initialized_) { - return; - } - mode_ = mode; - is_being_initialized_ = true; - switch (mode_) { - case HermesType::kServer: { - InitServer(std::move(server_config_path)); - break; - } - case HermesType::kClient: { - InitClient(std::move(server_config_path), - std::move(client_config_path)); - break; - } - case HermesType::kNone: { - HELOG(kFatal, "Cannot have none HermesType") - } - } - is_initialized_ = true; - is_being_initialized_ = false; -} - -/** Initialize Hermes as a server */ -void Hermes::InitServer(std::string server_config_path) { - LoadServerConfig(server_config_path); - InitSharedMemory(); - - // Initialize RPC - rpc_.InitServer(); - HERMES_THREAD_MODEL->SetThreadModel(hshm::ThreadType::kArgobots); - - // Load the trait libraries - traits_.Init(); - - // Construct the reference objects - mdm_.shm_init(header_->mdm_, main_alloc_, &server_config_); - rpc_.InitClient(); - bpm_.shm_init(header_->bpm_, main_alloc_); - borg_.shm_init(header_->borg_, main_alloc_); - prefetch_.Init(); -} - -/** Initialize Hermes as a client to the daemon */ -void Hermes::InitClient(std::string server_config_path, - std::string client_config_path) { - LoadServerConfig(server_config_path); - LoadClientConfig(client_config_path); - LoadSharedMemory(); - - // Initialize references to SHM types - mdm_.shm_deserialize(header_->mdm_); - rpc_.InitClient(); - HERMES_THREAD_MODEL->SetThreadModel(hshm::ThreadType::kArgobots); - bpm_.shm_deserialize(header_->bpm_); - borg_.shm_deserialize(header_->borg_); - prefetch_.Init(); - // mdm_.PrintDeviceInfo(); - - // Load the trait libraries - traits_.Init(); -} - -/** Load the server-side configuration */ -void Hermes::LoadServerConfig(std::string config_path) { - if (config_path.empty()) { - config_path = GetEnvSafe(kHermesServerConf); - } - if (mode_ == HermesType::kServer) { - HILOG(kInfo, "Loading server configuration: {}", config_path) - } - server_config_.LoadFromFile(config_path); -} - -/** Load the client-side configuration */ -void Hermes::LoadClientConfig(std::string config_path) { - if (config_path.empty()) { - config_path = GetEnvSafe(kHermesClientConf); - } - HILOG(kDebug, "Loading client configuration: {}", config_path) - client_config_.LoadFromFile(config_path); -} - -/** Initialize shared-memory between daemon and client */ -void Hermes::InitSharedMemory() { - // Create shared-memory allocator - auto mem_mngr = HERMES_MEMORY_MANAGER; - if (server_config_.max_memory_ == 0) { - server_config_.max_memory_ = hipc::MemoryManager::GetDefaultBackendSize(); - } - mem_mngr->CreateBackend( - server_config_.max_memory_, - server_config_.shmem_name_); - main_alloc_ = - mem_mngr->CreateAllocator( - server_config_.shmem_name_, - main_alloc_id, - sizeof(HermesShm)); - header_ = main_alloc_->GetCustomHeader(); -} - -/** Connect to a Daemon's shared memory */ -void Hermes::LoadSharedMemory() { - // Load shared-memory allocator - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->AttachBackend(hipc::MemoryBackendType::kPosixShmMmap, - server_config_.shmem_name_); - main_alloc_ = mem_mngr->GetAllocator(main_alloc_id); - header_ = main_alloc_->GetCustomHeader(); -} - -/** Finalize Daemon mode */ -void Hermes::FinalizeServer() { - // NOTE(llogan): rpc_.Finalize() is called internally by daemon in this case - // bpm_.shm_destroy(); - // mdm_.shm_destroy(); -} - -/** Finalize client mode */ -void Hermes::FinalizeClient() { - if (client_config_.stop_daemon_) { - StopDaemon(); - } - rpc_.Finalize(); -} - -/**==================================== - * PUBLIC Finalize Operations - * ===================================*/ - -/** Finalize Hermes explicitly */ -void Hermes::Finalize() { - if (!is_initialized_ || is_terminated_) { - return; - } - switch (mode_) { - case HermesType::kServer: { - FinalizeServer(); - break; - } - case HermesType::kClient: { - FinalizeClient(); - break; - } - default: { - throw std::logic_error("Invalid HermesType to launch in"); - } - } - is_initialized_ = false; - is_terminated_ = true; -} - -/** Run the Hermes core Daemon */ -void Hermes::RunDaemon() { - rpc_.RunDaemon(); -} - -/** Stop the Hermes core Daemon */ -void Hermes::StopDaemon() { - rpc_.StopDaemon(); -} - -/**==================================== - * PUBLIC Bucket Operations - * ===================================*/ - -/** Get or create a Bucket in Hermes */ -Bucket Hermes::GetBucket(std::string name, - Context ctx, - size_t backend_size) { - return Bucket(name, ctx, backend_size); -} - -/** Get an existing Bucket in Hermes */ -Bucket Hermes::GetBucket(TagId tag_id) { - return Bucket(tag_id); -} - -/**==================================== - * PUBLIC I/O Operations - * ===================================*/ - -/** Waits for all blobs to finish being flushed */ -void Hermes::Flush() { - borg_.GlobalWaitForFullFlush(); -} - -/** Destroy all buckets and blobs in this instance */ -void Hermes::Clear() { - Flush(); - mdm_.GlobalClear(); -} - -/**==================================== - * PUBLIC Tag Operations - * ===================================*/ - -/** Locate all blobs with a tag */ -std::vector Hermes::GroupBy(TagId tag_id) { - return mdm_.GlobalGroupByTag(tag_id); -} - - - -} // namespace hermes::api diff --git a/src/api/hermes.h b/src/api/hermes.h deleted file mode 100644 index 16f821776..000000000 --- a/src/api/hermes.h +++ /dev/null @@ -1,274 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_API_HERMES_H_ -#define HERMES_SRC_API_HERMES_H_ - -#include "config_client.h" -#include "config_server.h" -#include "hermes_types.h" -#include "utils.h" - -#include "rpc.h" -#include "metadata_manager.h" -#include "buffer_pool.h" -#include "buffer_organizer.h" -#include "prefetcher.h" -#include "trait_manager.h" -#include "bucket.h" - -#include "hermes_shm/util/singleton.h" - -// Singleton macros -#define HERMES hshm::Singleton::GetInstance() -#define HERMES_T hermes::api::Hermes* - -namespace hapi = hermes::api; - -namespace hermes::api { - -/** - * The Hermes shared-memory header - * */ -struct HermesShm { - hipc::Pointer ram_tier_; - hipc::ShmArchive mdm_; - hipc::ShmArchive bpm_; - hipc::ShmArchive borg_; -}; - -/** - * An index into all Hermes-related data structures. - * */ -class Hermes { - public: - HermesType mode_; - HermesShm *header_; - ServerConfig server_config_; - ClientConfig client_config_; - MetadataManager mdm_; - BufferPool bpm_; - BufferOrganizer borg_; - TraitManager traits_; - Prefetcher prefetch_; - RPC_TYPE rpc_; - hipc::Allocator *main_alloc_; - bool is_being_initialized_; - bool is_initialized_; - bool is_terminated_; - bool is_transparent_; - hshm::Mutex lock_; - - public: - /**==================================== - * PUBLIC Init Operations - * ===================================*/ - - /** Default constructor */ - Hermes() : is_being_initialized_(false), - is_initialized_(false), - is_terminated_(false), - is_transparent_(false) {} - - /** Destructor */ - ~Hermes() {} - - /** Whether or not Hermes is currently being initialized */ - bool IsBeingInitialized() { return is_being_initialized_; } - - /** Whether or not Hermes is initialized */ - bool IsInitialized() { return is_initialized_; } - - /** Whether or not Hermes is finalized */ - bool IsTerminated() { return is_terminated_; } - - /** Initialize Hermes explicitly */ - static Hermes* Create(HermesType mode = HermesType::kClient, - std::string server_config_path = "", - std::string client_config_path = "") { - auto hermes = HERMES; - hermes->Init(mode, server_config_path, client_config_path); - return hermes; - } - - public: - /**==================================== - * PUBLIC Finalize Operations - * ===================================*/ - - /** Finalize Hermes explicitly */ - void Finalize(); - - /** Run the Hermes core Daemon */ - void RunDaemon(); - - /** Stop the Hermes core Daemon */ - void StopDaemon(); - - public: - /**==================================== - * PUBLIC Bucket Operations - * ===================================*/ - - /** Get or create a Bucket in Hermes */ - Bucket GetBucket(std::string name, - Context ctx = Context(), - size_t backend_size = 0); - - /** Get an existing Bucket in Hermes */ - Bucket GetBucket(TagId bkt_id); - - /**==================================== - * PUBLIC I/O Operations - * ===================================*/ - - /** Waits for all blobs to finish being flushed */ - void Flush(); - - /** Destroy all buckets and blobs in this instance */ - void Clear(); - - /**==================================== - * PUBLIC Tag Operations - * ===================================*/ - - /** Create a generic tag in Hermes */ - TagId CreateTag(const std::string &tag_name) { - std::vector traits; - return mdm_.GlobalCreateTag(tag_name, false, traits); - } - - /** Get the TagId */ - TagId GetTagId(const std::string &tag_name) { - return mdm_.GlobalGetTagId(tag_name); - } - - /** Locate all blobs with a tag */ - std::vector GroupBy(TagId tag_id); - - /**==================================== - * PUBLIC Trait Operations - * ===================================*/ - - /** Create a trait */ - template - TraitId RegisterTrait(TraitT *trait) { - TraitId id = GetTraitId(trait->GetTraitUuid()); - if (!id.IsNull()) { - HILOG(kDebug, "Found existing trait trait: {}", trait->GetTraitUuid()) - return id; - } - HILOG(kDebug, "Registering a new trait: {}", trait->GetTraitUuid()) - id = HERMES->mdm_.GlobalRegisterTrait(TraitId::GetNull(), - trait->trait_info_); - HILOG(kDebug, "Giving trait {} id {}.{}", - trait->GetTraitUuid(), id.node_id_, id.unique_) - return id; - } - - /** Create a trait */ - template - TraitId RegisterTrait(const std::string &trait_uuid, - Args&& ...args) { - TraitId id = GetTraitId(trait_uuid); - if (!id.IsNull()) { - HILOG(kDebug, "Found existing trait trait: {}", trait_uuid) - return id; - } - HILOG(kDebug, "Registering new trait: {}", trait_uuid) - TraitT obj(trait_uuid, std::forward(args)...); - id = HERMES->mdm_.GlobalRegisterTrait(TraitId::GetNull(), - obj.trait_info_); - HILOG(kDebug, "Giving trait \"{}\" id {}.{}", - trait_uuid, id.node_id_, id.unique_) - return id; - } - - /** Get trait id */ - TraitId GetTraitId(const std::string &trait_uuid) { - return HERMES->mdm_.GlobalGetTraitId(trait_uuid); - } - - /** Get the trait */ - Trait* GetTrait(TraitId trait_id) { - return mdm_.GlobalGetTrait(trait_id); - } - - /** Attach a trait to a tag */ - void AttachTrait(TagId tag_id, TraitId trait_id) { - HERMES->mdm_.GlobalTagAddTrait(tag_id, trait_id); - } - - /** Get traits attached to tag */ - std::vector GetTraits(TagId tag_id, - uint32_t flags = ALL_BITS(uint32_t)) { - // HILOG(kDebug, "Getting the traits for tag {}", tag_id) - std::vector trait_ids = HERMES->mdm_.GlobalTagGetTraits(tag_id); - std::vector traits; - traits.reserve(trait_ids.size()); - for (TraitId &trait_id : trait_ids) { - auto trait = GetTrait(trait_id); - if (!trait) { continue; } - if (trait->GetTraitFlags().Any(flags)) { - traits.emplace_back(trait); - } - } - return traits; - } - - private: - /**==================================== - * PRIVATE Init + Finalize Operations - * ===================================*/ - - /** Internal initialization of Hermes */ - void Init(HermesType mode = HermesType::kClient, - std::string server_config_path = "", - std::string client_config_path = ""); - - /** Initialize Hermes as a server */ - void InitServer(std::string server_config_path); - - /** Initialize Hermes as a client to the daemon */ - void InitClient(std::string server_config_path, - std::string client_config_path); - - /** Load the server-side configuration */ - void LoadServerConfig(std::string config_path); - - /** Load the client-side configuration */ - void LoadClientConfig(std::string config_path); - - /** Initialize shared-memory between daemon and client */ - void InitSharedMemory(); - - /** Connect to a Daemon's shared memory */ - void LoadSharedMemory(); - - /** Finalize Daemon mode */ - void FinalizeServer(); - - /** Finalize client mode */ - void FinalizeClient(); -}; - -#define TRANSPARENT_HERMES\ - if (!HERMES->IsInitialized() && \ - !HERMES->IsBeingInitialized() && \ - !HERMES->IsTerminated()) {\ - HERMES->Create(hermes::HermesType::kClient);\ - HERMES->is_transparent_ = true;\ - } - -} // namespace hermes::api - -#endif // HERMES_SRC_API_HERMES_H_ diff --git a/src/api/hermes_daemon.cc b/src/api/hermes_daemon.cc deleted file mode 100644 index 4c6d199ec..000000000 --- a/src/api/hermes_daemon.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include -#include "hermes_shm/util/logging.h" -#include "hermes.h" - -#include - -namespace hapi = hermes::api; - -/** - * */ - -int main(int argc, char* argv[]) { - MPI_Init(&argc, &argv); - HILOG(kDebug, "Hermes start daemon begins") - std::string hermes_config = ""; - if (argc == 2) { - hermes_config = argv[1]; - } - - auto hermes = hapi::Hermes::Create( - hermes::HermesType::kServer, - hermes_config); - - hermes->RunDaemon(); - hermes->Finalize(); - MPI_Finalize(); - return 0; -} diff --git a/src/api/hermes_singleton.cc b/src/api/hermes_singleton.cc deleted file mode 100644 index 6edfc7a67..000000000 --- a/src/api/hermes_singleton.cc +++ /dev/null @@ -1,21 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_shm/util/singleton.h" - -#include "hermes.h" -DEFINE_SINGLETON_CC(hermes::api::Hermes) - -/** Finalize hermes when program exits */ -void __attribute__((destructor)) Finalize() { - HERMES->Finalize(); -} diff --git a/src/binlog.h b/src/binlog.h deleted file mode 100644 index 27c4316c6..000000000 --- a/src/binlog.h +++ /dev/null @@ -1,144 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_BINLOG_H_ -#define HERMES_SRC_BINLOG_H_ - -#include -#include "data_structures.h" -#include -#include -#include -#include -#include - -namespace hermes { - -template -struct BinaryLogRank { - std::vector log_; /**< Cached log entries */ - size_t backend_off_; /**< Entry offset in the backend file */ - - BinaryLogRank() : backend_off_(0) {} -}; - -/** - * A simple file-per-process log format for storing - * execution traces. - * - * This assumes only a single thread modifies or reads - * from the log. Intended for internal use by prefetcher. - * */ -template -class BinaryLog { - public: - std::vector> cache_; /**< The cached log entries */ - size_t max_ingest_; /**< Max number of elements to cache before flush */ - size_t cur_entry_count_; /**< Total number of cached entries */ - std::string path_; /**< Path to the backing log file */ - - public: - /** Default Constructor*/ - BinaryLog() : max_ingest_(0), cur_entry_count_(0) {} - - /** Constructor. */ - void Init(const std::string &path, - size_t max_ingest_bytes) { - max_ingest_ = max_ingest_bytes / sizeof(T); - path_ = path; - // Create + truncate the file - // This is ok because the Hermes daemons are assumed to be spawned before - // applications start running. - std::ofstream output_file(path_); - } - - /** - * Appends all entries in the queue to the cache. - * */ - void Ingest(const hipc::mpsc_queue &queue) { - T entry; - while (!queue.pop(entry).IsNull()) { - AppendEntry(entry); - } - } - - /** - * Appends all entries in the vector to the cache. - * */ - void Ingest(const std::vector &queue) { - for (auto &entry : queue) { - AppendEntry(entry); - } - } - - /** Appends an entry to the cache */ - void AppendEntry(const T &entry) { - if (entry.rank_ >= (int)cache_.size()) { - cache_.resize(entry.rank_ + 1); - } - auto &cache = cache_[entry.rank_]; - if (cache.log_.size() == 0) { - cache.log_.reserve(8192); - } - cache.log_.emplace_back(entry); - cur_entry_count_ += 1; - } - - /** - * Get the total number of ops stored in the rank's log throughout - * its lifetime. - * */ - size_t GetRankLogSize(int rank) { - auto &cache = cache_[rank]; - return cache.log_.size() + cache.backend_off_; - } - - /** - * Get the next entry corresponding to the rank - * */ - bool GetEntry(int rank, size_t off, T &entry) { - auto &cache = cache_[rank]; - if (off < cache.backend_off_ + cache.log_.size()) { - entry = cache.log_[off]; - return true; - } - return false; - } - - /** - * Flush all entries to the backing log - * */ - void Flush(bool force = false) { - if (!force && cur_entry_count_ < max_ingest_) { - return; - } - - // Serialize all contents into the log file - if (path_.empty()) { - std::ofstream output_file(path_, std::ios::out | std::ios::app); - cereal::BinaryOutputArchive oarch(output_file); - for (auto &cache : cache_) { - for (size_t i = 0; i < cache.log_.size(); ++i) { - auto &entry = cache.log_[i]; - oarch(entry); - cache.backend_off_ += 1; - } - cache.log_.clear(); - } - } - cur_entry_count_ = 0; - } -}; - -} // namespace hermes - -#endif // HERMES_SRC_BINLOG_H_ diff --git a/src/borg_io_clients/borg_io_client.h b/src/borg_io_clients/borg_io_client.h deleted file mode 100644 index deb7cf363..000000000 --- a/src/borg_io_clients/borg_io_client.h +++ /dev/null @@ -1,33 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_BORG_IO_CLIENTS_IO_CLIENT_H -#define HERMES_SRC_BORG_IO_CLIENTS_IO_CLIENT_H - -#include "hermes_types.h" -#include "metadata_types.h" - -namespace hermes { - -class BorgIoClient { - public: - virtual ~BorgIoClient() = default; - virtual bool Init(DeviceInfo &dev_info) = 0; - virtual bool Write(DeviceInfo &dev_info, - const char *data, size_t off, size_t size) = 0; - virtual bool Read(DeviceInfo &dev_info, - char *data, size_t off, size_t size) = 0; -}; - -} // namespace hermes - -#endif // HERMES_SRC_BORG_IO_CLIENTS_IO_CLIENT_H diff --git a/src/borg_io_clients/borg_io_client_factory.h b/src/borg_io_clients/borg_io_client_factory.h deleted file mode 100644 index d196cbc8b..000000000 --- a/src/borg_io_clients/borg_io_client_factory.h +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_BORG_IO_CLIENTS_IO_CLIENT_FACTORY_H -#define HERMES_SRC_BORG_IO_CLIENTS_IO_CLIENT_FACTORY_H - -#include "borg_io_client.h" -#include "metadata_types.h" -#include "borg_posix_client.h" -#include "borg_ram_client.h" - -namespace hermes::borg { - -/** - A class to represent I/O Client Factory -*/ -class BorgIoClientFactory { - public: - /** - * Get the I/O api implementation - * */ - static std::unique_ptr Get(IoInterface type) { - switch (type) { - case IoInterface::kPosix: { - return std::make_unique(); - } - case IoInterface::kRam: - default: { - return std::make_unique(); - } - } - } -}; - -} // namespace hermes::borg - -#endif // HERMES_SRC_BORG_IO_CLIENTS_IO_CLIENT_FACTORY_H diff --git a/src/borg_io_clients/borg_posix_client.h b/src/borg_io_clients/borg_posix_client.h deleted file mode 100644 index 02b1846e6..000000000 --- a/src/borg_io_clients/borg_posix_client.h +++ /dev/null @@ -1,85 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_BORG_IO_CLIENTS_POSIX_H -#define HERMES_SRC_BORG_IO_CLIENTS_POSIX_H - -#include "borg_io_client.h" -#include "adapter/posix/posix_api.h" - -#include - -namespace stdfs = std::filesystem; - -namespace hermes::borg { - -class PosixIoClient : public BorgIoClient { - public: - virtual ~PosixIoClient() = default; - - bool Init(DeviceInfo &dev_info) override { - auto api = HERMES_POSIX_API; - std::string text = (*dev_info.mount_dir_).str() + - "/" + "slab_" + (*dev_info.dev_name_).str(); - auto canon = stdfs::weakly_canonical(text).string(); - (*dev_info.mount_point_) = canon; - int fd = api->open((*dev_info.mount_point_).c_str(), - O_TRUNC | O_CREAT, 0666); - if (fd < 0) { return false; } - api->close(fd); - return true; - } - - bool Write(DeviceInfo &dev_info, const char *data, - size_t off, size_t size) override { - auto api = HERMES_POSIX_API; - auto mount_point = (*dev_info.mount_point_).str(); - int fd = api->open(mount_point.c_str(), O_RDWR); - if (fd < 0) { - HELOG(kError, "Failed to open (write): {}", - dev_info.mount_point_->str()) - return false; - } - size_t count = api->pwrite(fd, data, size, off); - api->close(fd); - if (count != size) { - HELOG(kError, "BORG: wrote {} bytes, but expected {}", - count, size); - return false; - } - return true; - } - - bool Read(DeviceInfo &dev_info, char *data, - size_t off, size_t size) override { - auto api = HERMES_POSIX_API; - auto mount_point = (*dev_info.mount_point_).str(); - int fd = api->open(mount_point.c_str(), O_RDWR); - if (fd < 0) { - HELOG(kError, "Failed to open (read): {}", - dev_info.mount_point_->str()) - return false; - } - size_t count = api->pread(fd, data, size, off); - api->close(fd); - if (count != size) { - HELOG(kError, "BORG: read {} bytes, but expected {}", - count, size); - return false; - } - return true; - } -}; - -} // namespace hermes::borg - -#endif // HERMES_SRC_BORG_IO_CLIENTS_POSIX_H diff --git a/src/borg_io_clients/borg_ram_client.h b/src/borg_io_clients/borg_ram_client.h deleted file mode 100644 index a018d84dd..000000000 --- a/src/borg_io_clients/borg_ram_client.h +++ /dev/null @@ -1,63 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_BORG_IO_CLIENTS_RAM_H_ -#define HERMES_SRC_BORG_IO_CLIENTS_RAM_H_ - -#include "borg_io_client.h" -#include "adapter/posix/posix_api.h" -#include "hermes.h" - -namespace hermes::borg { - -class RamIoClient : public BorgIoClient { - public: - virtual ~RamIoClient() = default; - - bool Init(DeviceInfo &dev_info) override { - auto &hermes_header = HERMES->header_; - auto &main_alloc = HERMES->main_alloc_; - hermes_header->ram_tier_ = main_alloc-> - Allocate(dev_info.capacity_); - if (hermes_header->ram_tier_.IsNull()) { - HELOG(kFatal, BUFFER_POOL_OUT_OF_RAM.Msg()); - } - return true; - } - - bool Write(DeviceInfo &dev_info, const char *data, - size_t off, size_t size) override { - auto &hermes_header = HERMES->header_; - auto &main_alloc = HERMES->main_alloc_; - char *ram_ptr = main_alloc->Convert(hermes_header->ram_tier_); - if (off + size > dev_info.capacity_) { - HILOG(kDebug, "Out of bounds: attempting to write to offset: {} / {}", - off + size, dev_info.capacity_); - return false; - } - memcpy(ram_ptr + off, data, size); - return true; - } - - bool Read(DeviceInfo &dev_info, char *data, - size_t off, size_t size) override { - auto &hermes_header = HERMES->header_; - auto &main_alloc = HERMES->main_alloc_; - char *ram_ptr = main_alloc->Convert(hermes_header->ram_tier_); - memcpy(data, ram_ptr + off, size); - return true; - } -}; - -} // namespace hermes::borg - -#endif // HERMES_SRC_BORG_IO_CLIENTS_RAM_H_ diff --git a/src/buffer_organizer.cc b/src/buffer_organizer.cc deleted file mode 100644 index 5f9a130b6..000000000 --- a/src/buffer_organizer.cc +++ /dev/null @@ -1,539 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "buffer_organizer.h" -#include "metadata_manager.h" -#include "borg_io_clients/borg_io_client_factory.h" -#include "hermes.h" -#include "bucket.h" - -namespace hermes { - -/**==================================== - * BORG I/O thread manager - * ===================================*/ - -/** Spawn a thread for re-organizing blobs */ -void BorgIoThreadManager::SpawnBlobMonitor() { - auto flush_scheduler = [](void *args) { - HILOG(kDebug, "Blob re-organization thread has started") - (void) args; - auto borg = &HERMES->borg_; - while (HERMES_THREAD_MANAGER->Alive()) { - borg->LocalAnalyzeBlobs(); - tl::thread::self().sleep(*HERMES->rpc_.server_engine_, - HERMES->server_config_.borg_.blob_reorg_period_); - } - HERMES_BORG_IO_THREAD_MANAGER->Join(); - HILOG(kDebug, "Blob re-organization thread has stopped") - }; - HERMES_THREAD_MANAGER->Spawn(flush_scheduler); -} - -/** Spawn the enqueuing thread */ -void BorgIoThreadManager::SpawnFlushMonitor() { - auto flush_scheduler = [](void *args) { - HILOG(kDebug, "Flushing scheduler thread has started") - (void) args; - auto borg = &HERMES->borg_; - while (HERMES_THREAD_MANAGER->Alive()) { - borg->LocalEnqueueFlushes(); - tl::thread::self().sleep(*HERMES->rpc_.server_engine_, - HERMES->server_config_.borg_.flush_period_); - } - HERMES_BORG_IO_THREAD_MANAGER->Join(); - HILOG(kDebug, "Flush scheduler thread has stopped") - }; - HERMES_THREAD_MANAGER->Spawn(flush_scheduler); -} - -/** Spawn the flushing I/O threads */ -void BorgIoThreadManager::SpawnFlushWorkers(int num_threads) { - // Define flush worker thread function - // The function will continue working until all pending flushes have - // been processed - auto flush = [](void *params) { - BufferOrganizer *borg = &HERMES->borg_; - int *id = reinterpret_cast(params); - BorgIoThreadQueue &bq = (*borg->queues_)[*id]; - BorgIoThreadQueueInfo& bq_info = bq.GetSecond(); - _BorgIoThreadQueue& queue = bq.GetFirst(); - HILOG(kDebug, "Flushing worker {} has started", bq_info.id_) - while (HERMES_BORG_IO_THREAD_MANAGER->Alive() || - (!HERMES_BORG_IO_THREAD_MANAGER->Alive() && bq_info.load_)) { - borg->LocalProcessFlushes(bq_info, queue); - tl::thread::self().sleep(*HERMES->rpc_.server_engine_, 25); - } - HILOG(kDebug, "Flushing worker {} has stopped", bq_info.id_) - }; - - // Create the flushing threads - for (int i = 0; i < num_threads; ++i) { - BorgIoThreadQueue &bq = (*queues_)[i]; - BorgIoThreadQueueInfo& bq_info = bq.GetSecond(); - bq_info.id_ = i; - bq_info.load_ = 0; - HERMES_THREAD_MANAGER->Spawn(flush, &bq_info.id_); - } -} - -/** Wait for flushing to complete */ -void BorgIoThreadManager::WaitForFlush() { - HILOG(kDebug, "Waiting for all flushing to complete {}", - HERMES->rpc_.node_id_) - while (IsFlushing()) { - tl::thread::self().sleep(*HERMES->rpc_.server_engine_, 5); - } - HILOG(kDebug, "Finished flushing {}", - HERMES->rpc_.node_id_) -} - -/**==================================== - * SHM Init - * ===================================*/ - -/** - * Initialize the BORG - * REQUIRES mdm to be initialized already. - * */ -void BufferOrganizer::shm_init(hipc::ShmArchive &header, - hipc::Allocator *alloc) { - shm_deserialize(header); - - // Initialize device information - for (TargetInfo &target : (*mdm_->targets_)) { - DeviceInfo &dev_info = - (*mdm_->devices_)[target.id_.GetDeviceId()]; - if (dev_info.mount_dir_->size() == 0) { - dev_info.io_api_ = IoInterface::kRam; - } else { - dev_info.io_api_ = IoInterface::kPosix; - } - auto io_client = borg::BorgIoClientFactory::Get(dev_info.io_api_); - io_client->Init(dev_info); - } - - // Print out device info - mdm_->PrintDeviceInfo(); - - // Spawn the thread for flushing blobs - int num_threads = HERMES->server_config_.borg_.num_threads_; - HSHM_MAKE_AR((*header).queues_, alloc, num_threads) - HERMES_BORG_IO_THREAD_MANAGER->queues_ = queues_; - HERMES_BORG_IO_THREAD_MANAGER->SpawnBlobMonitor(); - HERMES_BORG_IO_THREAD_MANAGER->SpawnFlushMonitor(); - HERMES_BORG_IO_THREAD_MANAGER->SpawnFlushWorkers(num_threads); -} - -/**==================================== - * SHM Deserialization - * ===================================*/ - -/** Deserialize the BORG from shared memory */ -void BufferOrganizer::shm_deserialize( - hipc::ShmArchive &header) { - mdm_ = &HERMES->mdm_; - rpc_ = &HERMES->rpc_; - queues_ = (*header).queues_.get(); - HERMES_BORG_IO_THREAD_MANAGER->queues_ = queues_; -} - -/**==================================== - * Destructors - * ===================================*/ - -/** Finalize the BORG */ -void BufferOrganizer::shm_destroy_main() { - queues_->shm_destroy(); -} - -/**==================================== - * BORG Methods - * ===================================*/ - -/** Stores a blob into a set of buffers */ -RPC void BufferOrganizer::LocalPlaceBlobInBuffers( - const Blob &blob, std::vector &buffers) { - AUTO_TRACE(1) - size_t blob_off = 0; - for (BufferInfo &buffer_info : buffers) { - if (buffer_info.tid_.GetNodeId() != mdm_->rpc_->node_id_) { - continue; - } - TIMER_START("DeviceInfo") - DeviceInfo &dev_info = - (*mdm_->devices_)[buffer_info.tid_.GetDeviceId()]; - if (buffer_info.t_off_ + buffer_info.blob_size_ > - dev_info.capacity_) { - HELOG(kFatal, "Out of bounds: attempting to write to offset: {} / {} " - "on device {}: {}", - buffer_info.t_off_ + buffer_info.blob_size_, - dev_info.capacity_, - buffer_info.tid_.GetDeviceId(), - dev_info.mount_point_->str()) - } - auto io_client = borg::BorgIoClientFactory::Get(dev_info.io_api_); - TIMER_END() - - TIMER_START("IO") - bool ret = io_client->Write(dev_info, - blob.data() + blob_off, - buffer_info.t_off_, - buffer_info.blob_size_); - TIMER_END() - blob_off += buffer_info.blob_size_; - if (!ret) { - mdm_->PrintDeviceInfo(); - HELOG(kFatal, "Could not perform I/O in BORG." - " Writing to target ID:" - " (node_id: {}, tgt_id: {}, dev_id: {}," - " t_off: {}, blob_size: {})", - buffer_info.tid_.GetNodeId(), - buffer_info.tid_.GetIndex(), - buffer_info.tid_.GetDeviceId(), - buffer_info.t_off_, - buffer_info.blob_size_) - } - } -} - -/** Globally store a blob into a set of buffers */ -void BufferOrganizer::GlobalPlaceBlobInBuffers( - const Blob &blob, std::vector &buffers) { - AUTO_TRACE(1) - // Get the nodes to transfer buffers to - size_t total_size; - auto unique_nodes = BufferPool::GroupByNodeId(buffers, total_size); - - // Send the buffers to each node - for (auto &[node_id, size] : unique_nodes) { - if (NODE_ID_IS_LOCAL(node_id)) { - LocalPlaceBlobInBuffers(blob, buffers); - } else { - rpc_->IoCall( - node_id, "RpcPlaceBlobInBuffers", - IoType::kWrite, blob.data(), blob.size(), - blob.size(), buffers); - } - } -} - -/** Stores a blob into a set of buffers */ -RPC void BufferOrganizer::LocalReadBlobFromBuffers( - Blob &blob, std::vector &buffers) { - AUTO_TRACE(1) - size_t blob_off = 0; - for (BufferInfo &buffer_info : buffers) { - if (buffer_info.tid_.GetNodeId() != mdm_->rpc_->node_id_) { - continue; - } - DeviceInfo &dev_info = - (*mdm_->devices_)[buffer_info.tid_.GetDeviceId()]; - if (buffer_info.t_off_ + buffer_info.blob_size_ > - dev_info.capacity_) { - HELOG(kFatal, "Out of bounds: attempting to read from offset: {} / {}" - " on device {}: {}", - buffer_info.t_off_ + buffer_info.blob_size_, - dev_info.capacity_, - buffer_info.tid_.GetDeviceId(), - dev_info.mount_point_->str()) - } - auto io_client = borg::BorgIoClientFactory::Get(dev_info.io_api_); - bool ret = io_client->Read(dev_info, - blob.data() + blob_off, - buffer_info.t_off_, - buffer_info.blob_size_); - blob_off += buffer_info.blob_size_; - if (!ret) { - HELOG(kFatal, "Could not perform I/O in BORG." - " reading from target ID:" - " (node_id: {}, tgt_id: {}, dev_id: {}," - " t_off: {}, blob_size: {})", - buffer_info.tid_.GetNodeId(), - buffer_info.tid_.GetIndex(), - buffer_info.tid_.GetDeviceId(), - buffer_info.t_off_, - buffer_info.blob_size_) - } - } -} - -/** The Global form of ReadBLobFromBuffers */ -Blob BufferOrganizer::GlobalReadBlobFromBuffers( - std::vector &buffers) { - AUTO_TRACE(1) - // Get the nodes to transfer buffers to - size_t total_size = 0; - auto unique_nodes = BufferPool::GroupByNodeId(buffers, total_size); - - // Send the buffers to each node - std::vector blobs; - blobs.reserve(unique_nodes.size()); - for (auto &[node_id, size] : unique_nodes) { - blobs.emplace_back(size); - if (NODE_ID_IS_LOCAL(node_id)) { - LocalReadBlobFromBuffers(blobs.back(), buffers); - } else { - rpc_->IoCall( - node_id, "RpcReadBlobFromBuffers", - IoType::kRead, blobs.back().data(), size, - size, buffers); - } - } - - // If the blob was only on one node - if (unique_nodes.size() == 1) { - return std::move(blobs.back()); - } - - // Merge the blobs at the end - hapi::Blob blob(total_size); - for (size_t i = 0; i < unique_nodes.size(); ++i) { - auto &[node_id, size] = unique_nodes[i]; - auto &tmp_blob = blobs[i]; - size_t tmp_blob_off = 0; - for (BufferInfo &info : buffers) { - if (info.tid_.GetNodeId() != node_id) { - continue; - } - memcpy(blob.data() + info.blob_off_, - tmp_blob.data() + tmp_blob_off, - info.blob_size_); - tmp_blob_off += info.blob_size_; - } - } - return blob; -} - -/** Re-organize blobs based on a score */ -void BufferOrganizer::GlobalOrganizeBlob(const std::string &bucket_name, - const std::string &blob_name, - float score) { - AUTO_TRACE(1) - auto bkt = HERMES->GetBucket(bucket_name); - BlobId blob_id; - bkt.GetBlobId(blob_name, blob_id); - float blob_score = bkt.GetBlobScore(blob_id); - GlobalOrganizeBlob(bkt, blob_name, blob_id, blob_score, score); -} - -/** Re-organize blobs based on a score */ -void BufferOrganizer::GlobalOrganizeBlob(hapi::Bucket &bkt, - const std::string &blob_name, - BlobId &blob_id, - float blob_score, - float score) { - AUTO_TRACE(1); - Context ctx; - - HILOG(kDebug, "Changing blob score from: {} to {}", blob_score, score) - - // Skip organizing if below threshold - if (abs(blob_score - score) < .05) { - return; - } - - // Lock the blob to ensure it doesn't get modified - bkt.LockBlob(blob_id, MdLockType::kExternalWrite); - - // Get the blob - hapi::Blob blob; - bkt.Get(blob_id, blob, ctx); - - // Re-emplace the blob with new score - BlobId tmp_id; - ctx.blob_score_ = score; - bkt.Put(blob_name, blob, tmp_id, ctx); - - // Unlock the blob - bkt.UnlockBlob(blob_id, MdLockType::kExternalWrite); -} - -/**==================================== - * BORG Flushing methods - * ===================================*/ - -/** Find blobs which should be re-organized */ -void BufferOrganizer::LocalAnalyzeBlobs() { - auto mdm = &HERMES->mdm_; - ScopedRwReadLock blob_map_lock(mdm->header_->lock_[kBlobMapLock], - kBORG_LocalEnqueueFlushes); - float recency_min = HERMES->server_config_.borg_.recency_min_; - float recency_max = HERMES->server_config_.borg_.recency_max_; - float freq_max = HERMES->server_config_.borg_.freq_max_; - float freq_min = HERMES->server_config_.borg_.freq_min_; - - // Only re-organize if there's a capacity trigger - bool is_below_thresh = false; - auto targets = mdm->LocalGetTargetInfo(); - for (TargetInfo &target : targets) { - DeviceInfo &dev_info = - (*mdm_->devices_)[target.id_.GetDeviceId()]; - float rem_cap = (float) target.rem_cap_ / (float)target.max_cap_; - if (rem_cap < dev_info.borg_min_thresh_) { - is_below_thresh = true; - } - } - if (!is_below_thresh) { - return; - } - - u64 time = BlobInfo::GetTimeFromStartNs(); - for (hipc::pair& blob_p : *mdm->blob_map_) { - BlobInfo &blob_info = blob_p.GetSecond(); - // Get the recency score [0, 1] - float last_access_elapse = (float)(time - blob_info.last_access_); - float recency_score; - if (last_access_elapse <= recency_min) { - recency_score = 1; - } else if (last_access_elapse >= recency_max) { - recency_score = 0; - } else { - recency_score = (last_access_elapse - recency_min) / - (recency_max - recency_min); - recency_score = 1 - recency_score; - } - - // Get the frequency score [0, 1] - float freq_score; - float freq = (float)blob_info.access_freq_; - if (freq <= freq_min) { - freq_score = 0; - } else if (freq >= freq_max) { - freq_score = 1; - } else { - freq_score = (freq - freq_min) / (freq_max - freq_min); - } - - // Update the current blob score - auto bkt = HERMES->GetBucket(blob_info.tag_id_); - GlobalOrganizeBlob(bkt, - blob_info.name_->str(), - blob_info.blob_id_, - blob_info.score_, - std::max(freq_score, recency_score)); - } -} - -/** Flush all blobs registered in this daemon */ -void BufferOrganizer::LocalEnqueueFlushes() { - auto mdm = &HERMES->mdm_; - // Acquire the read lock on the blob map - ScopedRwReadLock blob_map_lock(mdm->header_->lock_[kBlobMapLock], - kBORG_LocalEnqueueFlushes); - // Begin checking for blobs which need flushing - size_t count = 0; - for (hipc::pair& blob_p : *mdm->blob_map_) { - BlobId &blob_id = blob_p.GetFirst(); - BlobInfo &blob_info = blob_p.GetSecond(); - // Verify that flush is needing to happen - if (blob_info.mod_count_ == blob_info.last_flush_) { - continue; - } - // Check if bucket has flush trait - TagId &bkt_id = blob_info.tag_id_; - size_t blob_size = blob_info.blob_size_; - std::vector traits = HERMES->GetTraits(bkt_id, - HERMES_TRAIT_FLUSH); - if (traits.size() == 0) { - continue; - } - // Schedule the blob on an I/O worker thread - HERMES_BORG_IO_THREAD_MANAGER->Enqueue(bkt_id, blob_id, blob_size, - std::move(traits)); - count += 1; - } - if (count) { - HILOG(kDebug, "Flushing {} blobs", count); - } -} - -/** Actually process flush operations */ -void BufferOrganizer::LocalProcessFlushes( - BorgIoThreadQueueInfo &bq_info, - _BorgIoThreadQueue& queue) { - // Begin flushing - ScopedRwWriteLock flush_lock(mdm_->header_->lock_[kFlushLock], - kBORG_LocalProcessFlushes); - // Process tasks - auto entry = hipc::make_uptr(); - while (!queue.pop(*entry).IsNull()) { - BorgIoTask &task = *entry; - HILOG(kDebug, "Attempting to flush blob {}", task.blob_id_); - Blob blob; - - // Verify the blob exists and then read lock it - auto iter = mdm_->blob_map_->find(task.blob_id_); - if (iter.is_end()) { - bq_info.load_.fetch_sub(task.blob_size_); - HILOG(kDebug, "Finished BORG task for blob {} and load {}", - task.blob_id_, bq_info.load_.load()) - continue; - } - hipc::pair& blob_info_p = *iter; - BlobInfo &blob_info = blob_info_p.GetSecond(); - std::string blob_name = blob_info.name_->str(); - - // Verify that flush is needing to happen - if (blob_info.mod_count_ == blob_info.last_flush_) { - bq_info.load_.fetch_sub(task.blob_size_); - HILOG(kDebug, "Finished BORG task for blob {} and load {}", - task.blob_id_, bq_info.load_.load()) - continue; - } - size_t last_flush = blob_info.mod_count_; - blob_info.last_flush_ = last_flush; - - // Get the current blob from Hermes - api::Bucket bkt = HERMES->GetBucket(task.bkt_id_); - bkt.Get(task.blob_id_, blob, bkt.GetContext()); - HILOG(kDebug, "Flushing blob {} ({}) of size {}", - blob_name, - task.blob_id_, - blob.size()) - FlushTraitParams trait_params; - for (Trait *trait : task.traits_) { - trait_params.blob_ = &blob; - trait_params.blob_name_ = blob_name; - trait_params.bkt_ = &bkt; - trait->Run(HERMES_TRAIT_FLUSH, &trait_params); - } - - // Dequeue - bq_info.load_.fetch_sub(task.blob_size_); - HILOG(kDebug, "Finished BORG task for blob {} and load {}", - task.blob_id_, bq_info.load_.load()) - } -} - -/** Barrier for all flushing to complete */ -void BufferOrganizer::LocalWaitForFullFlush() { - HILOG(kInfo, "Full synchronous flush on node {}", rpc_->node_id_) - LocalEnqueueFlushes(); - HERMES_BORG_IO_THREAD_MANAGER->WaitForFlush(); - HILOG(kInfo, "Finished synchronous flush on node {}", rpc_->node_id_) -} - -/** Barrier for all I/O in Hermes to flush */ -void BufferOrganizer::GlobalWaitForFullFlush() { - for (int i = 0; i < (int)rpc_->hosts_.size(); ++i) { - int node_id = i + 1; - HILOG(kInfo, "Wait for flush on node {}", node_id) - if (node_id == rpc_->node_id_) { - LocalWaitForFullFlush(); - } else { - rpc_->Call(node_id, "RpcWaitForFullFlush"); - } - } -} - -} // namespace hermes diff --git a/src/buffer_organizer.h b/src/buffer_organizer.h deleted file mode 100644 index ea07eea61..000000000 --- a/src/buffer_organizer.h +++ /dev/null @@ -1,289 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_BUFFER_ORGANIZER_H_ -#define HERMES_SRC_BUFFER_ORGANIZER_H_ - -#include "rpc.h" -#include "hermes_types.h" -#include "buffer_pool.h" -#include "trait_manager.h" - -/** Forward declaration of Bucket */ -namespace hermes::api { -class Bucket; -} // namespace hermes::api - -namespace hermes { - -// NOTE(llogan): I may add this back to make flushing more intelligent -/** Calculates the total size of a blob's buffers -static inline size_t SumBufferBlobSizes(std::vector &buffers) { - size_t sum = 0; - for (BufferInfo &buffer_ref : buffers) { - sum += buffer_ref.blob_size_; - } - return sum; -}*/ - -/** Information needed by traits called internally by Flush */ -struct FlushTraitParams { - Blob *blob_; - std::string blob_name_; - api::Bucket *bkt_; -}; - -/** An I/O flushing task spawned by BORG */ -struct BorgIoTask { - TagId bkt_id_; - BlobId blob_id_; - size_t blob_size_; - size_t last_modified_; - std::vector traits_; - - /** Default constructor */ - BorgIoTask() = default; - - /** Flush emplace constructor */ - explicit BorgIoTask(TagId bkt_id, BlobId blob_id, size_t blob_size, - std::vector &&traits) - : bkt_id_(bkt_id), blob_id_(blob_id), blob_size_(blob_size), - traits_(std::forward>(traits)) {} - - /** Copy constructor */ - BorgIoTask(const BorgIoTask &other) = default; - - /** Copy assignment operator */ - BorgIoTask& operator=(const BorgIoTask &other) = default; - - /** Move constructor */ - BorgIoTask(BorgIoTask &&other) = default; - - /** Move assignment operator */ - BorgIoTask& operator=(BorgIoTask &&other) = default; -}; - -/** Info about a BORG I/O queue */ -struct BorgIoThreadQueueInfo { - std::atomic load_; /**< Data being processed on queue */ - int id_; /**< Unique ID of worker */ - - /** Default constructor */ - BorgIoThreadQueueInfo() { - load_ = 0; - id_ = 0; - } - - /** Copy constructor */ - BorgIoThreadQueueInfo(const BorgIoThreadQueueInfo &other) - : load_(other.load_.load()), id_(other.id_) {} - - /** Copy assignment */ - BorgIoThreadQueueInfo& operator=(const BorgIoThreadQueueInfo &other) { - if (this != &other) { - load_ = other.load_.load(); - id_ = other.id_; - } - return *this; - } - - /** Move constructor */ - BorgIoThreadQueueInfo(BorgIoThreadQueueInfo &&other) - : load_(other.load_.load()), id_(other.id_) {} - - /** Move assignment */ - BorgIoThreadQueueInfo& operator=(BorgIoThreadQueueInfo &&other) { - if (this != &other) { - load_ = other.load_.load(); - id_ = other.id_; - } - return *this; - } -}; - -/** A SHM queue for holding BORG I/O flushing tasks */ -typedef hipc::mpsc_queue _BorgIoThreadQueue; - -/** A SHM queue for holding tasks + statistics */ -typedef hipc::pair<_BorgIoThreadQueue, BorgIoThreadQueueInfo> - BorgIoThreadQueue; - -/** Macro to simplify thread manager singleton */ -#define HERMES_BORG_IO_THREAD_MANAGER \ - hshm::EasySingleton::GetInstance() - -/** Manages the I/O flushing threads for BORG */ -class BorgIoThreadManager { - public: - hipc::vector - *queues_; /**< Shared-memory request queues */ - std::atomic kill_requested_; /**< Kill flushing threads eventually */ - - public: - /** Constructor */ - BorgIoThreadManager() { - kill_requested_.store(false); - } - - /** Indicate that no more modifications will take place to blobs */ - void Join() { - kill_requested_.store(true); - } - - /** Check if IoThreadManager is still alive */ - bool Alive() { - return !kill_requested_.load(); - } - - /** Spawn a thread for re-organizing blobs */ - void SpawnBlobMonitor(); - - /** Spawn the enqueuing thread */ - void SpawnFlushMonitor(); - - /** Spawn the I/O threads */ - void SpawnFlushWorkers(int num_threads); - - /** Wait for flushing to complete */ - void WaitForFlush(); - - /** Check if a flush is still happening */ - bool IsFlushing() { - for (BorgIoThreadQueue &bq : *queues_) { - BorgIoThreadQueueInfo& bq_info = bq.GetSecond(); - if (bq_info.load_ > 0) { - return true; - } - } - return false; - } - - /** Enqueue a flushing task to a worker */ - void Enqueue(TagId bkt_id, BlobId blob_id, size_t blob_size, - std::vector &&traits) { - BorgIoThreadQueue &bq = HashToQueue(blob_id); - BorgIoThreadQueueInfo& bq_info = bq.GetSecond(); - _BorgIoThreadQueue& queue = bq.GetFirst(); - bq_info.load_.fetch_add(blob_size); - queue.emplace(bkt_id, blob_id, blob_size, - std::forward>(traits)); - } - - /** Hash request to a queue */ - BorgIoThreadQueue& HashToQueue(BlobId blob_id) { - size_t qid = std::hash{}(blob_id) % queues_->size(); - return (*queues_)[qid]; - } -}; - -/** - * Any state needed by BORG in SHM - * */ -struct BufferOrganizerShm { - hipc::ShmArchive> queues_; -}; - -/** - * Manages the organization of blobs in the hierarchy. - * */ -class BufferOrganizer { - public: - MetadataManager *mdm_; - RPC_TYPE *rpc_; - hipc::vector* - queues_; /** Async tasks for BORG. */ - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** Default constructor */ - BufferOrganizer() = default; - - /**==================================== - * SHM Init - * ===================================*/ - - /** - * Initialize the BORG - * REQUIRES mdm to be initialized already. - * */ - void shm_init(hipc::ShmArchive &header, - hipc::Allocator *alloc); - - /**==================================== - * SHM Deserialization - * ===================================*/ - - /** Deserialize the BORG from shared memory */ - void shm_deserialize(hipc::ShmArchive &header); - - /**==================================== - * Destructors - * ===================================*/ - - /** Finalize the BORG */ - void shm_destroy_main(); - - /**==================================== - * BORG Methods - * ===================================*/ - - /** Stores a blob into a set of buffers */ - RPC void LocalPlaceBlobInBuffers(const Blob &blob, - std::vector &buffers); - RPC void GlobalPlaceBlobInBuffers(const Blob &blob, - std::vector &buffers); - - /** Stores a blob into a set of buffers */ - RPC void LocalReadBlobFromBuffers(Blob &blob, - std::vector &buffers); - Blob GlobalReadBlobFromBuffers(std::vector &buffers); - - /** Re-organize blobs based on a score */ - void GlobalOrganizeBlob(const std::string &bucket_name, - const std::string &blob_name, - float score); - - /** Re-organize blobs based on a score */ - void GlobalOrganizeBlob(hermes::api::Bucket &bkt, - const std::string &blob_name, - BlobId &blob_id, - float blob_score, - float score); - - public: - /**==================================== - * BORG Flushing methods - * ===================================*/ - - /** Find blobs which should be re-organized */ - void LocalAnalyzeBlobs(); - - /** Flush all blobs registered in this daemon */ - void LocalEnqueueFlushes(); - - /** Actually process flush operations */ - void LocalProcessFlushes(BorgIoThreadQueueInfo &bq_info, - _BorgIoThreadQueue &queue); - - /** Barrier for all flushing to complete */ - void LocalWaitForFullFlush(); - - /** Barrier for all I/O in Hermes to flush */ - void GlobalWaitForFullFlush(); -}; - -} // namespace hermes - -#endif // HERMES_SRC_BUFFER_ORGANIZER_H_ diff --git a/src/buffer_pool.cc b/src/buffer_pool.cc deleted file mode 100644 index b772144ba..000000000 --- a/src/buffer_pool.cc +++ /dev/null @@ -1,444 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "buffer_pool.h" -#include "metadata_manager.h" -#include "hermes.h" -#include "buffer_organizer.h" - -namespace hermes { - -/**==================================== - * Default Constructor - * ===================================*/ - -/** -* Initialize the BPM and its shared memory. -* REQUIRES mdm to be initialized already. -* */ -void BufferPool::shm_init(hipc::ShmArchive &header, - hipc::Allocator *alloc) { - shm_deserialize(header); - // Initialize header - HSHM_MAKE_AR0(header_->free_lists_, alloc); - // [target] [cpu] [page_size] - header_->ntargets_ = mdm_->targets_->size(); - header_->concurrency_ = HERMES_SYSTEM_INFO->ncpu_; - header_->nslabs_ = 0; - // Get the maximum number of slab sizes over all targets - for (TargetInfo &target : *mdm_->targets_) { - int dev_id = target.id_.GetDeviceId(); - DeviceInfo &dev_info = (*mdm_->devices_)[dev_id]; - if (header_->nslabs_ < dev_info.slab_sizes_->size()) { - header_->nslabs_ = dev_info.slab_sizes_->size(); - } - } - // Create target free lists - target_allocs_->resize(header_->ntargets_ * header_->concurrency_ * - header_->nslabs_); - // Initialize target free lists - for (u16 target_id = 0; target_id < header_->ntargets_; ++target_id) { - for (size_t cpu = 0; cpu < header_->concurrency_; ++cpu) { - // Get the target + device info - TargetInfo &target = (*mdm_->targets_)[target_id]; - int dev_id = target.id_.GetDeviceId(); - DeviceInfo &dev_info = (*mdm_->devices_)[dev_id]; - size_t size_per_core = target.max_cap_ / header_->concurrency_; - if (size_per_core < MEGABYTES(1)) { - HELOG(kFatal, "The capacity of the target {} ({} bytes)" - " is not enough to give each of the {} CPUs at least" - " 1MB of space", - dev_info.mount_point_->str(), - target.max_cap_, header_->concurrency_) - } - - // Initialize the core's metadata - BpFreeListStat *target_stat; - GetTargetStatForCpu(target_id, cpu, target_stat); - target_stat->region_off_ = cpu * size_per_core; - target_stat->region_size_ = size_per_core; - target_stat->lock_.Init(); - } - } -} - -/**==================================== - * SHM Deserialize - * ===================================*/ - -/** Deserialize the BPM from shared memory */ -void BufferPool::shm_deserialize(hipc::ShmArchive &header) { - mdm_ = &HERMES->mdm_; - borg_ = &HERMES->borg_; - rpc_ = &HERMES->rpc_; - header_ = header.get(); - target_allocs_ = header_->free_lists_.get(); -} - -/**==================================== - * Destructor - * ===================================*/ - -/** Destroy the BPM shared memory. */ -void BufferPool::shm_destroy_main() { - target_allocs_->shm_destroy(); -} - -/**==================================== - * Allocate Buffers - * ===================================*/ - -/** -* Allocate buffers from the targets according to the schema -* */ -std::vector -BufferPool::LocalAllocateAndSetBuffers(PlacementSchema &schema, - const Blob &blob) { - AUTO_TRACE(1) - std::vector buffers; - size_t blob_off = 0; - int cpu = hshm::NodeThreadId().hash() % header_->concurrency_; - - TIMER_START("AllocateBuffers") - for (SubPlacement &plcmnt : schema.plcmnts_) { - // Get the target and device in the placement schema - if (plcmnt.tid_.GetNodeId() != mdm_->rpc_->node_id_) { - blob_off += plcmnt.size_; - continue; - } - int dev_id = plcmnt.tid_.GetDeviceId(); - DeviceInfo &dev_info = (*mdm_->devices_)[dev_id]; - - // Get the number of each buffer size to allocate - size_t buffer_count; - std::vector coins = CoinSelect(dev_info, - plcmnt.size_, - buffer_count); - - // Allocate buffers - AllocateBuffers(plcmnt.size_, - coins, - plcmnt.tid_, - cpu, - blob_off, - buffers); - } - TIMER_END() - borg_->LocalPlaceBlobInBuffers(blob, buffers); - return buffers; -} - -/** - * The RPC of LocalAllocateAndSendBuffers - * */ -std::vector -BufferPool::GlobalAllocateAndSetBuffers(PlacementSchema &schema, - const Blob &blob) { - AUTO_TRACE(1) - // Get the nodes to transfer buffers to - size_t total_size; - auto unique_nodes = GroupByNodeId(schema, total_size); - std::vector info; - - // Send the buffers to each node - for (auto &[node_id, size] : unique_nodes) { - std::vector sub_info; - if (NODE_ID_IS_LOCAL(node_id)) { - sub_info = LocalAllocateAndSetBuffers(schema, blob); - } else { - sub_info = rpc_->IoCall>( - node_id, "RpcAllocateAndSetBuffers", - IoType::kWrite, blob.data(), blob.size(), - blob.size(), schema); - } - - // Concatenate - info.reserve(info.size() + sub_info.size()); - for (BufferInfo &tmp_info : sub_info) { - info.emplace_back(tmp_info); - } - } - - return info; -} - -/** - * Allocate requested slabs from this target. - * If the target runs out of space, it will provision from the next target. - * We assume there is a baseline target, which can house an infinite amount - * of data. This would be the PFS in an HPC machine. - * - * @param total_size the total amount of data being placed in this target - * @param coins The requested number of slabs to allocate from this target - * @param target_id The ID of the (ideal) target to allocate from - * @param blob_off [out] The current size of the blob which has been placed - * @param buffers [out] The buffers which were allocated - * */ -void BufferPool::AllocateBuffers(size_t total_size, - std::vector &coins, - TargetId tid, - int cpu, - size_t &blob_off, - std::vector &buffers) { - // Get this target's stack allocator - size_t rem_size = total_size; - - // Allocate each slab size - for (size_t slab_id = 0; slab_id < coins.size(); ++slab_id) { - size_t slab_size = coins[slab_id].slab_size_; - size_t slab_count = coins[slab_id].count_; - while (slab_count) { - // Allocate slabs - AllocateSlabs(rem_size, slab_size, slab_count, slab_id, tid, - cpu, blob_off, buffers); - - // Go to the next target if there was not enough space in this target - if (slab_count > 0) { - i32 target_id = tid.GetIndex() + 1; - if (target_id >= (i32)mdm_->targets_->size()) { - HELOG(kFatal, "BORG ran out of space on all targets." - " This shouldn't happen." - " Please increase the amount of space dedicated to PFS.") - } - tid = (*mdm_->targets_)[target_id].id_; - } - } - } -} - -/** - * Allocate a set of slabs of particular size from this target. - * - * @param rem_size the amount of data remaining that needs to be allocated - * @param slab_size Size of the slabs to allocate - * @param slab_count The count of this slab size to allocate - * @param slab_id The offset of the slab in the device's slab list - * @param tid The target to allocate slabs from - * @param cpu the CPU this node is on - * @param blob_off [out] The current size of the blob which has been placed - * @param buffers [out] The buffers which were allocated - * */ -void BufferPool::AllocateSlabs(size_t &rem_size, - size_t slab_size, - size_t &slab_count, - size_t slab_id, - TargetId tid, - int cpu, - size_t &blob_off, - std::vector &buffers) { - int cpu_off = 0; - int ncpu = header_->concurrency_; - - // Get the free list for this CPU - BpFreeList *free_list; - BpFreeListStat *free_list_stat; - BpFreeListStat *target_stat; - GetFreeListForCpu(tid.GetIndex(), cpu, slab_id, free_list, - free_list_stat); - GetTargetStatForCpu(tid.GetIndex(), cpu, target_stat); - target_stat->lock_.Lock(10); - - while (slab_count > 0) { - BpSlot slot = - AllocateSlabSize(cpu, slab_size, - free_list, free_list_stat, target_stat); - if (slot.IsNull()) { - if (cpu_off < ncpu) { - cpu = (cpu + 1) % ncpu; - target_stat->lock_.Unlock(); - GetFreeListForCpu(tid.GetIndex(), cpu, slab_id, free_list, - free_list_stat); - GetTargetStatForCpu(tid.GetIndex(), cpu, target_stat); - target_stat->lock_.Lock(11); - cpu_off += 1; - continue; - } else { - target_stat->lock_.Unlock(); - return; - } - } - buffers.emplace_back(); - BufferInfo &info = buffers.back(); - info.t_off_ = slot.t_off_; - info.t_size_ = slot.t_size_; - info.t_slab_ = slab_id; - info.blob_off_ = blob_off; - info.blob_size_ = slot.t_size_; - if (rem_size < slot.t_size_) { - info.blob_size_ = rem_size; - } - rem_size -= info.blob_size_; - info.tid_ = tid; - blob_off += info.blob_size_; - mdm_->LocalUpdateTargetCapacity(tid, -static_cast(slab_size)); - --slab_count; - } - target_stat->lock_.Unlock(); -} - -/** - * Allocate a buffer of a particular size - * - * @param slab_id The size of slab to allocate - * @param tid The target to (ideally) allocate the slab from - * @param coin The buffer size information - * - * @return returns a BufferPool (BP) slot. The slot is NULL if the - * target didn't have enough remaining space. - * */ -BpSlot BufferPool::AllocateSlabSize(int cpu, - size_t slab_size, - BpFreeList *free_list, - BpFreeListStat *free_list_stat, - BpFreeListStat *target_stat) { - BpSlot slot(0, 0); - - // Case 1: Slab is cached on this core - if (free_list->size()) { - auto first = free_list->begin(); - slot = *first; - free_list->erase(first); - return slot; - } - - // Case 2: Allocate slab from stack - if (slot.IsNull() && target_stat->region_size_ >= slab_size) { - slot.t_off_ = target_stat->region_off_.fetch_add(slab_size); - slot.t_size_ = slab_size; - target_stat->region_size_.fetch_sub(slab_size); - return slot; - } - - // Case 3: Coalesce - // TOOD(llogan) - - // Case 4: No more space left in this target. - return slot; -} - -/** - * Determines a reasonable allocation of buffers based on the size of I/O. - * Returns the number of each slab size to allocate - * */ -std::vector BufferPool::CoinSelect(DeviceInfo &dev_info, - size_t total_size, - size_t &buffer_count) { - std::vector coins(dev_info.slab_sizes_->size()); - size_t rem_size = total_size; - buffer_count = 0; - - while (rem_size) { - // Find the slab size nearest to the rem_size - size_t i = 0, slab_size = 0; - for (size_t &tmp_slab_size : *dev_info.slab_sizes_) { - slab_size = tmp_slab_size; - if (slab_size >= rem_size) { - break; - } - ++i; - } - if (i == dev_info.slab_sizes_->size()) { i -= 1; } - - // Divide rem_size into slabs - if (rem_size > slab_size) { - coins[i].count_ += rem_size / slab_size; - coins[i].slab_size_ = slab_size; - rem_size %= slab_size; - } else { - coins[i].count_ += 1; - coins[i].slab_size_ = slab_size; - rem_size = 0; - } - buffer_count += coins[i].count_; - } - - return coins; -} - -/**==================================== - * Free Buffers - * ===================================*/ - -/** - * Free buffers from the BufferPool - * */ -bool BufferPool::LocalReleaseBuffers(std::vector &buffers) { - AUTO_TRACE(1) - HILOG(kDebug, "Releasing buffers") - int cpu = hshm::NodeThreadId().hash() % header_->concurrency_; - for (BufferInfo &info : buffers) { - // Acquire the main CPU lock for the target - BpFreeListStat *target_stat; - GetTargetStatForCpu(info.tid_.GetIndex(), cpu, target_stat); - hshm::ScopedMutex bpm_lock(target_stat->lock_, 12); - - // Get this core's free list for the page_size - BpFreeListStat *free_list_stat; - BpFreeList *free_list; - GetFreeListForCpu(info.tid_.GetIndex(), cpu, info.t_slab_, - free_list, free_list_stat); - free_list->emplace_front(info.t_off_, info.t_size_); - } - return true; -} - -/** - * Free buffers from the BufferPool (global) - * */ -bool BufferPool::GlobalReleaseBuffers(std::vector &buffers) { - AUTO_TRACE(1) - // Get the nodes to transfer buffers to - size_t total_size; - auto unique_nodes = GroupByNodeId(buffers, total_size); - - // Send the buffers to each node - for (auto &[node_id, size] : unique_nodes) { - if (NODE_ID_IS_LOCAL(node_id)) { - LocalReleaseBuffers(buffers); - } else { - rpc_->Call(node_id, "RpcReleaseBuffers", buffers); - } - } - - return true; -} - -/**==================================== - * Helper Methods - * ===================================*/ - -/** Get a free list reference */ -void BufferPool::GetFreeListForCpu(u16 target_id, int cpu, int slab_id, - BpFreeList* &free_list, - BpFreeListStat* &free_list_stat) { - size_t cpu_free_list_idx = header_->GetCpuFreeList(target_id, - cpu, slab_id); - if (cpu_free_list_idx >= target_allocs_->size()) { - HELOG(kFatal, "For some reason, the CPU free list was " - "not allocated properly and overflowed.") - } - BpFreeListPair &free_list_p = - (*target_allocs_)[cpu_free_list_idx]; - free_list_stat = &free_list_p.GetFirst(); - free_list = &free_list_p.GetSecond(); -} - -/** Get the stack allocator from the cpu */ -void BufferPool::GetTargetStatForCpu(u16 target_id, int cpu, - BpFreeListStat* &target_stat) { - size_t tgt_free_list_start = header_->GetCpuTargetStat(target_id, cpu); - BpFreeListPair &free_list_p = - (*target_allocs_)[tgt_free_list_start]; - target_stat = &free_list_p.GetFirst(); -} - -} // namespace hermes diff --git a/src/buffer_pool.h b/src/buffer_pool.h deleted file mode 100644 index 3f35e6132..000000000 --- a/src/buffer_pool.h +++ /dev/null @@ -1,321 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_BUFFER_POOL_H_ -#define HERMES_SRC_BUFFER_POOL_H_ - -#include "hermes_types.h" -#include "rpc.h" - -namespace hermes { - -class MetadataManager; -class BufferOrganizer; -class BufferPool; - -struct BpCoin { - size_t count_; - size_t slab_size_; - - BpCoin() : count_(0), slab_size_(0) {} -}; - -struct BpSlot { - size_t t_off_; /**< Offset of the buffer in the target */ - size_t t_size_; /**< Size of the buffer in the target*/ - - BpSlot() : t_size_(0) {} - - BpSlot(size_t t_off, size_t t_size) : t_off_(t_off), t_size_(t_size) {} - - bool IsNull() { - return t_size_ == 0; - } -}; - -struct BpFreeListStat { - std::atomic region_off_; /**< Current offset in the target */ - std::atomic region_size_; /**< Current space remaining in the tgt */ - size_t page_size_; /**< The size of page in this buffer list */ - size_t cur_count_; /**< Current number of pages allocated */ - size_t max_count_; /**< Max pages allocated at one time */ - Mutex lock_; /**< The modifier lock for this slot */ - - /** Default constructor */ - BpFreeListStat() = default; - - /** Copy constructor */ - BpFreeListStat(const BpFreeListStat &other) { - strong_copy(other); - } - - /** Copy assignment operator */ - BpFreeListStat& operator=(const BpFreeListStat &other) { - strong_copy(other); - return *this; - } - - /** Move constructor */ - BpFreeListStat(BpFreeListStat &&other) { - strong_copy(other); - } - - /** Move assignment operator */ - BpFreeListStat& operator=(BpFreeListStat &&other) { - strong_copy(other); - return *this; - } - - /** Internal copy */ - void strong_copy(const BpFreeListStat &other) { - region_off_ = other.region_off_.load(); - region_size_ = other.region_size_.load(); - page_size_ = other.page_size_; - cur_count_ = other.cur_count_; - max_count_ = other.max_count_; - } -}; - -/** Represents the list of available buffers */ -typedef hipc::slist BpFreeList; - -/** Represents a cache of buffer size in the target */ -typedef hipc::pair BpFreeListPair; - -/** Represents the set of targets */ -typedef hipc::vector BpTargetAllocs; - -/** - * The shared-memory representation of the BufferPool - * */ -struct BufferPoolShm { - hipc::ShmArchive free_lists_; - u16 ntargets_; - size_t concurrency_; - size_t nslabs_; - - /** - * Get the free list of the target - * This is where region_off_ & region_size_ in the BpFreeListStat are valid - * */ - size_t GetCpuTargetStat(u16 target, int cpu) { - return target * concurrency_ * nslabs_ + cpu * nslabs_; - } - - /** - * Get the start of the vector of the free list for the CPU in the target - * This is where page_size_, cur_count_, and max_count_ are valid. - * - * [target] [cpu] [slab_id] - * */ - size_t GetCpuFreeList(u16 target, int cpu, int slab_id) { - return target * concurrency_ * nslabs_ + cpu * nslabs_ + slab_id; - } -}; - -/** - * Responsible for managing the buffering space of all node-local targets. - * */ -class BufferPool { - private: - MetadataManager *mdm_; - BufferOrganizer *borg_; - RPC_TYPE *rpc_; - BufferPoolShm *header_; - /** Per-target allocator */ - BpTargetAllocs *target_allocs_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** Default constructor */ - BufferPool() = default; - - /**==================================== - * SHM Init - * ===================================*/ - - /** - * Initialize the BPM and its shared memory. - * REQUIRES mdm to be initialized already. - * */ - void shm_init(hipc::ShmArchive &header, - hipc::Allocator *alloc); - - /**==================================== - * SHM Deserialize - * ===================================*/ - - /** Deserialize the BPM from shared memory */ - void shm_deserialize(hipc::ShmArchive &header); - - /**==================================== - * Destructor - * ===================================*/ - - /** Destroy the BPM shared memory. */ - void shm_destroy_main(); - - /**==================================== - * Allocate Buffers - * ===================================*/ - - /** - * Allocate buffers from the targets according to the schema - * */ - RPC std::vector - LocalAllocateAndSetBuffers(PlacementSchema &schema, - const Blob &blob); - std::vector - GlobalAllocateAndSetBuffers(PlacementSchema &schema, - const Blob &blob); - - /** - * Determines a reasonable allocation of buffers based on the size of I/O. - * Returns the number of each slab size to allocate - * */ - std::vector CoinSelect(DeviceInfo &dev_info, - size_t total_size, - size_t &buffer_count); - - /** - * Allocate requested slabs from this target. - * If the target runs out of space, it will provision from the next target. - * We assume there is a baseline target, which can house an infinite amount - * of data. This would be the PFS in an HPC machine. - * - * @param total_size the total amount of data being placed in this target - * @param coins The requested number of slabs to allocate from this target - * @param tid The ID of the (ideal) target to allocate from - * @param cpu The CPU we are currently scheduled on - * @param blob_off [out] The current size of the blob which has been placed - * @param buffers [out] The buffers which were allocated - * */ - void AllocateBuffers(size_t total_size, - std::vector &coins, - TargetId tid, - int cpu, - size_t &blob_off, - std::vector &buffers); - - /** - * Allocate a set of slabs of particular size from this target. - * - * @param rem_size the amount of data remaining that needs to be allocated - * @param slab_size Size of the slabs to allocate - * @param slab_count The count of this slab size to allocate - * @param slab_id The offset of the slab in the device's slab list - * @param tid The target to allocate slabs from - * @param blob_off [out] The current size of the blob which has been placed - * @param buffers [out] The buffers which were allocated - * */ - void AllocateSlabs(size_t &rem_size, - size_t slab_size, - size_t &slab_count, - size_t slab_id, - TargetId tid, - int cpu, - size_t &blob_off, - std::vector &buffers); - - /** - * Allocate a buffer of a particular size - * - * @param slab_id The size of slab to allocate - * @param tid The target to (ideally) allocate the slab from - * @param coin The buffer size information - * - * @return returns a BufferPool (BP) slot. The slot is NULL if the - * target didn't have enough remaining space. - * */ - BpSlot AllocateSlabSize(int cpu, - size_t slab_size, - BpFreeList *free_list, - BpFreeListStat *stat, - BpFreeListStat *target_stat); - - - /**==================================== - * Free Buffers - * ===================================*/ - - /** - * Free buffers from the BufferPool - * */ - RPC bool LocalReleaseBuffers(std::vector &buffers); - bool GlobalReleaseBuffers(std::vector &buffers); - - /**==================================== - * Helper Methods - * ===================================*/ - - /** Get a free list reference */ - void GetFreeListForCpu(u16 target_id, int cpu, int slab_id, - BpFreeList* &free_list, - BpFreeListStat* &free_list_stat); - - /** Get the stack allocator from the cpu */ - void GetTargetStatForCpu(u16 target_id, int cpu, - BpFreeListStat* &target_stat); - - /** Find instance of unique target if it exists */ - static std::vector>::iterator - FindUniqueNodeId(std::vector> &unique_nodes, - i32 node_id) { - for (auto iter = unique_nodes.begin(); iter != unique_nodes.end(); ++iter) { - if (iter->first == node_id) { - return iter; - } - } - return unique_nodes.end(); - } - - /** Get the unique set of targets */ - static std::vector> - GroupByNodeId(std::vector &buffers, size_t &total_size) { - total_size = 0; - std::vector> unique_nodes; - for (BufferInfo &info : buffers) { - auto iter = FindUniqueNodeId(unique_nodes, info.tid_.GetNodeId()); - if (iter == unique_nodes.end()) { - unique_nodes.emplace_back(info.tid_.GetNodeId(), info.blob_size_); - } else { - (*iter).second += info.blob_size_; - } - total_size += info.blob_size_; - } - return unique_nodes; - } - - /** Get the unique set of targets */ - static std::vector> - GroupByNodeId(PlacementSchema &schema, size_t &total_size) { - total_size = 0; - std::vector> unique_nodes; - for (auto &plcmnt : schema.plcmnts_) { - auto iter = FindUniqueNodeId(unique_nodes, plcmnt.tid_.GetNodeId()); - if (iter == unique_nodes.end()) { - unique_nodes.emplace_back(plcmnt.tid_.GetNodeId(), plcmnt.size_); - } else { - (*iter).second += plcmnt.size_; - } - total_size += plcmnt.size_; - } - return unique_nodes; - } -}; - -} // namespace hermes - -#endif // HERMES_SRC_BUFFER_POOL_H_ diff --git a/src/config_client.cc b/src/config_client.cc index 1217aeaae..8f7d7dd26 100644 --- a/src/config_client.cc +++ b/src/config_client.cc @@ -10,96 +10,25 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "config_client.h" -#include "config_client_default.h" +#include "labstor/config/config_client.h" +#include "labstor/config/config_client_default.h" #include "hermes_shm/util/config_parse.h" #include namespace stdfs = std::filesystem; -namespace hermes::config { - -/** parse the AdapterConfig YAML node */ -void ClientConfig::ParseAdapterConfig(YAML::Node &yaml_conf, - AdapterObjectConfig &conf) { - std::string path = yaml_conf["path"].as(); - path = hshm::ConfigParse::ExpandPath(path); - path = stdfs::absolute(path).string(); - if (yaml_conf["mode"]) { - conf.mode_ = AdapterModeConv::to_enum(yaml_conf["mode"].as()); - } - if (yaml_conf["page_size"]) { - conf.page_size_ = hshm::ConfigParse::ParseSize( - yaml_conf["page_size"].as()); - } - SetAdapterConfig(path, conf); -} +namespace labstor::config { /** parse the YAML node */ void ClientConfig::ParseYAML(YAML::Node &yaml_conf) { - if (yaml_conf["stop_daemon"]) { - std::string stop_env = GetEnvSafe(kHermesStopDaemon); - if (stop_env.size() == 0) { - stop_daemon_ = yaml_conf["stop_daemon"].as(); - } else { - std::istringstream(stop_env) >> stop_daemon_; - } - } - if (yaml_conf["base_adapter_mode"]) { - std::string mode_env = GetEnvSafe(kHermesAdapterMode); - if (mode_env.size() == 0) { - base_adapter_config_.mode_ = AdapterModeConv::to_enum( - yaml_conf["base_adapter_mode"].as()); - } else { - base_adapter_config_.mode_ = AdapterModeConv::to_enum(mode_env); - } - } - if (yaml_conf["file_page_size"]) { - std::string page_size_env = GetEnvSafe(kHermesPageSize); - if (page_size_env.size() == 0) { - base_adapter_config_.page_size_ = - hshm::ConfigParse::ParseSize( - yaml_conf["file_page_size"].as()); - } else { - base_adapter_config_.page_size_ = - hshm::ConfigParse::ParseSize(page_size_env); - } - } - if (yaml_conf["path_inclusions"]) { - std::vector inclusions; - ParseVector(yaml_conf["path_inclusions"], inclusions); - for (auto &entry : inclusions) { - entry = hshm::ConfigParse::ExpandPath(entry); - SetAdapterPathTracking(std::move(entry), true); - } - } - if (yaml_conf["path_exclusions"]) { - std::vector exclusions; - ParseVector(yaml_conf["path_exclusions"], exclusions); - for (auto &entry : exclusions) { - entry = hshm::ConfigParse::ExpandPath(entry); - SetAdapterPathTracking(std::move(entry), false); - } - } - if (yaml_conf["flushing_mode"]) { - flushing_mode_ = - FlushingModeConv::GetEnum(yaml_conf["flushing_mode"].as()); - auto flush_mode_env = getenv("HERMES_FLUSH_MODE"); - if (flush_mode_env) { - flushing_mode_ = FlushingModeConv::GetEnum(flush_mode_env); - } - } - if (yaml_conf["file_adapter_configs"]) { - for (auto node : yaml_conf["file_adapter_configs"]) { - AdapterObjectConfig conf(base_adapter_config_); - ParseAdapterConfig(node, conf); - } + if (yaml_conf["thread_model"]) { + thread_model_ = yaml_conf["thread_model"].as(); } } /** Load the default configuration */ void ClientConfig::LoadDefault() { - LoadText(kClientDefaultConfigStr, false); + LoadText(kLabstorClientDefaultConfigStr, false); } -} // namespace hermes::config +} // namespace labstor::config diff --git a/src/config_server.cc b/src/config_server.cc index 59d1abb5c..df7d282c5 100644 --- a/src/config_server.cc +++ b/src/config_server.cc @@ -14,61 +14,55 @@ #include #include #include "hermes_shm/util/logging.h" -#include "utils.h" -#include "config.h" #include "hermes_shm/util/config_parse.h" +#include "labstor/config/config.h" +#include "labstor/config/config_server.h" +#include "labstor/config/config_server_default.h" -#include "config_server.h" -#include "config_server_default.h" +namespace labstor::config { -namespace hermes::config { +/** parse work orchestrator info from YAML config */ +void ServerConfig::ParseWorkOrchestrator(YAML::Node yaml_conf) { + if (yaml_conf["max_workers"]) { + wo_.max_workers_ = yaml_conf["max_workers"].as(); + } +} -/** parse device information from YAML config */ -void ServerConfig::ParseDeviceInfo(YAML::Node yaml_conf) { - devices_ = hipc::make_uptr>(); - for (auto device : yaml_conf) { - devices_->emplace_back(); - DeviceInfo &dev = devices_->back(); - auto dev_info = device.second; - (*dev.dev_name_) = device.first.as(); - (*dev.mount_dir_) = hshm::ConfigParse::ExpandPath( - dev_info["mount_point"].as()); - dev.borg_min_thresh_ = - dev_info["borg_capacity_thresh"][0].as(); - dev.borg_max_thresh_ = - dev_info["borg_capacity_thresh"][1].as(); - dev.is_shared_ = - dev_info["is_shared_device"].as(); - dev.block_size_ = - hshm::ConfigParse::ParseSize(dev_info["block_size"].as()); - dev.capacity_ = - hshm::ConfigParse::ParseSize(dev_info["capacity"].as()); - dev.bandwidth_ = - hshm::ConfigParse::ParseSize(dev_info["bandwidth"].as()); - dev.latency_ = - hshm::ConfigParse::ParseLatency(dev_info["latency"].as()); - std::vector size_vec; - ParseVector>( - dev_info["slab_sizes"], size_vec); - dev.slab_sizes_->reserve(size_vec.size()); - for (const std::string &size_str : size_vec) { - dev.slab_sizes_->emplace_back(hshm::ConfigParse::ParseSize(size_str)); - } +/** parse work orchestrator info from YAML config */ +void ServerConfig::ParseQueueManager(YAML::Node yaml_conf) { + if (yaml_conf["queue_depth"]) { + queue_manager_.queue_depth_ = yaml_conf["queue_depth"].as(); + } + if (yaml_conf["max_lanes"]) { + queue_manager_.max_lanes_ = yaml_conf["max_lanes"].as(); + } + if (yaml_conf["max_queues"]) { + queue_manager_.max_queues_ = yaml_conf["max_queues"].as(); + } + if (yaml_conf["shm_allocator"]) { + queue_manager_.shm_allocator_ = yaml_conf["shm_allocator"].as(); + } + if (yaml_conf["shm_name"]) { + queue_manager_.shm_name_ = yaml_conf["shm_name"].as(); + } + if (yaml_conf["shm_size"]) { + queue_manager_.shm_size_ = hshm::ConfigParse::ParseSize( + yaml_conf["shm_size"].as()); } } /** parse RPC information from YAML config */ void ServerConfig::ParseRpcInfo(YAML::Node yaml_conf) { std::string suffix; - + rpc_.host_names_.clear(); if (yaml_conf["host_file"]) { rpc_.host_file_ = hshm::ConfigParse::ExpandPath(yaml_conf["host_file"].as()); - rpc_.host_names_.clear(); + if (rpc_.host_file_.size() > 0) { + rpc_.host_names_ = hshm::ConfigParse::ParseHostfile(rpc_.host_file_); + } } - if (yaml_conf["host_names"] && rpc_.host_file_.size() == 0) { - // NOTE(llogan): host file is prioritized - rpc_.host_names_.clear(); + if (yaml_conf["host_names"] && rpc_.host_names_.size() == 0) { for (YAML::Node host_name_gen : yaml_conf["host_names"]) { std::string host_names = host_name_gen.as(); hshm::ConfigParse::ParseHostNameString(host_names, rpc_.host_names_); @@ -88,134 +82,25 @@ void ServerConfig::ParseRpcInfo(YAML::Node yaml_conf) { } } -/** parse dpe information from YAML config */ -void ServerConfig::ParseDpeInfo(YAML::Node yaml_conf) { - if (yaml_conf["default_placement_policy"]) { - std::string policy = - yaml_conf["default_placement_policy"].as(); - dpe_.default_policy_ = api::PlacementPolicyConv::to_enum(policy); - } -} - -/** parse buffer organizer information from YAML config */ -void ServerConfig::ParseBorgInfo(YAML::Node yaml_conf) { - if (yaml_conf["num_threads"]) { - borg_.num_threads_ = yaml_conf["num_threads"].as(); - } - if (yaml_conf["flush_period"]) { - borg_.flush_period_ = yaml_conf["flush_period"].as(); - } - if (yaml_conf["blob_reorg_period"]) { - borg_.blob_reorg_period_ = yaml_conf["blob_reorg_period"].as(); - } - if (yaml_conf["recency_min"]) { - borg_.recency_min_ = yaml_conf["recency_min"].as(); - } - if (yaml_conf["recency_max"]) { - borg_.recency_max_ = yaml_conf["recency_max"].as(); - } - if (yaml_conf["freq_max"]) { - borg_.freq_max_ = yaml_conf["freq_max"].as(); - } - if (yaml_conf["freq_min"]) { - borg_.freq_min_ = yaml_conf["freq_min"].as(); - } -} - -/** parse I/O tracing information from YAML config */ -void ServerConfig::ParseTracingInfo(YAML::Node yaml_conf) { - if (yaml_conf["enabled"]) { - tracing_.enabled_ = yaml_conf["enabled"].as(); - } - if (yaml_conf["output"]) { - tracing_.output_ = hshm::ConfigParse::ExpandPath( - yaml_conf["output"].as()); - } -} - -/** parse prefetch information from YAML config */ -void ServerConfig::ParsePrefetchInfo(YAML::Node yaml_conf) { - if (yaml_conf["enabled"]) { - prefetcher_.enabled_ = yaml_conf["enabled"].as(); - } - if (yaml_conf["io_trace_path"]) { - prefetcher_.trace_path_ = hshm::ConfigParse::ExpandPath( - yaml_conf["io_trace_path"].as()); - } - if (yaml_conf["epoch_ms"]) { - prefetcher_.epoch_ms_ = yaml_conf["epoch_ms"].as(); - } - if (yaml_conf["is_mpi"]) { - prefetcher_.is_mpi_ = yaml_conf["is_mpi"].as(); - } - if (yaml_conf["apriori_schema_path"]) { - prefetcher_.apriori_schema_path_ = - yaml_conf["apriori_schema_path"].as(); - } -} - -/** parse prefetch information from YAML config */ -void ServerConfig::ParseTraitInfo(YAML::Node yaml_conf) { - std::vector trait_names; - ParseVector>( - yaml_conf, trait_names); - trait_paths_.reserve(trait_names.size()); - for (auto &name : trait_names) { - name = hshm::ConfigParse::ExpandPath(name); - trait_paths_.emplace_back( - hshm::Formatter::format("lib{}.so", name)); - } -} - -/** parse prefetch information from YAML config */ -void ServerConfig::ParseMdmInfo(YAML::Node yaml_conf) { - mdm_.num_blobs_ = yaml_conf["est_blob_count"].as(); - mdm_.num_bkts_ = yaml_conf["est_blob_count"].as(); - mdm_.num_traits_ = yaml_conf["est_num_traits"].as(); -} - /** parse the YAML node */ void ServerConfig::ParseYAML(YAML::Node &yaml_conf) { - if (yaml_conf["devices"]) { - ParseDeviceInfo(yaml_conf["devices"]); + if (yaml_conf["work_orchestrator"]) { + ParseWorkOrchestrator(yaml_conf["work_orchestrator"]); + } + if (yaml_conf["queue_manager"]) { + ParseQueueManager(yaml_conf["queue_manager"]); } if (yaml_conf["rpc"]) { ParseRpcInfo(yaml_conf["rpc"]); } - if (yaml_conf["dpe"]) { - ParseDpeInfo(yaml_conf["dpe"]); - } - if (yaml_conf["buffer_organizer"]) { - ParseBorgInfo(yaml_conf["buffer_organizer"]); - } - if (yaml_conf["tracing"]) { - ParseTracingInfo(yaml_conf["tracing"]); - } - if (yaml_conf["prefetch"]) { - ParsePrefetchInfo(yaml_conf["prefetch"]); - } - if (yaml_conf["mdm"]) { - ParseMdmInfo(yaml_conf["mdm"]); - } - if (yaml_conf["system_view_state_update_interval_ms"]) { - system_view_state_update_interval_ms = - yaml_conf["system_view_state_update_interval_ms"].as(); - } - if (yaml_conf["traits"]) { - ParseTraitInfo(yaml_conf["traits"]); - } - if (yaml_conf["shmem_name"]) { - shmem_name_ = yaml_conf["shmem_name"].as(); - } - if (yaml_conf["max_memory"]) { - max_memory_ = hshm::ConfigParse::ParseSize( - yaml_conf["max_memory"].as()); + if (yaml_conf["task_registry"]) { + ParseVector(yaml_conf["task_registry"], task_libs_); } } /** Load the default configuration */ void ServerConfig::LoadDefault() { - LoadText(kServerDefaultConfigStr, false); + LoadText(kLabstorServerDefaultConfigStr, false); } -} // namespace hermes::config +} // namespace labstor::config diff --git a/src/config_server.h b/src/config_server.h deleted file mode 100644 index 4f2b461a5..000000000 --- a/src/config_server.h +++ /dev/null @@ -1,325 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_CONFIG_SERVER_H_ -#define HERMES_SRC_CONFIG_SERVER_H_ - -#include "config.h" - -namespace hermes::config { - -struct DeviceInfo; /** Forward declaration of DeviceInfo */ - -/** - * The type of interface the device exposes - * */ -enum class IoInterface { - kRam, - kPosix -}; - -/** - * DeviceInfo shared-memory representation - * */ -class DeviceInfo : public hipc::ShmContainer { - SHM_CONTAINER_TEMPLATE(DeviceInfo, DeviceInfo) - - /** The human-readable name of the device */ - hipc::ShmArchive dev_name_; - /** The unit of each slab, a multiple of the Device's block size */ - hipc::ShmArchive> slab_sizes_; - /** The directory the device is mounted on */ - hipc::ShmArchive mount_dir_; - /** The file to create on the device */ - hipc::ShmArchive mount_point_; - /** The I/O interface for the device */ - IoInterface io_api_; - /** The minimum transfer size of each device */ - size_t block_size_; - /** Device capacity (bytes) */ - size_t capacity_; - /** Bandwidth of a device (MBps) */ - f32 bandwidth_; - /** Latency of the device (us)*/ - f32 latency_; - /** Whether or not the device is shared among all nodes */ - bool is_shared_; - /** BORG's minimum and maximum capacity threshold for device */ - f32 borg_min_thresh_, borg_max_thresh_; - - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM Constructor. Default. */ - explicit DeviceInfo(hipc::Allocator *alloc) { - shm_init_container(alloc); - HSHM_MAKE_AR0(dev_name_, alloc); - HSHM_MAKE_AR0(slab_sizes_, alloc); - HSHM_MAKE_AR0(mount_dir_, alloc); - HSHM_MAKE_AR0(mount_point_, alloc); - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor. From DeviceInfo. */ - explicit DeviceInfo(hipc::Allocator *alloc, - const DeviceInfo &other) { - shm_init_container(alloc); - shm_strong_copy_constructor_main(alloc, other); - } - - /** Internal copy */ - void strong_copy(const DeviceInfo &other) { - io_api_ = other.io_api_; - block_size_ = other.block_size_; - capacity_ = other.capacity_; - bandwidth_ = other.bandwidth_; - latency_ = other.latency_; - is_shared_ = other.is_shared_; - borg_min_thresh_ = other.borg_min_thresh_; - borg_max_thresh_ = other.borg_max_thresh_; - } - - /** Copy constructor main. */ - void shm_strong_copy_constructor_main(hipc::Allocator *alloc, - const DeviceInfo &other) { - strong_copy(other); - HSHM_MAKE_AR(dev_name_, alloc, *other.dev_name_); - HSHM_MAKE_AR(slab_sizes_, alloc, *other.slab_sizes_); - HSHM_MAKE_AR(mount_dir_, alloc, *other.mount_dir_); - HSHM_MAKE_AR(mount_point_, alloc, *other.mount_point_); - } - - /** SHM copy assignment operator. From DeviceInfo. */ - DeviceInfo& operator=(const DeviceInfo &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_op_main(other); - } - return *this; - } - - /** Copy assignment operator main. */ - void shm_strong_copy_op_main(const DeviceInfo &other) { - strong_copy(other); - (*dev_name_) = (*other.dev_name_); - (*slab_sizes_) = (*other.slab_sizes_); - (*mount_dir_) = (*other.mount_dir_); - (*mount_point_) = (*other.mount_point_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - DeviceInfo(hipc::Allocator *alloc, - DeviceInfo &&other) { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - HSHM_MAKE_AR(dev_name_, alloc, std::move(*other.dev_name_)); - HSHM_MAKE_AR(slab_sizes_, alloc, std::move(*other.slab_sizes_)); - HSHM_MAKE_AR(mount_dir_, alloc, std::move(*other.mount_dir_)); - HSHM_MAKE_AR(mount_point_, alloc, std::move(*other.mount_point_)); - other.SetNull(); - } else { - shm_strong_copy_constructor_main(alloc, other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - DeviceInfo& operator=(DeviceInfo &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - (*dev_name_) = std::move(*other.dev_name_); - (*slab_sizes_) = std::move(*other.slab_sizes_); - (*mount_dir_) = std::move(*other.mount_dir_); - (*mount_point_) = std::move(*other.mount_point_); - other.SetNull(); - } else { - shm_strong_copy_op_main(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Whether DeviceInfo is NULL */ - bool IsNull() { return false; } - - /** Set DeviceInfo to NULL */ - void SetNull() {} - - /** Free shared memory */ - void shm_destroy_main() { - (*dev_name_).shm_destroy(); - (*slab_sizes_).shm_destroy(); - (*mount_dir_).shm_destroy(); - (*mount_point_).shm_destroy(); - } -}; - -/** - * RPC information defined in server config - * */ -struct RpcInfo { - /** The name of a file that contains host names, 1 per line */ - std::string host_file_; - /** The parsed hostnames from the hermes conf */ - std::vector host_names_; - /** The RPC protocol to be used. */ - std::string protocol_; - /** The RPC domain name for verbs transport. */ - std::string domain_; - /** The RPC port number. */ - int port_; - /** The number of handler threads per RPC server. */ - int num_threads_; -}; - -/** - * DPE information defined in server config - * */ -struct DpeInfo { - /** The default blob placement policy. */ - api::PlacementPolicy default_policy_; - - /** Whether blob splitting is enabled for Round-Robin blob placement. */ - bool default_rr_split_; -}; - -/** - * Buffer organizer information defined in server config - * */ -struct BorgInfo { - /** The number of buffer organizer threads. */ - int num_threads_; - /** Interval (seconds) where blobs are checked for flushing */ - size_t flush_period_; - /** Interval (seconds) where blobs are checked for re-organization */ - size_t blob_reorg_period_; - /** Time when score is equal to 1 (seconds) */ - float recency_min_; - /** Time when score is equal to 0 (seconds) */ - float recency_max_; - /** Number of accesses for score to be equal to 1 (count) */ - float freq_max_; - /** Number of accesses for score to be equal to 0 (count) */ - float freq_min_; -}; - -/** - * Prefetcher information in server config - * */ -struct PrefetchInfo { - bool enabled_; - std::string trace_path_; - std::string apriori_schema_path_; - size_t epoch_ms_; - bool is_mpi_; -}; - -/** - * Tracing information in server config - * */ -struct TracingInfo { - bool enabled_; - std::string output_; -}; - -/** MDM information */ -struct MdmInfo { - /** Number of buckets in mdm bucket map before collisions */ - size_t num_bkts_; - - /** Number of blobs in mdm blob map before collisions */ - size_t num_blobs_; - - /** Number of traits in mdm trait map before collisions */ - size_t num_traits_; -}; - -/** - * System configuration for Hermes - */ -class ServerConfig : public BaseConfig { - public: - /** The device information */ - hipc::uptr> devices_; - - /** The RPC information */ - RpcInfo rpc_; - - /** The DPE information */ - DpeInfo dpe_; - - /** Buffer organizer (BORG) information */ - BorgInfo borg_; - - /** Tracing information */ - TracingInfo tracing_; - - /** Prefetcher information */ - PrefetchInfo prefetcher_; - - /** Metadata Manager information */ - MdmInfo mdm_; - - /** Trait repo information */ - std::vector trait_paths_; - - /** The length of a view state epoch */ - u32 system_view_state_update_interval_ms; - - /** The max amount of memory hermes uses for non-buffering tasks */ - size_t max_memory_; - - /** A base name for the BufferPool shared memory segement. Hermes appends the - * value of the USER environment variable to this string. - */ - std::string shmem_name_; - - public: - ServerConfig() = default; - void LoadDefault(); - - private: - void ParseYAML(YAML::Node &yaml_conf); - void CheckConstraints(); - void ParseRpcInfo(YAML::Node yaml_conf); - void ParseDeviceInfo(YAML::Node yaml_conf); - void ParseDpeInfo(YAML::Node yaml_conf); - void ParseBorgInfo(YAML::Node yaml_conf); - void ParsePrefetchInfo(YAML::Node yaml_conf); - void ParseTracingInfo(YAML::Node yaml_conf); - void ParseTraitInfo(YAML::Node yaml_conf); - void ParseMdmInfo(YAML::Node yaml_conf); -}; - -} // namespace hermes::config - -namespace hermes { -using config::ServerConfig; -} // namespace hermes - -#endif // HERMES_SRC_CONFIG_SERVER_H_ diff --git a/src/data_placement_engine.cc b/src/data_placement_engine.cc deleted file mode 100644 index bdcf35906..000000000 --- a/src/data_placement_engine.cc +++ /dev/null @@ -1,75 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "data_placement_engine_factory.h" - -#include -#include - -#include -#include -#include - -#include "hermes.h" -#include "hermes_types.h" -#include "buffer_pool.h" - -namespace hermes { - -/** Constructor. */ -DPE::DPE() : mdm_(&HERMES->mdm_) {} - -/** calculate data placement */ -Status DPE::CalculatePlacement(const std::vector &blob_sizes, - std::vector &output, - api::Context &ctx) { - AUTO_TRACE(1) - Status result; - - // NOTE(chogan): Start with local targets and gradually expand the target - // list until the placement succeeds. - for (int i = 0; i < static_cast(TopologyType::kCount); ++i) { - // Reset the output schema - output.clear(); - // Get the capacity/bandwidth of targets - std::vector targets; - switch (static_cast(i)) { - case TopologyType::Local: { - targets = mdm_->LocalGetTargetInfo(); - break; - } - case TopologyType::Neighborhood: { - // targets = mdm_->GetNeighborhoodTargetInfo(); - break; - } - case TopologyType::Global: { - // targets = mdm_->GetGlobalTargetInfo(); - break; - } - case TopologyType::kCount: { - HELOG(kFatal, "Not a valid topology type") - } - } - if (targets.size() == 0) { - result = DPE_PLACEMENT_SCHEMA_EMPTY; - continue; - } - // Calculate a placement schema - result = Placement(blob_sizes, targets, ctx, output); - if (!result.Fail()) { - break; - } - } - return result; -} - -} // namespace hermes diff --git a/src/data_structures.h b/src/data_structures.h deleted file mode 100644 index d2380ccc5..000000000 --- a/src/data_structures.h +++ /dev/null @@ -1,43 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_DATA_STRUCTURES_H_ -#define HERMES_SRC_DATA_STRUCTURES_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using hshm::RwLock; -using hshm::Mutex; -using hshm::bitfield32_t; -using hshm::ScopedRwReadLock; -using hshm::ScopedRwWriteLock; - -#include -#include -#include -#include -#include -#include - -#endif // HERMES_SRC_DATA_STRUCTURES_H_ diff --git a/src/decorator.h b/src/decorator.h deleted file mode 100644 index 4d5016e82..000000000 --- a/src/decorator.h +++ /dev/null @@ -1,38 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_RPC_GENERATOR_DECORATOR_H_ -#define HERMES_SRC_RPC_GENERATOR_DECORATOR_H_ - -/** - * Decorators for RPC code generation - * - * code_generators/rpc.py - * */ -#define RPC -#define RPC_AUTOGEN_START -#define RPC_AUTOGEN_END -#define RPC_CLASS_INSTANCE_DEFS_START -#define RPC_CLASS_INSTANCE_DEFS_END - -/** - * In many cases, we define functions which have a large parameter - * set and then produce wrapper functions which set those parameters - * differently. These decorators automate generating those wrapper - * functions - * */ - -#define WRAP(...) -#define WRAP_AUTOGEN_START -#define WRAP_AUTOGEN_END - -#endif // HERMES_SRC_RPC_GENERATOR_DECORATOR_H_ diff --git a/src/dpe/minimize_io_time.cc b/src/dpe/minimize_io_time.cc deleted file mode 100644 index 6f58d2acd..000000000 --- a/src/dpe/minimize_io_time.cc +++ /dev/null @@ -1,74 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "minimize_io_time.h" -#include -#include -#include - -namespace hermes { - -Status MinimizeIoTime::Placement(const std::vector &blob_sizes, - std::vector &targets, - api::Context &ctx, - std::vector &output) { - for (size_t blob_size : blob_sizes) { - // Initialize blob's size, score, and schema - size_t rem_blob_size = blob_size; - float score = ctx.blob_score_; - if (ctx.blob_score_ == -1) { - score = 1; - } - output.emplace_back(); - PlacementSchema &blob_schema = output.back(); - - for (TargetInfo &target : targets) { - // NOTE(llogan): We skip targets that are too high of priority or - // targets that can't fit the ENTIRE blob - if (target.score_ > score || - target.rem_cap_ < blob_size) { - // TODO(llogan): add other considerations of this DPE - continue; - } - if (ctx.blob_score_ == -1) { - ctx.blob_score_ = target.score_; - } - - // NOTE(llogan): we assume the TargetInfo list is sorted - if (target.rem_cap_ >= rem_blob_size) { - blob_schema.plcmnts_.emplace_back(rem_blob_size, - target.id_); - target.rem_cap_ -= rem_blob_size; - rem_blob_size = 0; - } else { - // NOTE(llogan): this code technically never gets called, - // but it might in the future - blob_schema.plcmnts_.emplace_back(target.rem_cap_, - target.id_); - rem_blob_size -= target.rem_cap_; - target.rem_cap_ = 0; - } - - if (rem_blob_size == 0) { - break; - } - } - - if (rem_blob_size > 0) { - return DPE_MIN_IO_TIME_NO_SOLUTION; - } - } - - return Status(); -} - -} // namespace hermes diff --git a/src/dpe/minimize_io_time.h b/src/dpe/minimize_io_time.h deleted file mode 100644 index 94fe957e6..000000000 --- a/src/dpe/minimize_io_time.h +++ /dev/null @@ -1,34 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_DPE_MINIMIZE_IO_TIME_H_ -#define HERMES_SRC_DPE_MINIMIZE_IO_TIME_H_ - -#include "data_placement_engine.h" - -namespace hermes { -/** - A class to represent data placement engine that minimizes I/O time. -*/ -class MinimizeIoTime : public DPE { - public: - MinimizeIoTime() = default; - ~MinimizeIoTime() = default; - Status Placement(const std::vector &blob_sizes, - std::vector &targets, - api::Context &ctx, - std::vector &output); -}; - -} // namespace hermes - -#endif // HERMES_SRC_DPE_MINIMIZE_IO_TIME_H_ diff --git a/src/dpe/random.cc b/src/dpe/random.cc deleted file mode 100644 index cb51e637f..000000000 --- a/src/dpe/random.cc +++ /dev/null @@ -1,60 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "random.h" - -#include -#include -#include - -namespace hermes { - -Status Random::Placement(const std::vector &blob_sizes, - std::vector &targets, - api::Context &ctx, - std::vector &output) { - for (size_t blob_size : blob_sizes) { - // Initialize blob's size, score, and schema - size_t rem_blob_size = blob_size; - output.emplace_back(); - PlacementSchema &blob_schema = output.back(); - - // Choose RR target and iterate - size_t target_id = std::rand() % targets.size(); - for (size_t i = 0; i < targets.size(); ++i) { - TargetInfo &target = targets[(target_id + i) % targets.size()]; - - // NOTE(llogan): We skip targets that can't fit the ENTIRE blob - if (target.rem_cap_ < blob_size) { - continue; - } - if (ctx.blob_score_ == -1) { - ctx.blob_score_ = target.score_; - } - - // Place the blob on this target - blob_schema.plcmnts_.emplace_back(rem_blob_size, - target.id_); - target.rem_cap_ -= rem_blob_size; - rem_blob_size = 0; - break; - } - - if (rem_blob_size > 0) { - return DPE_MIN_IO_TIME_NO_SOLUTION; - } - } - - return Status(); -} - -} // namespace hermes diff --git a/src/dpe/round_robin.cc b/src/dpe/round_robin.cc deleted file mode 100644 index ed96b2a64..000000000 --- a/src/dpe/round_robin.cc +++ /dev/null @@ -1,57 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "round_robin.h" -#include - -namespace hermes { - -Status RoundRobin::Placement(const std::vector &blob_sizes, - std::vector &targets, - api::Context &ctx, - std::vector &output) { - for (size_t blob_size : blob_sizes) { - // Initialize blob's size, score, and schema - size_t rem_blob_size = blob_size; - output.emplace_back(); - PlacementSchema &blob_schema = output.back(); - - // Choose RR target and iterate - size_t target_id = counter_.fetch_add(1) % targets.size(); - for (size_t i = 0; i < targets.size(); ++i) { - TargetInfo &target = targets[(target_id + i) % targets.size()]; - - // NOTE(llogan): We skip targets that can't fit the ENTIRE blob - if (target.rem_cap_ < blob_size) { - continue; - } - if (ctx.blob_score_ == -1) { - ctx.blob_score_ = target.score_; - } - - // Place the blob on this target - blob_schema.plcmnts_.emplace_back(rem_blob_size, - target.id_); - target.rem_cap_ -= rem_blob_size; - rem_blob_size = 0; - break; - } - - if (rem_blob_size > 0) { - return DPE_MIN_IO_TIME_NO_SOLUTION; - } - } - - return Status(); -} - -} // namespace hermes diff --git a/src/hermes_types.cc b/src/hermes_types.cc deleted file mode 100644 index 4a17ce528..000000000 --- a/src/hermes_types.cc +++ /dev/null @@ -1,49 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_types.h" -#include "hermes.h" - -namespace hermes { -/** Identifier of the Hermes allocator */ -const hipc::allocator_id_t main_alloc_id(0, 1); - -/** Hermes server environment variable */ -const char* kHermesServerConf = "HERMES_CONF"; - -/** Hermes client environment variable */ -const char* kHermesClientConf = "HERMES_CLIENT_CONF"; - -/** Hermes adapter mode environment variable */ -const char* kHermesAdapterMode = "HERMES_ADAPTER_MODE"; - -/** Filesystem page size environment variable */ -const char* kHermesPageSize = "HERMES_PAGE_SIZE"; - -/** Stop daemon environment variable */ -const char* kHermesStopDaemon = "HERMES_STOP_DAEMON"; - -/** Maximum path length environment variable */ -const size_t kMaxPathLength = 4096; -} // namespace hermes - -namespace hermes::api { - -/** Default constructor */ -Context::Context() - : policy(HERMES->server_config_.dpe_.default_policy_), - rr_split(HERMES->server_config_.dpe_.default_rr_split_), - rr_retry(false), - disable_swap(false), - blob_score_(-1) {} - -} // namespace hermes::api diff --git a/src/hermes_types.h b/src/hermes_types.h deleted file mode 100644 index caeefd077..000000000 --- a/src/hermes_types.h +++ /dev/null @@ -1,567 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TYPES_H_ -#define HERMES_TYPES_H_ - -#include "hermes_shm/util/logging.h" -#include "hermes_shm/constants/macros.h" -#include - -#include -#include -#include -#include - -#include "data_structures.h" - -/** - * \file hermes_types.h - * Types used in Hermes. - */ - -/** - * \namespace hermes - */ -namespace hermes { - -typedef uint8_t u8; /**< 8-bit unsigned integer */ -typedef uint16_t u16; /**< 16-bit unsigned integer */ -typedef uint32_t u32; /**< 32-bit unsigned integer */ -typedef uint64_t u64; /**< 64-bit unsigned integer */ -typedef int8_t i8; /**< 8-bit signed integer */ -typedef int16_t i16; /**< 16-bit signed integer */ -typedef int32_t i32; /**< 32-bit signed integer */ -typedef int64_t i64; /**< 64-bit signed integer */ -typedef float f32; /**< 32-bit float */ -typedef double f64; /**< 64-bit float */ - -/** Identifier of the Hermes allocator */ -extern const hipc::allocator_id_t main_alloc_id; - -/** Hermes server environment variable */ -extern const char* kHermesServerConf; - -/** Hermes client environment variable */ -extern const char* kHermesClientConf; - -/** Hermes adapter mode environment variable */ -extern const char* kHermesAdapterMode; - -/** Filesystem page size environment variable */ -extern const char* kHermesPageSize; - -/** Stop daemon environment variable */ -extern const char* kHermesStopDaemon; - -/** Maximum path length environment variable */ -extern const size_t kMaxPathLength; - -/** The mode Hermes is launched in */ -enum class HermesType { - kNone, - kServer, - kClient -}; - -/** The flushing mode */ -enum class FlushingMode { - kSync, - kAsync -}; - -/** Convert flushing modes to strings */ -class FlushingModeConv { - public: - static FlushingMode GetEnum(const std::string &str) { - if (str == "kSync") { - return FlushingMode::kSync; - } - if (str == "kAsync") { - return FlushingMode::kAsync; - } - return FlushingMode::kAsync; - } -}; - -/** The types of I/O that can be performed (for IoCall RPC) */ -enum class IoType { - kRead, - kWrite, - kNone -}; - -typedef u16 DeviceID; /**< device id in unsigned 16-bit integer */ - -/** The types of topologies */ -enum class TopologyType { - Local, - Neighborhood, - Global, - - kCount -}; - -/** Represents unique ID for BlobId and TagId */ -template -struct UniqueId { - u64 unique_; /**< A unique id for the blob */ - i32 node_id_; /**< The node the content is on */ - - /** Default constructor */ - UniqueId() = default; - - /** Emplace constructor */ - UniqueId(u64 unique, i32 node_id) : unique_(unique), node_id_(node_id) {} - - /** Copy constructor */ - UniqueId(const UniqueId &other) { - unique_ = other.unique_; - node_id_ = other.node_id_; - } - - /** Copy assignment */ - UniqueId& operator=(const UniqueId &other) { - if (this != &other) { - unique_ = other.unique_; - node_id_ = other.node_id_; - } - return *this; - } - - /** Move constructor */ - UniqueId(UniqueId &&other) { - unique_ = other.unique_; - node_id_ = other.node_id_; - } - - /** Move assignment */ - UniqueId& operator=(UniqueId &&other) { - if (this != &other) { - unique_ = other.unique_; - node_id_ = other.node_id_; - } - return *this; - } - - /** Check if null */ - bool IsNull() const { return unique_ == 0; } - - /** Get null id */ - static inline UniqueId GetNull() { - static const UniqueId id(0, 0); - return id; - } - - /** Set to null id */ - void SetNull() { - node_id_ = 0; - unique_ = 0; - } - - /** Get id of node from this id */ - i32 GetNodeId() const { return node_id_; } - - /** Compare two ids for equality */ - bool operator==(const UniqueId &other) const { - return unique_ == other.unique_ && node_id_ == other.node_id_; - } - - /** Compare two ids for inequality */ - bool operator!=(const UniqueId &other) const { - return unique_ != other.unique_ || node_id_ != other.node_id_; - } - - /** Serialize a UniqueId - template - void serialize(Archive &ar) { - ar(unique_, node_id_); - }*/ -}; -typedef UniqueId<1> BlobId; -typedef UniqueId<2> TagId; -typedef UniqueId<3> TraitId; - -/** Allow unique ids to be printed as strings */ -template -std::ostream &operator<<(std::ostream &os, UniqueId const &obj) { - return os << (std::to_string(obj.node_id_) + "." - + std::to_string(obj.unique_)); -} - -/** Indicates a PUT or GET for a particular blob */ -struct IoStat { - IoType type_; - BlobId blob_id_; - TagId tag_id_; - size_t blob_size_; - int rank_; - - /** Default constructor */ - IoStat() = default; - - /** Copy constructor */ - IoStat(const IoStat &other) { - Copy(other); - } - - /** Copy assignment */ - IoStat& operator=(const IoStat &other) { - if (this != &other) { - Copy(other); - } - return *this; - } - - /** Move constructor */ - IoStat(IoStat &&other) { - Copy(other); - } - - /** Move assignment */ - IoStat& operator=(IoStat &&other) { - if (this != &other) { - Copy(other); - } - return *this; - } - - /** Generic copy / move */ - HSHM_ALWAYS_INLINE void Copy(const IoStat &other) { - type_ = other.type_; - blob_id_ = other.blob_id_; - tag_id_ = other.tag_id_; - blob_size_ = other.blob_size_; - rank_ = other.rank_; - } - - /** Serialize */ - template - void save(Archive &ar) const { - int type = static_cast(type_); - u64 ids[2] = {blob_id_.unique_, tag_id_.unique_}; - i32 nodes[2] = {blob_id_.node_id_, tag_id_.node_id_}; - ar(type, ids[0], nodes[0], ids[1], nodes[1], blob_size_, rank_); - } - - /** Deserialize */ - template - void load(Archive &ar) { - int type; - ar(type, - blob_id_.unique_, - blob_id_.node_id_, - tag_id_.unique_, - tag_id_.node_id_, - blob_size_, - rank_); - type_ = static_cast(type); - } -}; - -/** Used as hints to the prefetcher */ -struct IoTrace { - i32 node_id_; - IoType type_; - std::string blob_name_; - std::string tag_name_; - size_t blob_size_; - int organize_next_n_; - float score_; - int rank_; -}; - -/** A definition for logging something that is not yet implemented */ -#define HERMES_NOT_IMPLEMENTED_YET \ - HELOG(kFatal, "not implemented yet") - -/** A TargetId uniquely identifies a buffering target within the system. */ -union TargetId { - /** The Target ID as bitfield */ - struct { - /** The ID of the node in charge of this target. */ - i32 node_id_; - /** The ID of the virtual device that backs this target. It is an index into - * the Device array. */ - u16 device_id_; - /** The index into the Target array. */ - u16 index_; - } bits_; - - /** The TargetId as a unsigned 64-bit integer */ - u64 as_int_; - - TargetId() = default; - - TargetId(i32 node_id, u16 device_id, u16 index) { - bits_.node_id_ = node_id; - bits_.device_id_ = device_id; - bits_.index_ = index; - } - - TargetId(const TargetId &other) { - as_int_ = other.as_int_; - } - - i32 GetNodeId() { - return bits_.node_id_; - } - - u16 GetDeviceId() { - return bits_.device_id_; - } - - u16 GetIndex() { - return bits_.index_; - } - - bool IsNull() const { return as_int_ == 0; } - - bool operator==(const TargetId &other) const { - return as_int_ == other.as_int_; - } - - bool operator!=(const TargetId &other) const { - return as_int_ != other.as_int_; - } -}; - -/** - * Represents the fraction of a blob to place - * on a particular target during data placement - * */ -struct SubPlacement { - size_t size_; /**< Size (bytes) */ - TargetId tid_; /**< Target destination of data */ - - SubPlacement() = default; - - explicit SubPlacement(size_t size, TargetId tid) - : size_(size), tid_(tid) {} -}; - -/** - * The organization of a particular blob in the storage - * hierarchy during data placement. - */ -struct PlacementSchema { - std::vector plcmnts_; - - void AddSubPlacement(size_t size, TargetId tid) { - plcmnts_.emplace_back(size, tid); - } - - void Clear() { - plcmnts_.clear(); - } -}; - -/** - * A structure to represent thesholds with mimimum and maximum values - */ -struct Thresholds { - float min_; /**< minimum threshold value */ - float max_; /**< maximum threshold value */ -}; - -} // namespace hermes - - -namespace hermes::api { - -/** A blob is an uniterpreted array of bytes */ -typedef hshm::charbuf Blob; - -/** Supported data placement policies */ -enum class PlacementPolicy { - kRandom, /**< Random blob placement */ - kRoundRobin, /**< Round-Robin (around devices) blob placement */ - kMinimizeIoTime, /**< LP-based blob placement, minimize I/O time */ - kNone, /**< No DPE for cases we want it disabled */ -}; - -/** A class to convert placement policy enum value to string */ -class PlacementPolicyConv { - public: - /** A function to return string representation of \a policy */ - static std::string to_str(PlacementPolicy policy) { - switch (policy) { - case PlacementPolicy::kRandom: { - return "PlacementPolicy::kRandom"; - } - case PlacementPolicy::kRoundRobin: { - return "PlacementPolicy::kRoundRobin"; - } - case PlacementPolicy::kMinimizeIoTime: { - return "PlacementPolicy::kMinimizeIoTime"; - } - case PlacementPolicy::kNone: { - return "PlacementPolicy::kNone"; - } - } - return "PlacementPolicy::Invalid"; - } - - /** return enum value of \a policy */ - static PlacementPolicy to_enum(const std::string &policy) { - if (policy.find("Random") != std::string::npos) { - return PlacementPolicy::kRandom; - } else if (policy.find("RoundRobin") != std::string::npos) { - return PlacementPolicy::kRoundRobin; - } else if (policy.find("MinimizeIoTime") != std::string::npos) { - return PlacementPolicy::kMinimizeIoTime; - } else if (policy.find("None") != std::string::npos) { - return PlacementPolicy::kNone; - } - return PlacementPolicy::kNone; - } -}; - -/** - A structure to represent MinimizeIOTime options -*/ -struct MinimizeIoTimeOptions { - double minimum_remaining_capacity; /**< minimum remaining capacity */ - double capacity_change_threshold; /**< threshold for capacity change */ - bool use_placement_ratio; /**< use placement ratio or not */ - - /** A function for initialization */ - MinimizeIoTimeOptions(double minimum_remaining_capacity_ = 0.0, - double capacity_change_threshold_ = 0.0, - bool use_placement_ratio_ = false) - : minimum_remaining_capacity(minimum_remaining_capacity_), - capacity_change_threshold(capacity_change_threshold_), - use_placement_ratio(use_placement_ratio_) {} -}; - -enum class PrefetchHint { - kNone, - kFileSequential, - kApriori, - - kFileStrided, - kMachineLearning -}; - -/** Hermes API call context */ -struct Context { - /** The blob placement policy */ - PlacementPolicy policy; - - /** Options for controlling the MinimizeIoTime PlacementPolicy */ - MinimizeIoTimeOptions minimize_io_time_options; - - /** Whether random splitting of blobs is enabled for Round-Robin */ - bool rr_split; - - /** Whether Round-Robin can be retried after failure */ - bool rr_retry; - - /** Whether swapping is disabled */ - bool disable_swap; - - /** The blob's score */ - float blob_score_; - - Context(); -}; - -/** \brief Trait types. - * - */ -enum class TraitType : u8 { - META = 0, /**< The Trait only modifies metadata. */ - DATA = 1, /**< The Trait modifies raw data (Blob%s). */ - PERSIST = 2, -}; - -} // namespace hermes::api - -namespace hermes { - -/** Namespace simplification for Blob */ -using api::Blob; - -/** Namespace simplification for Context */ -using api::Context; - -} // namespace hermes - - -/** - * HASH FUNCTIONS - * */ - -namespace std { -template -struct hash> { - std::size_t operator()(const hermes::UniqueId &key) const { - return - std::hash{}(key.unique_) + - std::hash{}(key.node_id_); - } -}; -} // namespace std - -namespace hermes { - -/** Used for debugging concurrency issues with locks */ -enum LockOwners { - kNone = 0, - kMDM_Create = 1, - kMDM_Update = 2, - kMDM_Find = 3, - kMDM_Find2 = 4, - kMDM_Delete = 5, - kFS_GetBaseAdapterMode = 6, - kFS_GetAdapterMode = 7, - kFS_GetAdapterPageSize = 8, - kBORG_LocalEnqueueFlushes = 9, - kBORG_LocalProcessFlushes = 10, - kMDM_LocalGetBucketSize = 12, - kMDM_LocalSetBucketSize = 13, - kMDM_LocalLockBucket = 14, - kMDM_LocalUnlockBucket = 15, - kMDM_LocalClearBucket = 16, - kMDM_LocalTagBlob = 17, - kMDM_LocalBlobHasTag = 18, - kMDM_LocalTryCreateBlob = 19, - kMDM_LocalPutBlobMetadata = 20, - kMDM_LocalGetBlobId = 21, - kMDM_LocalGetBlobName = 22, - kMDM_LocalGetBlobScore = 24, - kMDM_LocalLockBlob = 26, - kMDM_LocalUnlockBlob = 27, - kMDM_LocalGetBlobBuffers = 28, - kMDM_LocalRenameBlob = 29, - kMDM_LocalDestroyBlob = 30, - kMDM_LocalClear = 31, - kMDM_LocalGetOrCreateTag = 32, - kMDM_LocalGetTagId = 33, - kMDM_LocalGetTagName = 34, - kMDM_LocalRenameTag = 35, - kMDM_LocalDestroyTag = 36, - kMDM_LocalTagAddBlob = 37, - kMDM_LocalTagRemoveBlob = 38, - kMDM_LocalGroupByTag = 39, - kMDM_LocalTagAddTrait = 40, - kMDM_LocalTagGetTraits = 41, - kMDM_LocalRegisterTrait = 42, - kMDM_LocalGetTraitId = 43, - kMDM_LocalGetTraitParams = 44, - kMDM_GlobalGetTrait = 45, - kMDM_AddIoStat = 46, - kMDM_ClearIoStats = 47 -}; - -} // namespace hermes -#endif // HERMES_TYPES_H_ diff --git a/src/hermes_version.h.in b/src/hermes_version.h.in deleted file mode 100644 index c98e9f5cc..000000000 --- a/src/hermes_version.h.in +++ /dev/null @@ -1,21 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_VERSION_H_ -#define HERMES_VERSION_H_ - -#define HERMES_VERSION_MAJOR @HERMES_VERSION_MAJOR@ -#define HERMES_VERSION_MINOR @HERMES_VERSION_MINOR@ -#define HERMES_VERSION_PATCH @HERMES_VERSION_PATCH@ -#define HERMES_VERSION_STRING "@HERMES_PACKAGE_VERSION@" - -#endif // HERMES_VERSION_H_ diff --git a/src/labstor_client.cc b/src/labstor_client.cc new file mode 100644 index 000000000..10e7fcf7a --- /dev/null +++ b/src/labstor_client.cc @@ -0,0 +1,9 @@ +// +// Created by lukemartinlogan on 6/27/23. +// + +#include "hermes_shm/util/singleton.h" +#include "labstor/api/labstor_client.h" + +/** Runtime singleton */ +DEFINE_SINGLETON_CC(labstor::Client) \ No newline at end of file diff --git a/src/labstor_runtime.cc b/src/labstor_runtime.cc new file mode 100644 index 000000000..7db986499 --- /dev/null +++ b/src/labstor_runtime.cc @@ -0,0 +1,9 @@ +// +// Created by lukemartinlogan on 6/17/23. +// + +#include "hermes_shm/util/singleton.h" +#include "labstor/api/labstor_runtime.h" + +/** Runtime singleton */ +DEFINE_SINGLETON_CC(labstor::Runtime) \ No newline at end of file diff --git a/src/labstor_start_runtime.cc b/src/labstor_start_runtime.cc new file mode 100644 index 000000000..c7749ccec --- /dev/null +++ b/src/labstor_start_runtime.cc @@ -0,0 +1,13 @@ +// +// Created by lukemartinlogan on 6/17/23. +// + +#include "hermes_shm/util/singleton.h" +#include "labstor/api/labstor_runtime.h" + +int main(int argc, char **argv) { + LABSTOR_RUNTIME->Create(); + LABSTOR_RUNTIME->RunDaemon(); + LABSTOR_RUNTIME->Finalize(); + return 0; +} diff --git a/src/labstor_stop_runtime.cc b/src/labstor_stop_runtime.cc new file mode 100644 index 000000000..3f9951fa5 --- /dev/null +++ b/src/labstor_stop_runtime.cc @@ -0,0 +1,10 @@ +// +// Created by lukemartinlogan on 6/22/23. +// + +#include "labstor_admin/labstor_admin.h" + +int main() { + TRANSPARENT_LABSTOR(); + LABSTOR_ADMIN->AsyncStopRuntimeRoot(labstor::DomainId::GetLocal()); +} \ No newline at end of file diff --git a/src/metadata_manager.cc b/src/metadata_manager.cc deleted file mode 100644 index db28a389d..000000000 --- a/src/metadata_manager.cc +++ /dev/null @@ -1,953 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes.h" -#include "metadata_manager.h" -#include "buffer_organizer.h" -#include "api/bucket.h" -#include - -namespace hermes { - -/** Namespace simplification for Bucket */ -using api::Bucket; - -/** - * Explicitly initialize the MetadataManager - * Requires RPC to be initialized - * */ -void MetadataManager::shm_init(hipc::ShmArchive &header, - hipc::Allocator *alloc, - ServerConfig *config) { - shm_deserialize(header); - header_->id_alloc_ = 1; - - // Put the node_id in SHM - header_->node_id_ = rpc_->node_id_; - - // Create the metadata maps - HSHM_MAKE_AR(header_->blob_id_map_, alloc, config->mdm_.num_blobs_) - HSHM_MAKE_AR(header_->blob_map_, alloc, config->mdm_.num_blobs_) - HSHM_MAKE_AR(header_->tag_id_map_, alloc, config->mdm_.num_bkts_) - HSHM_MAKE_AR(header_->tag_map_, alloc, config->mdm_.num_bkts_) - HSHM_MAKE_AR(header_->trait_id_map_, alloc, config->mdm_.num_traits_) - HSHM_MAKE_AR(header_->trait_map_, alloc, config->mdm_.num_traits_) - - // Create the DeviceInfo vector - HSHM_MAKE_AR(header_->devices_, alloc, *config->devices_) - - // Create the TargetInfo vector - HSHM_MAKE_AR0(header_->targets_, alloc) - targets_->reserve(devices_->size()); - int dev_id = 0; - float maxbw = 0; - for (DeviceInfo &dev_info : *config->devices_) { - targets_->emplace_back( - TargetId(rpc_->node_id_, dev_id, dev_id), - dev_info.capacity_, - dev_info.capacity_, - dev_info.bandwidth_, - dev_info.latency_); - if (maxbw < dev_info.bandwidth_) { - maxbw = dev_info.bandwidth_; - } - ++dev_id; - } - - // Assign a score to each target - for (TargetInfo &target_info : *targets_) { - target_info.score_ = (target_info.bandwidth_ / maxbw); - } - - // Create the log used to track I/O pattern - HSHM_MAKE_AR(header_->io_pattern_log_, alloc, 8192); -} - -/**==================================== - * SHM Deserialize - * ===================================*/ - -/** - * Store the MetadataManager in shared memory. - * Does not require anything to be initialized. - * */ -void MetadataManager::shm_deserialize( - hipc::ShmArchive &header) { - rpc_ = &HERMES->rpc_; - borg_ = &HERMES->borg_; - traits_ = &HERMES->traits_; - - header_ = header.get(); - blob_id_map_ = header_->blob_id_map_.get(); - blob_map_ = header_->blob_map_.get(); - tag_id_map_ = header_->tag_id_map_.get(); - tag_map_ = header_->tag_map_.get(); - trait_id_map_ = header_->trait_id_map_.get(); - trait_map_ = header_->trait_map_.get(); - targets_ = header_->targets_.get(); - devices_ = header_->devices_.get(); - io_pattern_log_ = header_->io_pattern_log_.get(); -} - -/**==================================== - * Destructor - * ===================================*/ - -/** - * Explicitly destroy the MetadataManager - * */ -void MetadataManager::shm_destroy() { - (*blob_id_map_).shm_destroy(); - (*blob_map_).shm_destroy(); - (*tag_id_map_).shm_destroy(); - (*tag_map_).shm_destroy(); - (*trait_id_map_).shm_destroy(); - (*trait_map_).shm_destroy(); - (*targets_).shm_destroy(); - (*devices_).shm_destroy(); - (*io_pattern_log_).shm_destroy(); -} - -/**==================================== - * Bucket Operations - * ===================================*/ - -/** - * Get the size of the bucket. May consider the impact the bucket has - * on the backing storage system's statistics using the io_ctx. - * */ -size_t MetadataManager::LocalGetBucketSize(TagId bkt_id) { - AUTO_TRACE(1); - // Acquire MD read lock (reading tag_map) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalGetBucketSize); - auto iter = tag_map_->find(bkt_id); - if (iter == tag_map_->end()) { - return 0; - } - hipc::pair &info = (*iter); - TagInfo &bkt_info = info.GetSecond(); - return bkt_info.internal_size_; -} - -/** - * Update \a bkt_id BUCKET stats - * */ -bool MetadataManager::LocalSetBucketSize(TagId bkt_id, - size_t new_size) { - AUTO_TRACE(1); - // Acquire MD read lock (reading tag_map) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalSetBucketSize); - auto iter = tag_map_->find(bkt_id); - if (iter == tag_map_->end()) { - return false; - } - hipc::pair &info = (*iter); - TagInfo &bkt_info = info.GetSecond(); - bkt_info.internal_size_ = new_size; - HILOG(kDebug, "The new size of the bucket {} ({}) is {}", - bkt_info.name_->str(), bkt_id, - bkt_info.internal_size_) - return true; -} - -/** - * Lock the bucket - * */ -bool MetadataManager::LocalLockBucket(TagId bkt_id, - MdLockType lock_type) { - AUTO_TRACE(1); - if (bkt_id.IsNull()) { - return false; - } - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalLockBucket); - return LockMdObject(*tag_map_, bkt_id, lock_type); -} - -/** - * Unlock the bucket - * */ -bool MetadataManager::LocalUnlockBucket(TagId bkt_id, - MdLockType lock_type) { - AUTO_TRACE(1); - if (bkt_id.IsNull()) { - return false; - } - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalUnlockBucket); - return UnlockMdObject(*tag_map_, bkt_id, lock_type); -} - -/** - * Clear \a bkt_id bucket - * */ -bool MetadataManager::LocalClearBucket(TagId bkt_id) { - AUTO_TRACE(1); - HILOG(kDebug, "Clearing the bucket {}", bkt_id) - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire tag map write lock - ScopedRwWriteLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalClearBucket); - auto iter = tag_map_->find(bkt_id); - if (iter == tag_map_->end()) { - return true; - } - hipc::pair &info = (*iter); - TagInfo &bkt_info = info.GetSecond(); - for (BlobId &blob_id : *bkt_info.blobs_) { - GlobalDestroyBlob(bkt_id, blob_id); - } - bkt_info.blobs_->clear(); - bkt_info.internal_size_ = 0; - return true; -} - -/**==================================== - * Blob Operations - * ===================================*/ - -/** - * Tag a blob - * - * @param blob_id id of the blob being tagged - * @param tag_name tag name - * */ -Status MetadataManager::LocalTagBlob( - BlobId blob_id, TagId tag_id) { - AUTO_TRACE(1); - // Acquire MD read lock (read blob_map_) - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalTagBlob); - auto iter = blob_map_->find(blob_id); - if (iter == blob_map_->end()) { - return Status(); // TODO(llogan): error status - } - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - // Acquire blob_info read lock (read buffers) - ScopedRwReadLock blob_info_lock(blob_info.lock_[0], - kMDM_LocalTagBlob); - blob_info.tags_->emplace_back(tag_id); - return Status(); -} - -/** - * Check whether or not \a blob_id BLOB has \a tag_id TAG - * */ -bool MetadataManager::LocalBlobHasTag(BlobId blob_id, - TagId tag_id) { - AUTO_TRACE(1); - // Acquire MD read lock (reading blob_map_) - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalBlobHasTag); - auto iter = blob_map_->find(blob_id); - if (iter == blob_map_->end()) { - return false; - } - // Get the blob info - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - // Iterate over tags - for (TagId &tag_cmp : *blob_info.tags_) { - if (tag_cmp == tag_id) { - return true; - } - } - return false; -} - -/** - * Creates the blob metadata - * - * @param bkt_id id of the bucket - * @param blob_name semantic blob name - * */ -std::pair MetadataManager::LocalTryCreateBlob( - TagId bkt_id, - const std::string &blob_name) { - AUTO_TRACE(1); - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire MD write lock (modify blob_map_) - ScopedRwWriteLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalTryCreateBlob); - // Get internal blob name - hipc::uptr internal_blob_name = - CreateBlobName(bkt_id, blob_name); - // Create unique ID for the Blob - BlobId blob_id; - blob_id.unique_ = header_->id_alloc_.fetch_add(1); - blob_id.node_id_ = rpc_->node_id_; - bool did_create = blob_id_map_->try_emplace(*internal_blob_name, blob_id); - if (did_create) { - blob_map_->emplace(blob_id); - auto iter = blob_map_->find(blob_id); - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - (*blob_info.name_) = blob_name; - blob_info.blob_id_ = blob_id; - blob_info.tag_id_ = bkt_id; - blob_info.blob_size_ = 0; - } - return std::pair(blob_id, did_create); -} - -/** - * Creates the blob metadata - * - * @param bkt_id id of the bucket - * @param blob_name semantic blob name - * @param data the data being placed - * @param buffers the buffers to place data in - * */ -std::tuple -MetadataManager::LocalPutBlobMetadata(TagId bkt_id, - const std::string &blob_name, - size_t blob_size, - std::vector &buffers, - float score) { - AUTO_TRACE(1); - size_t orig_blob_size = 0; - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire MD write lock (modify blob_map_) - ScopedRwWriteLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalPutBlobMetadata); - // Get internal blob name - hipc::uptr internal_blob_name = - CreateBlobName(bkt_id, blob_name); - // Create unique ID for the Blob - BlobId blob_id; - blob_id.unique_ = header_->id_alloc_.fetch_add(1); - blob_id.node_id_ = rpc_->node_id_; - bool did_create = blob_id_map_->try_emplace(*internal_blob_name, blob_id); - if (did_create) { - blob_map_->emplace(blob_id); - HILOG(kDebug, "Creating new blob: {}. Total num blobs: {}", - blob_name, blob_map_->size()) - auto iter = blob_map_->find(blob_id); - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - (*blob_info.name_) = blob_name; - (*blob_info.buffers_) = buffers; - blob_info.blob_id_ = blob_id; - blob_info.tag_id_ = bkt_id; - blob_info.blob_size_ = blob_size; - blob_info.score_ = score; - blob_info.mod_count_ = 0; - blob_info.access_freq_ = 0; - blob_info.last_flush_ = 0; - blob_info.UpdateWriteStats(); - } else { - HILOG(kDebug, "Found existing blob: {}. Total num blobs: {}", - blob_name, blob_map_->size()) - blob_id = (*blob_id_map_)[*internal_blob_name]; - auto iter = blob_map_->find(blob_id); - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - // Acquire blob_info write lock before modifying buffers - ScopedRwWriteLock blob_info_lock(blob_info.lock_[0], - kMDM_LocalPutBlobMetadata); - (*blob_info.buffers_) = buffers; - blob_info.blob_size_ = blob_size; - blob_info.score_ = score; - blob_info.UpdateWriteStats(); - } - return std::tuple(blob_id, did_create, orig_blob_size); -} - -/** - * Get \a blob_name blob from \a bkt_id bucket - * */ -BlobId MetadataManager::LocalGetBlobId(TagId bkt_id, - const std::string &blob_name) { - AUTO_TRACE(1); - // Acquire MD read lock (read blob_id_map_) - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalGetBlobId); - hipc::uptr internal_blob_name = - CreateBlobName(bkt_id, blob_name); - auto iter = blob_id_map_->find(*internal_blob_name); - if (iter == blob_id_map_->end()) { - return BlobId::GetNull(); - } - hipc::pair &info = *iter; - return info.GetSecond(); -} - -/** - * Get \a blob_name BLOB name from \a blob_id BLOB id - * */ -std::string MetadataManager::LocalGetBlobName(BlobId blob_id) { - AUTO_TRACE(1); - // Acquire MD read lock (read blob_id_map_) - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalGetBlobName); - auto iter = blob_map_->find(blob_id); - if (iter == blob_map_->end()) { - return ""; - } - hipc::pair& info = *iter; - BlobInfo &blob_info = info.GetSecond(); - return blob_info.name_->str(); -} - -/** - * Get \a score from \a blob_id BLOB id - * */ -float MetadataManager::LocalGetBlobScore(BlobId blob_id) { - AUTO_TRACE(1); - // Acquire MD read lock (read blob_id_map_) - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalGetBlobScore); - auto iter = blob_map_->find(blob_id); - if (iter == blob_map_->end()) { - return -1; - } - hipc::pair& info = *iter; - BlobInfo &blob_info = info.GetSecond(); - return blob_info.score_; -} - -/** - * Lock the blob - * */ -bool MetadataManager::LocalLockBlob(BlobId blob_id, - MdLockType lock_type) { - AUTO_TRACE(1); - if (blob_id.IsNull()) { - return false; - } - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalLockBlob); - return LockMdObject(*blob_map_, blob_id, lock_type); -} - -/** - * Unlock the blob - * */ -bool MetadataManager::LocalUnlockBlob(BlobId blob_id, - MdLockType lock_type) { - AUTO_TRACE(1); - if (blob_id.IsNull()) { - return false; - } - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalUnlockBlob); - return UnlockMdObject(*blob_map_, blob_id, lock_type); -} - -/** - * Get \a blob_id blob's buffers - * */ -std::vector MetadataManager::LocalGetBlobBuffers(BlobId blob_id) { - AUTO_TRACE(1); - // Acquire MD read lock (read blob_map_) - ScopedRwReadLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalGetBlobBuffers); - auto iter = blob_map_->find(blob_id); - if (iter == blob_map_->end()) { - return std::vector(); - } - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - // Acquire blob_info read lock - ScopedRwReadLock blob_info_lock(blob_info.lock_[0], - kMDM_LocalGetBlobBuffers); - blob_info.UpdateReadStats(); - auto vec = blob_info.buffers_->vec(); - return vec; -} - -/** - * Rename \a blob_id blob to \a new_blob_name new blob name - * in \a bkt_id bucket. - * */ -bool MetadataManager::LocalRenameBlob(TagId bkt_id, BlobId blob_id, - const std::string &new_blob_name) { - HILOG(kDebug, "Renaming the blob: {}.{}", blob_id.node_id_, blob_id.unique_) - AUTO_TRACE(1); - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire MD write lock (modify blob_id_map_) - ScopedRwWriteLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalRenameBlob); - auto iter = (*blob_map_).find(blob_id); - if (iter == blob_map_->end()) { - return true; - } - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - hipc::uptr old_blob_name_internal = - CreateBlobName(bkt_id, *blob_info.name_); - hipc::uptr new_blob_name_internal = - CreateBlobName(bkt_id, new_blob_name); - blob_id_map_->erase(*old_blob_name_internal); - blob_id_map_->emplace(*new_blob_name_internal, blob_id); - return true; -} - -/** - * Destroy \a blob_id blob in \a bkt_id bucket - * */ -bool MetadataManager::LocalDestroyBlob(TagId bkt_id, - BlobId blob_id) { - HILOG(kDebug, "Destroying the blob: {}", blob_id) - AUTO_TRACE(1); - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire MD write lock (modify blob_id_map & blob_map_) - ScopedRwWriteLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalDestroyBlob); - auto iter = (*blob_map_).find(blob_id); - if (iter.is_end()) { - return true; - } - hipc::pair& info = (*iter); - BlobInfo &blob_info = info.GetSecond(); - hipc::uptr blob_name_internal = - CreateBlobName(bkt_id, *blob_info.name_); - blob_id_map_->erase(*blob_name_internal); - blob_map_->erase(blob_id); - return true; -} - -/**==================================== - * Bucket + Blob Operations - * ===================================*/ - -/** - * Destroy all blobs + buckets - * */ -void MetadataManager::LocalClear() { - AUTO_TRACE(1); - HILOG(kDebug, "Clearing all buckets and blobs") - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Modify tag map - ScopedRwWriteLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalClear); - tag_id_map_->clear(); - tag_map_->clear(); - // Modify blob map - ScopedRwWriteLock blob_map_lock(header_->lock_[kBlobMapLock], - kMDM_LocalClear); - blob_id_map_->clear(); - blob_map_->clear(); -} - -/** - * Destroy all blobs + buckets globally - * */ -void MetadataManager::GlobalClear() { - AUTO_TRACE(1); - for (i32 i = 0; i < (i32)rpc_->hosts_.size(); ++i) { - i32 node_id = i + 1; - if (NODE_ID_IS_LOCAL(node_id)) { - LocalClear(); - } else { - rpc_->Call(node_id, "RpcClear"); - } - } -} - -/**==================================== - * Tag Operations - * ===================================*/ - -/** - * Get or create a bucket with \a bkt_name bucket name - * */ -std::pair -MetadataManager::LocalGetOrCreateTag(const std::string &tag_name, - bool owner, - std::vector &traits, - size_t backend_size) { - AUTO_TRACE(1); - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire MD write lock (modifying tag_map) - ScopedRwWriteLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalGetOrCreateTag); - - // Create unique ID for the Bucket - TagId tag_id; - tag_id.unique_ = header_->id_alloc_.fetch_add(1); - tag_id.node_id_ = rpc_->node_id_; - auto tag_name_shm = hipc::make_uptr(tag_name); - bool did_create = tag_id_map_->try_emplace(*tag_name_shm, tag_id); - - // Emplace bucket if it does not already exist - if (did_create) { - HILOG(kDebug, "Creating tag for the first time: {} {}", tag_name, tag_id) - tag_map_->emplace(tag_id); - auto iter = tag_map_->find(tag_id); - hipc::pair &info_pair = *iter; - TagInfo &info = info_pair.GetSecond(); - (*info.name_) = *tag_name_shm; - info.tag_id_ = tag_id; - info.owner_ = owner; - info.internal_size_ = backend_size; - } else { - HILOG(kDebug, "Found existing tag: {}", tag_name) - auto iter = tag_id_map_->find(*tag_name_shm); - hipc::pair &id_info = (*iter); - tag_id = id_info.GetSecond(); - } - - return {tag_id, did_create}; -} - -/** - * Get the TagId with \a bkt_name bucket name - * */ -TagId MetadataManager::LocalGetTagId(const std::string &tag_name) { - AUTO_TRACE(1); - // Acquire MD read lock (reading tag_id_map) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalGetTagId); - auto tag_name_shm = hipc::make_uptr(tag_name); - auto iter = tag_id_map_->find(*tag_name_shm); - if (iter == tag_id_map_->end()) { - return TagId::GetNull(); - } - hipc::pair &info_pair = (*iter); - TagId bkt_id = info_pair.GetSecond(); - return bkt_id; -} - -/** - * Get the name of a tag - * */ -std::string MetadataManager::LocalGetTagName(TagId tag_id) { - HILOG(kDebug, "Finding the name of tag: {}.{}", - tag_id.node_id_, tag_id.unique_) - AUTO_TRACE(1); - // Acquire MD read lock (reading tag_map_) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalGetTagName); - auto iter = tag_map_->find(tag_id); - if (iter == tag_map_->end()) { - return ""; - } - hipc::pair &info_pair = (*iter); - hipc::string &bkt_name = *info_pair.GetSecond().name_; - return bkt_name.str(); -} - -/** - * Rename a tag - * */ -bool MetadataManager::LocalRenameTag(TagId tag_id, - const std::string &new_name) { - HILOG(kDebug, "Renaming the tag: {}.{} to {}", - tag_id.node_id_, tag_id.unique_, new_name) - AUTO_TRACE(1); - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire MD write lock (modifying tag_map_) - ScopedRwWriteLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalRenameTag); - auto iter = tag_map_->find(tag_id); - if (iter == tag_map_->end()) { - return true; - } - auto new_name_shm = hipc::make_uptr(new_name); - hipc::pair &info_pair = (*iter); - hipc::string &old_bkt_name = *info_pair.GetSecond().name_; - tag_id_map_->erase(old_bkt_name); - tag_id_map_->emplace(*new_name_shm, tag_id); - return true; -} - -/** - * Delete a tag - * */ -bool MetadataManager::LocalDestroyTag(TagId tag_id) { - HILOG(kDebug, "Destroying the tag: {}", tag_id) - AUTO_TRACE(1); - // Avoid flushing - ScopedRwReadLock flush_lock(header_->lock_[kFlushLock], kMDM_LocalClear); - // Acquire MD write lock (modifying tag_map_) - ScopedRwWriteLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalDestroyTag); - auto iter = tag_map_->find(tag_id); - if (iter.is_end()) { - HILOG(kDebug, "Could NOT destroy: {}", tag_id) - return true; - } - hipc::pair &info_pair = *iter; - auto &tag_info = info_pair.GetSecond(); - tag_id_map_->erase(*tag_info.name_); - tag_map_->erase(tag_id); - HILOG(kDebug, "Destroyed the tag: {}", tag_id) - return true; -} - -/** - * Add a blob to a tag index - * */ -Status MetadataManager::LocalTagAddBlob(TagId tag_id, - BlobId blob_id) { - AUTO_TRACE(1); - // Acquire MD write lock (modify tag_map_) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalTagAddBlob); - auto iter = tag_map_->find(tag_id); - if (iter.is_end()) { - return Status(); - } - hipc::pair &blob_list = (*iter); - TagInfo &info = blob_list.GetSecond(); - ScopedRwWriteLock tag_lock(info.lock_[0], kMDM_LocalTagRemoveBlob); - info.blobs_->emplace_back(blob_id); - return Status(); -} - -/** - * Remove a blob from a tag index. - * */ -Status MetadataManager::LocalTagRemoveBlob(TagId tag_id, - BlobId blob_id) { - AUTO_TRACE(1); - // Acquire MD write lock (modify tag_map_) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalTagRemoveBlob); - auto iter = tag_map_->find(tag_id); - if (iter.is_end()) { - return Status(); - } - hipc::pair &blob_list = (*iter); - TagInfo &info = blob_list.GetSecond(); - ScopedRwWriteLock tag_lock(info.lock_[0], kMDM_LocalTagRemoveBlob); - info.blobs_->erase(blob_id); - return Status(); -} - - -/** - * Find all blobs pertaining to a tag - * */ -std::vector MetadataManager::LocalGroupByTag(TagId tag_id) { - AUTO_TRACE(1); - // Acquire MD read lock (read tag_map_) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalGroupByTag); - // Get the tag info - auto iter = tag_map_->find(tag_id); - if (iter.is_end()) { - return std::vector(); - } - auto &tag_info_pair = *iter; - TagInfo &tag_info = tag_info_pair.GetSecond(); - // Convert slist into std::vector - return hshm::to_stl_vector(*tag_info.blobs_); -} - -/** - * Add a trait to a tag index. Create tag if it does not exist. - * */ -bool MetadataManager::LocalTagAddTrait(TagId tag_id, TraitId trait_id) { - AUTO_TRACE(1); - HILOG(kDebug, "Adding trait {} to tag {}", - trait_id, tag_id); - // Acquire MD read lock (read tag_map_) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalTagAddTrait); - auto iter = tag_map_->find(tag_id); - if (iter.is_end()) { - HILOG(kDebug, "Could NOT add trait to tag {}") - return true; - } - auto &tag_info_pair = *iter; - TagInfo &tag_info = tag_info_pair.GetSecond(); - ScopedRwWriteLock tag_lock(tag_info.lock_[0], kMDM_LocalTagAddTrait); - tag_info.traits_->emplace_back(trait_id); - HILOG(kDebug, "Added trait {} to tag {}", - trait_id, tag_id); - return true; -} - -/** - * Find all traits pertaining to a tag - * */ -std::vector MetadataManager::LocalTagGetTraits(TagId tag_id) { - AUTO_TRACE(1); - // Acquire MD read lock (read tag_map_) - ScopedRwReadLock tag_map_lock(header_->lock_[kTagMapLock], - kMDM_LocalTagGetTraits); - auto iter = tag_map_->find(tag_id); - if (iter.is_end()) { - return std::vector(); - } - auto &tag_info_pair = *iter; - TagInfo &tag_info = tag_info_pair.GetSecond(); - return hshm::to_stl_vector(*tag_info.traits_); -} - -/**==================================== - * Trait Operations - * ===================================*/ - -/** - * Register a trait. Stores the state of the trait in a way that can - * be queried by all processes. - * */ -RPC TraitId MetadataManager::LocalRegisterTrait( - TraitId trait_id, - const hshm::charbuf &trait_params) { - HILOG(kDebug, "Registering trait {}", trait_id) - // Acquire md write lock (modifying trait map) - ScopedRwWriteLock md_lock(header_->lock_[kTraitMapLock], - kMDM_LocalRegisterTrait); - auto hdr = reinterpret_cast(trait_params.data()); - std::string trait_uuid = hdr->trait_uuid_; - - // Check if trait exists - auto trait_uuid_shm = hipc::make_uptr(trait_uuid); - auto iter = trait_id_map_->find(*trait_uuid_shm); - if (!iter.is_end()) { - trait_id = (*iter).GetSecond(); - return trait_id; - } - - // Create new trait - if (trait_id.IsNull()) { - trait_id.unique_ = header_->id_alloc_.fetch_add(1); - trait_id.node_id_ = rpc_->node_id_; - } - trait_id_map_->emplace(*trait_uuid_shm, trait_id); - trait_map_->emplace(trait_id, trait_params); - return trait_id; -} - -/** - * Get trait identifier - * */ -RPC TraitId -MetadataManager::LocalGetTraitId(const std::string &trait_uuid) { - HILOG(kDebug, "Getting trait id for {}", trait_uuid) - // Acquire md read lock (reading trait_id_map) - ScopedRwReadLock md_lock(header_->lock_[kTraitMapLock], - kMDM_LocalGetTraitId); - // Check if trait exists - auto trait_uuid_shm = hipc::make_uptr(trait_uuid); - auto iter = trait_id_map_->find(*trait_uuid_shm); - if (iter.is_end()) { - return TraitId::GetNull(); - } - TraitId trait_id = (*iter).GetSecond(); - return trait_id; -} - -/** - * Get trait parameters - * */ -RPC hshm::charbuf -MetadataManager::LocalGetTraitParams(TraitId trait_id) { - // Acquire md read lock (reading trait_map) - ScopedRwReadLock md_lock(header_->lock_[kTraitMapLock], - kMDM_LocalGetTraitParams); - - // Get the trait parameters from the map - auto iter = trait_map_->find(trait_id); - if (iter.is_end()) { - return hshm::charbuf(); - } - hipc::pair &trait_pair = *iter; - return hshm::to_charbuf(trait_pair.GetSecond()); -} - -/** - * Get an existing trait - * */ -Trait* MetadataManager::GlobalGetTrait(TraitId trait_id) { - // HILOG(kDebug, "Getting the trait {}", trait_id) - Trait *trait = nullptr; - - // Check if trait is already constructed - ScopedRwReadLock trait_lock(header_->lock_[kTraitMapLock], - kMDM_GlobalGetTrait); - auto iter = local_trait_map_.find(trait_id); - if (iter != local_trait_map_.end()) { - std::pair trait_pair = *iter; - trait = trait_pair.second; - trait_lock.Unlock(); - return trait; - } - trait_lock.Unlock(); - - // Get the trait state locally or globally - hshm::charbuf params = LocalGetTraitParams(trait_id); - if (params.size() == 0) { - params = GlobalGetTraitParams(trait_id); - if (params.size() == 0) { - HELOG(kError, "Could not find the trait {}.{}", - trait_id.node_id_, trait_id.unique_) - return nullptr; - } - LocalRegisterTrait(trait_id, params); - } - - // Construct the trait based on the parameters - ScopedRwWriteLock trait_lock_w(header_->lock_[kTraitMapLock], - kMDM_GlobalGetTrait); - HILOG(kDebug, "Acquired trait lock w for {}", trait_id) - - // Check if the trait exists now - iter = local_trait_map_.find(trait_id); - if (iter != local_trait_map_.end()) { - std::pair trait_pair = *iter; - trait = trait_pair.second; - return trait; - } - - // Construct the trait - trait = traits_->ConstructTrait(params); - if (trait == nullptr) { - return nullptr; - } - - // Induct the trait - local_trait_map_.emplace(trait_id, trait); - return trait; -} - -/**==================================== - * Statistics Operations - * ===================================*/ - -/** Add an I/O statistic to the internal log */ -void MetadataManager::AddIoStat(TagId tag_id, - BlobId blob_id, - size_t blob_size, - IoType type) { - if (!enable_io_tracing_) { - return; - } - IoStat stat; - stat.blob_id_ = blob_id; - stat.tag_id_ = tag_id; - stat.blob_size_ = blob_size; - stat.type_ = type; - stat.rank_ = 0; - if (is_mpi_) { - MPI_Comm_rank(MPI_COMM_WORLD, &stat.rank_); - } - io_pattern_log_->emplace(stat); -} - - -} // namespace hermes diff --git a/src/metadata_manager.h b/src/metadata_manager.h deleted file mode 100644 index 7ecf7864d..000000000 --- a/src/metadata_manager.h +++ /dev/null @@ -1,535 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_METADATA_MANAGER_H_ -#define HERMES_SRC_METADATA_MANAGER_H_ - -#include "decorator.h" -#include "hermes_types.h" -#include "status.h" -#include "rpc.h" -#include "metadata_types.h" -#include "trait_manager.h" -#include "statuses.h" -#include "rpc_thallium_serialization.h" - -namespace hermes { - -/** Forward declaration of borg */ -class BufferOrganizer; - -/** - * Type name simplification for the various map types - * */ -typedef hipc::unordered_map BLOB_ID_MAP_T; -typedef hipc::unordered_map TAG_ID_MAP_T; -typedef hipc::unordered_map TRAIT_ID_MAP_T; -typedef hipc::unordered_map BLOB_MAP_T; -typedef hipc::unordered_map TAG_MAP_T; -typedef hipc::unordered_map TRAIT_MAP_T; -typedef hipc::mpsc_queue IO_PATTERN_LOG_T; - -enum MdmLock { - kBlobMapLock, - kTagMapLock, - kTraitMapLock, - kFlushLock, - - kMdmLockCount -}; - -/** - * The SHM representation of the MetadataManager - * */ -struct MetadataManagerShm { - /// SHM representation of blob id map - hipc::ShmArchive blob_id_map_; - /// SHM representation of blob map - hipc::ShmArchive blob_map_; - /// SHM representation of tag id map - hipc::ShmArchive tag_id_map_; - /// SHM representation of tag map - hipc::ShmArchive tag_map_; - /// SHM representation of trait id map - hipc::ShmArchive trait_id_map_; - /// SHM representation of trait map - hipc::ShmArchive trait_map_; - /// SHM representation of device vector - hipc::ShmArchive> devices_; - /// SHM representation of target info vector - hipc::ShmArchive> targets_; - /// SHM representation of prefetcher stats log - hipc::ShmArchive io_pattern_log_; - /// Used to create unique ids. Starts at 1. - std::atomic id_alloc_; - /// Synchronization - RwLock lock_[kMdmLockCount]; - /// The ID of THIS node - i32 node_id_; -}; - -/** - * Manages the metadata for blobs, buckets, and vbuckets. - * */ -class MetadataManager { - public: - RPC_TYPE *rpc_; - BufferOrganizer *borg_; - TraitManager *traits_; - MetadataManagerShm *header_; - - /**==================================== - * Maps - * ===================================*/ - BLOB_ID_MAP_T *blob_id_map_; - TAG_ID_MAP_T *tag_id_map_; - TRAIT_ID_MAP_T *trait_id_map_; - BLOB_MAP_T *blob_map_; - TAG_MAP_T *tag_map_; - TRAIT_MAP_T *trait_map_; - std::unordered_map local_trait_map_; - RwLock local_lock_; - - /**==================================== - * I/O pattern log - * ===================================*/ - IO_PATTERN_LOG_T *io_pattern_log_; - bool enable_io_tracing_; - bool is_mpi_; - - /**==================================== - * Targets + devices - * ===================================*/ - hipc::vector *devices_; - hipc::vector *targets_; - - public: - /**==================================== - * Default Constructor - * ===================================*/ - - /** Default constructor */ - MetadataManager() = default; - - /**==================================== - * SHM Init - * ===================================*/ - - /** Default constructor */ - void shm_init(hipc::ShmArchive &header, - hipc::Allocator *alloc, - ServerConfig *config); - - /**==================================== - * SHM Deserialize - * ===================================*/ - - /** - * Unload the MetadataManager from shared memory - * */ - void shm_deserialize(hipc::ShmArchive &header); - - /**==================================== - * Destructor - * ===================================*/ - - /** Whether the MetadataManager is NULL */ - bool IsNull() { return false; } - - /** Set the MetadataManager to NULL */ - void SetNull() {} - - /** - * Explicitly destroy the MetadataManager - * */ - void shm_destroy(); - - /** - * Create a unique blob name using TagId - * */ - template - static hipc::uptr CreateBlobName( - TagId bkt_id, const StringT &blob_name) { - auto new_name = hipc::make_uptr( - sizeof(uint64_t) + sizeof(uint32_t) + blob_name.size()); - size_t off = 0; - memcpy(new_name->data() + off, &bkt_id.unique_, sizeof(uint64_t)); - off += sizeof(uint64_t); - memcpy(new_name->data() + off, &bkt_id.node_id_, sizeof(uint32_t)); - off += sizeof(uint32_t); - memcpy(new_name->data() + off, blob_name.data(), blob_name.size()); - return new_name; - } - - /**==================================== - * Bucket Operations - * ===================================*/ - - /** - * Get the size of a bucket - * */ - RPC size_t LocalGetBucketSize(TagId bkt_id); - DEFINE_RPC(size_t, GetBucketSize, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Update \a bkt_id BUCKET stats - * */ - RPC bool LocalSetBucketSize(TagId bkt_id, size_t new_size); - DEFINE_RPC(bool, SetBucketSize, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Lock the bucket - * */ - bool LocalLockBucket(TagId bkt_id, MdLockType lock_type); - DEFINE_RPC(bool, LockBucket, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Unlock the bucket - * */ - bool LocalUnlockBucket(TagId bkt_id, MdLockType lock_type); - DEFINE_RPC(bool, UnlockBucket, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Destroy \a bkt_id bucket - * */ - RPC bool LocalClearBucket(TagId bkt_id); - DEFINE_RPC(bool, ClearBucket, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /**==================================== - * Blob Operations - * ===================================*/ - - /** - * Create a blob's metadata - * - * @param bkt_id id of the bucket - * @param blob_name semantic blob name - * @param data the data being placed - * @param buffers the buffers to place data in - * */ - RPC std::tuple LocalPutBlobMetadata( - TagId bkt_id, const std::string &blob_name, size_t blob_size, - std::vector &buffers, float score); - DEFINE_RPC((std::tuple), PutBlobMetadata, 1, - STRING_HASH_LAMBDA) - - /** - * Creates the blob metadata - * - * @param bkt_id id of the bucket - * @param blob_name semantic blob name - * */ - std::pair LocalTryCreateBlob(TagId bkt_id, - const std::string &blob_name); - DEFINE_RPC((std::pair), TryCreateBlob, 1, - STRING_HASH_LAMBDA) - - /** - * Tag a blob - * - * @param blob_id id of the blob being tagged - * @param tag_name tag name - * */ - Status LocalTagBlob(BlobId blob_id, TagId tag_id); - DEFINE_RPC(Status, TagBlob, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Check if blob has a tag - * */ - bool LocalBlobHasTag(BlobId blob_id, TagId tag_id); - DEFINE_RPC(bool, BlobHasTag, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Get \a blob_name BLOB from \a bkt_id bucket - * */ - RPC BlobId LocalGetBlobId(TagId bkt_id, const std::string &blob_name); - DEFINE_RPC(BlobId, GetBlobId, 1, - STRING_HASH_LAMBDA) - - /** - * Get \a blob_name BLOB name from \a blob_id BLOB id - * */ - RPC std::string LocalGetBlobName(BlobId blob_id); - DEFINE_RPC(std::string, GetBlobName, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Get \a score from \a blob_id BLOB id - * */ - RPC float LocalGetBlobScore(BlobId blob_id); - DEFINE_RPC(float, GetBlobScore, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Lock the blob - * */ - RPC bool LocalLockBlob(BlobId blob_id, MdLockType lock_type); - DEFINE_RPC(bool, LockBlob, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Unlock the blob - * */ - RPC bool LocalUnlockBlob(BlobId blob_id, MdLockType lock_type); - DEFINE_RPC(bool, UnlockBlob, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Get \a blob_id blob's buffers - * */ - RPC std::vector LocalGetBlobBuffers(BlobId blob_id); - DEFINE_RPC(std::vector, GetBlobBuffers, 0, - UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Rename \a blob_id blob to \a new_blob_name new blob name - * in \a bkt_id bucket. - * */ - RPC bool LocalRenameBlob(TagId bkt_id, BlobId blob_id, - const std::string &new_blob_name); - DEFINE_RPC(bool, RenameBlob, 1, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Destroy \a blob_id blob in \a bkt_id bucket - * */ - RPC bool LocalDestroyBlob(TagId bkt_id, BlobId blob_id); - DEFINE_RPC(bool, DestroyBlob, 1, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /**==================================== - * Bucket + Blob Operations - * ===================================*/ - - /** - * Destroy all blobs + buckets - * */ - void LocalClear(); - - /** - * Destroy all blobs + buckets globally - * */ - void GlobalClear(); - - /**==================================== - * Target Operations - * ===================================*/ - - /** - * Update the capacity of the target device - * */ - RPC void LocalUpdateTargetCapacity(TargetId tid, off64_t offset) { - TargetInfo &target = (*targets_)[tid.GetIndex()]; - target.rem_cap_ += offset; - } - DEFINE_RPC(bool, UpdateTargetCapacity, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Update the capacity of the target device - * */ - RPC std::vector LocalGetTargetInfo() { - return hshm::to_stl_vector(*targets_); - } - - /** - * Get the TargetInfo for neighborhood - * */ - // hipc::vector GetNeighborhoodTargetInfo() { return {}; } - - /** - * Get all TargetInfo in the system - * */ - // hipc::vector GetGlobalTargetInfo() { return {}; } - - /**==================================== - * Tag Operations - * ===================================*/ - - /** - * Create a tag - * */ - std::pair LocalGetOrCreateTag(const std::string &tag_name, - bool owner, - std::vector &traits, - size_t backend_size); - DEFINE_RPC((std::pair), - GetOrCreateTag, 0, STRING_HASH_LAMBDA) - TagId GlobalCreateTag(const std::string &tag_name, - bool owner, - std::vector &traits) { - return LocalGetOrCreateTag(tag_name, owner, traits, 0).first; - } - - /** - * Get the id of a tag - * */ - TagId LocalGetTagId(const std::string &tag_name); - DEFINE_RPC(TagId, GetTagId, 0, STRING_HASH_LAMBDA) - - /** - * Get the name of a tag - * */ - RPC std::string LocalGetTagName(TagId tag); - DEFINE_RPC(std::string, GetTagName, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Rename a tag - * */ - RPC bool LocalRenameTag(TagId tag, const std::string &new_name); - DEFINE_RPC(bool, RenameTag, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Delete a tag - * */ - bool LocalDestroyTag(TagId tag); - DEFINE_RPC(bool, DestroyTag, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA) - - /** - * Add a blob to a tag index. - * */ - Status LocalTagAddBlob(TagId tag_id, BlobId blob_id); - DEFINE_RPC(Status, TagAddBlob, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA); - - /** - * Remove a blob from a tag index. - * */ - Status LocalTagRemoveBlob(TagId tag_id, BlobId blob_id); - DEFINE_RPC(Status, TagRemoveBlob, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA); - - /** - * Find all blobs pertaining to a tag - * */ - std::vector LocalGroupByTag(TagId tag_id); - DEFINE_RPC(std::vector, GroupByTag, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA); - - /** - * Add a trait to a tag index. Create tag if it does not exist. - * */ - RPC bool LocalTagAddTrait(TagId tag_id, TraitId trait_id); - DEFINE_RPC(bool, TagAddTrait, 0, UNIQUE_ID_TO_NODE_ID_LAMBDA); - - /** - * Find all traits pertaining to a tag - * */ - std::vector LocalTagGetTraits(TagId tag_id); - DEFINE_RPC(std::vector, TagGetTraits, 0, - UNIQUE_ID_TO_NODE_ID_LAMBDA); - - /**==================================== - * Trait Operations - * ===================================*/ - - /** - * Register a trait. Stores the state of the trait in a way that can - * be queried by all processes. - * */ - RPC TraitId LocalRegisterTrait(TraitId trait_id, - const hshm::charbuf &trait_params); - DEFINE_RPC(TraitId, RegisterTrait, 1, TRAIT_PARAMS_HASH_LAMBDA); - - /** - * Get trait identifier - * */ - RPC TraitId LocalGetTraitId(const std::string &trait_uuid); - DEFINE_RPC((TraitId), GetTraitId, 0, STRING_HASH_LAMBDA); - - /** - * Get trait parameters - * */ - RPC hshm::charbuf LocalGetTraitParams(TraitId trait_id); - DEFINE_RPC((hshm::charbuf), - GetTraitParams, 0, - UNIQUE_ID_TO_NODE_ID_LAMBDA); - - /** - * Get an existing trait. Will load the trait into this process's - * address space if necessary. - * */ - Trait* GlobalGetTrait(TraitId trait_id); - - /**==================================== - * Statistics Operations - * ===================================*/ - - /** Add an I/O statistic to the internal log */ - void AddIoStat(TagId tag_id, BlobId blob_id, size_t blob_size, IoType type); - - /**==================================== - * Private Operations - * ===================================*/ - - private: - /** Acquire the external lock to Bucket or Blob */ - template - bool LockMdObject(hipc::unordered_map &map, - IdT id, - MdLockType lock_type) { - auto iter = map.find(id); - if (iter == map.end()) { - return false; - } - hipc::pair &info = *iter; - MapSecond &obj_info = info.GetSecond(); - switch (lock_type) { - case MdLockType::kExternalRead: { - obj_info.lock_[1].ReadLock(0); - return true; - } - case MdLockType::kExternalWrite: { - obj_info.lock_[1].WriteLock(0); - return true; - } - default: { - return false; - } - } - } - - /** Release the external lock to Bucket or Blob */ - template - bool UnlockMdObject(hipc::unordered_map &map, - IdT id, - MdLockType lock_type) { - auto iter = map.find(id); - if (iter == map.end()) { - return false; - } - hipc::pair &info = *iter; - MapSecond &obj_info = info.GetSecond(); - switch (lock_type) { - case MdLockType::kExternalRead: { - obj_info.lock_[1].ReadUnlock(); - return true; - } - case MdLockType::kExternalWrite: { - obj_info.lock_[1].WriteUnlock(); - return true; - } - default: { - return false; - } - } - } - - /**==================================== - * Debugging Logs - * ===================================*/ - public: - void PrintDeviceInfo() { - int id = 0; - for (DeviceInfo &dev_info : *devices_) { - HILOG(kInfo, "Device {} is mounted on {} with capacity {} bytes", - id, - dev_info.mount_point_->str(), - dev_info.capacity_) - ++id; - } - } -}; - -} // namespace hermes - -#endif // HERMES_SRC_METADATA_MANAGER_H_ diff --git a/src/metadata_types.cc b/src/metadata_types.cc deleted file mode 100644 index 724a3f1d8..000000000 --- a/src/metadata_types.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "metadata_types.h" -#include "buffer_pool.h" -#include "metadata_manager.h" -#include "hermes.h" - -namespace hermes { - -void BlobInfo::shm_destroy_main() { - auto &bpm = HERMES->bpm_; - auto buffers_std = (*buffers_).vec(); - bpm.GlobalReleaseBuffers(buffers_std); - (*name_).shm_destroy(); - (*buffers_).shm_destroy(); - (*tags_).shm_destroy(); -} - -void TagInfo::shm_destroy_main() { - auto mdm = &HERMES->mdm_; - (*name_).shm_destroy(); - if (owner_) { - for (BlobId &blob_id : *blobs_) { - mdm->GlobalDestroyBlob(tag_id_, blob_id); - } - } - (*blobs_).shm_destroy(); - (*traits_).shm_destroy(); -} - -} // namespace hermes diff --git a/src/metadata_types.h b/src/metadata_types.h deleted file mode 100644 index 461b3c163..000000000 --- a/src/metadata_types.h +++ /dev/null @@ -1,395 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_METADATA_TYPES_H_ -#define HERMES_SRC_METADATA_TYPES_H_ - -#include "hermes_types.h" -#include "config_server.h" - -namespace hermes { - -#define TEXT_HASH(text) \ - std::hash{}(text); -#define ID_HASH(id) \ - id.bits_.node_id_ -#define BUCKET_HASH TEXT_HASH(bkt_name) -#define BLOB_HASH TEXT_HASH(blob_name) - -using api::Blob; /**< Namespace simplification for blob */ -struct TagInfo; /**< Forward declaration of TagInfo */ -struct BlobInfo; /**< Forward declaration of BlobInfo */ - -/** Any statistics which need to be globally maintained across ranks */ -struct GlobalIoClientState { - size_t true_size_; -}; - -/** Device information (e.g., path) */ -using config::DeviceInfo; -/** I/O interface to use for BORG (e.g., POSIX) */ -using config::IoInterface; - -/** Lock type used for internal metadata */ -enum class MdLockType { - kInternalRead, /**< Internal read lock used by Hermes */ - kInternalWrite, /**< Internal write lock by Hermes */ - kExternalRead, /**< External is used by programs */ - kExternalWrite, /**< External is used by programs */ -}; - -/** Represents the current status of a target */ -struct TargetInfo { - TargetId id_; /**< unique Target ID */ - size_t max_cap_; /**< maximum capacity of the target */ - size_t rem_cap_; /**< remaining capacity of the target */ - double bandwidth_; /**< the bandwidth of the device */ - double latency_; /**< the latency of the device */ - float score_; /**< Relative importance of this tier */ - - /** Default constructor */ - TargetInfo() = default; - - /** Primary constructor */ - TargetInfo(TargetId id, size_t max_cap, size_t rem_cap, - double bandwidth, double latency) - : id_(id), max_cap_(max_cap), rem_cap_(rem_cap), - bandwidth_(bandwidth), latency_(latency) {} -}; - -/** Represents an allocated fraction of a target */ -struct BufferInfo { - TargetId tid_; /**< The destination target */ - int t_slab_; /**< The index of the slab in the target */ - size_t t_off_; /**< Offset in the target */ - size_t t_size_; /**< Size in the target */ - size_t blob_off_; /**< Offset in the blob */ - size_t blob_size_; /**< The amount of the blob being placed */ - - /** Default constructor */ - BufferInfo() = default; - - /** Primary constructor */ - BufferInfo(TargetId tid, size_t t_off, size_t t_size, - size_t blob_off, size_t blob_size) - : tid_(tid), t_off_(t_off), t_size_(t_size), - blob_off_(blob_off), blob_size_(blob_size) {} - - /** Copy constructor */ - BufferInfo(const BufferInfo &other) { - Copy(other); - } - - /** Move constructor */ - BufferInfo(BufferInfo &&other) { - Copy(other); - } - - /** Copy assignment */ - BufferInfo& operator=(const BufferInfo &other) { - Copy(other); - return *this; - } - - /** Move assignment */ - BufferInfo& operator=(BufferInfo &&other) { - Copy(other); - return *this; - } - - /** Performs move/copy */ - void Copy(const BufferInfo &other) { - tid_ = other.tid_; - t_slab_ = other.t_slab_; - t_off_ = other.t_off_; - t_size_ = other.t_size_; - blob_off_ = other.blob_off_; - blob_size_ = other.blob_size_; - } -}; - -/** Represents BlobInfo in shared memory */ -struct BlobInfo : public hipc::ShmContainer { - SHM_CONTAINER_TEMPLATE(BlobInfo, BlobInfo) - - BlobId blob_id_; /**< The identifier of this blob */ - TagId tag_id_; /**< The bucket containing the blob */ - hipc::ShmArchive - name_; /**< SHM pointer to name string */ - hipc::ShmArchive> - buffers_; /**< SHM pointer to BufferInfo vector */ - hipc::ShmArchive> - tags_; /**< SHM pointer to tag list */ - RwLock lock_[2]; /**< Ensures BlobInfo access is synchronized */ - size_t blob_size_; /**< The overall size of the blob */ - float score_; /**< The priority of this blob */ - std::atomic access_freq_; /**< Number of times blob accessed in epoch */ - u64 last_access_; /**< Last time blob accessed */ - std::atomic mod_count_; /**< The number of times blob modified */ - std::atomic last_flush_; /**< The last mod that was flushed */ - - /**==================================== - * Default Constructor - * ===================================*/ - - /** Initialize the data structure */ - explicit BlobInfo(hipc::Allocator *alloc) { - shm_init_container(alloc); - HSHM_MAKE_AR0(name_, alloc) - HSHM_MAKE_AR0(buffers_, alloc) - HSHM_MAKE_AR0(tags_, alloc) - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor. From DeviceInfo. */ - explicit BlobInfo(hipc::Allocator *alloc, - const BlobInfo &other) { - shm_init_container(alloc); - shm_strong_copy_constructor_main(alloc, other); - } - - /** Copy the header POD types */ - void strong_copy(const BlobInfo &other) { - blob_id_ = other.blob_id_; - tag_id_ = other.tag_id_; - blob_size_ = other.blob_size_; - score_ = other.score_; - mod_count_ = other.mod_count_.load(); - last_flush_ = other.last_flush_.load(); - } - - /** Copy constructor main. */ - void shm_strong_copy_constructor_main(hipc::Allocator *alloc, - const BlobInfo &other) { - strong_copy(other); - HSHM_MAKE_AR(name_, alloc, *other.name_) - HSHM_MAKE_AR(buffers_, alloc, *other.buffers_) - HSHM_MAKE_AR(tags_, alloc, *other.tags_) - } - - /** SHM copy assignment operator. From DeviceInfo. */ - BlobInfo& operator=(const BlobInfo &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_op_main(other); - } - return *this; - } - - /** Copy assignment operator main. */ - void shm_strong_copy_op_main(const BlobInfo &other) { - strong_copy(other); - (*name_) = (*other.name_); - (*buffers_) = (*other.buffers_); - (*tags_) = (*other.tags_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - BlobInfo(hipc::Allocator *alloc, BlobInfo &&other) { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - HSHM_MAKE_AR(name_, alloc, std::move(*other.name_)) - HSHM_MAKE_AR(buffers_, alloc, std::move(*other.buffers_)) - HSHM_MAKE_AR(tags_, alloc, std::move(*other.tags_)) - other.SetNull(); - } else { - shm_strong_copy_constructor_main(alloc, other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - BlobInfo& operator=(BlobInfo &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - (*name_) = std::move(*other.name_); - (*buffers_) = std::move(*other.buffers_); - (*tags_) = std::move(*other.tags_); - other.SetNull(); - } else { - shm_strong_copy_op_main(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Whether the TagInfo is NULL */ - bool IsNull() { return false; } - - /** Set the TagInfo to NULL */ - void SetNull() {} - - /** Destroy all allocated data */ - void shm_destroy_main(); - - /**==================================== - * Statistics - * ===================================*/ - - void UpdateWriteStats() { - mod_count_.fetch_add(1); - UpdateReadStats(); - } - - void UpdateReadStats() { - last_access_ = GetTimeFromStartNs(); - access_freq_.fetch_add(1); - } - - static u64 GetTimeFromStartNs() { - struct timespec currentTime; - clock_gettime(CLOCK_MONOTONIC, ¤tTime); - unsigned long long nanoseconds = - currentTime.tv_sec * 1000000000ULL + currentTime.tv_nsec; - return nanoseconds; - } -}; - -/** Represents TagInfo in shared memory */ -struct TagInfo : public hipc::ShmContainer { - SHM_CONTAINER_TEMPLATE(TagInfo, TagInfo) - TagId tag_id_; /**< ID of the tag */ - hipc::ShmArchive - name_; /**< Archive of tag name */ - hipc::ShmArchive> - blobs_; /**< Archive of blob list */ - hipc::ShmArchive> - traits_; /**< Archive of trait list */ - size_t internal_size_; /**< Current bucket size */ - bool owner_; /**< Whether this tag owns the blobs */ - RwLock lock_[2]; /**< Lock the bucket */ - - /**==================================== - * Default Constructor - * ===================================*/ - - /** SHM constructor. Default. */ - explicit TagInfo(hipc::Allocator *alloc) { - shm_init_container(alloc); - HSHM_MAKE_AR0(name_, alloc) - HSHM_MAKE_AR0(blobs_, alloc) - HSHM_MAKE_AR0(traits_, alloc) - } - - /**==================================== - * Copy Constructors - * ===================================*/ - - /** SHM copy constructor. From DeviceInfo. */ - explicit TagInfo(hipc::Allocator *alloc, - const TagInfo &other) { - shm_init_container(alloc); - shm_strong_copy_constructor_main(alloc, other); - } - - /** Copy POD types */ - void strong_copy(const TagInfo &other) { - internal_size_ = other.internal_size_; - owner_ = other.owner_; - } - - /** Copy constructor main. */ - void shm_strong_copy_constructor_main(hipc::Allocator *alloc, - const TagInfo &other) { - strong_copy(other); - HSHM_MAKE_AR(name_, alloc, *other.name_) - HSHM_MAKE_AR(blobs_, alloc, *other.blobs_) - HSHM_MAKE_AR(traits_, alloc, *other.traits_) - } - - /** SHM copy assignment operator. From DeviceInfo. */ - TagInfo& operator=(const TagInfo &other) { - if (this != &other) { - shm_destroy(); - shm_strong_copy_op_main(other); - } - return *this; - } - - /** Copy assignment operator main. */ - void shm_strong_copy_op_main(const TagInfo &other) { - strong_copy(other); - (*name_) = (*other.name_); - (*blobs_) = (*other.blobs_); - (*traits_) = (*other.traits_); - } - - /**==================================== - * Move Constructors - * ===================================*/ - - /** SHM move constructor. */ - TagInfo(hipc::Allocator *alloc, - TagInfo &&other) { - shm_init_container(alloc); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - HSHM_MAKE_AR(name_, alloc, std::move(*other.name_)) - HSHM_MAKE_AR(blobs_, alloc, std::move(*other.blobs_)) - HSHM_MAKE_AR(traits_, alloc, std::move(*other.traits_)) - other.SetNull(); - } else { - shm_strong_copy_constructor_main(alloc, other); - other.shm_destroy(); - } - } - - /** SHM move assignment operator. */ - TagInfo& operator=(TagInfo &&other) noexcept { - if (this != &other) { - shm_destroy(); - if (GetAllocator() == other.GetAllocator()) { - strong_copy(other); - (*name_) = std::move(*other.name_); - (*blobs_) = std::move(*other.blobs_); - (*traits_) = std::move(*other.traits_); - other.SetNull(); - } else { - shm_strong_copy_op_main(other); - other.shm_destroy(); - } - } - return *this; - } - - /**==================================== - * Destructor - * ===================================*/ - - /** Whether the TagInfo is NULL */ - bool IsNull() { return false; } - - /** Set the TagInfo to NULL */ - void SetNull() {} - - /** Destroy all allocated data */ - void shm_destroy_main(); -}; - -} // namespace hermes - -#endif // HERMES_SRC_METADATA_TYPES_H_ diff --git a/src/prefetcher.cc b/src/prefetcher.cc deleted file mode 100644 index 364d31ead..000000000 --- a/src/prefetcher.cc +++ /dev/null @@ -1,98 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "prefetcher.h" -#include "hermes.h" -#include "prefetcher_factory.h" -#include - -namespace hermes { - -/** Initialize each candidate prefetcher, including trace info */ -void Prefetcher::Init() { - mdm_ = &HERMES->mdm_; - rpc_ = &HERMES->rpc_; - borg_ = &HERMES->borg_; - auto conf = HERMES->server_config_; - - // Make sure that prefetcher is enabled - is_enabled_ = conf.prefetcher_.enabled_; - mdm_->enable_io_tracing_ = is_enabled_; - if (!is_enabled_) { - return; - } - - // Info needed per-client and server - mdm_->is_mpi_ = conf.prefetcher_.is_mpi_; - if (HERMES->mode_ == HermesType::kClient) { - // NOTE(llogan): prefetcher runs only on daemon as thread - return; - } - - // Create the binary log - if (conf.prefetcher_.trace_path_.empty()) { - log_.Init("", MEGABYTES(64)); - } else { - log_.Init(conf.prefetcher_.trace_path_ + std::to_string(rpc_->node_id_), - MEGABYTES(64)); - } - - // Set the epoch - epoch_ms_ = (double)conf.prefetcher_.epoch_ms_; - - // Spawn the prefetcher thread - auto prefetcher = [](void *args) { - HILOG(kDebug, "Prefetcher has started") - (void) args; - Prefetcher *prefetch = &HERMES->prefetch_; - while (HERMES_THREAD_MANAGER->Alive()) { - prefetch->Run(); - tl::thread::self().sleep(*HERMES->rpc_.server_engine_, - prefetch->epoch_ms_); - } - prefetch->log_.Flush(true); - HILOG(kDebug, "Prefetcher has stopped") - }; - HERMES_THREAD_MANAGER->Spawn(prefetcher); -} - -/** Finalize the prefetcher thread */ -void Prefetcher::Finalize() { -} - -/** Parse the MDM's I/O pattern log */ -void Prefetcher::Run() { - // Get the set of buckets + Ingest log - std::unordered_set tags; - IoStat entry; - while (!mdm_->io_pattern_log_->pop(entry).IsNull()) { - log_.AppendEntry(entry); - tags.emplace(entry.tag_id_); - } - - // Enact the prefetchers for each bucket - for (auto &bkt_id : tags) { - std::vector traits = HERMES->GetTraits(bkt_id); - for (auto trait : traits) { - if (trait->header_->flags_.Any(HERMES_TRAIT_PREFETCHER)) { - auto *trait_hdr = - trait->GetHeader(); - auto *policy = PrefetcherFactory::Get(trait_hdr->type_); - policy->Prefetch(borg_, log_); - } - } - } - - log_.Flush(false); -} - -} // namespace hermes diff --git a/src/prefetcher.h b/src/prefetcher.h deleted file mode 100644 index 2904d53ab..000000000 --- a/src/prefetcher.h +++ /dev/null @@ -1,57 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_PREFETCHER_H_ -#define HERMES_SRC_PREFETCHER_H_ - -#include "hermes_types.h" -#include "thread_manager.h" -#include "metadata_manager.h" -#include "rpc.h" -#include "binlog.h" -#include -#include "traits/prefetcher/prefetcher_header.h" - -namespace hermes { - -class Prefetcher { - public: - std::vector> trace_; - std::vector::iterator> trace_off_; - MetadataManager *mdm_; - RPC_TYPE *rpc_; - BufferOrganizer *borg_; - tl::engine *engine; /**< Argobots execution engine */ - double epoch_ms_; /**< Milliseconds to sleep */ - bool is_enabled_; /**< Whether the prefetcher is enabled */ - BinaryLog log_; - - public: - /** Initialize each candidate prefetcher, including trace info */ - void Init(); - - /** Finalize the prefetcher thread */ - void Finalize(); - - /** Parse the MDM's I/O pattern log */ - void Run(); -}; - -class PrefetcherPolicy { - public: - /** Utilize the I/O pattern log to make prefetching decisions */ - virtual void Prefetch(BufferOrganizer *borg, BinaryLog &log) = 0; -}; - -} // namespace hermes - -#endif // HERMES_SRC_PREFETCHER_H_ diff --git a/src/prefetcher/apriori_prefetcher.cc b/src/prefetcher/apriori_prefetcher.cc deleted file mode 100644 index e6fd3999b..000000000 --- a/src/prefetcher/apriori_prefetcher.cc +++ /dev/null @@ -1,105 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "apriori_prefetcher.h" -#include "hermes.h" - -namespace hermes { - -/** Constructor. Parse YAML schema. */ -AprioriPrefetcher::AprioriPrefetcher() { - auto &path = HERMES->server_config_.prefetcher_.apriori_schema_path_; - auto real_path = hshm::ConfigParse::ExpandPath(path); - HILOG(kDebug, "Start load apriori schema {}", real_path) - try { - YAML::Node yaml_conf = YAML::LoadFile(real_path); - ParseSchema(yaml_conf); - HILOG(kDebug, "Complete load of apriori schema {}", real_path) - } catch (std::exception &e) { - HELOG(kFatal, e.what()) - } -} - -void ParseList(std::vector &list, YAML::Node node) { - list.reserve(node.size()); - for (YAML::Node sub_node : node) { - list.emplace_back(sub_node.as()); - } -} - -/** Parse YAML schema. */ -void AprioriPrefetcher::ParseSchema(YAML::Node &schema) { - rank_info_.resize(schema.size()); - for (const auto &rank_node_pair : schema) { - const YAML::Node& rank_node = rank_node_pair.first; - const YAML::Node& rank_instrs = rank_node_pair.second; - int rank = rank_node.as(); - auto &instr_list = rank_info_[rank]; - for (YAML::Node instr_list_node : rank_instrs) { - instr_list.emplace_back(); - auto &instr = instr_list.back(); - YAML::Node op_count_range_node = instr_list_node["op_count_range"]; - instr.min_op_count_ = op_count_range_node[0].as(); - instr.max_op_count_ = op_count_range_node[1].as(); - for (YAML::Node instr_node : instr_list_node["prefetch"]) { - instr.promotes_.emplace_back(); - auto &promote = instr.promotes_.back(); - promote.bkt_name_ = instr_node["bucket"].as(); - ParseList(promote.promote_, instr_node["promote_blobs"]); - ParseList(promote.demote_, instr_node["demote_blobs"]); - } - } - } -} - -/** Prefetch based on YAML schema */ -void AprioriPrefetcher::Prefetch(BufferOrganizer *borg, - BinaryLog &log) { - for (size_t rank = 0; rank < rank_info_.size(); ++rank) { - size_t num_ops = log.GetRankLogSize((int)rank); - auto &instr_list = rank_info_[rank]; - - // Find the instruction to execute for this rank - auto begin = instr_list.begin(); - auto cur = begin; - for (; cur != instr_list.end(); ++cur) { - auto &instr = *cur; - if (instr.min_op_count_ <= num_ops && instr.max_op_count_ <= num_ops) { - break; - } - } - - // First, demote blobs - if (cur != instr_list.end()) { - auto &instr = *cur; - for (auto &promote_instr : instr.promotes_) { - for (auto &blob_name : promote_instr.demote_) { - borg->GlobalOrganizeBlob(promote_instr.bkt_name_, - blob_name, 0); - } - } - - // Next, promote blobs - for (auto &promote_instr : instr.promotes_) { - for (auto &blob_name : promote_instr.promote_) { - borg->GlobalOrganizeBlob(promote_instr.bkt_name_, - blob_name, 1); - } - } - } - - // Erase unneeded logs - instr_list.erase(begin, cur); - } -} - -} // namespace hermes diff --git a/src/prefetcher/apriori_prefetcher.h b/src/prefetcher/apriori_prefetcher.h deleted file mode 100644 index ef5ec3e4b..000000000 --- a/src/prefetcher/apriori_prefetcher.h +++ /dev/null @@ -1,53 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_PREFETCHER_APRIORI_PREFETCHER_H_ -#define HERMES_SRC_PREFETCHER_APRIORI_PREFETCHER_H_ - -#include "prefetcher.h" -#include - -namespace hermes { - -struct AprioriPromoteInstr { - std::string bkt_name_; - std::vector promote_; - std::vector demote_; -}; - -struct AprioriPrefetchInstr { - size_t min_op_count_; - size_t max_op_count_; - std::vector promotes_; -}; - -class AprioriPrefetcher : public PrefetcherPolicy { - public: - std::vector> rank_info_; - - public: - /** Constructor. Parse YAML schema. */ - AprioriPrefetcher(); - - /** Parse YAML config */ - void ParseSchema(YAML::Node &schema); - - /** Destructor. */ - virtual ~AprioriPrefetcher() = default; - - /** Prefetch based on YAML schema */ - void Prefetch(BufferOrganizer *borg, BinaryLog &log); -}; - -} // namespace hermes - -#endif // HERMES_SRC_PREFETCHER_APRIORI_PREFETCHER_H_ diff --git a/src/prefetcher_factory.h b/src/prefetcher_factory.h deleted file mode 100644 index 82e9371b5..000000000 --- a/src/prefetcher_factory.h +++ /dev/null @@ -1,50 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_PREFETCHER_FACTORY_H_ -#define HERMES_SRC_PREFETCHER_FACTORY_H_ - -#include "prefetcher.h" -#include "prefetcher/apriori_prefetcher.h" - -namespace hermes { - -using hermes::PrefetcherType; - -/** - * A class to represent Data Placement Engine Factory - * */ -class PrefetcherFactory { - public: - /** - * return a pointer to prefetcher policy given a policy type. - * This uses factory pattern. - * - * @param[in] type a prefetcher policy type - * @return pointer to PrefetcherPolicy - */ - static PrefetcherPolicy* Get(const PrefetcherType &type) { - switch (type) { - case PrefetcherType::kApriori: { - return hshm::EasySingleton::GetInstance(); - } - default: { - HELOG(kFatal, "PlacementPolicy not implemented") - return NULL; - } - } - } -}; - -} // namespace hermes - -#endif // HERMES_SRC_PREFETCHER_FACTORY_H_ diff --git a/src/rpc.cc b/src/rpc.cc deleted file mode 100644 index 7403a9a14..000000000 --- a/src/rpc.cc +++ /dev/null @@ -1,210 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "rpc.h" -#include "hermes.h" -#include -#include -#include -#include -#include -#include - -namespace hermes { - -/** parse hostfile */ -std::vector RpcContext::ParseHostfile(const std::string &path) { - std::vector hosts; - std::ifstream file(path); - if (file.is_open()) { - std::string line; - while (std::getline(file, line)) { - hshm::ConfigParse::ParseHostNameString(line, hosts); - } - file.close(); - } else { - HELOG(kFatal, "Could not open the hostfile: {}", path) - } - return hosts; -} - -/** - * initialize host info list - * Requires the MetadataManager to be initialized. - * */ -void RpcContext::InitRpcContext() { - config_ = &HERMES->server_config_; - mdm_ = &HERMES->mdm_; - port_ = config_->rpc_.port_; - mode_ = HERMES->mode_; - if (hosts_.size()) { return; } - // Uses hosts produced by host_names - auto &hosts = config_->rpc_.host_names_; - // Load hosts from hostfile - if (!config_->rpc_.host_file_.empty()) { - hosts = ParseHostfile(config_->rpc_.host_file_); - } - - // Get all host info - hosts_.reserve(hosts.size()); - for (const auto& name : hosts) { - hosts_.emplace_back(name, _GetIpAddress(name)); - } - - // Get id of current host - if (HERMES->mode_ == HermesType::kServer) { - node_id_ = _FindThisHost(); - } else { - node_id_ = mdm_->header_->node_id_; - } - if (node_id_ == 0 || node_id_ > (i32)hosts_.size()) { - HELOG(kFatal, "Couldn't identify this host.") - } -} - -/** Check if we should skip an RPC and call a function locally */ -bool RpcContext::ShouldDoLocalCall(i32 node_id) { - switch (mode_) { - case HermesType::kClient: { - return false; - } - case HermesType::kServer: { - return node_id == node_id_; - } - default: { - HELOG(kFatal, "Invalid HermesType.") - exit(1); - } - } -} - -/** get RPC address */ -std::string RpcContext::GetRpcAddress(i32 node_id, int port) { - std::string result = config_->rpc_.protocol_ + "://"; - if (!config_->rpc_.domain_.empty()) { - result += config_->rpc_.domain_ + "/"; - } - std::string host_name = GetHostNameFromNodeId(node_id); - result += host_name + ":" + std::to_string(port); - return result; -} - -/** Get RPC address for this node */ -std::string RpcContext::GetMyRpcAddress() { - return GetRpcAddress(node_id_, port_); -} - -/** get host name from node ID */ -std::string RpcContext::GetHostNameFromNodeId(i32 node_id) { - // NOTE(llogan): node_id 0 is reserved as the NULL node - if (node_id <= 0 || node_id > (i32)hosts_.size()) { - HELOG(kFatal, "Attempted to get from node {}, which is out of " - "the range 1-{}", node_id, hosts_.size() + 1) - } - u32 index = node_id - 1; - return hosts_[index].hostname_; -} - -/** get host name from node ID */ -std::string RpcContext::GetIpAddressFromNodeId(i32 node_id) { - // NOTE(llogan): node_id 0 is reserved as the NULL node - if (node_id <= 0 || node_id > (i32)hosts_.size()) { - HELOG(kFatal, "Attempted to get from node {}, which is out of " - "the range 1-{}", node_id, hosts_.size() + 1) - } - u32 index = node_id - 1; - return hosts_[index].ip_addr_; -} - -/** Get RPC protocol */ -std::string RpcContext::GetProtocol() { - return config_->rpc_.protocol_; -} - -/** Get the node ID of this machine according to hostfile */ -int RpcContext::_FindThisHost() { - int node_id = 1; - for (HostInfo &host : hosts_) { - if (_IsAddressLocal(host.ip_addr_)) { - return node_id; - } - ++node_id; - } - HELOG(kFatal, "Could not identify this host"); - return -1; -} - -/** Check if an IP address is local to this machine */ -bool RpcContext::_IsAddressLocal(const std::string &addr) { - struct ifaddrs* ifAddrList = nullptr; - bool found = false; - - if (getifaddrs(&ifAddrList) == -1) { - perror("getifaddrs"); - return false; - } - - for (struct ifaddrs* ifAddr = ifAddrList; - ifAddr != nullptr; ifAddr = ifAddr->ifa_next) { - if (ifAddr->ifa_addr == nullptr || - ifAddr->ifa_addr->sa_family != AF_INET) { - continue; - } - - struct sockaddr_in* sin = - reinterpret_cast(ifAddr->ifa_addr); - char ipAddress[INET_ADDRSTRLEN] = {0}; - inet_ntop(AF_INET, &(sin->sin_addr), ipAddress, INET_ADDRSTRLEN); - - if (addr == ipAddress) { - found = true; - break; - } - } - - freeifaddrs(ifAddrList); - return found; -} - -/** Get IPv4 address from the host with "host_name" */ -std::string RpcContext::_GetIpAddress(const std::string &host_name) { - struct hostent hostname_info = {}; - struct hostent *hostname_result; - int hostname_error = 0; - char hostname_buffer[4096] = {}; -#ifdef __APPLE__ - hostname_result = gethostbyname(host_name.c_str()); - in_addr **addr_list = (struct in_addr **)hostname_result->h_addr_list; -#else - int gethostbyname_result = - gethostbyname_r(host_name.c_str(), &hostname_info, hostname_buffer, - 4096, &hostname_result, &hostname_error); - if (gethostbyname_result != 0) { - HELOG(kFatal, hstrerror(h_errno)) - } - in_addr **addr_list = (struct in_addr **)hostname_info.h_addr_list; -#endif - if (!addr_list[0]) { - HELOG(kFatal, hstrerror(h_errno)) - } - - char ip_address[INET_ADDRSTRLEN] = {0}; - const char *inet_result = - inet_ntop(AF_INET, addr_list[0], ip_address, INET_ADDRSTRLEN); - if (!inet_result) { - perror("inet_ntop"); - HELOG(kFatal, "inet_ntop failed"); - } - return ip_address; -} - -} // namespace hermes diff --git a/src/rpc.h b/src/rpc.h deleted file mode 100644 index 0e591b325..000000000 --- a/src/rpc.h +++ /dev/null @@ -1,201 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_RPC_H_ -#define HERMES_RPC_H_ - -#include -#include -#include -#include - -#include -#include - -#include "hermes_types.h" -#include "decorator.h" -#include "config_server.h" -#include "utils.h" -#include "thread_manager.h" - -namespace hermes::api { -class Hermes; -} // namespace hermes::api - -namespace hermes { - -class MetadataManager; -using api::Hermes; - -/** RPC types */ -enum class RpcType { - kThallium -}; - -/** Uniquely identify a host machine */ -struct HostInfo { - i32 node_id_; /**< Hermes-assigned node id */ - std::string hostname_; /**< Host name */ - std::string ip_addr_; /**< Host IP address */ - - HostInfo() = default; - explicit HostInfo(const std::string &hostname, const std::string &ip_addr) - : hostname_(hostname), ip_addr_(ip_addr) {} -}; - -/** A structure to represent RPC context. */ -class RpcContext { - public: - ServerConfig *config_; - MetadataManager *mdm_; - int port_; /**< port number */ - i32 node_id_; /**< the ID of this node*/ - std::vector hosts_; /**< Hostname and ip addr per-node */ - HermesType mode_; /**< The current mode hermes is executing in */ - - public: - RpcContext() = default; - - /** Parse a hostfile */ - static std::vector ParseHostfile(const std::string &path); - - /** initialize host info list */ - void InitRpcContext(); - - /** Check if we should skip an RPC and call a function locally */ - bool ShouldDoLocalCall(i32 node_id); - - /** get RPC address */ - std::string GetRpcAddress(i32 node_id, int port); - - /** Get RPC address for this node */ - std::string GetMyRpcAddress(); - - /** get host name from node ID */ - std::string GetHostNameFromNodeId(i32 node_id); - - /** get host name from node ID */ - std::string GetIpAddressFromNodeId(i32 node_id); - - /** Get RPC protocol */ - std::string GetProtocol(); - - private: - /** Get the node ID of this machine according to hostfile */ - int _FindThisHost(); - - /** Check if an IP address is local */ - bool _IsAddressLocal(const std::string &addr); - - /** Get IPv4 address from the host with "host_name" */ - std::string _GetIpAddress(const std::string &host_name); - - public: - virtual void InitServer() = 0; - virtual void InitClient() = 0; -}; - -} // namespace hermes - - -/** Decide the node to send an RPC based on a UniqueId template */ -#define UNIQUE_ID_TO_NODE_ID_LAMBDA \ - [](auto &¶m) { return param.GetNodeId(); } - -/** Decide the node to send an RPC based on a std::string */ -#define STRING_HASH_LAMBDA \ - [this](auto &¶m) { \ - auto hash = std::hash{}(param); \ - hash %= this->rpc_->hosts_.size(); \ - return hash + 1; \ - } - -/** Decide the node to send an RPC based on serialized trait parameters */ -#define TRAIT_PARAMS_HASH_LAMBDA \ - [this](auto &¶m) { \ - TraitHeader *hdr = reinterpret_cast(param.data()); \ - std::string trait_uuid = hdr->trait_uuid_; \ - auto hash = std::hash{}(trait_uuid); \ - hash %= this->rpc_->hosts_.size(); \ - return hash + 1; \ - } - -/** A helper to test RPCs in hermes */ -#ifdef HERMES_ONLY_RPC -#define NODE_ID_IS_LOCAL(node_id) false -#else -#define NODE_ID_IS_LOCAL(node_id) (node_id) == (rpc_->node_id_) -#endif - -/** - * For a function which need to be called as an RPC, - * this will create the global form of that function. - * - * E.g., let's say you have a function: - * BlobId LocalGetBlobId(std::string name) - * - * @param RET the return value of the RPC - * @param BaseName the base of the function name. E.g., GetBlobId() - * @param tuple_idx the offset of the parameter in the function prototype to - * determine where to send the RPC. E.g., 0 -> std::string name - * @param hashfn the hash function used to actually compute the node id. - * E.g., STRING_HASH_LAMBDA - * */ -#define DEFINE_RPC(RET, BaseName, tuple_idx, hashfn)\ - template\ - TYPE_UNWRAP(RET) Global##BaseName(Args&& ...args) {\ - if constexpr(std::is_same_v) {\ - _Global##BaseName(\ - hshm::make_argpack(std::forward(args)...));\ - } else {\ - return _Global##BaseName(\ - hshm::make_argpack(std::forward(args)...));\ - }\ - }\ - template\ - TYPE_UNWRAP(RET) _Global##BaseName(ArgPackT &&pack) {\ - i32 node_id = hashfn(pack.template \ - Forward()); \ - if (NODE_ID_IS_LOCAL(node_id)) {\ - if constexpr(std::is_same_v) {\ - hshm::PassArgPack::Call(\ - std::forward(pack), \ - [this](auto &&...args) constexpr {\ - this->Local##BaseName(std::forward(args)...);\ - });\ - } else {\ - return hshm::PassArgPack::Call(\ - std::forward(pack), \ - [this](auto &&...args) constexpr {\ - return this->Local##BaseName( \ - std::forward(args)...);\ - });\ - } \ - } else { \ - return hshm::PassArgPack::Call( \ - std::forward(pack), \ - [this, node_id](auto&& ...args) constexpr { \ - if constexpr(std::is_same_v) { \ - this->rpc_->Call( \ - node_id, "Rpc" #BaseName, \ - std::forward(args)...); \ - } else { \ - return this->rpc_->Call( \ - node_id, "Rpc" #BaseName, \ - std::forward(args)...); \ - } \ - }); \ - } \ - } -#include "rpc_factory.h" - -#endif // HERMES_RPC_H_ diff --git a/src/rpc_factory.h.in b/src/rpc_factory.h.in deleted file mode 100644 index 76cc7ae74..000000000 --- a/src/rpc_factory.h.in +++ /dev/null @@ -1,19 +0,0 @@ -// -// Created by lukemartinlogan on 12/4/22. -// - -#ifndef HERMES_SRC_RPC_FACTORY_H_ -#define HERMES_SRC_RPC_FACTORY_H_ - -#ifndef @CMAKE_HERMES_RPC_TYPE@ -#define @CMAKE_HERMES_RPC_TYPE@ -#endif - -#if defined(HERMES_RPC_THALLIUM) -#include "rpc_thallium.h" -#define RPC_TYPE ThalliumRpc -#else -#error "RPC implementation required (e.g., -DHERMES_RPC_THALLIUM)." -#endif - -#endif // HERMES_SRC_RPC_FACTORY_H_ diff --git a/src/rpc_thallium.cc b/src/rpc_thallium.cc deleted file mode 100644 index 32dec5a38..000000000 --- a/src/rpc_thallium.cc +++ /dev/null @@ -1,134 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include - -#include "hermes.h" -#include "metadata_manager.h" -#include "rpc_thallium.h" -#include "hermes_shm/util/singleton.h" -#include "prefetcher.h" -#include - -namespace tl = thallium; - -namespace hermes { - -/** start Thallium RPC server */ -void ThalliumRpc::InitServer() { - HILOG(kInfo, "Initializing RPC server"); - InitRpcContext(); - std::string addr = GetMyRpcAddress(); - HILOG(kInfo, "Attempting to start server on: {}", addr); - try { - server_engine_ = std::make_unique( - addr, THALLIUM_SERVER_MODE, true, config_->rpc_.num_threads_); - } catch (std::exception &e) { - HELOG(kFatal, "RPC init failed for host: {}\n{}", addr, e.what()); - } - std::string rpc_server_name = server_engine_->self(); - DefineRpcs(); - HILOG(kInfo, "Serving {} (i.e., {}) with {} RPC threads as node id {}", - rpc_server_name, - addr, - config_->rpc_.num_threads_, - node_id_); -} - -/** initialize RPC clients */ -void ThalliumRpc::InitClient() { - InitRpcContext(); - std::string protocol = GetProtocol(); - client_engine_ = std::make_unique(protocol, - THALLIUM_CLIENT_MODE, - true, 1); - HILOG(kInfo, "This client is on node {} (i.e., {})", - node_id_, GetHostNameFromNodeId(node_id_)); -} - -/** run daemon */ -void ThalliumRpc::RunDaemon() { - server_engine_->enable_remote_shutdown(); - auto prefinalize_callback = [this]() { - HILOG(kInfo, "Beginning finalization on node: {}", this->node_id_); - this->Finalize(); - HILOG(kInfo, "Finished finalization on node: {}", this->node_id_); - }; - - // TODO(llogan): add config param to do this - std::ofstream daemon_started_fs; - daemon_started_fs.open("/tmp/hermes_daemon_log.txt"); - daemon_started_fs << HERMES_SYSTEM_INFO->pid_; - daemon_started_fs.close(); - HILOG(kInfo, "Running the daemon on node: {}", node_id_); - - server_engine_->push_prefinalize_callback(prefinalize_callback); - server_engine_->wait_for_finalize(); - HILOG(kInfo, "Daemon has stopped on node: {}", node_id_); -} - -/** stop daemon (from client) */ -void ThalliumRpc::StopDaemon() { - try { - for (i32 node_id = 1; node_id < (int)hosts_.size() + 1; ++node_id) { - HILOG(kInfo, "Sending stop signal to: {}", node_id); - std::string server_name = GetServerName(node_id); - tl::endpoint server = client_engine_->lookup(server_name.c_str()); - client_engine_->shutdown_remote_engine(server); - } - } catch (std::exception &e) { - HELOG(kFatal, e.what()); - } -} - -/** get server name */ -std::string ThalliumRpc::GetServerName(i32 node_id) { - std::string ip_address = GetIpAddressFromNodeId(node_id); - return config_->rpc_.protocol_ + "://" + - std::string(ip_address) + - ":" + std::to_string(config_->rpc_.port_); -} - -/** finalize RPC context */ -void ThalliumRpc::Finalize() { - switch (mode_) { - case HermesType::kServer: { - HILOG(kInfo, "Stopping (server mode)"); - // NOTE(llogan): HERMES->Flush() is called in finalize_hermes to avoid - // margo exception. - HERMES_THREAD_MANAGER->Join(); - HERMES->prefetch_.Finalize(); - // NOTE(llogan): Don't use finalize with unique_ptr. finalize() is - // called in the destructor of the tl::enigne, and will segfault if - // called twice. - - // server_engine_->finalize() - // client_engine_->finalize() - break; - } - case HermesType::kClient: { - HILOG(kInfo, "Stopping (client mode)"); - HERMES_THREAD_MANAGER->Join(); - // NOTE(llogan): Don't use finalize with unique_ptr. finalize() is - // called in the destructor of the tl::enigne, and will segfault if - // called twice. - - // client_engine_.release(); - break; - } - default: { - throw std::logic_error("Invalid Hermes initialization type"); - } - } -} - -} // namespace hermes diff --git a/src/rpc_thallium.h b/src/rpc_thallium.h deleted file mode 100644 index 5cb5dc4a0..000000000 --- a/src/rpc_thallium.h +++ /dev/null @@ -1,170 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_RPC_THALLIUM_H_ -#define HERMES_RPC_THALLIUM_H_ - -#include -#include "rpc_thallium_serialization.h" -#include "config.h" -#include "utils.h" -#include "hermes_shm/data_structures/serialization/thallium.h" - -#include "rpc.h" - -namespace tl = thallium; - -namespace hermes { - -/** - A structure to represent Thallium state -*/ -class ThalliumRpc : public RpcContext { - public: - std::atomic kill_requested_; /**< is kill requested? */ - std::unique_ptr client_engine_; /**< pointer to client engine */ - std::unique_ptr server_engine_; /**< pointer to server engine */ - - /** initialize RPC context */ - ThalliumRpc() : RpcContext() {} - - void InitServer() override; - void InitClient() override; - void Finalize(); - void RunDaemon(); - void StopDaemon(); - std::string GetServerName(i32 node_id); - - template - void RegisterRpc(const char *name, RpcLambda &&lambda) { - server_engine_->define(name, std::forward(lambda)); - } - - /** RPC call */ - template - ReturnType Call(i32 node_id, const char *func_name, Args&&... args) { - HILOG(kDebug, "Calling {} {} -> {}", func_name, node_id_, node_id) - try { - std::string server_name = GetServerName(node_id); - tl::remote_procedure remote_proc = client_engine_->define(func_name); - tl::endpoint server = client_engine_->lookup(server_name); - HILOG(kDebug, "Found the server: {}", server_name) - if constexpr (std::is_same::value) { - remote_proc.disable_response(); - remote_proc.on(server)(std::forward(args)...); - } else { - ReturnType result = remote_proc.on(server)(std::forward(args)...); - return result; - } - } catch (tl::margo_exception &err) { - HELOG(kFatal, "Thallium failed on function: {}\n{}", - func_name, err.what()) - exit(1); - } - } - - /** I/O transfers */ - template - ReturnType IoCall(i32 node_id, const char *func_name, - IoType type, char *data, size_t size, Args&& ...args) { - HILOG(kDebug, "Calling {} {} -> {}", func_name, node_id_, node_id) - std::string server_name = GetServerName(node_id); - tl::bulk_mode flag; - switch (type) { - case IoType::kRead: { - // The "bulk" object will be modified - flag = tl::bulk_mode::write_only; - break; - } - case IoType::kWrite: { - // The "bulk" object will only be read from - flag = tl::bulk_mode::read_only; - break; - } - case IoType::kNone: { - // TODO(llogan) - HELOG(kFatal, "Cannot have none I/O type") - exit(1); - } - } - - tl::remote_procedure remote_proc = client_engine_->define(func_name); - tl::endpoint server = client_engine_->lookup(server_name); - - std::vector> segments(1); - segments[0].first = data; - segments[0].second = size; - - tl::bulk bulk = client_engine_->expose(segments, flag); - if constexpr(std::is_same_v) { - remote_proc.on(server)(bulk, std::forward(args)...); - } else { - return remote_proc.on(server)(bulk, std::forward(args)...); - } - } - - /** Io transfer at the server */ - size_t IoCallServer(const tl::request &req, tl::bulk &bulk, - IoType type, char *data, size_t size) { - tl::bulk_mode flag = tl::bulk_mode::write_only; - switch (type) { - case IoType::kRead: { - // The "local_bulk" object will only be read from - flag = tl::bulk_mode::read_only; - break; - } - case IoType::kWrite: { - // The "local_bulk" object will only be written to - flag = tl::bulk_mode::write_only; - break; - } - default: { - // NOTE(llogan): Avoids "uninitalized" warning - HELOG(kFatal, "Cannot have none I/O type") - } - } - - tl::endpoint endpoint = req.get_endpoint(); - std::vector> segments(1); - segments[0].first = data; - segments[0].second = size; - tl::bulk local_bulk = server_engine_->expose(segments, flag); - size_t io_bytes = 0; - - switch (type) { - case IoType::kRead: { - // Read from "local_bulk" to "bulk" - io_bytes = bulk.on(endpoint) << local_bulk; - break; - } - case IoType::kWrite: { - // Write to "local_bulk" from "bulk" - io_bytes = bulk.on(endpoint) >> local_bulk; - break; - } - case IoType::kNone: { - HELOG(kFatal, "Cannot have none I/O type") - } - } - if (io_bytes != size) { - HELOG(kFatal, "Failed to perform bulk I/O thallium") - } - return io_bytes; - } - - private: - void DefineRpcs(); -}; - -} // namespace hermes - -#endif // HERMES_RPC_THALLIUM_H_ diff --git a/src/rpc_thallium_defs.cc b/src/rpc_thallium_defs.cc deleted file mode 100644 index 7e647a789..000000000 --- a/src/rpc_thallium_defs.cc +++ /dev/null @@ -1,295 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "rpc_thallium.h" -#include "metadata_manager.h" -#include "rpc_thallium_serialization.h" -#include "data_structures.h" - -#include "hermes.h" - -namespace hermes { - -using thallium::request; - -void ThalliumRpc::DefineRpcs() { - RPC_CLASS_INSTANCE_DEFS_START - MetadataManager *mdm = &HERMES->mdm_; - BufferPool *bpm = &HERMES->bpm_; - BufferOrganizer *borg = &HERMES->borg_; - RPC_CLASS_INSTANCE_DEFS_END - - /**==================================== - * Bucket Operations - * ===================================*/ - - RegisterRpc("RpcGetBucketSize", [mdm](const request &req, - TagId bkt_id) { - auto ret = mdm->LocalGetBucketSize(bkt_id); - req.respond(ret); - }); - RegisterRpc("RpcSetBucketSize", [mdm](const request &req, - TagId bkt_id, - size_t new_size) { - auto ret = mdm->LocalSetBucketSize(bkt_id, new_size); - req.respond(ret); - }); - RegisterRpc("RpcLockBucket", [mdm](const request &req, - TagId bkt_id, - MdLockType lock_type) { - auto ret = mdm->LocalLockBucket(bkt_id, lock_type); - req.respond(ret); - }); - RegisterRpc("RpcUnlockBucket", [mdm](const request &req, - TagId bkt_id, - MdLockType lock_type) { - auto ret = mdm->LocalUnlockBucket(bkt_id, lock_type); - req.respond(ret); - }); - RegisterRpc("RpcClearBucket", [mdm](const request &req, - TagId bkt_id) { - auto ret = mdm->LocalClearBucket(bkt_id); - req.respond(ret); - }); - - /**==================================== - * Blob Operations - * ===================================*/ - - RegisterRpc("RpcPutBlobMetadata", [mdm](const request &req, - TagId bkt_id, - const std::string &blob_name, - size_t blob_size, - std::vector &buffers, - float score) { - auto ret = mdm->LocalPutBlobMetadata(bkt_id, blob_name, - blob_size, buffers, score); - req.respond(ret); - }); - RegisterRpc("RpcTryCreateBlob", [mdm](const request &req, - TagId bkt_id, - const std::string &blob_name) { - auto ret = mdm->LocalTryCreateBlob(bkt_id, blob_name); - req.respond(ret); - }); - RegisterRpc("RpcTagBlob", [mdm](const request &req, - BlobId blob_id, - TagId tag_id) { - auto ret = mdm->LocalTagBlob(blob_id, tag_id); - req.respond(ret); - }); - RegisterRpc("RpcBlobHasTag", [mdm](const request &req, - BlobId blob_id, - TagId tag_id) { - auto ret = mdm->LocalBlobHasTag(blob_id, tag_id); - req.respond(ret); - }); - RegisterRpc("RpcGetBlobId", [mdm](const request &req, - TagId bkt_id, - const std::string &blob_name) { - auto ret = mdm->LocalGetBlobId(bkt_id, blob_name); - req.respond(ret); - }); - RegisterRpc("RpcGetBlobName", [mdm](const request &req, - BlobId blob_id) { - auto ret = mdm->LocalGetBlobName(blob_id); - req.respond(ret); - }); - RegisterRpc("RpcGetBlobScore", [mdm](const request &req, - BlobId blob_id) { - auto ret = mdm->LocalGetBlobScore(blob_id); - req.respond(ret); - }); - RegisterRpc("RpcLockBlob", [mdm](const request &req, - BlobId blob_id, - MdLockType lock_type) { - auto ret = mdm->LocalLockBlob(blob_id, lock_type); - req.respond(ret); - }); - RegisterRpc("RpcUnlockBlob", [mdm](const request &req, - BlobId blob_id, - MdLockType lock_type) { - auto ret = mdm->LocalUnlockBlob(blob_id, lock_type); - req.respond(ret); - }); - RegisterRpc("RpcGetBlobBuffers", [mdm](const request &req, - BlobId blob_id) { - auto ret = mdm->LocalGetBlobBuffers(blob_id); - req.respond(ret); - }); - RegisterRpc("RpcRenameBlob", [mdm](const request &req, - TagId bkt_id, - BlobId blob_id, - const std::string &new_blob_name) { - auto ret = mdm->LocalRenameBlob(bkt_id, blob_id, new_blob_name); - req.respond(ret); - }); - RegisterRpc("RpcDestroyBlob", [mdm](const request &req, - TagId bkt_id, - BlobId blob_id) { - auto ret = mdm->LocalDestroyBlob(bkt_id, blob_id); - req.respond(ret); - }); - - /**==================================== - * Target Operations - * ===================================*/ - - RegisterRpc("RpcUpdateTargetCapacity", [mdm](const request &req, - TargetId tid, - off64_t offset) { - mdm->LocalUpdateTargetCapacity(tid, offset); - req.respond(true); - }); - RegisterRpc("RpcClear", [mdm](const request &req) { - mdm->LocalClear(); - req.respond(true); - }); - - /**==================================== - * Tag Operations - * ===================================*/ - - RegisterRpc("RpcGetOrCreateTag", [mdm](const request &req, - const std::string &tag_name, - bool owner, - std::vector &traits, - size_t backend_size) { - auto ret = mdm->LocalGetOrCreateTag(tag_name, owner, traits, backend_size); - req.respond(ret); - }); - RegisterRpc("RpcGetTagId", [mdm](const request &req, - std::string &tag_name) { - auto ret = mdm->LocalGetTagId(tag_name); - req.respond(ret); - }); - RegisterRpc("RpcGetTagName", [mdm](const request &req, - TagId tag) { - auto ret = mdm->LocalGetTagName(tag); - req.respond(ret); - }); - RegisterRpc("RpcRenameTag", [mdm](const request &req, - TagId tag, - const std::string &new_name) { - mdm->LocalRenameTag(tag, new_name); - req.respond(true); - }); - RegisterRpc("RpcDestroyTag", [mdm](const request &req, - TagId tag) { - mdm->LocalDestroyTag(tag); - req.respond(true); - }); - RegisterRpc("RpcTagAddBlob", [mdm](const request &req, - TagId tag_id, - BlobId blob_id) { - auto ret = mdm->LocalTagAddBlob(tag_id, blob_id); - req.respond(ret); - }); - RegisterRpc("RpcTagRemoveBlob", [mdm](const request &req, - TagId tag_id, - BlobId blob_id) { - auto ret = mdm->LocalTagRemoveBlob(tag_id, blob_id); - req.respond(ret); - }); - RegisterRpc("RpcGroupByTag", [mdm](const request &req, - TagId tag_id) { - auto ret = mdm->LocalGroupByTag(tag_id); - req.respond(ret); - }); - RegisterRpc("RpcTagAddTrait", [mdm](const request &req, - TagId tag_id, - TraitId trait_id) { - mdm->LocalTagAddTrait(tag_id, trait_id); - req.respond(true); - }); - RegisterRpc("RpcTagGetTraits", [mdm](const request &req, - TagId tag_id) { - auto ret = mdm->LocalTagGetTraits(tag_id); - req.respond(ret); - }); - - /**==================================== - * Trait Operations - * ===================================*/ - - RegisterRpc("RpcRegisterTrait", [mdm](const request &req, - TraitId trait_id, - hshm::charbuf &trait_params) { - auto ret = mdm->LocalRegisterTrait(trait_id, trait_params); - req.respond(ret); - }); - RegisterRpc("RpcGetTraitId", [mdm](const request &req, - const std::string &trait_uuid) { - auto ret = mdm->LocalGetTraitId(trait_uuid); - req.respond(ret); - }); - RegisterRpc("RpcGetTraitParams", [mdm](const request &req, - TraitId &trait_id) { - auto ret = mdm->LocalGetTraitParams(trait_id); - req.respond(ret); - }); - - /**==================================== - * Buffer Pool Operations - * ===================================*/ - - RegisterRpc("RpcReleaseBuffers", [bpm](const request &req, - std::vector &buffers) { - bpm->LocalReleaseBuffers(buffers); - req.respond(true); - }); - RegisterRpc("RpcAllocateAndSetBuffers", [this, bpm]( - const request &req, - tl::bulk &bulk, - size_t blob_size, - PlacementSchema &schema) { - hapi::Blob blob(blob_size); - this->IoCallServer(req, bulk, IoType::kWrite, blob.data(), - blob.size()); - auto ret = bpm->LocalAllocateAndSetBuffers(schema, blob); - req.respond(ret); - }); - - - /**==================================== - * Buffer Organizer Operations - * ===================================*/ - - RegisterRpc("RpcWaitForFullFlush", [borg](const request &req) { - borg->LocalWaitForFullFlush(); - req.respond(true); - }); - RegisterRpc("RpcPlaceBlobInBuffers", [this, borg]( - const request &req, - tl::bulk &bulk, - size_t blob_size, - std::vector &buffers) { - hapi::Blob blob(blob_size); - this->IoCallServer(req, bulk, IoType::kWrite, blob.data(), blob.size()); - borg->LocalPlaceBlobInBuffers(blob, buffers); - req.respond(true); - }); - RegisterRpc("RpcReadBlobFromBuffers", [this, borg]( - const request &req, - tl::bulk &bulk, - size_t size, - std::vector &buffers) { - hapi::Blob blob(size); - borg->LocalReadBlobFromBuffers(blob, buffers); - this->IoCallServer(req, bulk, IoType::kRead, blob.data(), blob.size()); - req.respond(true); - }); - - RPC_AUTOGEN_END -} - -} // namespace hermes diff --git a/src/rpc_thallium_serialization.h b/src/rpc_thallium_serialization.h deleted file mode 100644 index 07b52576a..000000000 --- a/src/rpc_thallium_serialization.h +++ /dev/null @@ -1,166 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_RPC_THALLIUM_SERIALIZATION_H_ -#define HERMES_SRC_RPC_THALLIUM_SERIALIZATION_H_ - -#include -#include -#include -#include -#include - -#include "hermes_types.h" -#include "adapter/adapter_types.h" -#include "statuses.h" -#include "metadata_types.h" -#include "data_structures.h" -#include "hermes_shm/data_structures/serialization/thallium.h" - -/** Lets thallium know how to serialize an enum */ -#define SERIALIZE_ENUM(T)\ - template \ - void save(A &ar, T &mode) {\ - int cast = static_cast(mode);\ - ar << cast;\ - }\ - template \ - void load(A &ar, T &mode) {\ - int cast;\ - ar >> cast;\ - mode = static_cast(cast);\ - } - -namespace hermes { - -/** - * Lets Thallium know how to serialize a TagId. - * - * This function is called implicitly by Thallium. - * - * @param ar An archive provided by Thallium. - * @param bucket_id The TagId to serialize. - */ -template -void serialize(A &ar, TagId &tag_id) { - ar &tag_id.unique_; - ar &tag_id.node_id_; -} - -/** - * Lets Thallium know how to serialize a BlobId. - * - * This function is called implicitly by Thallium. - * - * @param ar An archive provided by Thallium. - * @param blob_id The BlobId to serialize. - */ -template -void serialize(A &ar, BlobId &blob_id) { - ar &blob_id.unique_; - ar &blob_id.node_id_; -} - -/** - * Lets Thallium know how to serialize a TraitId. - * - * This function is called implicitly by Thallium. - * - * @param ar An archive provided by Thallium. - * @param blob_id The BlobId to serialize. - */ -template -void serialize(A &ar, TraitId &trait_id) { - ar &trait_id.unique_; - ar &trait_id.node_id_; -} - -/** - * Lets Thallium know how to serialize a TargetId. - * - * This function is called implicitly by Thallium. - * - * @param ar An archive provided by Thallium. - * @param target_id The TargetId to serialize. - */ -template -void serialize(A &ar, TargetId &target_id) { - ar &target_id.as_int_; -} - -/** - * Lets Thallium know how to serialize a TargetId. - * - * This function is called implicitly by Thallium. - * - * @param ar An archive provided by Thallium. - * @param target_id The TargetId to serialize. - */ -template -void serialize(A &ar, BufferInfo &info) { - ar &info.tid_; - ar &info.t_off_; - ar &info.t_size_; - ar &info.blob_off_; - ar &info.blob_size_; -} - -/** Lets thallium know how to serialize an MdLockType */ -SERIALIZE_ENUM(MdLockType) - -/** Lets thallium know how to serialize a SubPlacement */ -template -void serialize(A &ar, SubPlacement &plcmnt) { - ar &plcmnt.size_; - ar &plcmnt.tid_; -} - -/** Lets thallium know how to serialize a PlacementSchema */ -template -void serialize(A &ar, PlacementSchema &schema) { - ar &schema.plcmnts_; -} - -} // namespace hermes - -namespace hermes::adapter { - -/** Lets thallium know how to serialize an AdapterMode */ -SERIALIZE_ENUM(AdapterMode) - -/** Lets thallium know how to serialize an AdapterType */ -SERIALIZE_ENUM(AdapterType) - -} // namespace hermes::adapter - -namespace hermes::api { - -/** Lets thallium know how to serialize a Status */ -template -void save(A &ar, Status &status) { - ar << status.code_; -} - -/** Lets thallium know how to deserialize a Status */ -template -void load(A &ar, Status &status) { - int code; - ar >> code; - status = HERMES_STATUS->Get(code); -} - -/** Lets thallium know how to serialize a PlacementPolicy */ -SERIALIZE_ENUM(PlacementPolicy) - -} // namespace hermes::api - -#endif // HERMES_SRC_RPC_THALLIUM_SERIALIZATION_H_ diff --git a/src/statuses.h b/src/statuses.h deleted file mode 100644 index e17866eb5..000000000 --- a/src/statuses.h +++ /dev/null @@ -1,59 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_STATUSES_H_ -#define HERMES_SRC_STATUSES_H_ - -#include "status.h" - -namespace hermes { - -using api::Status; - -class StatusTable { - public: - std::vector table_; - - public: - StatusTable() { - table_.reserve(500); - Define(0, "The function was not implemented"); - Define(100, "Placement failed. Non-fatal."); - Define(102, "DPE could not find solution for the minimize I/O time DPE"); - Define(200, "Could not allocate the ram tier of storage in BPM"); - Define(300, "The read exceeds the size of the backend's data (PartialGet)"); - Define(301, "The read exceeds the size of the backend's data (PartialPut)"); - } - - void Define(int code, const char *msg) { - table_.emplace_back(code, msg); - } - - Status Get(int code) { - return table_[code]; - } -}; -#define HERMES_STATUS \ - hshm::EasySingleton::GetInstance() -#define HERMES_STATUS_T hermes::StatusTable* - -const Status NOT_IMPLEMENTED = HERMES_STATUS->Get(0); -const Status DPE_PLACEMENT_SCHEMA_EMPTY = HERMES_STATUS->Get(100); -const Status DPE_NO_SPACE = HERMES_STATUS->Get(101); -const Status DPE_MIN_IO_TIME_NO_SOLUTION = HERMES_STATUS->Get(102); -const Status BUFFER_POOL_OUT_OF_RAM = HERMES_STATUS->Get(200); -const Status PARTIAL_GET_OR_CREATE_OVERFLOW = HERMES_STATUS->Get(300); -const Status PARTIAL_PUT_OR_CREATE_OVERFLOW = HERMES_STATUS->Get(301); - -} // namespace hermes - -#endif // HERMES_SRC_STATUSES_H_ diff --git a/src/thread_manager.h b/src/thread_manager.h deleted file mode 100644 index 0f7f98ef4..000000000 --- a/src/thread_manager.h +++ /dev/null @@ -1,65 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_THREAD_MANAGER_H_ -#define HERMES_SRC_THREAD_MANAGER_H_ - -#include "hermes_types.h" -#include -#include - -namespace hermes { - -class ThreadManager { - public: - ABT_xstream execution_stream_; /**< Argobots execution stream */ - std::atomic kill_requested_; /**< Terminate threads spawned here */ - - public: - /** Default constructor */ - ThreadManager() : kill_requested_(false) { - int ret = ABT_xstream_create(ABT_SCHED_NULL, &execution_stream_); - if (ret != ABT_SUCCESS) { - HELOG(kFatal, "Could not create argobots xstream"); - } - } - - /** Spawn a for handling a function or lambda */ - template - void Spawn(FuncT &&func, ParamsT *params = nullptr) { - int ret = ABT_thread_create_on_xstream(execution_stream_, - func, (void*)params, - ABT_THREAD_ATTR_NULL, NULL); - if (ret != ABT_SUCCESS) { - HELOG(kFatal, "Couldn't spawn worker"); - } - } - - /** Begin terminating threads */ - void Join() { - kill_requested_.store(true); - ABT_xstream_join(execution_stream_); - ABT_xstream_free(&execution_stream_); - } - - /** Whether the threads in this thread manager should still be executing */ - bool Alive() { - return !kill_requested_.load(); - } -}; - -} // namespace hermes - -#define HERMES_THREAD_MANAGER \ - hshm::EasySingleton::GetInstance() - -#endif // HERMES_SRC_THREAD_MANAGER_H_ diff --git a/src/thread_pool.h b/src/thread_pool.h deleted file mode 100644 index 3e1f2a366..000000000 --- a/src/thread_pool.h +++ /dev/null @@ -1,104 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_THREAD_POOL_H_ -#define HERMES_THREAD_POOL_H_ - -#include -#include -#include -#include -#include - -namespace hermes { -/** - A class to represent thread pool - */ -class ThreadPool { - private: - std::vector threads; /**< a vector of threads */ - /** high-priority queue */ - mutable std::queue> queue_low; - /** low-priority queue */ - mutable std::queue> queue_high; - mutable std::mutex mutex; /**< mutex lock */ - mutable std::condition_variable condvar; /**< conditional variable */ - - public: - /** construct thread pool with \a num_threads number of threads */ - explicit ThreadPool( - unsigned num_threads = std::thread::hardware_concurrency()) { - while (num_threads--) { - threads.emplace_back([this] { - while (true) { - std::unique_lock lock(mutex); - condvar.wait(lock, [this]() { - return !queue_high.empty() || !queue_low.empty(); - }); - bool high_priority = !queue_high.empty(); - auto task = high_priority ? std::move(queue_high.front()) - : std::move(queue_low.front()); - if (task.valid()) { - if (high_priority) { - queue_high.pop(); - } else { - queue_low.pop(); - } - lock.unlock(); - // run the task - this cannot throw; any exception - // will be stored in the corresponding future - task(); - } else { - // an empty task is used to signal end of stream - // don't pop it off the top; all threads need to see it - break; - } - } - }); - } - } - - /** a template for running thread pool */ - template > - std::future run(F&& f, bool high_priority = false) const { - auto task = std::packaged_task(std::forward(f)); - auto future = task.get_future(); - { - std::lock_guard lock(mutex); - // conversion to packaged_task erases the return type - // so it can be stored in the queue. the future will still - // contain the correct type - if (high_priority) { - queue_high.push(std::packaged_task(std::move(task))); - } else { - queue_low.push(std::packaged_task(std::move(task))); - } - } - condvar.notify_one(); - return future; - } - - ~ThreadPool() { - // push a single empty task onto the queue and notify all threads, - // then wait for them to terminate - { - std::lock_guard lock(mutex); - queue_low.push({}); - } - condvar.notify_all(); - for (auto& thread : threads) { - thread.join(); - } - } -}; -} // namespace hermes -#endif // HERMES_THREAD_POOL_H_ diff --git a/src/trait_manager.cc b/src/trait_manager.cc deleted file mode 100644 index e89fdb9c0..000000000 --- a/src/trait_manager.cc +++ /dev/null @@ -1,87 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_shm/util/singleton.h" -#include "hermes.h" -#include "trait_manager.h" -#include - -namespace stdfs = std::filesystem; - -namespace hermes { - -void TraitManager::Init() { - ServerConfig *config = &HERMES->server_config_; - - // Load the LD_LIBRARY_PATH variable - auto ld_lib_path_env = getenv("LD_LIBRARY_PATH"); - std::string ld_lib_path; - if (ld_lib_path_env) { - ld_lib_path = ld_lib_path_env; - } - - // Load the HERMES_TRAIT_PATH variable - std::string hermes_trait_path; - auto hermes_trait_path_env = getenv("HERMES_TRAIT_PATH"); - if (hermes_trait_path_env) { - hermes_trait_path = hermes_trait_path_env; - } - - // Combine LD_LIBRARY_PATH and HERMES_TRAIT_PATH - std::string paths = hermes_trait_path + ":" + ld_lib_path; - - // Find each trait in LD_LIBRARY_PATH - for (std::string &trait_rel_path : config->trait_paths_) { - if (!FindLoadTrait(paths, trait_rel_path)) { - HELOG(kWarning, "Failed to load the trait: {}", trait_rel_path); - } - } -} - -bool TraitManager::FindLoadTrait(const std::string &paths, - const std::string &trait_rel_path) { - std::stringstream ss(paths); - std::string trait_dir; - while (std::getline(ss, trait_dir, ':')) { - std::string trait_path = hshm::Formatter::format("{}/{}", - trait_dir, - trait_rel_path); - if (!stdfs::exists(trait_path)) { - continue; - } - TraitLibInfo info; - info.lib_ = dlopen(trait_path.c_str(), RTLD_GLOBAL | RTLD_NOW); - if (!info.lib_) { - HELOG(kError, "Could not open the trait library: {}", trait_path); - return false; - } - info.create_trait_ = (create_trait_t)dlsym(info.lib_, "create_trait"); - if (!info.create_trait_) { - HELOG(kError, "The trait {} does not have create_trait symbol", - trait_path); - return false; - } - info.get_trait_id_ = (get_trait_id_t)dlsym(info.lib_, "get_trait_id"); - if (!info.get_trait_id_) { - HELOG(kError, "The trait {} does not have get_trait_id symbol", - trait_path); - return false; - } - std::string trait_name = info.get_trait_id_(); - HILOG(kDebug, "Finished loading the trait: {}", trait_name) - traits_.emplace(trait_name, std::move(info)); - return true; - } - return false; -} - -} // namespace hermes diff --git a/src/trait_manager.h b/src/trait_manager.h deleted file mode 100644 index 6c0f2394a..000000000 --- a/src/trait_manager.h +++ /dev/null @@ -1,213 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_SRC_TRAIT_MANAGER_H_ -#define HERMES_SRC_TRAIT_MANAGER_H_ - -#include -#include "hermes_types.h" - -namespace hermes { - -/** This trait is useful to the GroupBy function */ -#define HERMES_TRAIT_GROUP_BY BIT_OPT(uint32_t, 0) -/** This trait is useful to Put & Get */ -#define HERMES_TRAIT_PUT_GET BIT_OPT(uint32_t, 1) -/** This trait is useful to BORG's Flush operation */ -#define HERMES_TRAIT_FLUSH BIT_OPT(uint32_t, 2) -/** This trait is a prefetcher */ -#define HERMES_TRAIT_PREFETCHER BIT_OPT(uint32_t, 3) - -/** The basic state needed to be stored by every trait */ -struct TraitHeader { - char trait_uuid_[64]; /**< Unique name for this instance of trait */ - char trait_name_[64]; /**< The name of the trait shared object */ - bitfield32_t flags_; /**< Where the trait is useful */ - - /** Constructor. */ - explicit TraitHeader(const std::string &trait_uuid, - const std::string &trait_name, - uint32_t flags) { - memcpy(trait_uuid_, trait_uuid.c_str(), trait_uuid.size()); - memcpy(trait_name_, trait_name.c_str(), trait_name.size()); - trait_uuid_[trait_uuid.size()] = 0; - trait_name_[trait_name.size()] = 0; - flags_.SetBits(flags); - } -}; - -/** - * Represents a custom operation to perform. - * Traits are independent of Hermes. - * */ -class Trait { - public: - hshm::charbuf trait_info_; - TraitHeader *header_; - TraitId trait_id_; - std::string trait_uuid_; - - /** Default constructor */ - Trait() : trait_id_(TraitId::GetNull()) {} - - /** Virtual destructor */ - virtual ~Trait() = default; - - /** Deserialization constructor */ - explicit Trait(hshm::charbuf &trait_info) - : trait_info_(trait_info), - header_(reinterpret_cast(trait_info_.data())), - trait_id_(TraitId::GetNull()) {} - - /** Get trait ID */ - TraitId GetTraitId() { - return trait_id_; - } - - /** Get trait uuid */ - const std::string& GetTraitUuid() { - return trait_uuid_; - } - - /** Get trait class */ - bitfield32_t& GetTraitFlags() { - return header_->flags_; - } - - /** Run a method of the trait */ - virtual void Run(int method, void *params) = 0; - - /** Create the header for the trait */ - template - HeaderT* CreateHeader(const std::string &trait_uuid, - Args&& ...args) { - trait_info_ = hshm::charbuf(sizeof(HeaderT)); - trait_uuid_ = trait_uuid; - header_ = reinterpret_cast(trait_info_.data()); - hipc::Allocator::ConstructObj( - *GetHeader(), - trait_uuid, - std::forward(args)...); - return GetHeader(); - } - - /** Get the header of the trait */ - template - HeaderT* GetHeader() { - return reinterpret_cast(header_); - } - - /** - * Method to register a trait with Hermes. - * Defined automatically in HERMES_TRAIT_H - * */ - virtual void Register() = 0; -}; - -extern "C" { -/** The two methods provided by all traits */ -typedef Trait* (*create_trait_t)(hshm::charbuf &trait_info); -typedef const char* (*get_trait_id_t)(void); -} - -/** - * Used internally by trait header file - * Must be used after a "public:" - * */ -#define HERMES_TRAIT_H(TYPED_CLASS, TRAIT_ID) \ - static inline const char* trait_name_ = TRAIT_ID; \ - void Register() override { \ - if (trait_id_.IsNull()) { \ - trait_id_ = HERMES->RegisterTrait(this); \ - } \ - } - -/** Used internally by trait source file */ -#define HERMES_TRAIT_CC(TRAIT_CLASS) \ - extern "C" { \ - hermes::Trait* create_trait(hshm::charbuf &trait_info) { \ - return new TYPE_UNWRAP(TRAIT_CLASS)(trait_info); \ - } \ - const char* get_trait_id(void) { return TRAIT_CLASS::trait_name_; } \ - } - -/** All information needed to create a trait */ -struct TraitLibInfo { - void *lib_; - create_trait_t create_trait_; - get_trait_id_t get_trait_id_; - - /** Default constructor */ - TraitLibInfo() = default; - - /** Destructor */ - ~TraitLibInfo() { - if (lib_) { - dlclose(lib_); - } - } - - /** Emplace constructor */ - explicit TraitLibInfo(void *lib, - create_trait_t create_trait, - get_trait_id_t get_trait_id) - : lib_(lib), create_trait_(create_trait), get_trait_id_(get_trait_id) {} - - /** Copy constructor */ - TraitLibInfo(const TraitLibInfo &other) - : lib_(other.lib_), - create_trait_(other.create_trait_), - get_trait_id_(other.get_trait_id_) {} - - /** Move constructor */ - TraitLibInfo(TraitLibInfo &&other) noexcept - : lib_(other.lib_), - create_trait_(other.create_trait_), - get_trait_id_(other.get_trait_id_) { - other.lib_ = nullptr; - other.create_trait_ = nullptr; - other.get_trait_id_ = nullptr; - } -}; - -/** A module manager to keep track of trait binaries */ -class TraitManager { - private: - std::unordered_map traits_; - - public: - /** Identify the set of all traits and load them */ - void Init(); - - /** Construct a trait from a serialized parameter set */ - Trait* ConstructTrait(hshm::charbuf ¶ms) { - auto hdr = reinterpret_cast(params.data()); - auto iter = traits_.find(hdr->trait_name_); - if (iter == traits_.end()) { - HELOG(kError, "Could not find the trait with name: {}", - hdr->trait_name_); - return nullptr; - } - TraitLibInfo &info = iter->second; - Trait *trait = info.create_trait_(params); - return trait; - } - - private: - /** Find the trait library on disk and load it */ - bool FindLoadTrait(const std::string &paths, - const std::string &trait_rel_path); -}; - -} // namespace hermes - -#endif // HERMES_SRC_TRAIT_MANAGER_H_ diff --git a/src/utils.h b/src/utils.h deleted file mode 100644 index bbb416b3c..000000000 --- a/src/utils.h +++ /dev/null @@ -1,33 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_UTILS_H_ -#define HERMES_UTILS_H_ - -#include -#include -#include "hermes_shm/util/logging.h" - -namespace hermes { - -/** Get an environment variable with null safety. */ -static inline std::string GetEnvSafe(const char *env_name) { - char *val = getenv(env_name); - if (val == nullptr) { - return ""; - } - return val; -} - -} // namespace hermes - -#endif // HERMES_UTILS_H_ diff --git a/src/worker.cc b/src/worker.cc new file mode 100644 index 000000000..1d772e2bb --- /dev/null +++ b/src/worker.cc @@ -0,0 +1,95 @@ +// +// Created by lukemartinlogan on 6/27/23. +// + +#include "labstor/api/labstor_runtime.h" +#include "labstor/work_orchestrator/worker.h" +#include "labstor/work_orchestrator/work_orchestrator.h" + +namespace labstor { + +void Worker::Loop() { + pid_ = gettid(); + WorkOrchestrator *orchestrator = LABSTOR_WORK_ORCHESTRATOR; + while (orchestrator->IsAlive()) { + try { + Run(); + } catch (hshm::Error &e) { + e.print(); + exit(1); + } + // Yield(); + } + Run(); +} + +void Worker::Run() { + if (poll_queues_.size() > 0) { + _PollQueues(); + } + if (relinquish_queues_.size() > 0) { + _RelinquishQueues(); + } + for (WorkEntry &work_entry : work_queue_) { + if (!work_entry.lane_->flags_.Any(QUEUE_LOW_LATENCY)) { + work_entry.count_ += 1; + if (work_entry.count_ % 4096 != 0) { + continue; + } + } + PollGrouped(work_entry); + } +} + +void Worker::PollGrouped(WorkEntry &work_entry) { + Lane *lane = work_entry.lane_; + Task *task; + LaneData *entry; + int off = 0; + for (int i = 0; i < 1024; ++i) { + // Get the task message + if (lane->peek(entry, off).IsNull()) { + return; + } + if (entry->complete_) { + PopTask(lane, off); + continue; + } + task = LABSTOR_CLIENT->GetPrivatePointer(entry->p_); + // Get the task state + TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); + if (!exec) { + HELOG(kFatal, "(node {}) Could not find the task state: {}", + LABSTOR_CLIENT->node_id_, task->task_state_); + entry->complete_ = true; + EndTask(lane, task, off); + continue; + } + // Attempt to run the task if it's ready and runnable + bool is_remote = task->domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); + if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote)) { + // Execute or schedule task + if (is_remote) { + auto ids = LABSTOR_RUNTIME->ResolveDomainId(task->domain_id_); + LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); + task->DisableRun(); + task->SetUnordered(); + } else { + task->SetStarted(); + exec->Run(task->method_, task); + } + } + // Cleanup on task completion + if (task->IsModuleComplete()) { +// HILOG(kDebug, "(node {}) Ending task: task_node={} task_state={} lane={} queue={} worker={}", +// LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); + entry->complete_ = true; + RemoveTaskGroup(task, exec, work_entry.lane_id_, is_remote); + EndTask(lane, task, off); + } else { + off += 1; + } + } +} + +} // namespace labstor \ No newline at end of file diff --git a/tasks/CMakeLists.txt b/tasks/CMakeLists.txt new file mode 100644 index 000000000..1c73b9be4 --- /dev/null +++ b/tasks/CMakeLists.txt @@ -0,0 +1,8 @@ +add_subdirectory(hermes) +add_subdirectory(bdev) +add_subdirectory(ram_bdev) +add_subdirectory(posix_bdev) +add_subdirectory(hermes_mdm) +add_subdirectory(hermes_blob_mdm) +add_subdirectory(hermes_bucket_mdm) +add_subdirectory(hermes_adapters) diff --git a/tasks/bdev/CMakeLists.txt b/tasks/bdev/CMakeLists.txt new file mode 100644 index 000000000..c6f04b8f0 --- /dev/null +++ b/tasks/bdev/CMakeLists.txt @@ -0,0 +1,9 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h new file mode 100644 index 000000000..17b130b87 --- /dev/null +++ b/tasks/bdev/include/bdev/bdev.h @@ -0,0 +1,165 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_bdev_H_ +#define LABSTOR_bdev_H_ + +#include "bdev_tasks.h" + +namespace hermes::bdev { + +/** + * BDEV Client API + * */ +class Client : public TaskLibClient { + public: + DomainId domain_id_; + MonitorTask *monitor_task_; + size_t max_cap_; /**< maximum capacity of the target */ + double bandwidth_; /**< the bandwidth of the device */ + double latency_; /**< the latency of the device */ + float score_; /**< Relative importance of this tier */ + + public: + /** Copy dev info */ + void CopyDevInfo(DeviceInfo &dev_info) { + max_cap_ = dev_info.capacity_; + bandwidth_ = dev_info.bandwidth_; + latency_ = dev_info.latency_; + score_ = 1; + } + + /** Async create task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const std::string &lib_name, + DeviceInfo &dev_info) { + domain_id_ = domain_id; + id_ = TaskStateId::GetNull(); + CopyDevInfo(dev_info); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {4, 4, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, lib_name, id_, + queue_info, dev_info); + } + void AsyncCreateComplete(ConstructTask *task) { + if (task->IsComplete()) { + id_ = task->id_; + queue_id_ = QueueId(id_); + monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; + LABSTOR_CLIENT->DelTask(task); + } + } + LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + auto *task = AsyncCreateRoot(std::forward(args)...); + task->Wait(); + AsyncCreateComplete(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const std::string &state_name) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id_, id_); + monitor_task_->SetModuleComplete(); + } + + /** BDEV monitoring task */ + HSHM_ALWAYS_INLINE + void AsyncMonitorConstruct(MonitorTask *task, + const TaskNode &task_node, + size_t freq_ms) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id_, id_, freq_ms, max_cap_); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Monitor); + + /** Update bdev capacity */ + void AsyncUpdateCapacityConstruct(UpdateCapacityTask *task, + const TaskNode &task_node, + ssize_t size) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id_, id_, size); + } + LABSTOR_TASK_NODE_PUSH_ROOT(UpdateCapacity); + + /** Get bdev remaining capacity */ + HSHM_ALWAYS_INLINE + size_t GetRemCap() const { + return monitor_task_->rem_cap_; + } + + /** Allocate buffers from the bdev */ + HSHM_ALWAYS_INLINE + void AsyncAllocateConstruct(AllocateTask *task, + const TaskNode &task_node, + size_t size, + std::vector &buffers) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id_, id_, size, &buffers); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Allocate); + + /** Free data */ + HSHM_ALWAYS_INLINE + void AsyncFreeConstruct(FreeTask *task, + const TaskNode &task_node, + const std::vector &buffers, + bool fire_and_forget) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id_, id_, buffers, fire_and_forget); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Free); + + /** Allocate buffers from the bdev */ + HSHM_ALWAYS_INLINE + void AsyncWriteConstruct(WriteTask *task, + const TaskNode &task_node, + const char *data, size_t off, size_t size) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id_, id_, data, off, size); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Write); + + /** Allocate buffers from the bdev */ + HSHM_ALWAYS_INLINE + void AsyncReadConstruct(ReadTask *task, + const TaskNode &task_node, + char *data, size_t off, size_t size) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id_, id_, data, off, size); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Read); +}; + +class Server { + public: + ssize_t rem_cap_; + + public: + void UpdateCapacity(UpdateCapacityTask *task) { + rem_cap_ += task->diff_; + } + + void Monitor(MonitorTask *task) { + task->rem_cap_ = rem_cap_; + } +}; + +} // namespace hermes::bdev + +namespace hermes { +typedef bdev::Client TargetInfo; +} // namespace hermes + +#endif // LABSTOR_bdev_H_ diff --git a/tasks/bdev/include/bdev/bdev_lib_exec.h b/tasks/bdev/include/bdev/bdev_lib_exec.h new file mode 100644 index 000000000..d154585ad --- /dev/null +++ b/tasks/bdev/include/bdev/bdev_lib_exec.h @@ -0,0 +1,306 @@ +#ifndef LABSTOR_BDEV_LIB_EXEC_H_ +#define LABSTOR_BDEV_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kWrite: { + Write(reinterpret_cast(task)); + break; + } + case Method::kRead: { + Read(reinterpret_cast(task)); + break; + } + case Method::kAlloc: { + Alloc(reinterpret_cast(task)); + break; + } + case Method::kFree: { + Free(reinterpret_cast(task)); + break; + } + case Method::kMonitor: { + Monitor(reinterpret_cast(task)); + break; + } + case Method::kUpdateCapacity: { + UpdateCapacity(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kWrite: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kRead: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kAlloc: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kFree: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kMonitor: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kUpdateCapacity: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kWrite: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kRead: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kAlloc: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kFree: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kMonitor: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kUpdateCapacity: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kWrite: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRead: { + ar << *reinterpret_cast(task); + break; + } + case Method::kAlloc: { + ar << *reinterpret_cast(task); + break; + } + case Method::kFree: { + ar << *reinterpret_cast(task); + break; + } + case Method::kMonitor: { + ar << *reinterpret_cast(task); + break; + } + case Method::kUpdateCapacity: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kWrite: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kRead: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kAlloc: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kFree: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kMonitor: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kUpdateCapacity: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kWrite: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRead: { + ar << *reinterpret_cast(task); + break; + } + case Method::kAlloc: { + ar << *reinterpret_cast(task); + break; + } + case Method::kFree: { + ar << *reinterpret_cast(task); + break; + } + case Method::kMonitor: { + ar << *reinterpret_cast(task); + break; + } + case Method::kUpdateCapacity: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kWrite: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kRead: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kAlloc: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kFree: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kMonitor: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kUpdateCapacity: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kWrite: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kRead: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kAlloc: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kFree: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kMonitor: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kUpdateCapacity: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_BDEV_METHODS_H_ \ No newline at end of file diff --git a/tasks/bdev/include/bdev/bdev_methods.h b/tasks/bdev/include/bdev/bdev_methods.h new file mode 100644 index 000000000..715545386 --- /dev/null +++ b/tasks/bdev/include/bdev/bdev_methods.h @@ -0,0 +1,14 @@ +#ifndef LABSTOR_BDEV_METHODS_H_ +#define LABSTOR_BDEV_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kWrite = kLast + 0; + TASK_METHOD_T kRead = kLast + 1; + TASK_METHOD_T kAlloc = kLast + 2; + TASK_METHOD_T kFree = kLast + 3; + TASK_METHOD_T kMonitor = kLast + 4; + TASK_METHOD_T kUpdateCapacity = kLast + 5; +}; + +#endif // LABSTOR_BDEV_METHODS_H_ \ No newline at end of file diff --git a/tasks/bdev/include/bdev/bdev_methods.yaml b/tasks/bdev/include/bdev/bdev_methods.yaml new file mode 100644 index 000000000..616b06640 --- /dev/null +++ b/tasks/bdev/include/bdev/bdev_methods.yaml @@ -0,0 +1,7 @@ +kWrite: 0 +kRead: 1 +kAlloc: 2 +kFree: 3 +kMonitor: 4 +kUpdateCapacity: 5 +kLast: 6 \ No newline at end of file diff --git a/tasks/bdev/include/bdev/bdev_namespace.h b/tasks/bdev/include/bdev/bdev_namespace.h new file mode 100644 index 000000000..bebc9155a --- /dev/null +++ b/tasks/bdev/include/bdev/bdev_namespace.h @@ -0,0 +1,26 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ +#define LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ + +#include "bdev_tasks.h" +#include "bdev.h" +#include "labstor/labstor_namespace.h" + +/** The set of methods in the admin task */ +using ::hermes::bdev::Method; +using ::hermes::bdev::ConstructTask; +using ::hermes::bdev::DestructTask; +using ::hermes::bdev::AllocateTask; +using ::hermes::bdev::FreeTask; +using ::hermes::bdev::ReadTask; +using ::hermes::bdev::WriteTask; +using ::hermes::bdev::MonitorTask; +using ::hermes::bdev::UpdateCapacityTask; + +/** Create admin requests */ +using ::hermes::bdev::Client; + +#endif // LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ diff --git a/tasks/bdev/include/bdev/bdev_tasks.h b/tasks/bdev/include/bdev/bdev_tasks.h new file mode 100644 index 000000000..4837dd543 --- /dev/null +++ b/tasks/bdev/include/bdev/bdev_tasks.h @@ -0,0 +1,319 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ +#define LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "hermes/hermes_types.h" +#include "hermes/config_server.h" +#include "proc_queue/proc_queue.h" + +namespace hermes::bdev { + +#include "bdev_methods.h" +#include "labstor/labstor_namespace.h" + +/** + * A task to create hermes_mdm + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + IN DeviceInfo info_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const std::string &lib_name, + const TaskStateId &id, + const std::vector &queue_info, + DeviceInfo &info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + lib_name, id, queue_info) { + // Custom params + info_ = info; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy hermes_mdm */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + TaskStateId &state_id, + const DomainId &domain_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in bdev + * */ +struct AllocateTask : public Task, TaskFlags { + IN size_t size_; /**< Size in buf */ + OUT std::vector *buffers_; + OUT size_t alloc_size_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + AllocateTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + AllocateTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + size_t size, + std::vector *buffers) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kAlloc; + task_flags_.SetBits(TASK_UNORDERED); + domain_id_ = domain_id; + + // Free params + size_ = size; + buffers_ = buffers; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in bdev + * */ +struct FreeTask : public Task, TaskFlags { + IN std::vector buffers_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + FreeTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + FreeTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const std::vector &buffers, + bool fire_and_forget) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kFree; + task_flags_.SetBits(0); + if (fire_and_forget) { + task_flags_.SetBits(TASK_FIRE_AND_FORGET | TASK_UNORDERED); + } + domain_id_ = domain_id; + + // Free params + buffers_ = buffers; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in bdev + * */ +struct WriteTask : public Task, TaskFlags { + IN const char *buf_; /**< Data in memory */ + IN size_t disk_off_; /**< Offset on disk */ + IN size_t size_; /**< Size in buf */ + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + WriteTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + WriteTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const char *buf, + size_t disk_off, + size_t size) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kWrite; + task_flags_.SetBits(TASK_UNORDERED); + domain_id_ = domain_id; + + // Free params + buf_ = buf; + disk_off_ = disk_off; + size_ = size; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in bdev + * */ +struct ReadTask : public Task, TaskFlags { + IN char *buf_; /**< Data in memory */ + IN size_t disk_off_; /**< Offset on disk */ + IN size_t size_; /**< Size in disk buf */ + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ReadTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ReadTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + char *buf, + size_t disk_off, + size_t size) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kRead; + task_flags_.SetBits(TASK_UNORDERED); + domain_id_ = domain_id; + + // Free params + buf_ = buf; + disk_off_ = disk_off; + size_ = size; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to monitor bdev statistics */ +struct MonitorTask : public Task, TaskFlags { + IN size_t freq_ms_; /**< Frequency in ms */ + OUT size_t rem_cap_; /**< Remaining capacity of the target */ + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + MonitorTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + MonitorTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + size_t freq_ms, + size_t rem_cap) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLongRunning; + task_state_ = state_id; + method_ = Method::kMonitor; + task_flags_.SetBits(TASK_LONG_RUNNING); + domain_id_ = domain_id; + + // Custom + freq_ms_ = freq_ms; + rem_cap_ = rem_cap; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to update bdev capacity */ +struct UpdateCapacityTask : public Task, TaskFlags { + IN ssize_t diff_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + UpdateCapacityTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + UpdateCapacityTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + ssize_t diff) : Task(alloc) { + // Initialize task + task_node_ = task_node; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kUpdateCapacity; + task_flags_.SetBits(TASK_FIRE_AND_FORGET | TASK_UNORDERED); + domain_id_ = domain_id; + + // Custom + diff_ = diff; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace hermes::bdev + +#endif // LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ diff --git a/tasks/hermes/CMakeLists.txt b/tasks/hermes/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks/hermes/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/config/hermes_client_default.yaml b/tasks/hermes/config/hermes_client_default.yaml similarity index 100% rename from config/hermes_client_default.yaml rename to tasks/hermes/config/hermes_client_default.yaml diff --git a/config/hermes_server_default.yaml b/tasks/hermes/config/hermes_server_default.yaml similarity index 100% rename from config/hermes_server_default.yaml rename to tasks/hermes/config/hermes_server_default.yaml diff --git a/adapter/adapter_types.h b/tasks/hermes/include/hermes/adapter_types.h similarity index 99% rename from adapter/adapter_types.h rename to tasks/hermes/include/hermes/adapter_types.h index a13ad2d5a..faf3486da 100644 --- a/adapter/adapter_types.h +++ b/tasks/hermes/include/hermes/adapter_types.h @@ -80,7 +80,6 @@ class AdapterModeConv { } }; - } // namespace hermes::adapter #endif // HERMES_ADAPTER_ADAPTER_TYPES_H_ diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h new file mode 100644 index 000000000..241faf477 --- /dev/null +++ b/tasks/hermes/include/hermes/bucket.h @@ -0,0 +1,428 @@ +// +// Created by lukemartinlogan on 7/9/23. +// + +#ifndef LABSTOR_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ +#define LABSTOR_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ + +#include "hermes/hermes_types.h" +#include "hermes_mdm/hermes_mdm.h" +#include "hermes/config_manager.h" + +namespace hermes { + +#define HERMES_BUCKET_IS_FILE BIT_OPT(u32, 1) + +class Bucket { + public: + mdm::Client *mdm_; + blob_mdm::Client *blob_mdm_; + bucket_mdm::Client *bkt_mdm_; + TagId id_; + std::string name_; + Context ctx_; + bitfield32_t flags_; + + public: + /**==================================== + * Bucket Operations + * ===================================*/ + + /** + * Get or create \a bkt_name bucket. + * + * Called from hermes.h in GetBucket(). Should not + * be used directly. + * */ + explicit Bucket(const std::string &bkt_name, + size_t backend_size = 0, + u32 flags = 0) { + mdm_ = &HERMES_CONF->mdm_; + blob_mdm_ = &HERMES_CONF->blob_mdm_; + bkt_mdm_ = &HERMES_CONF->bkt_mdm_; + id_ = bkt_mdm_->GetOrCreateTagRoot(hshm::charbuf(bkt_name), true, + std::vector(), backend_size, flags); + name_ = bkt_name; + } + + /** + * Get or create \a bkt_name bucket. + * + * Called from hermes.h in GetBucket(). Should not + * be used directly. + * */ + explicit Bucket(const std::string &bkt_name, + Context &ctx, + size_t backend_size = 0, + u32 flags = 0) { + mdm_ = &HERMES_CONF->mdm_; + blob_mdm_ = &HERMES_CONF->blob_mdm_; + bkt_mdm_ = &HERMES_CONF->bkt_mdm_; + id_ = bkt_mdm_->GetOrCreateTagRoot(hshm::charbuf(bkt_name), true, + std::vector(), backend_size, flags); + name_ = bkt_name; + } + + /** + * Get an existing bucket. + * */ + explicit Bucket(TagId tag_id) { + id_ = tag_id; + mdm_ = &HERMES_CONF->mdm_; + blob_mdm_ = &HERMES_CONF->blob_mdm_; + bkt_mdm_ = &HERMES_CONF->bkt_mdm_; + } + + /** Default constructor */ + Bucket() = default; + + /** Default copy constructor */ + Bucket(const Bucket &other) = default; + + /** Default copy assign */ + Bucket& operator=(const Bucket &other) = default; + + /** Default move constructor */ + Bucket(Bucket &&other) = default; + + /** Default move assign */ + Bucket& operator=(Bucket &&other) = default; + + public: + /** + * Get the name of this bucket. Name is cached instead of + * making an RPC. Not coherent if Rename is called. + * */ + const std::string& GetName() const { + return name_; + } + + /** + * Get the identifier of this bucket + * */ + TagId GetId() const { + return id_; + } + + /** + * Get the context object of this bucket + * */ + Context& GetContext() { + return ctx_; + } + + /** + * Attach a trait to the bucket + * */ + void AttachTrait(TraitId trait_id) { + // TODO(llogan) + } + + /** + * Get the current size of the bucket + * */ + size_t GetSize() { + return bkt_mdm_->GetSizeRoot(id_); + } + + /** + * Rename this bucket + * */ + void Rename(const std::string &new_bkt_name) { + bkt_mdm_->RenameTagRoot(id_, hshm::to_charbuf(new_bkt_name)); + } + + /** + * Clears the buckets contents, but doesn't destroy its metadata + * */ + void Clear() { + bkt_mdm_->TagClearBlobsRoot(id_); + } + + /** + * Destroys this bucket along with all its contents. + * */ + void Destroy() { + bkt_mdm_->DestroyTagRoot(id_); + } + + /** + * Check if this bucket is valid + * */ + bool IsNull() { + return id_.IsNull(); + } + + public: + /**==================================== + * Blob Operations + * ===================================*/ + + /** + * Get the id of a blob from the blob name + * + * @param blob_name the name of the blob + * @param blob_id (output) the returned blob_id + * @return + * */ + BlobId GetBlobId(const std::string &blob_name) { + return blob_mdm_->GetBlobIdRoot(id_, hshm::to_charbuf(blob_name)); + } + + /** + * Get the name of a blob from the blob id + * + * @param blob_id the blob_id + * @param blob_name the name of the blob + * @return The Status of the operation + * */ + std::string GetBlobName(const BlobId &blob_id) { + return blob_mdm_->GetBlobNameRoot(id_, blob_id); + } + + + /** + * Get the score of a blob from the blob id + * + * @param blob_id the blob_id + * @return The Status of the operation + * */ + float GetBlobScore(const BlobId &blob_id) { + return blob_mdm_->GetBlobScoreRoot(id_, blob_id); + } + + /** + * Label \a blob_id blob with \a tag_name TAG + * */ + Status TagBlob(BlobId &blob_id, + TagId &tag_id) { + bkt_mdm_->TagAddBlobRoot(tag_id, blob_id); + return Status(); + } + + /** + * Put \a blob_name Blob into the bucket + * */ + template + HSHM_ALWAYS_INLINE + BlobId BasePut(const std::string &blob_name, + const Blob &blob, + size_t blob_off, + const BlobId &orig_blob_id, + Context &ctx) { + BlobId blob_id = orig_blob_id; + // Copy data to shared memory + LPointer p = LABSTOR_CLIENT->AllocateBuffer(blob.size()); + char *data = p.ptr_; + memcpy(data, blob.data(), blob.size()); + // Put to shared memory + hshm::charbuf blob_name_buf = hshm::to_charbuf(blob_name); + if (blob_id.IsNull()) { + blob_id = blob_mdm_->GetOrCreateBlobIdRoot(id_, blob_name_buf); + } + HILOG(kDebug, "The bucket's ID is: {}", blob_id); + if constexpr(!PARTIAL) { + blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, + blob_id, 0, blob.size(), p.shm_, ctx.blob_score_, + bitfield32_t(HERMES_BLOB_REPLACE), + ctx); + } else { + blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, + blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, + bitfield32_t(0), + ctx); + } + return blob_id; + } + + /** + * Put \a blob_name Blob into the bucket + * */ + BlobId Put(const std::string &blob_name, + const Blob &blob, + Context &ctx) { + return BasePut(blob_name, blob, 0, BlobId::GetNull(), ctx); + } + + /** + * Put \a blob_id Blob into the bucket + * */ + BlobId Put(const BlobId &blob_id, + const Blob &blob, + Context &ctx) { + return BasePut("", blob, 0, blob_id, ctx); + } + + /** + * Put \a blob_name Blob into the bucket + * */ + BlobId PartialPut(const std::string &blob_name, + const Blob &blob, + size_t blob_off, + Context &ctx) { + return BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); + } + + /** + * Put \a blob_id Blob into the bucket + * */ + BlobId PartialPut(const BlobId &blob_id, + const Blob &blob, + size_t blob_off, + Context &ctx) { + return BasePut("", blob, blob_off, blob_id, ctx); + } + + /** + * Serialized PUT + * */ + template + BlobId Put(const std::string &blob_name, + const T &data, + Context &ctx) { + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar << data; + Blob blob(ss.str()); + return Put(blob_name, blob, ctx); + } + + /** + * Append \a blob_name Blob into the bucket + * */ + Status Append(const Blob &blob, size_t page_size, Context &ctx) { + LPointer p = LABSTOR_CLIENT->AllocateBuffer(blob.size()); + char *data = p.ptr_; + memcpy(data, blob.data(), blob.size()); + bkt_mdm_->AppendBlobRoot(id_, blob.size(), p.shm_, page_size, ctx.blob_score_, ctx.node_id_, ctx); + return Status(); + } + + /** + * Reorganize a blob to a new score or node + * */ + void ReorganizeBlob(const BlobId &blob_id, + float score, + u32 node_id, + Context &ctx) { + blob_mdm_->AsyncReorganizeBlobRoot(id_, blob_id, score, node_id); + } + + /** + * Get the current size of the blob in the bucket + * */ + size_t GetBlobSize(const BlobId &blob_id) { + return blob_mdm_->GetBlobSizeRoot(id_, blob_id); + } + + /** + * Get \a blob_id Blob from the bucket + * */ + BlobId BaseGet(const std::string &blob_name, + const BlobId &orig_blob_id, + Blob &blob, + size_t blob_off, + Context &ctx) { + BlobId blob_id = orig_blob_id; + // Get the blob ID + if (blob_id.IsNull()) { + blob_id = blob_mdm_->GetBlobIdRoot(id_, hshm::to_charbuf(blob_name)); + } + if (blob_id.IsNull()) { + if (ctx.filename_.size() == 0) { + return blob_id; + } else { + // StageIn using PUT of an empty blob + hermes::Blob emtpy_blob; + blob_id = PartialPut(blob_name, emtpy_blob, 0, ctx); + } + } + // Get from shared memory + size_t data_size = blob.size(); + if (data_size == 0) { + data_size = GetBlobSize(blob_id); + } + LPointer data_p = LABSTOR_CLIENT->AllocateBuffer(data_size); + data_size = blob_mdm_->GetBlobRoot(id_, blob_id, blob_off, data_size, data_p.shm_, ctx); + char *data = data_p.ptr_; + // Copy data to blob + // TODO(llogan): intercept mmap to avoid copy + blob.resize(data_size); + memcpy(blob.data(), data, data_size); + LABSTOR_CLIENT->FreeBuffer(data_p); + return blob_id; + } + + /** + * Get \a blob_id Blob from the bucket + * */ + BlobId Get(const std::string &blob_name, + Blob &blob, + Context &ctx) { + return BaseGet(blob_name, BlobId::GetNull(), blob, 0, ctx); + } + + /** + * Put \a blob_name Blob into the bucket + * */ + BlobId PartialGet(const std::string &blob_name, + Blob &blob, + size_t blob_off, + Context &ctx) { + return BaseGet(blob_name, BlobId::GetNull(), blob, blob_off, ctx); + } + + /** + * Get \a blob_id Blob from the bucket + * */ + BlobId Get(const BlobId &blob_id, + Blob &blob, + Context &ctx) { + return BaseGet("", blob_id, blob, 0, ctx); + } + + /** + * Put \a blob_name Blob into the bucket + * */ + BlobId PartialGet(const BlobId &blob_id, + Blob &blob, + size_t blob_off, + Context &ctx) { + return BaseGet("", blob_id, blob, blob_off, ctx); + } + + /** + * Determine if the bucket contains \a blob_id BLOB + * */ + bool ContainsBlob(const std::string &blob_name) { + BlobId new_blob_id = blob_mdm_->GetBlobIdRoot(id_, hshm::to_charbuf(blob_name)); + return !new_blob_id.IsNull(); + } + + /** + * Rename \a blob_id blob to \a new_blob_name new name + * */ + void RenameBlob(const BlobId &blob_id, std::string new_blob_name, Context &ctx) { + blob_mdm_->RenameBlobRoot(id_, blob_id, hshm::to_charbuf(new_blob_name)); + } + + /** + * Delete \a blob_id blob + * */ + void DestroyBlob(const BlobId &blob_id, Context &ctx) { + // TODO(llogan): Make apart of bkt_mdm_ instead + blob_mdm_->DestroyBlobRoot(id_, blob_id); + } + + /** + * Get the set of blob IDs contained in the bucket + * */ + std::vector GetContainedBlobIds() { + // TODO(llogan) + return {}; + } +}; + +} // namespace hermes + +#endif // LABSTOR_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ diff --git a/src/config.h b/tasks/hermes/include/hermes/config.h similarity index 90% rename from src/config.h rename to tasks/hermes/include/hermes/config.h index 7772eb657..aa18b4d94 100644 --- a/src/config.h +++ b/tasks/hermes/include/hermes/config.h @@ -20,11 +20,9 @@ #include #include #include -#include "data_structures.h" #include "hermes_shm/util/config_parse.h" -#include "hermes_types.h" -#include "utils.h" +#include "hermes/hermes_types.h" namespace hermes::config { @@ -77,4 +75,16 @@ class BaseConfig { }; } // namespace hermes::config + +namespace hermes { +/** Get an environment variable with null safety. */ +static inline std::string GetEnvSafe(const char *env_name) { + char *val = getenv(env_name); + if (val == nullptr) { + return ""; + } + return val; +} +} // namespace hermes + #endif // HERMES_CONFIG_PARSER_H_ diff --git a/src/config_client.h b/tasks/hermes/include/hermes/config_client.h similarity index 57% rename from src/config_client.h rename to tasks/hermes/include/hermes/config_client.h index 36f4f54c2..aab5db713 100644 --- a/src/config_client.h +++ b/tasks/hermes/include/hermes/config_client.h @@ -15,7 +15,8 @@ #include #include "config.h" -#include "adapter/adapter_types.h" +#include "hermes_adapters/adapter_types.h" +#include "config_client_default.h" namespace stdfs = std::filesystem; @@ -63,7 +64,9 @@ class ClientConfig : public BaseConfig { public: ClientConfig() = default; - void LoadDefault() override; + void LoadDefault() override { + LoadText(kHermesClientDefaultConfigStr, false); + } void SetBaseAdapterMode(AdapterMode mode) { base_adapter_config_.mode_ = mode; @@ -118,10 +121,81 @@ class ClientConfig : public BaseConfig { } private: - void ParseYAML(YAML::Node &yaml_conf) override; + void ParseYAML(YAML::Node &yaml_conf) override { + if (yaml_conf["stop_daemon"]) { + std::string stop_env = GetEnvSafe(Constant::kHermesStopDaemon); + if (stop_env.size() == 0) { + stop_daemon_ = yaml_conf["stop_daemon"].as(); + } else { + std::istringstream(stop_env) >> stop_daemon_; + } + } + if (yaml_conf["base_adapter_mode"]) { + std::string mode_env = GetEnvSafe(Constant::kHermesAdapterMode); + if (mode_env.size() == 0) { + base_adapter_config_.mode_ = AdapterModeConv::to_enum( + yaml_conf["base_adapter_mode"].as()); + } else { + base_adapter_config_.mode_ = AdapterModeConv::to_enum(mode_env); + } + } + if (yaml_conf["file_page_size"]) { + std::string page_size_env = GetEnvSafe(Constant::kHermesPageSize); + if (page_size_env.size() == 0) { + base_adapter_config_.page_size_ = + hshm::ConfigParse::ParseSize( + yaml_conf["file_page_size"].as()); + } else { + base_adapter_config_.page_size_ = + hshm::ConfigParse::ParseSize(page_size_env); + } + } + if (yaml_conf["path_inclusions"]) { + std::vector inclusions; + ParseVector(yaml_conf["path_inclusions"], inclusions); + for (auto &entry : inclusions) { + entry = hshm::ConfigParse::ExpandPath(entry); + SetAdapterPathTracking(std::move(entry), true); + } + } + if (yaml_conf["path_exclusions"]) { + std::vector exclusions; + ParseVector(yaml_conf["path_exclusions"], exclusions); + for (auto &entry : exclusions) { + entry = hshm::ConfigParse::ExpandPath(entry); + SetAdapterPathTracking(std::move(entry), false); + } + } + if (yaml_conf["flushing_mode"]) { + flushing_mode_ = + FlushingModeConv::GetEnum(yaml_conf["flushing_mode"].as()); + auto flush_mode_env = getenv("HERMES_FLUSH_MODE"); + if (flush_mode_env) { + flushing_mode_ = FlushingModeConv::GetEnum(flush_mode_env); + } + } + if (yaml_conf["file_adapter_configs"]) { + for (auto node : yaml_conf["file_adapter_configs"]) { + AdapterObjectConfig conf(base_adapter_config_); + ParseAdapterConfig(node, conf); + } + } + } void ParseAdapterConfig(YAML::Node &yaml_conf, - AdapterObjectConfig &conf); + AdapterObjectConfig &conf) { + std::string path = yaml_conf["path"].as(); + path = hshm::ConfigParse::ExpandPath(path); + path = stdfs::absolute(path).string(); + if (yaml_conf["mode"]) { + conf.mode_ = AdapterModeConv::to_enum(yaml_conf["mode"].as()); + } + if (yaml_conf["page_size"]) { + conf.page_size_ = hshm::ConfigParse::ParseSize( + yaml_conf["page_size"].as()); + } + SetAdapterConfig(path, conf); + } }; } // namespace hermes::config diff --git a/src/config_client_default.h b/tasks/hermes/include/hermes/config_client_default.h similarity index 58% rename from src/config_client_default.h rename to tasks/hermes/include/hermes/config_client_default.h index b8df9e778..e33d413a0 100644 --- a/src/config_client_default.h +++ b/tasks/hermes/include/hermes/config_client_default.h @@ -1,6 +1,6 @@ -#ifndef HERMES_SRC_CONFIG_CLIENT_DEFAULT_H_ -#define HERMES_SRC_CONFIG_CLIENT_DEFAULT_H_ -const char* kClientDefaultConfigStr = +#ifndef LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ +#define LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ +static inline const char* kHermesClientDefaultConfigStr = "stop_daemon: false\n" "path_inclusions: [\"/tmp/test_hermes\"]\n" "path_exclusions: [\"/\"]\n" @@ -11,4 +11,4 @@ const char* kClientDefaultConfigStr = " - path: \"/\"\n" " page_size: 1MB\n" " mode: kDefault\n"; -#endif // HERMES_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file +#endif // LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/tasks/hermes/include/hermes/config_manager.h b/tasks/hermes/include/hermes/config_manager.h new file mode 100644 index 000000000..ed41040c4 --- /dev/null +++ b/tasks/hermes/include/hermes/config_manager.h @@ -0,0 +1,80 @@ +// +// Created by lukemartinlogan on 7/9/23. +// + +#ifndef LABSTOR_TASKS_HERMES_INCLUDE_hermes_H_ +#define LABSTOR_TASKS_HERMES_INCLUDE_hermes_H_ + +#include "hermes/hermes_types.h" +#include "labstor_admin/labstor_admin.h" +#include "hermes_mdm/hermes_mdm.h" +#include "hermes_bucket_mdm/hermes_bucket_mdm.h" +#include "hermes_blob_mdm/hermes_blob_mdm.h" +#include "hermes/config_client.h" +#include "hermes/config_server.h" + +namespace hermes { + +class ConfigurationManager { + public: + mdm::Client mdm_; + bucket_mdm::Client bkt_mdm_; + blob_mdm::Client blob_mdm_; + ServerConfig server_config_; + ClientConfig client_config_; + bool is_initialized_ = false; + + public: + ConfigurationManager() = default; + + void ClientInit() { + if (is_initialized_) { + return; + } + // Create connection to MDM + std::string config_path = ""; + LoadClientConfig(config_path); + LoadServerConfig(config_path); + mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_mdm"); + blob_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_blob_mdm"); + bkt_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_bkt_mdm"); + blob_mdm_.SetBucketMdmRoot(DomainId::GetGlobal(), bkt_mdm_.id_); + bkt_mdm_.SetBlobMdmRoot(DomainId::GetGlobal(), blob_mdm_.id_); + is_initialized_ = true; + } + + void LoadClientConfig(std::string &config_path) { + // Load hermes config + if (config_path.empty()) { + config_path = GetEnvSafe(Constant::kHermesServerConf); + } + HILOG(kInfo, "Loading client configuration: {}", config_path) + client_config_.LoadFromFile(config_path); + } + + void LoadServerConfig(std::string &config_path) { + // Load hermes config + if (config_path.empty()) { + config_path = GetEnvSafe(Constant::kHermesServerConf); + } + HILOG(kInfo, "Loading server configuration: {}", config_path) + server_config_.LoadFromFile(config_path); + } +}; + +} // namespace hermes + +#define HERMES_CONF hshm::Singleton::GetInstance() +#define HERMES_CLIENT_CONF HERMES_CONF->client_config_ +#define HERMES_SERVER_CONF HERMES_CONF->server_config_ + +/** Initialize client-side Hermes transparently */ +static inline bool TRANSPARENT_HERMES() { + if (TRANSPARENT_LABSTOR()) { + HERMES_CONF->ClientInit(); + return true; + } + return false; +} + +#endif // LABSTOR_TASKS_HERMES_INCLUDE_hermes_H_ diff --git a/tasks/hermes/include/hermes/config_server.h b/tasks/hermes/include/hermes/config_server.h new file mode 100644 index 000000000..24a8ae938 --- /dev/null +++ b/tasks/hermes/include/hermes/config_server.h @@ -0,0 +1,354 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SRC_CONFIG_SERVER_H_ +#define HERMES_SRC_CONFIG_SERVER_H_ + +#include "config.h" +#include "config_server_default.h" + +namespace hermes::config { + +struct DeviceInfo; /** Forward declaration of DeviceInfo */ + +/** + * The type of interface the device exposes + * */ +enum class IoInterface { + kRam, + kPosix +}; + +/** + * DeviceInfo shared-memory representation + * */ +struct DeviceInfo { + /** The human-readable name of the device */ + std::string dev_name_; + /** The unit of each slab, a multiple of the Device's block size */ + std::vector slab_sizes_; + /** The directory the device is mounted on */ + std::string mount_dir_; + /** The file to create on the device */ + std::string mount_point_; + /** The I/O interface for the device */ + IoInterface io_api_; + /** The minimum transfer size of each device */ + size_t block_size_; + /** Device capacity (bytes) */ + size_t capacity_; + /** Bandwidth of a device (MBps) */ + f32 bandwidth_; + /** Latency of the device (us)*/ + f32 latency_; + /** Whether or not the device is shared among all nodes */ + bool is_shared_; + /** BORG's minimum and maximum capacity threshold for device */ + f32 borg_min_thresh_, borg_max_thresh_; +}; + +/** + * RPC information defined in server config + * */ +struct RpcInfo { + /** The name of a file that contains host names, 1 per line */ + std::string host_file_; + /** The parsed hostnames from the hermes conf */ + std::vector host_names_; + /** The RPC protocol to be used. */ + std::string protocol_; + /** The RPC domain name for verbs transport. */ + std::string domain_; + /** The RPC port number. */ + int port_; + /** The number of handler threads per RPC server. */ + int num_threads_; +}; + +/** + * DPE information defined in server config + * */ +struct DpeInfo { + /** The default blob placement policy. */ + PlacementPolicy default_policy_; + + /** Whether blob splitting is enabled for Round-Robin blob placement. */ + bool default_rr_split_; +}; + +/** + * Buffer organizer information defined in server config + * */ +struct BorgInfo { + /** The number of buffer organizer threads. */ + int num_threads_; + /** Interval (seconds) where blobs are checked for flushing */ + size_t flush_period_; + /** Interval (seconds) where blobs are checked for re-organization */ + size_t blob_reorg_period_; + /** Time when score is equal to 1 (seconds) */ + float recency_min_; + /** Time when score is equal to 0 (seconds) */ + float recency_max_; + /** Number of accesses for score to be equal to 1 (count) */ + float freq_max_; + /** Number of accesses for score to be equal to 0 (count) */ + float freq_min_; +}; + +/** + * Prefetcher information in server config + * */ +struct PrefetchInfo { + bool enabled_; + std::string trace_path_; + std::string apriori_schema_path_; + size_t epoch_ms_; + bool is_mpi_; +}; + +/** + * Tracing information in server config + * */ +struct TracingInfo { + bool enabled_; + std::string output_; +}; + +/** MDM information */ +struct MdmInfo { + /** Number of buckets in mdm bucket map before collisions */ + size_t num_bkts_; + + /** Number of blobs in mdm blob map before collisions */ + size_t num_blobs_; + + /** Number of traits in mdm trait map before collisions */ + size_t num_traits_; +}; + +/** + * System configuration for Hermes + */ +class ServerConfig : public BaseConfig { + public: + /** The device information */ + std::vector devices_; + + /** The RPC information */ + RpcInfo rpc_; + + /** The DPE information */ + DpeInfo dpe_; + + /** Buffer organizer (BORG) information */ + BorgInfo borg_; + + /** Tracing information */ + TracingInfo tracing_; + + /** Prefetcher information */ + PrefetchInfo prefetcher_; + + /** Metadata Manager information */ + MdmInfo mdm_; + + /** Trait repo information */ + std::vector trait_paths_; + + /** The length of a view state epoch */ + u32 system_view_state_update_interval_ms; + + /** The max amount of memory hermes uses for non-buffering tasks */ + size_t max_memory_; + + /** A base name for the BufferPool shared memory segement. Hermes appends the + * value of the USER environment variable to this string. + */ + std::string shmem_name_; + + public: + /** Default constructor */ + ServerConfig() = default; + + /** Load the default configuration */ + void LoadDefault() { + LoadText(kHermesServerDefaultConfigStr, false); + } + + private: + /** parse the YAML node */ + void ParseYAML(YAML::Node &yaml_conf) { + if (yaml_conf["devices"]) { + ParseDeviceInfo(yaml_conf["devices"]); + } + if (yaml_conf["dpe"]) { + ParseDpeInfo(yaml_conf["dpe"]); + } + if (yaml_conf["buffer_organizer"]) { + ParseBorgInfo(yaml_conf["buffer_organizer"]); + } + if (yaml_conf["tracing"]) { + ParseTracingInfo(yaml_conf["tracing"]); + } + if (yaml_conf["prefetch"]) { + ParsePrefetchInfo(yaml_conf["prefetch"]); + } + if (yaml_conf["mdm"]) { + ParseMdmInfo(yaml_conf["mdm"]); + } + if (yaml_conf["system_view_state_update_interval_ms"]) { + system_view_state_update_interval_ms = + yaml_conf["system_view_state_update_interval_ms"].as(); + } + if (yaml_conf["traits"]) { + ParseTraitInfo(yaml_conf["traits"]); + } + if (yaml_conf["shmem_name"]) { + shmem_name_ = yaml_conf["shmem_name"].as(); + } + if (yaml_conf["max_memory"]) { + max_memory_ = hshm::ConfigParse::ParseSize( + yaml_conf["max_memory"].as()); + } + } + + + /** parse device information from YAML config */ + void ParseDeviceInfo(YAML::Node yaml_conf) { + devices_.clear(); + for (auto device : yaml_conf) { + devices_.emplace_back(); + DeviceInfo &dev = devices_.back(); + auto dev_info = device.second; + dev.dev_name_ = device.first.as(); + dev.mount_dir_ = hshm::ConfigParse::ExpandPath( + dev_info["mount_point"].as()); + dev.borg_min_thresh_ = + dev_info["borg_capacity_thresh"][0].as(); + dev.borg_max_thresh_ = + dev_info["borg_capacity_thresh"][1].as(); + dev.is_shared_ = + dev_info["is_shared_device"].as(); + dev.block_size_ = + hshm::ConfigParse::ParseSize(dev_info["block_size"].as()); + dev.capacity_ = + hshm::ConfigParse::ParseSize(dev_info["capacity"].as()); + dev.bandwidth_ = + hshm::ConfigParse::ParseSize(dev_info["bandwidth"].as()); + dev.latency_ = + hshm::ConfigParse::ParseLatency(dev_info["latency"].as()); + std::vector size_vec; + ParseVector>( + dev_info["slab_sizes"], size_vec); + dev.slab_sizes_.reserve(size_vec.size()); + for (const std::string &size_str : size_vec) { + dev.slab_sizes_.emplace_back(hshm::ConfigParse::ParseSize(size_str)); + } + } + } + + /** parse dpe information from YAML config */ + void ParseDpeInfo(YAML::Node yaml_conf) { + if (yaml_conf["default_placement_policy"]) { + std::string policy = + yaml_conf["default_placement_policy"].as(); + dpe_.default_policy_ = PlacementPolicyConv::to_enum(policy); + } + } + + /** parse buffer organizer information from YAML config */ + void ParseBorgInfo(YAML::Node yaml_conf) { + if (yaml_conf["num_threads"]) { + borg_.num_threads_ = yaml_conf["num_threads"].as(); + } + if (yaml_conf["flush_period"]) { + borg_.flush_period_ = yaml_conf["flush_period"].as(); + } + if (yaml_conf["blob_reorg_period"]) { + borg_.blob_reorg_period_ = yaml_conf["blob_reorg_period"].as(); + } + if (yaml_conf["recency_min"]) { + borg_.recency_min_ = yaml_conf["recency_min"].as(); + } + if (yaml_conf["recency_max"]) { + borg_.recency_max_ = yaml_conf["recency_max"].as(); + } + if (yaml_conf["freq_max"]) { + borg_.freq_max_ = yaml_conf["freq_max"].as(); + } + if (yaml_conf["freq_min"]) { + borg_.freq_min_ = yaml_conf["freq_min"].as(); + } + } + + /** parse I/O tracing information from YAML config */ + void ParsePrefetchInfo(YAML::Node yaml_conf) { + if (yaml_conf["enabled"]) { + tracing_.enabled_ = yaml_conf["enabled"].as(); + } + if (yaml_conf["output"]) { + tracing_.output_ = hshm::ConfigParse::ExpandPath( + yaml_conf["output"].as()); + } + } + + /** parse prefetch information from YAML config */ + void ParseTracingInfo(YAML::Node yaml_conf) { + if (yaml_conf["enabled"]) { + prefetcher_.enabled_ = yaml_conf["enabled"].as(); + } + if (yaml_conf["io_trace_path"]) { + prefetcher_.trace_path_ = hshm::ConfigParse::ExpandPath( + yaml_conf["io_trace_path"].as()); + } + if (yaml_conf["epoch_ms"]) { + prefetcher_.epoch_ms_ = yaml_conf["epoch_ms"].as(); + } + if (yaml_conf["is_mpi"]) { + prefetcher_.is_mpi_ = yaml_conf["is_mpi"].as(); + } + if (yaml_conf["apriori_schema_path"]) { + prefetcher_.apriori_schema_path_ = + yaml_conf["apriori_schema_path"].as(); + } + } + + /** parse prefetch information from YAML config */ + void ParseTraitInfo(YAML::Node yaml_conf) { + std::vector trait_names; + ParseVector>( + yaml_conf, trait_names); + trait_paths_.reserve(trait_names.size()); + for (auto &name : trait_names) { + name = hshm::ConfigParse::ExpandPath(name); + trait_paths_.emplace_back( + hshm::Formatter::format("lib{}.so", name)); + } + } + + /** parse prefetch information from YAML config */ + void ParseMdmInfo(YAML::Node yaml_conf) { + mdm_.num_blobs_ = yaml_conf["est_blob_count"].as(); + mdm_.num_bkts_ = yaml_conf["est_blob_count"].as(); + mdm_.num_traits_ = yaml_conf["est_num_traits"].as(); + } +}; + +} // namespace hermes::config + +namespace hermes { +using config::ServerConfig; +using config::DeviceInfo; +} // namespace hermes + +#endif // HERMES_SRC_CONFIG_SERVER_H_ diff --git a/src/config_server_default.h b/tasks/hermes/include/hermes/config_server_default.h similarity index 97% rename from src/config_server_default.h rename to tasks/hermes/include/hermes/config_server_default.h index 63ab8b94a..f6bdcdd3b 100644 --- a/src/config_server_default.h +++ b/tasks/hermes/include/hermes/config_server_default.h @@ -1,6 +1,6 @@ -#ifndef HERMES_SRC_CONFIG_SERVER_DEFAULT_H_ -#define HERMES_SRC_CONFIG_SERVER_DEFAULT_H_ -const char* kServerDefaultConfigStr = +#ifndef LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ +#define LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ +static inline const char* kHermesServerDefaultConfigStr = "# Example Hermes configuration file\n" "\n" "### Define properties of the storage devices\n" @@ -178,4 +178,4 @@ const char* kServerDefaultConfigStr = " - \"hermes_mpiio_io_client\"\n" " - \"hermes_example_trait\"\n" " - \"hermes_prefetcher_trait\"\n"; -#endif // HERMES_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file +#endif // LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file diff --git a/src/data_placement_engine.h b/tasks/hermes/include/hermes/dpe/dpe.h similarity index 70% rename from src/data_placement_engine.h rename to tasks/hermes/include/hermes/dpe/dpe.h index e2607ad2f..8392734b9 100644 --- a/src/data_placement_engine.h +++ b/tasks/hermes/include/hermes/dpe/dpe.h @@ -15,29 +15,22 @@ #include -#include "hermes.h" -#include "status.h" -#include "hermes_types.h" -#include "metadata_manager.h" +#include "hermes/hermes_types.h" +#include "hermes_mdm/hermes_mdm.h" +#include "bdev/bdev.h" namespace hermes { -using api::Status; -using hermes::api::PlacementPolicy; - /** A class to represent data placement engine */ -class DPE { - protected: - MetadataManager *mdm_; /**< A pointer to the MDM */ - +class Dpe { public: /** Constructor. */ - DPE(); + Dpe() = default; /** Destructor. */ - virtual ~DPE() = default; + virtual ~Dpe() = default; /** * Calculate the placement of a set of blobs using a particular @@ -45,16 +38,8 @@ class DPE { * */ virtual Status Placement(const std::vector &blob_sizes, std::vector &targets, - api::Context &ctx, + Context &ctx, std::vector &output) = 0; - - /** - * Calculate the total placement for all blobs and output a schema for - * each blob. - * */ - Status CalculatePlacement(const std::vector &blob_sizes, - std::vector &output, - api::Context &api_context); }; } // namespace hermes diff --git a/src/data_placement_engine_factory.h b/tasks/hermes/include/hermes/dpe/dpe_factory.h similarity index 87% rename from src/data_placement_engine_factory.h rename to tasks/hermes/include/hermes/dpe/dpe_factory.h index 8d8c1baba..99211f712 100644 --- a/src/data_placement_engine_factory.h +++ b/tasks/hermes/include/hermes/dpe/dpe_factory.h @@ -13,18 +13,18 @@ #ifndef HERMES_SRC_DPE_DATA_PLACEMENT_ENGINE_FACTORY_H_ #define HERMES_SRC_DPE_DATA_PLACEMENT_ENGINE_FACTORY_H_ -#include "dpe/minimize_io_time.h" -#include "dpe/random.h" -#include "dpe/round_robin.h" +#include "minimize_io_time.h" +#include "random.h" +#include "round_robin.h" +#include "dpe.h" +#include "hermes/hermes.h" namespace hermes { -using hermes::api::PlacementPolicy; - /** A class to represent Data Placement Engine Factory */ -class DPEFactory { +class DpeFactory { public: /** * return a pointer to data placement engine given a policy type. @@ -34,7 +34,10 @@ class DPEFactory { * data placement engine factory. * @return pointer to DataPlacementEngine given \a type PlacementPolicy. */ - static DPE* Get(const PlacementPolicy &type) { + static Dpe* Get(PlacementPolicy type) { + if (type == PlacementPolicy::kNone) { + type = HERMES_SERVER_CONF.dpe_.default_policy_; + } switch (type) { case PlacementPolicy::kRandom: { return hshm::EasySingleton::GetInstance(); diff --git a/tasks/hermes/include/hermes/dpe/minimize_io_time.h b/tasks/hermes/include/hermes/dpe/minimize_io_time.h new file mode 100644 index 000000000..7c09b6c96 --- /dev/null +++ b/tasks/hermes/include/hermes/dpe/minimize_io_time.h @@ -0,0 +1,84 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SRC_DPE_MINIMIZE_IO_TIME_H_ +#define HERMES_SRC_DPE_MINIMIZE_IO_TIME_H_ + +#include "dpe.h" + +namespace hermes { +/** + A class to represent data placement engine that minimizes I/O time. +*/ +class MinimizeIoTime : public Dpe { + public: + MinimizeIoTime() = default; + ~MinimizeIoTime() = default; + Status Placement(const std::vector &blob_sizes, + std::vector &targets, + Context &ctx, + std::vector &output) { + for (size_t blob_size : blob_sizes) { + // Initialize blob's size, score, and schema + size_t rem_blob_size = blob_size; + float score = ctx.blob_score_; + if (ctx.blob_score_ == -1) { + score = 1; + } + output.emplace_back(); + PlacementSchema &blob_schema = output.back(); + + for (u32 tgt_idx = 0; tgt_idx < targets.size(); ++tgt_idx) { + TargetInfo &target = targets[tgt_idx]; + // NOTE(llogan): We skip targets that are too high of priority or + // targets that can't fit the ENTIRE blob + size_t rem_cap = target.GetRemCap(); + if (target.score_ > score || rem_cap < blob_size) { + // TODO(llogan): add other considerations of this Dpe + continue; + } + if (ctx.blob_score_ == -1) { + ctx.blob_score_ = target.score_; + } + + // NOTE(llogan): we assume the TargetInfo list is sorted + if (rem_cap >= rem_blob_size) { + blob_schema.plcmnts_.emplace_back(rem_blob_size, + target.id_); + rem_cap -= rem_blob_size; + rem_blob_size = 0; + } else { + // NOTE(llogan): this code technically never gets called, + // but it might in the future + blob_schema.plcmnts_.emplace_back(rem_cap, + target.id_); + rem_blob_size -= rem_cap; + rem_cap = 0; + } + + if (rem_blob_size == 0) { + break; + } + } + + if (rem_blob_size > 0) { + return DPE_MIN_IO_TIME_NO_SOLUTION; + } + } + + return Status(); + } +}; + +} // namespace hermes + +#endif // HERMES_SRC_DPE_MINIMIZE_IO_TIME_H_ diff --git a/src/dpe/random.h b/tasks/hermes/include/hermes/dpe/random.h similarity index 55% rename from src/dpe/random.h rename to tasks/hermes/include/hermes/dpe/random.h index 6a4bbb1da..3d989ef93 100644 --- a/src/dpe/random.h +++ b/tasks/hermes/include/hermes/dpe/random.h @@ -13,15 +13,16 @@ #ifndef HERMES_SRC_DPE_RANDOM_H_ #define HERMES_SRC_DPE_RANDOM_H_ -#include "data_placement_engine.h" +#include "dpe.h" #include #include namespace hermes { + /** A class to represent data placement engine that places data randomly. */ -class Random : public DPE { +class Random : public Dpe { public: Random() { // TODO(llogan): make seed configurable @@ -30,8 +31,42 @@ class Random : public DPE { ~Random() = default; Status Placement(const std::vector &blob_sizes, std::vector &targets, - api::Context &ctx, - std::vector &output) override; + Context &ctx, + std::vector &output) override { + for (size_t blob_size : blob_sizes) { + // Initialize blob's size, score, and schema + size_t rem_blob_size = blob_size; + output.emplace_back(); + PlacementSchema &blob_schema = output.back(); + + // Choose RR target and iterate + size_t target_id = std::rand() % targets.size(); + for (size_t tgt_idx = 0; tgt_idx < targets.size(); ++tgt_idx) { + TargetInfo &target = targets[(target_id + tgt_idx) % targets.size()]; + size_t rem_cap = target.GetRemCap(); + + // NOTE(llogan): We skip targets that can't fit the ENTIRE blob + if (rem_cap < blob_size) { + continue; + } + if (ctx.blob_score_ == -1) { + ctx.blob_score_ = target.score_; + } + + // Place the blob on this target + blob_schema.plcmnts_.emplace_back(rem_blob_size, + target.id_); + rem_blob_size = 0; + break; + } + + if (rem_blob_size > 0) { + return DPE_MIN_IO_TIME_NO_SOLUTION; + } + } + + return Status(); + } }; } // namespace hermes diff --git a/tasks/hermes/include/hermes/dpe/round_robin.h b/tasks/hermes/include/hermes/dpe/round_robin.h new file mode 100644 index 000000000..440f7a072 --- /dev/null +++ b/tasks/hermes/include/hermes/dpe/round_robin.h @@ -0,0 +1,72 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_SRC_DPE_ROUND_ROBIN_H_ +#define HERMES_SRC_DPE_ROUND_ROBIN_H_ + +#include + +#include "dpe.h" + +namespace hermes { + +/** Represents the state of a Round-Robin data placement strategy */ +class RoundRobin : public Dpe { + private: + std::atomic counter_; + + public: + RoundRobin() : counter_(0) {} + + Status Placement(const std::vector &blob_sizes, + std::vector &targets, + Context &ctx, + std::vector &output) { + for (size_t blob_size : blob_sizes) { + // Initialize blob's size, score, and schema + size_t rem_blob_size = blob_size; + output.emplace_back(); + PlacementSchema &blob_schema = output.back(); + + // Choose RR target and iterate + size_t target_id = counter_.fetch_add(1) % targets.size(); + for (size_t tgt_idx = 0; tgt_idx < targets.size(); ++tgt_idx) { + TargetInfo &target = targets[(target_id + tgt_idx) % targets.size()]; + size_t rem_cap = target.GetRemCap(); + + // NOTE(llogan): We skip targets that can't fit the ENTIRE blob + if (rem_cap < blob_size) { + continue; + } + if (ctx.blob_score_ == -1) { + ctx.blob_score_ = target.score_; + } + + // Place the blob on this target + blob_schema.plcmnts_.emplace_back(rem_blob_size, + target.id_); + rem_blob_size = 0; + break; + } + + if (rem_blob_size > 0) { + return DPE_MIN_IO_TIME_NO_SOLUTION; + } + } + + return Status(); + } +}; + +} // namespace hermes + +#endif // HERMES_SRC_DPE_ROUND_ROBIN_H_ diff --git a/tasks/hermes/include/hermes/hermes.h b/tasks/hermes/include/hermes/hermes.h new file mode 100644 index 000000000..3c57b2311 --- /dev/null +++ b/tasks/hermes/include/hermes/hermes.h @@ -0,0 +1,47 @@ +// +// Created by lukemartinlogan on 7/9/23. +// + +#ifndef LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ +#define LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ + +#include "hermes/bucket.h" + +namespace hermes { + +class Hermes { + public: + /** Init hermes client */ + void ClientInit() { + HERMES_CONF->ClientInit(); + } + + /** Check if initialized */ + bool IsInitialized() { + return HERMES_CONF->is_initialized_; + } + + /** Get tag ID */ + TagId GetTagId(const std::string &tag_name) { + return HERMES_CONF->bkt_mdm_.GetTagIdRoot(hshm::to_charbuf(tag_name)); + } + + /** Get or create a bucket */ + hermes::Bucket GetBucket(const std::string &path, + Context ctx = Context(), + size_t backend_size = 0, + u32 flags = 0) { + return hermes::Bucket(path, ctx, backend_size, flags); + } + + /** Clear all data from hermes */ + void Clear() { + // TODO(llogan) + } +}; + +#define HERMES hshm::EasySingleton::GetInstance() + +} // namespace hermes + +#endif // LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ diff --git a/tasks/hermes/include/hermes/hermes_types.h b/tasks/hermes/include/hermes/hermes_types.h new file mode 100644 index 000000000..ab29f27fe --- /dev/null +++ b/tasks/hermes/include/hermes/hermes_types.h @@ -0,0 +1,442 @@ +// +// Created by lukemartinlogan on 7/8/23. +// + +#ifndef LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ +#define LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ + +#include "labstor/labstor_types.h" +#include "labstor/task_registry/task_registry.h" +#include "labstor/api/labstor_client.h" +#include "status.h" +#include "statuses.h" + +namespace hapi = hermes; + +namespace hermes { + +using labstor::TaskLib; +using labstor::TaskMethod; +using labstor::UniqueId; +using labstor::TaskStateId; +using labstor::DomainId; +using labstor::Task; +using labstor::TaskId; +using labstor::TaskNode; +using hshm::bitfield32_t; + +/** Queue id */ +using labstor::QueueId; + +/** Queue for interprocess-communication */ +using labstor::MultiQueue; + +/** Unique blob id */ +typedef UniqueId<100> BlobId; + +/** Unique bucket id */ +typedef UniqueId<101> BucketId; + +/** Represents a tag */ +typedef UniqueId<102> TagId; + +/** Represetnts a storage target */ +typedef TaskStateId TargetId; + +/** Represents a trait */ +typedef TaskStateId TraitId; + +/** Represents a blob */ +typedef hshm::charbuf Blob; + +/** Represents a blob in shared memory */ +typedef hipc::uptr IpcBlob; + +/** Supported data placement policies */ +enum class PlacementPolicy { + kRandom, /**< Random blob placement */ + kRoundRobin, /**< Round-Robin (around devices) blob placement */ + kMinimizeIoTime, /**< LP-based blob placement, minimize I/O time */ + kNone, /**< No Dpe for cases we want it disabled */ +}; + +/** A class to convert placement policy enum value to string */ +class PlacementPolicyConv { + public: + /** A function to return string representation of \a policy */ + static std::string to_str(PlacementPolicy policy) { + switch (policy) { + case PlacementPolicy::kRandom: { + return "PlacementPolicy::kRandom"; + } + case PlacementPolicy::kRoundRobin: { + return "PlacementPolicy::kRoundRobin"; + } + case PlacementPolicy::kMinimizeIoTime: { + return "PlacementPolicy::kMinimizeIoTime"; + } + case PlacementPolicy::kNone: { + return "PlacementPolicy::kNone"; + } + } + return "PlacementPolicy::Invalid"; + } + + /** return enum value of \a policy */ + static PlacementPolicy to_enum(const std::string &policy) { + if (policy.find("Random") != std::string::npos) { + return PlacementPolicy::kRandom; + } else if (policy.find("RoundRobin") != std::string::npos) { + return PlacementPolicy::kRoundRobin; + } else if (policy.find("MinimizeIoTime") != std::string::npos) { + return PlacementPolicy::kMinimizeIoTime; + } else if (policy.find("None") != std::string::npos) { + return PlacementPolicy::kNone; + } + return PlacementPolicy::kNone; + } +}; + +/** Hermes API call context */ +struct Context { + /** Data placement engine */ + PlacementPolicy dpe_; + + /** The blob's score */ + float blob_score_; + + /** Page size to use for FS reads / writes*/ + size_t page_size_; + + /** File name used for FS reads / writes */ + std::string filename_; + + /** The node id the blob will be accessed from */ + u32 node_id_; + + Context() + : dpe_(PlacementPolicy::kNone), + blob_score_(1), + node_id_(0) {} +}; + +/** + * Represents the fraction of a blob to place + * on a particular target during data placement + * */ +struct SubPlacement { + size_t size_; /**< Size (bytes) */ + TargetId tid_; /**< Index in the target vector */ + + SubPlacement() = default; + + explicit SubPlacement(size_t size, const TargetId &tid) + : size_(size), tid_(tid) {} +}; + +/** + * The organization of a particular blob in the storage + * hierarchy during data placement. + */ +struct PlacementSchema { + std::vector plcmnts_; + + void AddSubPlacement(size_t size, const TargetId &tid) { + plcmnts_.emplace_back(size, tid); + } + + void Clear() { + plcmnts_.clear(); + } +}; + +/** The types of topologies */ +enum class TopologyType { + Local, + Neighborhood, + Global, + + kCount +}; + +/** The flushing mode */ +enum class FlushingMode { + kSync, + kAsync +}; + +/** Convert flushing modes to strings */ +class FlushingModeConv { + public: + static FlushingMode GetEnum(const std::string &str) { + if (str == "kSync") { + return FlushingMode::kSync; + } + if (str == "kAsync") { + return FlushingMode::kAsync; + } + return FlushingMode::kAsync; + } +}; + +/** A class with static constants */ +#define CONST_T inline const static +class Constant { + public: + /** Hermes server environment variable */ + CONST_T char* kHermesServerConf = "HERMES_CONF"; + + /** Hermes client environment variable */ + CONST_T char* kHermesClientConf = "HERMES_CLIENT_CONF"; + + /** Hermes adapter mode environment variable */ + CONST_T char* kHermesAdapterMode = "HERMES_ADAPTER_MODE"; + + /** Filesystem page size environment variable */ + CONST_T char* kHermesPageSize = "HERMES_PAGE_SIZE"; + + /** Stop daemon environment variable */ + CONST_T char* kHermesStopDaemon = "HERMES_STOP_DAEMON"; + + /** Maximum path length environment variable */ + CONST_T size_t kMaxPathLength = 4096; +}; + +/** Represents an allocated fraction of a target */ +struct BufferInfo { + TargetId tid_; /**< The destination target */ + size_t t_slab_; /**< The index of the slab in the target */ + size_t t_off_; /**< Offset in the target */ + size_t t_size_; /**< Size in the target */ + + /** Serialization */ + template + void serialize(Ar &ar) { + ar(tid_, t_slab_, t_off_, t_size_); + } + + /** Default constructor */ + BufferInfo() = default; + + /** Primary constructor */ + BufferInfo(TaskStateId tid, size_t t_off, size_t t_size, + size_t blob_off, size_t blob_size) + : tid_(tid), t_off_(t_off), t_size_(t_size) {} + + /** Copy constructor */ + BufferInfo(const BufferInfo &other) { + Copy(other); + } + + /** Move constructor */ + BufferInfo(BufferInfo &&other) { + Copy(other); + } + + /** Copy assignment */ + BufferInfo& operator=(const BufferInfo &other) { + Copy(other); + return *this; + } + + /** Move assignment */ + BufferInfo& operator=(BufferInfo &&other) { + Copy(other); + return *this; + } + + /** Performs move/copy */ + void Copy(const BufferInfo &other) { + tid_ = other.tid_; + t_slab_ = other.t_slab_; + t_off_ = other.t_off_; + t_size_ = other.t_size_; + } +}; + +/** Data structure used to store Blob information */ +struct BlobInfo { + TagId tag_id_; /**< Tag the blob is on */ + BlobId blob_id_; /**< Unique ID of the blob */ + hshm::charbuf name_; /**< Name of the blob */ + std::vector buffers_; /**< Set of buffers */ + std::vector tags_; /**< Set of tags */ + size_t blob_size_; /**< The overall size of the blob */ + size_t max_blob_size_; /**< The amount of space current buffers support */ + float score_; /**< The priority of this blob */ + std::atomic access_freq_; /**< Number of times blob accessed in epoch */ + u64 last_access_; /**< Last time blob accessed */ + std::atomic mod_count_; /**< The number of times blob modified */ + std::atomic last_flush_; /**< The last mod that was flushed */ + + BlobInfo() = default; + + BlobInfo(const BlobInfo &other) { + blob_id_ = other.blob_id_; + tag_id_ = other.tag_id_; + blob_size_ = other.blob_size_; + score_ = other.score_; + mod_count_ = other.mod_count_.load(); + last_flush_ = other.last_flush_.load(); + } + + void UpdateWriteStats() { + mod_count_.fetch_add(1); + UpdateReadStats(); + } + + void UpdateReadStats() { + last_access_ = GetTimeFromStartNs(); + access_freq_.fetch_add(1); + } + + static u64 GetTimeFromStartNs() { + struct timespec currentTime; + clock_gettime(CLOCK_MONOTONIC, ¤tTime); + unsigned long long nanoseconds = + currentTime.tv_sec * 1000000000ULL + currentTime.tv_nsec; + return nanoseconds; + } +}; + +/** Data structure used to store Bucket information */ +struct TagInfo { + TagId tag_id_; + hshm::charbuf name_; + std::list blobs_; + std::list traits_; + size_t internal_size_; + size_t page_size_; + bool owner_; +}; + +/** The types of I/O that can be performed (for IoCall RPC) */ +enum class IoType { + kRead, + kWrite, + kNone +}; + +/** Indicates a PUT or GET for a particular blob */ +struct IoStat { + IoType type_; + BlobId blob_id_; + TagId tag_id_; + size_t blob_size_; + int rank_; + + /** Default constructor */ + IoStat() = default; + + /** Copy constructor */ + IoStat(const IoStat &other) { + Copy(other); + } + + /** Copy assignment */ + IoStat& operator=(const IoStat &other) { + if (this != &other) { + Copy(other); + } + return *this; + } + + /** Move constructor */ + IoStat(IoStat &&other) { + Copy(other); + } + + /** Move assignment */ + IoStat& operator=(IoStat &&other) { + if (this != &other) { + Copy(other); + } + return *this; + } + + /** Generic copy / move */ + HSHM_ALWAYS_INLINE void Copy(const IoStat &other) { + type_ = other.type_; + blob_id_ = other.blob_id_; + tag_id_ = other.tag_id_; + blob_size_ = other.blob_size_; + rank_ = other.rank_; + } + + /** Serialize */ + template + void save(Archive &ar) const { + int type = static_cast(type_); + u64 ids[2] = {blob_id_.unique_, tag_id_.unique_}; + u32 nodes[2] = {blob_id_.node_id_, tag_id_.node_id_}; + ar(type, ids[0], nodes[0], ids[1], nodes[1], blob_size_, rank_); + } + + /** Deserialize */ + template + void load(Archive &ar) { + int type; + ar(type, + blob_id_.unique_, + blob_id_.node_id_, + tag_id_.unique_, + tag_id_.node_id_, + blob_size_, + rank_); + type_ = static_cast(type); + } +}; + +/** Used for debugging concurrency issues with locks */ +enum LockOwners { + kNone = 0, + kMDM_Create = 1, + kMDM_Update = 2, + kMDM_Find = 3, + kMDM_Find2 = 4, + kMDM_Delete = 5, + kFS_GetBaseAdapterMode = 6, + kFS_GetAdapterMode = 7, + kFS_GetAdapterPageSize = 8, + kBORG_LocalEnqueueFlushes = 9, + kBORG_LocalProcessFlushes = 10, + kMDM_LocalGetBucketSize = 12, + kMDM_LocalSetBucketSize = 13, + kMDM_LocalLockBucket = 14, + kMDM_LocalUnlockBucket = 15, + kMDM_LocalClearBucket = 16, + kMDM_LocalTagBlob = 17, + kMDM_LocalBlobHasTag = 18, + kMDM_LocalTryCreateBlob = 19, + kMDM_LocalPutBlobMetadata = 20, + kMDM_LocalGetBlobId = 21, + kMDM_LocalGetBlobName = 22, + kMDM_LocalGetBlobScore = 24, + kMDM_LocalLockBlob = 26, + kMDM_LocalUnlockBlob = 27, + kMDM_LocalGetBlobBuffers = 28, + kMDM_LocalRenameBlob = 29, + kMDM_LocalDestroyBlob = 30, + kMDM_LocalClear = 31, + kMDM_LocalGetOrCreateTag = 32, + kMDM_LocalGetTagId = 33, + kMDM_LocalGetTagName = 34, + kMDM_LocalRenameTag = 35, + kMDM_LocalDestroyTag = 36, + kMDM_LocalTagAddBlob = 37, + kMDM_LocalTagRemoveBlob = 38, + kMDM_LocalGroupByTag = 39, + kMDM_LocalTagAddTrait = 40, + kMDM_LocalTagGetTraits = 41, + kMDM_LocalRegisterTrait = 42, + kMDM_LocalGetTraitId = 43, + kMDM_LocalGetTraitParams = 44, + kMDM_GlobalGetTrait = 45, + kMDM_AddIoStat = 46, + kMDM_ClearIoStats = 47 +}; + + +} // namespace hermes + +#endif // LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ diff --git a/tasks/hermes/include/hermes/slab_allocator.h b/tasks/hermes/include/hermes/slab_allocator.h new file mode 100644 index 000000000..8b177a473 --- /dev/null +++ b/tasks/hermes/include/hermes/slab_allocator.h @@ -0,0 +1,150 @@ +// +// Created by lukemartinlogan on 7/13/23. +// + +#ifndef LABSTOR_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ +#define LABSTOR_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ + +#include "labstor/labstor_types.h" +#include "hermes/hermes_types.h" + +namespace hermes { + +struct Slab { + size_t slab_size_; + std::list buffers_; +}; + +struct SlabCount { + size_t count_; + size_t slab_size_; + + SlabCount() : count_(0), slab_size_(0) {} +}; + +class SlabAllocator { + public: + std::vector slab_lists_; + std::atomic heap_; + size_t dev_size_; + TargetId target_id_; + + public: + /** Default constructor */ + SlabAllocator() = default; + + /** Destructor */ + ~SlabAllocator() = default; + + /** Initialize slab allocator */ + void Init(TargetId target_id, size_t dev_size, std::vector &slab_sizes_) { + heap_ = 0; + dev_size_ = dev_size; + target_id_ = target_id; + slab_lists_.reserve(slab_sizes_.size()); + for (auto &slab_size : slab_sizes_) { + slab_lists_.emplace_back(); + auto &slab = slab_lists_.back(); + slab.slab_size_ = slab_size; + } + } + + /** + * Allocate enough slabs to ideally fit the size + * It can allocate less than or more than the requested size + * + * @param size the amount of space to allocate + * @param[OUT] buffers the buffers allocated + * @return the amount of space actually allocated. Maximum value is size. + * */ + void Allocate(size_t size, + std::vector &buffers, + size_t &total_size) { + u32 buffer_count = 0; + std::vector coins = CoinSelect(size, buffer_count); + buffers.reserve(buffer_count); + total_size = 0; + int slab_idx = 0; + for (auto &coin : coins) { + AllocateSlabs(coin.slab_size_, + slab_idx, + coin.count_, + buffers, + total_size); + ++slab_idx; + } + } + + private: + /** Determine how many of each slab size to allocate */ + std::vector CoinSelect(size_t size, u32 &buffer_count) { + std::vector coins(slab_lists_.size()); + size_t rem_size = size; + + while (rem_size) { + // Find the slab size nearest to the rem_size + size_t i = 0, slab_size = 0; + for (auto &slab : slab_lists_) { + if (slab.slab_size_ >= rem_size) { + break; + } + ++i; + } + if (i == slab_lists_.size()) { i -= 1; } + slab_size = slab_lists_[i].slab_size_; + + // Divide rem_size into slabs + if (rem_size > slab_size) { + coins[i].count_ += rem_size / slab_size; + coins[i].slab_size_ = slab_size; + rem_size %= slab_size; + } else { + coins[i].count_ += 1; + coins[i].slab_size_ = slab_size; + rem_size = 0; + } + buffer_count += coins[i].count_; + } + + return coins; + } + + /** Allocate slabs of a certain size */ + void AllocateSlabs(size_t slab_size, int slab_idx, size_t count, + std::vector &buffers, + size_t &total_size) { + auto &slab = slab_lists_[slab_idx]; + for (size_t i = 0; i < count; ++i) { + if (slab.buffers_.size() > 0) { + buffers.push_back(slab.buffers_.front()); + slab.buffers_.pop_front(); + } else { + // Verify the heap has space + if (heap_.load() + slab_size > dev_size_) { + return; + } + // Allocate a new slab + buffers.emplace_back(); + BufferInfo &buf = buffers.back(); + buf.tid_ = target_id_; + buf.t_off_ = heap_.fetch_add(slab_size); + buf.t_size_ = slab_size; + buf.t_slab_ = slab_idx; + } + total_size += slab_size; + } + } + + public: + /** Free a set of buffers */ + void Free(const std::vector &buffers) { + for (const auto &buffer : buffers) { + auto &slab = slab_lists_[buffer.t_slab_]; + slab.buffers_.push_back(buffer); + } + } +}; + +} // namespace hermes + +#endif // LABSTOR_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ diff --git a/src/status.h b/tasks/hermes/include/hermes/status.h similarity index 98% rename from src/status.h rename to tasks/hermes/include/hermes/status.h index ea86f68e0..06863c7ab 100644 --- a/src/status.h +++ b/tasks/hermes/include/hermes/status.h @@ -17,7 +17,7 @@ /** \file hermes_status.h */ -namespace hermes::api { +namespace hermes { class Status { public: diff --git a/tasks/hermes/include/hermes/statuses.h b/tasks/hermes/include/hermes/statuses.h new file mode 100644 index 000000000..83dcd8168 --- /dev/null +++ b/tasks/hermes/include/hermes/statuses.h @@ -0,0 +1,21 @@ +// +// Created by lukemartinlogan on 7/13/23. +// + +#ifndef LABSTOR_TASKS_HERMES_INCLUDE_STATUSES_H_ +#define LABSTOR_TASKS_HERMES_INCLUDE_STATUSES_H_ + +#include "status.h" + +#define STATUS_T static inline const Status + +namespace hermes { + +STATUS_T NOT_IMPLEMENTED(0, "Function was not implemented"); +STATUS_T DPE_PLACEMENT_SCHEMA_EMPTY(1, "DPE placement schema is empty"); +STATUS_T DPE_NO_SPACE(1, "Placement failed. Non-fatal."); +STATUS_T DPE_MIN_IO_TIME_NO_SOLUTION(1, "DPE could not find solution for the minimize I/O time DPE"); + +} // namespace hermes + +#endif //LABSTOR_TASKS_HERMES_INCLUDE_STATUSES_H_ diff --git a/tasks/hermes/src/CMakeLists.txt b/tasks/hermes/src/CMakeLists.txt new file mode 100644 index 000000000..306b950d8 --- /dev/null +++ b/tasks/hermes/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(hermes SHARED + config_manager.cc) +add_dependencies(hermes ${Labstor_RUNTIME_DEPS}) +target_link_libraries(hermes ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + hermes + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + hermes + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(hermes) +endif() diff --git a/tasks/hermes/src/config_manager.cc b/tasks/hermes/src/config_manager.cc new file mode 100644 index 000000000..134861c35 --- /dev/null +++ b/tasks/hermes/src/config_manager.cc @@ -0,0 +1,7 @@ +// +// Created by lukemartinlogan on 7/28/23. +// + +#include "hermes/config_manager.h" + +DEFINE_SINGLETON_CC(hermes::ConfigurationManager) diff --git a/tasks/hermes_adapters/CMakeLists.txt b/tasks/hermes_adapters/CMakeLists.txt new file mode 100644 index 000000000..e0549ae9f --- /dev/null +++ b/tasks/hermes_adapters/CMakeLists.txt @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(filesystem) +if (HERMES_ENABLE_POSIX_ADAPTER) + add_subdirectory(posix) +endif() +if (HERMES_ENABLE_STDIO_ADAPTER) + add_subdirectory(stdio) +endif() +if (HERMES_ENABLE_MPIIO_ADAPTER) + add_subdirectory(mpiio) +endif() +if (HERMES_ENABLE_HDF5_ADAPTER) + add_subdirectory(vfd) +endif() +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/adapter/filesystem/CMakeLists.txt b/tasks/hermes_adapters/filesystem/CMakeLists.txt similarity index 76% rename from adapter/filesystem/CMakeLists.txt rename to tasks/hermes_adapters/filesystem/CMakeLists.txt index c9564a624..7014b106b 100644 --- a/adapter/filesystem/CMakeLists.txt +++ b/tasks/hermes_adapters/filesystem/CMakeLists.txt @@ -23,16 +23,16 @@ install( TARGETS hermes_fs_base EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} ) install( FILES filesystem_io_client.h DESTINATION - ${HERMES_INSTALL_INCLUDE_DIR} + ${LABSTOR_INSTALL_INCLUDE_DIR} COMPONENT headers ) @@ -40,21 +40,21 @@ install( #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS +set(LABSTOR_EXPORTED_LIBS hermes_fs_base - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${HERMES_EXPORTED_LIBS} + ${LABSTOR_EXPORTED_LIBS} FILE - ${HERMES_EXPORTED_TARGETS}.cmake + ${LABSTOR_EXPORTED_TARGETS}.cmake ) endif() #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) +if(LABSTOR_ENABLE_COVERAGE) set_coverage_flags(hermes_fs_base) endif() diff --git a/adapter/filesystem/filesystem.cc b/tasks/hermes_adapters/filesystem/filesystem.cc similarity index 50% rename from adapter/filesystem/filesystem.cc rename to tasks/hermes_adapters/filesystem/filesystem.cc index 24fb41c58..44d3a0a45 100644 --- a/adapter/filesystem/filesystem.cc +++ b/tasks/hermes_adapters/filesystem/filesystem.cc @@ -13,7 +13,7 @@ #include "filesystem.h" #include "hermes_shm/util/singleton.h" #include "filesystem_mdm.h" -#include "mapper/mapper_factory.h" +#include "hermes_adapters/mapper/mapper_factory.h" #include #include @@ -58,21 +58,14 @@ void Filesystem::Open(AdapterStat &stat, File &f, const std::string &path) { // Get or create the bucket if (stat.hflags_.Any(HERMES_FS_TRUNC)) { // The file was opened with TRUNCATION - stat.bkt_id_ = HERMES->GetBucket(stat.path_, ctx, 0); + stat.bkt_id_ = HERMES->GetBucket(stat.path_, ctx, 0, HERMES_BUCKET_IS_FILE); stat.bkt_id_.Clear(); } else { // The file was opened regularly stat.file_size_ = io_client_->GetSize(*path_shm); - stat.bkt_id_ = HERMES->GetBucket(stat.path_, ctx, stat.file_size_); + stat.bkt_id_ = HERMES->GetBucket(stat.path_, ctx, stat.file_size_, HERMES_BUCKET_IS_FILE); } HILOG(kDebug, "File has size: {}", stat.bkt_id_.GetSize()); - // Attach trait to bucket (if not scratch mode) - if (stat.bkt_id_.DidCreate()) { - io_client_->Register(); - if (stat.adapter_mode_ != AdapterMode::kScratch) { - stat.bkt_id_.AttachTrait(io_client_->GetTraitId()); - } - } // Update file position pointer if (stat.hflags_.Any(HERMES_FS_APPEND)) { stat.st_ptr_ = std::numeric_limits::max(); @@ -90,94 +83,6 @@ void Filesystem::Open(AdapterStat &stat, File &f, const std::string &path) { } } -/** - * The amount of data to read from the backend if the blob - * does not exist. - * */ -static inline size_t GetBackendSize(size_t file_off, - size_t file_size, - size_t page_size) { - if (file_off + page_size <= file_size) { - // Case 1: The file has more than "PageSize" bytes remaining - return page_size; - } else if (file_off < file_size) { - // Case 2: The file has less than "PageSize" bytes remaining - return file_size - file_off; - } else { - // Case 3: The offset is beyond the size of the backend file - return 0; - } -} - -/** - * Put \a blob_name Blob into the bucket. Load the blob from the - * I/O backend if it does not exist. - * - * @param blob_name the semantic name of the blob - * @param blob the buffer to put final data in - * @param blob_off the offset within the blob to begin the Put - * @param page_size the page size of the adapter - * @param io_ctx which adapter to route I/O request if blob DNE - * @param opts which adapter to route I/O request if blob DNE - * @param ctx any additional information - * */ -Status Filesystem::PartialPutOrCreate(hapi::Bucket &bkt, - const std::string &blob_name, - const Blob &blob, - size_t blob_off, - size_t page_size, - BlobId &blob_id, - IoStatus &status, - const FsIoOptions &opts, - Context &ctx) { - Blob full_blob(page_size); - if (bkt.ContainsBlob(blob_name, blob_id)) { - // Case 1: The blob already exists (read from hermes) - // Read blob from Hermes - HILOG(kDebug, "Blob existed. Reading from Hermes.") - bkt.Get(blob_id, full_blob, ctx); - } - if (blob_off == 0 && - blob.size() >= opts.backend_size_ && - blob.size() >= full_blob.size()) { - // Case 2: We're overriding the entire blob - // Put the entire blob, no need to load from storage - HILOG(kDebug, "Putting the entire blob.") - return bkt.Put(blob_name, blob, blob_id, ctx); - } - if (full_blob.size() < opts.backend_size_) { - // Case 3: The blob did not fully exist (need to read from backend) - // Read blob using adapter - HILOG(kDebug, "Blob did not fully exist. Reading blob from backend." - " cur_size: {}" - " backend_size: {}", - full_blob.size(), opts.backend_size_) - full_blob.resize(opts.backend_size_); - io_client_->ReadBlob(bkt.GetName(), - full_blob, opts, status); - if (!status.success_) { - HELOG(kFatal, "Failed to read blob from {} (PartialPut)." - " cur_size: {}" - " backend_off: {}" - " backend_size: {}", - bkt.GetName(), full_blob.size(), - opts.backend_off_, opts.backend_size_) - // return PARTIAL_PUT_OR_CREATE_OVERFLOW; - } - } - HILOG(kDebug, "Modifying full_blob at offset: {} for total size: {}", - blob_off, blob.size()) - // Ensure the blob can hold the update - full_blob.resize(std::max(full_blob.size(), blob_off + blob.size())); - // Modify the blob - memcpy(full_blob.data() + blob_off, blob.data(), blob.size()); - // Re-put the blob - bkt.Put(blob_name, full_blob, blob_id, ctx); - HILOG(kDebug, "Partially put to blob: ({}, {})", - blob_id.unique_, blob_id.node_id_) - return Status(); -} - size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr, size_t off, size_t total_size, IoStatus &io_status, FsIoOptions opts) { @@ -185,26 +90,14 @@ size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr, hapi::Bucket &bkt = stat.bkt_id_; std::string filename = bkt.GetName(); - // Update the size of the bucket - size_t orig_off = off; - stat.bkt_id_.LockBucket(MdLockType::kExternalWrite); - size_t backend_size = stat.bkt_id_.GetSize(); - if (off == std::numeric_limits::max()) { - off = backend_size; - } - size_t new_size = off + total_size; - if (new_size > backend_size) { - stat.bkt_id_.SetSize(off + total_size); - } - stat.bkt_id_.UnlockBucket(MdLockType::kExternalWrite); + bool is_append = stat.st_ptr_ == std::numeric_limits::max(); HILOG(kDebug, "Write called for filename: {}" - " on offset: {}" - " from position: {}" - " and size: {}" - " and current file size: {}" - " and adapter mode: {}", - filename, off, stat.st_ptr_, total_size, backend_size, + " on offset: {}" + " from position: {}" + " and current file size: {}" + " and adapter mode: {}", + filename, off, stat.st_ptr_, total_size, AdapterModeConv::str(stat.adapter_mode_)) if (stat.adapter_mode_ == AdapterMode::kBypass) { // Bypass mode is handled differently @@ -217,120 +110,40 @@ size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr, opts.backend_size_) return 0; } - if (opts.DoSeek() && orig_off != std::numeric_limits::max()) { + if (opts.DoSeek() && !is_append) { stat.st_ptr_ = off + total_size; } return total_size; } - size_t ret; - Context ctx; + // Fragment I/O request into pages BlobPlacements mapping; - size_t kPageSize = stat.page_size_; + auto mapper = MapperFactory::Get(MapperType::kBalancedMapper); + mapper->map(off, total_size, stat.page_size_, mapping); size_t data_offset = 0; - auto mapper = MapperFactory().Get(MapperType::kBalancedMapper); - mapper->map(off, total_size, kPageSize, mapping); - - for (const auto &p : mapping) { - const Blob blob_wrap((const char*)ptr + data_offset, p.blob_size_); - hshm::charbuf blob_name(p.CreateBlobName()); - BlobId blob_id; - opts.backend_off_ = p.page_ * kPageSize; - opts.backend_size_ = GetBackendSize(opts.backend_off_, - stat.file_size_, - kPageSize); - opts.adapter_mode_ = stat.adapter_mode_; - bkt.TryCreateBlob(blob_name.str(), blob_id, ctx); - bkt.LockBlob(blob_id, MdLockType::kExternalWrite); - auto status = PartialPutOrCreate(bkt, - blob_name.str(), - blob_wrap, - p.blob_off_, - kPageSize, - blob_id, - io_status, - opts, - ctx); - if (status.Fail()) { - data_offset = 0; - bkt.UnlockBlob(blob_id, MdLockType::kExternalWrite); - break; + + // Perform a PartialPut for each page + Context ctx; + ctx.page_size_ = stat.page_size_; + ctx.filename_ = stat.path_; + for (const BlobPlacement &p : mapping) { + const Blob page((const char*)ptr + data_offset, p.blob_size_); + if (!is_append) { + std::string blob_name(p.CreateBlobName().str()); + bkt.PartialPut(blob_name, page, p.blob_off_, ctx); + } else { + bkt.Append(page, stat.page_size_, ctx); } - bkt.UnlockBlob(blob_id, MdLockType::kExternalWrite); data_offset += p.blob_size_; } - if (opts.DoSeek() && orig_off != std::numeric_limits::max()) { + if (opts.DoSeek() && !is_append) { stat.st_ptr_ = off + total_size; } stat.UpdateTime(); HILOG(kDebug, "The size of file after write: {}", GetSize(f, stat)) - ret = data_offset; - return ret; -} - -/** - * Load \a blob_name Blob from the bucket. Load the blob from the - * I/O backend if it does not exist. - * - * @param blob_name the semantic name of the blob - * @param blob the buffer to put final data in - * @param blob_off the offset within the blob to begin the Put - * @param blob_size the total amount of data to read - * @param page_size the page size of the adapter - * @param blob_id [out] the blob id corresponding to blob_name - * @param io_ctx information required to perform I/O to the backend - * @param opts specific configuration of the I/O to perform - * @param ctx any additional information - * */ -Status Filesystem::PartialGetOrCreate(hapi::Bucket &bkt, - const std::string &blob_name, - Blob &blob, - size_t blob_off, - size_t blob_size, - size_t page_size, - BlobId &blob_id, - IoStatus &status, - const FsIoOptions &opts, - Context &ctx) { - Blob full_blob(page_size); - if (bkt.ContainsBlob(blob_name, blob_id)) { - // Case 1: The blob already exists (read from hermes) - // Read blob from Hermes - HILOG(kDebug, "Blob existed. Reading from Hermes." - " offset: {}" - " size: {}", - opts.backend_off_, blob_size) - bkt.Get(blob_id, full_blob, ctx); - } - if (full_blob.size() < opts.backend_size_) { - // Case 2: The blob did not exist (or at least not fully) - // Read blob using adapter - HILOG(kDebug, "Blob did not fully exist. Reading blob from backend." - " cur_size: {}" - " backend_size: {}", - full_blob.size(), opts.backend_size_) - full_blob.resize(opts.backend_size_); - io_client_->ReadBlob(bkt.GetName(), full_blob, opts, status); - if (!status.success_) { - HILOG(kDebug, "Failed to read blob from backend (PartialGet)." - " cur_size: {}" - " backend_size: {}", - full_blob.size(), opts.backend_size_) - return PARTIAL_GET_OR_CREATE_OVERFLOW; - } - bkt.Put(blob_name, full_blob, blob_id, ctx); - } - // Ensure the blob can hold the update - if (full_blob.size() < blob_off + blob_size) { - return PARTIAL_GET_OR_CREATE_OVERFLOW; - } - // Modify the blob - // TODO(llogan): we can avoid a copy here - blob.resize(blob_size); - memcpy(blob.data(), full_blob.data() + blob_off, blob.size()); - return Status(); + return data_offset; } size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr, @@ -338,14 +151,12 @@ size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr, IoStatus &io_status, FsIoOptions opts) { (void) f; hapi::Bucket &bkt = stat.bkt_id_; - std::string filename = bkt.GetName(); - size_t file_size = stat.bkt_id_.GetSize(); HILOG(kDebug, "Read called for filename: {}" " on offset: {}" " from position: {}" " and size: {}", - filename, off, stat.st_ptr_, total_size) + stat.path_, off, stat.st_ptr_, total_size) // SEEK_END is not a valid read position if (off == std::numeric_limits::max()) { @@ -353,9 +164,6 @@ size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr, } // Ensure the amount being read makes sense - if (off + total_size > file_size) { - total_size = file_size - off; - } if (total_size == 0) { return 0; } @@ -377,106 +185,49 @@ size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr, return total_size; } - size_t ret; - Context ctx; + // Fragment I/O request into pages BlobPlacements mapping; - size_t kPageSize = stat.page_size_; + auto mapper = MapperFactory::Get(MapperType::kBalancedMapper); + mapper->map(off, total_size, stat.page_size_, mapping); size_t data_offset = 0; - auto mapper = MapperFactory().Get(MapperType::kBalancedMapper); - mapper->map(off, total_size, kPageSize, mapping); - - for (const auto &p : mapping) { - Blob blob_wrap((const char*)ptr + data_offset, p.blob_size_); - hshm::charbuf blob_name(p.CreateBlobName()); - BlobId blob_id; - opts.backend_off_ = p.page_ * kPageSize; - opts.backend_size_ = GetBackendSize(opts.backend_off_, - stat.file_size_, - kPageSize); - opts.adapter_mode_ = stat.adapter_mode_; - bkt.TryCreateBlob(blob_name.str(), blob_id, ctx); - bkt.LockBlob(blob_id, MdLockType::kExternalRead); - auto status = PartialGetOrCreate(stat.bkt_id_, - blob_name.str(), - blob_wrap, - p.blob_off_, - p.blob_size_, - kPageSize, - blob_id, - io_status, - opts, - ctx); - if (status.Fail()) { - data_offset = 0; - bkt.UnlockBlob(blob_id, MdLockType::kExternalRead); - break; - } - bkt.UnlockBlob(blob_id, MdLockType::kExternalRead); + + // Perform a PartialPut for each page + Context ctx; + ctx.page_size_ = stat.page_size_; + ctx.filename_ = stat.path_; + for (const BlobPlacement &p : mapping) { + Blob page((const char*)ptr + data_offset, p.blob_size_); + std::string blob_name(p.CreateBlobName().str()); + bkt.PartialGet(blob_name, page, p.blob_off_, ctx); data_offset += p.blob_size_; } if (opts.DoSeek()) { stat.st_ptr_ = off + total_size; } stat.UpdateTime(); - - ret = data_offset; - return ret; + return data_offset; } -HermesRequest* Filesystem::AWrite(File &f, AdapterStat &stat, const void *ptr, - size_t off, size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { - (void) io_status; +Task* Filesystem::AWrite(File &f, AdapterStat &stat, const void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { HILOG(kDebug, "Starting an asynchronous write", - opts.backend_size_) - auto pool = HERMES_FS_THREAD_POOL; - HermesRequest *hreq = new HermesRequest(); - auto lambda = - [](Filesystem *fs, File &f, AdapterStat &stat, const void *ptr, - size_t off, size_t total_size, IoStatus &io_status, FsIoOptions opts) { - return fs->Write(f, stat, ptr, off, total_size, io_status, opts); - }; - auto func = std::bind(lambda, this, f, stat, ptr, off, - total_size, hreq->io_status, opts); - hreq->return_future = pool->run(func); - auto mdm = HERMES_FS_METADATA_MANAGER; - mdm->request_map.emplace(req_id, hreq); - return hreq;/**/ -} - -HermesRequest* Filesystem::ARead(File &f, AdapterStat &stat, void *ptr, - size_t off, size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { - (void) io_status; - auto pool = HERMES_FS_THREAD_POOL; - HermesRequest *hreq = new HermesRequest(); - auto lambda = - [](Filesystem *fs, File &f, AdapterStat &stat, void *ptr, - size_t off, size_t total_size, IoStatus &io_status, FsIoOptions opts) { - return fs->Read(f, stat, ptr, off, total_size, io_status, opts); - }; - auto func = std::bind(lambda, this, f, stat, - ptr, off, total_size, hreq->io_status, opts); - hreq->return_future = pool->run(func); - auto mdm = HERMES_FS_METADATA_MANAGER; - mdm->request_map.emplace(req_id, hreq); - return hreq; + opts.backend_size_); + return nullptr; +} + +Task* Filesystem::ARead(File &f, AdapterStat &stat, void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { + return nullptr; } size_t Filesystem::Wait(uint64_t req_id) { - auto mdm = HERMES_FS_METADATA_MANAGER; - auto req_iter = mdm->request_map.find(req_id); - if (req_iter == mdm->request_map.end()) { - return 0; - } - HermesRequest *req = (*req_iter).second; - size_t ret = req->return_future.get(); - delete req; - return ret; + return 0; } void Filesystem::Wait(std::vector &req_ids, - std::vector &ret) { + std::vector &ret) { for (auto &req_id : req_ids) { ret.emplace_back(Wait(req_id)); } @@ -538,10 +289,9 @@ off_t Filesystem::Tell(File &f, AdapterStat &stat) { } int Filesystem::Sync(File &f, AdapterStat &stat) { - if (HERMES->client_config_.flushing_mode_ == FlushingMode::kSync) { + if (HERMES_CLIENT_CONF.flushing_mode_ == FlushingMode::kSync) { // NOTE(llogan): only for the unit tests // Please don't enable synchronous flushing - HERMES->Flush(); } return 0; } @@ -562,7 +312,7 @@ int Filesystem::Close(File &f, AdapterStat &stat) { if (stat.amode_ & MPI_MODE_DELETE_ON_CLOSE) { Remove(stat.path_); } - if (HERMES->client_config_.flushing_mode_ == FlushingMode::kSync) { + if (HERMES_CLIENT_CONF.flushing_mode_ == FlushingMode::kSync) { // NOTE(llogan): only for the unit tests // Please don't enable synchronous flushing // stat.bkt_id_.Destroy(); @@ -619,16 +369,16 @@ size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr, return Read(f, stat, ptr, off, total_size, io_status, opts); } -HermesRequest* Filesystem::AWrite(File &f, AdapterStat &stat, const void *ptr, - size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { +Task* Filesystem::AWrite(File &f, AdapterStat &stat, const void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { off_t off = stat.st_ptr_; return AWrite(f, stat, ptr, off, total_size, req_id, io_status, opts); } -HermesRequest* Filesystem::ARead(File &f, AdapterStat &stat, void *ptr, - size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { +Task* Filesystem::ARead(File &f, AdapterStat &stat, void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { off_t off = stat.st_ptr_; return ARead(f, stat, ptr, off, total_size, req_id, io_status, opts); } @@ -693,9 +443,9 @@ size_t Filesystem::Read(File &f, bool &stat_exists, void *ptr, return Read(f, *stat, ptr, off, total_size, io_status, opts); } -HermesRequest* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, - size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { +Task* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { auto mdm = HERMES_FS_METADATA_MANAGER; auto stat = mdm->Find(f); if (!stat) { @@ -706,9 +456,9 @@ HermesRequest* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, return AWrite(f, *stat, ptr, total_size, req_id, io_status, opts); } -HermesRequest* Filesystem::ARead(File &f, bool &stat_exists, void *ptr, - size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { +Task* Filesystem::ARead(File &f, bool &stat_exists, void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { auto mdm = HERMES_FS_METADATA_MANAGER; auto stat = mdm->Find(f); if (!stat) { @@ -719,9 +469,9 @@ HermesRequest* Filesystem::ARead(File &f, bool &stat_exists, void *ptr, return ARead(f, *stat, ptr, total_size, req_id, io_status, opts); } -HermesRequest* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, - size_t off, size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { +Task* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { auto mdm = HERMES_FS_METADATA_MANAGER; auto stat = mdm->Find(f); if (!stat) { @@ -733,9 +483,9 @@ HermesRequest* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, return AWrite(f, *stat, ptr, off, total_size, req_id, io_status, opts); } -HermesRequest* Filesystem::ARead(File &f, bool &stat_exists, void *ptr, - size_t off, size_t total_size, size_t req_id, - IoStatus &io_status, FsIoOptions opts) { +Task* Filesystem::ARead(File &f, bool &stat_exists, void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, FsIoOptions opts) { auto mdm = HERMES_FS_METADATA_MANAGER; auto stat = mdm->Find(f); if (!stat) { diff --git a/adapter/filesystem/filesystem.h b/tasks/hermes_adapters/filesystem/filesystem.h similarity index 68% rename from adapter/filesystem/filesystem.h rename to tasks/hermes_adapters/filesystem/filesystem.h index 28062bf48..73bccb73c 100644 --- a/adapter/filesystem/filesystem.h +++ b/tasks/hermes_adapters/filesystem/filesystem.h @@ -23,13 +23,12 @@ #include #include -#include "bucket.h" -#include "hermes.h" +#include "hermes/bucket.h" +#include "hermes/hermes.h" #include "filesystem_io_client.h" #include -namespace hapi = hermes::api; namespace hermes::adapter::fs { @@ -61,71 +60,22 @@ class Filesystem { /** open \a f File in \a path*/ void Open(AdapterStat &stat, File &f, const std::string &path); - /** - * Put \a blob_name Blob into the bucket. Load the blob from the - * I/O backend if it does not exist or is not fully loaded. - * - * @param bkt the bucket to use for the partial operation - * @param blob_name the semantic name of the blob - * @param blob the buffer to put final data in - * @param blob_off the offset within the blob to begin the Put - * @param page_size the page size of the adapter - * @param blob_id [out] the blob id corresponding to blob_name - * @param io_ctx information required to perform I/O to the backend - * @param opts specific configuration of the I/O to perform - * @param ctx any additional information - * */ - Status PartialPutOrCreate(hapi::Bucket &bkt, - const std::string &blob_name, - const Blob &blob, - size_t blob_off, - size_t page_size, - BlobId &blob_id, - IoStatus &status, - const FsIoOptions &opts, - Context &ctx); - /** write */ size_t Write(File &f, AdapterStat &stat, const void *ptr, size_t off, size_t total_size, IoStatus &io_status, FsIoOptions opts = FsIoOptions()); - /** - * Load \a blob_name Blob from the bucket. Load the blob from the - * I/O backend if it does not exist or is not fully loaded. - * - * @param bkt the bucket to use for the partial operation - * @param blob_name the semantic name of the blob - * @param blob the buffer to put final data in - * @param blob_off the offset within the blob to begin the Put - * @param page_size the page size of the adapter - * @param blob_id [out] the blob id corresponding to blob_name - * @param io_ctx information required to perform I/O to the backend - * @param opts specific configuration of the I/O to perform - * @param ctx any additional information - * */ - Status PartialGetOrCreate(hapi::Bucket &bkt, - const std::string &blob_name, - Blob &blob, - size_t blob_off, - size_t blob_size, - size_t page_size, - BlobId &blob_id, - IoStatus &status, - const FsIoOptions &opts, - Context &ctx); - /** read */ size_t Read(File &f, AdapterStat &stat, void *ptr, size_t off, size_t total_size, IoStatus &io_status, FsIoOptions opts = FsIoOptions()); /** write asynchronously */ - HermesRequest *AWrite(File &f, AdapterStat &stat, const void *ptr, size_t off, + Task* AWrite(File &f, AdapterStat &stat, const void *ptr, size_t off, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts = FsIoOptions()); /** read asynchronously */ - HermesRequest *ARead(File &f, AdapterStat &stat, void *ptr, size_t off, + Task* ARead(File &f, AdapterStat &stat, void *ptr, size_t off, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts = FsIoOptions()); /** wait for \a req_id request ID */ @@ -160,11 +110,11 @@ class Filesystem { size_t Read(File &f, AdapterStat &stat, void *ptr, size_t total_size, IoStatus &io_status, FsIoOptions opts); /** write asynchronously */ - HermesRequest *AWrite(File &f, AdapterStat &stat, const void *ptr, + Task* AWrite(File &f, AdapterStat &stat, const void *ptr, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts); /** read asynchronously */ - HermesRequest *ARead(File &f, AdapterStat &stat, void *ptr, size_t total_size, + Task* ARead(File &f, AdapterStat &stat, void *ptr, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts); /* @@ -188,18 +138,18 @@ class Filesystem { size_t total_size, IoStatus &io_status, FsIoOptions opts = FsIoOptions()); /** write asynchronously */ - HermesRequest *AWrite(File &f, bool &stat_exists, const void *ptr, + Task* AWrite(File &f, bool &stat_exists, const void *ptr, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts); /** read asynchronously */ - HermesRequest *ARead(File &f, bool &stat_exists, void *ptr, size_t total_size, + Task* ARead(File &f, bool &stat_exists, void *ptr, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts); /** write \a off offset asynchronously */ - HermesRequest *AWrite(File &f, bool &stat_exists, const void *ptr, size_t off, + Task* AWrite(File &f, bool &stat_exists, const void *ptr, size_t off, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts); /** read \a off offset asynchronously */ - HermesRequest *ARead(File &f, bool &stat_exists, void *ptr, size_t off, + Task* ARead(File &f, bool &stat_exists, void *ptr, size_t off, size_t total_size, size_t req_id, IoStatus &io_status, FsIoOptions opts); /** seek */ @@ -218,11 +168,11 @@ class Filesystem { public: /** Whether or not \a path PATH is tracked by Hermes */ static bool IsPathTracked(const std::string &path) { - if (!HERMES->IsInitialized()) { + if (!HERMES_CONF->is_initialized_) { return false; } std::string abs_path = stdfs::absolute(path).string(); - auto &paths = HERMES->client_config_.path_list_; + auto &paths = HERMES_CLIENT_CONF.path_list_; // Check if path is included or excluded for (config::UserPathInfo &pth : paths) { if (abs_path.rfind(pth.path_) != std::string::npos) { diff --git a/adapter/filesystem/filesystem_io_client.h b/tasks/hermes_adapters/filesystem/filesystem_io_client.h similarity index 94% rename from adapter/filesystem/filesystem_io_client.h rename to tasks/hermes_adapters/filesystem/filesystem_io_client.h index f5cc32720..9c7edd1f9 100644 --- a/adapter/filesystem/filesystem_io_client.h +++ b/tasks/hermes_adapters/filesystem/filesystem_io_client.h @@ -13,9 +13,9 @@ #ifndef HERMES_ADAPTER_FILESYSTEM_FILESYSTEM_IO_CLIENT_H_ #define HERMES_ADAPTER_FILESYSTEM_FILESYSTEM_IO_CLIENT_H_ -#include "adapter/mapper/balanced_mapper.h" -#include "hermes.h" -#include "bucket.h" +#include "hermes_adapters/mapper/balanced_mapper.h" +#include "hermes/hermes.h" +#include "hermes/bucket.h" #include #include #include @@ -295,7 +295,7 @@ struct FilesystemIoClientState { * Defines I/O clients which are compatible with the filesystem * base class. * */ -class FilesystemIoClient : public Trait { +class FilesystemIoClient { public: /** Decode I/O client context from the original blob name */ FsIoOptions DecodeBlobName(const std::string &blob_name) { @@ -309,25 +309,6 @@ class FilesystemIoClient : public Trait { /** virtual destructor */ virtual ~FilesystemIoClient() = default; - /** Callback used for custom trait execution */ - void Run(int method, void *params) { - FlushTraitParams *info = (FlushTraitParams*)params; - switch (method) { - case HERMES_TRAIT_FLUSH: { - FsIoOptions opts = DecodeBlobName(info->blob_name_); - IoStatus status; - WriteBlob((*info->bkt_).GetName(), - (*info->blob_), - opts, - status); - break; - } - default: { - HELOG(kError, "Invalid I/O client run method: {}\n", method) - } - } - } - /** Get initial statistics from the backend */ virtual size_t GetSize(const hipc::charbuf &bkt_name) = 0; diff --git a/adapter/filesystem/filesystem_mdm.cc b/tasks/hermes_adapters/filesystem/filesystem_mdm.cc similarity index 99% rename from adapter/filesystem/filesystem_mdm.cc rename to tasks/hermes_adapters/filesystem/filesystem_mdm.cc index 0f29ab84f..69da5379d 100644 --- a/adapter/filesystem/filesystem_mdm.cc +++ b/tasks/hermes_adapters/filesystem/filesystem_mdm.cc @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "filesystem_mdm.h" -#include "hermes.h" +#include "hermes/hermes.h" #include namespace hermes::adapter::fs { diff --git a/adapter/filesystem/filesystem_mdm.h b/tasks/hermes_adapters/filesystem/filesystem_mdm.h similarity index 91% rename from adapter/filesystem/filesystem_mdm.h rename to tasks/hermes_adapters/filesystem/filesystem_mdm.h index 05c23d8e8..065d05893 100644 --- a/adapter/filesystem/filesystem_mdm.h +++ b/tasks/hermes_adapters/filesystem/filesystem_mdm.h @@ -17,7 +17,6 @@ #include #include "filesystem_io_client.h" #include "filesystem.h" -#include "thread_pool.h" namespace hermes::adapter::fs { @@ -42,20 +41,20 @@ class MetadataManager { /** Get the current adapter mode */ AdapterMode GetBaseAdapterMode() { - ScopedRwReadLock md_lock(lock_, kFS_GetBaseAdapterMode); - return HERMES->client_config_.GetBaseAdapterMode(); + ScopedRwReadLock md_lock(lock_, 1); + return HERMES_CLIENT_CONF.GetBaseAdapterMode(); } /** Get the adapter mode for a particular file */ AdapterMode GetAdapterMode(const std::string &path) { - ScopedRwReadLock md_lock(lock_, kFS_GetAdapterMode); - return HERMES->client_config_.GetAdapterConfig(path).mode_; + ScopedRwReadLock md_lock(lock_, 2); + return HERMES_CLIENT_CONF.GetAdapterConfig(path).mode_; } /** Get the adapter page size for a particular file */ size_t GetAdapterPageSize(const std::string &path) { - ScopedRwReadLock md_lock(lock_, kFS_GetAdapterPageSize); - return HERMES->client_config_.GetAdapterConfig(path).page_size_; + ScopedRwReadLock md_lock(lock_, 3); + return HERMES_CLIENT_CONF.GetAdapterConfig(path).page_size_; } /** diff --git a/adapter/filesystem/filesystem_mdm_singleton.cc b/tasks/hermes_adapters/filesystem/filesystem_mdm_singleton.cc similarity index 100% rename from adapter/filesystem/filesystem_mdm_singleton.cc rename to tasks/hermes_adapters/filesystem/filesystem_mdm_singleton.cc diff --git a/adapter/adapter_constants.h b/tasks/hermes_adapters/include/hermes_adapters/adapter_constants.h similarity index 100% rename from adapter/adapter_constants.h rename to tasks/hermes_adapters/include/hermes_adapters/adapter_constants.h diff --git a/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h b/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h new file mode 100644 index 000000000..5e4cabaeb --- /dev/null +++ b/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h @@ -0,0 +1,103 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_ADAPTER_TYPES_H_ +#define HERMES_ADAPTER_ADAPTER_TYPES_H_ + +#include "../../posix/posix_api.h" + +namespace hermes::adapter { + +/** Adapter types */ +enum class AdapterType { + kNone, + kPosix, + kStdio, + kMpiio, + kPubsub, + kVfd +}; + +/** Adapter modes */ +enum class AdapterMode { + kNone, + kDefault, + kBypass, + kScratch, + kWorkflow +}; + +/** + * Per-Object Adapter Settings. + * An object may be a file, for example. + * */ +struct AdapterObjectConfig { + AdapterMode mode_; + size_t page_size_; +}; + +/** Adapter Mode converter */ +class AdapterModeConv { + public: + static std::string str(AdapterMode mode) { + switch (mode) { + case AdapterMode::kDefault: { + return "AdapterMode::kDefault"; + } + case AdapterMode::kBypass: { + return "AdapterMode::kBypass"; + } + case AdapterMode::kScratch: { + return "AdapterMode::kScratch"; + } + case AdapterMode::kWorkflow: { + return "AdapterMode::kWorkflow"; + } + default: { + return "Unkown adapter mode"; + } + } + } + + static AdapterMode to_enum(const std::string &mode) { + if (mode.find("kDefault") != std::string::npos) { + return AdapterMode::kDefault; + } else if (mode.find("kBypass") != std::string::npos) { + return AdapterMode::kBypass; + } else if (mode.find("kScratch") != std::string::npos) { + return AdapterMode::kScratch; + } else if (mode.find("kWorkflow") != std::string::npos) { + return AdapterMode::kWorkflow; + } + return AdapterMode::kDefault; + } +}; + +struct AdapterInfo { + int file_id_; + int fd_; + int open_flags_; + int mode_flags_; + int refcnt_; + std::string path_; + AdapterMode adapter_mode_; + + ~AdapterInfo() { + if (fd_ >= 0) { + HERMES_POSIX_API->close(fd_); + } + } +}; + +} // namespace hermes::adapter + +#endif // HERMES_ADAPTER_ADAPTER_TYPES_H_ diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h new file mode 100644 index 000000000..1817cdf75 --- /dev/null +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h @@ -0,0 +1,72 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_hermes_adapters_H_ +#define LABSTOR_hermes_adapters_H_ + +#include "hermes_adapters_tasks.h" + +namespace labstor::hermes_adapters { + +/** Create hermes_adapters requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Async create a task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info); + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate) + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + auto *task = AsyncCreateRoot(std::forward(args)...); + task->Wait(); + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Call a custom method */ + HSHM_ALWAYS_INLINE + void AsyncCustomConstruct(CustomTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_); + } + HSHM_ALWAYS_INLINE + void CustomRoot(const DomainId &domain_id) { + LPointer> task = AsyncCustomRoot(domain_id); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Custom); +}; + +} // namespace labstor + +#endif // LABSTOR_hermes_adapters_H_ diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h new file mode 100644 index 000000000..a491992c9 --- /dev/null +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h @@ -0,0 +1,146 @@ +#ifndef LABSTOR_hermes_adapters_LIB_EXEC_H_ +#define LABSTOR_hermes_adapters_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + Custom(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kCustom: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kCustom: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kCustom: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_hermes_adapters_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h new file mode 100644 index 000000000..e6a468219 --- /dev/null +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_hermes_adapters_METHODS_H_ +#define LABSTOR_hermes_adapters_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kCustom = kLast + 0; +}; + +#endif // LABSTOR_hermes_adapters_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml new file mode 100644 index 000000000..b1b54e2a4 --- /dev/null +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml @@ -0,0 +1 @@ +kCustom: 0 \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h new file mode 100644 index 000000000..6df2570a4 --- /dev/null +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h @@ -0,0 +1,125 @@ +// +// Created by lukemartinlogan on 8/11/23. +// + +#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ +#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::hermes_adapters { + +#include "hermes_adapters_methods.h" +#include "labstor/labstor_namespace.h" +using labstor::proc_queue::TypedPushTask; +using labstor::proc_queue::PushTask; + +/** + * A task to create hermes_adapters + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) + : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "hermes_adapters", id, queue_info) { + // Custom params + } + + HSHM_ALWAYS_INLINE + ~ConstructTask() { + // Custom params + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy hermes_adapters */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) + : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in hermes_adapters + * */ +struct CustomTask : public Task, TaskFlags { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kCustom; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Custom params + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor::hermes_adapters + +#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ diff --git a/adapter/interceptor.h b/tasks/hermes_adapters/include/hermes_adapters/interceptor.h similarity index 93% rename from adapter/interceptor.h rename to tasks/hermes_adapters/include/hermes_adapters/interceptor.h index d291c4cd6..ba62e256a 100644 --- a/adapter/interceptor.h +++ b/tasks/hermes_adapters/include/hermes_adapters/interceptor.h @@ -13,8 +13,8 @@ #ifndef HERMES_ADAPTER_UTILS_H_ #define HERMES_ADAPTER_UTILS_H_ -#include "hermes.h" -#include "adapter/filesystem/filesystem_mdm.h" +#include "hermes/hermes.h" +#include "hermes_adapters/filesystem/filesystem_mdm.h" namespace stdfs = std::filesystem; diff --git a/adapter/mapper/abstract_mapper.h b/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h similarity index 87% rename from adapter/mapper/abstract_mapper.h rename to tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h index 016bf7f8c..dd0858099 100644 --- a/adapter/mapper/abstract_mapper.h +++ b/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h @@ -13,7 +13,7 @@ #ifndef HERMES_ABSTRACT_MAPPER_H #define HERMES_ABSTRACT_MAPPER_H -#include "hermes_types.h" +#include "hermes/hermes_types.h" namespace hermes::adapter { @@ -39,20 +39,20 @@ struct BlobPlacement { /** create a BLOB name from index. */ hshm::charbuf CreateBlobName() const { hshm::charbuf buf(sizeof(page_) + sizeof(blob_off_)); - size_t off = 0; - memcpy(buf.data() + off, &page_, sizeof(page_)); - off += sizeof(page_); - memcpy(buf.data() + off, &page_size_, sizeof(page_size_)); + labstor::LocalSerialize srl(buf); + srl << page_; + srl << page_size_; return buf; } /** decode \a blob_name BLOB name to index. */ template void DecodeBlobName(const StringT &blob_name) { - size_t off = 0; - memcpy(&page_, blob_name.data(), sizeof(page_)); - off += sizeof(page_); - memcpy(&page_size_, blob_name.data() + off, sizeof(page_size_)); + labstor::LocalDeserialize srl(blob_name); + srl >> page_; + srl >> page_size_; + bucket_off_ = page_ * page_size_; + blob_off_ = 0; } }; diff --git a/adapter/mapper/balanced_mapper.h b/tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h similarity index 69% rename from adapter/mapper/balanced_mapper.h rename to tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h index a240606f5..331901ba7 100644 --- a/adapter/mapper/balanced_mapper.h +++ b/tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h @@ -28,7 +28,22 @@ class BalancedMapper : public AbstractMapper { /** Divides an I/O size evenly by into units of page_size */ void map(size_t off, size_t size, size_t page_size, - BlobPlacements &ps) override; + BlobPlacements &ps) override { + size_t kPageSize = page_size; + size_t size_mapped = 0; + while (size > size_mapped) { + BlobPlacement p; + p.bucket_off_ = off + size_mapped; + p.page_ = p.bucket_off_ / kPageSize; + p.page_size_ = page_size; + p.blob_off_ = p.bucket_off_ % kPageSize; + auto left_size_page = kPageSize - p.blob_off_; + p.blob_size_ = left_size_page < size - size_mapped ? left_size_page + : size - size_mapped; + ps.emplace_back(p); + size_mapped += p.blob_size_; + } + } }; } // namespace hermes::adapter diff --git a/adapter/mapper/mapper_factory.h b/tasks/hermes_adapters/include/hermes_adapters/mapper/mapper_factory.h similarity index 95% rename from adapter/mapper/mapper_factory.h rename to tasks/hermes_adapters/include/hermes_adapters/mapper/mapper_factory.h index 7bc9b2f6c..bba844495 100644 --- a/adapter/mapper/mapper_factory.h +++ b/tasks/hermes_adapters/include/hermes_adapters/mapper/mapper_factory.h @@ -14,7 +14,6 @@ #define HERMES_ADAPTER_FACTORY_H #include "abstract_mapper.h" -#include "balanced_mapper.cc" #include "balanced_mapper.h" #include "hermes_shm/util/singleton.h" @@ -30,7 +29,7 @@ class MapperFactory { * @param[in] type type of mapper to be used by the POSIX adapter. * @return Instance of mapper given a type. */ - AbstractMapper* Get(const MapperType& type) { + static AbstractMapper* Get(const MapperType& type) { switch (type) { case MapperType::kBalancedMapper: { return hshm::EasySingleton::GetInstance(); diff --git a/adapter/real_api.h b/tasks/hermes_adapters/include/hermes_adapters/real_api.h similarity index 100% rename from adapter/real_api.h rename to tasks/hermes_adapters/include/hermes_adapters/real_api.h diff --git a/adapter/mpiio/CMakeLists.txt b/tasks/hermes_adapters/mpiio/CMakeLists.txt similarity index 82% rename from adapter/mpiio/CMakeLists.txt rename to tasks/hermes_adapters/mpiio/CMakeLists.txt index 1f3741e92..4cfd54d0e 100644 --- a/adapter/mpiio/CMakeLists.txt +++ b/tasks/hermes_adapters/mpiio/CMakeLists.txt @@ -1,5 +1,3 @@ -project(MPIIOAdapter VERSION ${HERMES_PACKAGE_VERSION}) - include_directories( ${CMAKE_SOURCE_DIR} ${HERMES_SRC_DIR} @@ -31,15 +29,15 @@ install( hermes_mpiio_io_client hermes_mpiio EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) +if(LABSTOR_ENABLE_COVERAGE) set_coverage_flags(hermes_mpiio_io_client) #set_coverage_flags(hermes_mpiio) endif() \ No newline at end of file diff --git a/adapter/mpiio/mpiio_api.cc b/tasks/hermes_adapters/mpiio/mpiio_api.cc similarity index 99% rename from adapter/mpiio/mpiio_api.cc rename to tasks/hermes_adapters/mpiio/mpiio_api.cc index 746b68e19..e61c857e2 100644 --- a/adapter/mpiio/mpiio_api.cc +++ b/tasks/hermes_adapters/mpiio/mpiio_api.cc @@ -19,7 +19,7 @@ bool mpiio_intercepted = true; #include "mpiio_fs_api.h" #include "hermes_shm/util/singleton.h" -#include "interceptor.h" +#include "hermes_adapters/interceptor.h" /** * Namespace declarations @@ -32,7 +32,6 @@ using hermes::adapter::fs::MpiioApi; using hermes::adapter::fs::MpiioFs; using hermes::adapter::fs::MpiioSeekModeConv; -namespace hapi = hermes::api; extern "C" { @@ -41,7 +40,7 @@ extern "C" { */ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { HILOG(kDebug, "MPI Init intercepted.") - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); auto real_api = HERMES_MPIIO_API; return real_api->MPI_Init(argc, argv); } diff --git a/adapter/mpiio/mpiio_api.h b/tasks/hermes_adapters/mpiio/mpiio_api.h similarity index 99% rename from adapter/mpiio/mpiio_api.h rename to tasks/hermes_adapters/mpiio/mpiio_api.h index 8ca86c43d..efdd2a101 100644 --- a/adapter/mpiio/mpiio_api.h +++ b/tasks/hermes_adapters/mpiio/mpiio_api.h @@ -18,7 +18,7 @@ #include "hermes_shm/util/logging.h" #include #include -#include "adapter/real_api.h" +#include "hermes_adapters/real_api.h" #ifndef MPI_MODE_TRUNCATE #define MPI_MODE_TRUNCATE 0 diff --git a/adapter/mpiio/mpiio_fs_api.h b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h similarity index 98% rename from adapter/mpiio/mpiio_fs_api.h rename to tasks/hermes_adapters/mpiio/mpiio_fs_api.h index 5a2e7718e..3e51e7558 100644 --- a/adapter/mpiio/mpiio_fs_api.h +++ b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h @@ -15,8 +15,8 @@ #include -#include "adapter/filesystem/filesystem.h" -#include "adapter/filesystem/filesystem_mdm.h" +#include "hermes_adapters/filesystem/filesystem.h" +#include "hermes_adapters/filesystem/filesystem_mdm.h" #include "mpiio_api.h" #include "mpiio_io_client.h" @@ -151,7 +151,7 @@ class MpiioFs : public Filesystem { HILOG(kDebug, "Starting an asynchronous write") auto mdm = HERMES_FS_METADATA_MANAGER; auto pool = HERMES_FS_THREAD_POOL; - HermesRequest *hreq = new HermesRequest(); + Task* hreq = new HermesRequest(); auto lambda = [](MpiioFs *fs, File &f, AdapterStat &stat, const void *ptr, int count, MPI_Datatype datatype, MPI_Status *status, FsIoOptions opts) { @@ -170,7 +170,7 @@ class MpiioFs : public Filesystem { auto real_api = HERMES_MPIIO_API; auto iter = mdm->request_map.find(reinterpret_cast(req)); if (iter != mdm->request_map.end()) { - HermesRequest *hreq = iter->second; + Task* hreq = iter->second; hreq->return_future.get(); memcpy(status, hreq->io_status.mpi_status_ptr_, sizeof(MPI_Status)); mdm->request_map.erase(iter); diff --git a/adapter/mpiio/mpiio_io_client.cc b/tasks/hermes_adapters/mpiio/mpiio_io_client.cc similarity index 99% rename from adapter/mpiio/mpiio_io_client.cc rename to tasks/hermes_adapters/mpiio/mpiio_io_client.cc index ab99fb1e9..09d23237f 100644 --- a/adapter/mpiio/mpiio_io_client.cc +++ b/tasks/hermes_adapters/mpiio/mpiio_io_client.cc @@ -219,5 +219,3 @@ void MpiioIoClient::UpdateIoStatus(size_t count, IoStatus &status) { } } // namespace hermes::adapter::fs - -HERMES_TRAIT_CC(hermes::adapter::fs::MpiioIoClient) diff --git a/adapter/mpiio/mpiio_io_client.h b/tasks/hermes_adapters/mpiio/mpiio_io_client.h similarity index 84% rename from adapter/mpiio/mpiio_io_client.h rename to tasks/hermes_adapters/mpiio/mpiio_io_client.h index 12e8d5ee1..0b7ad7008 100644 --- a/adapter/mpiio/mpiio_io_client.h +++ b/tasks/hermes_adapters/mpiio/mpiio_io_client.h @@ -15,7 +15,7 @@ #include -#include "adapter/filesystem/filesystem_io_client.h" +#include "hermes_adapters/filesystem/filesystem_io_client.h" #include "mpiio_api.h" using hermes::adapter::fs::AdapterStat; @@ -25,19 +25,8 @@ using hermes::adapter::fs::MpiioApi; namespace hermes::adapter::fs { -/** State for the MPI I/O trait */ -struct MpiioIoClientHeader : public TraitHeader { - explicit MpiioIoClientHeader(const std::string &trait_uuid, - const std::string &trait_name) - : TraitHeader(trait_uuid, trait_name, - HERMES_TRAIT_FLUSH) {} -}; - /** A class to represent STDIO IO file system */ class MpiioIoClient : public hermes::adapter::fs::FilesystemIoClient { - public: - HERMES_TRAIT_H(MpiioIoClient, "mpiio_io_client") - private: HERMES_MPIIO_API_T real_api; /**< pointer to real APIs */ @@ -45,14 +34,6 @@ class MpiioIoClient : public hermes::adapter::fs::FilesystemIoClient { /** Default constructor */ MpiioIoClient() { real_api = HERMES_MPIIO_API; - CreateHeader("mpiio_io_client_", trait_name_); - } - - /** Trait deserialization constructor */ - explicit MpiioIoClient(hshm::charbuf ¶ms) { - (void) params; - real_api = HERMES_MPIIO_API; - CreateHeader("mpiio_io_client_", trait_name_); } /** Virtual destructor */ diff --git a/adapter/posix/CMakeLists.txt b/tasks/hermes_adapters/posix/CMakeLists.txt similarity index 80% rename from adapter/posix/CMakeLists.txt rename to tasks/hermes_adapters/posix/CMakeLists.txt index d5e9eadad..0553be94a 100644 --- a/adapter/posix/CMakeLists.txt +++ b/tasks/hermes_adapters/posix/CMakeLists.txt @@ -30,32 +30,32 @@ install( hermes_posix_io_client hermes_posix EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS +set(LABSTOR_EXPORTED_LIBS hermes_posix_io_client hermes_posix - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${HERMES_EXPORTED_LIBS} + ${LABSTOR_EXPORTED_LIBS} FILE - ${HERMES_EXPORTED_TARGETS}.cmake + ${LABSTOR_EXPORTED_TARGETS}.cmake ) endif() #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) +if(LABSTOR_ENABLE_COVERAGE) set_coverage_flags(hermes_posix_io_client) #set_coverage_flags(hermes_posix) endif() diff --git a/adapter/posix/posix_api.cc b/tasks/hermes_adapters/posix/posix_api.cc similarity index 98% rename from adapter/posix/posix_api.cc rename to tasks/hermes_adapters/posix/posix_api.cc index 349a56188..2360ced2e 100644 --- a/adapter/posix/posix_api.cc +++ b/tasks/hermes_adapters/posix/posix_api.cc @@ -19,20 +19,19 @@ bool posix_intercepted = true; #include "hermes_shm/util/logging.h" #include -#include "hermes_types.h" +#include "hermes/hermes_types.h" #include "hermes_shm/util/singleton.h" -#include "interceptor.h" +#include "hermes_adapters/interceptor.h" #include "posix_api.h" #include "posix_fs_api.h" -#include "filesystem/filesystem.h" +#include "hermes_adapters/filesystem/filesystem.h" using hermes::adapter::fs::AdapterStat; using hermes::adapter::fs::IoStatus; using hermes::adapter::fs::File; using hermes::adapter::fs::SeekMode; -namespace hapi = hermes::api; namespace stdfs = std::filesystem; extern "C" { @@ -40,14 +39,14 @@ extern "C" { static __attribute__((constructor(101))) void init_posix(void) { HERMES_POSIX_API; HERMES_POSIX_FS; - TRANSPARENT_HERMES; + TRANSPARENT_HERMES();; }/**/ /** * POSIX */ int HERMES_DECL(open)(const char *path, int flags, ...) { -// TRANSPARENT_HERMES +// TRANSPARENT_HERMES(); int mode = 0; auto real_api = HERMES_POSIX_API; auto fs_api = HERMES_POSIX_FS; @@ -75,7 +74,7 @@ int HERMES_DECL(open)(const char *path, int flags, ...) { } int HERMES_DECL(open64)(const char *path, int flags, ...) { -// TRANSPARENT_HERMES +// TRANSPARENT_HERMES(); int mode = 0; auto real_api = HERMES_POSIX_API; auto fs_api = HERMES_POSIX_FS; @@ -101,7 +100,7 @@ int HERMES_DECL(open64)(const char *path, int flags, ...) { } int HERMES_DECL(__open_2)(const char *path, int oflag) { -// TRANSPARENT_HERMES +// TRANSPARENT_HERMES(); auto real_api = HERMES_POSIX_API; auto fs_api = HERMES_POSIX_FS; if (fs_api->IsPathTracked(path)) { @@ -117,7 +116,7 @@ int HERMES_DECL(__open_2)(const char *path, int oflag) { } int HERMES_DECL(creat)(const char *path, mode_t mode) { -// TRANSPARENT_HERMES +// TRANSPARENT_HERMES(); std::string path_str(path); auto real_api = HERMES_POSIX_API; auto fs_api = HERMES_POSIX_FS; @@ -134,7 +133,7 @@ int HERMES_DECL(creat)(const char *path, mode_t mode) { } int HERMES_DECL(creat64)(const char *path, mode_t mode) { -// TRANSPARENT_HERMES +// TRANSPARENT_HERMES(); std::string path_str(path); auto real_api = HERMES_POSIX_API; auto fs_api = HERMES_POSIX_FS; diff --git a/adapter/posix/posix_api.h b/tasks/hermes_adapters/posix/posix_api.h similarity index 99% rename from adapter/posix/posix_api.h rename to tasks/hermes_adapters/posix/posix_api.h index 318870b00..932ca5953 100644 --- a/adapter/posix/posix_api.h +++ b/tasks/hermes_adapters/posix/posix_api.h @@ -19,7 +19,7 @@ #include #include #include -#include "adapter/real_api.h" +#include "../include/hermes_adapters/real_api.h" #ifndef O_TMPFILE #define O_TMPFILE 0 diff --git a/adapter/posix/posix_fs_api.h b/tasks/hermes_adapters/posix/posix_fs_api.h similarity index 97% rename from adapter/posix/posix_fs_api.h rename to tasks/hermes_adapters/posix/posix_fs_api.h index b259e1aac..af6d0f74a 100644 --- a/adapter/posix/posix_fs_api.h +++ b/tasks/hermes_adapters/posix/posix_fs_api.h @@ -15,8 +15,8 @@ #include -#include "adapter/filesystem/filesystem.h" -#include "adapter/filesystem/filesystem_mdm.h" +#include "hermes_adapters/filesystem/filesystem.h" +#include "hermes_adapters/filesystem/filesystem_mdm.h" #include "posix_api.h" #include "posix_io_client.h" diff --git a/adapter/posix/posix_io_client.cc b/tasks/hermes_adapters/posix/posix_io_client.cc similarity index 99% rename from adapter/posix/posix_io_client.cc rename to tasks/hermes_adapters/posix/posix_io_client.cc index 603b33c4e..5fecbaef5 100644 --- a/adapter/posix/posix_io_client.cc +++ b/tasks/hermes_adapters/posix/posix_io_client.cc @@ -165,5 +165,3 @@ void PosixIoClient::ReadBlob(const std::string &bkt_name, } } // namespace hermes::adapter::fs - -HERMES_TRAIT_CC(hermes::adapter::fs::PosixIoClient) diff --git a/adapter/posix/posix_io_client.h b/tasks/hermes_adapters/posix/posix_io_client.h similarity index 83% rename from adapter/posix/posix_io_client.h rename to tasks/hermes_adapters/posix/posix_io_client.h index f922f8349..2b4849bb8 100644 --- a/adapter/posix/posix_io_client.h +++ b/tasks/hermes_adapters/posix/posix_io_client.h @@ -15,7 +15,7 @@ #include -#include "adapter/filesystem/filesystem_io_client.h" +#include "hermes_adapters/filesystem/filesystem_io_client.h" #include "posix_api.h" using hshm::Singleton; @@ -26,19 +26,8 @@ using hermes::adapter::fs::PosixApi; namespace hermes::adapter::fs { -/** State for the POSIX I/O trait */ -struct PosixIoClientHeader : public TraitHeader { - explicit PosixIoClientHeader(const std::string &trait_uuid, - const std::string &trait_name) - : TraitHeader(trait_uuid, trait_name, - HERMES_TRAIT_FLUSH) {} -}; - /** A class to represent POSIX IO file system */ class PosixIoClient : public hermes::adapter::fs::FilesystemIoClient { - public: - HERMES_TRAIT_H(PosixIoClient, "posix_io_client") - private: HERMES_POSIX_API_T real_api; /**< pointer to real APIs */ @@ -46,14 +35,6 @@ class PosixIoClient : public hermes::adapter::fs::FilesystemIoClient { /** Default constructor */ PosixIoClient() { real_api = HERMES_POSIX_API; - CreateHeader("posix_io_client_", trait_name_); - } - - /** Trait deserialization constructor */ - explicit PosixIoClient(hshm::charbuf ¶ms) { - (void) params; - real_api = HERMES_POSIX_API; - CreateHeader("posix_io_client_", trait_name_); } /** Virtual destructor */ diff --git a/tasks/hermes_adapters/src/CMakeLists.txt b/tasks/hermes_adapters/src/CMakeLists.txt new file mode 100644 index 000000000..3382fd743 --- /dev/null +++ b/tasks/hermes_adapters/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(hermes_adapters SHARED + hermes_adapters.cc) +add_dependencies(hermes_adapters ${Labstor_RUNTIME_DEPS}) +target_link_libraries(hermes_adapters ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + hermes_adapters + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + hermes_adapters + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(hermes_adapters) +endif() diff --git a/tasks/hermes_adapters/src/hermes_adapters.cc b/tasks/hermes_adapters/src/hermes_adapters.cc new file mode 100644 index 000000000..9192ef849 --- /dev/null +++ b/tasks/hermes_adapters/src/hermes_adapters.cc @@ -0,0 +1,39 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "hermes_adapters/hermes_adapters.h" + +namespace labstor::hermes_adapters { + +class Server : public TaskLib { + public: + Server() = default; + + void Construct(ConstructTask *task) { + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + void Custom(CustomTask *task) { + task->SetModuleComplete(); + } + + void FileStageIn() { + } + + void FileStageOut() { + } + + public: +#include "hermes_adapters/hermes_adapters_lib_exec.h" +}; + +} // namespace labstor::hermes_adapters + +LABSTOR_TASK_CC(labstor::hermes_adapters::Server, "hermes_adapters"); diff --git a/adapter/stdio/CMakeLists.txt b/tasks/hermes_adapters/stdio/CMakeLists.txt similarity index 78% rename from adapter/stdio/CMakeLists.txt rename to tasks/hermes_adapters/stdio/CMakeLists.txt index 654c331c4..142f395c7 100644 --- a/adapter/stdio/CMakeLists.txt +++ b/tasks/hermes_adapters/stdio/CMakeLists.txt @@ -1,4 +1,3 @@ -project(StdioAdapter VERSION ${HERMES_PACKAGE_VERSION}) include_directories( ${CMAKE_SOURCE_DIR} ${HERMES_SRC_DIR} @@ -31,32 +30,32 @@ install( hermes_stdio_io_client hermes_stdio EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS +set(LABSTOR_EXPORTED_LIBS hermes_stdio_io_client hermes_stdio - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${HERMES_EXPORTED_LIBS} + ${LABSTOR_EXPORTED_LIBS} FILE - ${HERMES_EXPORTED_TARGETS}.cmake + ${LABSTOR_EXPORTED_TARGETS}.cmake ) endif() #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) +if(LABSTOR_ENABLE_COVERAGE) set_coverage_flags(hermes_stdio_io_client) #set_coverage_flags(hermes_stdio) endif() diff --git a/adapter/stdio/stdio_api.cc b/tasks/hermes_adapters/stdio/stdio_api.cc similarity index 98% rename from adapter/stdio/stdio_api.cc rename to tasks/hermes_adapters/stdio/stdio_api.cc index 1cb9183b8..b88d7ea03 100644 --- a/adapter/stdio/stdio_api.cc +++ b/tasks/hermes_adapters/stdio/stdio_api.cc @@ -17,17 +17,14 @@ bool stdio_intercepted = true; #include #include "stdio_api.h" #include "stdio_fs_api.h" -#include "interceptor.h" +#include "hermes_adapters/interceptor.h" using hermes::adapter::fs::MetadataManager; using hermes::adapter::fs::SeekMode; using hermes::adapter::fs::AdapterStat; using hermes::adapter::fs::File; -namespace hapi = hermes::api; namespace stdfs = std::filesystem; -using hermes::u8; -using hermes::u64; extern "C" { @@ -36,7 +33,7 @@ extern "C" { */ FILE *HERMES_DECL(fopen)(const char *path, const char *mode) { - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); auto real_api = HERMES_STDIO_API; auto fs_api = HERMES_STDIO_FS; if (fs_api->IsPathTracked(path)) { @@ -50,7 +47,7 @@ FILE *HERMES_DECL(fopen)(const char *path, const char *mode) { } FILE *HERMES_DECL(fopen64)(const char *path, const char *mode) { - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); auto real_api = HERMES_STDIO_API; auto fs_api = HERMES_STDIO_FS; if (fs_api->IsPathTracked(path)) { @@ -64,7 +61,7 @@ FILE *HERMES_DECL(fopen64)(const char *path, const char *mode) { } FILE *HERMES_DECL(fdopen)(int fd, const char *mode) { - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); auto real_api = HERMES_STDIO_API; auto fs_api = HERMES_STDIO_FS; std::shared_ptr stat; @@ -77,7 +74,7 @@ FILE *HERMES_DECL(fdopen)(int fd, const char *mode) { } FILE *HERMES_DECL(freopen)(const char *path, const char *mode, FILE *stream) { - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); auto real_api = HERMES_STDIO_API; auto fs_api = HERMES_STDIO_FS; if (fs_api->IsFpTracked(stream)) { @@ -88,7 +85,7 @@ FILE *HERMES_DECL(freopen)(const char *path, const char *mode, FILE *stream) { } FILE *HERMES_DECL(freopen64)(const char *path, const char *mode, FILE *stream) { - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); auto real_api = HERMES_STDIO_API; auto fs_api = HERMES_STDIO_FS; if (fs_api->IsFpTracked(stream)) { diff --git a/adapter/stdio/stdio_api.h b/tasks/hermes_adapters/stdio/stdio_api.h similarity index 99% rename from adapter/stdio/stdio_api.h rename to tasks/hermes_adapters/stdio/stdio_api.h index 0e8c1bcda..0ed8cd871 100644 --- a/adapter/stdio/stdio_api.h +++ b/tasks/hermes_adapters/stdio/stdio_api.h @@ -17,7 +17,7 @@ #include #include "hermes_shm/util/logging.h" #include -#include "adapter/real_api.h" +#include "hermes_adapters/real_api.h" extern "C" { typedef FILE * (*fopen_t)(const char * path, const char * mode); diff --git a/adapter/stdio/stdio_fs_api.h b/tasks/hermes_adapters/stdio/stdio_fs_api.h similarity index 95% rename from adapter/stdio/stdio_fs_api.h rename to tasks/hermes_adapters/stdio/stdio_fs_api.h index b2b9f2531..1af7fb154 100644 --- a/adapter/stdio/stdio_fs_api.h +++ b/tasks/hermes_adapters/stdio/stdio_fs_api.h @@ -15,9 +15,9 @@ #include -#include "adapter/filesystem/filesystem.h" -#include "adapter/filesystem/filesystem_mdm.h" -#include "adapter/posix/posix_fs_api.h" +#include "hermes_adapters/filesystem/filesystem.h" +#include "hermes_adapters/filesystem/filesystem_mdm.h" +#include "hermes_adapters/posix/posix_fs_api.h" #include "stdio_api.h" #include "stdio_io_client.h" diff --git a/adapter/stdio/stdio_io_client.cc b/tasks/hermes_adapters/stdio/stdio_io_client.cc similarity index 99% rename from adapter/stdio/stdio_io_client.cc rename to tasks/hermes_adapters/stdio/stdio_io_client.cc index b72db9b64..6793115da 100644 --- a/adapter/stdio/stdio_io_client.cc +++ b/tasks/hermes_adapters/stdio/stdio_io_client.cc @@ -162,5 +162,3 @@ void StdioIoClient::ReadBlob(const std::string &bkt_name, } } // namespace hermes::adapter::fs - -HERMES_TRAIT_CC(hermes::adapter::fs::StdioIoClient) diff --git a/adapter/stdio/stdio_io_client.h b/tasks/hermes_adapters/stdio/stdio_io_client.h similarity index 83% rename from adapter/stdio/stdio_io_client.h rename to tasks/hermes_adapters/stdio/stdio_io_client.h index ec3bbd618..9ebca5add 100644 --- a/adapter/stdio/stdio_io_client.h +++ b/tasks/hermes_adapters/stdio/stdio_io_client.h @@ -15,7 +15,7 @@ #include -#include "adapter/filesystem/filesystem_io_client.h" +#include "hermes_adapters/filesystem/filesystem_io_client.h" #include "stdio_api.h" using hermes::adapter::fs::AdapterStat; @@ -25,18 +25,8 @@ using hermes::adapter::fs::StdioApi; namespace hermes::adapter::fs { -/** State for the STDIO I/O trait */ -struct StdioIoClientHeader : public TraitHeader { - explicit StdioIoClientHeader(const std::string &trait_uuid, - const std::string &trait_name) - : TraitHeader(trait_uuid, trait_name, HERMES_TRAIT_FLUSH) {} -}; - /** A class to represent STDIO IO file system */ class StdioIoClient : public hermes::adapter::fs::FilesystemIoClient { - public: - HERMES_TRAIT_H(StdioIoClient, "stdio_io_client") - private: HERMES_STDIO_API_T real_api; /**< pointer to real APIs */ @@ -44,14 +34,6 @@ class StdioIoClient : public hermes::adapter::fs::FilesystemIoClient { /** Default constructor */ StdioIoClient() { real_api = HERMES_STDIO_API; - CreateHeader(trait_name_, trait_name_); - } - - /** Trait deserialization constructor */ - explicit StdioIoClient(hshm::charbuf ¶ms) { - (void) params; - real_api = HERMES_STDIO_API; - CreateHeader(trait_name_, trait_name_); } /** Virtual destructor */ diff --git a/adapter/vfd/CMakeLists.txt b/tasks/hermes_adapters/vfd/CMakeLists.txt similarity index 77% rename from adapter/vfd/CMakeLists.txt rename to tasks/hermes_adapters/vfd/CMakeLists.txt index 9bb21f614..1ef53b42d 100644 --- a/adapter/vfd/CMakeLists.txt +++ b/tasks/hermes_adapters/vfd/CMakeLists.txt @@ -21,22 +21,14 @@ set(HDF5_HERMES_VFD_PACKAGE_TARNAME "${HDF5_HERMES_VFD_PACKAGE}") # Dynamically loaded VFDs must be shared libraries set(HDF5_HERMES_VFD_LIBTYPE SHARED) -#----------------------------------------------------------------------------- -# Targets built within this project are exported at Install time for use -# by other projects. -#----------------------------------------------------------------------------- -if(NOT HDF5_HERMES_VFD_EXPORTED_TARGETS) - set(HDF5_HERMES_VFD_EXPORTED_TARGETS "${HDF5_HERMES_VFD_PACKAGE}-targets") -endif() - set(HDF5_HERMES_VFD_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.h + ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.h ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.cc) add_library(hdf5_hermes_vfd SHARED ${HDF5_HERMES_VFD_SRCS}) target_include_directories(hdf5_hermes_vfd - SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} -) + SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ) add_dependencies(hdf5_hermes_vfd hermes hermes_posix_io_client) target_link_libraries(hdf5_hermes_vfd @@ -50,44 +42,44 @@ set(HDF5_HERMES_VFD_EXPORTED_LIBS hdf5_hermes_vfd ${HDF5_HERMES_VFD_EXPORTED_LIB # Specify project header files to be installed #----------------------------------------------------------------------------- set(HDF5_HERMES_VFD_HEADERS - ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.h -) + ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.h + ) #----------------------------------------------------------------------------- # Add file(s) to CMake Install #----------------------------------------------------------------------------- install( - FILES - ${HDF5_HERMES_VFD_HEADERS} - DESTINATION - ${HERMES_INSTALL_INCLUDE_DIR} - COMPONENT - headers + FILES + ${HDF5_HERMES_VFD_HEADERS} + DESTINATION + ${LABSTOR_INSTALL_INCLUDE_DIR} + COMPONENT + headers ) #----------------------------------------------------------------------------- # Add Target(s) to CMake Install #----------------------------------------------------------------------------- install( - TARGETS - hdf5_hermes_vfd - EXPORT - ${HDF5_HERMES_VFD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} + TARGETS + hdf5_hermes_vfd + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Add Target(s) to CMake Install for import into other projects #----------------------------------------------------------------------------- install( - EXPORT - ${HDF5_HERMES_VFD_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hdf5_hermes_vfd - FILE - ${HDF5_HERMES_VFD_EXPORTED_TARGETS}.cmake + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hdf5_hermes_vfd + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- @@ -95,10 +87,10 @@ install( if(NOT HDF5_HERMES_VFD_EXTERNALLY_CONFIGURED) export( - TARGETS - ${HDF5_HERMES_VFD_EXPORTED_LIBS} - FILE - ${HDF5_HERMES_VFD_EXPORTED_TARGETS}.cmake + TARGETS + ${HDF5_HERMES_VFD_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake ) endif() @@ -130,10 +122,10 @@ set(HDF5_HERMES_VFD_LIBRARIES ${HDF5_HERMES_VFD_LIBRARIES} PARENT_SCOPE) # Hermes VFD external library dependencies # Need to generate -lib if not already passed set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} - ${HDF5_HERMES_VFD_EXT_PKG_LIB_DEPENDENCIES} - PARENT_SCOPE -) + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_LIB_DEPENDENCIES} + PARENT_SCOPE + ) foreach(lib_dep ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES}) # get library name get_filename_component(lib_name ${lib_dep} NAME_WE) @@ -157,10 +149,10 @@ set(HDF5_HERMES_VFD_LIB_DEPENDENCIES ${HDF5_HERMES_VFD_LIB_DEPENDENCIES} PARENT_ # External include dependencies set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - ${HDF5_HERMES_VFD_EXT_PKG_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_INCLUDE_DEPENDENCIES} + PARENT_SCOPE + ) if(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES) list(REMOVE_DUPLICATES HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES) endif() @@ -170,15 +162,14 @@ endforeach() set(HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES ${HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES} PARENT_SCOPE) set(HDF5_HERMES_VFD_INCLUDES_BUILD_TIME - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE + ) set(HDF5_HERMES_VFD_INCLUDES_INSTALL_TIME - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - PARENT_SCOPE -) + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE) diff --git a/adapter/vfd/H5FDhermes.cc b/tasks/hermes_adapters/vfd/H5FDhermes.cc similarity index 99% rename from adapter/vfd/H5FDhermes.cc rename to tasks/hermes_adapters/vfd/H5FDhermes.cc index a9f2ce0f5..60fff9455 100644 --- a/adapter/vfd/H5FDhermes.cc +++ b/tasks/hermes_adapters/vfd/H5FDhermes.cc @@ -39,7 +39,7 @@ #include "H5PLextern.h" #include "H5FDhermes.h" /* Hermes file driver */ -#include "adapter/posix/posix_io_client.h" +#include "hermes_adapters/posix/posix_io_client.h" #include "posix/posix_fs_api.h" /** diff --git a/adapter/vfd/H5FDhermes.h b/tasks/hermes_adapters/vfd/H5FDhermes.h similarity index 100% rename from adapter/vfd/H5FDhermes.h rename to tasks/hermes_adapters/vfd/H5FDhermes.h diff --git a/adapter/vfd/README.md b/tasks/hermes_adapters/vfd/README.md similarity index 95% rename from adapter/vfd/README.md rename to tasks/hermes_adapters/vfd/README.md index 1c48793ca..9667f4762 100644 --- a/adapter/vfd/README.md +++ b/tasks/hermes_adapters/vfd/README.md @@ -31,7 +31,7 @@ Now we must tell the HDF5 library where to find the Hermes VFD. That is done with the following environment variable: ```sh -HDF5_PLUGIN_PATH=/lib/hermes_vfd +HDF5_PLUGIN_PATH=/lib/hermes_vfd ``` The Hermes VFD has two configuration options. @@ -65,14 +65,14 @@ initialization and finalization: ```sh HERMES_CONF=/hermes.yaml -LD_PRELOAD=/hermes_vfd/libhdf5_hermes_vfd.so +LD_PRELOAD=/hermes_vfd/libhdf5_hermes_vfd.so ``` Heres is a full example of running an HDF5 app with the Hermes VFD: ```sh HDF5_DRIVER=hermes \ - HDF5_PLUGIN_PATH=/hermes.yaml \ ./my_hdf5_app diff --git a/tasks/hermes_blob_mdm/CMakeLists.txt b/tasks/hermes_blob_mdm/CMakeLists.txt new file mode 100644 index 000000000..61dbbc349 --- /dev/null +++ b/tasks/hermes_blob_mdm/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Hermes Blob MDM Task Library +#------------------------------------------------------------------------------ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h new file mode 100644 index 000000000..abb5be9cf --- /dev/null +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -0,0 +1,428 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_hermes_blob_mdm_H_ +#define LABSTOR_hermes_blob_mdm_H_ + +#include "hermes_blob_mdm_tasks.h" + +namespace hermes::blob_mdm { + +/** Create hermes_blob_mdm requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Initialize directly using TaskStateId */ + void Init(const TaskStateId &id) { + id_ = id; + queue_id_ = QueueId(id_); + } + + /** Create a hermes_blob_mdm */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info); + } + void AsyncCreateComplete(ConstructTask *task) { + if (task->IsComplete()) { + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate); + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + LPointer task = AsyncCreateRoot(std::forward(args)...); + task->Wait(); + AsyncCreateComplete(task.ptr_); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /**==================================== + * Blob Operations + * ===================================*/ + + /** Sets the BUCKET MDM */ + void AsyncSetBucketMdmConstruct(SetBucketMdmTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &blob_mdm_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_, blob_mdm_id); + } + void SetBucketMdmRoot(const DomainId &domain_id, + const TaskStateId &blob_mdm_id) { + LPointer> push_task = + AsyncSetBucketMdmRoot(domain_id, blob_mdm_id); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(SetBucketMdm); + + /** + * Get \a blob_name BLOB from \a bkt_id bucket + * */ + void AsyncGetOrCreateBlobIdConstruct(GetOrCreateBlobIdTask* task, + const TaskNode &task_node, + TagId tag_id, + const hshm::charbuf &blob_name) { + u32 hash = std::hash{}(blob_name); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id, blob_name); + } + BlobId GetOrCreateBlobIdRoot(TagId tag_id, const hshm::charbuf &blob_name) { + LPointer> push_task = + AsyncGetOrCreateBlobIdRoot(tag_id, blob_name); + push_task->Wait(); + GetOrCreateBlobIdTask *task = push_task->get(); + BlobId blob_id = task->blob_id_; + LABSTOR_CLIENT->DelTask(push_task); + return blob_id; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateBlobId); + + /** + * Create a blob's metadata + * + * @param tag_id id of the bucket + * @param blob_name semantic blob name + * @param blob_id the id of the blob + * @param blob_off the offset of the data placed in existing blob + * @param blob_size the amount of data being placed + * @param blob a SHM pointer to the data to place + * @param score the current score of the blob + * @param replace whether to replace the blob if it exists + * @param[OUT] did_create whether the blob was created or not + * */ + void AsyncPutBlobConstruct( + PutBlobTask *task, + const TaskNode &task_node, + TagId tag_id, const hshm::charbuf &blob_name, + BlobId &blob_id, size_t blob_off, size_t blob_size, + const hipc::Pointer &blob, float score, + bitfield32_t flags, + Context ctx = Context(), + bitfield32_t task_flags = bitfield32_t(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER)) { + HILOG(kDebug, "Beginning PUT (task_node={})", task_node); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_name, blob_id, + blob_off, blob_size, + blob, score, flags, ctx); + task->task_flags_ = task_flags; + HILOG(kDebug, "Constructed PUT (task_node={})", task_node); + } + LABSTOR_TASK_NODE_PUSH_ROOT(PutBlob); + + /** Get a blob's data */ + void AsyncGetBlobConstruct(GetBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id, + size_t off, + ssize_t data_size, + hipc::Pointer &data, + Context ctx = Context()) { + HILOG(kDebug, "Beginning GET (task_node={})", task_node); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id, off, data_size, data, ctx); + } + size_t GetBlobRoot(const TagId &tag_id, + const BlobId &blob_id, + size_t off, + ssize_t data_size, + hipc::Pointer &data, + Context ctx = Context()) { + LPointer> push_task = + AsyncGetBlobRoot(tag_id, blob_id, off, data_size, data); + push_task->Wait(); + GetBlobTask *task = push_task->get(); + data = task->data_; + size_t true_size = task->data_size_; + LABSTOR_CLIENT->DelTask(push_task); + return true_size; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetBlob); + + /** + * Reorganize a blob + * + * @param blob_id id of the blob being reorganized + * @param score the new score of the blob + * @param node_id the node to reorganize the blob to + * */ + void AsyncReorganizeBlobConstruct(ReorganizeBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id, + float score, + u32 node_id) { + HILOG(kDebug, "Beginning REORGANIZE (task_node={})", task_node); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id, score, node_id); + } + LABSTOR_TASK_NODE_PUSH_ROOT(ReorganizeBlob); + + /** + * Tag a blob + * + * @param blob_id id of the blob being tagged + * @param tag_name tag name + * */ + void AsyncTagBlobConstruct(TagBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id, + const TagId &tag) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id, tag); + } + void TagBlobRoot(const TagId &tag_id, + const BlobId &blob_id, + const TagId &tag) { + LPointer> push_task = + AsyncTagBlobRoot(tag_id, blob_id, tag); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(TagBlob); + + /** + * Check if blob has a tag + * */ + void AsyncBlobHasTagConstruct(BlobHasTagTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id, + const TagId &tag) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id, tag); + } + bool BlobHasTagRoot(const TagId &tag_id, + const BlobId &blob_id, + const TagId &tag) { + LPointer> push_task = + AsyncBlobHasTagRoot(tag_id, blob_id, tag); + push_task->Wait(); + BlobHasTagTask *task = push_task->get(); + bool has_tag = task->has_tag_; + LABSTOR_CLIENT->DelTask(push_task); + return has_tag; + } + LABSTOR_TASK_NODE_PUSH_ROOT(BlobHasTag); + + /** + * Get \a blob_name BLOB from \a bkt_id bucket + * */ + void AsyncGetBlobIdConstruct(GetBlobIdTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const hshm::charbuf &blob_name) { + u32 hash = std::hash{}(blob_name); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id, blob_name); + } + BlobId GetBlobIdRoot(const TagId &tag_id, + const hshm::charbuf &blob_name) { + LPointer> push_task = + AsyncGetBlobIdRoot(tag_id, blob_name); + push_task->Wait(); + GetBlobIdTask *task = push_task->get(); + BlobId blob_id = task->blob_id_; + LABSTOR_CLIENT->DelTask(push_task); + return blob_id; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobId); + + /** + * Get \a blob_name BLOB name from \a blob_id BLOB id + * */ + void AsyncGetBlobNameConstruct(GetBlobNameTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id); + } + std::string GetBlobNameRoot(const TagId &tag_id, + const BlobId &blob_id) { + LPointer> push_task = + AsyncGetBlobNameRoot(tag_id, blob_id); + push_task->Wait(); + GetBlobNameTask *task = push_task->get(); + std::string blob_name = task->blob_name_->str(); + LABSTOR_CLIENT->DelTask(push_task); + return blob_name; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobName); + + /** + * Get \a size from \a blob_id BLOB id + * */ + void AsyncGetBlobSizeConstruct(GetBlobSizeTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id) { + HILOG(kDebug, "Getting blob size {}", task_node); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id); + } + size_t GetBlobSizeRoot(const TagId &tag_id, + const BlobId &blob_id) { + LPointer> push_task = + AsyncGetBlobSizeRoot(tag_id, blob_id); + push_task->Wait(); + GetBlobSizeTask *task = push_task->get(); + size_t size = task->size_; + LABSTOR_CLIENT->DelTask(push_task); + return size; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobSize); + + /** + * Get \a score from \a blob_id BLOB id + * */ + void AsyncGetBlobScoreConstruct(GetBlobScoreTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id); + } + float GetBlobScoreRoot(const TagId &tag_id, + const BlobId &blob_id) { + LPointer> push_task = + AsyncGetBlobScoreRoot(tag_id, blob_id); + push_task->Wait(); + GetBlobScoreTask *task = push_task->get(); + float score = task->score_; + LABSTOR_CLIENT->DelTask(push_task); + return score; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobScore); + + /** + * Get \a blob_id blob's buffers + * */ + void AsyncGetBlobBuffersConstruct(GetBlobBuffersTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id); + } + std::vector GetBlobBuffersRoot(const TagId &tag_id, + const BlobId &blob_id) { + LPointer> push_task = AsyncGetBlobBuffersRoot(tag_id, blob_id); + push_task->Wait(); + GetBlobBuffersTask *task = push_task->get(); + std::vector buffers = + hshm::to_stl_vector(*task->buffers_); + LABSTOR_CLIENT->DelTask(push_task); + return buffers; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobBuffers) + + /** + * Rename \a blob_id blob to \a new_blob_name new blob name + * in \a bkt_id bucket. + * */ + void AsyncRenameBlobConstruct(RenameBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id, + const hshm::charbuf &new_blob_name) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id, new_blob_name); + } + void RenameBlobRoot(const TagId &tag_id, + const BlobId &blob_id, + const hshm::charbuf &new_blob_name) { + LPointer> push_task = + AsyncRenameBlobRoot(tag_id, blob_id, new_blob_name); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(RenameBlob); + + /** + * Truncate a blob to a new size + * */ + void AsyncTruncateBlobConstruct(TruncateBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id, + size_t new_size) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), id_, + tag_id, blob_id, new_size); + } + void TruncateBlobRoot(const TagId &tag_id, + const BlobId &blob_id, + size_t new_size) { + LPointer> push_task = + AsyncTruncateBlobRoot(tag_id, blob_id, new_size); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(TruncateBlob); + + /** + * Destroy \a blob_id blob in \a bkt_id bucket + * */ + void AsyncDestroyBlobConstruct(DestroyBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(blob_id.node_id_), + id_, tag_id, blob_id); + } + void DestroyBlobRoot(const TagId &tag_id, + const BlobId &blob_id) { + LPointer> push_task = + AsyncDestroyBlobRoot(tag_id, blob_id); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(DestroyBlob); +}; + +} // namespace labstor + +#endif // LABSTOR_hermes_blob_mdm_H_ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h new file mode 100644 index 000000000..251fafc96 --- /dev/null +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -0,0 +1,594 @@ +#ifndef LABSTOR_HERMES_BLOB_MDM_LIB_EXEC_H_ +#define LABSTOR_HERMES_BLOB_MDM_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kPutBlob: { + PutBlob(reinterpret_cast(task)); + break; + } + case Method::kGetBlob: { + GetBlob(reinterpret_cast(task)); + break; + } + case Method::kTruncateBlob: { + TruncateBlob(reinterpret_cast(task)); + break; + } + case Method::kDestroyBlob: { + DestroyBlob(reinterpret_cast(task)); + break; + } + case Method::kTagBlob: { + TagBlob(reinterpret_cast(task)); + break; + } + case Method::kBlobHasTag: { + BlobHasTag(reinterpret_cast(task)); + break; + } + case Method::kGetBlobId: { + GetBlobId(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateBlobId: { + GetOrCreateBlobId(reinterpret_cast(task)); + break; + } + case Method::kGetBlobName: { + GetBlobName(reinterpret_cast(task)); + break; + } + case Method::kGetBlobSize: { + GetBlobSize(reinterpret_cast(task)); + break; + } + case Method::kGetBlobScore: { + GetBlobScore(reinterpret_cast(task)); + break; + } + case Method::kGetBlobBuffers: { + GetBlobBuffers(reinterpret_cast(task)); + break; + } + case Method::kRenameBlob: { + RenameBlob(reinterpret_cast(task)); + break; + } + case Method::kReorganizeBlob: { + ReorganizeBlob(reinterpret_cast(task)); + break; + } + case Method::kSetBucketMdm: { + SetBucketMdm(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kPutBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kTruncateBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestroyBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kTagBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kBlobHasTag: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetBlobId: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateBlobId: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetBlobName: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetBlobSize: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetBlobScore: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetBlobBuffers: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kRenameBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kReorganizeBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kSetBucketMdm: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kPutBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kTruncateBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestroyBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kTagBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kBlobHasTag: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetBlobId: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateBlobId: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetBlobName: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetBlobSize: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetBlobScore: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetBlobBuffers: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kRenameBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kReorganizeBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kSetBucketMdm: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kPutBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTruncateBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kBlobHasTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetOrCreateBlobId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobName: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobSize: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobScore: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobBuffers: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRenameBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kReorganizeBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetBucketMdm: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kPutBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kTruncateBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestroyBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kTagBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kBlobHasTag: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetBlobId: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetOrCreateBlobId: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetBlobName: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetBlobSize: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetBlobScore: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetBlobBuffers: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kRenameBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kReorganizeBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kSetBucketMdm: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kPutBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTruncateBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kBlobHasTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetOrCreateBlobId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobName: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobSize: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobScore: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetBlobBuffers: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRenameBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kReorganizeBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetBucketMdm: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kPutBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kTruncateBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestroyBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kTagBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kBlobHasTag: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetBlobId: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateBlobId: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetBlobName: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetBlobSize: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetBlobScore: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetBlobBuffers: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kRenameBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kReorganizeBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kSetBucketMdm: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kPutBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kTruncateBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestroyBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kTagBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kBlobHasTag: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetBlobId: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetOrCreateBlobId: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetBlobName: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetBlobSize: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetBlobScore: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetBlobBuffers: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kRenameBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kReorganizeBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kSetBucketMdm: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h new file mode 100644 index 000000000..2cbd3cb79 --- /dev/null +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h @@ -0,0 +1,23 @@ +#ifndef LABSTOR_HERMES_BLOB_MDM_METHODS_H_ +#define LABSTOR_HERMES_BLOB_MDM_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kPutBlob = kLast + 0; + TASK_METHOD_T kGetBlob = kLast + 1; + TASK_METHOD_T kTruncateBlob = kLast + 2; + TASK_METHOD_T kDestroyBlob = kLast + 3; + TASK_METHOD_T kTagBlob = kLast + 4; + TASK_METHOD_T kBlobHasTag = kLast + 6; + TASK_METHOD_T kGetBlobId = kLast + 8; + TASK_METHOD_T kGetOrCreateBlobId = kLast + 9; + TASK_METHOD_T kGetBlobName = kLast + 10; + TASK_METHOD_T kGetBlobSize = kLast + 11; + TASK_METHOD_T kGetBlobScore = kLast + 12; + TASK_METHOD_T kGetBlobBuffers = kLast + 13; + TASK_METHOD_T kRenameBlob = kLast + 14; + TASK_METHOD_T kReorganizeBlob = kLast + 15; + TASK_METHOD_T kSetBucketMdm = kLast + 16; +}; + +#endif // LABSTOR_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml new file mode 100644 index 000000000..d6c7b93db --- /dev/null +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml @@ -0,0 +1,17 @@ +kPutBlob: 0 +kGetBlob: 1 +kTruncateBlob: 2 +kDestroyBlob: 3 +kTagBlob: 4 +#kUntagBlob: 5 +kBlobHasTag: 6 +#kGetBlobTags: 7 +kGetBlobId: 8 +kGetOrCreateBlobId: 9 +kGetBlobName: 10 +kGetBlobSize: 11 +kGetBlobScore: 12 +kGetBlobBuffers: 13 +kRenameBlob: 14 +kReorganizeBlob: 15 +kSetBucketMdm: 16 \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h new file mode 100644 index 000000000..e8813efcb --- /dev/null +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -0,0 +1,1088 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ +#define LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "hermes/hermes_types.h" +#include "bdev/bdev.h" +#include "labstor/api/labstor_client.h" +#include "proc_queue/proc_queue.h" + +namespace hermes::blob_mdm { + +#include "hermes_blob_mdm_methods.h" +#include "labstor/labstor_namespace.h" + +using labstor::Task; +using labstor::TaskFlags; +using labstor::DataTransfer; + +/** Phases of the construct task */ +using labstor::Admin::CreateTaskStatePhase; +class ConstructTaskPhase : public CreateTaskStatePhase { + public: + TASK_METHOD_T kCreateTaskStates = kLast + 0; + TASK_METHOD_T kWaitForTaskStates = kLast + 1; +}; + +/** + * A task to create hermes_mdm + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "hermes_blob_mdm", id, queue_info) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy hermes_mdm */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** Set the BUCKET MDM ID */ +struct SetBucketMdmTask : public Task, TaskFlags { + IN TaskStateId bkt_mdm_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + SetBucketMdmTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + SetBucketMdmTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TaskStateId &bkt_mdm) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = state_id; + method_ = Method::kSetBucketMdm; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + bkt_mdm_ = bkt_mdm; + } + + /** Destructor */ + ~SetBucketMdmTask() {} + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(bkt_mdm_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } + + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} +}; + +/**==================================== + * Blob Operations + * ===================================*/ + +/** + * Get \a blob_name BLOB from \a bkt_id bucket + * */ +struct GetOrCreateBlobIdTask : public Task, TaskFlags { + IN TagId tag_id_; + IN hipc::ShmArchive blob_name_; + OUT BlobId blob_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetOrCreateBlobIdTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetOrCreateBlobIdTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const hshm::charbuf &blob_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = std::hash{}(blob_name); + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetOrCreateBlobId; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + HSHM_MAKE_AR(blob_name_, alloc, blob_name) + } + + /** Destructor */ + ~GetOrCreateBlobIdTask() { + HSHM_DESTROY_AR(blob_name_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_name_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(blob_id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** Phases for the put task */ +class PutBlobPhase { + public: + TASK_METHOD_T kCreate = 0; + TASK_METHOD_T kAllocate = 1; + TASK_METHOD_T kWaitAllocate = 2; + TASK_METHOD_T kModify = 3; + TASK_METHOD_T kWaitModify = 4; +}; + +#define HERMES_BLOB_REPLACE BIT_OPT(u32, 0) +#define HERMES_BLOB_APPEND BIT_OPT(u32, 1) +#define HERMES_DID_STAGE_IN BIT_OPT(u32, 2) +#define HERMES_BLOB_DID_CREATE BIT_OPT(u32, 3) + + +/** A task to put data in a blob */ +struct PutBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN hipc::ShmArchive blob_name_; + IN size_t blob_off_; + IN size_t data_size_; + IN hipc::Pointer data_; + IN float score_; + IN bitfield32_t flags_; + IN BlobId blob_id_; + IN hipc::ShmArchive filename_; + IN size_t page_size_; + + TEMP int phase_ = PutBlobPhase::kCreate; + TEMP int plcmnt_idx_ = 0; + TEMP int sub_plcmnt_idx_ = 0; + TEMP LPointer data_ptr_; + TEMP size_t data_off_; + TEMP hermes::bdev::AllocateTask *cur_bdev_alloc_; + TEMP hipc::ShmArchive> schema_; + TEMP hipc::ShmArchive> bdev_writes_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + PutBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + PutBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const hshm::charbuf &blob_name, + const BlobId &blob_id, + size_t blob_off, + size_t data_size, + const hipc::Pointer &data, + float score, + bitfield32_t flags, + const Context &ctx) : Task(alloc) { + // Initialize task + HILOG(kDebug, "Beginning PUT task constructor") + task_node_ = task_node; + lane_hash_ = blob_id_.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kPutBlob; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + HILOG(kDebug, "Setting blob name {}", blob_name.str()); + HSHM_MAKE_AR(blob_name_, alloc, blob_name); + blob_id_ = blob_id; + blob_off_ = blob_off; + data_size_ = data_size; + data_ = data; + score_ = score; + flags_ = flags; + HSHM_MAKE_AR(filename_, alloc, ctx.filename_); + page_size_ = ctx.page_size_; + HILOG(kDebug, "Finished setting blob name {}", blob_name.str()); + } + + /** Destructor */ + ~PutBlobTask() { + HSHM_DESTROY_AR(blob_name_); + HSHM_DESTROY_AR(filename_); + if (IsDataOwner()) { + LABSTOR_CLIENT->FreeBuffer(data_); + } + } + + /** (De)serialize message call */ + template + void SaveStart(Ar &ar) { + DataTransfer xfer(DT_RECEIVER_READ, + HERMES_MEMORY_MANAGER->Convert(data_), + data_size_, domain_id_); + task_serialize(ar); + ar & xfer; + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_); + } + + /** Deserialize message call */ + template + void LoadStart(Ar &ar) { + DataTransfer xfer; + task_serialize(ar); + ar & xfer; + data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** Phases for the get task */ +class GetBlobPhase { + public: + TASK_METHOD_T kStart = 0; + TASK_METHOD_T kWait = 1; +}; + +/** A task to get data from a blob */ +struct GetBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + IN size_t blob_off_; + IN hipc::Pointer data_; + IN hipc::ShmArchive filename_; + IN size_t page_size_; + INOUT ssize_t data_size_; + TEMP int phase_ = GetBlobPhase::kStart; + TEMP hipc::ShmArchive> bdev_reads_; + TEMP PutBlobTask *stage_task_ = nullptr; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id, + size_t off, + ssize_t data_size, + hipc::Pointer &data, + const Context &ctx) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetBlob; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + blob_id_ = blob_id; + blob_off_ = off; + data_size_ = data_size; + data_ = data; + HSHM_MAKE_AR(filename_, alloc, ctx.filename_); + page_size_ = ctx.page_size_; + } + + /** (De)serialize message call */ + template + void SaveStart(Ar &ar) { + // TODO(llogan): Make it so Get takes as input a buffer, instead of returning one + DataTransfer xfer(DT_RECEIVER_WRITE, + HERMES_MEMORY_MANAGER->Convert(data_), + data_size_, domain_id_); + task_serialize(ar); + ar & xfer; + ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_); + } + + /** Deserialize message call */ + template + void LoadStart(Ar &ar) { + DataTransfer xfer; + task_serialize(ar); + ar & xfer; + data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); + ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to tag a blob */ +struct TagBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + IN TagId tag_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + TagBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + TagBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id, + const TagId &tag) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kTagBlob; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + blob_id_ = blob_id; + tag_ = tag; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(blob_id_, tag_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** + * Check if blob has a tag + * */ +struct BlobHasTagTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + IN TagId tag_; + OUT bool has_tag_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + BlobHasTagTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + BlobHasTagTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id, + const TagId &tag) : Task(alloc) { + // Initialize task + task_node_ = task_node; + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kBlobHasTag; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + blob_id_ = blob_id; + tag_ = tag; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_, tag_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(has_tag_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** + * Get \a blob_name BLOB from \a bkt_id bucket + * */ +struct GetBlobIdTask : public Task, TaskFlags { + IN TagId tag_id_; + IN hipc::ShmArchive blob_name_; + OUT BlobId blob_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobIdTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobIdTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const hshm::charbuf &blob_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = std::hash{}(blob_name); + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetBlobId; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + HSHM_MAKE_AR(blob_name_, alloc, blob_name) + } + + /** Destructor */ + ~GetBlobIdTask() { + HSHM_DESTROY_AR(blob_name_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_name_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(blob_id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** + * Get \a blob_name BLOB name from \a blob_id BLOB id + * */ +struct GetBlobNameTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + OUT hipc::ShmArchive blob_name_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobNameTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobNameTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetBlobName; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + blob_id_ = blob_id; + HSHM_MAKE_AR0(blob_name_, alloc) + } + + /** Destructor */ + ~GetBlobNameTask() { + HSHM_DESTROY_AR(blob_name_) + }; + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(blob_name_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** Get \a score from \a blob_id BLOB id */ +struct GetBlobSizeTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + OUT size_t size_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobSizeTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobSizeTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetBlobSize; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + blob_id_ = blob_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(size_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** Get \a score from \a blob_id BLOB id */ +struct GetBlobScoreTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + OUT float score_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobScoreTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobScoreTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetBlobScore; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + blob_id_ = blob_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(score_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** Get \a blob_id blob's buffers */ +struct GetBlobBuffersTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + OUT hipc::ShmArchive> buffers_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobBuffersTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetBlobBuffersTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetBlobBuffers; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + blob_id_ = blob_id; + HSHM_MAKE_AR0(buffers_, alloc) + } + + /** Destructor */ + ~GetBlobBuffersTask() { + HSHM_DESTROY_AR(buffers_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(buffers_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** + * Rename \a blob_id blob to \a new_blob_name new blob name + * in \a bkt_id bucket. + * */ +struct RenameBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + IN hipc::ShmArchive new_blob_name_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + RenameBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + RenameBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id, + const hshm::charbuf &new_blob_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kRenameBlob; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom + tag_id_ = tag_id; + blob_id_ = blob_id; + HSHM_MAKE_AR(new_blob_name_, alloc, new_blob_name) + } + + /** Destructor */ + ~RenameBlobTask() { + HSHM_DESTROY_AR(new_blob_name_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(new_blob_name_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to truncate a blob */ +struct TruncateBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + IN u64 size_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + TruncateBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + TruncateBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id, + u64 size) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kTruncateBlob; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + blob_id_ = blob_id; + size_ = size; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_, size_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** Phases of the destroy blob task */ +struct DestroyBlobPhase { + TASK_METHOD_T kFreeBuffers = 0; + TASK_METHOD_T kWaitFreeBuffers = 1; +}; + +/** A task to destroy a blob */ +struct DestroyBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + TEMP int phase_ = DestroyBlobPhase::kFreeBuffers; + TEMP hipc::ShmArchive> free_tasks_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestroyBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestroyBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kDestroyBlob; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + blob_id_ = blob_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** Phases of the destroy blob task */ +struct ReorganizeBlobPhase { + TASK_METHOD_T kGet = 0; + TASK_METHOD_T kWaitGet = 1; + TASK_METHOD_T kPut = 2; +}; + +/** A task to reorganize a blob's composition in the hierarchy */ +struct ReorganizeBlobTask : public Task, TaskFlags { + IN BlobId blob_id_; + IN float score_; + IN u32 node_id_; + TEMP int phase_ = ReorganizeBlobPhase::kGet; + TEMP hipc::Pointer data_; + TEMP size_t data_size_; + TEMP GetBlobTask *get_task_; + TEMP PutBlobTask *put_task_; + TEMP TagId tag_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ReorganizeBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ReorganizeBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const BlobId &blob_id, + float score, + u32 node_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = blob_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kReorganizeBlob; + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + blob_id_ = blob_id; + score_ = score; + node_id_ = node_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_, score_, node_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +} // namespace hermes::blob_mdm + +#endif //LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ diff --git a/tasks/hermes_blob_mdm/src/CMakeLists.txt b/tasks/hermes_blob_mdm/src/CMakeLists.txt new file mode 100644 index 000000000..4f91d37a3 --- /dev/null +++ b/tasks/hermes_blob_mdm/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(hermes_blob_mdm SHARED + hermes_blob_mdm.cc) +add_dependencies(hermes_blob_mdm ${Labstor_RUNTIME_DEPS}) +target_link_libraries(hermes_blob_mdm ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + hermes_blob_mdm + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + hermes_blob_mdm + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(hermes_blob_mdm) +endif() diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc new file mode 100644 index 000000000..5a242883c --- /dev/null +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -0,0 +1,652 @@ +// +// Created by lukemartinlogan on 6/29/23. +// +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "hermes/config_server.h" +#include "hermes_blob_mdm/hermes_blob_mdm.h" +#include "hermes_adapters/mapper/mapper_factory.h" +#include "hermes/dpe/dpe_factory.h" +#include "hermes_adapters/posix/posix_api.h" +#include "bdev/bdev.h" + +namespace hermes::blob_mdm { + +/** Type name simplification for the various map types */ +typedef std::unordered_map BLOB_ID_MAP_T; +typedef std::unordered_map BLOB_MAP_T; +typedef hipc::mpsc_queue IO_PATTERN_LOG_T; + +class Server : public TaskLib { + public: + /**==================================== + * Configuration + * ===================================*/ + u32 node_id_; + + /**==================================== + * Maps + * ===================================*/ + BLOB_ID_MAP_T blob_id_map_; + BLOB_MAP_T blob_map_; + std::atomic id_alloc_; + + /**==================================== + * I/O pattern log + * ===================================*/ + IO_PATTERN_LOG_T *io_pattern_log_; + bool enable_io_tracing_; + bool is_mpi_; + + /**==================================== + * Targets + devices + * ===================================*/ + std::vector target_tasks_; + std::vector targets_; + std::unordered_map target_map_; + Client blob_mdm_; + bucket_mdm::Client bkt_mdm_; + + public: + Server() = default; + + void Construct(ConstructTask *task) { + id_alloc_ = 0; + node_id_ = LABSTOR_CLIENT->node_id_; + switch (task->phase_) { + case ConstructTaskPhase::kCreateTaskStates: { + target_tasks_.reserve(HERMES_SERVER_CONF.devices_.size()); + for (DeviceInfo &dev : HERMES_SERVER_CONF.devices_) { + std::string dev_type; + if (dev.mount_dir_.empty()) { + dev_type = "ram_bdev"; + dev.mount_point_ = hshm::Formatter::format("{}/{}", dev.mount_dir_, dev.dev_name_); + } else { + dev_type = "posix_bdev"; + } + targets_.emplace_back(); + bdev::Client &client = targets_.back(); + bdev::ConstructTask *create_task = client.AsyncCreate( + task->task_node_ + 1, + DomainId::GetLocal(), + "hermes_" + dev.dev_name_, + dev_type, + dev).ptr_; + target_tasks_.emplace_back(create_task); + } + task->phase_ = ConstructTaskPhase::kWaitForTaskStates; + } + + case ConstructTaskPhase::kWaitForTaskStates: { + for (int i = (int)target_tasks_.size() - 1; i >= 0; --i) { + bdev::ConstructTask *tgt_task = target_tasks_[i]; + if (!tgt_task->IsComplete()) { + return; + } + bdev::Client &client = targets_[i]; + client.AsyncCreateComplete(tgt_task); + target_map_.emplace(client.id_, &client); + target_tasks_.pop_back(); + } + blob_mdm_.Init(id_); + } + } + + // Create targets + HILOG(kInfo, "Created Blob MDM") + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + private: + /** Get the globally unique blob name */ + const hshm::charbuf GetBlobNameWithBucket(TagId tag_id, const hshm::charbuf &blob_name) { + hshm::charbuf new_name(sizeof(TagId) + blob_name.size()); + labstor::LocalSerialize srl(new_name); + srl << tag_id.node_id_; + srl << tag_id.unique_; + srl << blob_name; + return new_name; + } + + public: + /** + * Set the Bucket MDM + * */ + void SetBucketMdm(SetBucketMdmTask *task) { + bkt_mdm_.Init(task->bkt_mdm_); + task->SetModuleComplete(); + } + + /** + * Create a blob's metadata + * */ + void PutBlob(PutBlobTask *task) { + if (task->phase_ == PutBlobPhase::kCreate) { + PutBlobCreatePhase(task); + } + if (task->phase_ == PutBlobPhase::kAllocate) { + PutBlobAllocatePhase(task); + } + if (task->phase_ == PutBlobPhase::kWaitAllocate) { + if (!task->cur_bdev_alloc_->IsComplete()){ + return; + } + PutBlobWaitAllocatePhase(task); + } + if (task->phase_ == PutBlobPhase::kModify) { + PutBlobModifyPhase(task); + } + if (task->phase_ == PutBlobPhase::kWaitModify) { + PutBlobWaitModifyPhase(task); + if (task->phase_ == PutBlobPhase::kWaitModify) { + return; + } + } + } + + /** Create blob / update metadata for the PUT */ + void PutBlobCreatePhase(PutBlobTask *task) { + HILOG(kDebug, "PutBlobPhase::kCreate {}", task->blob_id_); + // Get the blob info data structure + hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->flags_.SetBits(HERMES_BLOB_DID_CREATE); + } + if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { + blob_map_.emplace(task->blob_id_, BlobInfo()); + } + BlobInfo &blob_info = blob_map_[task->blob_id_]; + + // Update the blob info + if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { + // Update blob info + blob_info.name_ = std::move(blob_name); + blob_info.blob_id_ = task->blob_id_; + blob_info.tag_id_ = task->tag_id_; + blob_info.blob_size_ = 0; + blob_info.max_blob_size_ = 0; + blob_info.score_ = task->score_; + blob_info.mod_count_ = 0; + blob_info.access_freq_ = 0; + blob_info.last_flush_ = 0; + blob_info.UpdateWriteStats(); + } else { + // Modify existing blob + blob_info.UpdateWriteStats(); + } + if (task->flags_.Any(HERMES_BLOB_REPLACE)) { + PutBlobFreeBuffersPhase(blob_info, task); + } + + // Stage in blob data from FS + task->data_ptr_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->data_); + task->data_ptr_.shm_ = task->data_; + task->data_off_ = 0; + if (task->filename_->size() > 0 && blob_info.blob_size_ == 0) { + adapter::BlobPlacement plcmnt; + plcmnt.DecodeBlobName(*task->blob_name_); + LPointer new_data_ptr = LABSTOR_CLIENT->AllocateBuffer(task->page_size_); + int fd = HERMES_POSIX_API->open(task->filename_->c_str(), O_RDONLY); + int ret = HERMES_POSIX_API->pread(fd, new_data_ptr.ptr_, task->page_size_, plcmnt.bucket_off_); + if (ret < 0) { + // TODO(llogan): ret != page_size_ will require knowing file size before-hand + HELOG(kError, "Failed to stage in {} bytes from {}", task->page_size_, task->filename_->str()); + } + HERMES_POSIX_API->close(fd); + memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, task->data_ptr_.ptr_, task->page_size_); + task->data_ptr_ = new_data_ptr; + task->blob_off_ = 0; + task->data_size_ = task->page_size_; + task->data_off_ = plcmnt.bucket_off_ + task->blob_off_ + task->data_size_; + task->flags_.SetBits(HERMES_DID_STAGE_IN); + } + + // Determine amount of additional buffering space needed + Context ctx; + size_t needed_space = task->blob_off_ + task->data_size_; + size_t size_diff = 0; + if (needed_space > blob_info.max_blob_size_) { + size_diff = needed_space - blob_info.max_blob_size_; + } + blob_info.blob_size_ += size_diff; + HILOG(kDebug, "The size diff is {} bytes", size_diff) + + // Initialize archives + HSHM_MAKE_AR0(task->schema_, nullptr); + HSHM_MAKE_AR0(task->bdev_writes_, nullptr); + + // Use DPE + if (size_diff > 0) { + auto *dpe = DpeFactory::Get(ctx.dpe_); + dpe->Placement({size_diff}, targets_, ctx, *task->schema_); + task->phase_ = PutBlobPhase::kAllocate; + } else { + task->phase_ = PutBlobPhase::kModify; + } + } + + /** Release buffers */ + void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task) { + for (BufferInfo &buf : blob_info.buffers_) { + TargetInfo &target = *target_map_[buf.tid_]; + std::vector buf_vec = {buf}; + target.AsyncFree(task->task_node_ + 1, std::move(buf_vec), true); + } + blob_info.buffers_.clear(); + blob_info.max_blob_size_ = 0; + blob_info.blob_size_ = 0; + } + + /** Resolve the current sub-placement using BPM */ + void PutBlobAllocatePhase(PutBlobTask *task) { + BlobInfo &blob_info = blob_map_[task->blob_id_]; + PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; + SubPlacement &placement = schema.plcmnts_[task->sub_plcmnt_idx_]; + TargetInfo &bdev = *target_map_[placement.tid_]; + HILOG(kDebug, "Allocating {} bytes of blob {}", placement.size_, task->blob_id_); + task->cur_bdev_alloc_ = bdev.AsyncAllocate(task->task_node_ + 1, + placement.size_, + blob_info.buffers_).ptr_; + task->phase_ = PutBlobPhase::kWaitAllocate; + } + + /** Wait for the current-subplacement to complete */ + void PutBlobWaitAllocatePhase(PutBlobTask *task) { + BlobInfo &blob_info = blob_map_[task->blob_id_]; + PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; + ++task->sub_plcmnt_idx_; + if (task->sub_plcmnt_idx_ >= schema.plcmnts_.size()) { + ++task->plcmnt_idx_; + task->sub_plcmnt_idx_ = 0; + } + if (task->cur_bdev_alloc_->alloc_size_ < task->cur_bdev_alloc_->size_) { + size_t diff = task->cur_bdev_alloc_->size_ - task->cur_bdev_alloc_->alloc_size_; + PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; + SubPlacement &sub_plcmnt = schema.plcmnts_[task->sub_plcmnt_idx_]; + sub_plcmnt.size_ += diff; + HILOG(kDebug, "Passing {} bytes to target {}", diff, sub_plcmnt.tid_); + } + LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); + if (task->plcmnt_idx_ < (*task->schema_).size()) { + task->phase_ = PutBlobPhase::kAllocate; + return; + } else { + task->phase_ = PutBlobPhase::kModify; + task->bdev_writes_->reserve(blob_info.buffers_.size()); + } + } + + /** Update the data on storage */ + void PutBlobModifyPhase(PutBlobTask *task) { + BlobInfo &blob_info = blob_map_[task->blob_id_]; + char *blob_buf = task->data_ptr_.ptr_; + std::vector &write_tasks = *task->bdev_writes_; + size_t blob_off = 0, buf_off = 0; + HILOG(kDebug, "Number of buffers {}", blob_info.buffers_.size()); + for (BufferInfo &buf : blob_info.buffers_) { + if (task->blob_off_ <= blob_off) { + size_t rel_off = blob_off - task->blob_off_; + size_t tgt_off = buf.t_off_ + rel_off; + size_t buf_size = buf.t_size_ - rel_off; + if (blob_off + buf_size > task->blob_off_ + task->data_size_) { + buf_size = task->blob_off_ + task->data_size_ - blob_off; + } + HILOG(kDebug, "Writing {} bytes at off {} from target {}", buf_size, tgt_off, buf.tid_) + TargetInfo &target = *target_map_[buf.tid_]; + bdev::WriteTask *write_task = target.AsyncWrite(task->task_node_ + 1, + blob_buf + buf_off, + tgt_off, buf_size).ptr_; + write_tasks.emplace_back(write_task); + buf_off += buf_size; + } + blob_off += buf.t_size_; + } + if (blob_off == 0) { + HELOG(kFatal, "Something annoying happened"); + } + blob_info.max_blob_size_ = blob_off; + task->phase_ = PutBlobPhase::kWaitModify; + HILOG(kDebug, "Modified {} bytes of blob {}", blob_off, task->blob_id_); + } + + /** Wait for the update to complete */ + void PutBlobWaitModifyPhase(PutBlobTask *task) { + std::vector &write_tasks = *task->bdev_writes_; + for (int i = (int)write_tasks.size() - 1; i >= 0; --i) { + bdev::WriteTask *write_task = write_tasks[i]; + if (!write_task->IsComplete()) { + return; + } + LABSTOR_CLIENT->DelTask(write_task); + write_tasks.pop_back(); + } + HILOG(kDebug, "PutBlobTask complete"); + HSHM_DESTROY_AR(task->schema_); + HSHM_DESTROY_AR(task->bdev_writes_); + if (task->flags_.Any(HERMES_DID_STAGE_IN)) { + LABSTOR_CLIENT->FreeBuffer(task->data_ptr_); + } + // Update the bucket statistics + if (task->data_off_) { + bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, + task->tag_id_, + task->data_off_); + } + if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { + bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, + task->tag_id_, + task->blob_id_); + } + task->SetModuleComplete(); + } + + /** Get a blob's data */ + void GetBlob(GetBlobTask *task) { + switch (task->phase_) { + case GetBlobPhase::kStart: { + GetBlobGetPhase(task); + } + case GetBlobPhase::kWait: { + GetBlobWaitPhase(task); + } + } + } + + void GetBlobGetPhase(GetBlobTask *task) { + HILOG(kDebug, "GetBlobTask start"); + BlobInfo &blob_info = blob_map_[task->blob_id_]; + HSHM_MAKE_AR0(task->bdev_reads_, nullptr); + std::vector &read_tasks = *task->bdev_reads_; + read_tasks.reserve(blob_info.buffers_.size()); + if (task->data_size_ < 0) { + task->data_size_ = (ssize_t)(blob_info.blob_size_ - task->blob_off_); + } + size_t blob_off = 0, buf_off = 0; + hipc::mptr blob_data_mptr(task->data_); + char *blob_buf = blob_data_mptr.get(); + for (BufferInfo &buf : blob_info.buffers_) { + if (task->blob_off_ <= blob_off) { + size_t rel_off = blob_off - task->blob_off_; + size_t tgt_off = buf.t_off_ + rel_off; + size_t buf_size = buf.t_size_ - rel_off; + if (blob_off + buf_size > task->blob_off_ + task->data_size_) { + buf_size = task->blob_off_ + task->data_size_ - blob_off; + } + HILOG(kDebug, "Loading {} bytes at off {} from target {}", buf_size, tgt_off, buf.tid_) + TargetInfo &target = *target_map_[buf.tid_]; + bdev::ReadTask *read_task = target.AsyncRead(task->task_node_ + 1, + blob_buf + buf_off, + tgt_off, buf_size).ptr_; + read_tasks.emplace_back(read_task); + buf_off += buf_size; + } + blob_off += buf.t_size_; + } + task->phase_ = GetBlobPhase::kWait; + } + + void GetBlobWaitPhase(GetBlobTask *task) { + std::vector &read_tasks = *task->bdev_reads_; + for (auto it = read_tasks.rbegin(); it != read_tasks.rend(); ++it) { + bdev::ReadTask *read_task = *it; + if (!read_task->IsComplete()) { + return; + } + LABSTOR_CLIENT->DelTask(read_task); + read_tasks.pop_back(); + } + HSHM_DESTROY_AR(task->bdev_reads_); + HILOG(kDebug, "GetBlobTask complete"); + task->SetModuleComplete(); + } + + /** + * Tag a blob + * */ + void TagBlob(TagBlobTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob = it->second; + blob.tags_.push_back(task->tag_); + task->SetModuleComplete(); + } + + /** + * Check if blob has a tag + * */ + void BlobHasTag(BlobHasTagTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob = it->second; + task->has_tag_ = std::find(blob.tags_.begin(), + blob.tags_.end(), + task->tag_) != blob.tags_.end(); + task->SetModuleComplete(); + } + + /** + * Create \a blob_id BLOB ID + * */ + void GetOrCreateBlobId(GetOrCreateBlobIdTask *task) { + hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); + hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); + auto it = blob_id_map_.find(blob_name_unique); + if (it == blob_id_map_.end()) { + task->blob_id_ = BlobId(node_id_, id_alloc_.fetch_add(1)); + blob_id_map_.emplace(blob_name_unique, task->blob_id_); + task->SetModuleComplete(); + return; + } + task->blob_id_ = it->second; + task->SetModuleComplete(); + } + + /** + * Get \a blob_name BLOB from \a bkt_id bucket + * */ + HSHM_ALWAYS_INLINE + void GetBlobId(GetBlobIdTask *task) { + hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); + hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); + auto it = blob_id_map_.find(blob_name_unique); + if (it == blob_id_map_.end()) { + task->blob_id_ = BlobId::GetNull(); + task->SetModuleComplete(); + HILOG(kDebug, "Failed to find blob {} in {}", blob_name.str(), task->tag_id_); + return; + } + task->blob_id_ = it->second; + HILOG(kDebug, "Found blob {} / {} in {}", task->blob_id_, blob_name.str(), task->tag_id_); + task->SetModuleComplete(); + } + + /** + * Get \a blob_name BLOB name from \a blob_id BLOB id + * */ + void GetBlobName(GetBlobNameTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob = it->second; + (*task->blob_name_) = blob.name_; + task->SetModuleComplete(); + } + + /** + * Get \a score from \a blob_id BLOB id + * */ + void GetBlobSize(GetBlobSizeTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob = it->second; + task->size_ = blob.blob_size_; + task->SetModuleComplete(); + } + + /** + * Get \a score from \a blob_id BLOB id + * */ + void GetBlobScore(GetBlobScoreTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob = it->second; + task->score_ = blob.score_; + task->SetModuleComplete(); + } + + /** + * Get \a blob_id blob's buffers + * */ + void GetBlobBuffers(GetBlobBuffersTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob = it->second; + (*task->buffers_) = blob.buffers_; + task->SetModuleComplete(); + } + + /** + * Rename \a blob_id blob to \a new_blob_name new blob name + * in \a bkt_id bucket. + * */ + void RenameBlob(RenameBlobTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob = it->second; + blob_id_map_.erase(blob.name_); + blob_id_map_[blob.name_] = task->blob_id_; + blob.name_ = hshm::to_charbuf(*task->new_blob_name_); + task->SetModuleComplete(); + } + + /** + * Truncate a blob to a new size + * */ + void TruncateBlob(TruncateBlobTask *task) { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob_info = it->second; + // TODO(llogan): truncate blob + task->SetModuleComplete(); + } + + /** + * Destroy \a blob_id blob in \a bkt_id bucket + * */ + void DestroyBlob(DestroyBlobTask *task) { + switch (task->phase_) { + case DestroyBlobPhase::kFreeBuffers: { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob_info = it->second; + hshm::charbuf unique_name = GetBlobNameWithBucket(blob_info.tag_id_, blob_info.name_); + blob_id_map_.erase(unique_name); + HSHM_MAKE_AR0(task->free_tasks_, nullptr); + task->free_tasks_->reserve(blob_info.buffers_.size()); + for (BufferInfo &buf : blob_info.buffers_) { + TargetInfo &tgt_info = *target_map_[buf.tid_]; + std::vector buf_vec = {buf}; + bdev::FreeTask *free_task = tgt_info.AsyncFree( + task->task_node_ + 1, std::move(buf_vec), false).ptr_; + task->free_tasks_->emplace_back(free_task); + } + task->phase_ = DestroyBlobPhase::kWaitFreeBuffers; + } + case DestroyBlobPhase::kWaitFreeBuffers: { + std::vector &free_tasks = *task->free_tasks_; + for (auto it = free_tasks.rbegin(); it != free_tasks.rend(); ++it) { + bdev::FreeTask *free_task = *it; + if (!free_task->IsComplete()) { + return; + } + LABSTOR_CLIENT->DelTask(free_task); + free_tasks.pop_back(); + } + HSHM_DESTROY_AR(task->free_tasks_); + blob_map_.erase(task->blob_id_); + task->SetModuleComplete(); + } + } + } + + /** + * Reorganize \a blob_id blob in \a bkt_id bucket + * */ + void ReorganizeBlob(ReorganizeBlobTask *task) { + switch (task->phase_) { + case ReorganizeBlobPhase::kGet: { + auto it = blob_map_.find(task->blob_id_); + if (it == blob_map_.end()) { + task->SetModuleComplete(); + return; + } + BlobInfo &blob_info = it->second; + task->data_ = LABSTOR_CLIENT->AllocateBuffer(blob_info.blob_size_).shm_; + task->data_size_ = blob_info.blob_size_; + task->get_task_ = blob_mdm_.AsyncGetBlob(task->task_node_ + 1, + task->tag_id_, + task->blob_id_, + 0, + task->data_size_, + task->data_).ptr_; + task->tag_id_ = blob_info.tag_id_; + task->phase_ = ReorganizeBlobPhase::kWaitGet; + } + case ReorganizeBlobPhase::kWaitGet: { + if (!task->get_task_->IsComplete()) { + return; + } + LABSTOR_CLIENT->DelTask(task->get_task_); + task->phase_ = ReorganizeBlobPhase::kPut; + } + case ReorganizeBlobPhase::kPut: { + task->put_task_ = blob_mdm_.AsyncPutBlob(task->task_node_ + 1, + task->tag_id_, hshm::charbuf(""), + task->blob_id_, 0, + task->data_size_, + task->data_, + task->score_, + bitfield32_t(HERMES_BLOB_REPLACE)).ptr_; + task->SetModuleComplete(); + } + } + } + + public: +#include "hermes_blob_mdm/hermes_blob_mdm_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(hermes::blob_mdm::Server, "hermes_blob_mdm"); diff --git a/tasks/hermes_bucket_mdm/CMakeLists.txt b/tasks/hermes_bucket_mdm/CMakeLists.txt new file mode 100644 index 000000000..67b8141e5 --- /dev/null +++ b/tasks/hermes_bucket_mdm/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Hermes Bucket MDM Task Library +#------------------------------------------------------------------------------ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h new file mode 100644 index 000000000..c72a1a036 --- /dev/null +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -0,0 +1,298 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_hermes_bucket_mdm_H_ +#define LABSTOR_hermes_bucket_mdm_H_ + +#include "hermes_bucket_mdm_tasks.h" + +namespace hermes::bucket_mdm { + +/** Create hermes_bucket_mdm requests */ +class Client : public TaskLibClient { + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Create a hermes_bucket_mdm */ + HSHM_ALWAYS_INLINE + void CreateRoot(const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + domain_id, state_name, id_, queue_info); + queue_id_ = QueueId(id_); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /**==================================== + * Tag Operations + * ===================================*/ + + /** Sets the BLOB MDM */ + void AsyncSetBlobMdmConstruct(SetBlobMdmTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &blob_mdm_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_, blob_mdm_id); + } + void SetBlobMdmRoot(const DomainId &domain_id, + const TaskStateId &blob_mdm_id) { + LPointer> push_task = + AsyncSetBlobMdmRoot(domain_id, blob_mdm_id); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(SetBlobMdm); + + /** Update statistics after blob PUT (fire & forget) */ + HSHM_ALWAYS_INLINE + void AsyncUpdateSizeConstruct(UpdateSizeTask *task, + const TaskNode &task_node, + TagId tag_id, + size_t update, bitfield32_t flags = bitfield32_t(0)) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(tag_id.node_id_), id_, + tag_id, update, flags); + } + LABSTOR_TASK_NODE_PUSH_ROOT(UpdateSize); + + /** Append data to the bucket (fire & forget) */ + HSHM_ALWAYS_INLINE + void AsyncAppendBlobSchemaConstruct(AppendBlobSchemaTask *task, + const TaskNode &task_node, + TagId tag_id, + size_t data_size, + size_t page_size) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(tag_id.node_id_), id_, + tag_id, data_size, page_size); + } + LABSTOR_TASK_NODE_PUSH_ROOT(AppendBlobSchema); + + /** Append data to the bucket (fire & forget) */ + HSHM_ALWAYS_INLINE + void AsyncAppendBlobConstruct( + AppendBlobTask *task, + const TaskNode &task_node, + TagId tag_id, + size_t data_size, + const hipc::Pointer &data, + size_t page_size, + float score, + u32 node_id, + const Context &ctx) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetLocal(), id_, + tag_id, data_size, data, page_size, score, node_id, ctx); + } + HSHM_ALWAYS_INLINE + void AppendBlobRoot(TagId tag_id, + size_t data_size, + const hipc::Pointer &data, + size_t page_size, + float score, + u32 node_id, + const Context &ctx) { + AsyncAppendBlobRoot(tag_id, data_size, data, page_size, score, node_id, ctx); + } + LABSTOR_TASK_NODE_PUSH_ROOT(AppendBlob); + + /** Create a tag or get the ID of existing tag */ + HSHM_ALWAYS_INLINE + void AsyncGetOrCreateTagConstruct(GetOrCreateTagTask *task, + const TaskNode &task_node, + const hshm::charbuf &tag_name, + bool blob_owner, + const std::vector &traits, + size_t backend_size, + u32 flags) { + HILOG(kDebug, "Creating a tag {}", tag_name.str()); + u32 hash = std::hash{}(tag_name); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_name, blob_owner, traits, backend_size, flags); + } + HSHM_ALWAYS_INLINE + TagId GetOrCreateTagRoot(const hshm::charbuf &tag_name, + bool blob_owner, + const std::vector &traits, + size_t backend_size, + u32 flags) { + LPointer> push_task = + AsyncGetOrCreateTagRoot(tag_name, blob_owner, traits, backend_size, flags); + push_task->Wait(); + GetOrCreateTagTask *task = push_task->get(); + TagId tag_id = task->tag_id_; + LABSTOR_CLIENT->DelTask(push_task); + return tag_id; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateTag); + + /** Get tag ID */ + void AsyncGetTagIdConstruct(GetTagIdTask *task, + const TaskNode &task_node, + const hshm::charbuf &tag_name) { + u32 hash = std::hash{}(tag_name); + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_name); + } + TagId GetTagIdRoot(const hshm::charbuf &tag_name) { + LPointer> push_task = + AsyncGetTagIdRoot(tag_name); + push_task->Wait(); + GetTagIdTask *task = push_task->get(); + TagId tag_id = task->tag_id_; + LABSTOR_CLIENT->DelTask(push_task); + return tag_id; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetTagId); + + /** Get tag name */ + void AsyncGetTagNameConstruct(GetTagNameTask *task, + const TaskNode &task_node, + const TagId &tag_id) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id); + } + hshm::string GetTagNameRoot(const TagId &tag_id) { + LPointer> push_task = + AsyncGetTagNameRoot(tag_id); + push_task->Wait(); + GetTagNameTask *task = push_task->get(); + hshm::string tag_name = hshm::to_charbuf(*task->tag_name_.get()); + LABSTOR_CLIENT->DelTask(push_task); + return tag_name; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetTagName); + + /** Rename tag */ + void AsyncRenameTagConstruct(RenameTagTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const hshm::charbuf &new_tag_name) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id, new_tag_name); + } + void RenameTagRoot(const TagId &tag_id, const hshm::charbuf &new_tag_name) { + LPointer> push_task = + AsyncRenameTagRoot(tag_id, new_tag_name); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(RenameTag); + + /** Destroy tag */ + void AsyncDestroyTagConstruct(DestroyTagTask *task, + const TaskNode &task_node, + const TagId &tag_id) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id); + } + void DestroyTagRoot(const TagId &tag_id) { + LPointer> push_task = + AsyncDestroyTagRoot(tag_id); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(DestroyTag); + + /** Add a blob to a tag */ + void AsyncTagAddBlobConstruct(TagAddBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, + const BlobId &blob_id) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id, blob_id); + } + void TagAddBlobRoot(const TagId &tag_id, const BlobId &blob_id) { + LPointer> push_task = + AsyncTagAddBlobRoot(tag_id, blob_id); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(TagAddBlob); + + /** Remove a blob from a tag */ + void AsyncTagRemoveBlobConstruct(TagRemoveBlobTask *task, + const TaskNode &task_node, + const TagId &tag_id, const BlobId &blob_id) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id, blob_id); + } + void TagRemoveBlobRoot(const TagId &tag_id, const BlobId &blob_id) { + LPointer> push_task = + AsyncTagRemoveBlobRoot(tag_id, blob_id); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(TagRemoveBlob); + + /** Clear blobs from a tag */ + void AsyncTagClearBlobsConstruct(TagClearBlobsTask *task, + const TaskNode &task_node, + const TagId &tag_id) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id); + } + void TagClearBlobsRoot(const TagId &tag_id) { + LPointer> push_task = + AsyncTagClearBlobsRoot(tag_id); + push_task->Wait(); + LABSTOR_CLIENT->DelTask(push_task); + } + LABSTOR_TASK_NODE_PUSH_ROOT(TagClearBlobs); + + /** Get the size of a bucket */ + void AsyncGetSizeConstruct(GetSizeTask *task, + const TaskNode &task_node, + const TagId &tag_id) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id); + } + size_t GetSizeRoot(const TagId &tag_id) { + LPointer> push_task = + AsyncGetSizeRoot(tag_id); + push_task->Wait(); + GetSizeTask *task = push_task->get(); + size_t size = task->size_; + LABSTOR_CLIENT->DelTask(push_task); + return size; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetSize); +}; + +} // namespace labstor + +#endif // LABSTOR_hermes_bucket_mdm_H_ diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h new file mode 100644 index 000000000..f1d58e0ab --- /dev/null +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -0,0 +1,530 @@ +#ifndef LABSTOR_HERMES_BUCKET_MDM_LIB_EXEC_H_ +#define LABSTOR_HERMES_BUCKET_MDM_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTag: { + GetOrCreateTag(reinterpret_cast(task)); + break; + } + case Method::kGetTagId: { + GetTagId(reinterpret_cast(task)); + break; + } + case Method::kGetTagName: { + GetTagName(reinterpret_cast(task)); + break; + } + case Method::kRenameTag: { + RenameTag(reinterpret_cast(task)); + break; + } + case Method::kDestroyTag: { + DestroyTag(reinterpret_cast(task)); + break; + } + case Method::kTagAddBlob: { + TagAddBlob(reinterpret_cast(task)); + break; + } + case Method::kTagRemoveBlob: { + TagRemoveBlob(reinterpret_cast(task)); + break; + } + case Method::kTagClearBlobs: { + TagClearBlobs(reinterpret_cast(task)); + break; + } + case Method::kUpdateSize: { + UpdateSize(reinterpret_cast(task)); + break; + } + case Method::kAppendBlobSchema: { + AppendBlobSchema(reinterpret_cast(task)); + break; + } + case Method::kAppendBlob: { + AppendBlob(reinterpret_cast(task)); + break; + } + case Method::kGetSize: { + GetSize(reinterpret_cast(task)); + break; + } + case Method::kSetBlobMdm: { + SetBlobMdm(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTag: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetTagId: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetTagName: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kRenameTag: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestroyTag: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kTagAddBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kTagRemoveBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kTagClearBlobs: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kUpdateSize: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kAppendBlobSchema: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kAppendBlob: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetSize: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kSetBlobMdm: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTag: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetTagId: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetTagName: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kRenameTag: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestroyTag: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kTagAddBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kTagRemoveBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kTagClearBlobs: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kUpdateSize: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kAppendBlobSchema: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kAppendBlob: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetSize: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kSetBlobMdm: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetOrCreateTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetTagId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetTagName: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRenameTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagAddBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagRemoveBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagClearBlobs: { + ar << *reinterpret_cast(task); + break; + } + case Method::kUpdateSize: { + ar << *reinterpret_cast(task); + break; + } + case Method::kAppendBlobSchema: { + ar << *reinterpret_cast(task); + break; + } + case Method::kAppendBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetSize: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetBlobMdm: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetOrCreateTag: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetTagId: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetTagName: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kRenameTag: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestroyTag: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kTagAddBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kTagRemoveBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kTagClearBlobs: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kUpdateSize: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kAppendBlobSchema: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kAppendBlob: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetSize: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kSetBlobMdm: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetOrCreateTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetTagId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetTagName: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRenameTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyTag: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagAddBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagRemoveBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kTagClearBlobs: { + ar << *reinterpret_cast(task); + break; + } + case Method::kUpdateSize: { + ar << *reinterpret_cast(task); + break; + } + case Method::kAppendBlobSchema: { + ar << *reinterpret_cast(task); + break; + } + case Method::kAppendBlob: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetSize: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetBlobMdm: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTag: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetTagId: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetTagName: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kRenameTag: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestroyTag: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kTagAddBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kTagRemoveBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kTagClearBlobs: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kUpdateSize: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kAppendBlobSchema: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kAppendBlob: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetSize: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kSetBlobMdm: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetOrCreateTag: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetTagId: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetTagName: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kRenameTag: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestroyTag: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kTagAddBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kTagRemoveBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kTagClearBlobs: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kUpdateSize: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kAppendBlobSchema: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kAppendBlob: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetSize: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kSetBlobMdm: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h new file mode 100644 index 000000000..bed26dd43 --- /dev/null +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h @@ -0,0 +1,21 @@ +#ifndef LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ +#define LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kGetOrCreateTag = kLast + 0; + TASK_METHOD_T kGetTagId = kLast + 1; + TASK_METHOD_T kGetTagName = kLast + 2; + TASK_METHOD_T kRenameTag = kLast + 3; + TASK_METHOD_T kDestroyTag = kLast + 4; + TASK_METHOD_T kTagAddBlob = kLast + 5; + TASK_METHOD_T kTagRemoveBlob = kLast + 6; + TASK_METHOD_T kTagClearBlobs = kLast + 10; + TASK_METHOD_T kUpdateSize = kLast + 11; + TASK_METHOD_T kAppendBlobSchema = kLast + 12; + TASK_METHOD_T kAppendBlob = kLast + 13; + TASK_METHOD_T kGetSize = kLast + 14; + TASK_METHOD_T kSetBlobMdm = kLast + 15; +}; + +#endif // LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml new file mode 100644 index 000000000..d56feaa39 --- /dev/null +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml @@ -0,0 +1,16 @@ +kGetOrCreateTag: 0 +kGetTagId: 1 +kGetTagName: 2 +kRenameTag: 3 +kDestroyTag: 4 +kTagAddBlob: 5 +kTagRemoveBlob: 6 +# kTagGroupBy: 7 +# kTagAddTrait: 8 +# kTagRemoveTrait: 9 +kTagClearBlobs: 10 +kUpdateSize: 11 +kAppendBlobSchema: 12 +kAppendBlob: 13 +kGetSize: 14 +kSetBlobMdm: 15 \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h new file mode 100644 index 000000000..e2bfb9ad3 --- /dev/null +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -0,0 +1,849 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ +#define LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "hermes/hermes_types.h" +#include "bdev/bdev.h" +#include "hermes_blob_mdm/hermes_blob_mdm.h" +#include "labstor/api/labstor_client.h" +#include "labstor/labstor_namespace.h" +#include "proc_queue/proc_queue.h" + +namespace hermes::bucket_mdm { + +#include "hermes_bucket_mdm_methods.h" +#include "labstor/labstor_namespace.h" + +/** + * A task to create hermes_bucket_mdm + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "hermes_bucket_mdm", id, queue_info) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy hermes_bucket_mdm */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** Set the BLOB MDM ID */ +struct SetBlobMdmTask : public Task, TaskFlags { + IN TaskStateId blob_mdm_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + SetBlobMdmTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + SetBlobMdmTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TaskStateId &blob_mdm) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = state_id; + method_ = Method::kSetBlobMdm; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + blob_mdm_ = blob_mdm; + } + + /** Destructor */ + ~SetBlobMdmTask() {} + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(blob_mdm_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } + + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} +}; + +/** Update bucket size */ +struct UpdateSizeTask : public Task, TaskFlags { + IN TagId tag_id_; + IN size_t update_; + IN bitfield32_t flags_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + UpdateSizeTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + UpdateSizeTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + size_t update, + bitfield32_t flags) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kUpdateSize; + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + update_ = update; + flags_ = flags; + } + + /** Destructor */ + ~UpdateSizeTask() {} + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, update_, flags_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** Phases for the append task */ +class AppendBlobPhase { + public: + TASK_METHOD_T kGetBlobIds = 0; + TASK_METHOD_T kWaitBlobIds = 1; + TASK_METHOD_T kWaitPutBlobs = 2; +}; + +/** A struct to store the */ +struct AppendInfo { + size_t blob_off_; + size_t data_size_; + hshm::charbuf blob_name_; + BlobId blob_id_; + blob_mdm::GetOrCreateBlobIdTask *blob_id_task_; + blob_mdm::PutBlobTask *put_task_; + + template + void serialize(Ar &ar) { + ar(blob_off_, data_size_, blob_name_, blob_id_); + } +}; + +/** A task to append data to a bucket */ +struct AppendBlobSchemaTask : public Task, TaskFlags { + IN TagId tag_id_; + IN size_t data_size_; + IN size_t page_size_; + TEMP int phase_ = AppendBlobPhase::kGetBlobIds; + TEMP hipc::ShmArchive> append_info_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + AppendBlobSchemaTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + AppendBlobSchemaTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + size_t data_size, + size_t page_size) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kAppendBlobSchema; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + data_size_ = data_size; + page_size_ = page_size; + } + + /** Destructor */ + ~AppendBlobSchemaTask() {} + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, data_size_, page_size_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(append_info_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to append data to a bucket */ +struct AppendBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN size_t data_size_; + IN hipc::Pointer data_; + IN size_t page_size_; + IN u32 node_id_; + IN float score_; + TEMP int phase_ = AppendBlobPhase::kGetBlobIds; + TEMP AppendBlobSchemaTask *schema_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + AppendBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + AppendBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + size_t data_size, + const hipc::Pointer &data, + size_t page_size, + float score, + u32 node_id, + const Context &ctx) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kAppendBlob; + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_UNORDERED); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + data_size_ = data_size; + data_ = data; + score_ = score; + page_size_ = page_size; + node_id_ = node_id; + } + + /** Destructor */ + ~AppendBlobTask() { + if (IsDataOwner()) { + LABSTOR_CLIENT->FreeBuffer(data_); + } + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to get or create a tag */ +struct GetOrCreateTagTask : public Task, TaskFlags { + IN hipc::ShmArchive tag_name_; + IN bool blob_owner_; + IN hipc::ShmArchive> traits_; + IN size_t backend_size_; + IN bitfield32_t flags_; + OUT TagId tag_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetOrCreateTagTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetOrCreateTagTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const hshm::charbuf &tag_name, + bool blob_owner, + const std::vector &traits, + size_t backend_size, + u32 flags) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = std::hash{}(tag_name); + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetOrCreateTag; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + blob_owner_ = blob_owner; + backend_size_ = backend_size; + HSHM_MAKE_AR(tag_name_, alloc, tag_name) + HSHM_MAKE_AR(traits_, alloc, traits) + flags_ = bitfield32_t(flags); + } + + /** Destructor */ + ~GetOrCreateTagTask() { + HSHM_DESTROY_AR(tag_name_) + HSHM_DESTROY_AR(traits_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_name_, blob_owner_, traits_, backend_size_, flags_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(tag_id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + group.resize(tag_name_->size()); + memcpy(group.data(), tag_name_->data(), tag_name_->size()); + return 0; + } +}; + +/** A task to get a tag id */ +struct GetTagIdTask : public Task, TaskFlags { + IN hipc::ShmArchive tag_name_; + OUT TagId tag_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetTagIdTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetTagIdTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const hshm::charbuf &tag_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = std::hash{}(tag_name); + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetTagId; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + HSHM_MAKE_AR(tag_name_, alloc, tag_name) + } + + /** Destructor */ + ~GetTagIdTask() { + HSHM_DESTROY_AR(tag_name_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_name_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(tag_id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + group.resize(tag_name_->size()); + memcpy(group.data(), tag_name_->data(), tag_name_->size()); + return 0; + } +}; + +/** A task to get a tag name */ +struct GetTagNameTask : public Task, TaskFlags { + IN TagId tag_id_; + OUT hipc::ShmArchive tag_name_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetTagNameTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetTagNameTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetTagName; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + } + + /** Destructor */ + ~GetTagNameTask() { + HSHM_DESTROY_AR(tag_name_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(tag_name_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to rename a tag */ +struct RenameTagTask : public Task, TaskFlags { + IN TagId tag_id_; + IN hipc::ShmArchive tag_name_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + RenameTagTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + RenameTagTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id, + const hshm::charbuf &tag_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kRenameTag; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + HSHM_MAKE_AR(tag_name_, alloc, tag_name) + } + + /** Destructor */ + ~RenameTagTask() { + HSHM_DESTROY_AR(tag_name_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(tag_name_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +class DestroyTagPhase { + public: + TASK_METHOD_T kDestroyBlobs = 0; + TASK_METHOD_T kWaitDestroyBlobs = 1; +}; + +/** A task to destroy a tag */ +struct DestroyTagTask : public Task, TaskFlags { + IN TagId tag_id_; + TEMP int phase_ = DestroyTagPhase::kDestroyBlobs; + TEMP hipc::ShmArchive> destroy_blob_tasks_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestroyTagTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestroyTagTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + TagId tag_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kDestroyTag; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to add a blob to the tag */ +struct TagAddBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + TagAddBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + TagAddBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + TagId tag_id, + const BlobId &blob_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kTagAddBlob; + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + blob_id_ = blob_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to remove a blob from a tag */ +struct TagRemoveBlobTask : public Task, TaskFlags { + IN TagId tag_id_; + IN BlobId blob_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + TagRemoveBlobTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + TagRemoveBlobTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + TagId tag_id, + const BlobId &blob_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kTagRemoveBlob; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + blob_id_ = blob_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_, blob_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to get the group of blobs associated with a tag */ +struct TagGroupByTask : public Task, TaskFlags {}; + +/** A task to associate a tag with a trait */ +struct TagAddTraitTask : public Task, TaskFlags {}; + +/** A task to remove a trait from a tag */ +struct TagRemoveTraitTask : public Task, TaskFlags {}; + +/** A task to destroy all blobs in the tag */ +struct TagClearBlobsTask : public Task, TaskFlags { + IN TagId tag_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + TagClearBlobsTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + TagClearBlobsTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + TagId tag_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kTagClearBlobs; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +/** A task to destroy all blobs in the tag */ +struct GetSizeTask : public Task, TaskFlags { + IN TagId tag_id_; + OUT size_t size_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetSizeTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetSizeTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + TagId tag_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.unique_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetSize; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(size_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + +} // namespace hermes::bucket_mdm + +#endif // LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ diff --git a/tasks/hermes_bucket_mdm/src/CMakeLists.txt b/tasks/hermes_bucket_mdm/src/CMakeLists.txt new file mode 100644 index 000000000..03b889366 --- /dev/null +++ b/tasks/hermes_bucket_mdm/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(hermes_bucket_mdm SHARED + hermes_bucket_mdm.cc) +add_dependencies(hermes_bucket_mdm ${Labstor_RUNTIME_DEPS}) +target_link_libraries(hermes_bucket_mdm ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + hermes_bucket_mdm + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + hermes_bucket_mdm + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(hermes_bucket_mdm) +endif() diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc new file mode 100644 index 000000000..e4b924fdc --- /dev/null +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -0,0 +1,344 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "hermes/config_server.h" +#include "hermes_bucket_mdm/hermes_bucket_mdm.h" +#include "hermes_adapters/mapper/abstract_mapper.h" +#include "hermes/dpe/dpe_factory.h" +#include "bdev/bdev.h" + +namespace hermes::bucket_mdm { + +typedef std::unordered_map TAG_ID_MAP_T; +typedef std::unordered_map TAG_MAP_T; + +class Server : public TaskLib { + public: + TAG_ID_MAP_T tag_id_map_; + TAG_MAP_T tag_map_; + u32 node_id_; + std::atomic id_alloc_; + Client bkt_mdm_; + blob_mdm::Client blob_mdm_; + + public: + Server() = default; + + void Construct(ConstructTask *task) { + id_alloc_ = 0; + node_id_ = LABSTOR_CLIENT->node_id_; + bkt_mdm_.Init(id_); + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + /** + * Set the Blob MDM + * */ + void SetBlobMdm(SetBlobMdmTask *task) { + blob_mdm_.Init(task->blob_mdm_); + task->SetModuleComplete(); + } + + /** Update the size of the bucket */ + void UpdateSize(UpdateSizeTask *task) { + TagInfo &tag_info = tag_map_[task->tag_id_]; + tag_info.internal_size_ = std::max(task->update_, + tag_info.internal_size_); + task->SetModuleComplete(); + } + + /** + * Create the PartialPuts for append operations. + * */ + void AppendBlobSchema(AppendBlobSchemaTask *task) { + switch (task->phase_) { + case AppendBlobPhase::kGetBlobIds: { + HILOG(kDebug, "(node {}) Getting blob IDs for tag {} (task_node={})", + LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) + TagInfo &tag_info = tag_map_[task->tag_id_]; + size_t bucket_size = tag_info.internal_size_; + size_t cur_page = bucket_size / task->page_size_; + size_t cur_page_off = bucket_size % task->page_size_; + size_t update_size = task->page_size_ - cur_page_off; + size_t max_pages = task->data_size_ / task->page_size_ + 1; + size_t cur_size = 0; + HSHM_MAKE_AR0(task->append_info_, nullptr); + std::vector &append_info = *task->append_info_; + append_info.reserve(max_pages); + while (cur_size < task->data_size_) { + if (update_size > task->data_size_) { + update_size = task->data_size_; + } + append_info.emplace_back(); + AppendInfo &append = append_info.back(); + append.blob_name_ = hshm::charbuf(std::to_string(cur_page)); + append.data_size_ = update_size; + append.blob_off_ = cur_page_off; + append.blob_id_task_ = blob_mdm_.AsyncGetOrCreateBlobId(task->task_node_ + 1, + task->tag_id_, + append.blob_name_).ptr_; + cur_size += update_size; + cur_page_off = 0; + ++cur_page; + update_size = task->page_size_; + } + task->phase_ = AppendBlobPhase::kWaitBlobIds; + } + case AppendBlobPhase::kWaitBlobIds: { + std::vector &append_info = *task->append_info_; + for (AppendInfo &append : append_info) { + if (!append.blob_id_task_->IsComplete()) { + return; + } + } + HILOG(kDebug, "(node {}) Finished blob IDs for tag {} (task_node={})", + LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) + for (AppendInfo &append : append_info) { + append.blob_id_ = append.blob_id_task_->blob_id_; + LABSTOR_CLIENT->DelTask(append.blob_id_task_); + } + task->SetModuleComplete(); + } + } + } + + /** + * Append data to a bucket. Assumes that the blobs in the bucket + * are named 0 ... N. Each blob is assumed to have a certain + * fixed page size. + * */ + void AppendBlob(AppendBlobTask *task) { + switch (task->phase_) { + case AppendBlobPhase::kGetBlobIds: { + HILOG(kDebug, "(node {}) Appending {} bytes to bucket {} (task_node={})", + LABSTOR_CLIENT->node_id_, task->data_size_, task->tag_id_, task->task_node_); + task->schema_ = bkt_mdm_.AsyncAppendBlobSchema(task->task_node_ + 1, + task->tag_id_, + task->data_size_, + task->page_size_).ptr_; + task->phase_ = AppendBlobPhase::kWaitBlobIds; + } + case AppendBlobPhase::kWaitBlobIds: { + if (!task->schema_->IsComplete()) { + return; + } + std::vector &append_info = *task->schema_->append_info_; + size_t buf_off = 0; + HILOG(kDebug, "(node {}) Got blob schema of size {} for tag {} (task_node={})", + LABSTOR_CLIENT->node_id_, append_info.size(), task->tag_id_, task->task_node_) + for (AppendInfo &append : append_info) { + HILOG(kDebug, "(node {}) Spawning blob {} of size {} for tag {} (task_node={} blob_mdm={})", + LABSTOR_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, + task->tag_id_, task->task_node_, blob_mdm_.id_); + append.put_task_ = blob_mdm_.AsyncPutBlob(task->task_node_ + 1, + task->tag_id_, + append.blob_name_, + append.blob_id_, + append.blob_off_, + append.data_size_, + task->data_ + buf_off, + task->score_, + bitfield32_t(0), + Context(), + bitfield32_t(0)).ptr_; + HILOG(kDebug, "(node {}) Finished spawning blob {} of size {} for tag {} (task_node={} blob_mdm={})", + LABSTOR_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, + task->tag_id_, task->task_node_, blob_mdm_.id_); + buf_off += append.data_size_; + } + task->phase_ = AppendBlobPhase::kWaitPutBlobs; + } + case AppendBlobPhase::kWaitPutBlobs: { + std::vector &append_info = *task->schema_->append_info_; + for (AppendInfo &append : append_info) { + if (!append.put_task_->IsComplete()) { + return; + } + } + HILOG(kDebug, "(node {}) PUT blobs for tag {} (task_node={})", + LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) + for (AppendInfo &append : append_info) { + LABSTOR_CLIENT->DelTask(append.put_task_); + } + HSHM_DESTROY_AR(task->schema_->append_info_); + LABSTOR_CLIENT->DelTask(task->schema_); + task->SetModuleComplete(); + } + } + } + + /** Get or create a tag */ + void GetOrCreateTag(GetOrCreateTagTask *task) { + TagId tag_id; + HILOG(kDebug, "Creating a tag") + + // Check if the tag exists + hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); + bool did_create = false; + if (tag_name.size() > 0) { + did_create = tag_id_map_.find(tag_name) == tag_id_map_.end(); + } + + // Emplace bucket if it does not already exist + if (did_create) { + tag_id.unique_ = id_alloc_.fetch_add(1); + tag_id.node_id_ = LABSTOR_RUNTIME->rpc_.node_id_; + HILOG(kDebug, "Creating tag for the first time: {} {}", tag_name.str(), tag_id) + tag_id_map_.emplace(tag_name, tag_id); + tag_map_.emplace(tag_id, TagInfo()); + TagInfo &info = tag_map_[tag_id]; + info.name_ = tag_name; + info.tag_id_ = tag_id; + info.owner_ = task->blob_owner_; + info.internal_size_ = task->backend_size_; + } else { + if (tag_name.size()) { + HILOG(kDebug, "Found existing tag: {}", tag_name.str()) + tag_id = tag_id_map_[tag_name]; + } else { + HILOG(kDebug, "Found existing tag: {}", task->tag_id_) + tag_id = task->tag_id_; + } + } + + task->tag_id_ = tag_id; + // task->did_create_ = did_create; + task->SetModuleComplete(); + } + + /** Get tag ID */ + void GetTagId(GetTagIdTask *task) { + hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); + auto it = tag_id_map_.find(tag_name); + if (it == tag_id_map_.end()) { + task->tag_id_ = TagId::GetNull(); + task->SetModuleComplete(); + return; + } + task->tag_id_ = it->second; + task->SetModuleComplete(); + } + + /** Get tag name */ + void GetTagName(GetTagNameTask *task) { + auto it = tag_map_.find(task->tag_id_); + if (it == tag_map_.end()) { + task->SetModuleComplete(); + return; + } + (*task->tag_name_) = it->second.name_; + task->SetModuleComplete(); + } + + /** Rename tag */ + void RenameTag(RenameTagTask *task) { + auto it = tag_map_.find(task->tag_id_); + if (it == tag_map_.end()) { + task->SetModuleComplete(); + return; + } + (*task->tag_name_) = (*task->tag_name_); + task->SetModuleComplete(); + } + + /** Destroy tag */ + void DestroyTag(DestroyTagTask *task) { + switch (task->phase_) { + case DestroyTagPhase::kDestroyBlobs: { + TagInfo &tag = tag_map_[task->tag_id_]; + tag_id_map_.erase(tag.name_); + HSHM_MAKE_AR0(task->destroy_blob_tasks_, nullptr); + std::vector blob_tasks = *task->destroy_blob_tasks_; + blob_tasks.reserve(tag.blobs_.size()); + for (BlobId &blob_id : tag.blobs_) { + blob_mdm::DestroyBlobTask *blob_task = + blob_mdm_.AsyncDestroyBlob(task->task_node_ + 1, task->tag_id_, blob_id).ptr_; + blob_tasks.emplace_back(blob_task); + } + task->phase_ = DestroyTagPhase::kWaitDestroyBlobs; + return; + } + case DestroyTagPhase::kWaitDestroyBlobs: { + std::vector blob_tasks = *task->destroy_blob_tasks_; + for (auto it = blob_tasks.rbegin(); it != blob_tasks.rend(); ++it) { + blob_mdm::DestroyBlobTask *blob_task = *it; + if (!blob_task->IsComplete()) { + return; + } + LABSTOR_CLIENT->DelTask(blob_task); + blob_tasks.pop_back(); + } + HSHM_DESTROY_AR(task->destroy_blob_tasks_); + tag_map_.erase(task->tag_id_); + task->SetModuleComplete(); + } + } + } + + /** Add a blob to a tag */ + void TagAddBlob(TagAddBlobTask *task) { + auto it = tag_map_.find(task->tag_id_); + if (it == tag_map_.end()) { + task->SetModuleComplete(); + return; + } + TagInfo &tag = it->second; + tag.blobs_.emplace_back(task->blob_id_); + task->SetModuleComplete(); + } + + /** Remove a blob from a tag */ + void TagRemoveBlob(TagRemoveBlobTask *task) { + auto it = tag_map_.find(task->tag_id_); + if (it == tag_map_.end()) { + task->SetModuleComplete(); + return; + } + TagInfo &tag = it->second; + auto blob_it = std::find(tag.blobs_.begin(), tag.blobs_.end(), task->blob_id_); + tag.blobs_.erase(blob_it); + task->SetModuleComplete(); + } + + /** Clear blobs from a tag */ + void TagClearBlobs(TagClearBlobsTask *task) { + auto it = tag_map_.find(task->tag_id_); + if (it == tag_map_.end()) { + task->SetModuleComplete(); + return; + } + TagInfo &tag = it->second; + tag.blobs_.clear(); + tag.internal_size_ = 0; + task->SetModuleComplete(); + } + + /** Get size of the bucket */ + void GetSize(GetSizeTask *task) { + auto it = tag_map_.find(task->tag_id_); + if (it == tag_map_.end()) { + task->size_ = 0; + task->SetModuleComplete(); + return; + } + TagInfo &tag = it->second; + task->size_ = tag.internal_size_; + task->SetModuleComplete(); + } + + + + public: +#include "hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(hermes::bucket_mdm::Server, "hermes_bucket_mdm"); diff --git a/tasks/hermes_mdm/CMakeLists.txt b/tasks/hermes_mdm/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks/hermes_mdm/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h new file mode 100644 index 000000000..11f6c42e9 --- /dev/null +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h @@ -0,0 +1,43 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_hermes_mdm_H_ +#define LABSTOR_hermes_mdm_H_ + +#include "hermes_mdm_tasks.h" + +namespace hermes::mdm { + +/** Create requests */ +class Client : public TaskLibClient { + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Create a hermes_mdm */ + HSHM_ALWAYS_INLINE + void CreateRoot(const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + std::vector queue_info = { + {1, 1, 1, 0}, + }; + id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + domain_id, state_name, id_, queue_info); + queue_id_ = QueueId(id_); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } +}; + +} // namespace labstor + +#endif // LABSTOR_hermes_mdm_H_ diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h new file mode 100644 index 000000000..127126f51 --- /dev/null +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h @@ -0,0 +1,114 @@ +#ifndef LABSTOR_HERMES_MDM_LIB_EXEC_H_ +#define LABSTOR_HERMES_MDM_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_HERMES_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h new file mode 100644 index 000000000..8a4383c14 --- /dev/null +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h @@ -0,0 +1,8 @@ +#ifndef LABSTOR_HERMES_MDM_METHODS_H_ +#define LABSTOR_HERMES_MDM_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { +}; + +#endif // LABSTOR_HERMES_MDM_METHODS_H_ \ No newline at end of file diff --git a/ci/jarvis-util/jarvis_util/serialize/__init__.py b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.yaml similarity index 100% rename from ci/jarvis-util/jarvis_util/serialize/__init__.py rename to tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.yaml diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h new file mode 100644 index 000000000..143f0b055 --- /dev/null +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h @@ -0,0 +1,95 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ +#define LABSTOR_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "hermes/hermes_types.h" +#include "bdev/bdev.h" +#include "proc_queue/proc_queue.h" + +namespace hermes::mdm { + +#include "hermes_mdm_methods.h" +#include "labstor/labstor_namespace.h" + +/** + * A task to create hermes_mdm + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + IN hipc::ShmArchive server_config_path_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info, + const std::string &server_config_path = "") + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "hermes_mdm", id, queue_info) { + // Custom params + HSHM_MAKE_AR(server_config_path_, alloc, server_config_path); + } + + /** Destructor */ + HSHM_ALWAYS_INLINE + ~ConstructTask() { + HSHM_DESTROY_AR(server_config_path_); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + CreateTaskStateTask::SerializeStart(ar); + ar(server_config_path_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy hermes_mdm */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace hermes::mdm + +#endif // LABSTOR_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ diff --git a/tasks/hermes_mdm/src/CMakeLists.txt b/tasks/hermes_mdm/src/CMakeLists.txt new file mode 100644 index 000000000..07107874a --- /dev/null +++ b/tasks/hermes_mdm/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(hermes_mdm SHARED + hermes_mdm.cc) +add_dependencies(hermes_mdm ${Labstor_RUNTIME_DEPS} hermes) +target_link_libraries(hermes_mdm ${Labstor_RUNTIME_LIBRARIES} hermes) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + hermes_mdm + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + hermes_mdm + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(hermes_mdm) +endif() diff --git a/tasks/hermes_mdm/src/hermes_mdm.cc b/tasks/hermes_mdm/src/hermes_mdm.cc new file mode 100644 index 000000000..d424753cb --- /dev/null +++ b/tasks/hermes_mdm/src/hermes_mdm.cc @@ -0,0 +1,46 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "hermes/config_server.h" +#include "hermes_mdm/hermes_mdm.h" +#include "hermes_blob_mdm/hermes_blob_mdm.h" +#include "hermes_bucket_mdm/hermes_bucket_mdm.h" +#include "hermes/dpe/dpe_factory.h" +#include "bdev/bdev.h" + +namespace hermes::mdm { + +class Server : public TaskLib { + public: + /**==================================== + * Configuration + * ===================================*/ + u32 node_id_; + blob_mdm::Client blob_mdm_; + bucket_mdm::Client bkt_mdm_; + + public: + Server() = default; + + void Construct(ConstructTask *task) { + HILOG(kDebug, "ConstructTaskPhase::kLoadConfig") + std::string config_path = task->server_config_path_->str(); + HERMES_CONF->LoadServerConfig(config_path); + node_id_ = LABSTOR_CLIENT->node_id_; + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + public: +#include "hermes_mdm/hermes_mdm_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(hermes::mdm::Server, "hermes_mdm"); diff --git a/tasks/posix_bdev/CMakeLists.txt b/tasks/posix_bdev/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks/posix_bdev/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/posix_bdev/include/posix_bdev/posix_bdev.h b/tasks/posix_bdev/include/posix_bdev/posix_bdev.h new file mode 100644 index 000000000..849d8f0c3 --- /dev/null +++ b/tasks/posix_bdev/include/posix_bdev/posix_bdev.h @@ -0,0 +1,20 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_posix_bdev_H_ +#define LABSTOR_posix_bdev_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "hermes/hermes_types.h" +#include "bdev/bdev.h" +#include "labstor/labstor_namespace.h" + +namespace hermes::posix_bdev { +#include "bdev/bdev_namespace.h" +} // namespace labstor + +#endif // LABSTOR_posix_bdev_H_ diff --git a/tasks/posix_bdev/src/CMakeLists.txt b/tasks/posix_bdev/src/CMakeLists.txt new file mode 100644 index 000000000..b8c763fb3 --- /dev/null +++ b/tasks/posix_bdev/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(posix_bdev SHARED + posix_bdev.cc) +add_dependencies(posix_bdev ${Labstor_RUNTIME_DEPS}) +target_link_libraries(posix_bdev ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + posix_bdev + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + posix_bdev + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(posix_bdev) +endif() diff --git a/tasks/posix_bdev/src/posix_bdev.cc b/tasks/posix_bdev/src/posix_bdev.cc new file mode 100644 index 000000000..f33cc8609 --- /dev/null +++ b/tasks/posix_bdev/src/posix_bdev.cc @@ -0,0 +1,91 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "posix_bdev/posix_bdev.h" +#include "hermes/slab_allocator.h" + +#include +#include +#include +#include + +namespace hermes::posix_bdev { + +class Server : public TaskLib { + public: + SlabAllocator alloc_; + char *mem_ptr_; + int fd_; + std::string path_; + + public: + void Construct(ConstructTask *task) { + DeviceInfo &dev_info = task->info_; + alloc_.Init(id_, dev_info.capacity_, dev_info.slab_sizes_); + std::string text = dev_info.mount_dir_ + + "/" + "slab_" + dev_info.dev_name_; + auto canon = stdfs::weakly_canonical(text).string(); + dev_info.mount_point_ = canon; + path_ = canon; + fd_ = open(dev_info.mount_point_.c_str(), + O_TRUNC | O_CREAT, 0666); + if (fd_ < 0) { + HELOG(kError, "Failed to open file: {}", dev_info.mount_point_); + } + HILOG(kInfo, "Created {} at {} of size {}", + dev_info.dev_name_, dev_info.mount_point_, dev_info.capacity_); + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + free(mem_ptr_); + task->SetModuleComplete(); + } + + void Alloc(AllocateTask *task) { + alloc_.Allocate(task->size_, *task->buffers_, task->alloc_size_); + HILOG(kDebug, "Allocated {}/{} bytes ({})", task->alloc_size_, task->size_, path_); + task->SetModuleComplete(); + } + + void Free(FreeTask *task) { + alloc_.Free(task->buffers_); + task->SetModuleComplete(); + } + + void Write(WriteTask *task) { + size_t count = pwrite(fd_, task->buf_, task->size_, (off_t)task->disk_off_); + if (count != task->size_) { + HELOG(kError, "BORG: wrote {} bytes, but expected {}", + count, task->size_); + } + task->SetModuleComplete(); + } + + void Read(ReadTask *task) { + memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); + size_t count = pread(fd_, task->buf_, task->size_, (off_t)task->disk_off_); + if (count != task->size_) { + HELOG(kError, "BORG: read {} bytes, but expected {}", + count, task->size_); + } + task->SetModuleComplete(); + } + + void Monitor(MonitorTask *task) { + } + + void UpdateCapacity(UpdateCapacityTask *task) { + task->SetModuleComplete(); + } + + public: +#include "bdev/bdev_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(hermes::posix_bdev::Server, "posix_bdev"); diff --git a/tasks/ram_bdev/CMakeLists.txt b/tasks/ram_bdev/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks/ram_bdev/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/ram_bdev/include/ram_bdev/ram_bdev.h b/tasks/ram_bdev/include/ram_bdev/ram_bdev.h new file mode 100644 index 000000000..b6e004101 --- /dev/null +++ b/tasks/ram_bdev/include/ram_bdev/ram_bdev.h @@ -0,0 +1,21 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_ram_bdev_H_ +#define LABSTOR_ram_bdev_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "hermes/hermes_types.h" +#include "bdev/bdev.h" +#include "labstor/labstor_namespace.h" + +namespace hermes::ram_bdev { +#include "bdev/bdev_namespace.h" +} // namespace labstor + + +#endif // LABSTOR_ram_bdev_H_ diff --git a/tasks/ram_bdev/src/CMakeLists.txt b/tasks/ram_bdev/src/CMakeLists.txt new file mode 100644 index 000000000..a0a22e62f --- /dev/null +++ b/tasks/ram_bdev/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(ram_bdev SHARED + ram_bdev.cc) +add_dependencies(ram_bdev ${Labstor_RUNTIME_DEPS}) +target_link_libraries(ram_bdev ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + ram_bdev + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + ram_bdev + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(ram_bdev) +endif() diff --git a/tasks/ram_bdev/src/ram_bdev.cc b/tasks/ram_bdev/src/ram_bdev.cc new file mode 100644 index 000000000..9131018df --- /dev/null +++ b/tasks/ram_bdev/src/ram_bdev.cc @@ -0,0 +1,67 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "ram_bdev/ram_bdev.h" +#include "hermes/slab_allocator.h" + +namespace hermes::ram_bdev { + +class Server : public TaskLib { + public: + SlabAllocator alloc_; + char *mem_ptr_; + + public: + void Construct(ConstructTask *task) { + DeviceInfo &dev_info = task->info_; + alloc_.Init(id_, dev_info.capacity_, dev_info.slab_sizes_); + mem_ptr_ = (char*)malloc(dev_info.capacity_); + HILOG(kDebug, "Created {} at {} of size {}", + dev_info.dev_name_, dev_info.mount_point_, dev_info.capacity_); + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + free(mem_ptr_); + task->SetModuleComplete(); + } + + void Alloc(AllocateTask *task) { + HILOG(kDebug, "Allocating {} bytes (RAM)", task->size_); + alloc_.Allocate(task->size_, *task->buffers_, task->alloc_size_); + HILOG(kDebug, "Allocated {} bytes (RAM)", task->alloc_size_); + task->SetModuleComplete(); + } + + void Free(FreeTask *task) { + alloc_.Free(task->buffers_); + task->SetModuleComplete(); + } + + void Write(WriteTask *task) { + memcpy(mem_ptr_ + task->disk_off_, task->buf_, task->size_); + task->SetModuleComplete(); + } + + void Read(ReadTask *task) { + memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); + task->SetModuleComplete(); + } + + void Monitor(MonitorTask *task) { + } + + void UpdateCapacity(UpdateCapacityTask *task) { + task->SetModuleComplete(); + } + + public: +#include "bdev/bdev_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(hermes::ram_bdev::Server, "ram_bdev"); diff --git a/tasks_required/CMakeLists.txt b/tasks_required/CMakeLists.txt new file mode 100644 index 000000000..0ab7c475e --- /dev/null +++ b/tasks_required/CMakeLists.txt @@ -0,0 +1,7 @@ +add_subdirectory(TASK_NAME) +add_subdirectory(small_message) +add_subdirectory(labstor_admin) +add_subdirectory(remote_queue) +add_subdirectory(worch_proc_round_robin) +add_subdirectory(worch_queue_round_robin) +add_subdirectory(proc_queue) \ No newline at end of file diff --git a/tasks_required/TASK_NAME/CMakeLists.txt b/tasks_required/TASK_NAME/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks_required/TASK_NAME/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h new file mode 100644 index 000000000..2495f6594 --- /dev/null +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h @@ -0,0 +1,72 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_TASK_NAME_H_ +#define LABSTOR_TASK_NAME_H_ + +#include "TASK_NAME_tasks.h" + +namespace labstor::TASK_NAME { + +/** Create TASK_NAME requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Async create a task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info); + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate) + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + auto *task = AsyncCreateRoot(std::forward(args)...); + task->Wait(); + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Call a custom method */ + HSHM_ALWAYS_INLINE + void AsyncCustomConstruct(CustomTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_); + } + HSHM_ALWAYS_INLINE + void CustomRoot(const DomainId &domain_id) { + LPointer> task = AsyncCustomRoot(domain_id); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Custom); +}; + +} // namespace labstor + +#endif // LABSTOR_TASK_NAME_H_ diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h new file mode 100644 index 000000000..c603dac50 --- /dev/null +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h @@ -0,0 +1,146 @@ +#ifndef LABSTOR_TASK_NAME_LIB_EXEC_H_ +#define LABSTOR_TASK_NAME_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + Custom(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kCustom: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kCustom: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kCustom: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_TASK_NAME_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h new file mode 100644 index 000000000..79d7137bb --- /dev/null +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_TASK_NAME_METHODS_H_ +#define LABSTOR_TASK_NAME_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kCustom = kLast + 0; +}; + +#endif // LABSTOR_TASK_NAME_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml new file mode 100644 index 000000000..b1b54e2a4 --- /dev/null +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml @@ -0,0 +1 @@ +kCustom: 0 \ No newline at end of file diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h new file mode 100644 index 000000000..52b4c6bce --- /dev/null +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h @@ -0,0 +1,125 @@ +// +// Created by lukemartinlogan on 8/11/23. +// + +#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ +#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::TASK_NAME { + +#include "TASK_NAME_methods.h" +#include "labstor/labstor_namespace.h" +using labstor::proc_queue::TypedPushTask; +using labstor::proc_queue::PushTask; + +/** + * A task to create TASK_NAME + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) + : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "TASK_NAME", id, queue_info) { + // Custom params + } + + HSHM_ALWAYS_INLINE + ~ConstructTask() { + // Custom params + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy TASK_NAME */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) + : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in TASK_NAME + * */ +struct CustomTask : public Task, TaskFlags { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kCustom; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Custom params + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor::TASK_NAME + +#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ diff --git a/tasks_required/TASK_NAME/src/CMakeLists.txt b/tasks_required/TASK_NAME/src/CMakeLists.txt new file mode 100644 index 000000000..b021e33ec --- /dev/null +++ b/tasks_required/TASK_NAME/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(TASK_NAME SHARED + TASK_NAME.cc) +add_dependencies(TASK_NAME ${Labstor_RUNTIME_DEPS}) +target_link_libraries(TASK_NAME ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + TASK_NAME + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + TASK_NAME + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(TASK_NAME) +endif() diff --git a/tasks_required/TASK_NAME/src/TASK_NAME.cc b/tasks_required/TASK_NAME/src/TASK_NAME.cc new file mode 100644 index 000000000..9142ee436 --- /dev/null +++ b/tasks_required/TASK_NAME/src/TASK_NAME.cc @@ -0,0 +1,33 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "TASK_NAME/TASK_NAME.h" + +namespace labstor::TASK_NAME { + +class Server : public TaskLib { + public: + Server() = default; + + void Construct(ConstructTask *task) { + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + void Custom(CustomTask *task) { + task->SetModuleComplete(); + } + + public: +#include "TASK_NAME/TASK_NAME_lib_exec.h" +}; + +} // namespace labstor::TASK_NAME + +LABSTOR_TASK_CC(labstor::TASK_NAME::Server, "TASK_NAME"); diff --git a/tasks_required/labstor_admin/CMakeLists.txt b/tasks_required/labstor_admin/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks_required/labstor_admin/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h new file mode 100644 index 000000000..8fc0ab097 --- /dev/null +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h @@ -0,0 +1,225 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_TASKS_LABSTOR_ADMIN_LABSTOR_ADMIN_H_ +#define LABSTOR_TASKS_LABSTOR_ADMIN_LABSTOR_ADMIN_H_ + +#include "labstor_admin_tasks.h" + +namespace labstor::Admin { + +/** Create admin requests */ +class Client : public TaskLibClient { + public: + /** Default constructor */ + Client() { + id_ = TaskStateId(LABSTOR_QM_CLIENT->admin_queue_); + queue_id_ = LABSTOR_QM_CLIENT->admin_queue_; + } + + /** Destructor */ + ~Client() = default; + + /** Register a task library */ + HSHM_ALWAYS_INLINE + void AsyncRegisterTaskLibConstruct(RegisterTaskLibTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &lib_name) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, lib_name); + } + HSHM_ALWAYS_INLINE + void RegisterTaskLibRoot(const DomainId &domain_id, + const std::string &lib_name) { + LPointer task = AsyncRegisterTaskLibRoot(domain_id, lib_name); + task->Wait(); + LABSTOR_CLIENT->DelTask(task); + } + LABSTOR_TASK_NODE_ADMIN_ROOT(RegisterTaskLib) + + /** Unregister a task */ + HSHM_ALWAYS_INLINE + void AsyncDestroyTaskLibConstruct(DestroyTaskLibTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &lib_name) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, lib_name); + } + HSHM_ALWAYS_INLINE + void DestroyTaskLibRoot(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &lib_name) { + LPointer task = + AsyncDestroyTaskLibRoot(domain_id, lib_name); + task->Wait(); + LABSTOR_CLIENT->DelTask(task); + } + LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskLib) + + /** Spawn a task state */ + template + HSHM_ALWAYS_INLINE + void AsyncCreateTaskStateConstruct(CreateTaskT *task, + const TaskNode &task_node, + const DomainId &domain_id, + Args&& ...args) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, std::forward(args)...); + } + template + HSHM_ALWAYS_INLINE + TaskStateId CreateTaskStateRoot(const DomainId &domain_id, + Args&& ...args) { + LPointer task = AsyncCreateTaskStateRoot( + domain_id, std::forward(args)...); + task->Wait(); + TaskStateId new_id = task->id_; + LABSTOR_CLIENT->DelTask(task); + if (new_id.IsNull()) { + HELOG(kWarning, "Failed to create task state"); + } + return new_id; + } + template + hipc::LPointer AsyncCreateTaskStateAlloc(const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = LABSTOR_CLIENT->AllocateTask(); + AsyncCreateTaskStateConstruct(task.ptr_, task_node, std::forward(args)...); + return task; + } + template + hipc::LPointer AsyncCreateTaskState(const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = + AsyncCreateTaskStateAlloc(task_node, std::forward(args)...); + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); + return task; + } + template + hipc::LPointer AsyncCreateTaskStateEmplace(MultiQueue *queue, + const TaskNode &task_node, + Args&& ...args) { + hipc::LPointer task = + AsyncCreateTaskStateAllocCreateTaskT(task_node, std::forward(args)...); + queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); + return task; + } + template + hipc::LPointer AsyncCreateTaskStateRoot(Args&& ...args) { + TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId(); + hipc::LPointer task = + AsyncCreateTaskState(task_node, std::forward(args)...); + return task; + } + + /** Get the ID of a task state */ + void AsyncGetOrCreateTaskStateIdConstruct(GetOrCreateTaskStateIdTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, state_name); + } + TaskStateId GetOrCreateTaskStateIdRoot(const DomainId &domain_id, + const std::string &state_name) { + LPointer task = + AsyncGetOrCreateTaskStateIdRoot(domain_id, state_name); + task->Wait(); + TaskStateId new_id = task->id_; + LABSTOR_CLIENT->DelTask(task); + return new_id; + } + LABSTOR_TASK_NODE_ADMIN_ROOT(GetOrCreateTaskStateId) + + /** Get the ID of a task state */ + void AsyncGetTaskStateIdConstruct(GetTaskStateIdTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, state_name); + } + TaskStateId GetTaskStateIdRoot(const DomainId &domain_id, + const std::string &state_name) { + LPointer task = + AsyncGetTaskStateIdRoot(domain_id, state_name); + task->Wait(); + TaskStateId new_id = task->id_; + LABSTOR_CLIENT->DelTask(task); + return new_id; + } + LABSTOR_TASK_NODE_ADMIN_ROOT(GetTaskStateId) + + /** Terminate a task state */ + HSHM_ALWAYS_INLINE + void AsyncDestroyTaskStateConstruct(DestroyTaskStateTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id); + } + HSHM_ALWAYS_INLINE + void DestroyTaskStateRoot(const DomainId &domain_id, + const TaskStateId &id) { + LPointer task = + AsyncDestroyTaskStateRoot(domain_id, id); + task->Wait(); + LABSTOR_CLIENT->DelTask(task); + } + LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskState) + + /** Terminate the runtime */ + void AsyncStopRuntimeConstruct(StopRuntimeTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id); + } + LABSTOR_TASK_NODE_ADMIN_ROOT(StopRuntime); + + /** Set work orchestrator queue policy */ + void AsyncSetWorkOrchQueuePolicyConstruct(SetWorkOrchQueuePolicyTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &policy) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, policy); + } + void SetWorkOrchQueuePolicyRoot(const DomainId &domain_id, + const TaskStateId &policy) { + LPointer task = + AsyncSetWorkOrchQueuePolicyRoot(domain_id, policy); + task->Wait(); + LABSTOR_CLIENT->DelTask(task); + } + LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchQueuePolicy); + + /** Set work orchestrator process policy */ + void AsyncSetWorkOrchProcPolicyConstruct(SetWorkOrchProcPolicyTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &policy) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, policy); + } + void SetWorkOrchProcPolicyRoot(const DomainId &domain_id, + const TaskStateId &policy) { + LPointer task = + AsyncSetWorkOrchProcPolicyRoot(domain_id, policy); + task->Wait(); + LABSTOR_CLIENT->DelTask(task); + } + LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchProcPolicy); +}; + +} // namespace labstor::Admin + +#define LABSTOR_ADMIN \ + hshm::EasySingleton::GetInstance() + +#endif // LABSTOR_TASKS_LABSTOR_ADMIN_LABSTOR_ADMIN_H_ diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h new file mode 100644 index 000000000..2cfe22c2b --- /dev/null +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h @@ -0,0 +1,338 @@ +#ifndef LABSTOR_LABSTOR_ADMIN_LIB_EXEC_H_ +#define LABSTOR_LABSTOR_ADMIN_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kCreateTaskState: { + CreateTaskState(reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskState: { + DestroyTaskState(reinterpret_cast(task)); + break; + } + case Method::kRegisterTaskLib: { + RegisterTaskLib(reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskLib: { + DestroyTaskLib(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTaskStateId: { + GetOrCreateTaskStateId(reinterpret_cast(task)); + break; + } + case Method::kGetTaskStateId: { + GetTaskStateId(reinterpret_cast(task)); + break; + } + case Method::kStopRuntime: { + StopRuntime(reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + SetWorkOrchQueuePolicy(reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchProcPolicy: { + SetWorkOrchProcPolicy(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kCreateTaskState: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskState: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kRegisterTaskLib: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskLib: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTaskStateId: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kGetTaskStateId: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kStopRuntime: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchProcPolicy: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kCreateTaskState: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskState: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kRegisterTaskLib: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskLib: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTaskStateId: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kGetTaskStateId: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kStopRuntime: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchProcPolicy: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kCreateTaskState: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyTaskState: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRegisterTaskLib: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyTaskLib: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetOrCreateTaskStateId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetTaskStateId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kStopRuntime: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetWorkOrchProcPolicy: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kCreateTaskState: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestroyTaskState: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kRegisterTaskLib: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestroyTaskLib: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetOrCreateTaskStateId: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kGetTaskStateId: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kStopRuntime: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kSetWorkOrchProcPolicy: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kCreateTaskState: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyTaskState: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRegisterTaskLib: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestroyTaskLib: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetOrCreateTaskStateId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kGetTaskStateId: { + ar << *reinterpret_cast(task); + break; + } + case Method::kStopRuntime: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSetWorkOrchProcPolicy: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kCreateTaskState: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskState: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kRegisterTaskLib: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskLib: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTaskStateId: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kGetTaskStateId: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kStopRuntime: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchProcPolicy: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kCreateTaskState: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestroyTaskState: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kRegisterTaskLib: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestroyTaskLib: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetOrCreateTaskStateId: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kGetTaskStateId: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kStopRuntime: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kSetWorkOrchQueuePolicy: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kSetWorkOrchProcPolicy: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_LABSTOR_ADMIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h new file mode 100644 index 000000000..6887eda8b --- /dev/null +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h @@ -0,0 +1,17 @@ +#ifndef LABSTOR_LABSTOR_ADMIN_METHODS_H_ +#define LABSTOR_LABSTOR_ADMIN_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kCreateTaskState = kLast + 0; + TASK_METHOD_T kDestroyTaskState = kLast + 1; + TASK_METHOD_T kRegisterTaskLib = kLast + 2; + TASK_METHOD_T kDestroyTaskLib = kLast + 3; + TASK_METHOD_T kGetOrCreateTaskStateId = kLast + 4; + TASK_METHOD_T kGetTaskStateId = kLast + 5; + TASK_METHOD_T kStopRuntime = kLast + 6; + TASK_METHOD_T kSetWorkOrchQueuePolicy = kLast + 7; + TASK_METHOD_T kSetWorkOrchProcPolicy = kLast + 8; +}; + +#endif // LABSTOR_LABSTOR_ADMIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml new file mode 100644 index 000000000..016aa0455 --- /dev/null +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml @@ -0,0 +1,9 @@ +kCreateTaskState: 0 +kDestroyTaskState: 1 +kRegisterTaskLib: 2 +kDestroyTaskLib: 3 +kGetOrCreateTaskStateId: 4 +kGetTaskStateId: 5 +kStopRuntime: 6 +kSetWorkOrchQueuePolicy: 7 +kSetWorkOrchProcPolicy: 8 \ No newline at end of file diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h new file mode 100644 index 000000000..d236af213 --- /dev/null +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -0,0 +1,410 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_TASKS_LABSTOR_ADMIN_INCLUDE_LABSTOR_ADMIN_LABSTOR_ADMIN_TASKS_H_ +#define LABSTOR_TASKS_LABSTOR_ADMIN_INCLUDE_LABSTOR_ADMIN_LABSTOR_ADMIN_TASKS_H_ + +#include "labstor/work_orchestrator/scheduler.h" +#include "labstor/api/labstor_client.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "labstor/labstor_namespace.h" + +namespace labstor::Admin { + +#include "labstor_admin_methods.h" + +/** A template to register or destroy a task library */ +template +struct RegisterTaskLibTaskTempl : public Task, TaskFlags { + IN hipc::ShmArchive lib_name_; + OUT TaskStateId id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterTaskLibTaskTempl(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterTaskLibTaskTempl(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &lib_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + if constexpr(method == 0) + { + method_ = Method::kRegisterTaskLib; + } else { + method_ = Method::kDestroyTaskLib; + } + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Initialize QueueManager task + HSHM_MAKE_AR(lib_name_, alloc, lib_name); + } + + /** Destructor */ + ~RegisterTaskLibTaskTempl() { + HSHM_DESTROY_AR(lib_name_); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(lib_name_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to register a Task Library */ +using RegisterTaskLibTask = RegisterTaskLibTaskTempl<0>; + +/** A task to destroy a Task Library */ +using DestroyTaskLibTask = RegisterTaskLibTaskTempl<1>; + +class CreateTaskStatePhase { + public: + // NOTE(llogan): kLast is intentially 0 so that the constructor + // can seamlessly pass data to submethods + TASK_METHOD_T kIdAllocStart = 0; + TASK_METHOD_T kIdAllocWait = 1; + TASK_METHOD_T kStateCreate = 2; + TASK_METHOD_T kLast = 0; +}; + +/** A task to get or retrieve the ID of a task */ +struct GetOrCreateTaskStateIdTask : public Task, TaskFlags { + IN hipc::ShmArchive state_name_; + OUT TaskStateId id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetOrCreateTaskStateIdTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetOrCreateTaskStateIdTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + method_ = Method::kGetOrCreateTaskStateId; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Initialize + HSHM_MAKE_AR(state_name_, alloc, state_name); + } + + ~GetOrCreateTaskStateIdTask() { + HSHM_DESTROY_AR(state_name_); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(state_name_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to register a Task state + Create a queue */ +struct CreateTaskStateTask : public Task, TaskFlags { + IN hipc::ShmArchive lib_name_; + IN hipc::ShmArchive state_name_; + IN hipc::ShmArchive> queue_info_; + INOUT TaskStateId id_; + TEMP int phase_ = 0; + TEMP GetOrCreateTaskStateIdTask *get_id_task_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + CreateTaskStateTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + CreateTaskStateTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const std::string &lib_name, + const TaskStateId &id, + const std::vector &queue_info) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + method_ = Method::kCreateTaskState; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Initialize + HSHM_MAKE_AR(state_name_, alloc, state_name); + HSHM_MAKE_AR(lib_name_, alloc, lib_name); + HSHM_MAKE_AR(queue_info_, alloc, queue_info); + id_ = id; + } + + /** Destructor */ + ~CreateTaskStateTask() { + HSHM_DESTROY_AR(state_name_); + HSHM_DESTROY_AR(lib_name_); + HSHM_DESTROY_AR(queue_info_); + } + + /** Replication (does nothing) */ + void ReplicateStart(u32 count) { + } + + /** Replicate end (does nothing) */ + void ReplicateEnd() { + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(lib_name_, state_name_, id_, queue_info_, phase_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to retrieve the ID of a task */ +struct GetTaskStateIdTask : public Task, TaskFlags { + IN hipc::ShmArchive + state_name_; + OUT TaskStateId + id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetTaskStateIdTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetTaskStateIdTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + method_ = Method::kGetTaskStateId; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Initialize + HSHM_MAKE_AR(state_name_, alloc, state_name); + } + + ~GetTaskStateIdTask() { + HSHM_DESTROY_AR(state_name_); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(state_name_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(id_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy a Task state */ +struct DestroyTaskStateTask : public Task, TaskFlags { + IN TaskStateId id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestroyTaskStateTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestroyTaskStateTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + method_ = Method::kDestroyTaskState; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Initialize + id_ = id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar & id_; + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy a Task state */ +struct StopRuntimeTask : public Task, TaskFlags { + /** SHM default constructor */ + StopRuntimeTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + StopRuntimeTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + method_ = Method::kStopRuntime; + task_flags_.SetBits(TASK_FIRE_AND_FORGET); + domain_id_ = domain_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy a Task state */ +template +struct SetWorkOrchestratorPolicyTask : public Task, TaskFlags { + IN TaskStateId policy_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + SetWorkOrchestratorPolicyTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + SetWorkOrchestratorPolicyTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &policy_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + if constexpr(method == 0) { + method_ = Method::kSetWorkOrchQueuePolicy; + } else { + method_ = Method::kSetWorkOrchProcPolicy; + } + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Initialize + policy_id_ = policy_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(policy_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; +using SetWorkOrchQueuePolicyTask = SetWorkOrchestratorPolicyTask<0>; +using SetWorkOrchProcPolicyTask = SetWorkOrchestratorPolicyTask<1>; + +} // namespace labstor::Admin + +#endif // LABSTOR_TASKS_LABSTOR_ADMIN_INCLUDE_LABSTOR_ADMIN_LABSTOR_ADMIN_TASKS_H_ diff --git a/tasks_required/labstor_admin/src/CMakeLists.txt b/tasks_required/labstor_admin/src/CMakeLists.txt new file mode 100644 index 000000000..bf3c06c61 --- /dev/null +++ b/tasks_required/labstor_admin/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +add_library(labstor_admin SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/labstor_admin.cc) +add_dependencies(labstor_admin ${Labstor_RUNTIME_DEPS}) +target_link_libraries(labstor_admin ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Labstor Admin Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + labstor_admin + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + labstor_admin + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(labstor_admin) +endif() diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc new file mode 100644 index 000000000..957756fc9 --- /dev/null +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -0,0 +1,179 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "labstor/work_orchestrator/scheduler.h" + +namespace labstor::Admin { + +class Server : public TaskLib { + public: + Task *queue_sched_; + Task *proc_sched_; + + public: + Server() : queue_sched_(nullptr), proc_sched_(nullptr) {} + + void RegisterTaskLib(RegisterTaskLibTask *task) { + std::string lib_name = task->lib_name_->str(); + LABSTOR_TASK_REGISTRY->RegisterTaskLib(lib_name); + task->SetModuleComplete(); + } + + void DestroyTaskLib(DestroyTaskLibTask *task) { + std::string lib_name = task->lib_name_->str(); + LABSTOR_TASK_REGISTRY->DestroyTaskLib(lib_name); + task->SetModuleComplete(); + } + + void GetOrCreateTaskStateId(GetOrCreateTaskStateIdTask *task) { + std::string state_name = task->state_name_->str(); + task->id_ = LABSTOR_TASK_REGISTRY->GetOrCreateTaskStateId(state_name); + task->SetModuleComplete(); + } + + void CreateTaskState(CreateTaskStateTask *task) { + switch (task->phase_) { + case CreateTaskStatePhase::kIdAllocStart: { + std::string lib_name = task->lib_name_->str(); + std::string state_name = task->state_name_->str(); + // Check local registry for task state + TaskState *task_state = LABSTOR_TASK_REGISTRY->GetTaskState(state_name, task->id_); + if (task_state) { + task->id_ = task_state->id_; + task->SetModuleComplete(); + return; + } + // Check global registry for task state + if (task->id_.IsNull()) { +// if (task->domain_id_ == DomainId::GetLocal()) { +// HILOG(kDebug, "Domain ID is local for {} (task_node={})", state_name, task->task_node_); +// task->id_ = LABSTOR_TASK_REGISTRY->GetOrCreateTaskStateId(state_name); +// task->phase_ = CreateTaskStatePhase::kStateCreate; +// } else { + HILOG(kDebug, "Domain ID is global for {} (task_node={})", state_name, task->task_node_); + DomainId domain = DomainId::GetNode(1); + task->get_id_task_ = LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId( + task->task_node_ + 1, domain, state_name).ptr_; + task->phase_ = CreateTaskStatePhase::kIdAllocWait; +// } + } else { + HILOG(kDebug, "Domain ID is given as {} for {} (task_node={})", task->id_, state_name, task->task_node_); + task->phase_ = CreateTaskStatePhase::kStateCreate; + } + return; + } + case CreateTaskStatePhase::kIdAllocWait: { + if (!task->get_id_task_->IsComplete()) { + return; + } + task->id_ = task->get_id_task_->id_; + task->phase_ = CreateTaskStatePhase::kStateCreate; + LABSTOR_CLIENT->DelTask(task->get_id_task_); + } + case CreateTaskStatePhase::kStateCreate: { + std::string lib_name = task->lib_name_->str(); + std::string state_name = task->state_name_->str(); + HILOG(kInfo, "(node {}) Creating task state {} with id {} (task_node={})", + LABSTOR_CLIENT->node_id_, state_name, task->id_, task->task_node_); + + // Verify the state isn't NULL + if (task->id_.IsNull()) { + HELOG(kError, "(node {}) The task state {} with id {} is NULL.", + LABSTOR_CLIENT->node_id_, state_name, task->id_); + task->SetModuleComplete(); + return; + } + + // Verify the state doesn't exist + if (LABSTOR_TASK_REGISTRY->TaskStateExists(task->id_)) { + HILOG(kInfo, "(node {}) The task state {} with id {} exists", + LABSTOR_CLIENT->node_id_, state_name, task->id_); + task->SetModuleComplete(); + return; + } + + // The state is being created + // NOTE(llogan): this does NOT return since task creations can have phases + task->method_ = Method::kConstruct; + + // Create the task queue for the state + QueueId qid(task->id_); + LABSTOR_QM_RUNTIME->CreateQueue( + qid, task->queue_info_->vec()); + + // Begin creating the task state + task->phase_ = 0; + task->task_state_ = task->id_; + bool ret = LABSTOR_TASK_REGISTRY->CreateTaskState( + lib_name.c_str(), + state_name.c_str(), + task->id_, + task); + if (!ret) { + task->SetModuleComplete(); + return; + } + HILOG(kInfo, "(node {}) Allocated task state {} with id {}", + LABSTOR_CLIENT->node_id_, state_name, task->task_state_); + } + } + } + + void GetTaskStateId(GetTaskStateIdTask *task) { + std::string state_name = task->state_name_->str(); + task->id_ = LABSTOR_TASK_REGISTRY->GetTaskStateId(state_name); + task->SetModuleComplete(); + } + + void DestroyTaskState(DestroyTaskStateTask *task) { + LABSTOR_TASK_REGISTRY->DestroyTaskState(task->id_); + task->SetModuleComplete(); + } + + void StopRuntime(StopRuntimeTask *task) { + HILOG(kInfo, "Stopping (server mode)"); + LABSTOR_WORK_ORCHESTRATOR->FinalizeRuntime(); + LABSTOR_THALLIUM->StopThisDaemon(); + task->SetModuleComplete(); + } + + void SetWorkOrchQueuePolicy(SetWorkOrchQueuePolicyTask *task) { + if (queue_sched_) { + queue_sched_->SetModuleComplete(); + } + if (queue_sched_ && !queue_sched_->IsComplete()) { + return; + } + auto queue_sched = LABSTOR_CLIENT->NewTask( + task->task_node_, DomainId::GetLocal(), task->policy_id_); + queue_sched_ = queue_sched.ptr_; + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(0, 0, queue_sched.shm_); + task->SetModuleComplete(); + } + + void SetWorkOrchProcPolicy(SetWorkOrchProcPolicyTask *task) { + if (proc_sched_) { + proc_sched_->SetModuleComplete(); + } + if (proc_sched_ && !proc_sched_->IsComplete()) { + return; + } + auto proc_sched = LABSTOR_CLIENT->NewTask( + task->task_node_, DomainId::GetLocal(), task->policy_id_); + proc_sched_ = proc_sched.ptr_; + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(0, 0, proc_sched.shm_); + task->SetModuleComplete(); + } + + public: +#include "labstor_admin/labstor_admin_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(labstor::Admin::Server, "labstor_admin"); diff --git a/tasks_required/proc_queue/CMakeLists.txt b/tasks_required/proc_queue/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks_required/proc_queue/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue.h b/tasks_required/proc_queue/include/proc_queue/proc_queue.h new file mode 100644 index 000000000..b6aa51aab --- /dev/null +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue.h @@ -0,0 +1,87 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_proc_queue_H_ +#define LABSTOR_proc_queue_H_ + +#include "proc_queue_tasks.h" + +namespace labstor::proc_queue { + +/** Create proc_queue requests */ +class Client : public TaskLibClient { + public: + /** Default constructor */ + Client() { + id_ = TaskStateId(LABSTOR_QM_CLIENT->process_queue_); + queue_id_ = LABSTOR_QM_CLIENT->process_queue_; + } + + /** Destructor */ + ~Client() = default; + + /** Async create a task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info); + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate); + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + auto *task = AsyncCreateRoot(std::forward(args)...); + task->Wait(); + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Call a custom method */ + template + HSHM_ALWAYS_INLINE + void AsyncPushConstruct(labpq::TypedPushTask *task, + const TaskNode &task_node, + const DomainId &domain_id, + const hipc::Pointer &subtask) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_, subtask); + } + template + HSHM_ALWAYS_INLINE + LPointer> + AsyncPush(const TaskNode &task_node, + const DomainId &domain_id, + const hipc::Pointer &subtask) { + LPointer> push_task = + LABSTOR_CLIENT->AllocateTask>(); + AsyncPushConstruct(push_task.ptr_, task_node, domain_id, subtask); + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(push_task->prio_, push_task->lane_hash_, push_task.shm_); + return push_task; + } + LABSTOR_TASK_NODE_ROOT(AsyncPush); +}; + +} // namespace labstor + +#define LABSTOR_PROCESS_QUEUE \ + hshm::EasySingleton::GetInstance() + +#endif // LABSTOR_proc_queue_H_ diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h new file mode 100644 index 000000000..bea645d52 --- /dev/null +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h @@ -0,0 +1,146 @@ +#ifndef LABSTOR_PROC_QUEUE_LIB_EXEC_H_ +#define LABSTOR_PROC_QUEUE_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kPush: { + Push(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kPush: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kPush: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kPush: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kPush: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kPush: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kPush: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kPush: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_PROC_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h new file mode 100644 index 000000000..095b84bf1 --- /dev/null +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_PROC_QUEUE_METHODS_H_ +#define LABSTOR_PROC_QUEUE_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kPush = kLast + 0; +}; + +#endif // LABSTOR_PROC_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml b/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml new file mode 100644 index 000000000..fb48aa97f --- /dev/null +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml @@ -0,0 +1 @@ +kPush: 0 \ No newline at end of file diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h new file mode 100644 index 000000000..002f42436 --- /dev/null +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -0,0 +1,139 @@ +// +// Created by lukemartinlogan on 8/11/23. +// + +#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ +#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" + +namespace labstor::proc_queue { + +#include "labstor/labstor_namespace.h" +#include "proc_queue_methods.h" + +/** + * A task to create proc_queue + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) + : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "proc_queue", id, queue_info) { + // Custom params + } + + HSHM_ALWAYS_INLINE + ~ConstructTask() { + // Custom params + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy proc_queue */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) + : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +class PushTaskPhase { + public: + TASK_METHOD_T kSchedule = 0; + TASK_METHOD_T kWaitSchedule = 1; +}; + +/** + * Push a task into the per-process queue + * */ +template +struct TypedPushTask : public Task, TaskFlags { + IN hipc::Pointer subtask_; //< SHM pointer to the subtask + TEMP TaskT *subtask_ptr_; //< Pointer to the subtask (client) + TEMP TaskT *ptr_; //< Pointer to the subtask (server) + TEMP int phase_ = PushTaskPhase::kSchedule; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + TypedPushTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + TypedPushTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const hipc::Pointer &subtask) : Task(alloc) { + // Initialize task + hshm::NodeThreadId tid; + task_node_ = task_node; + lane_hash_ = tid.bits_.tid_ + tid.bits_.pid_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kPush; + task_flags_.SetBits(TASK_DATA_OWNER | TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + subtask_ = subtask; + subtask_ptr_ = (TaskT*)LABSTOR_CLIENT->GetPrivatePointer(subtask_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + group.resize(sizeof(u32)); + memcpy(group.data(), &lane_hash_, sizeof(u32)); + return 0; + } + + /** Get the task address */ + HSHM_ALWAYS_INLINE + TaskT* get() { + return subtask_ptr_; + } +}; + +using PushTask = labstor::proc_queue::TypedPushTask; + +} // namespace labstor::proc_queue + +namespace labpq = labstor::proc_queue; + +#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ diff --git a/tasks_required/proc_queue/src/CMakeLists.txt b/tasks_required/proc_queue/src/CMakeLists.txt new file mode 100644 index 000000000..1d4946b6a --- /dev/null +++ b/tasks_required/proc_queue/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(proc_queue SHARED + proc_queue.cc) +add_dependencies(proc_queue ${Labstor_RUNTIME_DEPS}) +target_link_libraries(proc_queue ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + proc_queue + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + proc_queue + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(proc_queue) +endif() diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/tasks_required/proc_queue/src/proc_queue.cc new file mode 100644 index 000000000..4b82a2966 --- /dev/null +++ b/tasks_required/proc_queue/src/proc_queue.cc @@ -0,0 +1,52 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::proc_queue { + +class Server : public TaskLib { + public: + Server() = default; + + void Construct(ConstructTask *task) { + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + void Push(PushTask *task) { + switch (task->phase_) { + case PushTaskPhase::kSchedule: { + task->ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->subtask_); + HILOG(kDebug, "Scheduling task {} on state {} tid {}", + task->ptr_->task_node_, task->ptr_->task_state_, gettid()); + if (task->ptr_->IsFireAndForget()) { + task->ptr_->UnsetFireAndForget(); + } + MultiQueue *real_queue = LABSTOR_CLIENT->GetQueue(QueueId(task->ptr_->task_state_)); + real_queue->Emplace(task->ptr_->prio_, task->ptr_->lane_hash_, task->subtask_); + task->phase_ = PushTaskPhase::kWaitSchedule; + } + case PushTaskPhase::kWaitSchedule: { + if (!task->ptr_->IsComplete()) { + return; + } + LABSTOR_CLIENT->DelTask(task->ptr_); + task->SetModuleComplete(); + } + } + } + + public: +#include "proc_queue/proc_queue_lib_exec.h" +}; + +} // namespace labstor::proc_queue + +LABSTOR_TASK_CC(labstor::proc_queue::Server, "proc_queue"); diff --git a/tasks_required/remote_queue/CMakeLists.txt b/tasks_required/remote_queue/CMakeLists.txt new file mode 100644 index 000000000..da1eafa4d --- /dev/null +++ b/tasks_required/remote_queue/CMakeLists.txt @@ -0,0 +1,14 @@ +# find_package(PkgConfig REQUIRED) +# pkg_search_module(libfabric REQUIRED libfabric) +# message("LibFabric found at: ${libfabric_INCLUDE_DIRS} with libs ${libfabric_LIBRARIES} in dir ${libfabric_LIBRARY_DIRS}") + +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h new file mode 100644 index 000000000..e733c2b42 --- /dev/null +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -0,0 +1,95 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_remote_queue_H_ +#define LABSTOR_remote_queue_H_ + +#include "remote_queue_tasks.h" + +namespace labstor::remote_queue { + +/** + * Create remote_queue requests + * + * This is ONLY used in the Hermes runtime, and + * should never be called in client programs!!! + * */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Async create a task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &state_id) { + id_ = state_id; + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info); + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate); + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + LPointer task = + AsyncCreateRoot(std::forward(args)...); + task->Wait(); + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Call a custom method */ + HSHM_ALWAYS_INLINE + void Disperse(Task *orig_task, + TaskState *exec, + std::vector &domain_ids) { + // Serialize task + create the wait task + HILOG(kDebug, "Beginning dispersion for (task_node={}, task_state={}, method={})", + orig_task->task_node_ + 1, orig_task->task_state_, orig_task->method_) + BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); + auto xfer = exec->SaveStart(orig_task->method_, ar, orig_task); + + // Create subtasks + exec->ReplicateStart(orig_task->method_, domain_ids.size(), orig_task); + auto push_task = LABSTOR_CLIENT->NewTask( + orig_task->task_node_ + 1, DomainId::GetLocal(), id_, + domain_ids, orig_task, exec, orig_task->method_, xfer); + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(orig_task->prio_, orig_task->lane_hash_, push_task.shm_); + } + + /** Spawn task to accept new connections */ +// HSHM_ALWAYS_INLINE +// AcceptTask* AsyncAcceptThread() { +// hipc::Pointer p; +// MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); +// auto *task = LABSTOR_CLIENT->NewTask( +// p, TaskNode::GetNull(), DomainId::GetLocal(), id_); +// queue->Emplace(0, 0, p); +// return task; +// } +}; + +} // namespace labstor + +#endif // LABSTOR_remote_queue_H_ diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h new file mode 100644 index 000000000..e0ff029c2 --- /dev/null +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h @@ -0,0 +1,146 @@ +#ifndef LABSTOR_REMOTE_QUEUE_LIB_EXEC_H_ +#define LABSTOR_REMOTE_QUEUE_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kPush: { + Push(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kPush: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kPush: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kPush: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kPush: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kPush: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kPush: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kPush: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_REMOTE_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h new file mode 100644 index 000000000..d0a029c43 --- /dev/null +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_REMOTE_QUEUE_METHODS_H_ +#define LABSTOR_REMOTE_QUEUE_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kPush = kLast + 0; +}; + +#endif // LABSTOR_REMOTE_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml new file mode 100644 index 000000000..fb48aa97f --- /dev/null +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml @@ -0,0 +1 @@ +kPush: 0 \ No newline at end of file diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h new file mode 100644 index 000000000..a3c120771 --- /dev/null +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -0,0 +1,165 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ +#define LABSTOR_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +#include +#include +#include +#include +#include + +namespace tl = thallium; + +namespace labstor::remote_queue { + +#include "labstor/labstor_namespace.h" +#include "remote_queue_methods.h" + +/** + * A task to create remote_queue + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "remote_queue", id, queue_info) { + // Custom params + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy remote_queue */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * Phases of the push task + * */ +class PushPhase { + public: + TASK_METHOD_T kStart = 0; + TASK_METHOD_T kWait = 1; +}; + + +/** + * A task to push a serialized task onto the remote queue + * */ +struct PushTask : public Task, TaskFlags { + IN std::vector domain_ids_; + IN Task *orig_task_; + IN TaskState *exec_; + IN u32 exec_method_; + IN std::vector xfer_; + // TEMP std::vector tl_future_; + TEMP std::vector tl_future_; + TEMP int phase_ = PushPhase::kStart; + TEMP int replica_; + TEMP std::string params_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + PushTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + PushTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + std::vector &domain_ids, + Task *orig_task, + TaskState *exec, + u32 exec_method, + std::vector &xfer) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kPush; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + domain_ids_ = std::move(domain_ids); + orig_task_ = orig_task; + exec_ = exec; + exec_method_ = exec_method; + xfer_ = std::move(xfer); + replica_ = 0; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A task to push a serialized task onto the remote queue + * */ +//struct AcceptTask : public Task { +// /** Emplace constructor */ +// HSHM_ALWAYS_INLINE explicit +// AcceptTask(hipc::Allocator *alloc, +// const TaskNode &task_node, +// const DomainId &domain_id, +// const TaskStateId &state_id) : Task(alloc) { +// // Initialize task +// task_node_ = task_node; +// lane_hash_ = 0; +// task_state_ = state_id; +// method_ = Method::kAccept; +// task_flags_.SetBits(0); +// domain_id_ = domain_id; +// } +//}; + +} // namespace labstor::remote_queue + +#endif //LABSTOR_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ diff --git a/tasks_required/remote_queue/src/CMakeLists.txt b/tasks_required/remote_queue/src/CMakeLists.txt new file mode 100644 index 000000000..ead8373f0 --- /dev/null +++ b/tasks_required/remote_queue/src/CMakeLists.txt @@ -0,0 +1,57 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +include_directories(myapp PRIVATE ${libfabric_INCLUDE_DIRS}) +link_directories(${libfabric_LIBRARY_DIRS}) +add_library(remote_queue SHARED + remote_queue.cc) +add_dependencies(remote_queue ${Labstor_RUNTIME_DEPS}) +target_link_libraries(remote_queue + labstor_runtime ${libfabric_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + remote_queue + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + remote_queue + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(remote_queue) +endif() diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc new file mode 100644 index 000000000..6ae81d883 --- /dev/null +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -0,0 +1,378 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "remote_queue/remote_queue.h" + +namespace thallium { + +/** Serialize I/O type enum */ +SERIALIZE_ENUM(labstor::IoType); + +} // namespace thallium + +namespace labstor::remote_queue { + +/** Parameters for spawning async thread for thallium */ +struct ThalliumTask { + int replica_; + PushTask *task_; + DomainId domain_id_; + IoType io_type_; + char *data_; + size_t data_size_; + ABT_thread thread_; + bool done_; + + /** Default constructor */ + ThalliumTask() : done_(false) {}; + + /** Emplace constructor Small */ + ThalliumTask(int replica, PushTask *task, DomainId domain_id) : + replica_(replica), task_(task), domain_id_(domain_id), done_(false) {} + + /** Emplace constructor I/O */ + ThalliumTask(int replica, PushTask *task, DomainId domain_id, + IoType io_type, void *data, size_t data_size) : + replica_(replica), task_(task), domain_id_(domain_id), + io_type_(io_type), data_((char*)data), data_size_(data_size), done_(false) {} + + /** Check if the thread is finished */ + bool IsDone() { + if (done_) { + ABT_thread_join(thread_); + } + return done_; + } +}; + +class Server : public TaskLib { + public: + labstor::remote_queue::Client client_; + + public: + Server() = default; + + /** Construct remote queue */ + void Construct(ConstructTask *task) { + LABSTOR_THALLIUM->RegisterRpc("RpcPushSmall", [this](const tl::request &req, + TaskStateId state_id, + u32 method, + std::string ¶ms) { + this->RpcPushSmall(req, state_id, method, params); + }); + LABSTOR_THALLIUM->RegisterRpc("RpcPushBulk", [this](const tl::request &req, + const tl::bulk &bulk, + TaskStateId state_id, + u32 method, + std::string ¶ms, + size_t data_size, + IoType io_type) { + this->RpcPushBulk(req, state_id, method, params, bulk, data_size, io_type); + }); + task->SetModuleComplete(); + } + + /** Destroy remote queue */ + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + /** Handle output from replica PUSH */ + static void HandlePushReplicaOutput(int replica, std::string &ret, PushTask *task) { + try { + std::vector xfer(1); + xfer[0].data_ = ret.data(); + xfer[0].data_size_ = ret.size(); + HILOG(kDebug, "Wait got {} bytes of data (task_node={}, task_state={}, method={})", + xfer[0].data_size_, + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_); + BinaryInputArchive ar(xfer); + task->exec_->LoadEnd(replica, task->exec_method_, ar, task->orig_task_); + } catch (std::exception &e) { + HILOG(kFatal, "Error LoadEnd (task_node={}, task_state={}, method={}): {}", + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + e.what()); + } + } + + /** Handle finalization of PUSH replicate */ + static void HandlePushReplicaEnd(PushTask *task) { + task->exec_->ReplicateEnd(task->orig_task_->method_, task->orig_task_); + task->orig_task_->SetModuleComplete(); + HILOG(kDebug, "Completing task (task_node={}, task_state={}, method={})", + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_); + task->SetModuleComplete(); + } + + /** Push for small message */ + void ClientSmallPush(std::vector &xfer, PushTask *task) { + task->params_ = std::string((char *) xfer[0].data_, xfer[0].data_size_); + for (int replica = 0; replica < task->domain_ids_.size(); ++replica) { + DomainId domain_id = task->domain_ids_[replica]; + ThalliumTask *tl_task = new ThalliumTask(replica, task, domain_id); + tl_task->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread([](void *data) { + ThalliumTask *tl_task = (ThalliumTask *) data; + DomainId &domain_id = tl_task->domain_id_; + PushTask *task = tl_task->task_; + int replica = tl_task->replica_; + HILOG(kDebug, "(SM) Transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={})", + task->params_.size(), + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_); + std::string ret = LABSTOR_THALLIUM->SyncCall(domain_id.id_, + "RpcPushSmall", + task->exec_->id_, + task->exec_method_, + task->params_); + HandlePushReplicaOutput(replica, ret, task); + tl_task->done_ = true; + }, tl_task); + task->tl_future_.emplace_back(tl_task); + + +// tl::async_response future = LABSTOR_THALLIUM->AsyncCall(domain_id.id_, +// "RpcPushSmall", +// task->exec_->id_, +// task->exec_method_, +// params); +// task->tl_future_.emplace_back(std::move(future)); + } + } + + /** Push for I/O message */ + void ClientIoPush(std::vector &xfer, PushTask *task) { + task->params_ = std::string((char *) xfer[1].data_, xfer[1].data_size_); + IoType io_type = IoType::kRead; + if (xfer[0].flags_.Any(DT_RECEIVER_READ)) { + io_type = IoType::kWrite; + } + for (int replica = 0; replica < task->domain_ids_.size(); ++replica) { + DomainId domain_id = task->domain_ids_[replica]; + ThalliumTask *tl_task = new ThalliumTask( + replica, task, domain_id, io_type, + xfer[0].data_, xfer[0].data_size_); + tl_task->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread([](void *data) { + ThalliumTask *tl_task = (ThalliumTask *) data; + DomainId &domain_id = tl_task->domain_id_; + PushTask *task = tl_task->task_; + int replica = tl_task->replica_; + IoType &io_type = tl_task->io_type_; + HILOG(kDebug, "(IO) Transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={}, type={})", + tl_task->data_size_, + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_, + static_cast(io_type)); + std::string ret = LABSTOR_THALLIUM->SyncIoCall(domain_id.id_, + "RpcPushBulk", + io_type, + tl_task->data_, + tl_task->data_size_, + task->exec_->id_, + task->exec_method_, + task->params_, + tl_task->data_size_, + io_type); + HandlePushReplicaOutput(replica, ret, task); + tl_task->done_ = true; + }, tl_task); + task->tl_future_.emplace_back(tl_task); + +// tl::async_response future = LABSTOR_THALLIUM->AsyncIoCall(domain_id.id_, +// "RpcPushBulk", +// io_type, +// (char *) xfer[0].data_, +// xfer[0].data_size_, +// task->exec_->id_, +// task->exec_method_, +// params, +// xfer[0].data_size_, +// io_type); +// task->tl_future_.emplace_back(std::move(future)); + } + } + + /** Wait for client to finish message */ + void ClientWaitForMessage(PushTask *task) { + for (; task->replica_ < task->tl_future_.size(); ++task->replica_) { +// tl::async_response &future = task->tl_future_[task->replica_]; +// if (!LABSTOR_THALLIUM->IsDone(future)) { +// return; +// } +// std::string ret = LABSTOR_THALLIUM->Wait(future); +// HandlePushReplicaOutput(task->replica_, ret, task); + ThalliumTask *tl_task = (ThalliumTask *) task->tl_future_[task->replica_]; + if (!tl_task->IsDone()) { + return; + } + free(tl_task); + } + HandlePushReplicaEnd(task); + } + + /** Push operation called on client */ + void Push(PushTask *task) { + switch (task->phase_) { + case PushPhase::kStart: { + std::vector &xfer = task->xfer_; + task->tl_future_.reserve(task->domain_ids_.size()); + switch (task->xfer_.size()) { + case 1: { + ClientSmallPush(xfer, task); + break; + } + case 2: { + ClientIoPush(xfer, task); + break; + } + default: { + HELOG(kFatal, "The task {}/{} does not support remote calls", task->task_state_, task->method_); + } + } + task->phase_ = PushPhase::kWait; + } + case PushPhase::kWait: { + ClientWaitForMessage(task); + } + } + } + + private: + /** The RPC for processing a small message */ + void RpcPushSmall(const tl::request &req, + TaskStateId state_id, + u32 method, + std::string ¶ms) { + // Create the input data transfer object + std::vector xfer(1); + xfer[0].data_ = params.data(); + xfer[0].data_size_ = params.size(); + HILOG(kDebug, "Received small message of size {} " + "(task_state={}, method={})", xfer[0].data_size_, state_id, method); + + // Process the message + TaskState *exec; + Task *orig_task; + RpcExec(req, state_id, method, xfer, orig_task, exec); + RpcComplete(req, method, orig_task, exec, state_id); + } + + /** The RPC for processing a message with data */ + void RpcPushBulk(const tl::request &req, + TaskStateId state_id, + u32 method, + std::string ¶ms, + const tl::bulk &bulk, + size_t data_size, + IoType io_type) { + hshm::charbuf data(data_size); + + // Create the input data transfer object + std::vector xfer(2); + xfer[0].data_ = data.data(); + xfer[0].data_size_ = data.size(); + xfer[1].data_ = params.data(); + xfer[1].data_size_ = params.size(); + + HILOG(kDebug, "Received large message of size {} " + "(task_state={}, method={})", xfer[0].data_size_, state_id, method); + + // Process the message + if (io_type == IoType::kWrite) { + LABSTOR_THALLIUM->IoCallServer(req, bulk, io_type, data.data(), data_size); + HILOG(kDebug, "(node {}) Write blob integer: {}", LABSTOR_CLIENT->node_id_, (int)data[0]) + } + TaskState *exec; + Task *orig_task; + RpcExec(req, state_id, method, xfer, orig_task, exec); + if (io_type == IoType::kRead) { + HILOG(kDebug, "(node {}) Read blob integer: {}", LABSTOR_CLIENT->node_id_, (int)data[0]) + LABSTOR_THALLIUM->IoCallServer(req, bulk, io_type, data.data(), data_size); + } + + // Return + RpcComplete(req, method, orig_task, exec, state_id); + } + + /** Push operation called at the remote server */ + void RpcExec(const tl::request &req, + const TaskStateId &state_id, + u32 method, + std::vector &xfer, + Task *&orig_task, TaskState *&exec) { + size_t data_size = xfer[0].data_size_; + BinaryInputArchive ar(xfer); + + // Deserialize task + exec = LABSTOR_TASK_REGISTRY->GetTaskState(state_id); + if (exec == nullptr) { + HELOG(kFatal, "(node {}) Could not find the task state {}", + LABSTOR_CLIENT->node_id_, state_id); + req.respond(std::string()); + return; + } else { + HILOG(kDebug, "(node {}) Found task state {}", + LABSTOR_CLIENT->node_id_, + state_id); + } + TaskPointer task_ptr = exec->LoadStart(method, ar); + orig_task = task_ptr.task_; + hipc::Pointer &p = task_ptr.p_; + orig_task->domain_id_ = DomainId::GetNode(LABSTOR_CLIENT->node_id_); + + // Execute task + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(QueueId(state_id)); + orig_task->UnsetFireAndForget(); + orig_task->UnsetStarted(); + orig_task->UnsetMarked(); + orig_task->UnsetDataOwner(); + queue->Emplace(orig_task->prio_, orig_task->lane_hash_, p); + HILOG(kDebug, + "(node {}) Executing task (task_node={}, task_state={}/{}, state_name={}, method={}, size={}, lane_hash={})", + LABSTOR_CLIENT->node_id_, + orig_task->task_node_, + orig_task->task_state_, + state_id, + exec->name_, + method, + data_size, + orig_task->lane_hash_); + orig_task->Wait<1>(); + } + + void RpcComplete(const tl::request &req, + u32 method, Task *orig_task, + TaskState *exec, TaskStateId state_id) { + BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); + std::vector out_xfer = exec->SaveEnd(method, ar, orig_task); + LABSTOR_CLIENT->DelTask(orig_task); + HILOG(kDebug, "(node {}) Returning {} bytes of data (task_node={}, task_state={}/{}, method={})", + LABSTOR_CLIENT->node_id_, + out_xfer[0].data_size_, + orig_task->task_node_, + orig_task->task_state_, + state_id, + method); + req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); + } + + public: +#include "remote_queue/remote_queue_lib_exec.h" +}; +} // namespace labstor + +LABSTOR_TASK_CC(labstor::remote_queue::Server, "remote_queue"); diff --git a/tasks_required/small_message/CMakeLists.txt b/tasks_required/small_message/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks_required/small_message/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/small_message/include/small_message/small_message.h b/tasks_required/small_message/include/small_message/small_message.h new file mode 100644 index 000000000..6953a076e --- /dev/null +++ b/tasks_required/small_message/include/small_message/small_message.h @@ -0,0 +1,99 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_small_message_H_ +#define LABSTOR_small_message_H_ + +#include "small_message_tasks.h" + +namespace labstor::small_message { + +/** Create admin requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Create a small_message */ + HSHM_ALWAYS_INLINE + void CreateRoot(const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + domain_id, state_name, id_, queue_info); + queue_id_ = QueueId(id_); + } + + /** Destroy state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Metadata task */ + LPointer AsyncMd(const TaskNode &task_node, + const DomainId &domain_id) { + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + auto task = LABSTOR_CLIENT->NewTask( + task_node, domain_id, id_); + queue->Emplace(TaskPrio::kLowLatency, 3, task.shm_); + return task; + } + LABSTOR_TASK_NODE_ROOT(AsyncMd); + int MdRoot(const DomainId &domain_id) { + LPointer task = AsyncMdRoot(domain_id); + task->Wait(); + int ret = task->ret_[0]; + LABSTOR_CLIENT->DelTask(task); + return ret; + } + + /** Metadata PUSH task */ + void AsyncMdPushConstruct(MdPushTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_); + } + int MdPushRoot(const DomainId &domain_id) { + LPointer> push_task = + AsyncMdPushRoot(domain_id); + push_task->Wait(); + MdPushTask *task = push_task->subtask_ptr_; + int ret = task->ret_[0]; + LABSTOR_CLIENT->DelTask(push_task); + return ret; + } + LABSTOR_TASK_NODE_PUSH_ROOT(MdPush); + + /** Io task */ + void AsyncIoConstruct(IoTask *task, const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_); + } + int IoRoot(const DomainId &domain_id) { + LPointer> push_task = AsyncIoRoot(domain_id); + push_task.ptr_->Wait(); + IoTask *task = push_task.ptr_->subtask_ptr_; + int ret = task->ret_; + LABSTOR_CLIENT->DelTask(push_task); + return ret; + } + LABSTOR_TASK_NODE_PUSH_ROOT(Io) +}; + +} // namespace labstor + +#endif // LABSTOR_small_message_H_ diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/tasks_required/small_message/include/small_message/small_message_lib_exec.h new file mode 100644 index 000000000..2f217b7ba --- /dev/null +++ b/tasks_required/small_message/include/small_message/small_message_lib_exec.h @@ -0,0 +1,210 @@ +#ifndef LABSTOR_SMALL_MESSAGE_LIB_EXEC_H_ +#define LABSTOR_SMALL_MESSAGE_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kMd: { + Md(reinterpret_cast(task)); + break; + } + case Method::kIo: { + Io(reinterpret_cast(task)); + break; + } + case Method::kMdPush: { + MdPush(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kMd: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kIo: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kMdPush: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kMd: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kIo: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kMdPush: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kMd: { + ar << *reinterpret_cast(task); + break; + } + case Method::kIo: { + ar << *reinterpret_cast(task); + break; + } + case Method::kMdPush: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kMd: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kIo: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kMdPush: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kMd: { + ar << *reinterpret_cast(task); + break; + } + case Method::kIo: { + ar << *reinterpret_cast(task); + break; + } + case Method::kMdPush: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kMd: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kIo: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kMdPush: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kMd: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kIo: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kMdPush: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_SMALL_MESSAGE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/small_message/include/small_message/small_message_methods.h b/tasks_required/small_message/include/small_message/small_message_methods.h new file mode 100644 index 000000000..fc0e6e35d --- /dev/null +++ b/tasks_required/small_message/include/small_message/small_message_methods.h @@ -0,0 +1,11 @@ +#ifndef LABSTOR_SMALL_MESSAGE_METHODS_H_ +#define LABSTOR_SMALL_MESSAGE_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kMd = kLast + 0; + TASK_METHOD_T kIo = kLast + 1; + TASK_METHOD_T kMdPush = kLast + 2; +}; + +#endif // LABSTOR_SMALL_MESSAGE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/small_message/include/small_message/small_message_methods.yaml b/tasks_required/small_message/include/small_message/small_message_methods.yaml new file mode 100644 index 000000000..ddc39a75e --- /dev/null +++ b/tasks_required/small_message/include/small_message/small_message_methods.yaml @@ -0,0 +1,3 @@ +kMd: 0 +kIo: 1 +kMdPush: 2 \ No newline at end of file diff --git a/tasks_required/small_message/include/small_message/small_message_tasks.h b/tasks_required/small_message/include/small_message/small_message_tasks.h new file mode 100644 index 000000000..2ee7004b4 --- /dev/null +++ b/tasks_required/small_message/include/small_message/small_message_tasks.h @@ -0,0 +1,232 @@ +// +// Created by lukemartinlogan on 8/11/23. +// + +#ifndef LABSTOR_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ +#define LABSTOR_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::small_message { + +#include "small_message_methods.h" +#include "labstor/labstor_namespace.h" + +/** + * A task to create small_message + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "small_message", id, queue_info) { + } +}; + +/** A task to destroy small_message */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} +}; + +/** + * A custom task in small_message + * */ +struct MdTask : public Task, TaskFlags { + OUT hipc::pod_array ret_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + MdTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + MdTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kMd; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + ret_.construct(alloc, 1); + } + + /** Begin replication */ + void ReplicateStart(u32 count) { + ret_.resize(count); + } + + /** Finalize replication */ + void ReplicateEnd() {} + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(ret_[replica]); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in small_message + * */ +struct MdPushTask : public Task, TaskFlags { + OUT hipc::pod_array ret_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + MdPushTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + MdPushTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kMd; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + ret_.construct(alloc, 1); + } + + /** Begin replication */ + void ReplicateStart(u32 count) { + ret_.resize(count); + } + + /** Finalize replication */ + void ReplicateEnd() {} + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(ret_[replica]); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in small_message + * */ +struct IoTask : public Task, TaskFlags { + static inline int const DATA_SIZE = 4096; + IN char data_[DATA_SIZE]; + OUT int ret_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + IoTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + IoTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 3; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kIo; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Custom params + memset(data_, 10, DATA_SIZE); + } + + /** (De)serialize message call */ + template + void SaveStart(Ar &ar) { + DataTransfer xfer(DT_RECEIVER_READ, data_, DATA_SIZE, domain_id_); + task_serialize(ar); + ar & xfer; + } + + /** Deserialize message call */ + template + void LoadStart(Ar &ar) { + DataTransfer xfer; + task_serialize(ar); + ar & xfer; + memcpy(data_, xfer.data_, xfer.data_size_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(ret_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor + +#endif // LABSTOR_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ diff --git a/tasks_required/small_message/src/CMakeLists.txt b/tasks_required/small_message/src/CMakeLists.txt new file mode 100644 index 000000000..8bff4c3f7 --- /dev/null +++ b/tasks_required/small_message/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(small_message SHARED + small_message.cc) +add_dependencies(small_message ${Labstor_RUNTIME_DEPS}) +target_link_libraries(small_message ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + small_message + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + small_message + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(small_message) +endif() diff --git a/tasks_required/small_message/src/small_message.cc b/tasks_required/small_message/src/small_message.cc new file mode 100644 index 000000000..25eec506d --- /dev/null +++ b/tasks_required/small_message/src/small_message.cc @@ -0,0 +1,51 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "small_message/small_message.h" + +namespace labstor::small_message { + +class Server : public TaskLib { + public: + int count_ = 0; + + public: + void Construct(ConstructTask *task) { + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + void Md(MdTask *task) { + task->ret_[0] = 1; + task->SetModuleComplete(); + } + + void MdPush(MdPushTask *task) { + task->ret_[0] = 1; + task->SetModuleComplete(); + } + + void Io(IoTask *task) { + task->ret_ = 1; + for (int i = 0; i < 256; ++i) { + if (task->data_[i] != 10) { + task->ret_ = 0; + break; + } + } + task->SetModuleComplete(); + } + + public: +#include "small_message/small_message_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(labstor::small_message::Server, "small_message"); diff --git a/tasks_required/worch_proc_round_robin/CMakeLists.txt b/tasks_required/worch_proc_round_robin/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks_required/worch_proc_round_robin/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h new file mode 100644 index 000000000..0ffcffd08 --- /dev/null +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h @@ -0,0 +1,43 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_worch_proc_round_robin_H_ +#define LABSTOR_worch_proc_round_robin_H_ + +#include "worch_proc_round_robin_tasks.h" + +namespace labstor::worch_proc_round_robin { + +/** Create admin requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Create a worch_proc_round_robin */ + HSHM_ALWAYS_INLINE + void CreateRoot(const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + std::vector queue_info = { + {1, 1, 4, 0}, + }; + id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + domain_id, state_name, id_, queue_info); + } + + /** Destroy state */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } +}; + +} // namespace labstor + +#endif // LABSTOR_worch_proc_round_robin_H_ diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h new file mode 100644 index 000000000..10565c416 --- /dev/null +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h @@ -0,0 +1,146 @@ +#ifndef LABSTOR_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ +#define LABSTOR_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + Schedule(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSchedule: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kSchedule: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSchedule: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kSchedule: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h new file mode 100644 index 000000000..382707b85 --- /dev/null +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ +#define LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kSchedule = kLast + 0; +}; + +#endif // LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml new file mode 100644 index 000000000..49b54822b --- /dev/null +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml @@ -0,0 +1 @@ +kSchedule: 0 \ No newline at end of file diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h new file mode 100644 index 000000000..d0926a34b --- /dev/null +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h @@ -0,0 +1,74 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_WORCH_PROC_ROUND_ROBIN_TASKS_H__ +#define LABSTOR_WORCH_PROC_ROUND_ROBIN_TASKS_H__ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor/work_orchestrator/scheduler.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::worch_proc_round_robin { + +#include "labstor/labstor_namespace.h" + +/** The set of methods in the worch task */ +typedef SchedulerMethod Method; + +/** + * A task to create worch_proc_round_robin + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "worch_proc_round_robin", id, queue_info) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy worch_proc_round_robin */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor::worch_proc_round_robin + +#endif // LABSTOR_WORCH_PROC_ROUND_ROBIN_TASKS_H__ diff --git a/tasks_required/worch_proc_round_robin/src/CMakeLists.txt b/tasks_required/worch_proc_round_robin/src/CMakeLists.txt new file mode 100644 index 000000000..8188f116d --- /dev/null +++ b/tasks_required/worch_proc_round_robin/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(worch_proc_round_robin SHARED + worch_proc_round_robin.cc) +add_dependencies(worch_proc_round_robin ${Labstor_RUNTIME_DEPS}) +target_link_libraries(worch_proc_round_robin ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + worch_proc_round_robin + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + worch_proc_round_robin + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(worch_proc_round_robin) +endif() diff --git a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc new file mode 100644 index 000000000..c51a0eba5 --- /dev/null +++ b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc @@ -0,0 +1,34 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "worch_proc_round_robin/worch_proc_round_robin.h" + +namespace labstor::worch_proc_round_robin { + +class Server : public TaskLib { + public: + void Construct(ConstructTask *task) { + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + void Schedule(ScheduleTask *task) { + int rr = 0; + for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { + worker.SetCpuAffinity(rr % HERMES_SYSTEM_INFO->ncpu_); + ++rr; + } + } + +#include "worch_proc_round_robin/worch_proc_round_robin_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(labstor::worch_proc_round_robin::Server, "worch_proc_round_robin"); diff --git a/tasks_required/worch_queue_round_robin/CMakeLists.txt b/tasks_required/worch_queue_round_robin/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks_required/worch_queue_round_robin/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h new file mode 100644 index 000000000..1b4796079 --- /dev/null +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h @@ -0,0 +1,43 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_worch_queue_round_robin_H_ +#define LABSTOR_worch_queue_round_robin_H_ + +#include "worch_queue_round_robin_tasks.h" + +namespace labstor::worch_queue_round_robin { + +/** Create admin requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Create a worch_queue_round_robin */ + HSHM_ALWAYS_INLINE + void CreateRoot(const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + std::vector queue_info = { + {1, 1, 4, 0}, + }; + id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + domain_id, state_name, id_, queue_info); + } + + /** Destroy task state */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } +}; + +} // namespace labstor + +#endif // LABSTOR_worch_queue_round_robin_H_ diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h new file mode 100644 index 000000000..1321d5488 --- /dev/null +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h @@ -0,0 +1,146 @@ +#ifndef LABSTOR_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ +#define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + Schedule(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSchedule: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kDestruct: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + case Method::kSchedule: { + task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); + ar >> *reinterpret_cast(task_ptr.task_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kSchedule: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kSchedule: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h new file mode 100644 index 000000000..87ae85fc8 --- /dev/null +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ +#define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kSchedule = kLast + 0; +}; + +#endif // LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml new file mode 100644 index 000000000..49b54822b --- /dev/null +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml @@ -0,0 +1 @@ +kSchedule: 0 \ No newline at end of file diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h new file mode 100644 index 000000000..a5acbbc01 --- /dev/null +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h @@ -0,0 +1,78 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef LABSTOR_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ +#define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/work_orchestrator/scheduler.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::worch_queue_round_robin { + +#include "labstor/labstor_namespace.h" + +/** The set of methods in the worch task */ +typedef SchedulerMethod Method; + +/** + * A task to create worch_queue_round_robin + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "worch_queue_round_robin", id, queue_info) { + } + + /** Destructor */ + HSHM_ALWAYS_INLINE + ~ConstructTask() {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** A task to destroy worch_queue_round_robin */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + TaskStateId &state_id, + const DomainId &domain_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor::worch_queue_round_robin + +#endif // LABSTOR_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ diff --git a/tasks_required/worch_queue_round_robin/src/CMakeLists.txt b/tasks_required/worch_queue_round_robin/src/CMakeLists.txt new file mode 100644 index 000000000..ec0fcbd6a --- /dev/null +++ b/tasks_required/worch_queue_round_robin/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(worch_queue_round_robin SHARED + worch_queue_round_robin.cc) +add_dependencies(worch_queue_round_robin ${Labstor_RUNTIME_DEPS}) +target_link_libraries(worch_queue_round_robin ${Labstor_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + worch_queue_round_robin + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + DESTINATION + ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(LABSTOR_EXPORTED_LIBS + worch_queue_round_robin + ${LABSTOR_EXPORTED_LIBS}) +if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${LABSTOR_EXPORTED_LIBS} + FILE + ${LABSTOR_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(worch_queue_round_robin) +endif() diff --git a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc new file mode 100644 index 000000000..1ea856dcb --- /dev/null +++ b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc @@ -0,0 +1,59 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "worch_queue_round_robin/worch_queue_round_robin.h" + +namespace labstor::worch_queue_round_robin { + +class Server : public TaskLib { + public: + u32 count_; + + public: + void Construct(ConstructTask *task) { + count_ = 0; + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task) { + task->SetModuleComplete(); + } + + void Schedule(ScheduleTask *task) { + // Check if any new queues need to be scheduled + for (MultiQueue &queue : *LABSTOR_QM_RUNTIME->queue_map_) { + if (queue.id_.IsNull()) { + continue; + } + for (LaneGroup &lane_group : *queue.groups_) { + // NOTE(llogan): Assumes a minimum of two workers, admin on worker 0. + if (lane_group.IsLowPriority()) { + for (u32 lane_id = lane_group.num_scheduled_; lane_id < lane_group.num_lanes_; ++lane_id) { + HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); + Worker &worker = LABSTOR_WORK_ORCHESTRATOR->workers_[0]; + worker.PollQueues({WorkEntry(lane_group.prio_, lane_id, &queue)}); + } + lane_group.num_scheduled_ = lane_group.num_lanes_; + } else { + for (u32 lane_id = lane_group.num_scheduled_; lane_id < lane_group.num_lanes_; ++lane_id) { + HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); + u32 worker_id = (count_ % (LABSTOR_WORK_ORCHESTRATOR->workers_.size() - 1)) + 1; + Worker &worker = LABSTOR_WORK_ORCHESTRATOR->workers_[worker_id]; + worker.PollQueues({WorkEntry(lane_group.prio_, lane_id, &queue)}); + count_ += 1; + } + lane_group.num_scheduled_ = lane_group.num_lanes_; + } + } + } + } + +#include "worch_queue_round_robin/worch_queue_round_robin_lib_exec.h" +}; + +} // namespace labstor + +LABSTOR_TASK_CC(labstor::worch_queue_round_robin::Server, "worch_queue_round_robin"); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 507adf32c..670a02ac7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,63 +1,4 @@ -#------------------------------------------------------------------------------ -# Include source and build directories -#------------------------------------------------------------------------------ -include_directories( - ${PROJECT_SOURCE_DIR} - ${PROJECT_SOURCE_DIR}/src/api - ${PROJECT_SOURCE_DIR}/gotcha_intercept - ${PROJECT_SOURCE_DIR}/io_client - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_SOURCE_DIR} - ${HERMES_INCLUDES_BUILD_TIME} -) +cmake_minimum_required(VERSION 3.10) +project(labstor) -#------------------------------------------------------------------------------ -# Single-node API tests -#------------------------------------------------------------------------------ -set(API_TESTS - test_bucket - test_buffer_pool - test_trait - test_tag - test_binlog) - -find_program(BASH_PROGRAM bash) - -add_executable(test_config test_config.cc main.cc) -add_dependencies(test_config hermes) -target_link_libraries(test_config ${LIBRT} - hermes - MPI::MPI_CXX - $<$:thallium> - Catch2::Catch2) - -foreach(program ${API_TESTS}) - add_executable(${program} ${program}.cc main_mpi.cc) - add_dependencies(${program} hermes hermes_example_trait) - target_link_libraries(${program} ${LIBRT} - hermes hermes_example_trait - MPI::MPI_CXX - $<$:thallium> - Catch2::Catch2) - pytest(native ${program}) -endforeach() - -#------------------------------------------------------------------------------ -# Multi-node API tests -#------------------------------------------------------------------------------ -set(API_TESTS - test_multinode_put_get) - -find_program(BASH_PROGRAM bash) - -foreach(program ${API_TESTS}) - add_executable(${program} ${program}.cc main_mpi.cc) - add_dependencies(${program} hermes) - target_link_libraries(${program} ${LIBRT} - hermes - MPI::MPI_CXX - $<$:thallium> - Catch2::Catch2) - set(script ${CMAKE_SOURCE_DIR}/test/mpi.sh) - pytest(native ${program}) -endforeach() \ No newline at end of file +add_subdirectory(unit) \ No newline at end of file diff --git a/test/data/apriori_schema.yaml b/test/data/apriori_schema.yaml deleted file mode 100644 index ab909b8b0..000000000 --- a/test/data/apriori_schema.yaml +++ /dev/null @@ -1,21 +0,0 @@ -0: - - op_count_range: [5, 5] - prefetch: - - bucket: /tmp/test_hermes/hi.txt - promote_blobs: [ 3, 4 ] - demote_blobs: [ 1, 2 ] - - op_count_range: [7, 7] - prefetch: - - bucket: /tmp/test_hermes/hi.txt - promote_blobs: [ 5, 6 ] - demote_blobs: [ 3, 4 ] - - op_count_range: [9, 9] - prefetch: - - bucket: /tmp/test_hermes/hi.txt - promote_blobs: [ 5, 6 ] - demote_blobs: [ 3, 4 ] - - op_count_range: [11, 11] - prefetch: - - bucket: /tmp/test_hermes/hi.txt - promote_blobs: [ 7, 8 ] - demote_blobs: [ 5, 6 ] diff --git a/test/data/asan.supp b/test/data/asan.supp deleted file mode 100644 index 5ad2683cd..000000000 --- a/test/data/asan.supp +++ /dev/null @@ -1,10 +0,0 @@ -# Ignore leaks from external libraries -leak:libfabric.so -leak:libabt.so -leak:libmargo.so -leak:libmpi.so -leak:librdmacm.so -leak:libhwloc.so -leak:libmpich.so -leak:aiori-HDF5.c -leak:ior.c diff --git a/test/data/hermes_client.yaml b/test/data/hermes_client.yaml deleted file mode 100644 index e95d71736..000000000 --- a/test/data/hermes_client.yaml +++ /dev/null @@ -1,9 +0,0 @@ -stop_daemon: false -path_inclusions: ["/tmp/test_hermes"] -path_exclusions: ["/"] -file_page_size: 1024KB -base_adapter_mode: kDefault -file_adapter_configs: - - path: "/" - page_size: 1MB - mode: kDefault \ No newline at end of file diff --git a/test/data/hermes_client_specific.yaml b/test/data/hermes_client_specific.yaml deleted file mode 100644 index e95d71736..000000000 --- a/test/data/hermes_client_specific.yaml +++ /dev/null @@ -1,9 +0,0 @@ -stop_daemon: false -path_inclusions: ["/tmp/test_hermes"] -path_exclusions: ["/"] -file_page_size: 1024KB -base_adapter_mode: kDefault -file_adapter_configs: - - path: "/" - page_size: 1MB - mode: kDefault \ No newline at end of file diff --git a/test/data/hermes_server.yaml b/test/data/hermes_server.yaml deleted file mode 100644 index 348ce1e30..000000000 --- a/test/data/hermes_server.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# Example Hermes configuration file - -### Define properties of the storage devices -devices: - # The name of the device. - # It can be whatever the user wants, there are no special names - ram: - # The mount point of each device. RAM should be the empty string. For block - # devices, this is the directory where Hermes will create buffering files. For - # object storage or cloud targets, this will be a url. - mount_point: "" - - # The maximum buffering capacity in MiB of each device. Here we say that all 4 - # devices get 50 MiB of buffering capacity. - capacity: 1000MB - - # The size of the smallest available buffer in KiB. In general this should be - # the page size of your system for byte addressable storage, and the block size - # of the storage device for block addressable storage. - block_size: 4KB - - # The number of blocks (the size of which is chosen in block_sizes_kb) that each - # device should contain for each slab (controlled by num_slabs). This allows for - # precise control of the distibution of buffer sizes. - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - - # The maximum theoretical bandwidth (as advertised by the manufacturer) in - # Possible units: KBps, MBps, GBps - bandwidth: 6000MBps - - # The latency of each device (as advertised by the manufacturer). - # Possible units: ns, us, ms, s - latency: 15us - - # For each device, indicate '1' if it is shared among nodes (e.g., burst - # buffers), or '0' if it is per node (e.g., local NVMe). - is_shared_device: false - - # For each device, the minimum and maximum percent capacity threshold at which - # the BufferOrganizer will trigger. Decreasing the maximum thresholds will cause - # the BufferOrganizer to move data to lower devices, making more room in faster - # devices (ideal for write-heavy workloads). Conversely, increasing the minimum - # threshold will cause data to be moved from slower devices into faster devices - # (ideal for read-heavy workloads). For example, a maximum capacity threshold of - # 0.8 would have the effect of always keeping 20% of the device's space free for - # incoming writes. Conversely, a minimum capacity threshold of 0.3 would ensure - # that the device is always at least 30% occupied. - borg_capacity_thresh: [0.0, 1.0] - - nvme: - mount_point: "./" - capacity: 50MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 1GBps - latency: 600us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - ssd: - mount_point: "./" - capacity: 50MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 500MBps - latency: 1200us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - pfs: - mount_point: "./" - capacity: 16GB - block_size: 64KB # The stripe size of PFS - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 100MBps # Per-device bandwidth - latency: 200ms - is_shared_device: true - borg_capacity_thresh: [ 0.0, 1.0 ] - -### Define properties of RPCs -rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "" - - # Host names are constructed as "base_name + - # host_number + rpc_server_suffix." Each entry in the rpc_host_number_range_list - # can be either a single number, or a range, which is 2 numbers separated by a - # hyphen (-). For example the list {01, 03-05, 07, 08-10} will be expanded to - # {01, 03, 04, 05, 07, 08, 09, 10}. - host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. - protocol: "ofi+sockets" - - # RPC domain name for verbs transport. Blank for tcp. - domain: "" - - # Desired RPC port number. - port: 8080 - - # The number of handler threads for each RPC server. - num_threads: 4 - -### Define properties of the BORG -buffer_organizer: - # The number of threads used in the background organization of internal Hermes buffers. - num_threads: 1 - - # Desired RPC port number for buffer organizer. - port: 8081 - -### Define the default data placement policy -dpe: - # Choose Random, RoundRobin, or MinimizeIoTime - default_placement_policy: "MinimizeIoTime" - - # If true (1) the RoundRobin placement policy algorithm will split each Blob - # into a random number of smaller Blobs. - default_rr_split: 0 - -# The shared memory prefix for the hermes shared memory segment. A user name -# will be automatically appended. -shmem_name: "/hermes_shm_" - -# The interval in milliseconds at which to update the global system view. -system_view_state_update_interval_ms: 1000 \ No newline at end of file diff --git a/test/data/hermes_server_ares.yaml b/test/data/hermes_server_ares.yaml deleted file mode 100644 index 393d8adf3..000000000 --- a/test/data/hermes_server_ares.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# Example Hermes configuration file - -### Define properties of the storage devices -devices: - # The name of the device. - # It can be whatever the user wants, there are no special names - ram: - # The mount point of each device. RAM should be the empty string. For block - # devices, this is the directory where Hermes will create buffering files. For - # object storage or cloud targets, this will be a url. - mount_point: "" - - # The maximum buffering capacity in MiB of each device. Here we say that all 4 - # devices get 50 MiB of buffering capacity. - capacity: 5000MB - - # The size of the smallest available buffer in KiB. In general this should be - # the page size of your system for byte addressable storage, and the block size - # of the storage device for block addressable storage. - block_size: 4KB - - # The number of blocks (the size of which is chosen in block_sizes_kb) that each - # device should contain for each slab (controlled by num_slabs). This allows for - # precise control of the distibution of buffer sizes. - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - - # The maximum theoretical bandwidth (as advertised by the manufacturer) in - # Possible units: KBps, MBps, GBps - bandwidth: 6000MBps - - # The latency of each device (as advertised by the manufacturer). - # Possible units: ns, us, ms, s - latency: 15us - - # For each device, indicate '1' if it is shared among nodes (e.g., burst - # buffers), or '0' if it is per node (e.g., local NVMe). - is_shared_device: false - - # For each device, the minimum and maximum percent capacity threshold at which - # the BufferOrganizer will trigger. Decreasing the maximum thresholds will cause - # the BufferOrganizer to move data to lower devices, making more room in faster - # devices (ideal for write-heavy workloads). Conversely, increasing the minimum - # threshold will cause data to be moved from slower devices into faster devices - # (ideal for read-heavy workloads). For example, a maximum capacity threshold of - # 0.8 would have the effect of always keeping 20% of the device's space free for - # incoming writes. Conversely, a minimum capacity threshold of 0.3 would ensure - # that the device is always at least 30% occupied. - borg_capacity_thresh: [0.0, 1.0] - - nvme: - mount_point: "./" - capacity: 50MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 1GBps - latency: 600us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - ssd: - mount_point: "./" - capacity: 50MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 500MBps - latency: 1200us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - pfs: - mount_point: "./" - capacity: inf - block_size: 64KB # The stripe size of PFS - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 100MBps # Per-device bandwidth - latency: 200ms - is_shared_device: true - borg_capacity_thresh: [ 0.0, 1.0 ] - -### Define properties of RPCs -rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "${HOME}/hostfile.txt" - - # Host names are constructed as "base_name + - # host_number + rpc_server_suffix." Each entry in the rpc_host_number_range_list - # can be either a single number, or a range, which is 2 numbers separated by a - # hyphen (-). For example the list {01, 03-05, 07, 08-10} will be expanded to - # {01, 03, 04, 05, 07, 08, 09, 10}. - host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. - protocol: "ofi+sockets" - - # RPC domain name for verbs transport. Blank for tcp. - domain: "" - - # Desired RPC port number. - port: 8080 - - # The number of handler threads for each RPC server. - num_threads: 4 - -### Define properties of the BORG -buffer_organizer: - # The number of threads used in the background organization of internal Hermes buffers. - num_threads: 1 - - # Desired RPC port number for buffer organizer. - port: 8081 - -### Define the default data placement policy -dpe: - # Choose Random, RoundRobin, or MinimizeIoTime - default_placement_policy: "MinimizeIoTime" - - # If true (1) the RoundRobin placement policy algorithm will split each Blob - # into a random number of smaller Blobs. - default_rr_split: 0 - -# The shared memory prefix for the hermes shared memory segment. A user name -# will be automatically appended. -shmem_name: "/hermes_shm_" - -# The interval in milliseconds at which to update the global system view. -system_view_state_update_interval_ms: 1000 \ No newline at end of file diff --git a/test/data/hermes_server_bpm.yaml b/test/data/hermes_server_bpm.yaml deleted file mode 100644 index 092a1a0dd..000000000 --- a/test/data/hermes_server_bpm.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# Example Hermes configuration file - -### Define properties of the storage devices -devices: - # The name of the device. - # It can be whatever the user wants, there are no special names - ram: - # The mount point of each device. RAM should be the empty string. For block - # devices, this is the directory where Hermes will create buffering files. For - # object storage or cloud targets, this will be a url. - mount_point: "" - - # The maximum buffering capacity in MiB of each device. Here we say that all 4 - # devices get 50 MiB of buffering capacity. - capacity: 50MB - - # The size of the smallest available buffer in KiB. In general this should be - # the page size of your system for byte addressable storage, and the block size - # of the storage device for block addressable storage. - block_size: 4KB - - # The number of blocks (the size of which is chosen in block_sizes_kb) that each - # device should contain for each slab (controlled by num_slabs). This allows for - # precise control of the distibution of buffer sizes. - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - - # The maximum theoretical bandwidth (as advertised by the manufacturer) in - # Possible units: KBps, MBps, GBps - bandwidth: 6000MBps - - # The latency of each device (as advertised by the manufacturer). - # Possible units: ns, us, ms, s - latency: 15us - - # For each device, indicate '1' if it is shared among nodes (e.g., burst - # buffers), or '0' if it is per node (e.g., local NVMe). - is_shared_device: false - - # For each device, the minimum and maximum percent capacity threshold at which - # the BufferOrganizer will trigger. Decreasing the maximum thresholds will cause - # the BufferOrganizer to move data to lower devices, making more room in faster - # devices (ideal for write-heavy workloads). Conversely, increasing the minimum - # threshold will cause data to be moved from slower devices into faster devices - # (ideal for read-heavy workloads). For example, a maximum capacity threshold of - # 0.8 would have the effect of always keeping 20% of the device's space free for - # incoming writes. Conversely, a minimum capacity threshold of 0.3 would ensure - # that the device is always at least 30% occupied. - borg_capacity_thresh: [0.0, 1.0] - - nvme: - mount_point: "./" - capacity: 50MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 1GBps - latency: 600us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - ssd: - mount_point: "./" - capacity: 50MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 500MBps - latency: 1200us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - pfs: - mount_point: "./" - capacity: 16GB - block_size: 64KB # The stripe size of PFS - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 100MBps # Per-device bandwidth - latency: 200ms - is_shared_device: true - borg_capacity_thresh: [ 0.0, 1.0 ] - -### Define properties of RPCs -rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "" - - # Host names are constructed as "base_name + - # host_number + rpc_server_suffix." Each entry in the rpc_host_number_range_list - # can be either a single number, or a range, which is 2 numbers separated by a - # hyphen (-). For example the list {01, 03-05, 07, 08-10} will be expanded to - # {01, 03, 04, 05, 07, 08, 09, 10}. - host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. - protocol: "ofi+sockets" - - # RPC domain name for verbs transport. Blank for tcp. - domain: "" - - # Desired RPC port number. - port: 8080 - - # The number of handler threads for each RPC server. - num_threads: 4 - -### Define properties of the BORG -buffer_organizer: - # The number of threads used in the background organization of internal Hermes buffers. - num_threads: 1 - - # Desired RPC port number for buffer organizer. - port: 8081 - -### Define the default data placement policy -dpe: - # Choose Random, RoundRobin, or MinimizeIoTime - default_placement_policy: "MinimizeIoTime" - - # If true (1) the RoundRobin placement policy algorithm will split each Blob - # into a random number of smaller Blobs. - default_rr_split: 0 - -# The shared memory prefix for the hermes shared memory segment. A user name -# will be automatically appended. -shmem_name: "/hermes_shm_" - -# The interval in milliseconds at which to update the global system view. -system_view_state_update_interval_ms: 1000 \ No newline at end of file diff --git a/test/data/hermes_server_prefetch.yaml b/test/data/hermes_server_prefetch.yaml deleted file mode 100644 index 24306b43d..000000000 --- a/test/data/hermes_server_prefetch.yaml +++ /dev/null @@ -1,138 +0,0 @@ -# Example Hermes configuration file - -### Define properties of the storage devices -devices: - # The name of the device. - # It can be whatever the user wants, there are no special names - ram: - # The mount point of each device. RAM should be the empty string. For block - # devices, this is the directory where Hermes will create buffering files. For - # object storage or cloud targets, this will be a url. - mount_point: "" - - # The maximum buffering capacity in MiB of each device. Here we say that all 4 - # devices get 50 MiB of buffering capacity. - capacity: 6000MB - - # The size of the smallest available buffer in KiB. In general this should be - # the page size of your system for byte addressable storage, and the block size - # of the storage device for block addressable storage. - block_size: 4KB - - # The number of blocks (the size of which is chosen in block_sizes_kb) that each - # device should contain for each slab (controlled by num_slabs). This allows for - # precise control of the distibution of buffer sizes. - slab_sizes: [ 4KB, 16KB, 64KB, 1MB, 16MB ] - - # The maximum theoretical bandwidth (as advertised by the manufacturer) in - # Possible units: KBps, MBps, GBps - bandwidth: 6000MBps - - # The latency of each device (as advertised by the manufacturer). - # Possible units: ns, us, ms, s - latency: 15us - - # For each device, indicate '1' if it is shared among nodes (e.g., burst - # buffers), or '0' if it is per node (e.g., local NVMe). - is_shared_device: false - - # For each device, the minimum and maximum percent capacity threshold at which - # the BufferOrganizer will trigger. Decreasing the maximum thresholds will cause - # the BufferOrganizer to move data to lower devices, making more room in faster - # devices (ideal for write-heavy workloads). Conversely, increasing the minimum - # threshold will cause data to be moved from slower devices into faster devices - # (ideal for read-heavy workloads). For example, a maximum capacity threshold of - # 0.8 would have the effect of always keeping 20% of the device's space free for - # incoming writes. Conversely, a minimum capacity threshold of 0.3 would ensure - # that the device is always at least 30% occupied. - borg_capacity_thresh: [0.0, 1.0] - - nvme: - mount_point: "./" - capacity: 10GB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 1GBps - latency: 600us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - ssd: - mount_point: "./" - capacity: 10GB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 500MBps - latency: 1200us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - pfs: - mount_point: "./" - capacity: inf - block_size: 64KB # The stripe size of PFS - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 100MBps # Per-device bandwidth - latency: 200ms - is_shared_device: true - borg_capacity_thresh: [ 0.0, 1.0 ] - -### Define properties of RPCs -rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "" - - # Host names are constructed as "base_name + - # host_number + rpc_server_suffix." Each entry in the rpc_host_number_range_list - # can be either a single number, or a range, which is 2 numbers separated by a - # hyphen (-). For example the list {01, 03-05, 07, 08-10} will be expanded to - # {01, 03, 04, 05, 07, 08, 09, 10}. - host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. - protocol: "ofi+sockets" - - # RPC domain name for verbs transport. Blank for tcp. - domain: "" - - # Desired RPC port number. - port: 8080 - - # The number of handler threads for each RPC server. - num_threads: 4 - -### Define properties of the BORG -buffer_organizer: - # The number of threads used in the background organization of internal Hermes buffers. - num_threads: 1 - - # Desired RPC port number for buffer organizer. - port: 8081 - -### Define the default data placement policy -dpe: - # Choose Random, RoundRobin, or MinimizeIoTime - default_placement_policy: "MinimizeIoTime" - - # If true (1) the RoundRobin placement policy algorithm will split each Blob - # into a random number of smaller Blobs. - default_rr_split: 0 - -# The shared memory prefix for the hermes shared memory segment. A user name -# will be automatically appended. -shmem_name: "/hermes_shm_" - -# The interval in milliseconds at which to update the global system view. -system_view_state_update_interval_ms: 1000 - -### Define prefetcher properties -prefetch: - enabled: true - io_trace_path: "${HOME}/test/io_trace.yaml" - apriori_schema_path: "${HOME}/apriori_schema.yaml" - epoch_ms: 50 - is_mpi: true \ No newline at end of file diff --git a/test/hostfile.txt b/test/hostfile.txt deleted file mode 100644 index 6c2d49ee5..000000000 --- a/test/hostfile.txt +++ /dev/null @@ -1,4 +0,0 @@ -ares-comp-01 -ares-comp-02 - -ares-comp-[03-05,08-10] diff --git a/test/test_binlog.cc b/test/test_binlog.cc deleted file mode 100644 index 9fbe9daa6..000000000 --- a/test/test_binlog.cc +++ /dev/null @@ -1,89 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include - -#include "binlog.h" -#include "hermes_types.h" -#include "basic_test.h" -#include - -namespace hapi = hermes::api; -namespace stdfs = std::filesystem; - -void MainPretest() { -} - -void MainPosttest() { -} - -std::vector create_stats( - size_t bytes, int num_ranks, size_t &entries_per_rank) { - std::vector stats; - size_t max_entries = bytes / sizeof(hermes::IoStat); - entries_per_rank = max_entries / num_ranks; - stats.reserve(num_ranks * entries_per_rank); - for (int rank = 0; rank < num_ranks; ++rank) { - for (size_t i = 0; i < entries_per_rank; ++i) { - stats.emplace_back(); - auto &stat = stats.back(); - stat.rank_ = rank; - stat.type_ = (i % 2) ? hermes::IoType::kRead : hermes::IoType::kWrite; - stat.blob_id_ = hermes::BlobId(i, rank); - stat.blob_size_ = 8 * (i + 1); - } - } - return stats; -} - -void verify_log(hermes::BinaryLog log, - int num_ranks, size_t entries_per_rank) { - for (int rank = 0; rank < num_ranks; ++rank) { - hermes::IoStat stat; - size_t i = 0; - while (log.GetEntry(rank, i, stat)) { - REQUIRE(stat.blob_id_.node_id_ == rank); - REQUIRE(stat.blob_id_.unique_ == i); - i += 1; - } - REQUIRE(i == entries_per_rank); - } -} - -TEST_CASE("TestBinlog") { - int num_ranks = 16; - size_t log_bytes = MEGABYTES(1); - size_t chunk_bytes = log_bytes / 4; - size_t entries_per_rank; - std::string path = "/tmp/log.bin"; - - // Create chunk - std::vector stats = create_stats( - chunk_bytes, num_ranks, entries_per_rank); - - // Attempt flushing the log - hermes::BinaryLog log; - log.Init(path, log_bytes); - log.Ingest(stats); - verify_log(log, num_ranks, entries_per_rank); - log.Flush(); - REQUIRE(stdfs::file_size(path) == 0); - - // Actually flush the log when capacity reached - log.Ingest(stats); - log.Ingest(stats); - log.Ingest(stats); - log.Ingest(stats); - log.Flush(); - REQUIRE(stdfs::file_size(path) > 0); -} diff --git a/test/test_bucket.cc b/test/test_bucket.cc deleted file mode 100644 index 5325f8b1c..000000000 --- a/test/test_bucket.cc +++ /dev/null @@ -1,212 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include - -#include -#include "hermes_shm/util/logging.h" -#include "hermes.h" -#include "bucket.h" - -#include "basic_test.h" - -namespace hapi = hermes::api; - -void MainPretest() { - hapi::Hermes::Create(hermes::HermesType::kClient); -} - -void MainPosttest() { - HERMES->Clear(); - HERMES->Finalize(); -} - -void TestBlobCreates(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - size_t num_blobs = 256; - size_t blob_size = KILOBYTES(4); - hermes::api::Context ctx; - hermes::BlobId blob_id; - - for (size_t i = 0; i < num_blobs; ++i) { - hermes::Blob blob(blob_size); - std::string name = std::to_string(i); - char nonce = i % 256; - memset(blob.data(), nonce, blob_size); - bkt.Put(name, blob, blob_id, ctx); - } - - for (size_t i = 0; i < num_blobs; ++i) { - std::string name = std::to_string(i); - char nonce = i % 256; - hermes::Blob blob; - bkt.GetBlobId(name, blob_id); - REQUIRE(!blob_id.IsNull()); - bkt.Get(blob_id, blob, ctx); - REQUIRE(blob.size() == blob_size); - REQUIRE(VerifyBuffer(blob.data(), blob_size, nonce)); - } -} - -void TestBlobOverride(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - auto bkt2 = hermes->GetBucket("hello"); - hermes::api::Context ctx; - hermes::BlobId blob_id; - hermes::Blob blob(1024); - for (size_t i = 0; i < 1024; ++i) { - memset(blob.data(), i, 1024); - bkt2.Put("0", blob, blob_id, ctx); - hermes::Blob ret; - bkt.Get(blob_id, ret, ctx); - REQUIRE(ret.size() == 1024); - REQUIRE(VerifyBuffer(ret.data(), 1024, i)); - } -} - -void TestBucketRename(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - bkt.Rename("hello2"); - auto bkt1 = hermes->GetBucket("hello"); - auto bkt2 = hermes->GetBucket("hello2"); - - REQUIRE(!bkt2.IsNull()); - REQUIRE(bkt1.GetId() != bkt2.GetId()); - REQUIRE(bkt.GetId() == bkt2.GetId()); -} - -void TestBucketClear(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - size_t num_blobs = 16; - size_t blob_size = MEGABYTES(32); - hermes::api::Context ctx; - hermes::BlobId blob_id; - - for (size_t i = 0; i < num_blobs; ++i) { - hermes::Blob blob(blob_size); - std::string name = std::to_string(i); - char nonce = i % 256; - memset(blob.data(), nonce, blob_size); - bkt.Put(name, std::move(blob), blob_id, ctx); - } - - bkt.Clear(); - - REQUIRE(bkt.GetSize() == 0); - auto blobs = bkt.GetContainedBlobIds(); - REQUIRE(blobs.size() == 0); -} - -void TestBucketDestroy(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - size_t num_blobs = 1; - size_t blob_size = MEGABYTES(32); - hermes::api::Context ctx; - hermes::BlobId blob_id; - - for (size_t i = 0; i < num_blobs; ++i) { - hermes::Blob blob(blob_size); - std::string name = std::to_string(i); - char nonce = i % 256; - memset(blob.data(), nonce, blob_size); - bkt.Put(name, blob, blob_id, ctx); - } - - bkt.Destroy(); - - // Check if the bucket can be re-obtained - auto new_bkt = hermes->GetBucket("hello"); - REQUIRE(bkt.GetId() != new_bkt.GetId()); - REQUIRE(new_bkt.GetSize() == 0); -} - -void TestBlobRename(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - hapi::Blob blob(1024); - hermes::BlobId blob_id; - hapi::Context ctx; - bkt.Put("0", blob, blob_id, ctx); - bkt.RenameBlob(blob_id, "1", ctx); - - { - hermes::BlobId blob_get_id; - bkt.GetBlobId("0", blob_get_id); - REQUIRE(blob_get_id.IsNull()); - } - - { - hermes::BlobId blob_get_id; - bkt.GetBlobId("1", blob_get_id); - REQUIRE(!blob_get_id.IsNull()); - REQUIRE(blob_get_id == blob_id); - } -} - -void TestBlobDestroy(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - hapi::Blob blob(1024); - hermes::BlobId blob_id; - hapi::Context ctx; - bkt.Put("0", blob, blob_id, ctx); - bkt.DestroyBlob(blob_id, ctx); - { - hermes::BlobId blob_id_get; - bkt.GetBlobId("0", blob_id_get); - REQUIRE(blob_id_get.IsNull()); - } -} - -TEST_CASE("TestCreateBlobName") { - HERMES->Clear(); - hermes::TagId bkt_id(1, 1); - std::string blob_name("0"); - hipc::uptr n1 = - hermes::MetadataManager::CreateBlobName(bkt_id, blob_name); - hipc::uptr n2 = - hermes::MetadataManager::CreateBlobName(bkt_id, blob_name); - REQUIRE(*n1 == *n2); -} - -TEST_CASE("TestBlobCreates") { - HERMES->Clear(); - TestBlobCreates(HERMES); -} - -TEST_CASE("TestBlobOverride") { - TestBlobOverride(HERMES); -} - -TEST_CASE("TestBucketRename") { - HERMES->Clear(); - TestBucketRename(HERMES); -} - -TEST_CASE("TestBucketClear") { - HERMES->Clear(); - TestBucketClear(HERMES); -} - -TEST_CASE("TestBucketDestroy") { - HERMES->Clear(); - TestBucketDestroy(HERMES); -} - -TEST_CASE("TestBlobRename") { - HERMES->Clear(); - TestBlobRename(HERMES); -} - -TEST_CASE("TestBlobDestroy") { - HERMES->Clear(); - TestBlobDestroy(HERMES); -} diff --git a/test/test_buffer_pool.cc b/test/test_buffer_pool.cc deleted file mode 100644 index db0f4caa8..000000000 --- a/test/test_buffer_pool.cc +++ /dev/null @@ -1,87 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include - -#include -#include "hermes_shm/util/logging.h" -#include "hermes.h" -#include "buffer_pool.h" - -#include "basic_test.h" - -namespace hapi = hermes::api; - -void MainPretest() { - hapi::Hermes::Create(hermes::HermesType::kClient); -} - -void MainPosttest() { - HERMES->Finalize(); -} - -TEST_CASE("TestBufferPool") { - std::vector> sizes = { - {23, 1}, // 23B blob on 1 target - {KILOBYTES(1), 1}, // 1KB blob on 1 target - {KILOBYTES(4), 1}, // 4KB blob on 1 target - {KILOBYTES(8), 1}, // 8KB blob on 1 target - {KILOBYTES(16), 1}, // 16KB blob on 1 target - {KILOBYTES(64), 1}, // 64KB blob on 1 target - {MEGABYTES(4), 1}, // 4MB blob on 1 target - - {KILOBYTES(4), 3}, // 4KB blob on 3 targets - {KILOBYTES(8), 3}, // 8KB blob on 3 targets - {KILOBYTES(16), 3}, // 16KB blob on 3 targets - {KILOBYTES(64), 3}, // 64KB blob on 3 targets - {MEGABYTES(4), 3}, // 4MB blob on 3 targets - {MEGABYTES(4) + KILOBYTES(32), 3}, // 4MB blob on 3 targets - - // This will test the ability to "trickle down" - {MEGABYTES(256), 1}, // 256MB blob on 1 target - }; - - for (auto &[size, num_targets] : sizes) { - // Divide the blob among targets - hermes::PlacementSchema schema; - size_t tgt_size = size / num_targets; - size_t last_size = tgt_size + (size % num_targets); - for (int i = 0; i < num_targets; ++i) { - hermes::TargetInfo &target = (*HERMES->mdm_.targets_)[i]; - if (i < num_targets - 1) { - schema.plcmnts_.emplace_back(tgt_size, target.id_); - } else { - schema.plcmnts_.emplace_back(last_size, target.id_); - } - } - - // Create the blob to write - hermes::Blob write_blob(size); - memset(write_blob.data(), 10, size); - - // Allocate the buffers and set them - std::vector buffers = - HERMES->bpm_.LocalAllocateAndSetBuffers(schema, write_blob); - - // Read back the buffers - hermes::Blob read_blob(size); - HERMES->borg_.LocalReadBlobFromBuffers(read_blob, buffers); - - // Verify they are the same - REQUIRE(read_blob.size() == size); - REQUIRE(VerifyBuffer(read_blob.data(), size, 10)); - - // Release buffers - HERMES->bpm_.LocalReleaseBuffers(buffers); - } -} diff --git a/test/test_config.cc b/test/test_config.cc deleted file mode 100644 index 27aabdf85..000000000 --- a/test/test_config.cc +++ /dev/null @@ -1,91 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes.h" -#include "basic_test.h" - -void MainPretest() { -} - -void MainPosttest() { -} - -TEST_CASE("Hostfile") { - auto vec = hermes::RpcContext::ParseHostfile("hostfile.txt"); - REQUIRE(vec.size() == 8); - REQUIRE(vec[0] == "ares-comp-01"); - REQUIRE(vec[1] == "ares-comp-02"); - REQUIRE(vec[2] == "ares-comp-03"); - REQUIRE(vec[3] == "ares-comp-04"); - REQUIRE(vec[4] == "ares-comp-05"); - REQUIRE(vec[5] == "ares-comp-08"); - REQUIRE(vec[6] == "ares-comp-09"); - REQUIRE(vec[7] == "ares-comp-10"); -} - -TEST_CASE("HostName") { - SECTION("Simple host name") { - std::vector host_names; - hshm::ConfigParse::ParseHostNameString("localhost", host_names); - REQUIRE(host_names.size() == 1); - REQUIRE(host_names[0] == "localhost"); - } - - SECTION("Host name with range at the end") { - std::vector host_names; - hshm::ConfigParse::ParseHostNameString("ares-comp-[0-4]", - host_names); - REQUIRE(host_names.size() == 5); - REQUIRE(host_names[0] == "ares-comp-0"); - REQUIRE(host_names[1] == "ares-comp-1"); - REQUIRE(host_names[2] == "ares-comp-2"); - REQUIRE(host_names[3] == "ares-comp-3"); - REQUIRE(host_names[4] == "ares-comp-4"); - } - - SECTION("Host name with fixed number range") { - std::vector host_names; - hshm::ConfigParse::ParseHostNameString("ares-comp-[08-12]", - host_names); - REQUIRE(host_names.size() == 5); - REQUIRE(host_names[0] == "ares-comp-08"); - REQUIRE(host_names[1] == "ares-comp-09"); - REQUIRE(host_names[2] == "ares-comp-10"); - REQUIRE(host_names[3] == "ares-comp-11"); - REQUIRE(host_names[4] == "ares-comp-12"); - } - - SECTION("Host name with fixed number range and suffix") { - std::vector host_names; - hshm::ConfigParse::ParseHostNameString("ares-comp-[08-12]-hello", - host_names); - REQUIRE(host_names.size() == 5); - REQUIRE(host_names[0] == "ares-comp-08-hello"); - REQUIRE(host_names[1] == "ares-comp-09-hello"); - REQUIRE(host_names[2] == "ares-comp-10-hello"); - REQUIRE(host_names[3] == "ares-comp-11-hello"); - REQUIRE(host_names[4] == "ares-comp-12-hello"); - } - - SECTION("Host name with multiple ranges") { - std::vector host_names; - hshm::ConfigParse::ParseHostNameString( - "ares-comp-[08-10,13-14,25]-hello", host_names); - REQUIRE(host_names.size() == 6); - REQUIRE(host_names[0] == "ares-comp-08-hello"); - REQUIRE(host_names[1] == "ares-comp-09-hello"); - REQUIRE(host_names[2] == "ares-comp-10-hello"); - REQUIRE(host_names[3] == "ares-comp-13-hello"); - REQUIRE(host_names[4] == "ares-comp-14-hello"); - REQUIRE(host_names[5] == "ares-comp-25-hello"); - } -} diff --git a/test/test_multinode_put_get.cc b/test/test_multinode_put_get.cc deleted file mode 100644 index 7f7a88358..000000000 --- a/test/test_multinode_put_get.cc +++ /dev/null @@ -1,75 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include -#include -#include "hermes.h" -#include "hermes_shm/util/formatter.h" -#include "bucket.h" -#include "basic_test.h" - -static const int kBlobSize = KILOBYTES(64); -static const int kBlobsPerRank = 1024; - -void MainPretest() { - hapi::Hermes::Create(hermes::HermesType::kClient); -} - -void MainPosttest() { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Barrier(MPI_COMM_WORLD); - if (rank == 0) { - HERMES->Finalize(); - } -} - -// Put to my bucket -void PutTest(int rank) { - std::string bkt_name = hshm::Formatter::format("bucket{}", rank); - auto bkt = HERMES->GetBucket(bkt_name); - hermes::api::Context ctx; - hermes::BlobId blob_id; - hermes::Blob blob(kBlobSize); - for (size_t i = 0; i < kBlobsPerRank; ++i) { - std::string name = std::to_string(i); - memset(blob.data(), i, kBlobSize); - bkt.Put(name, blob, blob_id, ctx); - } -} - -// Get from a different rank -void GetTest(int rank, int nprocs) { - std::string bkt_name = hshm::Formatter::format( - "bucket{}", (rank + 1) % nprocs); - auto bkt = HERMES->GetBucket(bkt_name); - hermes::api::Context ctx; - hermes::BlobId blob_id; - for (size_t i = 0; i < kBlobsPerRank; ++i) { - std::string name = std::to_string(i); - hermes::Blob ret; - bkt.GetBlobId(name, blob_id); - bkt.Get(blob_id, ret, ctx); - REQUIRE(VerifyBuffer(ret.data(), kBlobSize, i)); - } -} - -TEST_CASE("ParallelPutGetDifferentBuckets") { - int rank, nprocs; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Barrier(MPI_COMM_WORLD); - PutTest(rank); - MPI_Barrier(MPI_COMM_WORLD); - GetTest(rank, nprocs); -} diff --git a/test/test_tag.cc b/test/test_tag.cc deleted file mode 100644 index 48b2964c5..000000000 --- a/test/test_tag.cc +++ /dev/null @@ -1,66 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include - -#include -#include "hermes_shm/util/logging.h" -#include "hermes.h" -#include "bucket.h" - -#include "basic_test.h" - -namespace hapi = hermes::api; - -using hermes::TagId; - -void MainPretest() { - hapi::Hermes::Create(hermes::HermesType::kClient); -} - -void MainPosttest() { - HERMES->Finalize(); -} - -void TestTag(hapi::Hermes *hermes) { - auto bkt = hermes->GetBucket("hello"); - size_t num_blobs = 100; - size_t blob_size = KILOBYTES(4); - hermes::api::Context ctx; - std::vector blob_ids(num_blobs); - - // Create some blobs - for (size_t i = 0; i < num_blobs; ++i) { - hermes::Blob blob(blob_size); - std::string name = std::to_string(i); - char nonce = i % 256; - memset(blob.data(), nonce, blob_size); - bkt.Put(name, std::move(blob), blob_ids[i], ctx); - } - - // Create two tags - TagId tag1 = HERMES->CreateTag("tag1"); - - // Tag some blobs - for (size_t i = 0; i < num_blobs; ++i) { - bkt.TagBlob(blob_ids[i], tag1); - } - - // Query by tag - auto tags = HERMES->GroupBy(tag1); - REQUIRE(tags.size() == 100); -} - -TEST_CASE("TestTag") { - TestTag(HERMES); -} diff --git a/test/test_trait.cc b/test/test_trait.cc deleted file mode 100644 index f23c052ee..000000000 --- a/test/test_trait.cc +++ /dev/null @@ -1,60 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include - -#include -#include "hermes_shm/util/logging.h" -#include "hermes.h" -#include "bucket.h" -#include "traits/example/example_trait.h" - -#include "basic_test.h" - -namespace hapi = hermes::api; - -using hermes::TagId; - -void MainPretest() { - hapi::Hermes::Create(hermes::HermesType::kClient); -} - -void MainPosttest() { - HERMES->Finalize(); -} - -void TestTrait() { - auto bkt = HERMES->GetBucket("hello"); - size_t num_blobs = 100; - hermes::api::Context ctx; - std::vector blob_ids(num_blobs); - - // Create a trait - HERMES->RegisterTrait("ex", 5); - - // Get the trait id - hermes::TraitId trait_id = HERMES->GetTraitId("ex"); - REQUIRE(!trait_id.IsNull()); - - // Get the trait itself - auto *trait = HERMES->GetTrait(trait_id); - hapi::ExampleTraitParams params; - trait->Run(0, ¶ms); - auto hello = params.hello_; - REQUIRE(hello == 5); -} - -TEST_CASE("TestTrait") { - TestTrait(); -} - diff --git a/test/test_utils.h b/test/test_utils.h deleted file mode 100644 index eaaa83ea9..000000000 --- a/test/test_utils.h +++ /dev/null @@ -1,43 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TEST_TEST_UTILS_H_ -#define HERMES_TEST_TEST_UTILS_H_ - -#include -#include - -static inline bool VerifyBuffer(char *ptr, size_t size, char nonce) { - for (size_t i = 0; i < size; ++i) { - if (ptr[i] != nonce) { - std::cout << "Mismatch at: " << (int)i << std::endl; - return false; - } - } - return true; -} - -static inline bool CompareBuffers(char *p1, size_t s1, char *p2, size_t s2, - size_t off) { - if (s1 != s2) { - return false; - } - for (size_t i = off; i < s1; ++i) { - if (p1[i] != p2[i]) { - std::cout << "Mismatch at: " << (int)i << std::endl; - return false; - } - } - return true; -} - -#endif // HERMES_TEST_TEST_UTILS_H_ diff --git a/test/tests.py b/test/tests.py deleted file mode 100644 index 4c7921189..000000000 --- a/test/tests.py +++ /dev/null @@ -1,60 +0,0 @@ -from py_hermes_ci.test_manager import TestManager -from jarvis_util import * - - -class NativeTestManager(TestManager): - def spawn_all_nodes(self): - return self.spawn_info() - - def set_paths(self): - self.TEST_BUCKET_CMD = f"{self.CMAKE_BINARY_DIR}/bin/test_bucket" - self.TEST_BUFFER_POOL_CMD = f"{self.CMAKE_BINARY_DIR}/bin/test_buffer_pool" - self.TEST_TRAIT_CMD = f"{self.CMAKE_BINARY_DIR}/bin/test_trait" - self.TEST_TAG_CMD = f"{self.CMAKE_BINARY_DIR}/bin/test_tag" - self.TEST_BINLOG_CMD = f"{self.CMAKE_BINARY_DIR}/bin/test_binlog" - self.TEST_MULTINODE_PUT_GET_CMD = f"{self.CMAKE_BINARY_DIR}/bin/test_multinode_put_get" - - def test_bucket(self): - spawn_info = self.spawn_info(nprocs=1, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(self.TEST_BUCKET_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_buffer_pool(self): - spawn_info = self.spawn_info(nprocs=1, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(self.TEST_BUFFER_POOL_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_trait(self): - spawn_info = self.spawn_info(nprocs=1, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(self.TEST_TRAIT_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_tag(self): - spawn_info = self.spawn_info(nprocs=1, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(self.TEST_TAG_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_binlog(self): - node = Exec(self.TEST_BINLOG_CMD) - - def test_multinode_put_get(self): - spawn_info = self.spawn_info(nprocs=2, - ppn=1, - hostfile=None, # TODO(llogan) - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(self.TEST_MULTINODE_PUT_GET_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt new file mode 100644 index 000000000..b8b63529d --- /dev/null +++ b/test/unit/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.10) +project(labstor) + +set(CMAKE_CXX_STANDARD 17) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_SOURCE_DIR}/tasks/labstor_admin/include) +add_subdirectory(ipc) +add_subdirectory(hermes) +add_subdirectory(hermes_adapters) \ No newline at end of file diff --git a/test/basic_test.h b/test/unit/basic_test.h similarity index 57% rename from test/basic_test.h rename to test/unit/basic_test.h index 14ee8d480..278bf6d85 100644 --- a/test/basic_test.h +++ b/test/unit/basic_test.h @@ -10,17 +10,45 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef HERMES_SHM_TEST_UNIT_BASIC_TEST_H_ -#define HERMES_SHM_TEST_UNIT_BASIC_TEST_H_ +#ifndef LABSTOR_TEST_UNIT_BASIC_TEST_H_ +#define LABSTOR_TEST_UNIT_BASIC_TEST_H_ #define CATCH_CONFIG_RUNNER #include -#include "test_utils.h" namespace cl = Catch::Clara; cl::Parser define_options(); +#include +#include + +static inline bool VerifyBuffer(char *ptr, size_t size, char nonce) { + for (size_t i = 0; i < size; ++i) { + if (ptr[i] != nonce) { + std::cout << (int)ptr[i] << std::endl; + return false; + } + } + return true; +} + +static inline bool CompareBuffers(char *p1, size_t s1, char *p2, size_t s2, + size_t off) { + if (s1 != s2) { + return false; + } + for (size_t i = off; i < s1; ++i) { + if (p1[i] != p2[i]) { + std::cout << "Mismatch at: " << (int)i << std::endl; + return false; + } + } + return true; +} + void MainPretest(); void MainPosttest(); -#endif // HERMES_SHM_TEST_UNIT_BASIC_TEST_H_ +#define PAGE_DIVIDE(TEXT) + +#endif // LABSTOR_TEST_UNIT_BASIC_TEST_H_ diff --git a/hermes_shm/test/unit/thread/CMakeLists.txt b/test/unit/hermes/CMakeLists.txt similarity index 52% rename from hermes_shm/test/unit/thread/CMakeLists.txt rename to test/unit/hermes/CMakeLists.txt index 69a28e6eb..7230f31f6 100644 --- a/hermes_shm/test/unit/thread/CMakeLists.txt +++ b/test/unit/hermes/CMakeLists.txt @@ -1,44 +1,44 @@ cmake_minimum_required(VERSION 3.10) -project(hermes_shm) +project(labstor) set(CMAKE_CXX_STANDARD 17) #------------------------------------------------------------------------------ # Build Tests #------------------------------------------------------------------------------ -add_executable(test_thread_exec - ${TEST_MAIN}/main.cc + +add_executable(test_hermes_exec + ${TEST_MAIN}/main_mpi.cc test_init.cc - test_thread.cc - test_lock.cc) -add_dependencies(test_thread_exec hermes_shm_data_structures) -target_link_libraries(test_thread_exec - hermes_shm_data_structures - thallium - Catch2::Catch2 - MPI::MPI_CXX - OpenMP::OpenMP_CXX) + test_bucket.cc +) +add_dependencies(test_hermes_exec + ${Labstor_CLIENT_DEPS} hermes) +target_link_libraries(test_hermes_exec + ${Labstor_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) #------------------------------------------------------------------------------ # Test Cases #------------------------------------------------------------------------------ -add_test(NAME test_thread COMMAND - ${CMAKE_BINARY_DIR}/bin/test_thread_exec) + +# STRING TESTS +add_test(NAME test_ipc COMMAND + ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") #------------------------------------------------------------------------------ # Install Targets #------------------------------------------------------------------------------ install(TARGETS - test_thread_exec + test_hermes_exec EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) #----------------------------------------------------------------------------- # Coverage #----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(test_thread_exec) -endif() \ No newline at end of file +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(test_messages) +endif() diff --git a/test/data/hermes_server_ci.yaml b/test/unit/hermes/config/hermes_server.yaml similarity index 71% rename from test/data/hermes_server_ci.yaml rename to test/unit/hermes/config/hermes_server.yaml index 7183dbd97..0dd5b841c 100644 --- a/test/data/hermes_server_ci.yaml +++ b/test/unit/hermes/config/hermes_server.yaml @@ -10,9 +10,8 @@ devices: # object storage or cloud targets, this will be a url. mount_point: "" - # The maximum buffering capacity in MiB of each device. Here we say that all 4 - # devices get 50 MiB of buffering capacity. - capacity: 100MB + # The maximum buffering capacity in MiB of each device. + capacity: 4GB # The size of the smallest available buffer in KiB. In general this should be # the page size of your system for byte addressable storage, and the block size @@ -69,7 +68,7 @@ devices: pfs: mount_point: "./" - capacity: 4GB + capacity: 100MB block_size: 64KB # The stripe size of PFS slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] bandwidth: 100MBps # Per-device bandwidth @@ -77,20 +76,24 @@ devices: is_shared_device: true borg_capacity_thresh: [ 0.0, 1.0 ] +# Define the maximum amount of memory Hermes can use for non-buffering tasks. +# This includes metadata management and memory allocations. +# This memory will not be preallocated, so if you don't know, 0 indicates +# any amount of memory +max_memory: 0g + ### Define properties of RPCs rpc: # A path to a file containing a list of server names, 1 per line. If your # servers are named according to a pattern (e.g., server-1, server-2, etc.), # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "hermes_hosts" + host_file: "" - # Host names are constructed as "base_name + - # host_number + rpc_server_suffix." Each entry in the rpc_host_number_range_list - # can be either a single number, or a range, which is 2 numbers separated by a - # hyphen (-). For example the list {01, 03-05, 07, 08-10} will be expanded to - # {01, 03, 04, 05, 07, 08, 09, 10}. - host_names: [] + # Host names can be defined using the following syntax: + # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... + # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... + host_names: ["localhost"] # The RPC protocol. This must come from the documentation of the specific RPC # library in use. @@ -110,8 +113,23 @@ buffer_organizer: # The number of threads used in the background organization of internal Hermes buffers. num_threads: 1 - # Desired RPC port number for buffer organizer. - port: 8081 + # Interval (ms) where blobs are checked for flushing + flush_period: 1024 + + # Interval (ms) where blobs are checked for re-organization + blob_reorg_period: 1024 + + ## What does "recently accessed" mean? + # Time when score is equal to 1 (seconds) + recency_min: 0 + # Time when score is equal to 0 (seconds) + recency_max: 60 + + ## What does "frequently accessed" mean? + # Number of accesses for score to be equal to 1 (count) + freq_max: 15 + # Number of accesses for score to be equal to 0 (count) + freq_min: 0 ### Define the default data placement policy dpe: @@ -122,16 +140,38 @@ dpe: # into a random number of smaller Blobs. default_rr_split: 0 -# The shared memory prefix for the hermes shared memory segment. A user name +### Define I/O tracing properties +tracing: + enabled: false + output: "" + +### Define prefetcher properties +prefetch: + enabled: false + io_trace_path: "" + apriori_schema_path: "" + epoch_ms: 50 + is_mpi: false + +### Define mdm properties +mdm: + # This represents the number of blobs and buckets before collisions start + # to happen in the unordered_map tables. + est_blob_count: 100000 + est_bucket_count: 100000 + est_num_traits: 256 + +# The shared memory prefix for the hermes shared memory segment. A username # will be automatically appended. shmem_name: "/hermes_shm_" # The interval in milliseconds at which to update the global system view. system_view_state_update_interval_ms: 1000 -### Define prefetcher properties -prefetch: - enabled: true - io_trace_path: "${HOME}/test/io_trace.yaml" - epoch_ms: 50 - is_mpi: true \ No newline at end of file +### Define the names of the traits to search LD_LIBRARY_PATH for +traits: + - "hermes_posix_io_client" + - "hermes_stdio_io_client" + - "hermes_mpiio_io_client" + - "hermes_example_trait" + - "hermes_prefetcher_trait" diff --git a/test/unit/hermes/config/labstor_server.yaml b/test/unit/hermes/config/labstor_server.yaml new file mode 100644 index 000000000..e68550377 --- /dev/null +++ b/test/unit/hermes/config/labstor_server.yaml @@ -0,0 +1,54 @@ +### Runtime orchestration settings +work_orchestrator: + # The number of worker threads to spawn + max_workers: 2 + +### Queue Manager settings +queue_manager: + # The default depth of allocated queues + queue_depth: 8192 + # The maximum number of lanes per queue + max_lanes: 4 + # The maximum number of queues + max_queues: 1024 + # The shared memory allocator to use + shm_allocator: kScalablePageAllocator + # The name of the shared memory region to create + shm_name: "labstor_shm" + # The size of the shared memory region to allocate + shm_size: 0g + +### Define properties of RPCs +rpc: + # A path to a file containing a list of server names, 1 per line. If your + # servers are named according to a pattern (e.g., server-1, server-2, etc.), + # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this + # option is not empty, it will override anything in `rpc_server_base_name`. + host_file: "" + + # Host names can be defined using the following syntax: + # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... + # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... + host_names: ["localhost"] + + # The RPC protocol. This must come from the documentation of the specific RPC + # library in use. + protocol: "sockets" + + # RPC domain name for verbs transport. Blank for tcp. + domain: "" + + # Desired RPC port number. + port: 9192 + + # The number of handler threads for each RPC server. + num_threads: 4 + +### Task Registry +task_registry: [ + 'hermes_mdm', + 'hermes_blob_mdm', + 'hermes_bucket_mdm', + 'posix_bdev', + 'ram_bdev' +] \ No newline at end of file diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc new file mode 100644 index 000000000..e9d131e96 --- /dev/null +++ b/test/unit/hermes/test_bucket.cc @@ -0,0 +1,326 @@ +// +// Created by llogan on 7/1/23. +// + +#include "basic_test.h" +#include "labstor/api/labstor_client.h" +#include "labstor_admin/labstor_admin.h" +#include "hermes/hermes.h" +#include "hermes/bucket.h" +#include + +TEST_CASE("TestHermesPut1n") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + if (rank == 0) { + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("hello"); + + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t max_blobs = 4; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i % max_blobs), blob, ctx); + + // Get a blob + HILOG(kInfo, "Put {} returned successfully", i); + size_t size = bkt.GetBlobSize(blob_id); + REQUIRE(blob.size() == size); + } + } + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_CASE("TestHermesPut") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("hello"); + + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + + // Get a blob + HILOG(kInfo, "Put {} returned successfully", i); + size_t size = bkt.GetBlobSize(blob_id); + REQUIRE(blob.size() == size); + } + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_CASE("TestHermesPutGet") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + HILOG(kInfo, "WE ARE HERE!!!") + hermes::Context ctx; + hermes::Bucket bkt("hello"); + HILOG(kInfo, "BUCKET LOADED!!!") + + size_t count_per_proc = 4; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (int rep = 0; rep < 4; ++rep) { + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {} with blob name {}", i, std::to_string(i)); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + + // Get a blob + hermes::Blob blob2; + bkt.Get(blob_id, blob2, ctx); + REQUIRE(blob.size() == blob2.size()); + REQUIRE(blob == blob2); + } + } +} + +TEST_CASE("TestHermesPartialPutGet") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + HILOG(kInfo, "WE ARE HERE!!!") + hermes::Context ctx; + hermes::Bucket bkt("hello"); + HILOG(kInfo, "BUCKET LOADED!!!") + + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + size_t half_blob = KILOBYTES(4); + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Make left and right blob + hermes::Blob lblob(half_blob); + memset(lblob.data(), i % 256, lblob.size()); + hermes::Blob rblob(half_blob); + memset(rblob.data(), (i + 1) % 256, rblob.size()); + + // PartialPut a blob + hermes::BlobId lblob_id = bkt.PartialPut(std::to_string(i), lblob, 0, ctx); + hermes::BlobId rblob_id = bkt.PartialPut(std::to_string(i), rblob, half_blob, ctx); + REQUIRE(lblob_id == rblob_id); + + // PartialGet a blob + hermes::Blob lblob2(half_blob); + hermes::Blob rblob2(half_blob); + bkt.PartialGet(lblob_id, lblob2, 0, ctx); + bkt.PartialGet(rblob_id, rblob2, half_blob, ctx); + REQUIRE(lblob2.size() == half_blob); + REQUIRE(rblob2.size() == half_blob); + REQUIRE(lblob == lblob2); + REQUIRE(rblob == rblob2); + REQUIRE(lblob2 != rblob2); + } +} + +TEST_CASE("TestHermesBlobDestroy") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("hello"); + + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + bkt.DestroyBlob(blob_id, ctx); + REQUIRE(!bkt.ContainsBlob(std::to_string(i))); + } +} + +TEST_CASE("TestHermesBucketDestroy") { + // TODO(llogan): need to inform bucket when a blob has been placed in it + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("hello"); + + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + bkt.Put(std::to_string(i), blob, ctx); + } + + bkt.Destroy(); + MPI_Barrier(MPI_COMM_WORLD); + + for (size_t i = off; i < proc_count; ++i) { + REQUIRE(!bkt.ContainsBlob(std::to_string(i))); + } +} + +TEST_CASE("TestHermesReorganizeBlob") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("hello"); + + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + bkt.ReorganizeBlob(blob_id, .5, 0, ctx); + hermes::Blob blob2; + bkt.Get(blob_id, blob2, ctx); + REQUIRE(blob.size() == blob2.size()); + REQUIRE(blob == blob2); + } + + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "ContainsBlob Iteration: {}", i); + REQUIRE(bkt.ContainsBlob(std::to_string(i))); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_CASE("TestHermesBucketAppend") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("append_test"); + + // Put a few blobs in the bucket + size_t page_size = KILOBYTES(4); + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + bkt.Append(blob, page_size, ctx); + } + MPI_Barrier(MPI_COMM_WORLD); + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "ContainsBlob Iteration: {}", i); + REQUIRE(bkt.ContainsBlob(std::to_string(i))); + HILOG(kInfo, "ContainsBlob Iteration: {} SUCCESS", i); + } + // REQUIRE(bkt.GetSize() == count_per_proc * nprocs * page_size); + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_CASE("TestHermesBucketAppend1n") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + if (rank == 0) { + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("append_test"); + + // Put a few blobs in the bucket + size_t page_size = KILOBYTES(4); + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + bkt.Append(blob, page_size, ctx); + hermes::Blob blob2; + bkt.Get(std::to_string(i), blob2, ctx); + REQUIRE(blob.size() == blob2.size()); + REQUIRE(blob == blob2); + } + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "ContainsBlob Iteration: {}", i); + REQUIRE(bkt.ContainsBlob(std::to_string(i))); + } + } + MPI_Barrier(MPI_COMM_WORLD); +} \ No newline at end of file diff --git a/hermes_shm/benchmark/allocator/test_init.cc b/test/unit/hermes/test_init.cc similarity index 89% rename from hermes_shm/benchmark/allocator/test_init.cc rename to test/unit/hermes/test_init.cc index 06c925a4e..63132b16a 100644 --- a/hermes_shm/benchmark/allocator/test_init.cc +++ b/test/unit/hermes/test_init.cc @@ -10,11 +10,13 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "labstor/api/labstor_client.h" +#include "basic_test.h" #include "test_init.h" -#include "hermes_shm/data_structures/ipc/vector.h" -#include "hermes_shm/memory/allocator/stack_allocator.h" void MainPretest() { + TRANSPARENT_LABSTOR(); } void MainPosttest() { diff --git a/hermes_shm/test/unit/thread/test_init.cc b/test/unit/hermes/test_init.h similarity index 83% rename from hermes_shm/test/unit/thread/test_init.cc rename to test/unit/hermes/test_init.h index 82bac123b..a6c71f3ec 100644 --- a/hermes_shm/test/unit/thread/test_init.cc +++ b/test/unit/hermes/test_init.h @@ -11,8 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "basic_test.h" +#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ -void MainPretest() {} +#include "labstor/labstor_types.h" -void MainPosttest() {} +#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/adapter/test/CMakeLists.txt b/test/unit/hermes_adapters/CMakeLists.txt similarity index 94% rename from adapter/test/CMakeLists.txt rename to test/unit/hermes_adapters/CMakeLists.txt index db34230ac..72962ce60 100644 --- a/adapter/test/CMakeLists.txt +++ b/test/unit/hermes_adapters/CMakeLists.txt @@ -2,6 +2,8 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_PRELOAD -DHERMES_RPC_THALLIUM") set(ADAPTER_COMMON ${CMAKE_CURRENT_SOURCE_DIR}/catch_config.h) set(HERMES_ADAPTER_TEST_DIR ${HERMES_ADAPTER_DIR}/test) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + find_package(Catch2 REQUIRED) find_program(BASH_PROGRAM bash) find_package(OpenMP REQUIRED COMPONENTS C CXX) diff --git a/adapter/test/adapter_test_utils.h b/test/unit/hermes_adapters/adapter_test_utils.h similarity index 98% rename from adapter/test/adapter_test_utils.h rename to test/unit/hermes_adapters/adapter_test_utils.h index 3b668c052..bfdfe9f7d 100644 --- a/adapter/test/adapter_test_utils.h +++ b/test/unit/hermes_adapters/adapter_test_utils.h @@ -15,7 +15,7 @@ #include #include #include -#include +#include bool FilesystemSupportsTmpfile() { bool result = false; diff --git a/adapter/test/catch_config.h b/test/unit/hermes_adapters/catch_config.h similarity index 100% rename from adapter/test/catch_config.h rename to test/unit/hermes_adapters/catch_config.h diff --git a/adapter/test/mpiio/CMakeLists.txt b/test/unit/hermes_adapters/mpiio/CMakeLists.txt similarity index 92% rename from adapter/test/mpiio/CMakeLists.txt rename to test/unit/hermes_adapters/mpiio/CMakeLists.txt index 10e6c2fb6..7fafaf3f0 100644 --- a/adapter/test/mpiio/CMakeLists.txt +++ b/test/unit/hermes_adapters/mpiio/CMakeLists.txt @@ -3,7 +3,7 @@ add_executable(mpiio_adapter_test mpiio_adapter_test.cpp ${ADAPTER_COMMON}) add_executable(hermes_mpiio_adapter_test mpiio_adapter_test.cpp ${ADAPTER_COMMON}) target_link_libraries(hermes_mpiio_adapter_test hermes_mpiio) -add_dependencies(hermes_mpiio_adapter_test hermes_mpiio hermes_daemon) +add_dependencies(hermes_mpiio_adapter_test hermes_mpiio) set_target_properties(hermes_mpiio_adapter_test PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") pytest(mpiio test_hermes_mpiio_basic_sync) pytest(mpiio test_hermes_mpiio_basic_async) @@ -32,7 +32,7 @@ if(HERMES_INSTALL_TESTS) endif() add_executable(mpi_parallel parallel.cc) -add_dependencies(mpi_parallel hermes_mpiio hermes_daemon) +add_dependencies(mpi_parallel hermes_mpiio) target_link_libraries(mpi_parallel hermes_mpiio Catch2::Catch2 -lstdc++fs -lc MPI::MPI_CXX) set_target_properties(mpi_parallel PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") diff --git a/adapter/test/mpiio/mpiio_adapter_basic_test.cpp b/test/unit/hermes_adapters/mpiio/mpiio_adapter_basic_test.cpp similarity index 100% rename from adapter/test/mpiio/mpiio_adapter_basic_test.cpp rename to test/unit/hermes_adapters/mpiio/mpiio_adapter_basic_test.cpp diff --git a/adapter/test/mpiio/mpiio_adapter_test.cpp b/test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp similarity index 97% rename from adapter/test/mpiio/mpiio_adapter_test.cpp rename to test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp index addcee72d..0e0402a6b 100644 --- a/adapter/test/mpiio/mpiio_adapter_test.cpp +++ b/test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp @@ -63,7 +63,7 @@ hermes::adapter::mpiio::test::Info info; int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); info.write_data = GenRandom(args.request_size); @@ -155,11 +155,11 @@ int pretest() { MPI_Barrier(MPI_COMM_WORLD); REQUIRE(info.total_size > 0); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.shared_new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.shared_existing_file_cmp, false); #endif return 0; @@ -173,8 +173,8 @@ void Clear() { int posttest(bool compare_data = true) { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); #endif if (compare_data && stdfs::exists(info.new_file) && stdfs::exists(info.new_file_cmp)) { @@ -243,10 +243,10 @@ int posttest(bool compare_data = true) { Clear(); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); #endif return 0; } diff --git a/adapter/test/mpiio/parallel.cc b/test/unit/hermes_adapters/mpiio/parallel.cc similarity index 100% rename from adapter/test/mpiio/parallel.cc rename to test/unit/hermes_adapters/mpiio/parallel.cc diff --git a/adapter/test/mpiio/tests.py b/test/unit/hermes_adapters/mpiio/tests.py similarity index 100% rename from adapter/test/mpiio/tests.py rename to test/unit/hermes_adapters/mpiio/tests.py diff --git a/adapter/test/posix/CMakeLists.txt b/test/unit/hermes_adapters/posix/CMakeLists.txt similarity index 88% rename from adapter/test/posix/CMakeLists.txt rename to test/unit/hermes_adapters/posix/CMakeLists.txt index e24b5d9b6..3f6775206 100644 --- a/adapter/test/posix/CMakeLists.txt +++ b/test/unit/hermes_adapters/posix/CMakeLists.txt @@ -15,14 +15,14 @@ pytest(posix test_posix_basic_mpi) add_executable(hermes_posix_adapter_test posix_adapter_test.cpp ${ADAPTER_COMMON}) target_link_libraries(hermes_posix_adapter_test hermes_posix) -add_dependencies(hermes_posix_adapter_test hermes_posix hermes_daemon) +add_dependencies(hermes_posix_adapter_test hermes_posix) set_target_properties(hermes_posix_adapter_test PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") pytest(posix test_hermes_posix_basic_small) pytest(posix test_hermes_posix_basic_large) add_executable(hermes_posix_adapter_mpi_test posix_adapter_mpi_test.cpp ${ADAPTER_COMMON}) target_link_libraries(hermes_posix_adapter_mpi_test hermes_posix) -add_dependencies(hermes_posix_adapter_mpi_test hermes_posix hermes_daemon) +add_dependencies(hermes_posix_adapter_mpi_test hermes_posix) set_target_properties(hermes_posix_adapter_mpi_test PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") pytest(posix test_hermes_posix_basic_mpi_small) pytest(posix test_hermes_posix_basic_mpi_large) @@ -34,11 +34,12 @@ target_link_libraries(posix_simple_io_mpi hermes_posix) add_executable(hermes_posix_simple_io_omp posix_simple_io_omp.cc) add_dependencies(hermes_posix_simple_io_omp hermes_posix) -target_link_libraries(hermes_posix_simple_io_omp hermes_posix) +target_link_libraries(hermes_posix_simple_io_omp + hermes_posix Catch2::Catch2 OpenMP::OpenMP_CXX) add_executable(posix_simple_io_omp posix_simple_io_omp.cc) -add_dependencies(posix_simple_io_omp hermes_shm_data_structures) -target_link_libraries(posix_simple_io_omp hermes_shm_data_structures) +target_link_libraries(posix_simple_io_omp + ${HermesShm_LIBRARIES} Catch2::Catch2 OpenMP::OpenMP_CXX) pytest(posix test_hermes_posix_simple_io_omp_default) pytest(posix test_hermes_posix_simple_io_omp_scratch) diff --git a/adapter/test/posix/hdf5_write_read.py b/test/unit/hermes_adapters/posix/hdf5_write_read.py similarity index 100% rename from adapter/test/posix/hdf5_write_read.py rename to test/unit/hermes_adapters/posix/hdf5_write_read.py diff --git a/adapter/test/posix/posix_adapter_basic_test.cpp b/test/unit/hermes_adapters/posix/posix_adapter_basic_test.cpp similarity index 100% rename from adapter/test/posix/posix_adapter_basic_test.cpp rename to test/unit/hermes_adapters/posix/posix_adapter_basic_test.cpp diff --git a/adapter/test/posix/posix_adapter_mpi_test.cpp b/test/unit/hermes_adapters/posix/posix_adapter_mpi_test.cpp similarity index 93% rename from adapter/test/posix/posix_adapter_mpi_test.cpp rename to test/unit/hermes_adapters/posix/posix_adapter_mpi_test.cpp index db9260928..501b5e07f 100644 --- a/adapter/test/posix/posix_adapter_mpi_test.cpp +++ b/test/unit/hermes_adapters/posix/posix_adapter_mpi_test.cpp @@ -22,8 +22,8 @@ #include "adapter_test_utils.h" #if HERMES_INTERCEPT == 1 -#include "adapter/posix/posix_api.h" -#include "adapter/posix/posix_fs_api.h" +#include "hermes_adapters/posix/posix_api.h" +#include "hermes_adapters/posix/posix_fs_api.h" #endif namespace stdfs = std::filesystem; @@ -84,7 +84,7 @@ std::vector gen_random(const int len) { int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); info.write_data = gen_random(args.request_size); @@ -192,9 +192,9 @@ int pretest() { REQUIRE(info.total_size > 0); MPI_Barrier(MPI_COMM_WORLD); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file_cmp, false); #endif return 0; @@ -208,9 +208,9 @@ void Clear() { int posttest(bool compare_data = true) { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file, false); #endif if (compare_data && stdfs::exists(info.new_file) && @@ -316,13 +316,13 @@ int posttest(bool compare_data = true) { Clear(); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file, true); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file_cmp, true); #endif return 0; diff --git a/adapter/test/posix/posix_adapter_rs_test.cpp b/test/unit/hermes_adapters/posix/posix_adapter_rs_test.cpp similarity index 100% rename from adapter/test/posix/posix_adapter_rs_test.cpp rename to test/unit/hermes_adapters/posix/posix_adapter_rs_test.cpp diff --git a/adapter/test/posix/posix_adapter_shared_test.cpp b/test/unit/hermes_adapters/posix/posix_adapter_shared_test.cpp similarity index 100% rename from adapter/test/posix/posix_adapter_shared_test.cpp rename to test/unit/hermes_adapters/posix/posix_adapter_shared_test.cpp diff --git a/adapter/test/posix/posix_adapter_test.cpp b/test/unit/hermes_adapters/posix/posix_adapter_test.cpp similarity index 94% rename from adapter/test/posix/posix_adapter_test.cpp rename to test/unit/hermes_adapters/posix/posix_adapter_test.cpp index 96d0dea18..0c5e1350d 100644 --- a/adapter/test/posix/posix_adapter_test.cpp +++ b/test/unit/hermes_adapters/posix/posix_adapter_test.cpp @@ -16,14 +16,14 @@ #include #include -#include "hermes.h" +#include "hermes/hermes.h" #include #include "catch_config.h" -#include "adapter/posix/posix_api.h" +#include "hermes_adapters/posix/posix_api.h" #if HERMES_INTERCEPT == 1 -#include "adapter/posix/posix_fs_api.h" +#include "hermes_adapters/posix/posix_fs_api.h" #endif #include "adapter_test_utils.h" @@ -76,7 +76,7 @@ std::vector gen_random(const int len) { int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); info.write_data = gen_random(args.request_size); @@ -92,17 +92,17 @@ int finalize() { void IgnoreAllFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); #endif } void TrackFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); #endif } diff --git a/adapter/test/posix/posix_simple_io_mpi.cc b/test/unit/hermes_adapters/posix/posix_simple_io_mpi.cc similarity index 99% rename from adapter/test/posix/posix_simple_io_mpi.cc rename to test/unit/hermes_adapters/posix/posix_simple_io_mpi.cc index 3efe24c0b..5299e1a18 100644 --- a/adapter/test/posix/posix_simple_io_mpi.cc +++ b/test/unit/hermes_adapters/posix/posix_simple_io_mpi.cc @@ -21,7 +21,7 @@ #include #include #include -#include "test/test_utils.h" +#include "basic_test.h" int main(int argc, char **argv) { int rank = 0, nprocs = 1; diff --git a/adapter/test/posix/posix_simple_io_omp.cc b/test/unit/hermes_adapters/posix/posix_simple_io_omp.cc similarity index 95% rename from adapter/test/posix/posix_simple_io_omp.cc rename to test/unit/hermes_adapters/posix/posix_simple_io_omp.cc index 6918d1397..f470be1a5 100644 --- a/adapter/test/posix/posix_simple_io_omp.cc +++ b/test/unit/hermes_adapters/posix/posix_simple_io_omp.cc @@ -15,15 +15,16 @@ #include #include #include "hermes_shm/util/logging.h" +#include "hermes_shm/util/config_parse.h" #include #include #include #include #include #include -#include "test/test_utils.h" +#include "basic_test.h" -static const int kNumProcs = 4; +static const int kNumProcs = 1; void TestThread(char *path, int do_read, @@ -130,14 +131,14 @@ void TestThread(char *path, int main(int argc, char **argv) { if (argc != 6) { std::cout << "USAGE: ./posix_simple_io" - << " [path] [read] [block_size (kb)] [count]" + << " [path] [read] [block_size] [count]" << " [off (blocks)]"; exit(1); } char *path = argv[1]; int do_read = atoi(argv[2]); - int block_size = atoi(argv[3])*1024; + int block_size = hshm::ConfigParse::ParseSize(argv[3]); int count = atoi(argv[4]); int block_off = atoi(argv[5]); if (do_read) { diff --git a/adapter/test/posix/tests.py b/test/unit/hermes_adapters/posix/tests.py similarity index 100% rename from adapter/test/posix/tests.py rename to test/unit/hermes_adapters/posix/tests.py diff --git a/adapter/test/stdio/CMakeLists.txt b/test/unit/hermes_adapters/stdio/CMakeLists.txt similarity index 91% rename from adapter/test/stdio/CMakeLists.txt rename to test/unit/hermes_adapters/stdio/CMakeLists.txt index 800efc3ad..d14367619 100644 --- a/adapter/test/stdio/CMakeLists.txt +++ b/test/unit/hermes_adapters/stdio/CMakeLists.txt @@ -22,20 +22,20 @@ add_executable(stdio_adapter_mpi_test stdio_adapter_mpi_test.cpp ${ADAPTER_COMMO add_executable(hermes_stdio_adapter_test stdio_adapter_test.cpp ${ADAPTER_COMMON}) target_link_libraries(hermes_stdio_adapter_test hermes_stdio) -add_dependencies(hermes_stdio_adapter_test hermes_stdio hermes_daemon) +add_dependencies(hermes_stdio_adapter_test hermes_stdio) set_target_properties(hermes_stdio_adapter_test PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") pytest(stdio test_hermes_stdio_basic_small) pytest(stdio test_hermes_stdio_basic_large) add_executable(hermes_stdio_low_buf_adapter_test stdio_adapter_low_buffer_space_test.cpp ${ADAPTER_COMMON}) target_link_libraries(hermes_stdio_low_buf_adapter_test hermes_stdio) -add_dependencies(hermes_stdio_low_buf_adapter_test hermes_stdio hermes_daemon) +add_dependencies(hermes_stdio_low_buf_adapter_test hermes_stdio) set_target_properties(hermes_stdio_low_buf_adapter_test PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") pytest(stdio test_hermes_stdio_low_buf) add_executable(hermes_stdio_adapter_mode_test stdio_adapter_mode_test.cpp ${ADAPTER_COMMON}) target_link_libraries(hermes_stdio_adapter_mode_test hermes_stdio) -add_dependencies(hermes_stdio_adapter_mode_test hermes_stdio hermes_daemon) +add_dependencies(hermes_stdio_adapter_mode_test hermes_stdio) set_target_properties(hermes_stdio_adapter_mode_test PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") pytest(stdio test_hermes_stdio_bypass) pytest(stdio test_hermes_stdio_default) @@ -43,7 +43,7 @@ pytest(stdio test_hermes_stdio_scratch) add_executable(hermes_stdio_adapter_mpi_test stdio_adapter_mpi_test.cpp ${ADAPTER_COMMON}) target_link_libraries(hermes_stdio_adapter_mpi_test hermes_stdio) -add_dependencies(hermes_stdio_adapter_mpi_test hermes_stdio hermes_daemon) +add_dependencies(hermes_stdio_adapter_mpi_test hermes_stdio) set_target_properties(hermes_stdio_adapter_mpi_test PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") pytest(stdio test_hermes_stdio_mpi_small) pytest(stdio test_hermes_stdio_mpi_large) diff --git a/adapter/test/stdio/stdio_adapter_basic_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_basic_test.cpp similarity index 100% rename from adapter/test/stdio/stdio_adapter_basic_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_basic_test.cpp diff --git a/adapter/test/stdio/stdio_adapter_func_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_func_test.cpp similarity index 100% rename from adapter/test/stdio/stdio_adapter_func_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_func_test.cpp diff --git a/adapter/test/stdio/stdio_adapter_low_buffer_space_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_low_buffer_space_test.cpp similarity index 93% rename from adapter/test/stdio/stdio_adapter_low_buffer_space_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_low_buffer_space_test.cpp index 943ce94b4..337c81f86 100644 --- a/adapter/test/stdio/stdio_adapter_low_buffer_space_test.cpp +++ b/test/unit/hermes_adapters/stdio/stdio_adapter_low_buffer_space_test.cpp @@ -20,8 +20,8 @@ #include "adapter_test_utils.h" #include "catch_config.h" #if HERMES_INTERCEPT == 1 -#include "adapter/stdio/stdio_api.h" -#include "adapter/stdio/stdio_fs_api.h" +#include "hermes_adapters/stdio/stdio_api.h" +#include "hermes_adapters/stdio/stdio_fs_api.h" #endif namespace stdfs = std::filesystem; @@ -59,7 +59,7 @@ hermes::adapter::stdio::test::Info info; int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); info.write_data = GenRandom(args.request_size); @@ -104,8 +104,8 @@ int pretest() { } REQUIRE(info.total_size > 0); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); #endif return 0; } @@ -118,8 +118,8 @@ void Clear() { int posttest(bool compare_data = true) { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); #endif if (compare_data && stdfs::exists(info.new_file) && stdfs::exists(info.new_file_cmp)) { @@ -191,10 +191,10 @@ int posttest(bool compare_data = true) { Clear(); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); #endif return 0; } diff --git a/adapter/test/stdio/stdio_adapter_mapper_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_mapper_test.cpp similarity index 96% rename from adapter/test/stdio/stdio_adapter_mapper_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_mapper_test.cpp index e3cd45dd2..5214074fe 100644 --- a/adapter/test/stdio/stdio_adapter_mapper_test.cpp +++ b/test/unit/hermes_adapters/stdio/stdio_adapter_mapper_test.cpp @@ -13,10 +13,10 @@ #include #include "catch_config.h" -#include "adapter_constants.h" -#include "src/hermes_types.h" -#include "mapper/mapper_factory.h" -#include "adapter/stdio/stdio_fs_api.h" +#include "hermes/hermes_types.h" +#include "hermes_adapters/adapter_constants.h" +#include "hermes_adapters/mapper/mapper_factory.h" +#include "hermes_adapters/stdio/stdio_fs_api.h" using hermes::adapter::BlobPlacements; using hermes::adapter::MapperFactory; @@ -53,7 +53,7 @@ hermes::adapter::stdio::test::Info info; int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); diff --git a/adapter/test/stdio/stdio_adapter_mode_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_mode_test.cpp similarity index 90% rename from adapter/test/stdio/stdio_adapter_mode_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_mode_test.cpp index e6d81d5c6..78926c123 100644 --- a/adapter/test/stdio/stdio_adapter_mode_test.cpp +++ b/test/unit/hermes_adapters/stdio/stdio_adapter_mode_test.cpp @@ -19,12 +19,12 @@ #include #if HERMES_INTERCEPT == 1 -#include "adapter/stdio/stdio_api.h" +#include "hermes_adapters/stdio/stdio_api.h" #endif #include "adapter_test_utils.h" -#include "adapter_types.h" -#include "hermes.h" +#include "hermes_adapters/adapter_types.h" +#include "hermes/hermes.h" namespace stdfs = std::filesystem; using hermes::adapter::AdapterMode; @@ -62,7 +62,7 @@ hermes::adapter::stdio::test::Info info; int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif stdfs::path fullpath = args.directory; fullpath /= args.filename; @@ -73,8 +73,8 @@ int init(int* argc, char*** argv) { fullpath.string() + "_ext_cmp" + std::to_string(getpid()); char* set_path = getenv("SET_PATH"); if (set_path && strcmp(set_path, "1") == 0) { - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); } MPI_Init(argc, argv); info.write_data = GenRandom(args.request_size); @@ -88,17 +88,17 @@ int finalize() { void IgnoreAllFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); #endif } void TrackFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); #endif } @@ -286,8 +286,8 @@ TEST_CASE("BatchedWriteSequentialPersistent", std::to_string(info.num_iterations) + "]" "[pattern=sequential][file=1]") { - HERMES->client_config_.SetBaseAdapterMode(AdapterMode::kDefault); - REQUIRE(HERMES->client_config_.GetBaseAdapterMode() == AdapterMode::kDefault); + HERMES_CLIENT_CONF.SetBaseAdapterMode(AdapterMode::kDefault); + REQUIRE(HERMES_CLIENT_CONF.GetBaseAdapterMode() == AdapterMode::kDefault); pretest(); SECTION("write to new file always at end") { test::test_fopen(info.new_file.c_str(), "w+"); @@ -314,8 +314,8 @@ TEST_CASE("BatchedWriteSequentialBypass", std::to_string(info.num_iterations) + "]" "[pattern=sequential][file=1]") { - HERMES->client_config_.SetBaseAdapterMode(AdapterMode::kBypass); - REQUIRE(HERMES->client_config_.GetBaseAdapterMode() == AdapterMode::kBypass); + HERMES_CLIENT_CONF.SetBaseAdapterMode(AdapterMode::kBypass); + REQUIRE(HERMES_CLIENT_CONF.GetBaseAdapterMode() == AdapterMode::kBypass); pretest(); SECTION("write to new file always at end") { test::test_fopen(info.new_file.c_str(), "w+"); @@ -342,8 +342,8 @@ TEST_CASE("BatchedWriteSequentialScratch", std::to_string(info.num_iterations) + "]" "[pattern=sequential][file=1]") { - HERMES->client_config_.SetBaseAdapterMode(AdapterMode::kScratch); - REQUIRE(HERMES->client_config_.GetBaseAdapterMode() == AdapterMode::kScratch); + HERMES_CLIENT_CONF.SetBaseAdapterMode(AdapterMode::kScratch); + REQUIRE(HERMES_CLIENT_CONF.GetBaseAdapterMode() == AdapterMode::kScratch); pretest(); SECTION("write to new file always at end") { test::test_fopen(info.new_file.c_str(), "w+"); diff --git a/adapter/test/stdio/stdio_adapter_mpi_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_mpi_test.cpp similarity index 93% rename from adapter/test/stdio/stdio_adapter_mpi_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_mpi_test.cpp index 3cb86dde7..0a4970665 100644 --- a/adapter/test/stdio/stdio_adapter_mpi_test.cpp +++ b/test/unit/hermes_adapters/stdio/stdio_adapter_mpi_test.cpp @@ -19,8 +19,8 @@ #include #include #if HERMES_INTERCEPT == 1 -#include "adapter/stdio/stdio_api.h" -#include "adapter/stdio/stdio_fs_api.h" +#include "hermes_adapters/stdio/stdio_api.h" +#include "hermes_adapters/stdio/stdio_fs_api.h" #endif namespace stdfs = std::filesystem; @@ -62,7 +62,7 @@ hermes::adapter::stdio::test::Info info; int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); info.write_data = GenRandom(args.request_size); @@ -211,9 +211,9 @@ int pretest() { REQUIRE(info.total_size > 0); MPI_Barrier(MPI_COMM_WORLD); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file_cmp, false); #endif return 0; @@ -227,9 +227,9 @@ void Clear() { int posttest(bool compare_data = true) { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file, false); #endif if (compare_data && stdfs::exists(info.new_file) && @@ -335,13 +335,13 @@ int posttest(bool compare_data = true) { Clear(); #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, true); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file, true); - HERMES->client_config_.SetAdapterPathTracking( + HERMES_CLIENT_CONF.SetAdapterPathTracking( info.existing_shared_file_cmp, true); #endif return 0; diff --git a/adapter/test/stdio/stdio_adapter_rs_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_rs_test.cpp similarity index 100% rename from adapter/test/stdio/stdio_adapter_rs_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_rs_test.cpp diff --git a/adapter/test/stdio/stdio_adapter_shared_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_shared_test.cpp similarity index 100% rename from adapter/test/stdio/stdio_adapter_shared_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_shared_test.cpp diff --git a/adapter/test/stdio/stdio_adapter_test.cpp b/test/unit/hermes_adapters/stdio/stdio_adapter_test.cpp similarity index 93% rename from adapter/test/stdio/stdio_adapter_test.cpp rename to test/unit/hermes_adapters/stdio/stdio_adapter_test.cpp index e43d951af..92c690771 100644 --- a/adapter/test/stdio/stdio_adapter_test.cpp +++ b/test/unit/hermes_adapters/stdio/stdio_adapter_test.cpp @@ -19,9 +19,9 @@ #include "adapter_test_utils.h" #include "catch_config.h" -#include "adapter/stdio/stdio_api.h" +#include "hermes_adapters/stdio/stdio_api.h" #if HERMES_INTERCEPT == 1 -#include "adapter/stdio/stdio_fs_api.h" +#include "hermes_adapters/stdio/stdio_fs_api.h" #endif #include "hermes_shm/util/logging.h" @@ -63,7 +63,7 @@ hermes::adapter::stdio::test::Info info; int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); info.write_data = GenRandom(args.request_size); @@ -78,17 +78,17 @@ int finalize() { void IgnoreAllFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); #endif } void TrackFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); #endif } diff --git a/adapter/test/stdio/tests.py b/test/unit/hermes_adapters/stdio/tests.py similarity index 100% rename from adapter/test/stdio/tests.py rename to test/unit/hermes_adapters/stdio/tests.py diff --git a/adapter/test/vfd/CMakeLists.txt b/test/unit/hermes_adapters/vfd/CMakeLists.txt similarity index 100% rename from adapter/test/vfd/CMakeLists.txt rename to test/unit/hermes_adapters/vfd/CMakeLists.txt diff --git a/adapter/test/vfd/hermes_vfd_basic_test.cc b/test/unit/hermes_adapters/vfd/hermes_vfd_basic_test.cc similarity index 99% rename from adapter/test/vfd/hermes_vfd_basic_test.cc rename to test/unit/hermes_adapters/vfd/hermes_vfd_basic_test.cc index 216d20e6f..f0c2eca7f 100644 --- a/adapter/test/vfd/hermes_vfd_basic_test.cc +++ b/test/unit/hermes_adapters/vfd/hermes_vfd_basic_test.cc @@ -709,7 +709,7 @@ TEST_CASE("ScratchMode", "[scratch]") { test::TestClose(); REQUIRE(test::hermes_herr >= 0); - if (HERMES->client_config_.GetBaseAdapterMode() + if (HERMES_CLIENT_CONF.GetBaseAdapterMode() == hermes::adapter::AdapterMode::kScratch) { REQUIRE(!stdfs::exists(info.new_file)); } diff --git a/adapter/test/vfd/hermes_vfd_test.cc b/test/unit/hermes_adapters/vfd/hermes_vfd_test.cc similarity index 96% rename from adapter/test/vfd/hermes_vfd_test.cc rename to test/unit/hermes_adapters/vfd/hermes_vfd_test.cc index 25a45da61..ce2711315 100644 --- a/adapter/test/vfd/hermes_vfd_test.cc +++ b/test/unit/hermes_adapters/vfd/hermes_vfd_test.cc @@ -22,7 +22,7 @@ #include #include -#include "hermes.h" +#include "hermes/hermes.h" #include "hermes_types.h" #include "adapter_test_utils.h" #include "catch_config.h" @@ -364,17 +364,17 @@ using hermes::adapter::vfd::test::GenRandom0to1; void IgnoreAllFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.existing_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file_cmp, false); - HERMES->client_config_.SetAdapterPathTracking(info.new_file, false); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file_cmp, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, false); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, false); #endif } void TrackFiles() { #if HERMES_INTERCEPT == 1 - HERMES->client_config_.SetAdapterPathTracking(info.new_file, true); - HERMES->client_config_.SetAdapterPathTracking(info.existing_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.new_file, true); + HERMES_CLIENT_CONF.SetAdapterPathTracking(info.existing_file, true); #endif } @@ -400,7 +400,7 @@ int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); TRANSPARENT_HERMES - HERMES->client_config_.flushing_mode_ = hermes::FlushingMode::kSync; + HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); @@ -477,7 +477,7 @@ void CheckResults(const std::string &file1, const std::string &file2) { * Called after each individual test. */ int Posttest() { - if (HERMES->client_config_.GetBaseAdapterMode() + if (HERMES_CLIENT_CONF.GetBaseAdapterMode() != hermes::adapter::AdapterMode::kScratch) { // NOTE(chogan): This is necessary so that h5diff doesn't use the Hermes VFD // in CheckResults. We don't need to reset LD_PRELOAD because it only has an diff --git a/adapter/test/vfd/tests.py b/test/unit/hermes_adapters/vfd/tests.py similarity index 100% rename from adapter/test/vfd/tests.py rename to test/unit/hermes_adapters/vfd/tests.py diff --git a/test/unit/ipc/CMakeLists.txt b/test/unit/ipc/CMakeLists.txt new file mode 100644 index 000000000..27f886489 --- /dev/null +++ b/test/unit/ipc/CMakeLists.txt @@ -0,0 +1,46 @@ +cmake_minimum_required(VERSION 3.10) +project(labstor) + +set(CMAKE_CXX_STANDARD 17) + +#------------------------------------------------------------------------------ +# Build Tests +#------------------------------------------------------------------------------ + +add_executable(test_ipc_exec + ${TEST_MAIN}/main_mpi.cc + test_init.cc + test_finalize.cc + test_ipc.cc + test_serialize.cc +) + +add_dependencies(test_ipc_exec ${Labstor_CLIENT_DEPS}) +target_link_libraries(test_ipc_exec + ${Labstor_CLIENT_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) + +#------------------------------------------------------------------------------ +# Test Cases +#------------------------------------------------------------------------------ + +# STRING TESTS +add_test(NAME test_ipc COMMAND + ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") + +#------------------------------------------------------------------------------ +# Install Targets +#------------------------------------------------------------------------------ +install(TARGETS + test_ipc_exec + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(test_messages) +endif() diff --git a/test/unit/ipc/test_finalize.cc b/test/unit/ipc/test_finalize.cc new file mode 100644 index 000000000..8834bd3d2 --- /dev/null +++ b/test/unit/ipc/test_finalize.cc @@ -0,0 +1,11 @@ +// +// Created by llogan on 7/1/23. +// + +#include "basic_test.h" +#include "labstor/api/labstor_client.h" +#include "labstor_admin/labstor_admin.h" + +TEST_CASE("TestFinalize") { + LABSTOR_ADMIN->AsyncStopRuntimeRoot(labstor::DomainId::GetGlobal()); +} \ No newline at end of file diff --git a/hermes_shm/test/unit/allocators/test_init.cc b/test/unit/ipc/test_init.cc similarity index 87% rename from hermes_shm/test/unit/allocators/test_init.cc rename to test/unit/ipc/test_init.cc index e3026af72..63132b16a 100644 --- a/hermes_shm/test/unit/allocators/test_init.cc +++ b/test/unit/ipc/test_init.cc @@ -11,11 +11,13 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include "labstor/api/labstor_client.h" +#include "basic_test.h" #include "test_init.h" -void Posttest() { +void MainPretest() { + TRANSPARENT_LABSTOR(); } -void MainPretest() {} - -void MainPosttest() {} +void MainPosttest() { +} diff --git a/hermes_shm/test/unit/types/test_init.cc b/test/unit/ipc/test_init.h similarity index 83% rename from hermes_shm/test/unit/types/test_init.cc rename to test/unit/ipc/test_init.h index 82bac123b..a6c71f3ec 100644 --- a/hermes_shm/test/unit/types/test_init.cc +++ b/test/unit/ipc/test_init.h @@ -11,8 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "basic_test.h" +#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ -void MainPretest() {} +#include "labstor/labstor_types.h" -void MainPosttest() {} +#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/test/unit/ipc/test_ipc.cc b/test/unit/ipc/test_ipc.cc new file mode 100644 index 000000000..41ecaf5ee --- /dev/null +++ b/test/unit/ipc/test_ipc.cc @@ -0,0 +1,111 @@ +// +// Created by llogan on 7/1/23. +// + +#include "basic_test.h" +#include +#include "labstor/api/labstor_client.h" +#include "labstor_admin/labstor_admin.h" + +#include "small_message/small_message.h" +#include "hermes_shm/util/timer.h" +#include "labstor/work_orchestrator/affinity.h" +#include "omp.h" + +TEST_CASE("TestIpc") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + labstor::small_message::Client client; + LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + MPI_Barrier(MPI_COMM_WORLD); + hshm::Timer t; + + int pid = getpid(); + ProcessAffiner::SetCpuAffinity(pid, 8); + + t.Resume(); + size_t ops = 256; + for (size_t i = 0; i < ops; ++i) { + int ret; + HILOG(kInfo, "Sending message {}", i); + int node_id = 1 + ((rank + 1) % nprocs); + ret = client.MdRoot(labstor::DomainId::GetNode(node_id)); + REQUIRE(ret == 1); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +void TestIpcMultithread(int nprocs) { + labstor::small_message::Client client; + LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + +#pragma omp parallel shared(client, nprocs) num_threads(nprocs) + { + int rank = omp_get_thread_num(); + size_t ops = 256; + for (size_t i = 0; i < ops; ++i) { + int ret; + HILOG(kInfo, "Sending message {}", i); + int node_id = 1 + ((rank + 1) % nprocs); + ret = client.MdRoot(labstor::DomainId::GetNode(node_id)); + REQUIRE(ret == 1); + } + } +} + +TEST_CASE("TestIpcMultithread4") { + TestIpcMultithread(4); +} + +TEST_CASE("TestIpcMultithread8") { + TestIpcMultithread(8); +} + +TEST_CASE("TestIpcMultithread16") { + TestIpcMultithread(16); +} + +TEST_CASE("TestIpcMultithread32") { + TestIpcMultithread(32); +} + +TEST_CASE("TestIO") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + labstor::small_message::Client client; + LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + hshm::Timer t; + + int pid = getpid(); + ProcessAffiner::SetCpuAffinity(pid, 8); + + HILOG(kInfo, "Starting IO test: {}", nprocs); + + t.Resume(); + size_t ops = 16; + for (size_t i = 0; i < ops; ++i) { + int ret; + HILOG(kInfo, "Sending message {}", i); + int node_id = 1 + ((rank + 1) % nprocs); + ret = client.IoRoot(labstor::DomainId::GetNode(node_id)); + REQUIRE(ret == 1); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +//TEST_CASE("TestHostfile") { +// for (u32 node_id = 1; node_id < LABSTOR_THALLIUM->rpc_->hosts_.size() + 1; ++node_id) { +// HILOG(kInfo, "Node {}: {}", node_id, LABSTOR_THALLIUM->GetServerName(node_id)); +// } +//} \ No newline at end of file diff --git a/test/unit/ipc/test_serialize.cc b/test/unit/ipc/test_serialize.cc new file mode 100644 index 000000000..e3f1c7ed9 --- /dev/null +++ b/test/unit/ipc/test_serialize.cc @@ -0,0 +1,63 @@ +// +// Created by lukemartinlogan on 8/7/23. +// + +#include "basic_test.h" +#include "labstor/network/serialize.h" +#include "labstor/task_registry/task.h" + +using labstor::DomainId; +using labstor::BinaryOutputArchive; +using labstor::BinaryInputArchive; +using labstor::DataTransfer; +using labstor::Task; +using labstor::TaskFlags; + +struct TestObj : public Task, TaskFlags { + std::vector data_; + DataTransfer xfer1_; + DataTransfer xfer2_; + int a_, b_, c_; + + TestObj(int a, int b, int c, const std::vector &data) + : a_(a), b_(b), c_(c), data_(data), + xfer1_(DT_RECEIVER_READ, data_.data(), data_.size()), + xfer2_(DT_RECEIVER_WRITE, data_.data(), data_.size()), + Task(nullptr) {} + + template + void serialize(Ar &ar) { + + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(a_, b_, xfer1_, c_, xfer2_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + bool operator==(const TestObj &other) const { + return a_ == other.a_ && b_ == other.b_ && c_ == other.c_ && + xfer1_ == other.xfer1_ && xfer2_ == other.xfer2_; + } +}; + +TEST_CASE("TestSerialize") { + std::vector data(256); + BinaryOutputArchive out(DomainId::GetLocal()); + TestObj obj(25, 30, 35, std::vector(256, 10)); + out << obj; + std::vector xfer = out.Get(); + + BinaryInputArchive in(xfer); + TestObj obj2(40, 50, 60, std::vector(256, 0)); + in >> obj2; + + REQUIRE(obj == obj2); +} \ No newline at end of file diff --git a/test/main.cc b/test/unit/main.cc similarity index 98% rename from test/main.cc rename to test/unit/main.cc index da1e76728..982afa67a 100644 --- a/test/main.cc +++ b/test/unit/main.cc @@ -10,8 +10,8 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + #include "basic_test.h" -#include int main(int argc, char **argv) { int rc; diff --git a/test/main_mpi.cc b/test/unit/main_mpi.cc similarity index 100% rename from test/main_mpi.cc rename to test/unit/main_mpi.cc diff --git a/traits/CMakeLists.txt b/traits/CMakeLists.txt deleted file mode 100644 index 070331d27..000000000 --- a/traits/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ - -set(HERMES_IO_CLIENT_DIR ${CMAKE_SOURCE_DIR}/io_client) -set(HERMES_SRC_DIR ${CMAKE_SOURCE_DIR}/src) -set(HERMES_API_DIR ${CMAKE_SOURCE_DIR}/src/api) - -include_directories( - ${CMAKE_SOURCE_DIR} - ${HERMES_SRC_DIR} - ${HERMES_API_DIR} - ${HERMES_IO_CLIENT_DIR}) - -add_subdirectory(example) -add_subdirectory(prefetcher) - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DESTINATION ${HERMES_INSTALL_INCLUDE_DIR} - FILES_MATCHING PATTERN "*.h") diff --git a/traits/example/CMakeLists.txt b/traits/example/CMakeLists.txt deleted file mode 100644 index dca853117..000000000 --- a/traits/example/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ - - -# Creates an example trait -add_library(hermes_example_trait SHARED - example_trait.cc) -add_dependencies(hermes_example_trait hermes) -target_link_libraries(hermes_example_trait - hermes MPI::MPI_CXX stdc++fs dl) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - hermes_example_trait - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_example_trait - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() -#----------------------------------------------------------------------------- -# Add Target(s) to Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_example_trait) -endif() diff --git a/traits/example/example_trait.cc b/traits/example/example_trait.cc deleted file mode 100644 index 120e0ba69..000000000 --- a/traits/example/example_trait.cc +++ /dev/null @@ -1,25 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "example_trait.h" - -namespace hermes::api { - -void ExampleTrait::Run(int method, void *params) { - (void) method; - auto cast = reinterpret_cast(params); - cast->hello_ = GetHeader()->hello_; -} - -} // namespace hermes::api - -HERMES_TRAIT_CC(hermes::api::ExampleTrait) \ No newline at end of file diff --git a/traits/example/example_trait.h b/traits/example/example_trait.h deleted file mode 100644 index 22699c21f..000000000 --- a/traits/example/example_trait.h +++ /dev/null @@ -1,50 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TRAITS_EXAMPLE_EXAMPLE_TRAIT_H_ -#define HERMES_TRAITS_EXAMPLE_EXAMPLE_TRAIT_H_ - -#include "hermes.h" - -namespace hermes::api { - -struct ExampleTraitHeader : public TraitHeader { - int hello_; - explicit ExampleTraitHeader(const std::string &trait_uuid, - const std::string &trait_name, - int hello) - : TraitHeader(trait_uuid, trait_name, HERMES_TRAIT_PUT_GET), - hello_(hello) {} -}; - -struct ExampleTraitParams { - int hello_; -}; - -class ExampleTrait : public Trait { - public: - HERMES_TRAIT_H(ExampleTrait, "ExampleTrait"); - - public: - explicit ExampleTrait(hshm::charbuf &data) : Trait(data) {} - - explicit ExampleTrait(const std::string &trait_uuid, int hello) { - auto hdr = CreateHeader(trait_uuid, trait_name_, hello); - hdr->hello_ = hello; - } - - void Run(int method, void *params) override; -}; - -} // namespace hermes::api - -#endif // HERMES_TRAITS_EXAMPLE_EXAMPLE_TRAIT_H_ diff --git a/traits/prefetcher/CMakeLists.txt b/traits/prefetcher/CMakeLists.txt deleted file mode 100644 index 7a8ec1efd..000000000 --- a/traits/prefetcher/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ - - -# Creates an example trait -add_library(hermes_prefetcher_trait SHARED - prefetcher_trait.cc) -add_dependencies(hermes_prefetcher_trait hermes) -target_link_libraries(hermes_prefetcher_trait - hermes MPI::MPI_CXX stdc++fs dl) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - hermes_prefetcher_trait - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_prefetcher_trait - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() -#----------------------------------------------------------------------------- -# Add Target(s) to Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_prefetcher_trait) -endif() diff --git a/traits/prefetcher/prefetcher_header.h b/traits/prefetcher/prefetcher_header.h deleted file mode 100644 index 6499ffd2a..000000000 --- a/traits/prefetcher/prefetcher_header.h +++ /dev/null @@ -1,37 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TRAITS_PREFETCHER_PREFETCHER_HEADER_H_ -#define HERMES_TRAITS_PREFETCHER_PREFETCHER_HEADER_H_ - -#include "trait_manager.h" - -namespace hermes { - -/** Types of prefetchers available */ -enum class PrefetcherType { - kApriori -}; - -/** Header for prefetcher trait */ -struct PrefetcherTraitHeader : public TraitHeader { - hermes::PrefetcherType type_; - explicit PrefetcherTraitHeader(const std::string &trait_uuid, - const std::string &trait_name, - hermes::PrefetcherType type) - : TraitHeader(trait_uuid, trait_name, HERMES_TRAIT_PREFETCHER), - type_(type) {} -}; - -} // namespace hermes::api - -#endif // HERMES_TRAITS_PREFETCHER_PREFETCHER_HEADER_H_ diff --git a/traits/prefetcher/prefetcher_trait.cc b/traits/prefetcher/prefetcher_trait.cc deleted file mode 100644 index 6150363a5..000000000 --- a/traits/prefetcher/prefetcher_trait.cc +++ /dev/null @@ -1,24 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "prefetcher_trait.h" - -namespace hermes { - -void PrefetcherTrait::Run(int method, void *params) { - (void) method; - (void) params; -} - -} // namespace hermes::api - -HERMES_TRAIT_CC(hermes::PrefetcherTrait) \ No newline at end of file diff --git a/traits/prefetcher/prefetcher_trait.h b/traits/prefetcher/prefetcher_trait.h deleted file mode 100644 index 2599ad674..000000000 --- a/traits/prefetcher/prefetcher_trait.h +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_TRAITS_EXAMPLE_EXAMPLE_TRAIT_H_ -#define HERMES_TRAITS_EXAMPLE_EXAMPLE_TRAIT_H_ - -#include "hermes.h" -#include "prefetcher_header.h" - -namespace hermes { - -/** Prefetcher trait */ -class PrefetcherTrait : public Trait { - public: - HERMES_TRAIT_H(PrefetcherTrait, "PrefetcherTrait"); - - public: - explicit PrefetcherTrait(hshm::charbuf &data) : Trait(data) {} - - explicit PrefetcherTrait(const std::string &trait_uuid, - hermes::PrefetcherType prefetch_type) { - CreateHeader(trait_uuid, - trait_name_, - prefetch_type); - } - - void Run(int method, void *params) override; -}; - -} // namespace hermes - -#endif // HERMES_TRAITS_EXAMPLE_EXAMPLE_TRAIT_H_ diff --git a/wrapper/CMakeLists.txt b/wrapper/CMakeLists.txt deleted file mode 100644 index 10b7e232d..000000000 --- a/wrapper/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ - -set(HERMES_IO_CLIENT_DIR ${CMAKE_SOURCE_DIR}/io_client) -set(HERMES_SRC_DIR ${CMAKE_SOURCE_DIR}/src) -set(HERMES_API_DIR ${CMAKE_SOURCE_DIR}/src/api) -set(HERMES_WRAPPER_DIR ${CMAKE_SOURCE_DIR}/wrapper) - -include_directories( - ${CMAKE_SOURCE_DIR} - ${HERMES_SRC_DIR} - ${HERMES_API_DIR} - ${HERMES_WRAPPER_DIR} - ${HERMES_IO_CLIENT_DIR}) - -if (HERMES_ENABLE_C_BINDINGS OR - HERMES_ENABLE_JAVA_BINDINGS OR - HERMES_ENABLE_PYTHON_BINDINGS) - add_subdirectory(c) -endif() - -if (HERMES_ENABLE_JAVA_BINDINGS) - add_subdirectory(java) -endif() - -if (HERMES_ENABLE_PYTHON_BINDINGS) - add_subdirectory(python) -endif() diff --git a/wrapper/c/CMakeLists.txt b/wrapper/c/CMakeLists.txt deleted file mode 100644 index a81c4fc12..000000000 --- a/wrapper/c/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ - - -# Creates an example trait -add_library(hermes_c_wrapper SHARED - c_wrapper.cc) -add_dependencies(hermes_c_wrapper hermes) -target_link_libraries(hermes_c_wrapper - hermes MPI::MPI_CXX stdc++fs dl) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - hermes_c_wrapper - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_c_wrapper - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() -#----------------------------------------------------------------------------- -# Add Target(s) to Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_example_trait) -endif() diff --git a/wrapper/c/c_wrapper.cc b/wrapper/c/c_wrapper.cc deleted file mode 100644 index 39821cb6c..000000000 --- a/wrapper/c/c_wrapper.cc +++ /dev/null @@ -1,78 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes.h" -#include "c_wrapper.h" - -extern "C" { - -/** Get the Hermes instance */ -hapi::Hermes* HERMES_Get() { - return HERMES; -} - -/** Initialize a Hermes instance */ -void HERMES_Create(hermes::HermesType mode, - std::string server_config_path, - std::string client_config_path) { - HERMES->Create(mode, server_config_path, client_config_path); -} - -/** Bucket create */ -hapi::Bucket HERMES_Bucket_get(const std::string &name) { - hapi::Context ctx; - return HERMES->GetBucket(name, ctx, 0); -} - -/** Bucket delete */ -void HERMES_Bucket_destroy(hapi::Bucket &bucket) { - bucket.Destroy(); -} - -/** Blob put */ -hermes::BlobId HERMES_Bucket_put_blob(hapi::Bucket &bucket, - const char *blob_name, - const char *blob, - size_t blob_size) { - hermes::Blob blob_wrap(blob, blob_size); - hermes::BlobId blob_id; - bucket.Put(blob_name, blob_wrap, blob_id, bucket.GetContext()); - return blob_id; -} - -/** Get blob ID */ -hermes::BlobId HERMES_Bucket_get_blob_id(hapi::Bucket &bucket, - const char *blob_name) { - hermes::BlobId blob_id; - bucket.GetBlobId(blob_name, blob_id); - return blob_id; -} - -/** Blob get */ -void HERMES_Bucket_get_blob(hapi::Bucket &bucket, - hermes::BlobId &blob_id, - char* &blob, - size_t &blob_size) { - hermes::Blob blob_h; - bucket.Get(blob_id, blob_h, bucket.GetContext()); - blob = blob_h.data(); - blob_size = blob_h.size(); - std::move(blob_h); // Don't allow blob to be destroyed -} - -/** Blob delete */ -void HERMES_Bucket_destroy_blob(hapi::Bucket &bucket, - hermes::BlobId &blob_id) { - bucket.DestroyBlob(blob_id, bucket.GetContext()); -} - -} // extern C \ No newline at end of file diff --git a/wrapper/c/c_wrapper.h b/wrapper/c/c_wrapper.h deleted file mode 100644 index 9e70a8d4f..000000000 --- a/wrapper/c/c_wrapper.h +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_WRAPPER_C_C_WRAPPER_H_ -#define HERMES_WRAPPER_C_C_WRAPPER_H_ - -#include "hermes.h" - -extern "C" { - -hapi::Hermes* HERMES_Get(); -void HERMES_Create(hermes::HermesType mode, - std::string server_config_path, - std::string client_config_path); -hapi::Bucket HERMES_Bucket_get(const std::string &name); -void HERMES_Bucket_destroy(hapi::Bucket &bucket); -hermes::BlobId HERMES_Bucket_put_blob(hapi::Bucket &bucket, - const char *blob_name, - const char *blob, - size_t blob_size); -hermes::BlobId HERMES_Bucket_get_blob_id(hapi::Bucket &bucket, - const char *blob_name); -void HERMES_Bucket_get_blob(hapi::Bucket &bucket, - hermes::BlobId &blob_id, - char* &blob, - size_t &blob_size); -void HERMES_Bucket_destroy_blob(hapi::Bucket &bucket, - hermes::BlobId &blob_id); - -} // extern C - -#endif // HERMES_WRAPPER_C_C_WRAPPER_H_ diff --git a/wrapper/java/CMakeLists.txt b/wrapper/java/CMakeLists.txt deleted file mode 100644 index 03a03c75e..000000000 --- a/wrapper/java/CMakeLists.txt +++ /dev/null @@ -1,73 +0,0 @@ - -#----------------------------------------------------------------------------- -# Find Java JNI -#----------------------------------------------------------------------------- -find_package(JNI REQUIRED) -find_package(Java REQUIRED) -include(UseJava) -message("Found JNI at ${JNI_INCLUDE_DIRS} and ${JNI_LIBRARIES}") - -#----------------------------------------------------------------------------- -# Build C++ Java wrapper -#----------------------------------------------------------------------------- -function(java_wrapper name path) - include_directories(${JNI_INCLUDE_DIRS}) - add_library(${name} SHARED ${path}) - add_dependencies(${name} hermes) - target_link_libraries(${name} - hermes - MPI::MPI_CXX stdc++fs dl ${JNI_LIBRARIES}) -endfunction() -java_wrapper(hermes_java_Blob src/hermes/cpp/src_main_Blob.cc) -java_wrapper(hermes_java_Bucket src/hermes/cpp/src_main_Bucket.cc) -java_wrapper(hermes_java_Hermes src/hermes/cpp/src_main_Hermes.cc) - -#----------------------------------------------------------------------------- -# Build Java API -#----------------------------------------------------------------------------- -add_custom_command(TARGET hermes_java_Blob POST_BUILD - COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && - ${CMAKE_CURRENT_SOURCE_DIR}/gradlew build -x test) -add_custom_target(build_hermes_java - COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && - ${CMAKE_CURRENT_SOURCE_DIR}/gradlew build -x test) -pytest(java_wrapper test_hermes_java) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - hermes_java_Blob - hermes_java_Bucket - hermes_java_Hermes - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_java_Blob - hermes_java_Bucket - hermes_java_Hermes - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() -#----------------------------------------------------------------------------- -# Add Target(s) to Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) -# set_coverage_flags(hermes_src_main_Blob) -# set_coverage_flags(hermes_src_main_Bucket) -# set_coverage_flags(hermes_src_main_Hermes) -endif() diff --git a/wrapper/java/build.gradle b/wrapper/java/build.gradle deleted file mode 100644 index e4ed3c207..000000000 --- a/wrapper/java/build.gradle +++ /dev/null @@ -1,47 +0,0 @@ -/* - * This build file was generated by the Gradle 'init' task. - * - * This generated file contains a commented-out sample Java project to get you started. - * For more details take a look at the Java Quickstart chapter in the Gradle - * user guide available at https://docs.gradle.org/4.4.1/userguide/tutorial_java_projects.html - */ - -// Apply the java plugin to add support for Java -apply plugin: 'java' - -// In this section you declare where to find the dependencies of your project -repositories { - // Use 'jcenter' for resolving your dependencies. - // You can declare any Maven/Ivy/file repository here. - jcenter() -} - -// In this section you declare the dependencies for your production and test code -dependencies { - // The production code uses the SLF4J logging API at compile time - compile 'org.slf4j:slf4j-api:1.7.25' - - // Declare the dependency for your favourite test framework you want to use in your tests. - // TestNG is also supported by the Gradle Test task. Just change the - // testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add - // 'test.useTestNG()' to your build script. - testCompile 'junit:junit:4.12' -} - -sourceSets { - main { - java { - srcDir 'src/hermes/java' - } - } - - test { - java { - srcDir 'src/test/java' - } - } -} - -test { - useJUnit() -} \ No newline at end of file diff --git a/wrapper/java/gradle/wrapper/gradle-wrapper.jar b/wrapper/java/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index d9b7505b1096bc428abaa017cb79c9d2da247111..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55627 zcmb5V1#D$IlP#PMGcz+YbvPYnW@ct)PEMGaI?T+>%*;uLnW2+(@MY$H`rq6;^G4r) zBuhtom%L?rRjuVJTV4tT6dDK!5(wy6HbM^Qzx<%S{>z9c3(!f(iZaN5nW27}LH?)N z#J9l!fv?x9U+wGZ|22~lkd+V>QBtOt5xtk0oRF5Hqo0G5qNARkoM}*GTx8ifyqlyO zm6(#5qLHBe27LxFPB|#lq-w<=D?TVR9PMw)I7vUjx^ujB{08**Pa=E{sQt&MkiQzi zS2J@oFt#?KGqSe$Y6ebD_6E-8^!AQ+_9l+b7A8(Kl;bm!GW1lL5C5wp+^-$@G_e2bNwJF5a>a!LBL*5P zHu|ZsLIr;P+xZNvbAtpsvCO~l;^b2r9}JFvae7_1IUW7l;`j6W0GAhC`vJ_&K)Tsn^J$}0mOBJWui~zyQMyt`iLrT1SSF6)& zJOt;0BA1-M0|a0=fFV1E)Va_krMg15v;jiZN*4R38|#(re=+(rZ9`e zr^Xv4-?Bob3@Ot#8VN99y&sixubAp3dx{qVTA7S>QRri;Ri6MOR2Tvi+E_`KpKF!y zu%6P#OFJ6<)P}hlXkew=JK48gff=FeJbnhdoYemtw}?&>8!Ku>kG6wBXB+2a)u?sJ z883DsFPDZ}JtMl*G2%+7$puqY0Y*qI49ca?{6)xM&2BR{HofAdnP8F0V254krTv^) zVX!+K8D4sGKnlA7o*_hE-k!WIU^arMv;6T_R zbh#B+Nj+}~7eXIRqTY>xh^jt{t3}k~+t&Ab?6oPqFz?PE4;*^90uvZ8B!iElVSYKP z>O?T(Qu^7CIOU4IS=?o)?&l8xs%t)8qzKu)Z}^I~_A!lnt65n_xS!_OpJ{tJaWzPH zs540qth`1HL3Ac9pWpuOvMWJN4eDRmzxcvD_x~G;^8cUH{=Y$4rK+Wdqk{S&%RJW0 zc!&%oo`+Vu7+g3gCDkOMAXyd-h%63V#pad^mh<~|!``jH81-c0ovROd@7-FU|IH1Jb`PI| z|DfU(dj46wZ)EgAA8LX(Z3vn`wc{onn1Jt6uo)uAy*GU3xt820``nM-NN5-XwlB#d z{TFT!KeCQd9Qs9P-)t%qol#q7iv*I!JFGV#EwL zb#3&aQ&_ANBtGXXRw0~=N1y8el%$K0%rGvS&Me_InAT~vRM9oayF%V6AF!^X%&M}D zkZErs^(CmYNO!CMEWBJ-Fq2S=k8iA0)9JM@0gGl#=g&*7Id-V+#O(pLKc-E?294R+Biuo>YmK*dHpmblLvn0Iu*LA!)Y33kh20+$6$)x_`{R3gt_e?muIW@@kq2?hy1#xr13=8i@J`(AQc?hdz4P0mQa zFNJ1DNU=|Y7At_A=xv759>uer2V_@9sjmenGlb}%j#yPP=qv8*uF4Ot!OHCqsOT3; zM|e#llSx%U8e%QlZ1Y6g`a`xUE#l(soPn8P*23f<=<Uq@x?$HH8bHZ$SE;M;_wIeuFPnt;r70bK5+xM57UC?z@fcLZ7Mni$i_Q z5VEDiCFAK2b6F3L#nM%6vU7*Y&qd?d0@;#{?uK1)?rv5AKWsoBKQktS2Q~w8a&EdM zO2BDOuWw-pLIEo{7)un>O*1{XRQTRH~m|21co|>ho6zOHjA4}RB*lxH_ zXRwb8fN)|`6=5xE4Ph%Pg<^HC)%)`zA}O{Q)Cr~!gF%Gs0pxJSayjInWrT0k!At48 zuz2R}k$~dc!}iGo;Zz$Ye@oyMbP~w}Bds2H;P?$5x`ji)E~j@(ArjT1mcX_(A?`Ci zhy(F!EaT3|3j!wDI03veB6fWLw^IkOn#JaiKb#-IvT>ki69f0vdW8% zj7oo)hNSF6ziNXXn}$8BP?(&-0D=uywnv^58RFcJ5{RN#J`00}CpHN~e5EtaQEX^U zh(SGRNIqY2?9W@M9YG&}(?FV+SM=d1Fsr+FnaP|+`N#N3YhcLVG@N`XRTVMmg<0ua z$I-CeIfGHj7mXvuK%oUr^XrP#zOsGNqfo`J?IF`0{ANf?24O;Rc6M1_@Q}Th?O*}( zsC%N}N~v65Oy*W!DjBYnJhw{O%Z+lGVyqm`aw#x7_ey|IgZZg z2G3GL&0aYqv!U|}`uA07*xq}=`dSsUuT`P??=4 z3^6OTC=YuPp@;$`C8_&%@p(Pj)~Kt}%LdOg$eY3*dkC^Lw2r##(Ny;nAU$Wh#t!I- z!)?~ZUeZ26ny!^S*q0%z9_(JJIQY}B)pS1zg6zVD=P(?qlUn$MhaTm_!npVdej3u2 zG{lg6CCgrof``%|eoir5A9K)QXHCleDkL4fd)7L^kzb*mHRJH&sNoPLs7xpN-WN1P1g-;w8+ywAhr<1 zX===^+SH^U(w$vKkAPe-mWC)!XH2(8x^{bm=zK2jydk6yP|=t;OhoAZgYkw%pV~TO z%$2@bI1!q4pY7D$(6jL&d-UYz`vKNF@k}0N!$Gm{jrlFhN`xRq)8U&P%xOO6ukam2 zdXXDw`bdG~I$Qy4EUb(uw*hm8t#N1z5phpQwf?M{h!CXya*na^K2#yy&2DmVDNz9c zodElNv+WTyGFNeR>B^Gw+;a3{mT9b{dGPkvx*i%a?zO;6_)fYMaPXABVqe?90iATbXDt`qJk4? zKNl_skPRJj{zzywESd=6T|$iZ`9LDHWtH$q(CE|(2}p_kafYI(5Wft|??A+!yrWmW}=Xchei2Q{BM?DA5({t?*HMwLQx7HK} z9Lp{9&InIpJd)ebfWukOX)qzSC#HfO5O2!7hIUwHKE)d*K7+aOTqVzd#&9;mz`!ndfj&|`l{n$7 z(@uHc0eWhVkOk<+WccxGG6cSX;J}&Nkk;3tz?z$;K<@sBX|tlz{@b-f8qJIk5v%Q} z-!c@?W((|ep^Px+-?(02ie*(nk7sgtiHHilBDT<^)gkoDOqe}IJ0vRlvRG6ewa3Wj zWYL;ROxT2kbZ#|u1a<@3ZRT1vgqk(h4Og%<>1jqMR>{d8hDJWm4PmzQYi_7+8k{y+ zVcUr*DLZ$)>MV6vZg@UhTW-&D!4&$aYx%UnS_u#b@0PuuNlqFy?;i`J;U_(#8~--r98xBtBFVrJU_8D$u@^np$(vV zh?x_gq51;YD8JGE@_IQ2$p?$yYlR_-*=UQ%1K~XxR( z6#mQ%W_Ux#2FA7EhM(Ow7>7QdpN1?7O6ov#ACJp0ei-8*y^nzZ{eulKkdrTK!IC& zm5_k}xJW6-1+1>^qaBwT9bt+Ym+B9Ze!YPw~izkn_I5w4Oy4( z>H5c}vtwsF(r^)Nd2gcJ8i@6I1Oi1Namd6Gs7PTcBnfq+J#_{IGe|3dW1NTc4g^By z1I$@w^hT1-N0IpPxU3X;WcFF$P6)YF5FInH$!bw-`?%6a&>w%f#RWYfTDM<=DeawQ zcIUgahe6=|0$-1X9@Ri_><_s>jGJq|#v!DCM&zbO+Q1zw94R-zP>=;BAl5{=MZVU) z$s5Bjdt3@VDGz8B4x4Zxn)Y<~?Z4!UySrexH9)Ss` z7teX@kme@9n##`ftzXOqCP#*de7QE+Fum(?+9-{}sU}`9C03GYrX}sQMBZ0pP`2Mo z<5zy;j(FrJ@ZVYF?J}aK=!-oLzZ&KL$|CXxj!q_yq)dNFrjRHB=zc~-kx%ocCTQ2g z_Asj;-8@Rjh(L<4OL?t%(1f#HZFe99vf@_fUD*3jgzL|h*IUrqVGdzTL*cNQBU7he zsu55PldRE;L*4V<3Cndl%Xjop3(?cb4$1Tgo4PVc#N?CFbRrfedFlt{$o#uQ5sYgV zQ|wCfoUXxz=O~hAint7H$8<#RYKet}4>#G}dhQtdJAVFoE2DZBSZDsEFaO9KrOysm z$%1?h^(#a6KkRG&iy-;ePz6lCvMZt%j!w=>E{6Yl{nzM+RL^Bm6c9gTwKz<)a+L)- zNmPTIXJLp5!_bo=QK1DSZnocqkG!4M(%0_qXIAWh=F?JV{e|)9B%q?j#l?y6elT1)JUDD(Bz%Ly4Z=m>0FZ$) z5)1T`#Ejr#0W{B>RHI0-)*ArnatAUVMY-fjfhm|Q%PS^6WEH1pS=Q((PGOCqY&s31 znB-hrOP@=%YcwXFLOPMK-AqOzCg!M7Yg4~Xmb~d1Fv3u#u_LtAlVoJFq@;7~Q+ExI zl8Z9-Gc+2}*;A$BVEd?a)TZhJwQ0)>m#)u@4k&RWSaNaA@#8bb#WBy|eEJe(Xj%*` z0Mksvvj?*=RLGjazw22#sC(j3z%*tk&}o$x?uC-G=-p;$YR4dItdB(YsBcatu$uWe zRnRr72f0r-xVeab)aO@Djp96(?6~*Wy3(uc;9HGXqt0v;sw)~wun3p8jS$jdW)|ni zSmk?bQagJJR|i7w3nCJ+I`F_5;-cUt3P|r_31wZZf2lK#pGQ}^Qy{7#|%xTSeJ(yammK<`7Fp^ zcBsvTi|kM;WRFGUqesREkg8bNp+(OfHl7w2p&>7--D>AyjHMop5`V?&_WM<1&I2Yh?YitL7eGE=ui+o&2fC8xx zum6;=n&9F)F>Vu4z?z05hnQl(z$I*L5bPo}CA+R^#C7K039-GYk^(nGU<8rvaqGtL zNuSBO*qAj4@33aLQ_y23ACw!I@TlB(#gkC?kRnFmJD^JhqF;7YQ7=4c#k}WB;F2Q&TtF z*ZY+g0w7a|4gVMg5Q`|6MAqs8bHZ>55T#mY>GL#$QRPn0>`2QoB=!OMu|&2Ca*w5^ zs)%!Sxi;(7r6QjkFu)?77{`YWAimE1>6Tf@ajXKFrv#Q} zmn1^8gx`b(qw#gugUz3V=xcYuvI)Ue$Faj`fK0gyqa&$lTrh6TRY~Q&%ZKlR?(epT zOi{o`-8{`liWWTvo7GZkOetNI9V2OLW!b?xI+iC7lE6ut$%)nYRB7Xh5*!ZRD4yXE z#@w*#@pZ@_GoHG=aQ-FZk8%AvC7avZ{CH<=BXz!hyb&X_K)zYm;-w!t-@TLQV28_F z=x82=#C{L`$ec+kXN-icw`~0>@*NiMJLZwmsnuT7&988HweRk$2@y;s`=;nEj5CZL zmEgu{9MfwollVPsC->3(DuB6W9Nvp=Xa`?d0xq@}Kl3zmf)A^98;8^T!=$Bi7gE%ml@fYAD zVm94KS0azhNRK0`JOSe;G z5A)q|3Ei(1zC5uw#2*C$A1pjTm>U8Q+x?w{3r9wFzI|m}Gr#HyJpT^l1x-xt98LaT z)TFo}iu$pPUP%%;*qRpHTnNwh&AV?iK|hleEJMq|UX!AcylYI*0nM6|WI{$aFLt&? z+-#kQYN7CxYoX<2yIY8xmxfc{Fi4u5<-K5w_w_W(jz621%kSg$90thAozMs}lHw#j z9;uI5OcoGKhNYnmzXx6I=uL_~flCAYSO<%>!ch%jAGBW_rS&X447AK~$K0=eOliMq zaMIGlf?bCNsPo<`FQiFnS6N!nToSKYwqje_9LFj46>Gw7PbPUtFu;C91K=e7x>-Sr z@u+}H&0N=yQ$5$>8lW;zw_6@>cTuBL8$+I6VQXs7(*&ruL?z@-AZpAJEYiKV8?ap% zHf-LF!bND7P%$6kpw85eLX1+k+Gtr4n;to&WVY@}?!4pcuT1pnreN7)qQQ59t<(Kw z6c_osxJY3-EExByR~qILEvqjq&>-umbs~-{rFieq3!43ga|8oga@9D@^R#S-uiJRk zU9{cIHuLw0Y3d}8(`rhf6DCS!w}Ch=zW!P{t5v&&o>%^96%WUXBbLpfc)4V2X{P^N z`0el@tB^Ur2Fzh-yQ!0R#Sak;;YCb(4}6QuI1o6uZA9NPb#3p4y;&Hd?+15AOY;cG zT-HWc&n?;X>2B@#=^XT722~{yi|0iNDl!W=(W`r~I@N#hukMp@DlBl8SkXWfFyS#v z%T=5+cfGk~zANc=ZP>1x#Ua5Cdl}S@33!4dl)egmSNSD)YQeDDWr^*X4VCO9Ih7$< zZ`^*<>DQ>}Rm44isDpHU{he|E{}bw(p#PK1_KrgrspyJYPzDPoMA)Mwoq$i_vhRkz z%65KTt^ir?4OovSely^cLi6vxEz(}1Gafr$muTDL3|DfN7e>~ZMVKv8A!PtlOE5u4G)AZB z$YCh2X9szspvDL|QCMrT*!)NcF*A%rf=^81Cu(t-iwrl!@An|w&_N_j=m6m?8@!eu z>fxeSCnJx)&oN@wvQRj#1RsH&ER*E812~WHz=o{9VPr8l&)qse5zMvupQ?oEaz<_hN6tQ%9So?k`qGX4vzXIJU7;!S#irL^;YIY{ z2`xefP9_qzPA0Za7S0x~CQ8nZ7Pe+z;@955(ZJd6U&Y2EC7myVMdfvSt&?w2oCu8| z4pT7~KzC;<4lb3E&RC2s41UVPJKJ$howN=28~W^xWJr6~_kJY6y4|0?0uK_g+7mjS z-sF7R^pZ8>_xrp->Lq7~sa2rTpQPp7WSii{ks|N1E-*D38m1=dDe^|ML1CU)(h+P% zawap#FY^B5J-eOgDR$wJMb#1*bK@GKdXz=pFh_%ie1t=sB*UNNTT1Lgx)HDHTZ8m~ z5Q46d3njboRAlQ;3`>T=V*%EF{MdNnIe+OsZep$Xth4aDXf|Wavna4TGnXL#Q6 z{0A=73X1QHOx*8X>83HZK7&GFzkw4h;4B@x`ughiZ3@H{)Q$L)PG53(rMctwu2t^1 z>{7n0GIpEI*gldX+r{Yn;IA(yexGYj7vY5ciy{$_kp)}C(#6-t z6y}d_Kcp6N7=8LOnHY@UU}e-XMP_VQ-$ESF{G2q@hbj0ty&XA`N3<__E8RNl_NVhU;m8pzJX0nAo>{wcNJg>?{tZ>`>hsnl|UJ3YI_t=t-6qU{=VA zxhkPRl(xJ1UO;IWD zM8QajhzLYL(M++o2|FfX7}=>H(th%S%wYylqL3~9umZ!7La`7A$pe4^;W9*Q#_R@ypynYt<6uF-;2H$Mfg6*2tLbkdxr0k zbq$wO$vTO5Rlcp^Iyf(!xY2Hji@Rqta+?{LZ&+bwEF~)2Xf=SXlN2YL%c)!Plr@bc zczVTh@>F8qhS{2TC{HX`wy(Xp!6bQNgW9&iSBwZP9@fz^BScmZ^F_w1OInM$utM1( zwwMX>lYp*6Z*WEal)+O;r0O;iIq>HyZ!W>*UBjQV37sB)`5m0cMHLCUGR6Jt;xe&3 z-u&r$YiKWvXsi0X^<#<`Ce5?K_Cz+dH#ItB3|Zq+9ZT;L{mP<~X&JckblFdQ%)P&mOMe-p83D|evdnF%t$+Bc-r0<@wpR&TsrXU(z=;v_lZQD|f2sUMvvR}nf0pmqSF1NAfac(GcdnN7- z7ej1KARUV<8ZjidQb{N4a|vZMEimwU48oatW+v)$&OJ5=_GcphO1Nq6{p?22&`9&_ z%4`$a$Lp*-E=o*=>$E;NWT*cG3(ISL$fTkvwprr*SmI>0$zo$}zX8F@bU+a5K~j&u zq}@s>!ry!B7YjAn43}Zfn4>QMIXA)P!7;WT#AALzL^JPhmN^Yx`;w$@u&X-+E##-# zw_J_$@^;@T9h4f-2AomFcuT3u^9oVAqSp=C=iILNbB9LjxA*Ya?F5IBaM4ybZ7A1o z!kj1vilfS!nGMqmJhdZfV$)NQI{;_wP$R~jk^9W9#Kilc`OLD!)VprLEf~%oIkp=~ zD!NCr$?wey$rv;2pFaylr>+f4qm(%4c6qU`_oJftcS@rWZt|k|9ZB0?6eYw;JSW4N zlbNRG(P`gkKS=jyykBGOZDUrt`r{pb;(aiE5bvUVE0V_=OB+n;#oMm)gLbB`tv+U? zn+dYgQ4T^h*N}FBsnk-}DoDw$s@D~3P#$SCm&B;H3;^`o%w7{MW?9e`sUuV@R(wB9 z=kg-Chlv1J_-UY+Y-d<^Q)o(>MWm+2FcsQXgDhBa`^&S+_VBKN+J`8_0%nP%ejCJb4!}+Pmi6w}@>!`^k zhd;Le+^#IbSfcoTHzO8u9J(Qe$m3>Gh>@YLI$M+jNMw)XcHf6W)odMZ%x8 zaQ*F(Fd6XrA10aT7Z2_{uf1^+mp&=!*lPt&8FV=$U3x|UGAG1ALyLhU^Wyol8jR+~ z9`M}Rnrz1R3~CXwSdKtykMTA=Z0D;~)9lwGPRM``Nc#RQG&CL_ooV8qMb39Z>v*C} z2L37b_oNa1hS)>9k&v*m1;)rlyD1XlyUP-w18e~jNg-B{I9%6~7sW1cq(5*4#5;qi z9Z;M0Fqb3T6c}O(G_rcTgd32^XbE^%fm;zbbPfcqJ?dcUF27+Io=c6kSGCTx+>%-t zyqLQ~ABxq!5Fhl7lLbtubE)HgfGU21(PoMcZD*F$^7v?{dr8|{+{3D(8X!Pvdc=^T zceIq6AcQH!4O-NH_*9~p&64r`sCl045@;&nbqb0QEpbxr8L8D>K;5C)xmkn=ihGVe zuXCMF2n*Xja7J-(tnUV>tqrk`=vlh=?;xchx;`nkM> z6niP^sUx#Xj{VeDm;uk?v-P3=N*g^&4rjQls5YFS)lT)VVP2jq#@`m~Di(Xkg*&cz1?5Zb{ z9t`mjyIv8evr`OB{%b#UBEMD~H_qNk%1S1;*cf~7tuwCcFr#*=V1hI|Do6egGzRtq zjQDU+w+8d$3mEA>QHwvOCuVHx;YVqO0l~UFb-rBKtMm=qcMjD2bt%aa*b=yu%sx6yVF3(tbcA%hsc zK>SI!o6VEoq`)Tke)G;II;-G7337RoV8sv)qx@Kb0lh$M|j23MCj?#T4qB+`Ka}Z0NCvROAvi z6m7GK#!Z9{-44hHa!KF`mpcgCf4rXijw8{;tRf_HF>XU_IbP!(>H9a5a7p8bb1nUx zsv2S%xhbv*e|n^P)k69sub9d_aNwYg`Jhrl_I+w3Wj$ezgq@`v3mg!qnz{k*oW;x- z%LIZ(i`g0`1%9Q^6C(on^N|u}@Iv}-lE&)+f<^Ax5@c|OIRJ&7UAy@XHvev6w^j_T zH2%|F%ee=ws+fugAkAQAd~%sw_=L?XVTny@5mpK{H^yioCmuQLM?Le1IELX%+%9Ap zJSlLbaJShHU;6*4=7Sbhl85$0oc&)SaAywx|ijth6} z(04qW5}aj*n#0_y{{fNTfk`QSE%@pS`7}0S0zANO%Q#3SK|v*WXA%{<<7E&i0alE= zADWlL?I_)xH|83Hr46TmriY-Aw1572xsc5L8VLG&wjb5Y*jpr^)QT>MpJ2jjzKXv( zooAeZhvv#|J#W{UP`wY(n@hS*FUkmj8Z_r1!({pmMq>ba_Nvm4Hcx>{nEed#_e_g2 zs1#rM7yJjm;Qw!_%YWhC&A`#*zp2YAl?~g4ABen}g${eIq)F`B<79+hpKeEwo&l{nYvy!6dTKqL)QmTRQ#;y!LBu+=*Yog?fq8RdcqAV)x++xI5S++@SB&+UoO<*+}X1LN8;@W}J`X@EF~iYy0>^O)eYk zLzY1u)e@~Cy|hkU=_fi$G9S&B31pob|NA@vlr9o?)SSwxc}I^>&Z zg()1503!lK(il>&9t<7(VX=jalIXhMX?Cw@iW9O@m`538(atNU7G1U?RjKicOLVFezw)Yu!%@Tg+EojY+=)~KEu}XVc`(oMd zCZqJq)YQ~09X{`HR}xCcZ@sH)2djo(MXrw3){OKCc{4?#1oMdChu$D%Nm=f9!AN;T z*gykV^^(Dm3mNkiu6@prXhc4a7`t*G8nYS^4YbPafus`#Vcr%Ib6A=)AeY&TkVwa1 zu&LHhGK$0QQQQN=>)SfCTR`j2#D8rjZd$Cne7e>uVTe`kyJmlOOm z36QM(pW)y4YI#n93%3Tk+aGodI*9oPVgw}p9IZyLF<>y(gtej2N@4@a55OPH;QNgv ze7E6qxKuCtxEW5Cor+#(X2NIM>5BLG!T0_59~gg;Br;=0lDwRr7e*RG*+3wbeqc#r zbP-w!aVPF*3Sw0bZ*oSnK3%98Di}fuQ3yZYStU}6De07;6y3qF&pwY)xP>z zmRrm!{p>EU{v{q&9Y1$v^$uq-2R-!_^3}eiR=+qMex7pJ0@MTYfU!NIIaA<-)?-pQOJS7^SyCr)ITS zaGI}L+~>+a*JUQY)BTn!`o0HKC$XqFWzpucUo^(ezaH=VQ)yz<@W(Lt&+y!I%`StL z0m0^(8S^Kw=^T_>?P2t0i<|1lKm!_f6OE^+h)b+yvzMc~xpXgLa3UjE22Aq(@?nR4 z*rHGT_a1n3*BFCVt#k8;{gFV;6~;G4D4$3Wup%2+OF)f|S4v!nK_PFC$=pxDcG4-n ze0u1I45&e-iiN@R`^LG%0Q;6G`y&?`cgIg|O%j8bl{9BbS?5kjo#hp6bY^6Cu zoE_7v2qcIG-o&vKgcla!Lm_s4ba#wlqtP@y?I%GP$FfVPB zRl_jS)7M}g&`$lJ{~TnHnD!fMd>jxbuqA|-@>P2Ihzj){$||J|lW5BAIrz^MAc}*v zNXRNoF;gloa-2R)6ptwl@~l}TOkjS({hbsJOoh@)zD}wYd=-Z|{(XP=Z+wuPuq}xr zfI8gMpwZA?jQz>F)Vy5!tpIgJ(oV8a9;7HFH$E&LhU2e$^4;RVR2 zbe<*CKP-f`17kbAVfzo>bk_U(@h*lx$)X@?EmHr8sQ(m@ir}rfg$X|j^zSbCKD%Nc z33Y2KI=RrGYS!cYOncL|MF_K2nNg+>#FJ>$}iPTtW(0q z>BbWys+N0{?D;vPrKzQ*>G0r!PdYgYI+s|50_x&yp~FANwO)q?zcoYN%e~+(Kam&Y z2o5N-SaHOyBg>vu(KwLg3!iXUY85zdre(AuO1;V$$0~PEMs^~p=~B#(18oFvy%^tw z03G%7xGTWYQ}8}?hK0NZzVR?qA9ul` zX{+8dsYXj>;67f*{0Lvd893d(86RZ~Jf7pyJvOm({M52Vm_~i}3H9I~B(_Tbo!pRW z_Rj+l`NK~l6<@)y`Rn4vzg2GhD;WNj7yT>Cl&t(I`&DlIc=S9RcSzC%6;?osY!sVN z7|bEWwvvv5B&P)b)T&N(bnmjdf-i{OLwRN(1r8_Y^S>^1aMePzP83X>zM7o&zMq=B zdVjfpr|{?GIThiT&K_)xVT}-^D`G6z$EJmA;9^A$p@d;d?YG(GuGy^yc)>#9FJsnE zY`l@Q@La7_HgPz=uLBFo&4z9DmQ3ZJcieu(s_w0tEk$)Rc^ZKw159;SA2XAZ?$cnq z=i=iq>42#&7yNcXJFF%5iD=9ymGoLDp);6mxBiHcg3&xBD^5r&Rk{O}nL6v4CTfAM z>GyK++lMR zNFoTU{bTqCdXOLwE2BRw6GJjAt$w`0f!}5cceR>}AV_Sj@ebE8CMKs|Um9Qpw#ez$ zbo6aNZ)?-_;yLMb)GFT#;k5c#@&_4?dBuvxH^_2X3SWc}o`FrdV}q zasJj;m{E3yK}Jq3&sCAZA{C9Un2pLNC^u4a9ozy+6Jk<`bn?`v)YO5%S`Gj|#)zi> z;F%VvoAM*043qrToduSp6?QaReBeNqRMA{+!MrHE5S}(;JJ&Q3;^a?Y9lz!2-$fcajtyLM3zxXE3WC;>V7l}Mz&NQJ$01_YyP zY=spu7DZfwOkq^%n<$NY@?AObS{6ON;2Jkx%r^6;bvy2N;Q`?aB5d1nK$s4EqPVeQ zYe*Tpsq(3<*a?X#3bLkv?l`dMp*Je_Z#?ZlS(ZFK^4}PL4}4|<4)sZ2*mwWJ{{Ig( z{6E(OvXy84qbBfS>benP0)v)Et@)#JR?(P>FgOWO93_EVMcPvQfjvn^L)&%iD&@JR zcPukixARAS_*{iwB2)KTH6?o0?W_m?QRY+klZ$DOx7R09FR`lggOO+`JqDb|7ygwe z#}4BG9&Iwqi?z9VgK^p|I^3K3@^>LR;7xm%TChTF8%;l(V;*Tg+g6<@S+wu5``6IA z>GnCw01&Rdaci9LHZxVB-Iwe%zvk+0L{9#|iY~d(_3VWa)LTFq~qn0}lN0uj(zS8#|J`#r` z;=22ab3ERO{a91xMU7UI8+72gOdjU;3RNYcbu)ujfr~_HJ+3XPPs-}i)gOI_$;j)m z{jP)bRi8cgGhBeS8F{1n<@t1hqbynDMb~3sv*Fk%t=8e;G3pjLrq0SeM z9yx9J_Oq8zj(CDi`kwEgEe4sO7g>xmvB3kAV4g9>)(=&1yD|SSKtW%JkP3sNRCdT*p#OhNdQ8`plb2!lYxkV_D(ReJ@bDw>kwej(Be~;l0(r9C>&=4;~7dyg!=5A;( zA{Z3swJX8kDKW~LF#_6v6Q$j6Ig@y``^R{v9KmqMsdyd+(9e%PzHrNGAzGp+2cg?w_&y3_)$W7|y87_Hjm86E=Xc_gAVXbcAO~u$ z@E?=>;4yIIWt^SvSRhP5N4$jO%GUVe7&kXd15n{Yp$+%qW%aQ-~Y?YLu}P=oQB^v6h|Hygy|N zxJgjTJ!r)z!3`XY(Kc9r&e`D17E;$lsEk`Zypn<9yL`z#{-N z5^(yGEi%x-FcE26r^sv_yC_q%LUpm4wChPToYFGtC|5XNH7((E0kX~ww`h~jL~i9R zEW4JO(kExFy0he$rh9)O;nzRD44Nsyu4G*d29N_TwyNSn`);?%;_)~}!nyA)hR1ti z;-n;ovB%XOKbx1QG=d18pSHjhtEDiT`Sbm_gotqobpvQ|i!jAT#Q0^29Ya!|)MefP zmfZ^!7_VNwwd1hOQAQJO{fW)90I~D0wp1Gg{(xkov(rnl@&vqdokaUey`Gk4wrCZ`dBA=pxIc`51esF3N|6C(#az$JhO3L!=EoF*) z4tKq+s4j9(P9~oZ+V!`}MkVz@nuszTc_eB^aQWRgGNbHdsWW8HIm-#437BSzdgi*P z4q?5eW+gDi5rOza6w!#1h=0^Gft6KE&%mUSl`{of9ooeI_h@XanYoDmbuHZb>v9nJ zzu`H5Y3*zc3w`aO_z%g9^zWXY%_eL0y!zI=Am=Ge&9#S0d_Y)I>?w8e2$Slb38MHj zVd+TC9D=R=NHK^p`Y6Bw`k|nKZN@MGYcj3<$fH-tt5fv`YeBsJa75NXBO|`=zcS}0 zPq4z@nuOi*m_p#&~wb6gWwtX=5DDzN}XO< z;66)s<^DXw@vY?T!{B*5D-IZZU57aK#_gN)z8GOz!d(eQRf~Kk6C1%CTN*4d>U4kK zQ|e(`dPWoZ8KaZ`N~Sl3S&o%wD*9rA>Jks!6SZ6@KjQ4&pKTIFZsd}*gVKXqm+DZC zRl`AEN}davRe12TG9z44F7i6bn};EBb*Gd((PdG*j=bb7jYf$zy?l%n@7|<6Mw3%U zVS2Vuc*M|CK+>aX#U5HESGtut z71h!?4PQii zs%~V0bK*+G=n;(!kyj_s!-3)E(uppCV-tZ64SdMoNp9tdox{^*NGWUE7XD0QNOSPl z^CE9y)nTlF8ac2mCqa@+%Pc#MILXyE3ox#myZ{R*2vOo6FF#u9RBmbJ)4A9{v6#*b zvss`=HJMIB5AB5)U)d~Li80Pgev_JNHlLi#STQyl|K`FS#KdB>Efe?_6+X4exTO`y zRIyT7p|{oAR99^YYrYkE~o2k7_<)3W!lhl9x^D;gft`O2L_Ssc{kq&*grswf+LY2F~hj40Mad5qs=q_fuf3u8DgkJE6}P)1wQ2B^*^G}n&A zw%}*z1&J;yLYb)+U)TEmPfjJlLo!En1hv6#Afc_+rcrCv+1li5EmtoFxCX~Ss0738 zfhlz{u*B0W$bvxILTYTGG^gIIaBN{a!m+jzvd8qY`DB|*@&lnU;E|DLK(okM1OP0| zX6UdGimVrN#uX8;nn@h|_}0fB;PY~Y)fJK7Y$%X+`@>9s9Iu*JE=7?!4gXXbcNv;w zKZi_U0kasdf*wllZrhl`&mSb~v;Y46cSgLh^t-OXlSq0T=TwHO_S{u)goqkkx_u|1 zt2#$E)n4Vnm)tU?lsW7)cnS8oirJy>TeRkH7rkqw4U+xAzC8u|{mt$0C(A?EZhy&` zNCqg|nrP=$6adp6MsgmME)psGHx-b?FSM)m4=!i26$-@^&%Z__~iJ zJhM!3prxv#N(OH_SOABvNp;cH<}7fgGXMgVo>J+j#)WOXCT_8U9(R#adKrY4=A^s# zdR96M*n$*5G(b^zj!oA6%kb z(GhktLJM}&9|vx;QgplTQn)mLZW?NsyEU4e4$v53vE$V$C(na=0iV#<$W$n^-WKFs z3TV-uid%`caw%!$W?QorS9irkjxQzBab!Xwd%b}ThZJhFG7HhC(m>*oMeY~D5A~Nb3w0BzQRmxXJ0@$(xHm2@ z)oTvfw1%iev8c5bl}9<`fNPJPS&Pu3M`JlgD|)`|G(v`GsSHD2wyRdN9B4Uq5q+l= z+oV*(bWHT#h|4^*6teBR1Z+~bVBO)7QNM1257iU3Ot@*NxAhiAL}M^dyXZ`^dQ=B4 zgleF}@lYgLUCU2p12)OrD59NfS?af}U4UaXIHh;jVljofDV~he6fq*fyil20NU0;> zI)nTeja#%C{E55{bcMPZp3EI{0=^$Y05)@megazw;mhwHrQ_$|kA-pJe1rRwr)@nN zbcgtqZi_b;iz#xuXE>xkIzT-%+-8(b?uGWwrwVF8hFDk0i)jkA))nMtSl(%3VU(%M zW8~>A>OiTB3p+BYZzbnIbG;t6rdibZT<3UC0(cWH61=R}CqsoR^#YFv1*nIp@wn>HgVi#DWsmjkRgC4`40=}sD~I-ik5t0HlC5++ z{~lLQocAYs3>7byI$R{zD+p%ClASFQ8itWR#S?}Fd-BZSC=7<_%nyXQKDjpQEwA`w z!9fPf-iqWfaslw}jRYs{4LVpwxLsQd2ln zX~4RGc{G=$*a6}^6VZ}d69aITJU;fX_1Ln7W4WM_7ZDw1b#1zbH(H7WvkY}QJv2AN zmh$6&uOPbw!kjRnu(YS%rf_L$_gF?cu_($kciT?l4s&u`p!k1%#?m_$1J(r-&X??$ zJ2HwjcMXg3^vX8oJh!^;vNp)QtAD$^_^SjXvuYXVX3nL-q3~a}xVMO3C?t270{}EhGIBWZst(SGJ+1sXf*E@XwNFE;E zqiUdEh~o=1;}FBEh@AEZ?j@fNfS$w#r_8 zU=Q^NH%NthIO$iee=U(5*!LaQzPXLR>{;`G>@beH_wRQGR88Zjen&jsmx+eUz<1rV zc~k|qc_C6HRt8<7S?j3(}tC-KfEW@wF4Rc7HM`mGJ0GO!|p2|b>v093)_!!d#~T9*V0)n z66D|0pQ>SN+`AD`*Kt(m+frMKao|#rOK^~Zb`d!2vKy^GD9Rffp8S8_kWgT8JV zeHQfXsBLtgJ4wnk!90A(sa*I1$We%j zbNiy;l#nyH$W2c4z@%`)@6B6$M=s;x2OuebplJBs%`-Wg*D*|?&SNSn$=P`{TD=62 zn9A!cuO}cyRnkyfj&Kmx+Kzx(lC6AzeCsORjOzF0HK$STleTCtpVLfGO)Sw;dH;Dl zPx+zXg0^y=M&vA7n2oHySj+?UDbhR=urj@B# zwG|ExFCmlfd*qVRLzWUoIj=>avgg@&fHL^rd;of%i=}Enyr73-Xw8T}jLsDW3tU+Z z*F7NWlpKGmh0iGfADN>h)*E+g9HTHxF>n&2;5d@9O8FINjN&3jmOmdv(*}w$AssrV zGk7Hb;SIrg#C|_I(WGSRYlc50@P|{FA-%UyG?zuwl*WVO4wtD?xGKdNp@Zk? z?Th(Z$?60>oW!O)R5trQ%kA$@Z3SB zy05<>`ATfbo(24Cj9zpw{ZNBxB(FbmbaZ}pRj{^b>6|`*W76ymx#So(IA)*`(BgLv z9i20%h=4n_7byT!HF*c;F!5-bH(RCj(aeN8kINWHRZV0R8_G$(FN1-kO3#vACmfp6 zY?--Hd3y|E!koyef?p0iTy~4-6wZcB8yDVIZ|V*=lF5z>;F+B63%2ZveBDvL*N@Jv zn};BbtZ||%6(LjGV74jx#O3&yt=Z z%bR7Nx_e%cgu5r_R#a=pf}{pKFrvKN0;pxO^SBpgu$+Sx_t>Z9?#)jZZ!?2W)?l`P z;Kg|I4X=oXL}>_n;2WfEGj^Q)wQuDFju{h(dieG&$!=UdX}4{=Am}VA#lCHb5vLK} zfa06v#ov_Fu8YWTxFKIJRJ5pPb1Da^3B{1w4~bH>WYqdh0hJ%fOqvopKWF0Vb)s;s zR?uFQJn(xHeRtq9U+mC>t_BYMj`I5^MxG$vf=mep**AkI7pdxKiFe|=AT{RE9))@{lFvp#oPcl%i>YNQ> zM(=3)JBAi?;+{RSeyr6rOJa6D#DUFJLu5Ylm#HY!tgYKFz&epNpmXg&tPv|_XOgXR zH%tjmXM9xQG;zcYnr=Z7i?pGZz%n(5|JXF}tSJG@fz|QQdw7d*c+D0|_cQY3_FMMI zNGAKF9(!<3M6~V?N!>)yPLc*-J)n=O92h(&d1g{FiB zq{A#5>Db*%b`Hx;Ctv_s-Llkw#FhZ5{+t*jZ$_}Hak!TP@7xm9PnzmQjxwsoY82Do zsOhJ+_iorsCr)L@o;`jm8n3`(y5ly;TFdb3X&(Bfn6|=pDw@18^X^GHgkeDFLgJJsiK3i}?l(&yLE(3}!r3 zm=i7CdxC|zZwZI@iMM|dG{}8&b3Z{_=E}|m^X@xTYe(_YPw<#V`|uzBjd%B8bA~aT z)8C@hsgADwBeq2uqR=unQU$(4Y0UN82@7|8jElHD?jU zJgf;642ILAE(^oYlz^*ER^LMZ`KJpPxZ|y6nXl-dR#KV~HKNS7S)#vC^01boNV586 zY9ppv7&&$r4KZ~&Gd$Bgecc)RH2OUWnH{T7Zccw(_8F#QOnum2saAagE{NB3$38MH zYyKSGHgej-pVb)q;GLXM)&ZK#=igc%P4RA`CLh-GV5+#epMpq8o(t5jMhFNi~F zNu-f8Tg$UDUt|3a9g7;T=b+Zzx5v5jJe-r8nVEXxWlAEs+)6KcGP(IQ?UnPieL}c=8Pnza%12E@klQw*mR;8GPRsanZWwt6% z08E;W*ldvO0!Tzt_N>IM(?6l#nZLgT)#AnBkM1clWQpX|4|mY8|ItRLgYm=~ICL^J zO}lsy(Mpo6iwvTx0!*Uo#^%T9R*9)Okk-G0xBpw*+F7=*34=Y=nMLmyL`=oloHSNc zme!d|d+}YEGXXDk{%!!~+=SsQ(*}cDuy5?Vgm{{v;$;3iLho&%c@gA%a`rg(ci6Sb z-h_5#I)#Jd`S@%}X%mNG3ith6@;ID7$?|j#%>|T71U!bTOzwpm2llyO^$*RbfTnE#rFdgpc5p<%SLA=rYaNlAhPn%d?$*f~JCuy6G?%aYb%gHU`-)aqL zQCMF?sxd3HXUt$mC)qJ@HC&dG#hwsp-b}OJ9)>~AMr?-a+AZ!TxqF&u+eBzFGB^k(jq3`ke=aaX8Dbrqq zqo%?H)IE1$Wi;}upFh$NYwppS`Svwv0UO}}272q~S*wnkQH15`$VWOP?qBTfnD5vK zd%pgVEZNcO$=yUy%eUV=a0G{d*5;EUGsJ&5eNTZHQ6c+COajsh2&q(lZU1@nx^VS*Gxc14v&}{p^2MBSE(Y2-%iY09@3x69|<%>!+8t~miGdDsGns%G8Wf28V zq%FnChI2&+c;2^v;@qHe=de1IMa8Ld_cwc&1!7EQR+$RjSK-q`fogV|kbjqj zaSvLP#cmWVQLK+T*V8F*u`J_&IIk)OMeqw7sVyg5jx*3lt2e3>m~*6i&3 z_;K#e8_7Q|=*xh>t7Ne)wiz7X`hDlk`q`Fa5i_h#w@ct;MY16BKV6fVS6wnDjYbcf3>!h*nF)qdEzrSC|+K4FSf>es;2hm z%J~Z?t5NL0A71}qOET4}w{F{l#pxMf7TjO{50up)tjpjiZZ|ZD2Y#mwKCIhl<}0m zls;~Hxt%J^l;#GPd}iNukn%-4%Jbb-`r{=g zMzhnmWlIr%`KK6 zksi&kviqRW%|D$&L05tg*8_<3>nHgeRf1PKFTYFbBcogRhiL07oJm?@6c|(mJL|Wq z+Z{nA;Sz!N8=~U;BW|fX{LY4&UcnecuxM+geSju%xwFFCzhLU{M3DqDohHdWpNtCC|0?m)ju&? zo7+EZKbst6)=4T%SAR4Zxkn3g#NfYBMMuYt$#9S$+)qpl)=M9oQ@i{bD7sj6Rmn2MVrZ8#b8>86lcm0}etE~gk%_ zCRb4E$SXS)`M^UHEWziex1HXF0bgfujw0DfAGydbVVzEdy9IRiB@d1qmR%eXc+Znp z_15P2&vbg?KjYix4vjNt92i$4>OQ040^0)y0r_A&}4g>#f@A~f}LDW~9 z>mna2_g2ldsGsp)IWT>+Kx6sx(&ukPp=M$#^y&e9E&8qs5FJ;=SIB!Mw^sWRh^eJu z;)|v}i9i`mJ#`Xb-a^`j#|-z$=GhJJjt@v`e*#lId3mZBvA+QfUTb#oQE>Pe$vqQt zYg!+*6sf&t_f2Ul)y{%LTgiTRI3x_R=A@OKDt~k1Saa(3ZD3hS;KdZVSW=Kaea)U7 zI5BmOX+XmmK#d*;YKzy_ zKc;jHyc0s-F& z0yZR~HYRbaxIg(X4d9m*z*r=5QktW4|Ni9`y$DyKf`MBge~?^~TVlr|$q{QbLeb^} zl(eB^RpLwTQjVqBfG_FKfgW?&#XUhKFFA&{gE}N|7D^)fl)J z+B)PfmMn|mTbOC4T)M@)!$v2`oBPDMp-4=6kk%nY+>nh4zK*1F0li5!FWMWI2e9e} zrZK*v>?h!Jvu^q|N8vB71!V@P;FtzHuNU{>PJ{pW_!%x z%s$2-LM%}pLBHJ#IW9NfFMr+H$5MuK*GtmqTph}XQ<69SbGcqQ#iz5{*Pq<yuKFAMH%js8z4gl9@ne03FF2%Mtv^5})b((zW zIPO-*bxEQWAy>FAP-`eiFkT{Js+O*SQi|=FwVTL^*{q3iKVDVXbimSs?~6(KPDGCl#)-aOL#ClO~vqP%GjiILM++yU8>f-vBU4JeLyu*K*l z-wGm7JfSD(wPN<}HO}oJsJ#RibWbHf)3b#q!cHwf)6;~hpiBR{l9fFm{=2*mGsDLH zk2!t&+WXl5=bS3KJ31L#|KCmNgv0_T&vc;0{CbIyF++Yw!EINh)t=yS+ zI{rc&3Cgbpfo%y@e^`Rv5M(m*A7%W5>Y0ppGoMqLZJj=y-QRk@S*83$j3Tl^%~KDY z0<}WR75?cn&h={s+GzbHmalnYL@amR6n|T43n|=#!y!@O#xa6JkW!6FX0S4|*0)za z2X> z?x5U=DyrYXHwo)%Gw=GUVFD(0{woW&s;0SG_^JiR0kg+Rc+?RJ$Y%gPSoI2fPBs}v#(#8d<~d#9?MIO&;M$T|KG{;DE~gC|HR+Z{`(pz zUo%<#{`K8E|014I|L=bgu(fq^{6fpyN$Z>2eD(JIAE12H7rPrp74;)J%}5Uy+>kFI zJb)FMCah^$rM6L`>*qJ>Z&pGZ2xEkiGA8q5gOWEG9`BIdk43WD6<*Md)$2W|6@cY5#@EY@rz@|=!yD5cUT`}YUh>d=fB$b%<8G3}!r)!4g}V&Z6g%A{UUh!GnYjqyt*Nh#t#(Ksgq2cxHd5imifFd zzm|J+-*EW*-F1GEDk-uGgAMwiJ7{R@zF(_&PYg4K+){S1wuG>m3^{M;FPy9D6{w(Y zfd7E1RJQBF2bTFG3(8Y1ZY+kSvxkkD|Gg`c>fa8=-)o8qbpm}fJfnB@ma8hlB5~yn z25Lc(I!%8nHi5SfWE!QGq=Fjsfep$I$xRh9*|6;Lv(*w5`W5?7(dBF*D_D=#11JIO zZu$Macv!B=spMm|N~`1*&wK#U-O zI-(S;r-kawjCn9xC@jQMuUa(>;`%OdTu)-CKXQo@=rB(Z>yS5x-Onske-UeWswQ}n+K50(vsBASBfNy|5*2i? zcWN_#Jsy<5wTAr|DZGy);%`R7Bz5CqJ6P27!viqsEr=K(Ci5qr(@#d@mHhczzx|Ja z`#IhyZ>@fP0N;RhzeX-f+#4Ei-|s&LtdU5FKSszEz(@poaON9fT zj#CQEoROVQ!HYgs>HEL~)8a}{@)#9joK8h?@z3I9pyMr0Bk)W2b`6F0_jK{aFH257 zx%%?78hLju@IqE{-_Md%W)~#OpyQc9xg~04=Re^3(4EW*3-WMKz|_p7ZGSLc&Gm7~ zbs+r?PK{*6KpxwlIayG{SQiJ(2f~%6Qv8SKkN!bmp|{WfR>y8YVay@e7uqeD+YeAU>L>zGS)FX zV)v8I%?L^_wjjhX-A@uz5684k3=QOA%?(Uz;cmc?_K#nj3Fa5D1=020>zRavla-2E z>~WT3IArA#93KOdtP&$XuRo=fn9rFi3qlq?!JIhWK@MaC7H(b~5|oshb=AA2v|ZKiNPH z)K=QfYO2=4hk|^Y`7RN3$rro%o8$EW$T|9{GpgHnh+LsT2*CjoEY}QxXJXizWT};9ojCn#+@Apr6dUD~ z%Yq9YFKo(mJX(eH;@Br{2B}tNTL32M9$QU;wbqk-Utb!a?SAS6NCIG8&Dp#iQ7SHk z1)m=EH1!00^>X6A5z*O6%LH5-A!+l3N&f=z`XE;qHpa1JFGB!W7YGM<<2)Q) zvjfNll{LWVly@7}dyq8s?>)$^UD7g1{piD34Y`Fw8-ZBoU3>`U-7xp-k22d_D#*K;=aw~VdxRMzqr z?pnbCmJrF}@BfiAf)d9_TKokscfSgt(@Q5K0p1S+2fUZE6IOL;W)bp z|Kt4qv+DjY0kfwd{AWe*r{ch8#a@Pmo8Z|!HOdZxJ_uuRFDiW@Wnk8GC&iMdgfBp$ zz=XeDJo3lyL8c;>z!OpX)N81UwBV+PO}^?@bh3WCX4^_z)k#g$AX z2IJ*Ff{mIeETz;drnN<-4>>!IgRPL&xT`PRw!}HJ zT;~1ldQv!eo2U+AIID9oo8J@3%uWf|I&Jl20grr7AAQ{cpl3*nt|gm|wjCd_S(9u_ z;{pEd)qPzaPjFm6F4((Eu+4}lQl+Az*tv|IZO3sgFz+to3K#6B8_3A;Y6?_ouYX{A z-ZYTmmd|>u^H{o)Si4pdAA3N%MotjYAhPRl#5sURCFod8I?kVM3$43qIlE~V4Y-y} zw~Zxo&+~?4oq*p+v_kuo1-)`MA(iRSS@2j^*i6c7 zMHXU3o40*8=*yd(Td_h@$e=7mUAp#Uc%8v)IAu4ECmm_bWomi?Qhrl%~TVE^>kXMYGo4O<%(Fk zwVh_udy%|pz%!?j>|F^6BhRnTpuoB-D=9cyUaO~BIm>9QCs%(wqDCaoi2w)HqUb=*0NHn^{bFB z?&cIJ@@ivJtsx;wIvZMjxbsnm`h24hkrAD)-cl=cfQdY!X?*hy96GnX;gN4*Miuvo zMJgki`)OOEsAa$L$e9%yz`EQ`Kx{4;*H!47qxLl^Vnl>#N_e9lWtQO5x>9>$>jaKT ze276=Bp5kj{LM&<%aw;x?}C)`d&zDeCpZ=(P|Owy2Fz z6VL{qgW5<7B8zjZOP@%r3E#FAZ(Y9?&v_^#$zdE4Bnp&)mE9vw{_+>1>W#`1BUR$kB zJYrFP{GL2`SV|a>3^ztExlLsM4Zuz#3SiIc*Gu}9P5QJaEGbjI=UsI2-QqD9+X{i} zDKQAcPFSuWKc(47Ej0im`U?B>;%oy0Pl|Ca|1}2xD#l_lGA+Lk1(9d6Jq1Ig&19(_$%WcKVKo zy}NHJeo>PgRFv}|>K_~2)f)cTy0>a1dSs;*VU{_0AP5U)CdFC9TUD?YbFvf{NaHMU z-uFn8DoRd|xf_TF!Q8BAUm7pd1nOnLGN+7S=Q|g@W)-V;D@X!+sr6~854YK!s<7?w$7>n0oENFlerJxqJAJ37j<^X&R+QB2(&Amz z-u9b!@iqbH+R9#L$~sl3i87{$yU}9I{$sM&)oDK}Pm#wHcb(RyCy_Dj9Yi85ovidz zu{#*&x_oh$BFAS%7EMI4s~y$d^6dE9ej|cnr+q+1y`fRku|lhag?eYi6`_f0ljltY z$Y>sJH9=BUlXqM(HgZQSmqCcme4QS+lBU+ znc7lvpXi>0M%A-{{hn$!Q#?&~@2YZp5C&6Sbx)X&yUcu4u~P}CG0J;^u-0J{4Ij;U z6|z0oAy=V%HQ^QaFQBmPe>_Q*>r+IN8glrTY2{3KXHPgbs__Np4+A*TBSz0iAVMNM&m@-;E74=hPI^8ky zneeUl!&lyO^Nl(S(?j^-0Gixyi5h< z(3hIjfdDa*L2mthW+N10GqR`^9EnMvn#EkZ`B(;UUSC~0!C`HZl+9h}EYDb^?5pHW zchtmD*#wpxJXKNJ6&-8|HDrwKtg26;UEV|k|J#SkYc`nlbhf@~v%B-s)l5fSV4v8H zO!%-x@tj<$LEgjX4E*=FG`UW+O9Tm+7)L+&6 z!GQc2!?luH9#2DbR_%dcH@5>O8*g*>_xg3%$H$i==NI*s&S?3JQm>r^Gjs26OMBAl zOau!;rXF|{eHTy_q7(JAL0<^keA3Y7Q1z}#mWX9j)9r$%M&mjBvNi-pg#=;I4QHkV zhl_^RGDrC%XTd5_^sf6%nf8@i4(&&d?&6%b*zOz(@Xfvl8HY{br=2OVHSu$9xce@+ zahgj$j8B&WNRQc=E~jT6IDK9bv9#WAzgGEw%@aZtBJz$YwF_07+JW_ie&dq8RvmrE z-hU>FKA2$htJx*#4u35Z#@+dO1)urD8%_-VIz@wv9Nj7l^rnaf-VKX4n7~x7Wf!_L zB+OFo9Q+N8hCv(OD5xjpK}Ryow&ZdFuz;y}TyncmTCjNcj5U33K2`Co-YJ_@xrm_< z-ry-=pDbBMSvFvJFWywzIDJ)$ghJ2@d4`v?S(ED$47q}%W~kTTHv+H4a7*}I7W28* z?^}R(EXdJhEf2;%^3;R0)0Eu8uBV8=B3u=PreA)9ADO9|UDTZljL&GVe(<9bS4*Lt z)I&t1XA*uPhO&EAPw*7*7|;wAvhm!I#M^RbisREPdk{S_)DC|RUm|Iz$IT;=G9CVm z$)01va47VqfUo1q(js**-QBE;RHe=Ni{9kmOcyS|Y(a(+8iU9#8DdNac)J;EZ6aZwMvk1lmKS${4)(2pAr4 zGMT7$`Ed-Jna_p78|}AvB2bcw=kZr-6>_?m)}6>%y4j+?_P&}&3&gD13EstXU|S&oM5oQ#@5G!R}rzv5S*CB(4F^0>h4+j{9N zlHCQOzxnrKplq2a9oMGWISAwD_c}M7MB7x@Y26--sHxY=f+u$VM}7iQIPc&*;4c4hHn^_R<^Zzd$@iHqiTQ zJXKtw_jS1) z|9bu-o9;i0MqdRLQwL*5M=A3!X~yPXxvtJpeK7ufsKPf?3l?ZX6OY2Pd>hHau~5Pa zui^y7p~Q@aADR(7xiR5P>n<1}9qgI^m9x^{8*q{*n>tud<3M^aE1{V-C5;X}5(os0 z9%_gCSFz1ib0R{fKFAcVGFGog z|3vLS&6vpOWYc1du9W=gG#ia8fFbibYq{$8`X85iLUvE6<}a>PAl(1Jsr+laU*UUJ z&W>MfX%TZPV}3)!uOUn8+vuDAYs5*a8;&@tSf6<6^XnDPp62B$Q_?aTl$>=*5E9{5 z0=O$ON0OPt(V8i-PSjbeI5jRq7HA`zis=HH3SNd*tdBO#D}tl z#6|I=+wGkbfk!5861qiL-k`F1<%Xy!&VW4$xbu>$K_FN%5Di{;>6Y2=5jd7t2TS}H zxCg6fDk}142F58`mw`4zeZ%RvL9yGE{q7yKO0ZfHC_R!xe^riZ2CRyAs`wBR<~sy; zNz~q$=Ip+w$!I1Th6IfKiGdzu`0M!;$>%gmtx#*d>2O$PVJ`D~;W(2ziMvH_ow3-o z%rOh|0Rvz$7TvM!>9Wlj(L71xZqf$my1Y&D*RE1aISMtN=o8>qp+qS*oR*A<9-i|3 zNtX(JP`g>Ve}O2gyi?x*zCxkIEz9BUGTLZuT-W2Fy_n#bV$`QPewb8$M9nxT-)__q zB<#0ESRmLH1ioNjs0|HA7cQJc-nPy8%cKB`Sl})7j$Y|r?jx3&TIb9X$LwU~|Z_Ro0r5sI$h_2J?a@5he`cp|lNP%%M#_IppT*q4(bDlRhbsVg~?_ zNj>6|8d&+Ncnq*IdrI<|{xw=_2_@?{8&7wY>N%Hx8tiqlu|$gY%4Y~haB?g{xtq`V zB|!$*P*JlOcWMa7BXx9_L0Gd9=Fx5{v}H^h<7DxV`0-=VN;1zLwbd2}Uxe{UrAwe| z`Sm(_l84kP7f}@+caZR_SA#^a5^<^*_PM$Ddq&5N&{BQGJmW19u?_R@H9^7NdV0tu;P{<50C1x1ct3)DQ3&tI985QCQ>IRvUHDFl z*i-~lsr9Z@6~E8KyPwjCFtGKhIt@oHJnNS+*{&Z(7; zQcuU``Ef^y>ib92Qco@coYk{?s9|Z7%NJG1&b4L0gAPLR-QH&IR(_lCxn&6awP3vR23W#6oe;PrFZ-X!#tO=_Xtk=KPM4#d%L#cB%924u_Qz zr`O<0$%MaQ%2pH;mr0YQ_1&BaJ*qDGnK6&Yg*9EQOtS0qm80;%#mQ9N0Q$Pg!Wt>V&7kdvHYxHzum%r`pGc!wcaQ(2F{S*yu=>;PAyUY!&HXhH3J`- ztxfQAzTx#dQx^slFM6nOn~VKc3_5Nv0@q{=`cJ&@y@h9~=b|5}NPRL`;dY;FJd!l& zVQ*OUjiCEsXhNol%Re;vJs%;MY%(m(&ZO+h`da;#6}vj8gcawoU9?6F&(|K&k z@wBN!LiTogh#pO}qYx6#{DM~AS+-??vy9j3U-!?YyJk76*D8Ves^HrU$z53OuZ zzO&xy(Ikg2#f}dt0R&l_Fu4nY25?wr(oDOO@`cS-V15lESbbU9Jy4 zb|F|1wgfbOmXwH{!l2Gq@R8wl8vZQm$&j>~NodVtT&BlvN&DB@{=aslL4#vkO@Dtb6db)w~7pyLcYT6>q#Um)yJ+|dNM6|=Q7ebE+ zLJvQ$cf_|v@+N#~a3QZMZ#BU*RlU6+GNRMgmb-=W39A9fsjP z$PI50^lxSggoG1&1dE_FM0IN%r_r>55OOA>x#iNnk6rJ9#K{8Dv1Vj<-3!<+zI<$v{S{b z$d#J|AFm(C_0a45KvOZ1bHa#xe<*5p)fOZQw$1PK#vNZ^WL30Q%%_c9mmBnV2lVX3 z-7)(8?;FR1b)-+aY!Wnl9CTEDCmW6%)7^hq&|@zH0q$QyO7NGE^8cc|i#q5VSs5#T zWvYqUS{qB*+FCl>{mXW?A+zzt6@?{niS%1cl!QT*pde`+1Ye(r70%b=BhpIbE94UX zr4AlYM?8|3ue%fhRx0LoDcDn1&T$ba`4jO_-{kS|a+3XGi}%y}1C%ZR99%w9-0hd4 zRv*|tGBS)nY9dgpQdel!p8{GO9A?<)l&B|>p88K@pnC9!v7xa&Yg2-*G%Za@H_E4ivxSv+|KK&z#%mSTSRkzMcCNnzH z79D|g(t%E-p9;dgTW=+Z607ALt#y}SFg;2<+?(?E+z6e$tT{>Hu!IVyaOW!KvCLvk_?lj*dcDF8k%8)BY+VoS?pe&Mrp4w8 z@78|@WT~B5fDA2%fW9%MwdbnMXqHzWGiIlYEzHEVtOP60Mj&+ph3KvJ2-bwKn&l2^wZAU|Dr2awEqE*>YnKXF>>1;nQ zSaSOCJ&?;j;&6*7-*0dV(RxrZ7a2WZQ@-X^54hHC4?A+SKgQ)NZdIi@LN=ko_Bo02 zE2$tTpyhE3O}T{HLnxy4a0^reNTrZ}dBQ^uXny2t_J=~GRqCdvk}mDD+I=H+4R(ofnohE>1pBX-6P z|8S7}Q>amc^h8;~`s4+?9^UEWV8IdXhMH>*yY$1*gV8mAgY@f*U?U+6@mvK}H!Vad zP{%W$kNZWPCGNVwBAwApVr|SSBfzmx>`~}()8)$e;^P`$Y47Xv4&Jlyy4|w!s*~;c zxHke)iGHDMHEfP{6R6)08KM=f!4|2x23arFoZ+8D^0l_+2RINq1`Ut zy`n59HDKN*M7&W3%5+Nhbtq3d_CA%ZMn8!)KTRBUIk@M9L*9CW6Ug22(@u-c22h^g z#`+_csf8m}s71z*i!FY8%m4v_JtI!9D9@dB+{YllD{tax!AW+;J8aetv zq2vjZEqi9lmIGpWsthGOXZp`jJ^P2lKj(*Q0!eAR0npw1L#k+=(jRZ_T%Hp6-hvM9 z;}NzJB0GQfH(0(@Tw#%)lc-o!L-qMEjczkqyxkY8YWyh1xgAZ z6#$xz;FvmJE)bVmN5<|l8v1k*k{_%z4|g%g&y64V$~0>jC4vk}C1~PV=uA?LC>phl zV(;a-8QOAP+wz*VWy6hJAfHX|4Q_#hfSO??xpF{!sCwW4$Sr2qEgJ!+#ET4FYv)F0 zcGFg7?7({C#4bKP{Qk$}cM4v8&fi^qE8S2i53u%_@&C7(W zfFy01N6$^ym$EYuC3THb!w|m4Iw_lrt%wt6HcIW&x(@8*#E@=nJ2O}rO_t4tgur_J zw6hoVX8PmGP+LwsJl9$c(p5*7>^Q%EI9_T)E^sH37Ozle#W4t1;hfu&Ve0RIe)8>A zW(e?Zw2Y~4;FXNe)2>O-(xBdFJ4QzBJ{wAAHSEY}09$_78nBxP)f9mP&4`7Fa|OMR z`I6e(S=qi-2P{1D6)(q=5Xm3t5?Kw6)Kq86cyDxo@h;hY0bIGUSlB6zax!U zH^n(XRj`&PZ4vXsuI_8X7O`sx&af*8CT@+hJ{LeM-a~vQ##1>}IzEs}ha#3-w1BK) z7Z08)kfMb5>CZuX6CTh7vhL%ekaH(TVmjTXe-;EXzu|q(-pam7_VqBoL4F!5$Y31G zaMO=B&fPM7^;-b47h-#}ux0k#u>qS!bmIb{>Y%XNR_DoH+}Or_?YJ)KWCrqz1SW{~ zGBRdv^fm5sVyy{|^~j)LNK$i!ly&|BH3;iH zM&L$gtm>`hCmzAv=3^@-Lq51y-L1t!Ec8O&^+IODWgP+4*YOIu4@4@}jVrYso0!WM z6pgGc3M(>5Lv7JQp(G~lh`yTFQM1G|i5L2N{T0Yd zT`8=wUJ2!3ftFI+KUfs5Ru#tFDve}mJia4^nAs2|cJ7?kC=tS3Amgn#saWsBWRq=I z9D%J8=5FxQg=skdbWDqX0tegFAu1)!T#?ohk7arwpUjarvn1uAFikb`JnZpV%waNA z9YcY7LRAJcgqV1Y&00%-u_7WW4#TXBdth=AdxelVWYWg3b4Ub^>M2?~w(T-}1c8(r z4caQB?<#&E$jiaduumu2NW4P(kxy5UkS0p|_g-J%czy5_#x%6djfN(Z-lW<5SWD%u z0^qUz6^gS&jzxbZE2VA&Ev^khOZJYVo0^Z_ipuQeTg#gr+TNx>wW&=@Ahl`QMXu_p ze=h7qzosjhA}J+V3HI@B5ZU-XT)m@eOv80%YKrO8pL3>4?si<}rdE3Vr6Up)*;_rq zfyn+-7U0rMWa6=^v>@kU(7~$M1we=J5+tkm4n`xrtQpJ@BSx)61J<6r`T7m>|EcUN zpz=tXwF3cyyK8WFcXxMpch?}nH9&B8_ux)&cLKqKySxAI-retJmz(`||2fP#1BZF4 zyQZh7y1MGANW9I|(d6&9OO)#zUJ~9=W6pg#A~M;BLh0!Z>9aTsNvlJGR^kcf4VS=D z>|V43&MNh}iz}b)_o1RNucv$25@vFpCR(*TcPf^@yD=lEZ5VRAb!`g2foq&-6t>BL zD%WOm=~K&<@$?7>yA@$=c7(pdH8s(k_~Aj5uPk-5py)}Tnd;hpp1)Q^{@{K3xR-dc z(G^}=#Lo!b+Q^WYRO-L`BwZn)6XJTEqRGxE)xNQta{01lm#Xf%$?Yas;LgM@6UC~v z%oajS)^G9#_m)273BEC`MP055Z)jfC&c}JIJ?7yp`vV};@SMNKx-Vi>$Jw`#UAUP&WpdL*5-yHshsgqa6CgtiT;8&f zL*5;Qm-24G5}p>T(fz8EiGB%Phvs(qAgbc0&*Vz64l9ZFO8j~RN9%+|)SNs9%y zC<4O;bWvn=Ticc1Hkm13PTL7-O{T4SwUkIw? zt||NuSKwiiDo;qt0dr%>t7(@T1DZJ$rwa)U4cQxJ10FxM2$Hg1ibEl&>q&FfHHCW< z7-#`m18$p6uy&mVyLT*WOch_-zWa&Za)-MB(z+ruLwvr&t`mC7L{3$1CGiDx7-iwE zcRL^x$GP%G5=fMJ(a{H?d--s`Fb?KNV8i}EUza1RNzdIQYmIA8sQg0IlwRQ7Z|`T% zZa$3geRS2cBEK7lq*DEHb~;zl3?1?wlN3Hp2q`@ojCt#00gj@0??;)Ltni9?8!Y96 zsxBBM;Ok1ldifGh{6f$Q$sJYDw@pKLt_AOhn;ommx~`!H)(fb@ubEanHP?_+?&j7l zue4*j@{!cE2W(tnTkh0dJlhKAxuxdB+giM~$jij_tsxH-)H)7$Mqs7&yXB@qXm}+N ztlK}4TQ$FfBARdw+>y7!rGU7WHW&t}&yYTt;_WgIOI;pkUYNg^%dHYev8S=oC4A_4yLh{oPm_Cl|y=mG3ce&r(`pp z1=6VM11Ib)!#s=KG82voBL_+2n&f(v_4B$_K+e-`noUEMh6g4ZK+oq>6rZ?r64+Jx zskHl~;HHkKp}&HW2w^dLFm`C(EsvEQo9O5crX{q4D`-T=HB5`v3>4$gc1l|FvaA`8 zIP;G{-tTOgS2A^8Fj=51ok1s%eXscXkOoZww8T&5*6X+ld?#7p1MFI0$ZM1cg7MDF z0`kMj7%~v+gTiinXPQvvNKs5$Sp#|+5o&c=0b}VHji_ikVn`se;aWv)5+EodUU8T7 zMj|ZbE1!PON_2Mx!>G)Uc$V4H=U25R?$^<`O8|EBBH$$Wwf!vqpMb&N?Pg(sueXa1 zI_PC&ZyQj#9a87Yp!1-D0vd#W^274bce10R={qZ2K=BBPLnV{2&(z!3GPOf@;a#oPQ zCeKZoE8!pvtrZe5GsHS6y2Ay~XU)L7quMqpj29JY)??`Iqf%sXfIIndHW4-Ks2ixi z)frCLw7q>zpxzCol6DSo`bxt6eNSy1o%F4&{u3CPu6pN#Y>M&HK5v*Xnio(D(f}HU zI&5NIjU6Zft|s<1h-M8O-%mLG5H=M*Hrd%FA=l!-_a;<>Qu$y~qcvY5L9X_yY&P@p z-J|GZ?fy;j{3uB(u^;c@2cG^LuZ)w2)*7Fe8_G)H?|!Ta*>HH#+x}DNQ&M8Fi3QO@ zs0BgO;zV*Ybjd=91yWQP>Ee149-1lz`SyAC`Q*g*ka!YZg6u%LDiqZYEWVC84m_by zZVDJv|NvS4vQrvL3{wO!zJg{OS*_y5QkPI}90A>9-Pg$o!T;G|Jwp(VN)k<>{ z4B#_%rPPw5C2phxFE_cWS-H+e%{^wXfX8uyinu>d+e#vg--DRuG?5a+ZFY!B*a$Km zY^_I+SEShOjugD8m#0ZhtI%F2I$*t+kk>Fs;F#tf9hAPG&TpC#g^XF-tX-2%)@hts zOESsk+RSjYnj<4Cip}GjXS0T7R^+=`Rv6>5T*!lF3YebLFPpO9lE5QhWolzJiMM55pLP*LrhR8||x zFilcMHd?mcQ`&hndpu5JL08Ei2R6#=_d;N6Vo5DtAqz2BT#G-FZYbgkPq5I+G3avb zCJuZzRZgMn2w~YUBzR@?&Q6_hU4^};DUIz$Do(p)JvA7{C&ril+`VQ*qb>9shikub zV=q1Cm*&dAFCu4Cd3K-NQ7KJ4^btD##ZD2kyKh}aPi=rCL- z4UhYi7(f{BS2aA_ewQ8!&rAtmc6Dx#N4 zs$J?0XDh;m?zsDjgTIe6)gs97=f$rKnIH|0<_EC#;r{6_Hw_b>icw4UOL-N6#tuDP zq>^g=a5GfMEch$)HUb`iqgEr1LYyhK;^Y5brgGAJ+=LW*}ZXwW+t?_kYGSUL0lC_=*b++$qL& zMk$6%dORt+7#WLZn2^9>bDF zWzrdASxc@O-DZh9{6j@hiuO2vxDG#4b@;_Dl}cn;dqgqulhgR!UQ|Jjav3JtL5-WH zbS#Ae=vu;s6dD7<1WgS$9k zwjMdWE1R1@9OAw13GmXR@)SJbR~xF&7aF4uWnr!=$$0l0QK?udBYDNRIS7K<+-poz zRt+W}`52u?xYqn541K6hu~K_Q`tGyOlnyCs&|%IWy~`k(klDCqiQ zN6v{I_SCEQFl!z;n4f*yZb{uwNrGlDy;H2dddGJfTEM?k0*xvSXDiC+4DrAXo`GUR zJl!FCan#N^_P!5Qa3oymme=~Ou%V>E9oXMka2mkzQ6Kw#DWnB0F6lJkWlj4;Ddz106~NU zq1ZC84h~S+Z^MdH(O7>`J2EKOm~5t13x^_rKu57Yuqjt5t*@)QR?qa3wLUudP}!d2 z{&KQ3XcPktyq%c#;Y7Xtg8Idlos`z|c^65DDQfjbf%RZ2BF3ba$uMtlRD@9_h3gJ` zC*J_K(!SmQ_KLJ^a|?n&<^8A-?SoqkQ*0YoV4=sJM+V8L)2AmL%yH6zM-0r&-L+&L zk7y|7jIDfeu0=^W`_Gril8QO@#fb$MZnTN^O^GE<0z8!Y$%Laz*c<~FOc^I@pqR9A z7~!oeZ9NBgS!5swLSD2Mkjh)@4+^r%UW9L(Qttu%Zv~8USOgbm|@mwq2Ses4qDNQ;& zsi{mWd1ofcPLYW!7fm`AKQ5S7*7HT?8+^xas_^kfr$)Xt_DV`)Pkg_eGJ$n#{MK!a zYlG(PKko@-i&89r3p>n6Fe1c!e5z1tRvk|}k{!iPBCTaOEUD?9&jgX&D7_)`sik!W z-!XTI8j1#yzhghY>R?}m)klQLlH1qJln9v_SZbOxl#55-~G;MlPL7i2|$h69TW=}y?YAR1q!8o`9lcO76gdR4>-ejd~48S+eV z680n~_O+?nQrJDZI8^f`1Ezs%h>Ayh_BF%^P#Vker5}ag*t=JaMrc-7D@SBxF67co zeXio0W9mXTF6vxPGqXl1GPTwLv$Z@9x`G+j8|o`yc_)P=YF8&!jwr~gAu>FxEE3$6 z+^0Kti-o!TF0U?G#hIw8GXhkzWjel*1rxM!8yd+oZ;6mjIojrO%cPYU^{~o%4;sWU znsH9Y_e{>%OO9NxOHOz982$9!OI1@L^s@CH;wM+VJC@X^Jk#0QRBFu)gN8t1LJXb~ zK06I+D^2iwW{t7{1O>8zNKvFOsEYdB{fGN>npBKgB3$@bHqm357!>|f7M|>4%wUTI zLGdBUgw9-^z2M3EdyVGbT!3T}ip)P@K)u_r_oYiSV`C19B|c3_+Ha882b)b1`5GBy zPYExrMh;$5t}0}x-(5WeVhTP9<;0PK5IU_?Pl6iX*i3~0Js;QTZ@x4D!C(N(f4d*I zNG-0eUS1TgH>{TPwnRE}y-x7!kQus@$UGkNq}}(BuN_WhA$>7A`Dd)zt9qV>bbfgt zojAh1sqYP$R`dqe4?kis5WDp;kU=5Uxd{+MRYlOsN^%;7Cx1L?9K~qE&6?fCAyFl8 zkcVHF9p7cx+qO$apqx#w4w+&}P$3}sHWUIvoDM@`Y;7j^TS$@P3bq-y!ah~AN9usv1T#@r4LXAovLW<5h8j~rxD-b+ zJ9dE_;mYhi%rCpOk%(eo=G5>STm6ET(?8Y&t8dE;)jV7InMy66f6#;*B@*{rA3HT* zysx`$%#ZK=MB%bHF&BG5VU!{xJI%=P6#6V(C|WFAxH@&>eQUpUOdfe7TmWwd&l{xo!-%}YWe7O*C?`scUs4x+doHugQs4U z>=^j{isL2P_9OTnVplnqs!C$6#-uyiD)_1lp0uNq+{ZFcq!+QB)H~F4+pR@-ZH7b` z%+-nu9M8fr*Ry4g)$W%3CeNgyqC*6ipPP!^mr=-w z!ESVyJ%O5?pCL{V0E`?LZ#`M3n?wQ0l&;vCLQiY7{k~&x=Ykx|0uB4EiFmW_c(q+e zB`J{gMQ;wH>hBrPCM>=(`659Q27V^tW86rqnmn>)txtY7nU1)nCS1ZdUtGngU7&;{ zEO{Rl_@vUJpH2T%!O0o&jq-|Xt$hMbI)QGh;h zb&qY&R}F>`p!XInn+`sfRG8OT-joR;;q8Y6#R4D~g=lf!Gt$b>U2ghD?=kQ5_ha!} zIP=dEjGk2Tp+Om*EtCU3p-sFr3-XD3h%SGZYwS#A;v_jRb4TwX zxzm3Kkq$Sw7+ZzWMlN>E@{UlvAL*7W@X#}l!4&)u%)(Yx@&j9#LeCesHGy;=NI0PQ z%)LiPZ;c&OkWf&Yp;F@7&vczUqo1JFuOa z4vbzWjnSTlcTGu7JsN~~CP=J6IeYXWp1q%`0z3f*G`7vFj2T@f@FY+Vp9?ii4JZP<1D}mpF%#P}v zDrQH)#lfRx`sxo>`dRZtUyhF6j>)@*N6xrx3@E&Go*xa)&uu@};U336ridz;a=gf| zueJcld%CH=c!(F~mY=qd)`5w{)%cvHx5U*-gvHqv9<<9_u3(G_*bK(*I%&REtDWY- z$SFxwXfCRd$*74sTLGq8|H`R^*CgEAO-n-pn;PeWLf1~Vv9{{NBuQt@o)Mlx>wNH; zHNx7tPdt>>44HO2pN7xHb1%oLXf{&4apYUhqF6?A31dbkvPN_@kSumz*$^fjDIdB6 zis9TFc)EAM6c7Z|(RMaLnjhD5wFcRI+dlrVyr^F1uuI$pM^8Z3*sa3ffhF?lR#&1* zWH1cw6Lh9eHWXHcT^Zj*`)1q|iS;g*rBqvA;~Fu2@QaIs{OG`4F=cu+p=7*ybAK4@ zX*TFa-5M+ushn$R*welfli&C4&ncq=-rgi4o7;@Itzy_H_2~E?h7@cr1bXgn=_uYV z4mK_GR4hhnE@MXpAtsPl&@sWd=3akv<%kZRI>DIznA$mg-Uz)cCmQS-xal`^2ub0; z0M?fnRz2^~La3%RN7+kgpd~}s?wLgd0wwaMp7U%M&Gb6~8pAjA&t@RK1_P)T0`ftY zU?}@Jg(8<)+crtjn?0D95&*Akno@;v3Eh0c05yNIP9-s4y4TWWAnZ4>9)Kp41FmLAs2P^=bg!-dGH>=S+Q81z6Tp+Ue_pO(4NJ2RTb?D zEG0%pp>EIZy1hX+1QIAvt~()1E03A9qYzo@d3|1fas5-fgo8rks=^U|qma z+bcYCB5+F*pDu#8t=XlBi2Kg8w|I#|a7DuP#RI9LdM5RBi|@pB@53&F4QzBG6IWuw z^+b}g86fc$H+#|zLv!nr{R3I$5OI{fvdv}GdDDZLr()>%kz;&(!i|DAoU~gEBray3 zDZLRD&=$j~hD>ZRaN4-MXGN{6_@;_WtcD z!wd@5cE$sh6oq5QtBE+_G0z?9Cm+g-ChR+ShyCN{n;{P~4V1jJl^MxriqS6vu_tFu znT`J25&7PkoU4p4F>jp&l8Yx3?dQU4wdWThy&OOEV(v8W6#qf_kG=Tpc7D* zX;+qhewPZk8;{B=7_sqma&5FtA?E<0K;euB`!WP^Cv(+4O*7UIhT!5Z?gEVWIho9j zK_C*7z@=d`oP~CnwvQhxYlfsb+%g;^_ZyC_&Ha9+29ae6sRmif#Pq9XPDK z2Y9YAIB5e(@Nh63C#-@nDlz?zRJRtiHMW7q*E^Q?6|ojs_57=5V&sMV`k~(z3FQn> zRVwnHX5MaH=BTQM#b&(SMPN2koYhY$p6oOi6!MI_k0EFW#(_kt+ZvaX7FVZLR$(9R zwcTD4y8tR~!n)Wtmx!6_BO&G3W@Tv_!b)A@DHKxyMAxB$6QFUxP2{wZe~%-h(v5(j z&BwmIaJ#LBTcP6N#T$=(Ke>0hy>*&h8pql+SvYMNJrsySI=xY=Ev}my)1HtaHz~Q& z6I(sAh`(hIS9II=0^I4uo|#t|yhR<0M6O>$>TK*&xl-u;;PAlLV^u(wg6gRz()GHuJC_b#G3ct5&X*gm{I@8i87M zWea^j0YRa51|BPAL>(zkDUn+5b$Wi$QyKDe83(!yQHmX$j3NQu^v8UP%MK`h3HZvZ zxOT8}`ynOcOSh8>1j~@f%frXz;+BD?hA7AsiHn5mDf80C>?j#q2VR+P(vhvkbY?m* zBM_xDwuku|oHiAFO9A}jjh|(Ne$3pP5nn1Nx|+aw36DgH!hO4$C|Rz}iNZ{|H*=01 z(Vdnwy_(Vp<@=;oRUv@{Q(;?j&Ls%SLv8DS1ji$52xq+%;zsPdeHgUN)Vbn@Q`Mt@!jd7 zv~p4hG(sN6ms}+I4Thq73J>^BH(Ikx11CI#A;L@em77jfA|%0l#;HIgwa!4!J9b5);Ks!>R8o3q(c$BmPf`S~MFs7sRDpP)|Vfct(#Tu8*7r_i5)18XlVmCiXZ1syMz5qO^2-8W9S-ulL| z6&%hJLVVvnE3zDb*OM=*RYf+H~mdzA_5lkbW=FqKTQ0sWJFvH%FWaOAjI zt;oswj4VHq~BSqUkj^Q!)t zmfPq%wFCI{{5UTdviZFu%zN3u59C#c?mD+K!C8k&1$1H$5f04vYPqxBO2w12-YJqA zm}ABu%a~8`(bq56&5agWn@w>(o1AFzT4WxHllUn($Okw@9yo^dgY1{=7ik9VaOE8H z@;qdS6-Kp-=5MI6C{W%XAkk~oC%$G_L&Hc(lz)~ed;d(!k>>P6g3CBTQozAvs+qf! z@=oIuiABdD7K=ewhZ}` zZm(>%i0L{M??=2Lw+sc0NM_0D=^cl$7grhpuOY7wkXaUnD0G8Cqu(OElH8}fV#UJu-% z7BnVU*bTtPL++mdpRsCqtDH-i$XT6gP2g6HLQ&3VTQIq35aN3DB{!5?5H&vO(n7P3 z=(7M}C`b(CF|j=!95R>GI)Z~Z@~K)1=@b{SjNTkZ!q{ykTvVV4M1@uy;+EED!=$ql zZ|o)mKD6tk9EVs5XiYCcLVI&MKe(dkL(qPv(d?yURDX|o&uwH^+LLE3Der}SM^&=v zRrt5eDJ#bFGWUu(wKDpjtx{Y*fKR^dvE=<~(R%{!m8}gfX@rIVC?bPU79)X+VY}lV ztATo|q7dH~in8fB>ZQx5{SY)k+S5Ux#VafX8EhI5E+?0Bbsu5pL6KGvtBg4EL0)QG z^DA@a#05k$1U@~!(CREtreR#@7jb+|#)y%)&)L`#rV+LMY8MzJ-*B>RUAn}?Ma++o zp)TSeQkH#No;Nkg4Gw~btZ z(?yGDA62g|fn3rt;j-#mG2-*5Jf|Y2&fyk|s=I`$l>sNxW3x#aBhPH%4qE3&;mzfH zz4zlOp)(N|JVi=dco0_$hu7v)du9g#KX8q#0Y$rulyKq3QxtrVpos!Ll@Z4vWygn- ziqC-)Q_U~n56_jbk0>V~`pKG8T=5emrD(2<#?{}A!+UFf|9rO`r!Cu5@9g7Be7sSf z65?MuFW0ci23{z34Nb!b5d{oV1eVmV^#;$8XBOh&rSkJPHS$m~&a)j8*?I?oY!emr zWM&xUXAInzWMF|@-?j;G;2~qeNfozPw-FcLTpH{~&q^VHiRc7Hx`2@le-!yc$8n`I zCtcqIS$3a(A9+|zZd;0}&ma`8%#>}wBk0?gj7fbp)ia|m%nR%0ut^w5RUdP%)EO5i zI-8^j5-uHUXiu`axDD5 z9}=%(ZiuCd!fhytsgBzpX+Tgnj;tFh$_nfL0`5uk?I7ad$yjIe6q6 z5`w3~bNEIw9S=%p;aM`clc!3asj{cc_PD4WqHgqBeb|l?7hWG4VaS@KzNdOz2SK-L z@5G$YeRG&?z?n2c2TC85z9*;O$&VJl7X$QFsEr4QTVWJhayAZawdspklff^e->MB! zsX#Rrs-Zs+R3=J@h8SP>?Ab{*sWuNP$$}77C?@lgMq$Ft7dEQ&>TC2Wa=|$t`^tf0U&fE%RzuIg-Os zI1kT;3wO(C&!%6|d(`nur}BwzJY<8GbDOd~YiB*$<5Ro&3;1`kYE`+&gPXgu*pFVX-8(K`$m&IURf7}w0)OU(w zGTc@5L+kclMH;qK2zT5Q6nMX;U3*cNv+}Lrj9{3r7YY)C!fS@Qt6+z7D*2NFhmbVBx0A5?> zm2-cpzF`v{%kP-pR2;mWvKVo3L79UV$h>o>P@la+Rgu%YZiH!Byz|%R035i(;Tg%E ze7&WZ-u|l=w>aLFfH2=&KbgdDVAlD4qgo3N5ovPwv1Abjn!Bw9Kh*q3D0hob#^R~_ zm*}n^=Q-ucc53KLfAq|%7tv2sB%~*)QV<$Ej5{TE>7F(RaTb{sg|)bX&;SyQD0bPt zGcqsr>Z;>;Gj3{A*Svf(lGg0S!gL|x&%MPOR;JkSjQEDeDEXz@xtn5%U^TLtv8HtH z6SL{+Y^(12O-J{`aQ9iKKd8qc-v*YeWd-9()dU)Pb|0q8!Bct%Le~l_-y#+!DCvtV<@tipg4g-%z}CoG2HwJO z*A#rK3u*|QgdkLWxcYK-liuTgBORo%SFIe^(UDW{Gz?>`+05JlC)9z}37$*J9R>K7cz?C%rR7%ER8q{Xozk+~~n;8t6zT zA9$Wp>fvh`z&H{7=D1gs%sc~;Cyai~KyQill#W@Ll>iKJkkhP0lU+dLhpX~OCe4Wx z4US{;hXf%+4FLn-*!IyEMDZ<8&d+68O<{Z_^h&~ZB=l)z1Wr6vZ=KbjAyp@<{RbRS z%C*cR)A9VZ zguX}-`yfG8q55-mlDK;*mJRg1r&TJa&@>Ea(p=858awC-NL9D@X@+Jb=tH9De(#TJ z*W@MeEzXV4{pB|)zPi8X4;n>rQnvRjI1Ux{{!ZB8_1@Pssj-HpA;2`Z&-&|DcH>lB zsu0Z0`wa)nn0EuBKJh!7n_`ze+DozcdN(8RlO8U14(xm)ZOWI<&`NeEE++53Z0(Yp zl)OmKos$glzKUX7~ zTxXtRg#M(SC^^EZDR|IR9bTZY0a<}K*#CsKa9c;bio|nV!$r>W0ln~nK`RBTpM@1$utmU%sPU=~rBMzs01V-0nWxj<_u?u4Wgd2!<9 z)t5EEXY`;8kbmY0V1K3i^@`v%3$U2J4In4Y;rDOsCzWMIEM?>u?U*WjD(VSfm?_k7 z>NUplFeQbm1hCL^Vt%r#TpImUs(|Ij`M7n}C-1}94JohF(N2v1N3IgVwtXL3~xvSd5J9g+BjchJ7HVNBCxjnV*TORXq=5p0kw5}`&e6$i;gLEE(MF8gnseB z4?^|masnt(ewkhvtCGghxG-u@s)>ig8-G(u&2wmqTcO^IF zPs#~3cNcfyAx!eDW71Q4X`3E_p?a2Z*AKL0+q?DKYE-$iWVqfu(5mxILxhBQP~wqa zzi`hzr;;JG3oQ%{sfrxW6S)AGFNvRe zwt1@aq+M?&Z0LB2oIobuSm?d|gy>#weA=JKAR=x)kMTq@sWXi;R7%Co|0$mNy!vdu zKK(Fe?Xqq+KR^H(DoY3T@R?jknA>=4Qy}8UvG{m`| zkVW?I)jdUoc&rak+4^uQb+x_rgwWpJ(sDW}$JqvDoSgi``1IC&<_zt75XXAF#KZ?5 ztaQc}xkT4VzRT?GZg(aIVMW~0w>Ng1-afdZt`h`wj8*~7KtCe-!%Jj zVM9VP6T}l3#T;%e4-%39LQGi-MGzEmHJUnYOROoTrIf^*LYj!A{q*f^b_Amb`NA*hn_daa2bCGW~Bf9BUdtTGQJpAbS}B#%boK3{$4^W zmdL}WHzcQ(F`ANKidWV3K(GUR-#n zXh6``eI3uN(b3jX4v%wf9Pg1Yp^&*pWU{tp*s{@`Lm`kS+a~&uzT2Y^k@!nd@E><^ zqlig?k>~au0tvtgV#!kQGhrBsKse~a94$OU zb@gTtBDxxp2;ZMe)i22+Ism#lETs-NHdEb-PX9er5Lh#GkB~2grX2}bYejukzPC4aKTBk%GJ53}fp;}?GW+N>) zowPleZcbmjuW;tYvXMRUpLk^4JRI(Dx1FD$>4H^(zQmk`M2HHyN%5nsP;6L3`mq)A zbuHd)+GCu6ucb4rQCmip`ns|XXT>#wK zBXXu^lB#U*DfIY`yBkYD)aj!gg4{ zPTWLQ2PZq*#4T&)$0PuBV%XC}qaNE!1jT|P?)94FE znSW}C($wI{ZOE*y#a_1LJse5*E7UWu*EiWySWYtkoVMXHv|_8qlrkF^P(JB!x`^+R zKC1}YETZQt!zi6XeMn#Jp{wtp)Bsi|Kg+yiSWr8HKg!^Cf?K^@p> z>ITbQqHGe_A6h*qKzI@+@?)7z100N#Bj346r$oGfat@g==sCmofRNnr zp~YSRDYbw|0;2jHNw`b&Z0N4GCQD_+y zTr|08uBa^y4R4+eO}$iKB4BruWv_6{gSlR^#%dYS9@1=?OM^f)#>EMZBF;8Va3h_=UF(r(bhA8!kOqzqktTB!tGmnq=UAe z%>WTv`D(i2by8SBQa!IR%N0IhVjkIUWI!hU)s8X9yIVA=TVkYr&t_uxs%a{tsV>Q- z9D+hmY>VXBu57G%#SR7g22w=ZBF%4eK9Pq?DR^7+FlW~RVXwo>qJI`rkUHBH27eivz?n?MNFE*9DnNJ^jp`ggWHy+RA(aeE3CSHCO z?(H^s0Rze6jN5@}td$f(2Q`nC(8AK*T&Fb4t+PmnX~N~uVplWjYE&4c;FpwV{E=KA zfjYCVI!mH0a}yPisb(GHHM$}Xu;bpKN1WUb%C)N}^Q_bD!!^3$bPbjo!A*wY$sTN@ z7ve5(9-%c+F6l+tC=+z=%-vv>yT907Q2{%G!;a^epq|%$JSd#h*9r#tRN@F>3qMhZ zP6iw)h*~@Q9a>=A;11g&xLe`}@d#E$BylBQdhcT?6_pB2uEu~GA0s?r1Y63)_UCW= zDW8#B6_BJa^*99bANB-3$`lyZsPTP;f^eHOri!f75n)5^h-hXrcf^(L)2!4JKSY!_ zk>~Z#H^9*8n}$vp&(R0_G&T>pQzo8P?lh*tZtD6un1E632%Cj7*b>Ar+_8R^U6XWd zR+rB()2)mm`jFbTyZm&&qMlJ?-ifs%!L0J?pZLox9(Ho=kr z$;6y}UJwQ8&RBWk=?;~YT&JN*u1^3HiLTVl27jmb!$5BN9J?)&X9rWT zqv(XG)aThCK!D~+p5T!*;Fdem!oqs~% zy{R=zNZ_QoXTny=Mi)^Fk}Kves5FkhemfxM7;>4C(`ef2{)P!sJm_Tcm?kKX=r#OW4LpDYlCg=oNFq@^ZNa*!m1?D1sX;Tug>f zCOV2yO{!@mL^EBpR|^KoT0sng5K7TV3Z0!x%oRmpksuf}3^oKYou^&v?RDckW>|0j z8x*U275*{n4uxp31LVD z*0`%?;oHG5!}#vaT7uWhRB}ZvsHGu#XiyN_GwS=&Axp`-x`lY!2T`$wZ@DGK>>=dv zZl4)S$|ok|&`?*$85h841-DRm> zdo%a^sa|I-v7Z~)hGKtebDv<1+V5rU1^D+>1vof3z>wkxoIei$QOX+wXLBnfY6k#| zj=7x^jh%xnfEmTf+}M#)lA2;{Wc-41R9b?%nt76Ok`V;puz3fLW3H}(0=V(Fiav6Z zK){f10GulT>>Zgf*`LsF-T=(ESBJE)5A-WIDadg@$)zUT>b&}$7avp0{)#^01mD{9TI?D#oy|3 z{5M5H{ptVTys`S@j^CT{UvC6txBlsf|LctaW&EG#@NJ2=f(x+XR0QnF|3?1!^XU4& zH}f0(R)@*Z=}(1;0qal~!07ppL+-U+=lttx)BycI1PmC?pV)vaX~19qRG#Fww*3eG z-<14%Ey$~U%v}`FVo5+>{@Racz(9Q^=uhqEcQAFf29yT@Fg_aF{aj1GhOb#&zsCT; zn*!j;|I&{*K+paj{vXt%Xae~S7?720`p3@VPvlF$5cqr8KUaW!-PFgUu*!S@&=%nQ zZ6WpZ-~hZTe~21R5+-1-`A3}z>N^^X+c+BAIGQ^Fs7Mt7 zkp(uU0NvT?1H3-}S^mCOAJ{Gf1PDNNod|E93t4 zr~DU^{t)4+?_m6ook}A9G13AMRRR#j`WOAt(EMvr|MZYHO$3t)0kcW&AHgL*NtP^H-Lb@J{DG%|4#w` zq%l2dZ)FgGs7Ao6#PCyHfGZL}<;Z{R%RfGLyHPcmWdOb<;ML*%3%&9aBs#`xL)S3NP3BMj}uldb?!lk(WSGZrZoxg7FYdZ0t z@L_Jhg#RaU@z(-g6Se*n@Y&~=0)D5w{1t8MYv|YfnLnYG{r(H|ACCB+Y?`leU-Jt6 z#Et*-OWap1L$C2(v-kYO^NIX5-e0+VUgN#yHu#D6CH}wR{d$1Cj%)sj_O|4gXn&4! ze%-X!fu}#gD$9Ne_OHTFUrTu%()d$KOT|A+`L!Be2Xp+yM5+HJ=IaUfFQFZ;Tl{)! z^d~xT%P-OY7^J`0D18n6dh6*Y^!xVz2L0EKsQ;k;$LsQEr^#<#m!HRn{=fIZ$Vq|% VEYIJJ9ELZ#02_W5@V{@~{68A?Eg=8^ diff --git a/wrapper/java/gradle/wrapper/gradle-wrapper.properties b/wrapper/java/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 2c2bbe5f9..000000000 --- a/wrapper/java/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-bin.zip diff --git a/wrapper/java/gradlew b/wrapper/java/gradlew deleted file mode 100755 index cccdd3d51..000000000 --- a/wrapper/java/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/wrapper/java/gradlew.bat b/wrapper/java/gradlew.bat deleted file mode 100644 index e95643d6a..000000000 --- a/wrapper/java/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/wrapper/java/settings.gradle b/wrapper/java/settings.gradle deleted file mode 100644 index 4444894ae..000000000 --- a/wrapper/java/settings.gradle +++ /dev/null @@ -1,18 +0,0 @@ -/* - * This settings file was generated by the Gradle 'init' task. - * - * The settings file is used to specify which projects to include in your build. - * In a single project build this file can be empty or even removed. - * - * Detailed information about configuring a multi-project build in Gradle can be found - * in the user guide at https://docs.gradle.org/4.4.1/userguide/multi_project_builds.html - */ - -/* -// To declare projects as part of a multi-project build use the 'include' method -include 'shared' -include 'api' -include 'services:webservice' -*/ - -rootProject.name = 'hermes-1.0.0' diff --git a/wrapper/java/src/hermes/cpp/hermes_java_wrapper.h b/wrapper/java/src/hermes/cpp/hermes_java_wrapper.h deleted file mode 100644 index 22a59f57f..000000000 --- a/wrapper/java/src/hermes/cpp/hermes_java_wrapper.h +++ /dev/null @@ -1,177 +0,0 @@ -// -// Created by lukemartinlogan on 4/28/23. -// - -#ifndef HERMES_WRAPPER_Java_src_hermes_CPP_HERMES_JAVA_WRAPPER_H_ -#define HERMES_WRAPPER_Java_src_hermes_CPP_HERMES_JAVA_WRAPPER_H_ - -#include -#include "hermes.h" - -struct JavaStringWrap { - const char *data_; - JNIEnv *&env_; - jstring &string_java_; - - JavaStringWrap(JNIEnv *&env, jstring &string_java) - : env_(env), string_java_(string_java) { - data_ = env_->GetStringUTFChars(string_java, nullptr); - } - - ~JavaStringWrap() { - env_->ReleaseStringUTFChars(string_java_, data_); - } -}; - -class HermesJavaWrapper { - public: - JNIEnv *env_; - /** UniqueId Java class */ - jclass id_class; - jmethodID id_cstor; - jfieldID unique_fid; - jfieldID node_id_fid; - - /** Bucket Java class */ - jclass bkt_class; - jmethodID bkt_constructor; - jfieldID bkt_id_fid; - - /** Blob java class */ - jclass blob_class; - jmethodID blob_cstor; - jfieldID blob_data_fid; - jfieldID blob_size_fid; - jfieldID blob_alloc_fid; - jfieldID is_native_fid; - - /** Blob ID vector methods */ - jclass vector_class; - jmethodID vector_cstor; - jmethodID vector_add; - - public: - HermesJavaWrapper(JNIEnv *env) : env_(env) { - /* UniqueId methods */ - id_class = FindClass("hermes/java/UniqueId"); - id_cstor = env->GetMethodID(id_class, "", "(JI)V"); - unique_fid = env->GetFieldID(id_class, "unique_", "J"); - node_id_fid = env->GetFieldID(id_class, "node_id_", "I"); - - /* Bucket methods */ - bkt_class = FindClass("hermes/java/Bucket"); - bkt_id_fid = env->GetFieldID(bkt_class, "bkt_id_", - "Lhermes/java/UniqueId;"); - bkt_constructor = env->GetMethodID( - bkt_class, - "", - "(Lhermes/java/UniqueId;)V"); - - /* Blob methods */ - blob_class = FindClass("hermes/java/Blob"); - blob_cstor = env->GetMethodID( - blob_class, - "", - "(Ljava/nio/ByteBuffer;IJ)V"); - blob_data_fid = env->GetFieldID(blob_class, "data_", "Ljava/nio/ByteBuffer;"); - blob_size_fid = env->GetFieldID(blob_class, "size_", "I"); - blob_alloc_fid = env->GetFieldID(blob_class, "alloc_", "J"); - is_native_fid = env->GetFieldID(blob_class, "is_native_", "Z"); - - /* Blob vector methods */ - vector_class = FindClass("java/util/Vector"); - vector_cstor = env-> - GetMethodID(vector_class, "", "(I)V"); - vector_add = env->GetMethodID( - vector_class, "add", "(Ljava/lang/Object;)Z"); - } - - HSHM_ALWAYS_INLINE jclass FindClass(const std::string &java_class) { - return (jclass)env_->NewGlobalRef(env_->FindClass(java_class.c_str())); - } - - /** Convert a C++ UniqueId to Java UniqueId */ - template - jobject ConvertUniqueIdToJava(JNIEnv *env, - const IdT &id) { - jobject id_java = env->NewObject(id_class, id_cstor, - id.unique_, id.node_id_); - return id_java; - } - - /** Get a C++ UniqueId from Java UniqueId */ - template - IdT GetUniqueIdFromJava(JNIEnv *env, - jobject bkt_id_java) { - IdT tag_id; - tag_id.unique_ = env->GetLongField(bkt_id_java, unique_fid); - tag_id.node_id_ = env->GetIntField(bkt_id_java, node_id_fid); - return tag_id; - } - - /** Convert a C++ BUCKET to Java Bucket */ - jobject ConvertBucketToJava(JNIEnv *env, - hapi::Bucket &bkt) { - auto bkt_id_java = ConvertUniqueIdToJava(env, bkt.GetId()); - jobject bkt_java = env->NewObject(bkt_class, bkt_constructor, bkt_id_java); - return bkt_java; - } - - /** Get a C++ BUCKET from Java Bucket */ - hapi::Bucket GetBucketFromJava(JNIEnv *env, - jobject bkt_java) { - jobject bkt_id_java = env->GetObjectField(bkt_java, bkt_id_fid); - auto tag_id = GetUniqueIdFromJava(env, bkt_id_java); - return hapi::Bucket(tag_id); - } - - /** Convert a C++ BLOB to Java Blob */ - jobject ConvertBlobToJava(JNIEnv *env, - hapi::Blob &blob) { - // Prepare data to place into Java Blob - blob.destructable_ = false; - jobject data_java = env->NewDirectByteBuffer(blob.data(), blob.size()); - hipc::Allocator *alloc = blob.GetAllocator(); - - // Allocate new Blob java - jobject blob_java = env->NewObject(blob_class, blob_cstor, - data_java, - (jint)blob.size(), - (jlong)alloc); - return blob_java; - } - - /** Get a C++ BLOB from Java Blob */ - hapi::Blob GetBlobFromJava(JNIEnv *env, - jobject blob_java) { - hapi::Blob blob; - bool is_native = env->GetBooleanField(blob_java, is_native_fid); - jobject data = env->GetObjectField(blob_java, blob_data_fid); - blob.data_ = reinterpret_cast( - env->GetDirectBufferAddress(data)); - blob.size_ = env->GetIntField(blob_java, blob_size_fid); - if (is_native) { - blob.alloc_ = reinterpret_cast( - env->GetLongField(blob_java, blob_alloc_fid)); - } - return blob; - } - - /** Convert a blob ID vector to a Java vector */ - jobject ConvertBlobIdVectorToJava(JNIEnv *env, - std::vector &blob_ids) { - // Allocate new Blob java - jobject blob_ids_java = env->NewObject(vector_class, vector_cstor, - (jint)blob_ids.size()); - for (hermes::BlobId &blob_id : blob_ids) { - jobject blob_id_java = ConvertUniqueIdToJava(env, blob_id); - env->CallBooleanMethod(blob_ids_java, vector_add, blob_id_java); - } - return blob_ids_java; - } -}; - -#define HERMES_JAVA_WRAPPER \ - hshm::EasySingleton::GetInstance(env) - -#endif //HERMES_WRAPPER_Java_src_hermes_CPP_HERMES_JAVA_WRAPPER_H_ diff --git a/wrapper/java/src/hermes/cpp/src_main_Blob.cc b/wrapper/java/src/hermes/cpp/src_main_Blob.cc deleted file mode 100644 index 1264cfa9d..000000000 --- a/wrapper/java/src/hermes/cpp/src_main_Blob.cc +++ /dev/null @@ -1,22 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -#include "hermes_java_wrapper.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Class: hermes_java_Blob - * Method: freeNative - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Blob_freeNative(JNIEnv *env, - jobject blob_java) { - hapi::Blob blob = HERMES_JAVA_WRAPPER->GetBlobFromJava(env, blob_java); - blob.destructable_ = true; -} - -#ifdef __cplusplus -} -#endif diff --git a/wrapper/java/src/hermes/cpp/src_main_Blob.h b/wrapper/java/src/hermes/cpp/src_main_Blob.h deleted file mode 100644 index 47214c0bc..000000000 --- a/wrapper/java/src/hermes/cpp/src_main_Blob.h +++ /dev/null @@ -1,22 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class hermes_java_Blob */ - -#ifndef _Included_hermes_java_Blob -#define _Included_hermes_java_Blob -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Class: hermes_java_Blob - * Method: freeNative - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Blob_freeNative - (JNIEnv *, jobject); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/wrapper/java/src/hermes/cpp/src_main_Bucket.cc b/wrapper/java/src/hermes/cpp/src_main_Bucket.cc deleted file mode 100644 index 90a94d231..000000000 --- a/wrapper/java/src/hermes/cpp/src_main_Bucket.cc +++ /dev/null @@ -1,167 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -#include "src_main_Bucket.h" -#include "hermes_java_wrapper.h" -#include "src/api/hermes.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/**==================================== - * BUCKET operations - * ===================================*/ - -/* - * Class: src_main_java_Bucket - * Method: lock - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_lock(JNIEnv *env, - jobject bkt_java, - jint lock_type_java) { - auto bkt = HERMES_JAVA_WRAPPER-> - GetBucketFromJava(env, bkt_java); - auto lock_type = static_cast(lock_type_java); - bkt.LockBucket(lock_type); -} - -/* - * Class: src_main_java_Bucket - * Method: unlock - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_unlock(JNIEnv *env, - jobject bkt_java, - jint lock_type_java) { - auto bkt = HERMES_JAVA_WRAPPER-> - GetBucketFromJava(env, bkt_java); - auto lock_type = static_cast(lock_type_java); - bkt.UnlockBucket(lock_type); -} - -/* - * Class: src_main_java_Bucket - * Method: getContainedBlobIds - * Signature: ()Ljava/util/List; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_getContainedBlobIds( - JNIEnv *env, jobject bkt_java) { - auto bkt = HERMES_JAVA_WRAPPER-> - GetBucketFromJava(env, bkt_java); - std::vector blob_ids = bkt.GetContainedBlobIds(); - return HERMES_JAVA_WRAPPER->ConvertBlobIdVectorToJava(env, blob_ids); -} - -/* - * Class: src_main_Bucket - * Method: Destroy - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_destroy(JNIEnv *env, - jobject bkt_java) { - hapi::Bucket bkt = HERMES_JAVA_WRAPPER->GetBucketFromJava(env, bkt_java); - bkt.Destroy(); -} - -/**==================================== - * BLOB operations - * ===================================*/ - -/* - * Class: src_main_java_Bucket - * Method: getBlobId - * Signature: (Ljava/lang/String;)Lsrc/main/java/UniqueId; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_getBlobId( - JNIEnv *env, jobject bkt_java, jstring blob_name_java) { - JavaStringWrap blob_name(env, blob_name_java); - auto bkt = HERMES_JAVA_WRAPPER->GetBucketFromJava(env, bkt_java); - hermes::BlobId blob_id; - bkt.GetBlobId(blob_name.data_, blob_id); - return HERMES_JAVA_WRAPPER->ConvertUniqueIdToJava(env, blob_id); -} - -/* - * Class: src_main_Bucket - * Method: get - * Signature: (Lsrc/main/UniqueId;)Ljava/lang/String; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_get(JNIEnv *env, - jobject bkt_java, - jobject blob_id_java) { - hapi::Bucket bkt = HERMES_JAVA_WRAPPER->GetBucketFromJava(env, bkt_java); - auto blob_id = HERMES_JAVA_WRAPPER->GetUniqueIdFromJava( - env, blob_id_java); - hapi::Blob blob; - hapi::Context ctx; - bkt.Get(blob_id, blob, ctx); - jobject blob_java = HERMES_JAVA_WRAPPER->ConvertBlobToJava(env, blob); - return blob_java; -} - -/* - * Class: src_main_Bucket - * Method: put - * Signature: (Ljava/lang/String;Ljava/lang/String;)Lsrc/main/UniqueId; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_put(JNIEnv *env, - jobject bkt_java, - jstring blob_name_java, - jobject blob_java) { - hapi::Context ctx; - hermes::BlobId blob_id; - hapi::Blob blob = HERMES_JAVA_WRAPPER->GetBlobFromJava(env, blob_java); - hapi::Bucket bkt = HERMES_JAVA_WRAPPER->GetBucketFromJava(env, bkt_java); - JavaStringWrap blob_name(env, blob_name_java); - bkt.Put(blob_name.data_, blob, blob_id, ctx); - return HERMES_JAVA_WRAPPER->ConvertUniqueIdToJava(env, blob_id); -} - -/* - * Class: src_main_java_Bucket - * Method: lockBlob - * Signature: (Lsrc/main/java/UniqueId;)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_lockBlob - (JNIEnv *env, jobject bkt_java, jobject blob_id_java, jint lock_type_java) { - auto blob_id = HERMES_JAVA_WRAPPER-> - GetUniqueIdFromJava(env, blob_id_java); - auto bkt = HERMES_JAVA_WRAPPER-> - GetBucketFromJava(env, bkt_java); - auto lock_type = static_cast(lock_type_java); - bkt.LockBlob(blob_id, lock_type); -} - -/* - * Class: src_main_java_Bucket - * Method: unlockBlob - * Signature: (Lsrc/main/java/UniqueId;)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_unlockBlob - (JNIEnv *env, jobject bkt_java, jobject blob_id_java, jint lock_type_java) { - auto blob_id = HERMES_JAVA_WRAPPER-> - GetUniqueIdFromJava(env, blob_id_java); - auto bkt = HERMES_JAVA_WRAPPER-> - GetBucketFromJava(env, bkt_java); - auto lock_type = static_cast(lock_type_java); - bkt.UnlockBlob(blob_id, lock_type); -} - -/* - * Class: src_main_Bucket - * Method: DestroyBlob - * Signature: (Lsrc/main/UniqueId;)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_destroyBlob( - JNIEnv *env, jobject bkt_java, jobject blob_id_java) { - hapi::Context ctx; - auto blob_id = HERMES_JAVA_WRAPPER->GetUniqueIdFromJava( - env, blob_id_java); - hapi::Bucket bkt = HERMES_JAVA_WRAPPER->GetBucketFromJava(env, bkt_java); - bkt.DestroyBlob(blob_id, ctx); -} - -#ifdef __cplusplus -} -#endif diff --git a/wrapper/java/src/hermes/cpp/src_main_Bucket.h b/wrapper/java/src/hermes/cpp/src_main_Bucket.h deleted file mode 100644 index 9f9feda7d..000000000 --- a/wrapper/java/src/hermes/cpp/src_main_Bucket.h +++ /dev/null @@ -1,93 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class src_main_java_Bucket */ - -#ifndef _Included_src_main_java_Bucket -#define _Included_src_main_java_Bucket -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: src_main_java_Bucket - * Method: lock - * Signature: (I)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_lock - (JNIEnv *, jobject, jint); - -/* - * Class: src_main_java_Bucket - * Method: unlock - * Signature: (I)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_unlock - (JNIEnv *, jobject, jint); - -/* - * Class: src_main_java_Bucket - * Method: getContainedBlobIds - * Signature: ()Ljava/util/Vector; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_getContainedBlobIds - (JNIEnv *, jobject); - -/* - * Class: src_main_java_Bucket - * Method: destroy - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_destroy - (JNIEnv *, jobject); - -/* - * Class: src_main_java_Bucket - * Method: getBlobId - * Signature: (Ljava/lang/String;)Lsrc/main/java/UniqueId; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_getBlobId - (JNIEnv *, jobject, jstring); - -/* - * Class: src_main_java_Bucket - * Method: get - * Signature: (Lsrc/main/java/UniqueId;)Lsrc/main/java/Blob; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_get - (JNIEnv *, jobject, jobject); - -/* - * Class: src_main_java_Bucket - * Method: put - * Signature: (Ljava/lang/String;Lsrc/main/java/Blob;)Lsrc/main/java/UniqueId; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Bucket_put - (JNIEnv *, jobject, jstring, jobject); - -/* - * Class: src_main_java_Bucket - * Method: lockBlob - * Signature: (Lsrc/main/java/UniqueId;I)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_lockBlob - (JNIEnv *, jobject, jobject, jint); - -/* - * Class: src_main_java_Bucket - * Method: unlockBlob - * Signature: (Lsrc/main/java/UniqueId;I)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_unlockBlob - (JNIEnv *, jobject, jobject, jint); - -/* - * Class: src_main_java_Bucket - * Method: destroyBlob - * Signature: (Lsrc/main/java/UniqueId;)V - */ -JNIEXPORT void JNICALL Java_hermes_java_Bucket_destroyBlob - (JNIEnv *, jobject, jobject); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/wrapper/java/src/hermes/cpp/src_main_Hermes.cc b/wrapper/java/src/hermes/cpp/src_main_Hermes.cc deleted file mode 100644 index fc48f1003..000000000 --- a/wrapper/java/src/hermes/cpp/src_main_Hermes.cc +++ /dev/null @@ -1,50 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "hermes_java_wrapper.h" -#include "src_main_Hermes.h" -#include "src_main_Bucket.h" -#include "src/api/hermes.h" -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Class: src_main_Hermes - * Method: create - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Hermes_create( - JNIEnv *env, jobject obj) { - (void) env; (void) obj; - HERMES->Create(hermes::HermesType::kClient); -} - -/* - * Class: src_main_Hermes - * Method: getBucket - * Signature: (Ljava/lang/String;)Lsrc/main/Bucket; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Hermes_getBucket( - JNIEnv *env, jobject obj, jstring bkt_name_java) { - (void) obj; - JavaStringWrap bkt_name(env, bkt_name_java); - hapi::Bucket bkt = HERMES->GetBucket(bkt_name.data_); - auto bkt_java = HERMES_JAVA_WRAPPER->ConvertBucketToJava(env, bkt); - return bkt_java; -} - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/wrapper/java/src/hermes/cpp/src_main_Hermes.h b/wrapper/java/src/hermes/cpp/src_main_Hermes.h deleted file mode 100644 index 59a868764..000000000 --- a/wrapper/java/src/hermes/cpp/src_main_Hermes.h +++ /dev/null @@ -1,29 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class src_main_Hermes */ - -#ifndef _Included_src_main_Hermes -#define _Included_src_main_Hermes -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: src_main_Hermes - * Method: create - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_hermes_java_Hermes_create - (JNIEnv *, jobject); - -/* - * Class: src_main_Hermes - * Method: getBucket - * Signature: (Ljava/lang/String;)Lsrc/main/Bucket; - */ -JNIEXPORT jobject JNICALL Java_hermes_java_Hermes_getBucket - (JNIEnv *, jobject, jstring); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/wrapper/java/src/hermes/java/Blob.java b/wrapper/java/src/hermes/java/Blob.java deleted file mode 100644 index 293b79423..000000000 --- a/wrapper/java/src/hermes/java/Blob.java +++ /dev/null @@ -1,58 +0,0 @@ -package hermes.java; -import java.nio.ByteBuffer; - -public class Blob { - public ByteBuffer data_; - int size_; - long alloc_; - boolean is_native_; - - /** Create a blob from JNI */ - private Blob(ByteBuffer data, int size, long alloc) { - data_ = data; - size_ = size; - alloc_ = alloc; - is_native_ = true; - data_.position(0); - } - - /** Create a blob within Java */ - public Blob(ByteBuffer data) { - data_ = ByteBuffer.allocateDirect(data.capacity()); - data_.put(data); - size_ = data.capacity(); - alloc_ = 0; - is_native_ = false; - data_.position(0); - } - - /** Convert a string to a Blob */ - public static Blob fromString(String data) { - return new Blob(ByteBuffer.wrap(data.getBytes())); - } - - /** Call ByteBuffer array method */ - public byte[] array() { - byte[] bytes = new byte[size_]; - data_.get(bytes); - return bytes; - } - - /** Free data allocated by JNI */ - private native void freeNative(); - - /** Release this blob */ - public void close() { - if (is_native_) { - freeNative(); - } - } - - public boolean equals(Blob other) { - return data_.equals(other.data_); - } - - static { - System.loadLibrary("hermes_java_Blob"); - } -} diff --git a/wrapper/java/src/hermes/java/Bucket.java b/wrapper/java/src/hermes/java/Bucket.java deleted file mode 100644 index ca7f00e97..000000000 --- a/wrapper/java/src/hermes/java/Bucket.java +++ /dev/null @@ -1,68 +0,0 @@ -package hermes.java; -import java.util.Vector; -import hermes.java.UniqueId; -import hermes.java.MdLockType; -import hermes.java.Blob; - -public class Bucket { - public UniqueId bkt_id_; - - /**==================================== - * BUCKET operatoins - * ===================================*/ - - /** Used to construct bucket in JNI */ - private Bucket(UniqueId bkt_id) { - bkt_id_ = bkt_id; - } - - /** Lock this bucket */ - public native void lock(int lock_type); - public void lock(MdLockType lock_type) { - lock(lock_type.ordinal()); - } - - /** Unlock this bucket */ - public native void unlock(int lock_type); - public void unlock(MdLockType lock_type) { - unlock(lock_type.ordinal()); - } - - /** Get all blob IDs contained in this bucket */ - public native Vector getContainedBlobIds(); - - /** Destroy this bucket */ - public native void destroy(); - - /**==================================== - * BLOB operations - * ===================================*/ - - /** Get the ID of a blob from a bucket */ - public native UniqueId getBlobId(String name); - - /** Get a blob from the bucket */ - public native Blob get(UniqueId blob_id); - - /** Put a blob into the bucket */ - public native UniqueId put(String blob_name, Blob data); - - /** Lock a blob */ - public native void lockBlob(UniqueId blob_id, int lock_type); - public void lockBlob(UniqueId blob_id, MdLockType lock_type) { - unlockBlob(blob_id, lock_type.ordinal()); - } - - /** Unlock a blob */ - public native void unlockBlob(UniqueId blob_id, int lock_type); - public void unlockBlob(UniqueId blob_id, MdLockType lock_type) { - unlockBlob(blob_id, lock_type.ordinal()); - } - - /** Destroy a blob */ - public native void destroyBlob(UniqueId blob_id); - - static { - System.loadLibrary("hermes_java_Bucket"); - } -} diff --git a/wrapper/java/src/hermes/java/Hermes.java b/wrapper/java/src/hermes/java/Hermes.java deleted file mode 100644 index 6dfd745e9..000000000 --- a/wrapper/java/src/hermes/java/Hermes.java +++ /dev/null @@ -1,20 +0,0 @@ -package hermes.java; -import hermes.java.Bucket; - -public class Hermes { - private static Hermes instance_ = null; - - static public Hermes getInstance() { - if (instance_ == null) { - instance_ = new Hermes(); - } - return instance_; - } - - public native void create(); - public native Bucket getBucket(String bkt_name); - - static { - System.loadLibrary("hermes_java_Hermes"); - } -} diff --git a/wrapper/java/src/hermes/java/MdLockType.java b/wrapper/java/src/hermes/java/MdLockType.java deleted file mode 100644 index 5d0f6f7d1..000000000 --- a/wrapper/java/src/hermes/java/MdLockType.java +++ /dev/null @@ -1,7 +0,0 @@ -package hermes.java; -public enum MdLockType { - kInternalRead, /**< Internal read lock used by Hermes */ - kInternalWrite, /**< Internal write lock by Hermes */ - kExternalRead, /**< External is used by programs */ - kExternalWrite, /**< External is used by programs */ -} diff --git a/wrapper/java/src/hermes/java/UniqueId.java b/wrapper/java/src/hermes/java/UniqueId.java deleted file mode 100644 index 9473bee13..000000000 --- a/wrapper/java/src/hermes/java/UniqueId.java +++ /dev/null @@ -1,17 +0,0 @@ -package hermes.java; - -public class UniqueId { - public long unique_; - public int node_id_; - - UniqueId(long unique, int node_id) { - unique_ = unique; - node_id_ = node_id; - } - - public boolean equals(UniqueId other) { - return unique_ == other.unique_ && node_id_ == other.node_id_; - } - - public boolean isNull() { return unique_ == 0; } -} diff --git a/wrapper/java/src/test/java/HermesJniTest.java b/wrapper/java/src/test/java/HermesJniTest.java deleted file mode 100644 index 6244cd8cf..000000000 --- a/wrapper/java/src/test/java/HermesJniTest.java +++ /dev/null @@ -1,40 +0,0 @@ -import org.junit.Test; -import static org.junit.Assert.*; - -import java.util.Vector; -import hermes.java.Blob; -import hermes.java.Bucket; -import hermes.java.Hermes; -import hermes.java.UniqueId; -import hermes.java.MdLockType; - -import java.lang.management.ManagementFactory; - -public class HermesJniTest { - @Test - public void testBucketPutGet() { - String pid = ManagementFactory.getRuntimeMXBean().getName(); - System.out.println(pid); - Hermes hermes = Hermes.getInstance(); - hermes.create(); - Bucket bkt = hermes.getBucket("hello"); - Blob old_data = Blob.fromString("this is my DATA!"); - - bkt.lock(MdLockType.kExternalWrite); - UniqueId blob_id = bkt.put("0", old_data); - bkt.unlock(MdLockType.kExternalWrite); - - bkt.lockBlob(blob_id, MdLockType.kExternalRead); - Blob new_data = bkt.get(blob_id); - bkt.unlockBlob(blob_id, MdLockType.kExternalRead); - - Vector blob_ids = bkt.getContainedBlobIds(); - assertEquals(blob_ids.size(), 1); - assertTrue(blob_ids.get(0).equals(blob_id)); - - bkt.destroy(); - assertTrue(new_data.equals(old_data)); - new_data.close(); - old_data.close(); - } -} diff --git a/wrapper/java/tests.py b/wrapper/java/tests.py deleted file mode 100644 index b9fae8a53..000000000 --- a/wrapper/java/tests.py +++ /dev/null @@ -1,21 +0,0 @@ -from py_hermes_ci.test_manager import TestManager -from jarvis_util import * - - -class JavaWrapperTestManager(TestManager): - def spawn_all_nodes(self): - return self.spawn_info() - - def set_paths(self): - self.KVSTORE_CMD = f"./gradlew test" - self.disable_testing = False - - def test_hermes_java(self): - return - spawn_info = self.spawn_info(nprocs=1, - hermes_conf='hermes_server', - cwd=f"{self.MY_DIR}/java") - self.start_daemon(spawn_info) - node = Exec(self.KVSTORE_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code From 6cabe2ff5f7ffaf1b02bb58aa52269527a1c9584 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 15 Sep 2023 10:09:38 -0500 Subject: [PATCH 009/191] Update spack --- README.md | 24 ++++--------------- .../hermes/packages/hermes => ci}/__init__.py | 0 {scripts/ci => ci}/coverage.sh | 0 {scripts/ci => ci}/external/CMakeLists.txt | 0 {scripts/ci => ci}/external/test.cc | 0 ci/hermes/__init__.py | 0 ci/hermes/packages/__init__.py | 0 ci/hermes/packages/hermes/__init__.py | 0 .../hermes/packages/hermes/package.py | 22 ++++++++++++----- {scripts => ci}/hermes/repo.yaml | 0 {scripts/ci => ci}/install_deps.sh | 0 {scripts/ci => ci}/install_docs.sh | 0 {scripts/ci => ci}/install_hshm.sh | 0 {scripts/ci => ci}/packages.yaml | 0 {scripts/ci => ci}/py_hermes_ci/bin/run_test | 0 .../py_hermes_ci/py_hermes_ci/test_manager.py | 0 16 files changed, 21 insertions(+), 25 deletions(-) rename {scripts/hermes/packages/hermes => ci}/__init__.py (100%) rename {scripts/ci => ci}/coverage.sh (100%) rename {scripts/ci => ci}/external/CMakeLists.txt (100%) rename {scripts/ci => ci}/external/test.cc (100%) create mode 100644 ci/hermes/__init__.py create mode 100644 ci/hermes/packages/__init__.py create mode 100644 ci/hermes/packages/hermes/__init__.py rename {scripts => ci}/hermes/packages/hermes/package.py (68%) rename {scripts => ci}/hermes/repo.yaml (100%) rename {scripts/ci => ci}/install_deps.sh (100%) rename {scripts/ci => ci}/install_docs.sh (100%) rename {scripts/ci => ci}/install_hshm.sh (100%) rename {scripts/ci => ci}/packages.yaml (100%) rename {scripts/ci => ci}/py_hermes_ci/bin/run_test (100%) rename {scripts/ci => ci}/py_hermes_ci/py_hermes_ci/test_manager.py (100%) diff --git a/README.md b/README.md index 53d224120..d96b02b9b 100644 --- a/README.md +++ b/README.md @@ -22,28 +22,14 @@ Hermes is a heterogeneous-aware, multi-tiered, dynamic, and distributed I/O buff [Spack](https://spack.io/) is the easiest way to get Hermes and all its dependencies installed. -```bash -# Install spack if you don't already have it -SPACK_DIR=~/spack -git clone https://github.com/spack/spack ${SPACK_DIR} -. ${SPACK_DIR}/share/spack/setup-env.sh -spack install hermes -``` - -If this doesn't work, the Hermes and Mochi spack recipes might be out of sync -with the version of spack you're using. In that case, you can try the most -up-to-date repos. - ```bash # set location of hermes_file_staging -STAGE_DIR=~/hermes_stage -MOCHI_REPO=${STAGE_DIR}/mochi -HERMES_REPO=${STAGE_DIR}/hermes -git clone https://github.com/mochi-hpc/mochi-spack-packages.git ${MOCHI_REPO} -git clone https://github.com/HDFGroup/hermes ${HERMES_REPO} -spack repo add ${MOCHI_REPO} +git clone https://github.com/HDFGroup/hermes spack repo add ${HERMES_REPO}/ci/hermes -spack install hermes +# The stable 1.1. development branch +spack install hermes@hdf-1.1 +# The unstable 1.1. development branch +spack install hermes@dev-1.1 ``` ### CMake diff --git a/scripts/hermes/packages/hermes/__init__.py b/ci/__init__.py similarity index 100% rename from scripts/hermes/packages/hermes/__init__.py rename to ci/__init__.py diff --git a/scripts/ci/coverage.sh b/ci/coverage.sh similarity index 100% rename from scripts/ci/coverage.sh rename to ci/coverage.sh diff --git a/scripts/ci/external/CMakeLists.txt b/ci/external/CMakeLists.txt similarity index 100% rename from scripts/ci/external/CMakeLists.txt rename to ci/external/CMakeLists.txt diff --git a/scripts/ci/external/test.cc b/ci/external/test.cc similarity index 100% rename from scripts/ci/external/test.cc rename to ci/external/test.cc diff --git a/ci/hermes/__init__.py b/ci/hermes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ci/hermes/packages/__init__.py b/ci/hermes/packages/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ci/hermes/packages/hermes/__init__.py b/ci/hermes/packages/hermes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py similarity index 68% rename from scripts/hermes/packages/hermes/package.py rename to ci/hermes/packages/hermes/package.py index f4f69caba..8d0cda713 100644 --- a/scripts/hermes/packages/hermes/package.py +++ b/ci/hermes/packages/hermes/package.py @@ -1,9 +1,22 @@ from spack import * class Hermes(CMakePackage): - homepage = "https://github.com/lukemartinlogan/labstor/wiki" - git = "https://github.com/lukemartinlogan/labstor.git" - version('master', branch='dev') + homepage = "http://www.cs.iit.edu/~scs/assets/projects/Hermes/Hermes.html" + url = "https://github.com/HDFGroup/hermes/tarball/master" + git = "https://github.com/HDFGroup/hermes.git" + + version('master', branch='master') + version('pnnl', branch='pnnl') + version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', branch='dev') + version('dev-1.1', git='https://github.com/lukemartinlogan/hermes.git', branch='hermes-1.1') + version('hdf-1.1', git='https://github.com/HDFGroup/hermes.git', branch='hermes-1.1') + + variant('vfd', default=False, description='Enable HDF5 VFD') + variant('ares', default=False, description='Enable full libfabric install') + variant('debug', default=False, description='Enable debug mode') + variant('debug', default=False, description='Build shared libraries') + variant('zmq', default=False, description='Build ZeroMQ tests') + depends_on('mochi-thallium~cereal@0.10.1') depends_on('cereal') depends_on('catch2@3.0.1') @@ -13,9 +26,6 @@ class Hermes(CMakePackage): depends_on('hermes_shm') depends_on('libzmq', '+zmq') - variant('debug', default=False, description='Build shared libraries') - variant('zmq', default=False, description='Build ZeroMQ tests') - def cmake_args(self): args = ['-DCMAKE_INSTALL_PREFIX={}'.format(self.prefix)] if '+debug' in self.spec: diff --git a/scripts/hermes/repo.yaml b/ci/hermes/repo.yaml similarity index 100% rename from scripts/hermes/repo.yaml rename to ci/hermes/repo.yaml diff --git a/scripts/ci/install_deps.sh b/ci/install_deps.sh similarity index 100% rename from scripts/ci/install_deps.sh rename to ci/install_deps.sh diff --git a/scripts/ci/install_docs.sh b/ci/install_docs.sh similarity index 100% rename from scripts/ci/install_docs.sh rename to ci/install_docs.sh diff --git a/scripts/ci/install_hshm.sh b/ci/install_hshm.sh similarity index 100% rename from scripts/ci/install_hshm.sh rename to ci/install_hshm.sh diff --git a/scripts/ci/packages.yaml b/ci/packages.yaml similarity index 100% rename from scripts/ci/packages.yaml rename to ci/packages.yaml diff --git a/scripts/ci/py_hermes_ci/bin/run_test b/ci/py_hermes_ci/bin/run_test similarity index 100% rename from scripts/ci/py_hermes_ci/bin/run_test rename to ci/py_hermes_ci/bin/run_test diff --git a/scripts/ci/py_hermes_ci/py_hermes_ci/test_manager.py b/ci/py_hermes_ci/py_hermes_ci/test_manager.py similarity index 100% rename from scripts/ci/py_hermes_ci/py_hermes_ci/test_manager.py rename to ci/py_hermes_ci/py_hermes_ci/test_manager.py From 01601cccad9421b039fd713f48160ae9ed6afdf1 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 15 Sep 2023 10:35:03 -0500 Subject: [PATCH 010/191] Fix spack repo file --- ci/hermes/repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/hermes/repo.yaml b/ci/hermes/repo.yaml index 0fdccdc29..039fd3384 100644 --- a/ci/hermes/repo.yaml +++ b/ci/hermes/repo.yaml @@ -1,2 +1,2 @@ repo: - namespace: 'labstor' + namespace: 'hermes' From 735ea4ca56a061481e5de27e66ea00c9459755ba Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 15 Sep 2023 10:41:49 -0500 Subject: [PATCH 011/191] Add hermes_shm package --- ci/hermes/packages/hermes_shm/__init__.py | 0 ci/hermes/packages/hermes_shm/package.py | 44 +++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 ci/hermes/packages/hermes_shm/__init__.py create mode 100644 ci/hermes/packages/hermes_shm/package.py diff --git a/ci/hermes/packages/hermes_shm/__init__.py b/ci/hermes/packages/hermes_shm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py new file mode 100644 index 000000000..58cfbea28 --- /dev/null +++ b/ci/hermes/packages/hermes_shm/package.py @@ -0,0 +1,44 @@ +from spack import * + +class HermesShm(CMakePackage): + homepage = "https://github.com/lukemartinlogan/hermes_shm/wiki" + git = "https://github.com/lukemartinlogan/hermes_shm.git" + version('master', branch='master') + depends_on('mochi-thallium~cereal@0.10.1') + depends_on('catch2@3.0.1') + depends_on('mpi') + depends_on('boost@1.7:') + depends_on('cereal') + depends_on('doxygen@1.9.3') + + variant('debug', default=False, description='Build shared libraries') + + def cmake_args(self): + args = [] + if '+debug' in self.spec: + args.append('-DCMAKE_BUILD_TYPE=Debug') + return args + + def set_include(self, env, path): + env.append_flags('CFLAGS', '-I{}'.format(path)) + env.append_flags('CXXFLAGS', '-I{}'.format(path)) + env.prepend_path('INCLUDE', '{}'.format(path)) + env.prepend_path('CPATH', '{}'.format(path)) + + def set_lib(self, env, path): + env.prepend_path('LIBRARY_PATH', path) + env.prepend_path('LD_LIBRARY_PATH', path) + env.append_flags('LDFLAGS', '-L{}'.format(path)) + + def set_flags(self, env): + self.set_include(env, '{}/include'.format(self.prefix)) + self.set_include(env, '{}/include'.format(self.prefix)) + self.set_lib(env, '{}/lib'.format(self.prefix)) + self.set_lib(env, '{}/lib64'.format(self.prefix)) + env.prepend_path('CMAKE_PREFIX_PATH', '{}/cmake'.format(self.prefix)) + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + self.set_flags(spack_env) + + def setup_run_environment(self, env): + self.set_flags(env) From 208362aa856047fb5d333bf9a51d25f44809899f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 15 Sep 2023 10:58:11 -0500 Subject: [PATCH 012/191] Add gettid wrapper --- include/labstor/work_orchestrator/worker.h | 6 +++++- src/worker.cc | 2 +- tasks_required/proc_queue/src/proc_queue.cc | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 6ab2f7509..1894df6ad 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -12,6 +12,10 @@ #include "affinity.h" #include "labstor/network/rpc_thallium.h" +static inline pid_t GetLinuxTid() { + return syscall(SYS_gettid); +} + namespace labstor { #define WORKER_CONTINUOUS_POLLING BIT_OPT(u32, 0) @@ -167,7 +171,7 @@ class Worker { EnableContinuousPolling(); retries_ = 1; pid_ = 0; - pthread_id_ = gettid(); + pthread_id_ = GetLinuxTid(); // TODO(llogan): implement reserve for group group_.resize(512); group_.resize(0); diff --git a/src/worker.cc b/src/worker.cc index 1d772e2bb..ee514fdae 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -9,7 +9,7 @@ namespace labstor { void Worker::Loop() { - pid_ = gettid(); + pid_ = GetLinuxTid(); WorkOrchestrator *orchestrator = LABSTOR_WORK_ORCHESTRATOR; while (orchestrator->IsAlive()) { try { diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/tasks_required/proc_queue/src/proc_queue.cc index 4b82a2966..31016b865 100644 --- a/tasks_required/proc_queue/src/proc_queue.cc +++ b/tasks_required/proc_queue/src/proc_queue.cc @@ -25,7 +25,7 @@ class Server : public TaskLib { case PushTaskPhase::kSchedule: { task->ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->subtask_); HILOG(kDebug, "Scheduling task {} on state {} tid {}", - task->ptr_->task_node_, task->ptr_->task_state_, gettid()); + task->ptr_->task_node_, task->ptr_->task_state_, GetLinuxTid()); if (task->ptr_->IsFireAndForget()) { task->ptr_->UnsetFireAndForget(); } From 08f75ba303c76155fe94138d1bba3d7fc7fbbc38 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 15 Sep 2023 11:48:33 -0500 Subject: [PATCH 013/191] Load client config test --- tasks/hermes/include/hermes/config_manager.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/hermes/include/hermes/config_manager.h b/tasks/hermes/include/hermes/config_manager.h index ed41040c4..17656f77e 100644 --- a/tasks/hermes/include/hermes/config_manager.h +++ b/tasks/hermes/include/hermes/config_manager.h @@ -46,7 +46,7 @@ class ConfigurationManager { void LoadClientConfig(std::string &config_path) { // Load hermes config if (config_path.empty()) { - config_path = GetEnvSafe(Constant::kHermesServerConf); + config_path = GetEnvSafe(Constant::kHermesClientConf); } HILOG(kInfo, "Loading client configuration: {}", config_path) client_config_.LoadFromFile(config_path); @@ -55,7 +55,7 @@ class ConfigurationManager { void LoadServerConfig(std::string &config_path) { // Load hermes config if (config_path.empty()) { - config_path = GetEnvSafe(Constant::kHermesServerConf); + config_path = GetEnvSafe(Constant::kHermesClientConf); } HILOG(kInfo, "Loading server configuration: {}", config_path) server_config_.LoadFromFile(config_path); From 74513160a45ffd8742af5f99a2ec9727e1cd9f05 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 15 Sep 2023 11:48:49 -0500 Subject: [PATCH 014/191] Load client config test --- tasks/hermes/include/hermes/config_manager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/hermes/include/hermes/config_manager.h b/tasks/hermes/include/hermes/config_manager.h index 17656f77e..0a4edf314 100644 --- a/tasks/hermes/include/hermes/config_manager.h +++ b/tasks/hermes/include/hermes/config_manager.h @@ -55,7 +55,7 @@ class ConfigurationManager { void LoadServerConfig(std::string &config_path) { // Load hermes config if (config_path.empty()) { - config_path = GetEnvSafe(Constant::kHermesClientConf); + config_path = GetEnvSafe(Constant::kHermesServerConf); } HILOG(kInfo, "Loading server configuration: {}", config_path) server_config_.LoadFromFile(config_path); From b70c290e50697f3e76ac59da2299e489cac9b89c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 16 Sep 2023 02:04:52 -0500 Subject: [PATCH 015/191] Use hermes/hermes.h --- tasks/hermes_adapters/mpiio/mpiio_api.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/hermes_adapters/mpiio/mpiio_api.cc b/tasks/hermes_adapters/mpiio/mpiio_api.cc index e61c857e2..3a05de83f 100644 --- a/tasks/hermes_adapters/mpiio/mpiio_api.cc +++ b/tasks/hermes_adapters/mpiio/mpiio_api.cc @@ -12,8 +12,8 @@ bool mpiio_intercepted = true; -#include -#include +#include +#include #include "mpiio_api.h" #include "mpiio_fs_api.h" From 808b6684fc1d4ddc4114e7334d03fc1d5f74e8f9 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 16 Sep 2023 07:35:51 -0500 Subject: [PATCH 016/191] Make it so bucket size always gets updated after PUT and GET --- .../filesystem/filesystem_mdm.h | 2 - tasks/hermes_adapters/mpiio/mpiio_api.cc | 1 - tasks/hermes_adapters/mpiio/mpiio_fs_api.h | 10 +++-- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 38 +++++++++++-------- .../hermes_bucket_mdm/hermes_bucket_mdm.h | 5 ++- .../hermes_bucket_mdm_tasks.h | 18 ++++++--- .../src/hermes_bucket_mdm.cc | 11 +++++- 7 files changed, 53 insertions(+), 32 deletions(-) diff --git a/tasks/hermes_adapters/filesystem/filesystem_mdm.h b/tasks/hermes_adapters/filesystem/filesystem_mdm.h index 065d05893..ac2b6d0ec 100644 --- a/tasks/hermes_adapters/filesystem/filesystem_mdm.h +++ b/tasks/hermes_adapters/filesystem/filesystem_mdm.h @@ -108,7 +108,5 @@ class MetadataManager { hshm::Singleton::GetInstance() #define HERMES_FS_METADATA_MANAGER_T hermes::adapter::fs::MetadataManager* -#define HERMES_FS_THREAD_POOL \ - hshm::EasySingleton::GetInstance() #endif // HERMES_ADAPTER_METADATA_MANAGER_H diff --git a/tasks/hermes_adapters/mpiio/mpiio_api.cc b/tasks/hermes_adapters/mpiio/mpiio_api.cc index 3a05de83f..e9dbb397d 100644 --- a/tasks/hermes_adapters/mpiio/mpiio_api.cc +++ b/tasks/hermes_adapters/mpiio/mpiio_api.cc @@ -24,7 +24,6 @@ bool mpiio_intercepted = true; /** * Namespace declarations */ -using hermes::ThreadPool; using hermes::adapter::fs::MetadataManager; using hermes::adapter::fs::File; using hermes::adapter::fs::AdapterStat; diff --git a/tasks/hermes_adapters/mpiio/mpiio_fs_api.h b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h index 3e51e7558..53a8d3a07 100644 --- a/tasks/hermes_adapters/mpiio/mpiio_fs_api.h +++ b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h @@ -149,7 +149,8 @@ class MpiioFs : public Filesystem { MPI_Datatype datatype, MPI_Request *request, FsIoOptions opts) { HILOG(kDebug, "Starting an asynchronous write") - auto mdm = HERMES_FS_METADATA_MANAGER; + // TODO(llogan): FIX + /*auto mdm = HERMES_FS_METADATA_MANAGER; auto pool = HERMES_FS_THREAD_POOL; Task* hreq = new HermesRequest(); auto lambda = [](MpiioFs *fs, File &f, AdapterStat &stat, const void *ptr, @@ -161,12 +162,13 @@ class MpiioFs : public Filesystem { auto func = std::bind(lambda, this, f, stat, ptr, count, datatype, &hreq->io_status.mpi_status_, opts); hreq->return_future = pool->run(func); - mdm->request_map.emplace(reinterpret_cast(request), hreq); + mdm->request_map.emplace(reinterpret_cast(request), hreq);*/ return MPI_SUCCESS; } int Wait(MPI_Request *req, MPI_Status *status) { - auto mdm = HERMES_FS_METADATA_MANAGER; + // TODO(llogan): FIX + /*auto mdm = HERMES_FS_METADATA_MANAGER; auto real_api = HERMES_MPIIO_API; auto iter = mdm->request_map.find(reinterpret_cast(req)); if (iter != mdm->request_map.end()) { @@ -176,7 +178,7 @@ class MpiioFs : public Filesystem { mdm->request_map.erase(iter); delete (hreq); return MPI_SUCCESS; - } + }*/ return real_api->MPI_Wait(req, status); } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 5a242883c..fc0420a47 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -183,10 +183,20 @@ class Server : public TaskLib { PutBlobFreeBuffersPhase(blob_info, task); } + // Determine amount of additional buffering space needed + Context ctx; + size_t needed_space = task->blob_off_ + task->data_size_; + size_t size_diff = 0; + if (needed_space > blob_info.max_blob_size_) { + size_diff = needed_space - blob_info.max_blob_size_; + } + blob_info.blob_size_ += size_diff; + HILOG(kDebug, "The size diff is {} bytes", size_diff) + // Stage in blob data from FS task->data_ptr_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->data_); task->data_ptr_.shm_ = task->data_; - task->data_off_ = 0; + task->data_off_ = size_diff; if (task->filename_->size() > 0 && blob_info.blob_size_ == 0) { adapter::BlobPlacement plcmnt; plcmnt.DecodeBlobName(*task->blob_name_); @@ -206,16 +216,6 @@ class Server : public TaskLib { task->flags_.SetBits(HERMES_DID_STAGE_IN); } - // Determine amount of additional buffering space needed - Context ctx; - size_t needed_space = task->blob_off_ + task->data_size_; - size_t size_diff = 0; - if (needed_space > blob_info.max_blob_size_) { - size_diff = needed_space - blob_info.max_blob_size_; - } - blob_info.blob_size_ += size_diff; - HILOG(kDebug, "The size diff is {} bytes", size_diff) - // Initialize archives HSHM_MAKE_AR0(task->schema_, nullptr); HSHM_MAKE_AR0(task->bdev_writes_, nullptr); @@ -332,11 +332,14 @@ class Server : public TaskLib { LABSTOR_CLIENT->FreeBuffer(task->data_ptr_); } // Update the bucket statistics - if (task->data_off_) { - bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, - task->tag_id_, - task->data_off_); + int update_mode = bucket_mdm::UpdateSizeMode::kAdd; + if (task->flags_.Any(HERMES_DID_STAGE_IN)) { + update_mode = bucket_mdm::UpdateSizeMode::kCap; } + bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, + task->tag_id_, + task->data_off_, + update_mode); if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, task->tag_id_, @@ -593,6 +596,11 @@ class Server : public TaskLib { LABSTOR_CLIENT->DelTask(free_task); free_tasks.pop_back(); } + BlobInfo &blob_info = blob_map_[task->blob_id_]; + bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, + task->tag_id_, + -(ssize_t)blob_info.blob_size_, + bucket_mdm::UpdateSizeMode::kAdd); HSHM_DESTROY_AR(task->free_tasks_); blob_map_.erase(task->blob_id_); task->SetModuleComplete(); diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index c72a1a036..479085856 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -66,10 +66,11 @@ class Client : public TaskLibClient { void AsyncUpdateSizeConstruct(UpdateSizeTask *task, const TaskNode &task_node, TagId tag_id, - size_t update, bitfield32_t flags = bitfield32_t(0)) { + size_t update, + int mode) { LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(tag_id.node_id_), id_, - tag_id, update, flags); + tag_id, update, mode); } LABSTOR_TASK_NODE_PUSH_ROOT(UpdateSize); diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index e2bfb9ad3..f40c18ea2 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -126,11 +126,17 @@ struct SetBlobMdmTask : public Task, TaskFlags { void ReplicateEnd() {} }; +class UpdateSizeMode { + public: + TASK_METHOD_T kAdd = 0; + TASK_METHOD_T kCap = 0; +}; + /** Update bucket size */ struct UpdateSizeTask : public Task, TaskFlags { IN TagId tag_id_; - IN size_t update_; - IN bitfield32_t flags_; + IN ssize_t update_; + IN int mode_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -143,8 +149,8 @@ struct UpdateSizeTask : public Task, TaskFlags { const DomainId &domain_id, const TaskStateId &state_id, const TagId &tag_id, - size_t update, - bitfield32_t flags) : Task(alloc) { + ssize_t update, + int mode) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = tag_id.unique_; @@ -157,7 +163,7 @@ struct UpdateSizeTask : public Task, TaskFlags { // Custom params tag_id_ = tag_id; update_ = update; - flags_ = flags; + mode_ = mode; } /** Destructor */ @@ -167,7 +173,7 @@ struct UpdateSizeTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); - ar(tag_id_, update_, flags_); + ar(tag_id_, update_, mode_); } /** (De)serialize message return */ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index e4b924fdc..e03639c1a 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -49,8 +49,13 @@ class Server : public TaskLib { /** Update the size of the bucket */ void UpdateSize(UpdateSizeTask *task) { TagInfo &tag_info = tag_map_[task->tag_id_]; - tag_info.internal_size_ = std::max(task->update_, - tag_info.internal_size_); + ssize_t internal_size = (ssize_t) tag_info.internal_size_; + if (task->mode_ == UpdateSizeMode::kAdd) { + internal_size += task->update_; + } else { + internal_size = std::max(task->update_, internal_size); + } + tag_info.internal_size_ = (size_t) internal_size; task->SetModuleComplete(); } @@ -69,6 +74,8 @@ class Server : public TaskLib { size_t update_size = task->page_size_ - cur_page_off; size_t max_pages = task->data_size_ / task->page_size_ + 1; size_t cur_size = 0; + HILOG(kDebug, "(node {}) Bucket size {}, page_size {}, cur_page {} (task_node={})", + LABSTOR_CLIENT->node_id_, bucket_size, task->page_size_, cur_page, task->task_node_) HSHM_MAKE_AR0(task->append_info_, nullptr); std::vector &append_info = *task->append_info_; append_info.reserve(max_pages); From 82490f4fa079793453bce0e3de81856e890c3d78 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 16 Sep 2023 09:07:00 -0500 Subject: [PATCH 017/191] IOR read seems to pass again --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 55 +++++++++++++------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index fc0420a47..049445810 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -183,45 +183,55 @@ class Server : public TaskLib { PutBlobFreeBuffersPhase(blob_info, task); } - // Determine amount of additional buffering space needed - Context ctx; - size_t needed_space = task->blob_off_ + task->data_size_; - size_t size_diff = 0; - if (needed_space > blob_info.max_blob_size_) { - size_diff = needed_space - blob_info.max_blob_size_; - } - blob_info.blob_size_ += size_diff; - HILOG(kDebug, "The size diff is {} bytes", size_diff) - // Stage in blob data from FS task->data_ptr_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->data_); task->data_ptr_.shm_ = task->data_; - task->data_off_ = size_diff; if (task->filename_->size() > 0 && blob_info.blob_size_ == 0) { adapter::BlobPlacement plcmnt; plcmnt.DecodeBlobName(*task->blob_name_); + HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", + task->page_size_, task->filename_->str(), plcmnt.bucket_off_); LPointer new_data_ptr = LABSTOR_CLIENT->AllocateBuffer(task->page_size_); int fd = HERMES_POSIX_API->open(task->filename_->c_str(), O_RDONLY); - int ret = HERMES_POSIX_API->pread(fd, new_data_ptr.ptr_, task->page_size_, plcmnt.bucket_off_); + if (fd < 0) { + HELOG(kError, "Failed to open file {}", task->filename_->str()); + } + int ret = HERMES_POSIX_API->pread(fd, new_data_ptr.ptr_, task->page_size_, (off_t)plcmnt.bucket_off_); if (ret < 0) { // TODO(llogan): ret != page_size_ will require knowing file size before-hand HELOG(kError, "Failed to stage in {} bytes from {}", task->page_size_, task->filename_->str()); } HERMES_POSIX_API->close(fd); - memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, task->data_ptr_.ptr_, task->page_size_); + memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, task->data_ptr_.ptr_, task->data_size_); task->data_ptr_ = new_data_ptr; task->blob_off_ = 0; - task->data_size_ = task->page_size_; + task->data_size_ = ret; task->data_off_ = plcmnt.bucket_off_ + task->blob_off_ + task->data_size_; task->flags_.SetBits(HERMES_DID_STAGE_IN); + HILOG(kDebug, "Staged {} bytes from the backend file {}", + task->data_size_, task->filename_->str()); } + // Determine amount of additional buffering space needed + size_t needed_space = task->blob_off_ + task->data_size_; + size_t size_diff = 0; + if (needed_space > blob_info.max_blob_size_) { + size_diff = needed_space - blob_info.max_blob_size_; + } + if (!task->flags_.Any(HERMES_DID_STAGE_IN)) { + task->data_off_ = size_diff; + } + blob_info.blob_size_ += size_diff; + HILOG(kDebug, "The size diff is {} bytes", size_diff) + + // Initialize archives HSHM_MAKE_AR0(task->schema_, nullptr); HSHM_MAKE_AR0(task->bdev_writes_, nullptr); // Use DPE if (size_diff > 0) { + Context ctx; auto *dpe = DpeFactory::Get(ctx.dpe_); dpe->Placement({size_diff}, targets_, ctx, *task->schema_); task->phase_ = PutBlobPhase::kAllocate; @@ -289,8 +299,10 @@ class Server : public TaskLib { size_t blob_off = 0, buf_off = 0; HILOG(kDebug, "Number of buffers {}", blob_info.buffers_.size()); for (BufferInfo &buf : blob_info.buffers_) { - if (task->blob_off_ <= blob_off) { - size_t rel_off = blob_off - task->blob_off_; + size_t blob_left = blob_off; + size_t blob_right = blob_off + buf.t_size_; + if (blob_left <= task->blob_off_ && task->blob_off_ < blob_right) { + size_t rel_off = task->blob_off_ - blob_off; size_t tgt_off = buf.t_off_ + rel_off; size_t buf_size = buf.t_size_ - rel_off; if (blob_off + buf_size > task->blob_off_ + task->data_size_) { @@ -306,7 +318,7 @@ class Server : public TaskLib { } blob_off += buf.t_size_; } - if (blob_off == 0) { + if (blob_off < task->data_size_) { HELOG(kFatal, "Something annoying happened"); } blob_info.max_blob_size_ = blob_off; @@ -361,7 +373,6 @@ class Server : public TaskLib { } void GetBlobGetPhase(GetBlobTask *task) { - HILOG(kDebug, "GetBlobTask start"); BlobInfo &blob_info = blob_map_[task->blob_id_]; HSHM_MAKE_AR0(task->bdev_reads_, nullptr); std::vector &read_tasks = *task->bdev_reads_; @@ -369,12 +380,16 @@ class Server : public TaskLib { if (task->data_size_ < 0) { task->data_size_ = (ssize_t)(blob_info.blob_size_ - task->blob_off_); } + HILOG(kDebug, "Getting blob {} of size {} starting at offset {} (total_blob_size={}, buffers={})", + task->blob_id_, task->data_size_, task->blob_off_, blob_info.blob_size_, blob_info.buffers_.size()); size_t blob_off = 0, buf_off = 0; hipc::mptr blob_data_mptr(task->data_); char *blob_buf = blob_data_mptr.get(); for (BufferInfo &buf : blob_info.buffers_) { - if (task->blob_off_ <= blob_off) { - size_t rel_off = blob_off - task->blob_off_; + size_t blob_left = blob_off; + size_t blob_right = blob_off + buf.t_size_; + if (blob_left <= task->blob_off_ && task->blob_off_ < blob_right) { + size_t rel_off = task->blob_off_ - blob_off; size_t tgt_off = buf.t_off_ + rel_off; size_t buf_size = buf.t_size_ - rel_off; if (blob_off + buf_size > task->blob_off_ + task->data_size_) { From 2158144f5d2e2476fe8a45cfb1dc3649ff05d1cd Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 17 Sep 2023 16:31:17 -0500 Subject: [PATCH 018/191] Partially fix file size inconsistency --- CMakeLists.txt | 2 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 3 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 55 +++++++++++-------- .../hermes_bucket_mdm_tasks.h | 2 +- .../src/hermes_bucket_mdm.cc | 12 ++-- tasks/posix_bdev/src/posix_bdev.cc | 10 ++-- 6 files changed, 48 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b42e4d49..481df1597 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10) -project(labstor) +project(hermes) #----------------------------------------------------------------------------- # Define Options diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index e8813efcb..f5d8e1b64 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -213,7 +213,8 @@ class PutBlobPhase { #define HERMES_BLOB_REPLACE BIT_OPT(u32, 0) #define HERMES_BLOB_APPEND BIT_OPT(u32, 1) #define HERMES_DID_STAGE_IN BIT_OPT(u32, 2) -#define HERMES_BLOB_DID_CREATE BIT_OPT(u32, 3) +#define HERMES_IS_FILE BIT_OPT(u32, 3) +#define HERMES_BLOB_DID_CREATE BIT_OPT(u32, 4) /** A task to put data in a blob */ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 049445810..78928210d 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -186,30 +186,39 @@ class Server : public TaskLib { // Stage in blob data from FS task->data_ptr_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->data_); task->data_ptr_.shm_ = task->data_; - if (task->filename_->size() > 0 && blob_info.blob_size_ == 0) { + if (task->filename_->size() > 0) { adapter::BlobPlacement plcmnt; plcmnt.DecodeBlobName(*task->blob_name_); - HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", - task->page_size_, task->filename_->str(), plcmnt.bucket_off_); - LPointer new_data_ptr = LABSTOR_CLIENT->AllocateBuffer(task->page_size_); - int fd = HERMES_POSIX_API->open(task->filename_->c_str(), O_RDONLY); - if (fd < 0) { - HELOG(kError, "Failed to open file {}", task->filename_->str()); - } - int ret = HERMES_POSIX_API->pread(fd, new_data_ptr.ptr_, task->page_size_, (off_t)plcmnt.bucket_off_); - if (ret < 0) { - // TODO(llogan): ret != page_size_ will require knowing file size before-hand - HELOG(kError, "Failed to stage in {} bytes from {}", task->page_size_, task->filename_->str()); - } - HERMES_POSIX_API->close(fd); - memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, task->data_ptr_.ptr_, task->data_size_); - task->data_ptr_ = new_data_ptr; - task->blob_off_ = 0; - task->data_size_ = ret; + task->flags_.SetBits(HERMES_IS_FILE); task->data_off_ = plcmnt.bucket_off_ + task->blob_off_ + task->data_size_; - task->flags_.SetBits(HERMES_DID_STAGE_IN); - HILOG(kDebug, "Staged {} bytes from the backend file {}", - task->data_size_, task->filename_->str()); + if (blob_info.blob_size_ == 0 && + task->blob_off_ == 0 && + task->data_size_ < task->page_size_) { + HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", + task->page_size_, task->filename_->str(), plcmnt.bucket_off_); + LPointer new_data_ptr = LABSTOR_CLIENT->AllocateBuffer(task->page_size_); + int fd = HERMES_POSIX_API->open(task->filename_->c_str(), O_RDONLY); + if (fd < 0) { + HELOG(kError, "Failed to open file {}", task->filename_->str()); + } + int ret = HERMES_POSIX_API->pread(fd, new_data_ptr.ptr_, task->page_size_, (off_t)plcmnt.bucket_off_); + if (ret < 0) { + // TODO(llogan): ret != page_size_ will require knowing file size before-hand + HELOG(kError, "Failed to stage in {} bytes from {}", task->page_size_, task->filename_->str()); + } + HERMES_POSIX_API->close(fd); + memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, task->data_ptr_.ptr_, task->data_size_); + task->data_ptr_ = new_data_ptr; + task->blob_off_ = 0; + if (ret < task->blob_off_ + task->data_size_) { + task->data_size_ = task->blob_off_ + task->data_size_; + } else { + task->data_size_ = ret; + } + task->flags_.SetBits(HERMES_DID_STAGE_IN); + HILOG(kDebug, "Staged {} bytes from the backend file {}", + task->data_size_, task->filename_->str()); + } } // Determine amount of additional buffering space needed @@ -218,7 +227,7 @@ class Server : public TaskLib { if (needed_space > blob_info.max_blob_size_) { size_diff = needed_space - blob_info.max_blob_size_; } - if (!task->flags_.Any(HERMES_DID_STAGE_IN)) { + if (!task->flags_.Any(HERMES_IS_FILE)) { task->data_off_ = size_diff; } blob_info.blob_size_ += size_diff; @@ -345,7 +354,7 @@ class Server : public TaskLib { } // Update the bucket statistics int update_mode = bucket_mdm::UpdateSizeMode::kAdd; - if (task->flags_.Any(HERMES_DID_STAGE_IN)) { + if (task->flags_.Any(HERMES_IS_FILE)) { update_mode = bucket_mdm::UpdateSizeMode::kCap; } bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index f40c18ea2..5e81e71ac 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -129,7 +129,7 @@ struct SetBlobMdmTask : public Task, TaskFlags { class UpdateSizeMode { public: TASK_METHOD_T kAdd = 0; - TASK_METHOD_T kCap = 0; + TASK_METHOD_T kCap = 1; }; /** Update bucket size */ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index e03639c1a..8c9bcecf7 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -55,6 +55,8 @@ class Server : public TaskLib { } else { internal_size = std::max(task->update_, internal_size); } + HILOG(kInfo, "Updating size of tag {} from {} to {} with update {} (mode={})", + task->tag_id_, tag_info.internal_size_, internal_size, task->update_, task->mode_) tag_info.internal_size_ = (size_t) internal_size; task->SetModuleComplete(); } @@ -200,11 +202,11 @@ class Server : public TaskLib { HILOG(kDebug, "Creating tag for the first time: {} {}", tag_name.str(), tag_id) tag_id_map_.emplace(tag_name, tag_id); tag_map_.emplace(tag_id, TagInfo()); - TagInfo &info = tag_map_[tag_id]; - info.name_ = tag_name; - info.tag_id_ = tag_id; - info.owner_ = task->blob_owner_; - info.internal_size_ = task->backend_size_; + TagInfo &tag_info = tag_map_[tag_id]; + tag_info.name_ = tag_name; + tag_info.tag_id_ = tag_id; + tag_info.owner_ = task->blob_owner_; + tag_info.internal_size_ = task->backend_size_; } else { if (tag_name.size()) { HILOG(kDebug, "Found existing tag: {}", tag_name.str()) diff --git a/tasks/posix_bdev/src/posix_bdev.cc b/tasks/posix_bdev/src/posix_bdev.cc index f33cc8609..c6f87a83a 100644 --- a/tasks/posix_bdev/src/posix_bdev.cc +++ b/tasks/posix_bdev/src/posix_bdev.cc @@ -31,7 +31,7 @@ class Server : public TaskLib { dev_info.mount_point_ = canon; path_ = canon; fd_ = open(dev_info.mount_point_.c_str(), - O_TRUNC | O_CREAT, 0666); + O_TRUNC | O_CREAT | O_RDWR, 0666); if (fd_ < 0) { HELOG(kError, "Failed to open file: {}", dev_info.mount_point_); } @@ -57,17 +57,17 @@ class Server : public TaskLib { } void Write(WriteTask *task) { - size_t count = pwrite(fd_, task->buf_, task->size_, (off_t)task->disk_off_); + ssize_t count = pwrite(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { - HELOG(kError, "BORG: wrote {} bytes, but expected {}", - count, task->size_); + HELOG(kError, "BORG: wrote {} bytes, but expected {}: {}", + count, task->size_, strerror(errno)); } task->SetModuleComplete(); } void Read(ReadTask *task) { memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); - size_t count = pread(fd_, task->buf_, task->size_, (off_t)task->disk_off_); + ssize_t count = pread(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { HELOG(kError, "BORG: read {} bytes, but expected {}", count, task->size_); From fce5da27f22122cbe8f0fbca3cdf22844f95ba75 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 20 Sep 2023 07:21:35 -0500 Subject: [PATCH 019/191] Use catch2 instaed of specific version --- ci/hermes/packages/__init__.py | 0 ci/hermes/packages/hermes/package.py | 5 +++-- ci/hermes/packages/hermes_shm/package.py | 3 ++- tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) delete mode 100644 ci/hermes/packages/__init__.py diff --git a/ci/hermes/packages/__init__.py b/ci/hermes/packages/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ci/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py index 8d0cda713..8e110b39b 100644 --- a/ci/hermes/packages/hermes/package.py +++ b/ci/hermes/packages/hermes/package.py @@ -19,8 +19,9 @@ class Hermes(CMakePackage): depends_on('mochi-thallium~cereal@0.10.1') depends_on('cereal') - depends_on('catch2@3.0.1') - depends_on('mpich@3.3.2:') + # depends_on('catch2@3.0.1') + depends_on('catch2') + depends_on('mpich@3.3.2') depends_on('yaml-cpp') depends_on('boost@1.7:') depends_on('hermes_shm') diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index 58cfbea28..9225ceb29 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -5,7 +5,8 @@ class HermesShm(CMakePackage): git = "https://github.com/lukemartinlogan/hermes_shm.git" version('master', branch='master') depends_on('mochi-thallium~cereal@0.10.1') - depends_on('catch2@3.0.1') + # depends_on('catch2@3.0.1') + depends_on('catch2') depends_on('mpi') depends_on('boost@1.7:') depends_on('cereal') diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 8c9bcecf7..656a4fccc 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -55,7 +55,7 @@ class Server : public TaskLib { } else { internal_size = std::max(task->update_, internal_size); } - HILOG(kInfo, "Updating size of tag {} from {} to {} with update {} (mode={})", + HILOG(kDebug, "Updating size of tag {} from {} to {} with update {} (mode={})", task->tag_id_, tag_info.internal_size_, internal_size, task->update_, task->mode_) tag_info.internal_size_ = (size_t) internal_size; task->SetModuleComplete(); From 92a32751b1be1cd1f080852868b7fd0687435b1c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 16:23:19 -0500 Subject: [PATCH 020/191] hermes_api_bench --- benchmark/CMakeLists.txt | 8 ++-- benchmark/hermes_api_bench.cc | 71 ++++++++++++++++++++++++++-- ci/hermes/packages/hermes/package.py | 1 - 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 4eb76ca54..28c07cf82 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -29,11 +29,11 @@ target_link_libraries(test_performance_exec # ${Labstor_CLIENT_LIBRARIES} hermes Catch2::Catch2 # MPI::MPI_CXX ${ZMQ_LIBRARIES}) -add_executable(test_hermes_api +add_executable(hermes_api_bench hermes_api_bench.cc) -add_dependencies(test_hermes_api +add_dependencies(hermes_api_bench ${Labstor_CLIENT_DEPS} hermes) -target_link_libraries(test_hermes_api +target_link_libraries(hermes_api_bench ${Labstor_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) @@ -50,7 +50,7 @@ add_test(NAME test_performance COMMAND #------------------------------------------------------------------------------ install(TARGETS test_performance_exec - test_hermes_api + hermes_api_bench EXPORT ${LABSTOR_EXPORTED_TARGETS} LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index 928d53053..3ba8d0cb0 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -69,9 +69,7 @@ void GetTest(int nprocs, int rank, for (size_t i = 0; i < blobs_per_rank; ++i) { size_t blob_name_int = rank * blobs_per_rank + i; std::string name = std::to_string(blob_name_int); - hermes::Blob ret; - hermes::BlobId blob_id = bkt.GetBlobId(name); - bkt.Get(blob_id, ret, ctx); + bkt.Get(name, ret, ctx); } } t.Pause(); @@ -86,6 +84,62 @@ void PutGetTest(int nprocs, int rank, int repeat, GetTest(nprocs, rank, repeat, blobs_per_rank, blob_size); } +/** Each process PUTS into the same bucket, but with different blob names */ +void PartialPutTest(int nprocs, int rank, + int repeat, size_t blobs_per_rank, + size_t blob_size, size_t part_size) { + Timer t; + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); + hermes::Blob blob(blob_size); + t.Resume(); + for (int j = 0; j < repeat; ++j) { + for (size_t i = 0; i < blobs_per_rank; ++i) { + size_t blob_name_int = rank * blobs_per_rank + i; + std::string name = std::to_string(blob_name_int); + for (size_t cur_size = 0; cur_size < blob_size; cur_size += part_size) { + bkt.PartialPut(name, blob, cur_size, ctx); + } + } + } + t.Pause(); + GatherTimes("PartialPut", nprocs * blobs_per_rank * blob_size * repeat, t); +} + +/** + * Each process GETS from the same bucket, but with different blob names + * MUST run PutTest first. + * */ +void PartialGetTest(int nprocs, int rank, + int repeat, size_t blobs_per_rank, + size_t blob_size, size_t part_size) { + Timer t; + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); + t.Resume(); + for (int j = 0; j < repeat; ++j) { + for (size_t i = 0; i < blobs_per_rank; ++i) { + size_t blob_name_int = rank * blobs_per_rank + i; + std::string name = std::to_string(blob_name_int); + hermes::Blob ret(blob_size); + for (size_t cur_size = 0; cur_size < blob_size; cur_size += part_size) { + bkt.PartialGet(name, ret, cur_size, ctx); + } + } + } + t.Pause(); + GatherTimes("PartialGet", nprocs * blobs_per_rank * blob_size * repeat, t); +} + +/** Each process PUTs then GETs */ +void PartialPutGetTest(int nprocs, int rank, int repeat, + size_t blobs_per_rank, size_t blob_size, + size_t part_size) { + PartialPutTest(nprocs, rank, repeat, blobs_per_rank, blob_size); + MPI_Barrier(MPI_COMM_WORLD); + PartialGetTest(nprocs, rank, repeat, blobs_per_rank, blob_size); +} + /** Each process creates a set of buckets */ void CreateBucketTest(int nprocs, int rank, size_t bkts_per_rank) { @@ -120,7 +174,7 @@ void GetBucketTest(int nprocs, int rank, hapi::Bucket bkt(std::to_string(bkt_name), ctx); } t.Pause(); - GatherTimes("CreateBucket", bkts_per_rank * nprocs, t); + GatherTimes("GetBucket", bkts_per_rank * nprocs, t); } /** Each process deletes a number of buckets */ @@ -172,6 +226,7 @@ void help() { printf("USAGE: ./api_bench [mode] ...\n"); printf("USAGE: ./api_bench put [blob_size (K/M/G)] [blobs_per_rank]\n"); printf("USAGE: ./api_bench putget [blob_size (K/M/G)] [blobs_per_rank]\n"); + printf("USAGE: ./api_bench pputget [blob_size (K/M/G)] [part_size (K/M/G)] [blobs_per_rank]\n"); printf("USAGE: ./api_bench create_bkt [bkts_per_rank]\n"); printf("USAGE: ./api_bench get_bkt [bkts_per_rank]\n"); printf("USAGE: ./api_bench create_blob_1bkt [blobs_per_rank]\n"); @@ -212,7 +267,13 @@ int main(int argc, char **argv) { size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); size_t blobs_per_rank = atoi(argv[3]); PutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size); - } else if (mode == "create_bkt") { + } else if (mode == "pputget") { + REQUIRE_ARGC(5) + size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); + size_t part_size = hshm::ConfigParse::ParseSize(argv[3]); + size_t blobs_per_rank = atoi(argv[4]); + PartialPutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size, part_size); + } else if (mode == "create_bkt") { REQUIRE_ARGC(3) size_t bkts_per_rank = atoi(argv[2]); CreateBucketTest(nprocs, rank, bkts_per_rank); diff --git a/ci/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py index 8e110b39b..184922e1b 100644 --- a/ci/hermes/packages/hermes/package.py +++ b/ci/hermes/packages/hermes/package.py @@ -14,7 +14,6 @@ class Hermes(CMakePackage): variant('vfd', default=False, description='Enable HDF5 VFD') variant('ares', default=False, description='Enable full libfabric install') variant('debug', default=False, description='Enable debug mode') - variant('debug', default=False, description='Build shared libraries') variant('zmq', default=False, description='Build ZeroMQ tests') depends_on('mochi-thallium~cereal@0.10.1') From 447aad09e0293d3e20baa19092d3a43138c8a8ee Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 16:24:28 -0500 Subject: [PATCH 021/191] Fix minor compile error --- benchmark/hermes_api_bench.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index 3ba8d0cb0..ef57ff695 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -67,6 +67,7 @@ void GetTest(int nprocs, int rank, t.Resume(); for (int j = 0; j < repeat; ++j) { for (size_t i = 0; i < blobs_per_rank; ++i) { + hermes::Blob ret; size_t blob_name_int = rank * blobs_per_rank + i; std::string name = std::to_string(blob_name_int); bkt.Get(name, ret, ctx); @@ -135,9 +136,9 @@ void PartialGetTest(int nprocs, int rank, void PartialPutGetTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size, size_t part_size) { - PartialPutTest(nprocs, rank, repeat, blobs_per_rank, blob_size); + PartialPutTest(nprocs, rank, repeat, blobs_per_rank, blob_size, part_size); MPI_Barrier(MPI_COMM_WORLD); - PartialGetTest(nprocs, rank, repeat, blobs_per_rank, blob_size); + PartialGetTest(nprocs, rank, repeat, blobs_per_rank, blob_size, part_size); } /** Each process creates a set of buckets */ From d1cbfe5c2cd93b465ed0e7db1fb1a4a5fdb9b5c8 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 21:09:54 -0500 Subject: [PATCH 022/191] Add debug statement to remote_queue --- tasks_required/remote_queue/src/remote_queue.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 6ae81d883..039ef67f1 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -57,6 +57,8 @@ class Server : public TaskLib { /** Construct remote queue */ void Construct(ConstructTask *task) { + HILOG(kInfo, "(node {}) Constructing remote queue (task_node={}, task_state={}, method={})", + LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, task->method_); LABSTOR_THALLIUM->RegisterRpc("RpcPushSmall", [this](const tl::request &req, TaskStateId state_id, u32 method, From 1b0d1eb9463b765b827a5dff7ff8adddf039d401 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 21:16:13 -0500 Subject: [PATCH 023/191] Change worker catch statement --- src/worker.cc | 3 +-- .../remote_queue/src/remote_queue.cc | 26 ------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/src/worker.cc b/src/worker.cc index ee514fdae..15e8ebbff 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -15,8 +15,7 @@ void Worker::Loop() { try { Run(); } catch (hshm::Error &e) { - e.print(); - exit(1); + HELOG(kFatal, "(node {}) Worker {} caught an error: {}", LABSTOR_CLIENT->node_id_, id_, e.what()); } // Yield(); } diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 039ef67f1..fd4916fad 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -142,14 +142,6 @@ class Server : public TaskLib { tl_task->done_ = true; }, tl_task); task->tl_future_.emplace_back(tl_task); - - -// tl::async_response future = LABSTOR_THALLIUM->AsyncCall(domain_id.id_, -// "RpcPushSmall", -// task->exec_->id_, -// task->exec_method_, -// params); -// task->tl_future_.emplace_back(std::move(future)); } } @@ -193,30 +185,12 @@ class Server : public TaskLib { tl_task->done_ = true; }, tl_task); task->tl_future_.emplace_back(tl_task); - -// tl::async_response future = LABSTOR_THALLIUM->AsyncIoCall(domain_id.id_, -// "RpcPushBulk", -// io_type, -// (char *) xfer[0].data_, -// xfer[0].data_size_, -// task->exec_->id_, -// task->exec_method_, -// params, -// xfer[0].data_size_, -// io_type); -// task->tl_future_.emplace_back(std::move(future)); } } /** Wait for client to finish message */ void ClientWaitForMessage(PushTask *task) { for (; task->replica_ < task->tl_future_.size(); ++task->replica_) { -// tl::async_response &future = task->tl_future_[task->replica_]; -// if (!LABSTOR_THALLIUM->IsDone(future)) { -// return; -// } -// std::string ret = LABSTOR_THALLIUM->Wait(future); -// HandlePushReplicaOutput(task->replica_, ret, task); ThalliumTask *tl_task = (ThalliumTask *) task->tl_future_[task->replica_]; if (!tl_task->IsDone()) { return; From 25c2234dcffba2d8617d8c24e7a090563ec12f18 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 21:28:36 -0500 Subject: [PATCH 024/191] Add node_id to thallium --- include/labstor/network/rpc_thallium.h | 4 +-- .../remote_queue/src/remote_queue.cc | 28 +++++++------------ 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/include/labstor/network/rpc_thallium.h b/include/labstor/network/rpc_thallium.h index 3a722e3b3..c48d24b28 100644 --- a/include/labstor/network/rpc_thallium.h +++ b/include/labstor/network/rpc_thallium.h @@ -139,8 +139,8 @@ class ThalliumRpc { return remote_proc.on(server).async(std::forward(args)...); } } catch (tl::margo_exception &err) { - HELOG(kFatal, "Thallium failed on function: {}\n{}", - func_name, err.what()) + HELOG(kFatal, "(node {}) Thallium failed on function: {}: {}", + rpc_->node_id_, func_name, err.what()) exit(1); } } diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index fd4916fad..8118cb87f 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -84,24 +84,16 @@ class Server : public TaskLib { /** Handle output from replica PUSH */ static void HandlePushReplicaOutput(int replica, std::string &ret, PushTask *task) { - try { - std::vector xfer(1); - xfer[0].data_ = ret.data(); - xfer[0].data_size_ = ret.size(); - HILOG(kDebug, "Wait got {} bytes of data (task_node={}, task_state={}, method={})", - xfer[0].data_size_, - task->orig_task_->task_node_, - task->orig_task_->task_state_, - task->orig_task_->method_); - BinaryInputArchive ar(xfer); - task->exec_->LoadEnd(replica, task->exec_method_, ar, task->orig_task_); - } catch (std::exception &e) { - HILOG(kFatal, "Error LoadEnd (task_node={}, task_state={}, method={}): {}", - task->orig_task_->task_node_, - task->orig_task_->task_state_, - task->orig_task_->method_, - e.what()); - } + std::vector xfer(1); + xfer[0].data_ = ret.data(); + xfer[0].data_size_ = ret.size(); + HILOG(kDebug, "Wait got {} bytes of data (task_node={}, task_state={}, method={})", + xfer[0].data_size_, + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_); + BinaryInputArchive ar(xfer); + task->exec_->LoadEnd(replica, task->exec_method_, ar, task->orig_task_); } /** Handle finalization of PUSH replicate */ From b951b316ae84f9e9647613c163184307110e615f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 21:29:46 -0500 Subject: [PATCH 025/191] Print more node info for thallium error --- include/labstor/network/rpc_thallium.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/labstor/network/rpc_thallium.h b/include/labstor/network/rpc_thallium.h index c48d24b28..7f7abe10d 100644 --- a/include/labstor/network/rpc_thallium.h +++ b/include/labstor/network/rpc_thallium.h @@ -139,8 +139,8 @@ class ThalliumRpc { return remote_proc.on(server).async(std::forward(args)...); } } catch (tl::margo_exception &err) { - HELOG(kFatal, "(node {}) Thallium failed on function: {}: {}", - rpc_->node_id_, func_name, err.what()) + HELOG(kFatal, "(node {} -> {}) Thallium failed on function: {}: {}", + rpc_->node_id_, node_id, func_name, err.what()) exit(1); } } From c18ad6157ce19d604037c0c3cf5e0f2b6acac8ec Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 21:58:27 -0500 Subject: [PATCH 026/191] Use const std::string --- include/labstor/network/rpc_thallium.h | 78 ++++++++++++++------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/include/labstor/network/rpc_thallium.h b/include/labstor/network/rpc_thallium.h index 7f7abe10d..bc504fede 100644 --- a/include/labstor/network/rpc_thallium.h +++ b/include/labstor/network/rpc_thallium.h @@ -93,7 +93,7 @@ class ThalliumRpc { tl::endpoint server = client_engine_->lookup(server_name.c_str()); client_engine_->shutdown_remote_engine(server); } catch (std::exception &e) { - HELOG(kFatal, e.what()); + HELOG(kFatal, "Stop daemon failed: {}", e.what()); } } @@ -120,7 +120,7 @@ class ThalliumRpc { /** RPC call */ template - RetT Call(u32 node_id, const char *func_name, Args&&... args) { + RetT Call(u32 node_id, const std::string &func_name, Args&&... args) { HILOG(kDebug, "Calling {} {} -> {}", func_name, rpc_->node_id_, node_id) try { std::string server_name = GetServerName(node_id); @@ -147,65 +147,71 @@ class ThalliumRpc { /** RPC call */ template - RetT SyncCall(u32 node_id, const char *func_name, Args&&... args) { + RetT SyncCall(u32 node_id, const std::string &func_name, Args&&... args) { return Call( node_id, func_name, std::forward(args)...); } /** Async RPC call */ template - thallium::async_response AsyncCall(u32 node_id, const char *func_name, Args&&... args) { + thallium::async_response AsyncCall(u32 node_id, const std::string &func_name, Args&&... args) { return Call( node_id, func_name, std::forward(args)...); } /** I/O transfers */ template - RetT IoCall(i32 node_id, const char *func_name, + RetT IoCall(i32 node_id, const std::string &func_name, IoType type, char *data, size_t size, Args&& ...args) { HILOG(kDebug, "Calling {} {} -> {}", func_name, rpc_->node_id_, node_id) - std::string server_name = GetServerName(node_id); - tl::bulk_mode flag; - switch (type) { - case IoType::kRead: { - // The "bulk" object will be modified - flag = tl::bulk_mode::write_only; - break; - } - case IoType::kWrite: { - // The "bulk" object will only be read from - flag = tl::bulk_mode::read_only; - break; - } - case IoType::kNone: { - // TODO(llogan) - HELOG(kFatal, "Cannot have none I/O type") - exit(1); + try { + std::string server_name = GetServerName(node_id); + tl::bulk_mode flag; + switch (type) { + case IoType::kRead: { + // The "bulk" object will be modified + flag = tl::bulk_mode::write_only; + break; + } + case IoType::kWrite: { + // The "bulk" object will only be read from + flag = tl::bulk_mode::read_only; + break; + } + case IoType::kNone: { + // TODO(llogan) + HELOG(kFatal, "Cannot have none I/O type") + exit(1); + } } - } - tl::remote_procedure remote_proc = client_engine_->define(func_name); - tl::endpoint server = client_engine_->lookup(server_name); + tl::remote_procedure remote_proc = client_engine_->define(func_name); + tl::endpoint server = client_engine_->lookup(server_name); - std::vector> segments(1); - segments[0].first = data; - segments[0].second = size; + std::vector> segments(1); + segments[0].first = data; + segments[0].second = size; - tl::bulk bulk = client_engine_->expose(segments, flag); - if constexpr (!ASYNC) { - if constexpr (std::is_same_v) { - remote_proc.on(server)(bulk, std::forward(args)...); + tl::bulk bulk = client_engine_->expose(segments, flag); + if constexpr (!ASYNC) { + if constexpr (std::is_same_v) { + remote_proc.on(server)(bulk, std::forward(args)...); + } else { + return remote_proc.on(server)(bulk, std::forward(args)...); + } } else { - return remote_proc.on(server)(bulk, std::forward(args)...); + return remote_proc.on(server).async(bulk, std::forward(args)...); } - } else { - return remote_proc.on(server).async(bulk, std::forward(args)...); + } catch (tl::margo_exception &err) { + HELOG(kFatal, "(node {} -> {}) Thallium failed on function: {}: {}", + rpc_->node_id_, node_id, func_name, err.what()) + exit(1); } } /** Synchronous I/O transfer */ template - RetT SyncIoCall(i32 node_id, const char *func_name, + RetT SyncIoCall(i32 node_id, const std::string &func_name, IoType type, char *data, size_t size, Args&& ...args) { return IoCall( node_id, func_name, type, data, size, std::forward(args)...); From f24cb5159cb2331a06fa4c11ac436273b354563c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 22 Sep 2023 14:02:45 -0500 Subject: [PATCH 027/191] Add run context --- benchmark/hermes_api_bench.cc | 70 ++-- codegen/refresh_methods | 342 +++++++++--------- include/labstor/labstor_namespace.h | 1 + include/labstor/task_registry/task_lib.h | 16 +- src/worker.cc | 4 +- tasks/bdev/include/bdev/bdev_lib_exec.h | 34 +- tasks/bdev/include/bdev/bdev_methods.h | 2 +- tasks/bdev/include/bdev/bdev_methods.yaml | 2 +- tasks/bdev/include/bdev/bdev_tasks.h | 2 +- .../hermes_adapters_lib_exec.h | 14 +- .../hermes_adapters/hermes_adapters_methods.h | 6 +- tasks/hermes_adapters/src/hermes_adapters.cc | 6 +- .../hermes_blob_mdm_lib_exec.h | 36 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 66 ++-- .../hermes_bucket_mdm_lib_exec.h | 32 +- .../src/hermes_bucket_mdm.cc | 30 +- .../include/hermes_mdm/hermes_mdm_lib_exec.h | 6 +- tasks/hermes_mdm/src/hermes_mdm.cc | 4 +- tasks/posix_bdev/src/posix_bdev.cc | 16 +- tasks/ram_bdev/src/ram_bdev.cc | 16 +- .../include/TASK_NAME/TASK_NAME_lib_exec.h | 8 +- tasks_required/TASK_NAME/src/TASK_NAME.cc | 6 +- .../labstor_admin/labstor_admin_lib_exec.h | 20 +- .../labstor_admin/src/labstor_admin.cc | 18 +- .../include/proc_queue/proc_queue_lib_exec.h | 8 +- tasks_required/proc_queue/src/proc_queue.cc | 6 +- .../remote_queue/remote_queue_lib_exec.h | 8 +- .../remote_queue/src/remote_queue.cc | 6 +- .../small_message/small_message_lib_exec.h | 12 +- .../small_message/src/small_message.cc | 10 +- .../worch_proc_round_robin_lib_exec.h | 8 +- .../src/worch_proc_round_robin.cc | 6 +- .../worch_queue_round_robin_lib_exec.h | 8 +- .../src/worch_queue_round_robin.cc | 6 +- 34 files changed, 432 insertions(+), 403 deletions(-) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index ef57ff695..18871c95a 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -258,39 +258,43 @@ int main(int argc, char **argv) { HIPRINT("Beginning {}\n", mode) // Run tests - if (mode == "put") { - REQUIRE_ARGC(4) - size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); - size_t blobs_per_rank = atoi(argv[3]); - PutTest(nprocs, rank, 1, blobs_per_rank, blob_size); - } else if (mode == "putget") { - REQUIRE_ARGC(4) - size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); - size_t blobs_per_rank = atoi(argv[3]); - PutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size); - } else if (mode == "pputget") { - REQUIRE_ARGC(5) - size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); - size_t part_size = hshm::ConfigParse::ParseSize(argv[3]); - size_t blobs_per_rank = atoi(argv[4]); - PartialPutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size, part_size); - } else if (mode == "create_bkt") { - REQUIRE_ARGC(3) - size_t bkts_per_rank = atoi(argv[2]); - CreateBucketTest(nprocs, rank, bkts_per_rank); - } else if (mode == "get_bkt") { - REQUIRE_ARGC(3) - size_t bkts_per_rank = atoi(argv[2]); - GetBucketTest(nprocs, rank, bkts_per_rank); - } else if (mode == "del_bkt") { - REQUIRE_ARGC(4) - size_t bkt_per_rank = atoi(argv[2]); - size_t blobs_per_bkt = atoi(argv[3]); - DeleteBucketTest(nprocs, rank, bkt_per_rank, blobs_per_bkt); - } else if (mode == "del_blobs") { - REQUIRE_ARGC(4) - size_t blobs_per_rank = atoi(argv[2]); - DeleteBlobOneBucket(nprocs, rank, blobs_per_rank); + try { + if (mode == "put") { + REQUIRE_ARGC(4) + size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); + size_t blobs_per_rank = atoi(argv[3]); + PutTest(nprocs, rank, 1, blobs_per_rank, blob_size); + } else if (mode == "putget") { + REQUIRE_ARGC(4) + size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); + size_t blobs_per_rank = atoi(argv[3]); + PutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size); + } else if (mode == "pputget") { + REQUIRE_ARGC(5) + size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); + size_t part_size = hshm::ConfigParse::ParseSize(argv[3]); + size_t blobs_per_rank = atoi(argv[4]); + PartialPutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size, part_size); + } else if (mode == "create_bkt") { + REQUIRE_ARGC(3) + size_t bkts_per_rank = atoi(argv[2]); + CreateBucketTest(nprocs, rank, bkts_per_rank); + } else if (mode == "get_bkt") { + REQUIRE_ARGC(3) + size_t bkts_per_rank = atoi(argv[2]); + GetBucketTest(nprocs, rank, bkts_per_rank); + } else if (mode == "del_bkt") { + REQUIRE_ARGC(4) + size_t bkt_per_rank = atoi(argv[2]); + size_t blobs_per_bkt = atoi(argv[3]); + DeleteBucketTest(nprocs, rank, bkt_per_rank, blobs_per_bkt); + } else if (mode == "del_blobs") { + REQUIRE_ARGC(4) + size_t blobs_per_rank = atoi(argv[2]); + DeleteBlobOneBucket(nprocs, rank, blobs_per_rank); + } + } catch (hshm::Error &err) { + HELOG(kFatal, "Error: {}", err.what()); } MPI_Finalize(); } diff --git a/codegen/refresh_methods b/codegen/refresh_methods index 600e355d8..404f6a705 100755 --- a/codegen/refresh_methods +++ b/codegen/refresh_methods @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -USAGE: ./referesh_methods [TASK_ROOT] +USAGE: ./referesh_methods [TASK_DIR] """ import yaml @@ -9,168 +9,178 @@ import os import sys from codegen.util.paths import LABSTOR_ROOT -TASK_ROOT = sys.argv[1] -TASK_NAME = os.path.basename(TASK_ROOT) -METHODS_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.h' -METHODS_YAML = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.yaml' -LIB_EXEC_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_lib_exec.h' -METHOD_MACRO = f'LABSTOR_{TASK_NAME.upper()}_METHODS_H_' -LIB_EXEC_MACRO = f'LABSTOR_{TASK_NAME.upper()}_LIB_EXEC_H_' - -with open(METHODS_YAML) as fp: - methods = yaml.load(fp, Loader=yaml.FullLoader) -if methods is None: - methods = {} -if 'kLast' in methods: - del methods['kLast'] -methods = sorted(methods.items(), key=lambda x: x[1]) -if TASK_NAME != 'labstor_admin': - methods.insert(0, ('kConstruct', -2)) - methods.insert(1, ('kDestruct', -1)) - -# Produce the TASK_NAME_methods.h file -lines = [] -lines += [f'#ifndef {METHOD_MACRO}', - f'#define {METHOD_MACRO}', - '', - '/** The set of methods in the admin task */', - 'struct Method : public TaskMethod {'] -for method_enum_name, method_off in methods: - if method_enum_name == 'kConstruct': - continue - if method_enum_name == 'kDestruct': - continue - lines += f' TASK_METHOD_T {method_enum_name} = kLast + {method_off};', -lines += ['};', '', f'#endif // {METHOD_MACRO}'] -with open(METHODS_H, 'w') as fp: - fp.write('\n'.join(lines)) - - -# Produce the TASK_NAME_lib_exec.h file -lines = [] -lines += [f'#ifndef {LIB_EXEC_MACRO}', - f'#define {LIB_EXEC_MACRO}', - ''] -## Create the Run method -lines += ['/** Execute a task */', - 'void Run(u32 method, Task *task) override {', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' {method_name}(reinterpret_cast<{task_name} *>(task));', - f' break;', - f' }}'] -lines += [' }'] -lines += ['}'] - -## Create the ReplicateStart method -lines += ['/** Ensure there is space to store replicated outputs */', - 'void ReplicateStart(u32 method, u32 count, Task *task) override {', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' labstor::CALL_REPLICA_START(count, reinterpret_cast<{task_name}*>(task));', - f' break;', - f' }}'] -lines += [' }'] -lines += ['}'] - -## Create the ReplicateEnd method -lines += ['/** Determine success and handle failures */', - 'void ReplicateEnd(u32 method, Task *task) override {', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' labstor::CALL_REPLICA_END(reinterpret_cast<{task_name}*>(task));', - f' break;', - f' }}'] -lines += [' }'] -lines += ['}'] - -## Create the SaveStart Method -lines += ['/** Serialize a task when initially pushing into remote */', - 'std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override {', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' ar << *reinterpret_cast<{task_name}*>(task);', - f' break;', - f' }}'] -lines += [' }'] -lines += [' return ar.Get();'] -lines += ['}'] - -## Create the LoadStart Method -lines += ['/** Deserialize a task when popping from remote queue */', - 'TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override {', - ' TaskPointer task_ptr;', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask<{task_name}>(task_ptr.p_);', - f' ar >> *reinterpret_cast<{task_name}*>(task_ptr.task_);', - f' break;', - f' }}'] -lines += [' }'] -lines += [' return task_ptr;'] -lines += ['}'] - -## Create the SaveEnd Method -lines += ['/** Serialize a task when returning from remote queue */', - 'std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override {', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' ar << *reinterpret_cast<{task_name}*>(task);', - f' break;', - f' }}'] -lines += [' }'] -lines += [' return ar.Get();'] -lines += ['}'] - -## Create the LoadEnd Method -lines += ['/** Deserialize a task when returning from remote queue */', - 'void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override {', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' ar.Deserialize(replica, *reinterpret_cast<{task_name}*>(task));', - f' break;', - f' }}'] -lines += [' }'] -lines += ['}'] - -## Create the CheckIfConcurrent Method -lines += ['/** Get the grouping of the task */', - 'u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override {', - ' switch (method) {'] -for method_enum_name, method_off in methods: - method_name = method_enum_name.replace('k', '', 1) - task_name = method_name + "Task" - lines += [f' case Method::{method_enum_name}: {{', - f' return reinterpret_cast<{task_name}*>(task)->GetGroup(group);', - f' }}'] -lines += [' }'] -lines += [' return -1;'] -lines += ['}'] - -## Finish the file -lines += ['', f'#endif // {METHOD_MACRO}'] - -## Write TASK_NAME_lib_exec.h -with open(LIB_EXEC_H, 'w') as fp: - fp.write('\n'.join(lines)) +def refresh_methods(TASK_ROOT): + if not os.path.exists(f'{TASK_ROOT}/include'): + return + TASK_NAME = os.path.basename(TASK_ROOT) + METHODS_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.h' + METHODS_YAML = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.yaml' + LIB_EXEC_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_lib_exec.h' + METHOD_MACRO = f'LABSTOR_{TASK_NAME.upper()}_METHODS_H_' + LIB_EXEC_MACRO = f'LABSTOR_{TASK_NAME.upper()}_LIB_EXEC_H_' + + with open(METHODS_YAML) as fp: + methods = yaml.load(fp, Loader=yaml.FullLoader) + if methods is None: + methods = {} + if 'kLast' in methods: + del methods['kLast'] + methods = sorted(methods.items(), key=lambda x: x[1]) + if TASK_NAME != 'labstor_admin': + methods.insert(0, ('kConstruct', -2)) + methods.insert(1, ('kDestruct', -1)) + + # Produce the TASK_NAME_methods.h file + lines = [] + lines += [f'#ifndef {METHOD_MACRO}', + f'#define {METHOD_MACRO}', + '', + '/** The set of methods in the admin task */', + 'struct Method : public TaskMethod {'] + for method_enum_name, method_off in methods: + if method_enum_name == 'kConstruct': + continue + if method_enum_name == 'kDestruct': + continue + lines += f' TASK_METHOD_T {method_enum_name} = kLast + {method_off};', + lines += ['};', '', f'#endif // {METHOD_MACRO}'] + with open(METHODS_H, 'w') as fp: + fp.write('\n'.join(lines)) + + + # Produce the TASK_NAME_lib_exec.h file + lines = [] + lines += [f'#ifndef {LIB_EXEC_MACRO}', + f'#define {LIB_EXEC_MACRO}', + ''] + ## Create the Run method + lines += ['/** Execute a task */', + 'void Run(u32 method, Task *task, RunContext &ctx) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' {method_name}(reinterpret_cast<{task_name} *>(task), ctx);', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the ReplicateStart method + lines += ['/** Ensure there is space to store replicated outputs */', + 'void ReplicateStart(u32 method, u32 count, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' labstor::CALL_REPLICA_START(count, reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the ReplicateEnd method + lines += ['/** Determine success and handle failures */', + 'void ReplicateEnd(u32 method, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' labstor::CALL_REPLICA_END(reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the SaveStart Method + lines += ['/** Serialize a task when initially pushing into remote */', + 'std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar << *reinterpret_cast<{task_name}*>(task);', + f' break;', + f' }}'] + lines += [' }'] + lines += [' return ar.Get();'] + lines += ['}'] + + ## Create the LoadStart Method + lines += ['/** Deserialize a task when popping from remote queue */', + 'TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override {', + ' TaskPointer task_ptr;', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask<{task_name}>(task_ptr.p_);', + f' ar >> *reinterpret_cast<{task_name}*>(task_ptr.task_);', + f' break;', + f' }}'] + lines += [' }'] + lines += [' return task_ptr;'] + lines += ['}'] + + ## Create the SaveEnd Method + lines += ['/** Serialize a task when returning from remote queue */', + 'std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar << *reinterpret_cast<{task_name}*>(task);', + f' break;', + f' }}'] + lines += [' }'] + lines += [' return ar.Get();'] + lines += ['}'] + + ## Create the LoadEnd Method + lines += ['/** Deserialize a task when returning from remote queue */', + 'void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar.Deserialize(replica, *reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the CheckIfConcurrent Method + lines += ['/** Get the grouping of the task */', + 'u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' return reinterpret_cast<{task_name}*>(task)->GetGroup(group);', + f' }}'] + lines += [' }'] + lines += [' return -1;'] + lines += ['}'] + + ## Finish the file + lines += ['', f'#endif // {METHOD_MACRO}'] + + ## Write TASK_NAME_lib_exec.h + with open(LIB_EXEC_H, 'w') as fp: + fp.write('\n'.join(lines)) + +TASK_DIR = sys.argv[1] +TASK_ROOTS = [os.path.join(TASK_DIR, item) for item in os.listdir(TASK_DIR)] +for TASK_ROOT in TASK_ROOTS: + try: + refresh_methods(TASK_ROOT) + except: + pass diff --git a/include/labstor/labstor_namespace.h b/include/labstor/labstor_namespace.h index d80e18f6f..230834691 100644 --- a/include/labstor/labstor_namespace.h +++ b/include/labstor/labstor_namespace.h @@ -22,6 +22,7 @@ using labstor::TaskLib; using labstor::TaskLibClient; using labstor::config::QueueManagerInfo; using labstor::TaskPrio; +using labstor::RunContext; using hshm::RwLock; using hshm::Mutex; diff --git a/include/labstor/task_registry/task_lib.h b/include/labstor/task_registry/task_lib.h index e67de5852..5dc4eda98 100644 --- a/include/labstor/task_registry/task_lib.h +++ b/include/labstor/task_registry/task_lib.h @@ -53,6 +53,17 @@ struct TaskPointer { } }; +/** Context passed to the Run method of a task */ +struct RunContext { + u32 lane_id_; /**< The lane id of the task */ + + /** Default constructor */ + RunContext() {} + + /** Emplace constructor */ + RunContext(u32 lane_id) : lane_id_(lane_id) {} +}; + /** * Represents a custom operation to perform. * Tasks are independent of Hermes. @@ -77,7 +88,7 @@ class TaskLib { virtual ~TaskLib() = default; /** Run a method of the task */ - virtual void Run(u32 method, Task *task) = 0; + virtual void Run(u32 method, Task *task, RunContext &ctx) = 0; /** Allow task to store replicas of completion */ virtual void ReplicateStart(u32 method, u32 count, Task *task) = 0; @@ -132,7 +143,8 @@ typedef const char* (*get_task_lib_name_t)(void); labstor::TaskState *exec = reinterpret_cast( \ new TYPE_UNWRAP(TRAIT_CLASS)()); \ exec->Init(task->id_, state_name); \ - exec->Run(labstor::TaskMethod::kConstruct, task); \ + RunContext ctx(0); \ + exec->Run(labstor::TaskMethod::kConstruct, task, ctx); \ return exec; \ } \ const char* get_task_lib_name(void) { return TASK_NAME; } \ diff --git a/src/worker.cc b/src/worker.cc index 15e8ebbff..37c696d27 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -45,6 +45,8 @@ void Worker::PollGrouped(WorkEntry &work_entry) { Task *task; LaneData *entry; int off = 0; + RunContext ctx; + ctx.lane_id_ = work_entry.lane_id_; for (int i = 0; i < 1024; ++i) { // Get the task message if (lane->peek(entry, off).IsNull()) { @@ -75,7 +77,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->SetUnordered(); } else { task->SetStarted(); - exec->Run(task->method_, task); + exec->Run(task->method_, task, ctx); } } // Cleanup on task completion diff --git a/tasks/bdev/include/bdev/bdev_lib_exec.h b/tasks/bdev/include/bdev/bdev_lib_exec.h index d154585ad..943c61ed2 100644 --- a/tasks/bdev/include/bdev/bdev_lib_exec.h +++ b/tasks/bdev/include/bdev/bdev_lib_exec.h @@ -2,38 +2,38 @@ #define LABSTOR_BDEV_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kWrite: { - Write(reinterpret_cast(task)); + Write(reinterpret_cast(task), ctx); break; } case Method::kRead: { - Read(reinterpret_cast(task)); + Read(reinterpret_cast(task), ctx); break; } - case Method::kAlloc: { - Alloc(reinterpret_cast(task)); + case Method::kAllocate: { + Allocate(reinterpret_cast(task), ctx); break; } case Method::kFree: { - Free(reinterpret_cast(task)); + Free(reinterpret_cast(task), ctx); break; } case Method::kMonitor: { - Monitor(reinterpret_cast(task)); + Monitor(reinterpret_cast(task), ctx); break; } case Method::kUpdateCapacity: { - UpdateCapacity(reinterpret_cast(task)); + UpdateCapacity(reinterpret_cast(task), ctx); break; } } @@ -57,7 +57,7 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } - case Method::kAlloc: { + case Method::kAllocate: { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } @@ -94,7 +94,7 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } - case Method::kAlloc: { + case Method::kAllocate: { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } @@ -131,7 +131,7 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } - case Method::kAlloc: { + case Method::kAllocate: { ar << *reinterpret_cast(task); break; } @@ -174,7 +174,7 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.task_); break; } - case Method::kAlloc: { + case Method::kAllocate: { task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); ar >> *reinterpret_cast(task_ptr.task_); break; @@ -216,7 +216,7 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } - case Method::kAlloc: { + case Method::kAllocate: { ar << *reinterpret_cast(task); break; } @@ -254,7 +254,7 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } - case Method::kAlloc: { + case Method::kAllocate: { ar.Deserialize(replica, *reinterpret_cast(task)); break; } @@ -287,7 +287,7 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kRead: { return reinterpret_cast(task)->GetGroup(group); } - case Method::kAlloc: { + case Method::kAllocate: { return reinterpret_cast(task)->GetGroup(group); } case Method::kFree: { diff --git a/tasks/bdev/include/bdev/bdev_methods.h b/tasks/bdev/include/bdev/bdev_methods.h index 715545386..7ca9b12f4 100644 --- a/tasks/bdev/include/bdev/bdev_methods.h +++ b/tasks/bdev/include/bdev/bdev_methods.h @@ -5,7 +5,7 @@ struct Method : public TaskMethod { TASK_METHOD_T kWrite = kLast + 0; TASK_METHOD_T kRead = kLast + 1; - TASK_METHOD_T kAlloc = kLast + 2; + TASK_METHOD_T kAllocate = kLast + 2; TASK_METHOD_T kFree = kLast + 3; TASK_METHOD_T kMonitor = kLast + 4; TASK_METHOD_T kUpdateCapacity = kLast + 5; diff --git a/tasks/bdev/include/bdev/bdev_methods.yaml b/tasks/bdev/include/bdev/bdev_methods.yaml index 616b06640..70173aa3e 100644 --- a/tasks/bdev/include/bdev/bdev_methods.yaml +++ b/tasks/bdev/include/bdev/bdev_methods.yaml @@ -1,6 +1,6 @@ kWrite: 0 kRead: 1 -kAlloc: 2 +kAllocate: 2 kFree: 3 kMonitor: 4 kUpdateCapacity: 5 diff --git a/tasks/bdev/include/bdev/bdev_tasks.h b/tasks/bdev/include/bdev/bdev_tasks.h index 4837dd543..6a502882d 100644 --- a/tasks/bdev/include/bdev/bdev_tasks.h +++ b/tasks/bdev/include/bdev/bdev_tasks.h @@ -99,7 +99,7 @@ struct AllocateTask : public Task, TaskFlags { lane_hash_ = 0; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; - method_ = Method::kAlloc; + method_ = Method::kAllocate; task_flags_.SetBits(TASK_UNORDERED); domain_id_ = domain_id; diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h index a491992c9..3ca8e4e03 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h @@ -1,19 +1,19 @@ -#ifndef LABSTOR_hermes_adapters_LIB_EXEC_H_ -#define LABSTOR_hermes_adapters_LIB_EXEC_H_ +#ifndef LABSTOR_HERMES_ADAPTERS_LIB_EXEC_H_ +#define LABSTOR_HERMES_ADAPTERS_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kCustom: { - Custom(reinterpret_cast(task)); + Custom(reinterpret_cast(task), ctx); break; } } @@ -143,4 +143,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_hermes_adapters_METHODS_H_ \ No newline at end of file +#endif // LABSTOR_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h index e6a468219..19d2b80a6 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h @@ -1,9 +1,9 @@ -#ifndef LABSTOR_hermes_adapters_METHODS_H_ -#define LABSTOR_hermes_adapters_METHODS_H_ +#ifndef LABSTOR_HERMES_ADAPTERS_METHODS_H_ +#define LABSTOR_HERMES_ADAPTERS_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kCustom = kLast + 0; }; -#endif // LABSTOR_hermes_adapters_METHODS_H_ \ No newline at end of file +#endif // LABSTOR_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/src/hermes_adapters.cc b/tasks/hermes_adapters/src/hermes_adapters.cc index 9192ef849..03d166b76 100644 --- a/tasks/hermes_adapters/src/hermes_adapters.cc +++ b/tasks/hermes_adapters/src/hermes_adapters.cc @@ -12,15 +12,15 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Custom(CustomTask *task) { + void Custom(CustomTask *task, RunContext &ctx) { task->SetModuleComplete(); } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index 251fafc96..55f37fe6d 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -2,74 +2,74 @@ #define LABSTOR_HERMES_BLOB_MDM_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kPutBlob: { - PutBlob(reinterpret_cast(task)); + PutBlob(reinterpret_cast(task), ctx); break; } case Method::kGetBlob: { - GetBlob(reinterpret_cast(task)); + GetBlob(reinterpret_cast(task), ctx); break; } case Method::kTruncateBlob: { - TruncateBlob(reinterpret_cast(task)); + TruncateBlob(reinterpret_cast(task), ctx); break; } case Method::kDestroyBlob: { - DestroyBlob(reinterpret_cast(task)); + DestroyBlob(reinterpret_cast(task), ctx); break; } case Method::kTagBlob: { - TagBlob(reinterpret_cast(task)); + TagBlob(reinterpret_cast(task), ctx); break; } case Method::kBlobHasTag: { - BlobHasTag(reinterpret_cast(task)); + BlobHasTag(reinterpret_cast(task), ctx); break; } case Method::kGetBlobId: { - GetBlobId(reinterpret_cast(task)); + GetBlobId(reinterpret_cast(task), ctx); break; } case Method::kGetOrCreateBlobId: { - GetOrCreateBlobId(reinterpret_cast(task)); + GetOrCreateBlobId(reinterpret_cast(task), ctx); break; } case Method::kGetBlobName: { - GetBlobName(reinterpret_cast(task)); + GetBlobName(reinterpret_cast(task), ctx); break; } case Method::kGetBlobSize: { - GetBlobSize(reinterpret_cast(task)); + GetBlobSize(reinterpret_cast(task), ctx); break; } case Method::kGetBlobScore: { - GetBlobScore(reinterpret_cast(task)); + GetBlobScore(reinterpret_cast(task), ctx); break; } case Method::kGetBlobBuffers: { - GetBlobBuffers(reinterpret_cast(task)); + GetBlobBuffers(reinterpret_cast(task), ctx); break; } case Method::kRenameBlob: { - RenameBlob(reinterpret_cast(task)); + RenameBlob(reinterpret_cast(task), ctx); break; } case Method::kReorganizeBlob: { - ReorganizeBlob(reinterpret_cast(task)); + ReorganizeBlob(reinterpret_cast(task), ctx); break; } case Method::kSetBucketMdm: { - SetBucketMdm(reinterpret_cast(task)); + SetBucketMdm(reinterpret_cast(task), ctx); break; } } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 78928210d..fa8812f01 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -50,7 +50,7 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { id_alloc_ = 0; node_id_ = LABSTOR_CLIENT->node_id_; switch (task->phase_) { @@ -97,7 +97,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } @@ -116,7 +116,7 @@ class Server : public TaskLib { /** * Set the Bucket MDM * */ - void SetBucketMdm(SetBucketMdmTask *task) { + void SetBucketMdm(SetBucketMdmTask *task, RunContext &ctx) { bkt_mdm_.Init(task->bkt_mdm_); task->SetModuleComplete(); } @@ -124,24 +124,24 @@ class Server : public TaskLib { /** * Create a blob's metadata * */ - void PutBlob(PutBlobTask *task) { + void PutBlob(PutBlobTask *task, RunContext &ctx) { if (task->phase_ == PutBlobPhase::kCreate) { - PutBlobCreatePhase(task); + PutBlobCreatePhase(task, ctx); } if (task->phase_ == PutBlobPhase::kAllocate) { - PutBlobAllocatePhase(task); + PutBlobAllocatePhase(task, ctx); } if (task->phase_ == PutBlobPhase::kWaitAllocate) { if (!task->cur_bdev_alloc_->IsComplete()){ return; } - PutBlobWaitAllocatePhase(task); + PutBlobWaitAllocatePhase(task, ctx); } if (task->phase_ == PutBlobPhase::kModify) { - PutBlobModifyPhase(task); + PutBlobModifyPhase(task, ctx); } if (task->phase_ == PutBlobPhase::kWaitModify) { - PutBlobWaitModifyPhase(task); + PutBlobWaitModifyPhase(task, ctx); if (task->phase_ == PutBlobPhase::kWaitModify) { return; } @@ -149,7 +149,7 @@ class Server : public TaskLib { } /** Create blob / update metadata for the PUT */ - void PutBlobCreatePhase(PutBlobTask *task) { + void PutBlobCreatePhase(PutBlobTask *task, RunContext &ctx) { HILOG(kDebug, "PutBlobPhase::kCreate {}", task->blob_id_); // Get the blob info data structure hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); @@ -180,7 +180,7 @@ class Server : public TaskLib { blob_info.UpdateWriteStats(); } if (task->flags_.Any(HERMES_BLOB_REPLACE)) { - PutBlobFreeBuffersPhase(blob_info, task); + PutBlobFreeBuffersPhase(blob_info, task, ctx); } // Stage in blob data from FS @@ -250,7 +250,7 @@ class Server : public TaskLib { } /** Release buffers */ - void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task) { + void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task, RunContext &ctx) { for (BufferInfo &buf : blob_info.buffers_) { TargetInfo &target = *target_map_[buf.tid_]; std::vector buf_vec = {buf}; @@ -262,7 +262,7 @@ class Server : public TaskLib { } /** Resolve the current sub-placement using BPM */ - void PutBlobAllocatePhase(PutBlobTask *task) { + void PutBlobAllocatePhase(PutBlobTask *task, RunContext &ctx) { BlobInfo &blob_info = blob_map_[task->blob_id_]; PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; SubPlacement &placement = schema.plcmnts_[task->sub_plcmnt_idx_]; @@ -275,7 +275,7 @@ class Server : public TaskLib { } /** Wait for the current-subplacement to complete */ - void PutBlobWaitAllocatePhase(PutBlobTask *task) { + void PutBlobWaitAllocatePhase(PutBlobTask *task, RunContext &ctx) { BlobInfo &blob_info = blob_map_[task->blob_id_]; PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; ++task->sub_plcmnt_idx_; @@ -301,7 +301,7 @@ class Server : public TaskLib { } /** Update the data on storage */ - void PutBlobModifyPhase(PutBlobTask *task) { + void PutBlobModifyPhase(PutBlobTask *task, RunContext &ctx) { BlobInfo &blob_info = blob_map_[task->blob_id_]; char *blob_buf = task->data_ptr_.ptr_; std::vector &write_tasks = *task->bdev_writes_; @@ -336,7 +336,7 @@ class Server : public TaskLib { } /** Wait for the update to complete */ - void PutBlobWaitModifyPhase(PutBlobTask *task) { + void PutBlobWaitModifyPhase(PutBlobTask *task, RunContext &ctx) { std::vector &write_tasks = *task->bdev_writes_; for (int i = (int)write_tasks.size() - 1; i >= 0; --i) { bdev::WriteTask *write_task = write_tasks[i]; @@ -370,18 +370,18 @@ class Server : public TaskLib { } /** Get a blob's data */ - void GetBlob(GetBlobTask *task) { + void GetBlob(GetBlobTask *task, RunContext &ctx) { switch (task->phase_) { case GetBlobPhase::kStart: { - GetBlobGetPhase(task); + GetBlobGetPhase(task, ctx); } case GetBlobPhase::kWait: { - GetBlobWaitPhase(task); + GetBlobWaitPhase(task, ctx); } } } - void GetBlobGetPhase(GetBlobTask *task) { + void GetBlobGetPhase(GetBlobTask *task, RunContext &ctx) { BlobInfo &blob_info = blob_map_[task->blob_id_]; HSHM_MAKE_AR0(task->bdev_reads_, nullptr); std::vector &read_tasks = *task->bdev_reads_; @@ -417,7 +417,7 @@ class Server : public TaskLib { task->phase_ = GetBlobPhase::kWait; } - void GetBlobWaitPhase(GetBlobTask *task) { + void GetBlobWaitPhase(GetBlobTask *task, RunContext &ctx) { std::vector &read_tasks = *task->bdev_reads_; for (auto it = read_tasks.rbegin(); it != read_tasks.rend(); ++it) { bdev::ReadTask *read_task = *it; @@ -435,7 +435,7 @@ class Server : public TaskLib { /** * Tag a blob * */ - void TagBlob(TagBlobTask *task) { + void TagBlob(TagBlobTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -449,7 +449,7 @@ class Server : public TaskLib { /** * Check if blob has a tag * */ - void BlobHasTag(BlobHasTagTask *task) { + void BlobHasTag(BlobHasTagTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -465,7 +465,7 @@ class Server : public TaskLib { /** * Create \a blob_id BLOB ID * */ - void GetOrCreateBlobId(GetOrCreateBlobIdTask *task) { + void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &ctx) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); auto it = blob_id_map_.find(blob_name_unique); @@ -483,7 +483,7 @@ class Server : public TaskLib { * Get \a blob_name BLOB from \a bkt_id bucket * */ HSHM_ALWAYS_INLINE - void GetBlobId(GetBlobIdTask *task) { + void GetBlobId(GetBlobIdTask *task, RunContext &ctx) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); auto it = blob_id_map_.find(blob_name_unique); @@ -501,7 +501,7 @@ class Server : public TaskLib { /** * Get \a blob_name BLOB name from \a blob_id BLOB id * */ - void GetBlobName(GetBlobNameTask *task) { + void GetBlobName(GetBlobNameTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -515,7 +515,7 @@ class Server : public TaskLib { /** * Get \a score from \a blob_id BLOB id * */ - void GetBlobSize(GetBlobSizeTask *task) { + void GetBlobSize(GetBlobSizeTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -529,7 +529,7 @@ class Server : public TaskLib { /** * Get \a score from \a blob_id BLOB id * */ - void GetBlobScore(GetBlobScoreTask *task) { + void GetBlobScore(GetBlobScoreTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -543,7 +543,7 @@ class Server : public TaskLib { /** * Get \a blob_id blob's buffers * */ - void GetBlobBuffers(GetBlobBuffersTask *task) { + void GetBlobBuffers(GetBlobBuffersTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -558,7 +558,7 @@ class Server : public TaskLib { * Rename \a blob_id blob to \a new_blob_name new blob name * in \a bkt_id bucket. * */ - void RenameBlob(RenameBlobTask *task) { + void RenameBlob(RenameBlobTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -574,7 +574,7 @@ class Server : public TaskLib { /** * Truncate a blob to a new size * */ - void TruncateBlob(TruncateBlobTask *task) { + void TruncateBlob(TruncateBlobTask *task, RunContext &ctx) { auto it = blob_map_.find(task->blob_id_); if (it == blob_map_.end()) { task->SetModuleComplete(); @@ -588,7 +588,7 @@ class Server : public TaskLib { /** * Destroy \a blob_id blob in \a bkt_id bucket * */ - void DestroyBlob(DestroyBlobTask *task) { + void DestroyBlob(DestroyBlobTask *task, RunContext &ctx) { switch (task->phase_) { case DestroyBlobPhase::kFreeBuffers: { auto it = blob_map_.find(task->blob_id_); @@ -635,7 +635,7 @@ class Server : public TaskLib { /** * Reorganize \a blob_id blob in \a bkt_id bucket * */ - void ReorganizeBlob(ReorganizeBlobTask *task) { + void ReorganizeBlob(ReorganizeBlobTask *task, RunContext &ctx) { switch (task->phase_) { case ReorganizeBlobPhase::kGet: { auto it = blob_map_.find(task->blob_id_); diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index f1d58e0ab..def2f8430 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -2,66 +2,66 @@ #define LABSTOR_HERMES_BUCKET_MDM_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kGetOrCreateTag: { - GetOrCreateTag(reinterpret_cast(task)); + GetOrCreateTag(reinterpret_cast(task), ctx); break; } case Method::kGetTagId: { - GetTagId(reinterpret_cast(task)); + GetTagId(reinterpret_cast(task), ctx); break; } case Method::kGetTagName: { - GetTagName(reinterpret_cast(task)); + GetTagName(reinterpret_cast(task), ctx); break; } case Method::kRenameTag: { - RenameTag(reinterpret_cast(task)); + RenameTag(reinterpret_cast(task), ctx); break; } case Method::kDestroyTag: { - DestroyTag(reinterpret_cast(task)); + DestroyTag(reinterpret_cast(task), ctx); break; } case Method::kTagAddBlob: { - TagAddBlob(reinterpret_cast(task)); + TagAddBlob(reinterpret_cast(task), ctx); break; } case Method::kTagRemoveBlob: { - TagRemoveBlob(reinterpret_cast(task)); + TagRemoveBlob(reinterpret_cast(task), ctx); break; } case Method::kTagClearBlobs: { - TagClearBlobs(reinterpret_cast(task)); + TagClearBlobs(reinterpret_cast(task), ctx); break; } case Method::kUpdateSize: { - UpdateSize(reinterpret_cast(task)); + UpdateSize(reinterpret_cast(task), ctx); break; } case Method::kAppendBlobSchema: { - AppendBlobSchema(reinterpret_cast(task)); + AppendBlobSchema(reinterpret_cast(task), ctx); break; } case Method::kAppendBlob: { - AppendBlob(reinterpret_cast(task)); + AppendBlob(reinterpret_cast(task), ctx); break; } case Method::kGetSize: { - GetSize(reinterpret_cast(task)); + GetSize(reinterpret_cast(task), ctx); break; } case Method::kSetBlobMdm: { - SetBlobMdm(reinterpret_cast(task)); + SetBlobMdm(reinterpret_cast(task), ctx); break; } } diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 656a4fccc..5db76ac26 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -27,27 +27,27 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { id_alloc_ = 0; node_id_ = LABSTOR_CLIENT->node_id_; bkt_mdm_.Init(id_); task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } /** * Set the Blob MDM * */ - void SetBlobMdm(SetBlobMdmTask *task) { + void SetBlobMdm(SetBlobMdmTask *task, RunContext &ctx) { blob_mdm_.Init(task->blob_mdm_); task->SetModuleComplete(); } /** Update the size of the bucket */ - void UpdateSize(UpdateSizeTask *task) { + void UpdateSize(UpdateSizeTask *task, RunContext &ctx) { TagInfo &tag_info = tag_map_[task->tag_id_]; ssize_t internal_size = (ssize_t) tag_info.internal_size_; if (task->mode_ == UpdateSizeMode::kAdd) { @@ -64,7 +64,7 @@ class Server : public TaskLib { /** * Create the PartialPuts for append operations. * */ - void AppendBlobSchema(AppendBlobSchemaTask *task) { + void AppendBlobSchema(AppendBlobSchemaTask *task, RunContext &ctx) { switch (task->phase_) { case AppendBlobPhase::kGetBlobIds: { HILOG(kDebug, "(node {}) Getting blob IDs for tag {} (task_node={})", @@ -123,7 +123,7 @@ class Server : public TaskLib { * are named 0 ... N. Each blob is assumed to have a certain * fixed page size. * */ - void AppendBlob(AppendBlobTask *task) { + void AppendBlob(AppendBlobTask *task, RunContext &ctx) { switch (task->phase_) { case AppendBlobPhase::kGetBlobIds: { HILOG(kDebug, "(node {}) Appending {} bytes to bucket {} (task_node={})", @@ -184,7 +184,7 @@ class Server : public TaskLib { } /** Get or create a tag */ - void GetOrCreateTag(GetOrCreateTagTask *task) { + void GetOrCreateTag(GetOrCreateTagTask *task, RunContext &ctx) { TagId tag_id; HILOG(kDebug, "Creating a tag") @@ -223,7 +223,7 @@ class Server : public TaskLib { } /** Get tag ID */ - void GetTagId(GetTagIdTask *task) { + void GetTagId(GetTagIdTask *task, RunContext &ctx) { hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); auto it = tag_id_map_.find(tag_name); if (it == tag_id_map_.end()) { @@ -236,7 +236,7 @@ class Server : public TaskLib { } /** Get tag name */ - void GetTagName(GetTagNameTask *task) { + void GetTagName(GetTagNameTask *task, RunContext &ctx) { auto it = tag_map_.find(task->tag_id_); if (it == tag_map_.end()) { task->SetModuleComplete(); @@ -247,7 +247,7 @@ class Server : public TaskLib { } /** Rename tag */ - void RenameTag(RenameTagTask *task) { + void RenameTag(RenameTagTask *task, RunContext &ctx) { auto it = tag_map_.find(task->tag_id_); if (it == tag_map_.end()) { task->SetModuleComplete(); @@ -258,7 +258,7 @@ class Server : public TaskLib { } /** Destroy tag */ - void DestroyTag(DestroyTagTask *task) { + void DestroyTag(DestroyTagTask *task, RunContext &ctx) { switch (task->phase_) { case DestroyTagPhase::kDestroyBlobs: { TagInfo &tag = tag_map_[task->tag_id_]; @@ -292,7 +292,7 @@ class Server : public TaskLib { } /** Add a blob to a tag */ - void TagAddBlob(TagAddBlobTask *task) { + void TagAddBlob(TagAddBlobTask *task, RunContext &ctx) { auto it = tag_map_.find(task->tag_id_); if (it == tag_map_.end()) { task->SetModuleComplete(); @@ -304,7 +304,7 @@ class Server : public TaskLib { } /** Remove a blob from a tag */ - void TagRemoveBlob(TagRemoveBlobTask *task) { + void TagRemoveBlob(TagRemoveBlobTask *task, RunContext &ctx) { auto it = tag_map_.find(task->tag_id_); if (it == tag_map_.end()) { task->SetModuleComplete(); @@ -317,7 +317,7 @@ class Server : public TaskLib { } /** Clear blobs from a tag */ - void TagClearBlobs(TagClearBlobsTask *task) { + void TagClearBlobs(TagClearBlobsTask *task, RunContext &ctx) { auto it = tag_map_.find(task->tag_id_); if (it == tag_map_.end()) { task->SetModuleComplete(); @@ -330,7 +330,7 @@ class Server : public TaskLib { } /** Get size of the bucket */ - void GetSize(GetSizeTask *task) { + void GetSize(GetSizeTask *task, RunContext &ctx) { auto it = tag_map_.find(task->tag_id_); if (it == tag_map_.end()) { task->size_ = 0; diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h index 127126f51..2d5c8cd7b 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h @@ -2,14 +2,14 @@ #define LABSTOR_HERMES_MDM_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } } diff --git a/tasks/hermes_mdm/src/hermes_mdm.cc b/tasks/hermes_mdm/src/hermes_mdm.cc index d424753cb..9c5557b88 100644 --- a/tasks/hermes_mdm/src/hermes_mdm.cc +++ b/tasks/hermes_mdm/src/hermes_mdm.cc @@ -25,7 +25,7 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { HILOG(kDebug, "ConstructTaskPhase::kLoadConfig") std::string config_path = task->server_config_path_->str(); HERMES_CONF->LoadServerConfig(config_path); @@ -33,7 +33,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } diff --git a/tasks/posix_bdev/src/posix_bdev.cc b/tasks/posix_bdev/src/posix_bdev.cc index c6f87a83a..11fc832fe 100644 --- a/tasks/posix_bdev/src/posix_bdev.cc +++ b/tasks/posix_bdev/src/posix_bdev.cc @@ -22,7 +22,7 @@ class Server : public TaskLib { std::string path_; public: - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { DeviceInfo &dev_info = task->info_; alloc_.Init(id_, dev_info.capacity_, dev_info.slab_sizes_); std::string text = dev_info.mount_dir_ + @@ -40,23 +40,23 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { free(mem_ptr_); task->SetModuleComplete(); } - void Alloc(AllocateTask *task) { + void Allocate(AllocateTask *task, RunContext &ctx) { alloc_.Allocate(task->size_, *task->buffers_, task->alloc_size_); HILOG(kDebug, "Allocated {}/{} bytes ({})", task->alloc_size_, task->size_, path_); task->SetModuleComplete(); } - void Free(FreeTask *task) { + void Free(FreeTask *task, RunContext &ctx) { alloc_.Free(task->buffers_); task->SetModuleComplete(); } - void Write(WriteTask *task) { + void Write(WriteTask *task, RunContext &ctx) { ssize_t count = pwrite(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { HELOG(kError, "BORG: wrote {} bytes, but expected {}: {}", @@ -65,7 +65,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Read(ReadTask *task) { + void Read(ReadTask *task, RunContext &ctx) { memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); ssize_t count = pread(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { @@ -75,10 +75,10 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Monitor(MonitorTask *task) { + void Monitor(MonitorTask *task, RunContext &ctx) { } - void UpdateCapacity(UpdateCapacityTask *task) { + void UpdateCapacity(UpdateCapacityTask *task, RunContext &ctx) { task->SetModuleComplete(); } diff --git a/tasks/ram_bdev/src/ram_bdev.cc b/tasks/ram_bdev/src/ram_bdev.cc index 9131018df..d5f2a042a 100644 --- a/tasks/ram_bdev/src/ram_bdev.cc +++ b/tasks/ram_bdev/src/ram_bdev.cc @@ -15,7 +15,7 @@ class Server : public TaskLib { char *mem_ptr_; public: - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { DeviceInfo &dev_info = task->info_; alloc_.Init(id_, dev_info.capacity_, dev_info.slab_sizes_); mem_ptr_ = (char*)malloc(dev_info.capacity_); @@ -24,37 +24,37 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { free(mem_ptr_); task->SetModuleComplete(); } - void Alloc(AllocateTask *task) { + void Allocate(AllocateTask *task, RunContext &ctx) { HILOG(kDebug, "Allocating {} bytes (RAM)", task->size_); alloc_.Allocate(task->size_, *task->buffers_, task->alloc_size_); HILOG(kDebug, "Allocated {} bytes (RAM)", task->alloc_size_); task->SetModuleComplete(); } - void Free(FreeTask *task) { + void Free(FreeTask *task, RunContext &ctx) { alloc_.Free(task->buffers_); task->SetModuleComplete(); } - void Write(WriteTask *task) { + void Write(WriteTask *task, RunContext &ctx) { memcpy(mem_ptr_ + task->disk_off_, task->buf_, task->size_); task->SetModuleComplete(); } - void Read(ReadTask *task) { + void Read(ReadTask *task, RunContext &ctx) { memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); task->SetModuleComplete(); } - void Monitor(MonitorTask *task) { + void Monitor(MonitorTask *task, RunContext &ctx) { } - void UpdateCapacity(UpdateCapacityTask *task) { + void UpdateCapacity(UpdateCapacityTask *task, RunContext &ctx) { task->SetModuleComplete(); } diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h index c603dac50..95ba64a49 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_TASK_NAME_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kCustom: { - Custom(reinterpret_cast(task)); + Custom(reinterpret_cast(task), ctx); break; } } diff --git a/tasks_required/TASK_NAME/src/TASK_NAME.cc b/tasks_required/TASK_NAME/src/TASK_NAME.cc index 9142ee436..03ff393dd 100644 --- a/tasks_required/TASK_NAME/src/TASK_NAME.cc +++ b/tasks_required/TASK_NAME/src/TASK_NAME.cc @@ -12,15 +12,15 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Custom(CustomTask *task) { + void Custom(CustomTask *task, RunContext &ctx) { task->SetModuleComplete(); } diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h index 2cfe22c2b..0bcdc2d02 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h @@ -2,42 +2,42 @@ #define LABSTOR_LABSTOR_ADMIN_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kCreateTaskState: { - CreateTaskState(reinterpret_cast(task)); + CreateTaskState(reinterpret_cast(task), ctx); break; } case Method::kDestroyTaskState: { - DestroyTaskState(reinterpret_cast(task)); + DestroyTaskState(reinterpret_cast(task), ctx); break; } case Method::kRegisterTaskLib: { - RegisterTaskLib(reinterpret_cast(task)); + RegisterTaskLib(reinterpret_cast(task), ctx); break; } case Method::kDestroyTaskLib: { - DestroyTaskLib(reinterpret_cast(task)); + DestroyTaskLib(reinterpret_cast(task), ctx); break; } case Method::kGetOrCreateTaskStateId: { - GetOrCreateTaskStateId(reinterpret_cast(task)); + GetOrCreateTaskStateId(reinterpret_cast(task), ctx); break; } case Method::kGetTaskStateId: { - GetTaskStateId(reinterpret_cast(task)); + GetTaskStateId(reinterpret_cast(task), ctx); break; } case Method::kStopRuntime: { - StopRuntime(reinterpret_cast(task)); + StopRuntime(reinterpret_cast(task), ctx); break; } case Method::kSetWorkOrchQueuePolicy: { - SetWorkOrchQueuePolicy(reinterpret_cast(task)); + SetWorkOrchQueuePolicy(reinterpret_cast(task), ctx); break; } case Method::kSetWorkOrchProcPolicy: { - SetWorkOrchProcPolicy(reinterpret_cast(task)); + SetWorkOrchProcPolicy(reinterpret_cast(task), ctx); break; } } diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index 957756fc9..53bc527c4 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -16,25 +16,25 @@ class Server : public TaskLib { public: Server() : queue_sched_(nullptr), proc_sched_(nullptr) {} - void RegisterTaskLib(RegisterTaskLibTask *task) { + void RegisterTaskLib(RegisterTaskLibTask *task, RunContext &ctx) { std::string lib_name = task->lib_name_->str(); LABSTOR_TASK_REGISTRY->RegisterTaskLib(lib_name); task->SetModuleComplete(); } - void DestroyTaskLib(DestroyTaskLibTask *task) { + void DestroyTaskLib(DestroyTaskLibTask *task, RunContext &ctx) { std::string lib_name = task->lib_name_->str(); LABSTOR_TASK_REGISTRY->DestroyTaskLib(lib_name); task->SetModuleComplete(); } - void GetOrCreateTaskStateId(GetOrCreateTaskStateIdTask *task) { + void GetOrCreateTaskStateId(GetOrCreateTaskStateIdTask *task, RunContext &ctx) { std::string state_name = task->state_name_->str(); task->id_ = LABSTOR_TASK_REGISTRY->GetOrCreateTaskStateId(state_name); task->SetModuleComplete(); } - void CreateTaskState(CreateTaskStateTask *task) { + void CreateTaskState(CreateTaskStateTask *task, RunContext &ctx) { switch (task->phase_) { case CreateTaskStatePhase::kIdAllocStart: { std::string lib_name = task->lib_name_->str(); @@ -122,25 +122,25 @@ class Server : public TaskLib { } } - void GetTaskStateId(GetTaskStateIdTask *task) { + void GetTaskStateId(GetTaskStateIdTask *task, RunContext &ctx) { std::string state_name = task->state_name_->str(); task->id_ = LABSTOR_TASK_REGISTRY->GetTaskStateId(state_name); task->SetModuleComplete(); } - void DestroyTaskState(DestroyTaskStateTask *task) { + void DestroyTaskState(DestroyTaskStateTask *task, RunContext &ctx) { LABSTOR_TASK_REGISTRY->DestroyTaskState(task->id_); task->SetModuleComplete(); } - void StopRuntime(StopRuntimeTask *task) { + void StopRuntime(StopRuntimeTask *task, RunContext &ctx) { HILOG(kInfo, "Stopping (server mode)"); LABSTOR_WORK_ORCHESTRATOR->FinalizeRuntime(); LABSTOR_THALLIUM->StopThisDaemon(); task->SetModuleComplete(); } - void SetWorkOrchQueuePolicy(SetWorkOrchQueuePolicyTask *task) { + void SetWorkOrchQueuePolicy(SetWorkOrchQueuePolicyTask *task, RunContext &ctx) { if (queue_sched_) { queue_sched_->SetModuleComplete(); } @@ -155,7 +155,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void SetWorkOrchProcPolicy(SetWorkOrchProcPolicyTask *task) { + void SetWorkOrchProcPolicy(SetWorkOrchProcPolicyTask *task, RunContext &ctx) { if (proc_sched_) { proc_sched_->SetModuleComplete(); } diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h index bea645d52..f7f4ef23f 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_PROC_QUEUE_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kPush: { - Push(reinterpret_cast(task)); + Push(reinterpret_cast(task), ctx); break; } } diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/tasks_required/proc_queue/src/proc_queue.cc index 31016b865..de580f9a7 100644 --- a/tasks_required/proc_queue/src/proc_queue.cc +++ b/tasks_required/proc_queue/src/proc_queue.cc @@ -12,15 +12,15 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Push(PushTask *task) { + void Push(PushTask *task, RunContext &ctx) { switch (task->phase_) { case PushTaskPhase::kSchedule: { task->ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->subtask_); diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h index e0ff029c2..1bb6e92a9 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_REMOTE_QUEUE_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kPush: { - Push(reinterpret_cast(task)); + Push(reinterpret_cast(task), ctx); break; } } diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 8118cb87f..61f4001aa 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -56,7 +56,7 @@ class Server : public TaskLib { Server() = default; /** Construct remote queue */ - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { HILOG(kInfo, "(node {}) Constructing remote queue (task_node={}, task_state={}, method={})", LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, task->method_); LABSTOR_THALLIUM->RegisterRpc("RpcPushSmall", [this](const tl::request &req, @@ -78,7 +78,7 @@ class Server : public TaskLib { } /** Destroy remote queue */ - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } @@ -193,7 +193,7 @@ class Server : public TaskLib { } /** Push operation called on client */ - void Push(PushTask *task) { + void Push(PushTask *task, RunContext &ctx) { switch (task->phase_) { case PushPhase::kStart: { std::vector &xfer = task->xfer_; diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/tasks_required/small_message/include/small_message/small_message_lib_exec.h index 2f217b7ba..68aab730b 100644 --- a/tasks_required/small_message/include/small_message/small_message_lib_exec.h +++ b/tasks_required/small_message/include/small_message/small_message_lib_exec.h @@ -2,26 +2,26 @@ #define LABSTOR_SMALL_MESSAGE_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kMd: { - Md(reinterpret_cast(task)); + Md(reinterpret_cast(task), ctx); break; } case Method::kIo: { - Io(reinterpret_cast(task)); + Io(reinterpret_cast(task), ctx); break; } case Method::kMdPush: { - MdPush(reinterpret_cast(task)); + MdPush(reinterpret_cast(task), ctx); break; } } diff --git a/tasks_required/small_message/src/small_message.cc b/tasks_required/small_message/src/small_message.cc index 25eec506d..16282873d 100644 --- a/tasks_required/small_message/src/small_message.cc +++ b/tasks_required/small_message/src/small_message.cc @@ -13,25 +13,25 @@ class Server : public TaskLib { int count_ = 0; public: - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Md(MdTask *task) { + void Md(MdTask *task, RunContext &ctx) { task->ret_[0] = 1; task->SetModuleComplete(); } - void MdPush(MdPushTask *task) { + void MdPush(MdPushTask *task, RunContext &ctx) { task->ret_[0] = 1; task->SetModuleComplete(); } - void Io(IoTask *task) { + void Io(IoTask *task, RunContext &ctx) { task->ret_ = 1; for (int i = 0; i < 256; ++i) { if (task->data_[i] != 10) { diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h index 10565c416..21d3493ee 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kSchedule: { - Schedule(reinterpret_cast(task)); + Schedule(reinterpret_cast(task), ctx); break; } } diff --git a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc index c51a0eba5..238143a27 100644 --- a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc +++ b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc @@ -10,15 +10,15 @@ namespace labstor::worch_proc_round_robin { class Server : public TaskLib { public: - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Schedule(ScheduleTask *task) { + void Schedule(ScheduleTask *task, RunContext &ctx) { int rr = 0; for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { worker.SetCpuAffinity(rr % HERMES_SYSTEM_INFO->ncpu_); diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h index 1321d5488..9223e7f1e 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task) override { +void Run(u32 method, Task *task, RunContext &ctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task)); + Construct(reinterpret_cast(task), ctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task)); + Destruct(reinterpret_cast(task), ctx); break; } case Method::kSchedule: { - Schedule(reinterpret_cast(task)); + Schedule(reinterpret_cast(task), ctx); break; } } diff --git a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc index 1ea856dcb..10f53ed33 100644 --- a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc +++ b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc @@ -13,16 +13,16 @@ class Server : public TaskLib { u32 count_; public: - void Construct(ConstructTask *task) { + void Construct(ConstructTask *task, RunContext &ctx) { count_ = 0; task->SetModuleComplete(); } - void Destruct(DestructTask *task) { + void Destruct(DestructTask *task, RunContext &ctx) { task->SetModuleComplete(); } - void Schedule(ScheduleTask *task) { + void Schedule(ScheduleTask *task, RunContext &ctx) { // Check if any new queues need to be scheduled for (MultiQueue &queue : *LABSTOR_QM_RUNTIME->queue_map_) { if (queue.id_.IsNull()) { From 68189cbd71c5e2d625af29147684f1e58f554470 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 22 Sep 2023 21:00:09 -0500 Subject: [PATCH 028/191] Improve bucket operation throughput by dividing unordered_map into partitions --- config/labstor_server_default.yaml | 3 +- include/labstor/api/labstor_client.h | 2 +- .../labstor_task_node_push_root.template | 2 +- include/labstor/labstor_types.h | 12 +- .../queue_manager/queue_manager_runtime.h | 2 + .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 29 +++-- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 104 +++++++++++------- .../hermes_bucket_mdm_tasks.h | 20 ++-- .../src/hermes_bucket_mdm.cc | 69 +++++++----- .../include/proc_queue/proc_queue.h | 4 +- .../include/proc_queue/proc_queue_tasks.h | 21 ++-- tasks_required/proc_queue/src/proc_queue.cc | 19 ++-- .../include/small_message/small_message.h | 6 +- 13 files changed, 179 insertions(+), 114 deletions(-) diff --git a/config/labstor_server_default.yaml b/config/labstor_server_default.yaml index d0fc09001..2a498ff9b 100644 --- a/config/labstor_server_default.yaml +++ b/config/labstor_server_default.yaml @@ -15,8 +15,9 @@ queue_manager: shm_allocator: kScalablePageAllocator # The name of the shared memory region to create shm_name: "labstor_shm" - # The size of the shared memory region to allocate + # The size of the shared memory region to allocate for general data structures shm_size: 0g + # The size of the shared memory to allocate for data buffers ### Define properties of RPCs rpc: diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 68c53535a..f3ef03473 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -269,7 +269,7 @@ class Client : public ConfigurationManager { hipc::LPointer> push_task =\ LABSTOR_PROCESS_QUEUE->AsyncPush(task_node,\ DomainId::GetLocal(),\ - task.shm_);\ + task);\ return push_task;\ } diff --git a/include/labstor/api/template/labstor_task_node_push_root.template b/include/labstor/api/template/labstor_task_node_push_root.template index 703048125..37b909500 100644 --- a/include/labstor/api/template/labstor_task_node_push_root.template +++ b/include/labstor/api/template/labstor_task_node_push_root.template @@ -28,6 +28,6 @@ hipc::LPointer> Async##CUSTOM##Root(Args&& .. hipc::LPointer> push_task = LABSTOR_PROCESS_QUEUE->AsyncPush(task_node, DomainId::GetLocal(), - task.shm_); + task); return push_task; } \ No newline at end of file diff --git a/include/labstor/labstor_types.h b/include/labstor/labstor_types.h index c6594db35..7ecce655b 100644 --- a/include/labstor/labstor_types.h +++ b/include/labstor/labstor_types.h @@ -225,12 +225,14 @@ struct DomainId { template struct UniqueId { u32 node_id_; /**< The node the content is on */ + u32 hash_; /**< The hash of the content the ID represents */ u64 unique_; /**< A unique id for the blob */ /** Serialization */ template void serialize(Ar &ar) { ar & node_id_; + ar & hash_; ar & unique_; } @@ -239,8 +241,14 @@ struct UniqueId { UniqueId() = default; /** Emplace constructor */ - HSHM_ALWAYS_INLINE - UniqueId(u32 node_id, u64 unique) : node_id_(node_id), unique_(unique) {} + HSHM_ALWAYS_INLINE explicit + UniqueId(u32 node_id, u64 unique) + : node_id_(node_id), unique_(unique) {} + + /** Emplace constructor (+hash) */ + HSHM_ALWAYS_INLINE explicit + UniqueId(u32 node_id, u32 hash, u64 unique) + : node_id_(node_id), hash_(hash), unique_(unique) {} /** Copy constructor */ HSHM_ALWAYS_INLINE diff --git a/include/labstor/queue_manager/queue_manager_runtime.h b/include/labstor/queue_manager/queue_manager_runtime.h index 8164b1520..a489e6d9f 100644 --- a/include/labstor/queue_manager/queue_manager_runtime.h +++ b/include/labstor/queue_manager/queue_manager_runtime.h @@ -18,6 +18,7 @@ class QueueManagerRuntime : public QueueManager { public: ServerConfig *config_; size_t max_queues_; + size_t max_lanes_; hipc::split_ticket_queue *tickets_; u32 node_id_; @@ -35,6 +36,7 @@ class QueueManagerRuntime : public QueueManager { QueueManagerInfo &qm = config_->queue_manager_; // Initialize ticket queue (ticket 0 is for admin queue) max_queues_ = qm.max_queues_; + max_lanes_ = qm.max_lanes_; HSHM_MAKE_AR(shm.tickets_, alloc, max_queues_) for (u64 i = 1; i <= max_queues_; ++i) { shm.tickets_->emplace(i); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index f5d8e1b64..015cbcc2b 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -261,7 +261,7 @@ struct PutBlobTask : public Task, TaskFlags // Initialize task HILOG(kDebug, "Beginning PUT task constructor") task_node_ = task_node; - lane_hash_ = blob_id_.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPutBlob; @@ -365,7 +365,7 @@ struct GetBlobTask : public Task, TaskFlags const Context &ctx) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetBlob; @@ -382,6 +382,11 @@ struct GetBlobTask : public Task, TaskFlags page_size_ = ctx.page_size_; } + /** Destructor */ + ~GetBlobTask() { + HSHM_DESTROY_AR(filename_); + } + /** (De)serialize message call */ template void SaveStart(Ar &ar) { @@ -439,7 +444,7 @@ struct TagBlobTask : public Task, TaskFlags { const TagId &tag) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kTagBlob; @@ -499,7 +504,7 @@ struct BlobHasTagTask : public Task, TaskFlags { // Initialize task task_node_ = task_node; task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kBlobHasTag; @@ -619,7 +624,7 @@ struct GetBlobNameTask : public Task, TaskFlags { const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetBlobName; @@ -680,7 +685,7 @@ struct GetBlobSizeTask : public Task, TaskFlags { const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetBlobSize; @@ -735,7 +740,7 @@ struct GetBlobScoreTask : public Task, TaskFlags { const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetBlobScore; @@ -790,7 +795,7 @@ struct GetBlobBuffersTask : public Task, TaskFlags { const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetBlobBuffers; @@ -855,7 +860,7 @@ struct RenameBlobTask : public Task, TaskFlags { const hshm::charbuf &new_blob_name) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kRenameBlob; @@ -917,7 +922,7 @@ struct TruncateBlobTask : public Task, TaskFlags { u64 size) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kTruncateBlob; @@ -979,7 +984,7 @@ struct DestroyBlobTask : public Task, TaskFlags { const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kDestroyBlob; @@ -1048,7 +1053,7 @@ struct ReorganizeBlobTask : public Task, TaskFlags { u32 node_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.unique_; + lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kReorganizeBlob; diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index fa8812f01..b3a284023 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -27,8 +27,8 @@ class Server : public TaskLib { /**==================================== * Maps * ===================================*/ - BLOB_ID_MAP_T blob_id_map_; - BLOB_MAP_T blob_map_; + std::vector blob_id_map_; + std::vector blob_map_; std::atomic id_alloc_; /**==================================== @@ -55,6 +55,8 @@ class Server : public TaskLib { node_id_ = LABSTOR_CLIENT->node_id_; switch (task->phase_) { case ConstructTaskPhase::kCreateTaskStates: { + blob_id_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + blob_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); target_tasks_.reserve(HERMES_SERVER_CONF.devices_.size()); for (DeviceInfo &dev : HERMES_SERVER_CONF.devices_) { std::string dev_type; @@ -153,14 +155,15 @@ class Server : public TaskLib { HILOG(kDebug, "PutBlobPhase::kCreate {}", task->blob_id_); // Get the blob info data structure hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->flags_.SetBits(HERMES_BLOB_DID_CREATE); } if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { - blob_map_.emplace(task->blob_id_, BlobInfo()); + blob_map.emplace(task->blob_id_, BlobInfo()); } - BlobInfo &blob_info = blob_map_[task->blob_id_]; + BlobInfo &blob_info = blob_map[task->blob_id_]; // Update the blob info if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { @@ -263,7 +266,8 @@ class Server : public TaskLib { /** Resolve the current sub-placement using BPM */ void PutBlobAllocatePhase(PutBlobTask *task, RunContext &ctx) { - BlobInfo &blob_info = blob_map_[task->blob_id_]; + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BlobInfo &blob_info = blob_map[task->blob_id_]; PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; SubPlacement &placement = schema.plcmnts_[task->sub_plcmnt_idx_]; TargetInfo &bdev = *target_map_[placement.tid_]; @@ -276,7 +280,8 @@ class Server : public TaskLib { /** Wait for the current-subplacement to complete */ void PutBlobWaitAllocatePhase(PutBlobTask *task, RunContext &ctx) { - BlobInfo &blob_info = blob_map_[task->blob_id_]; + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BlobInfo &blob_info = blob_map[task->blob_id_]; PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; ++task->sub_plcmnt_idx_; if (task->sub_plcmnt_idx_ >= schema.plcmnts_.size()) { @@ -302,7 +307,8 @@ class Server : public TaskLib { /** Update the data on storage */ void PutBlobModifyPhase(PutBlobTask *task, RunContext &ctx) { - BlobInfo &blob_info = blob_map_[task->blob_id_]; + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BlobInfo &blob_info = blob_map[task->blob_id_]; char *blob_buf = task->data_ptr_.ptr_; std::vector &write_tasks = *task->bdev_writes_; size_t blob_off = 0, buf_off = 0; @@ -382,7 +388,8 @@ class Server : public TaskLib { } void GetBlobGetPhase(GetBlobTask *task, RunContext &ctx) { - BlobInfo &blob_info = blob_map_[task->blob_id_]; + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BlobInfo &blob_info = blob_map[task->blob_id_]; HSHM_MAKE_AR0(task->bdev_reads_, nullptr); std::vector &read_tasks = *task->bdev_reads_; read_tasks.reserve(blob_info.buffers_.size()); @@ -436,8 +443,9 @@ class Server : public TaskLib { * Tag a blob * */ void TagBlob(TagBlobTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } @@ -450,8 +458,9 @@ class Server : public TaskLib { * Check if blob has a tag * */ void BlobHasTag(BlobHasTagTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } @@ -468,10 +477,11 @@ class Server : public TaskLib { void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &ctx) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); - auto it = blob_id_map_.find(blob_name_unique); - if (it == blob_id_map_.end()) { - task->blob_id_ = BlobId(node_id_, id_alloc_.fetch_add(1)); - blob_id_map_.emplace(blob_name_unique, task->blob_id_); + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; + auto it = blob_id_map.find(blob_name_unique); + if (it == blob_id_map.end()) { + task->blob_id_ = BlobId(node_id_, task->lane_hash_, id_alloc_.fetch_add(1)); + blob_id_map.emplace(blob_name_unique, task->blob_id_); task->SetModuleComplete(); return; } @@ -486,8 +496,9 @@ class Server : public TaskLib { void GetBlobId(GetBlobIdTask *task, RunContext &ctx) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); - auto it = blob_id_map_.find(blob_name_unique); - if (it == blob_id_map_.end()) { + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; + auto it = blob_id_map.find(blob_name_unique); + if (it == blob_id_map.end()) { task->blob_id_ = BlobId::GetNull(); task->SetModuleComplete(); HILOG(kDebug, "Failed to find blob {} in {}", blob_name.str(), task->tag_id_); @@ -502,8 +513,9 @@ class Server : public TaskLib { * Get \a blob_name BLOB name from \a blob_id BLOB id * */ void GetBlobName(GetBlobNameTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } @@ -516,8 +528,9 @@ class Server : public TaskLib { * Get \a score from \a blob_id BLOB id * */ void GetBlobSize(GetBlobSizeTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } @@ -530,8 +543,9 @@ class Server : public TaskLib { * Get \a score from \a blob_id BLOB id * */ void GetBlobScore(GetBlobScoreTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } @@ -544,8 +558,9 @@ class Server : public TaskLib { * Get \a blob_id blob's buffers * */ void GetBlobBuffers(GetBlobBuffersTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } @@ -559,14 +574,16 @@ class Server : public TaskLib { * in \a bkt_id bucket. * */ void RenameBlob(RenameBlobTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; BlobInfo &blob = it->second; - blob_id_map_.erase(blob.name_); - blob_id_map_[blob.name_] = task->blob_id_; + blob_id_map.erase(blob.name_); + blob_id_map[blob.name_] = task->blob_id_; blob.name_ = hshm::to_charbuf(*task->new_blob_name_); task->SetModuleComplete(); } @@ -575,8 +592,9 @@ class Server : public TaskLib { * Truncate a blob to a new size * */ void TruncateBlob(TruncateBlobTask *task, RunContext &ctx) { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } @@ -591,14 +609,16 @@ class Server : public TaskLib { void DestroyBlob(DestroyBlobTask *task, RunContext &ctx) { switch (task->phase_) { case DestroyBlobPhase::kFreeBuffers: { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; BlobInfo &blob_info = it->second; hshm::charbuf unique_name = GetBlobNameWithBucket(blob_info.tag_id_, blob_info.name_); - blob_id_map_.erase(unique_name); + blob_id_map.erase(unique_name); HSHM_MAKE_AR0(task->free_tasks_, nullptr); task->free_tasks_->reserve(blob_info.buffers_.size()); for (BufferInfo &buf : blob_info.buffers_) { @@ -620,13 +640,14 @@ class Server : public TaskLib { LABSTOR_CLIENT->DelTask(free_task); free_tasks.pop_back(); } - BlobInfo &blob_info = blob_map_[task->blob_id_]; + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BlobInfo &blob_info = blob_map[task->blob_id_]; bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, task->tag_id_, -(ssize_t)blob_info.blob_size_, bucket_mdm::UpdateSizeMode::kAdd); HSHM_DESTROY_AR(task->free_tasks_); - blob_map_.erase(task->blob_id_); + blob_map.erase(task->blob_id_); task->SetModuleComplete(); } } @@ -638,8 +659,9 @@ class Server : public TaskLib { void ReorganizeBlob(ReorganizeBlobTask *task, RunContext &ctx) { switch (task->phase_) { case ReorganizeBlobPhase::kGet: { - auto it = blob_map_.find(task->blob_id_); - if (it == blob_map_.end()) { + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + auto it = blob_map.find(task->blob_id_); + if (it == blob_map.end()) { task->SetModuleComplete(); return; } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 5e81e71ac..2ec787bad 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -153,7 +153,7 @@ struct UpdateSizeTask : public Task, TaskFlags { int mode) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kUpdateSize; @@ -236,7 +236,7 @@ struct AppendBlobSchemaTask : public Task, TaskFlags { size_t page_size) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kAppendBlobSchema; @@ -305,7 +305,7 @@ struct AppendBlobTask : public Task, TaskFlags { const Context &ctx) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kAppendBlob; @@ -481,7 +481,7 @@ struct GetTagNameTask : public Task, TaskFlags { const TagId &tag_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetTagName; @@ -539,7 +539,7 @@ struct RenameTagTask : public Task, TaskFlags { const hshm::charbuf &tag_name) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kRenameTag; @@ -604,7 +604,7 @@ struct DestroyTagTask : public Task, TaskFlags { TagId tag_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kDestroyTag; @@ -655,7 +655,7 @@ struct TagAddBlobTask : public Task, TaskFlags { const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kTagAddBlob; @@ -707,7 +707,7 @@ struct TagRemoveBlobTask : public Task, TaskFlags { const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kTagRemoveBlob; @@ -766,7 +766,7 @@ struct TagClearBlobsTask : public Task, TaskFlags { TagId tag_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kTagClearBlobs; @@ -816,7 +816,7 @@ struct GetSizeTask : public Task, TaskFlags { TagId tag_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = tag_id.unique_; + lane_hash_ = tag_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetSize; diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 5db76ac26..75cae81b3 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -17,8 +17,8 @@ typedef std::unordered_map TAG_MAP_T; class Server : public TaskLib { public: - TAG_ID_MAP_T tag_id_map_; - TAG_MAP_T tag_map_; + std::vector tag_id_map_; + std::vector tag_map_; u32 node_id_; std::atomic id_alloc_; Client bkt_mdm_; @@ -31,6 +31,8 @@ class Server : public TaskLib { id_alloc_ = 0; node_id_ = LABSTOR_CLIENT->node_id_; bkt_mdm_.Init(id_); + tag_id_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + tag_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); task->SetModuleComplete(); } @@ -48,7 +50,8 @@ class Server : public TaskLib { /** Update the size of the bucket */ void UpdateSize(UpdateSizeTask *task, RunContext &ctx) { - TagInfo &tag_info = tag_map_[task->tag_id_]; + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + TagInfo &tag_info = tag_map[task->tag_id_]; ssize_t internal_size = (ssize_t) tag_info.internal_size_; if (task->mode_ == UpdateSizeMode::kAdd) { internal_size += task->update_; @@ -69,7 +72,8 @@ class Server : public TaskLib { case AppendBlobPhase::kGetBlobIds: { HILOG(kDebug, "(node {}) Getting blob IDs for tag {} (task_node={})", LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) - TagInfo &tag_info = tag_map_[task->tag_id_]; + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + TagInfo &tag_info = tag_map[task->tag_id_]; size_t bucket_size = tag_info.internal_size_; size_t cur_page = bucket_size / task->page_size_; size_t cur_page_off = bucket_size % task->page_size_; @@ -189,20 +193,23 @@ class Server : public TaskLib { HILOG(kDebug, "Creating a tag") // Check if the tag exists + TAG_ID_MAP_T &tag_id_map = tag_id_map_[ctx.lane_id_]; hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); bool did_create = false; if (tag_name.size() > 0) { - did_create = tag_id_map_.find(tag_name) == tag_id_map_.end(); + did_create = tag_id_map.find(tag_name) == tag_id_map.end(); } // Emplace bucket if it does not already exist if (did_create) { + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; tag_id.unique_ = id_alloc_.fetch_add(1); + tag_id.hash_ = task->lane_hash_; tag_id.node_id_ = LABSTOR_RUNTIME->rpc_.node_id_; HILOG(kDebug, "Creating tag for the first time: {} {}", tag_name.str(), tag_id) - tag_id_map_.emplace(tag_name, tag_id); - tag_map_.emplace(tag_id, TagInfo()); - TagInfo &tag_info = tag_map_[tag_id]; + tag_id_map.emplace(tag_name, tag_id); + tag_map.emplace(tag_id, TagInfo()); + TagInfo &tag_info = tag_map[tag_id]; tag_info.name_ = tag_name; tag_info.tag_id_ = tag_id; tag_info.owner_ = task->blob_owner_; @@ -210,7 +217,7 @@ class Server : public TaskLib { } else { if (tag_name.size()) { HILOG(kDebug, "Found existing tag: {}", tag_name.str()) - tag_id = tag_id_map_[tag_name]; + tag_id = tag_id_map[tag_name]; } else { HILOG(kDebug, "Found existing tag: {}", task->tag_id_) tag_id = task->tag_id_; @@ -224,9 +231,10 @@ class Server : public TaskLib { /** Get tag ID */ void GetTagId(GetTagIdTask *task, RunContext &ctx) { + TAG_ID_MAP_T &tag_id_map = tag_id_map_[ctx.lane_id_]; hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); - auto it = tag_id_map_.find(tag_name); - if (it == tag_id_map_.end()) { + auto it = tag_id_map.find(tag_name); + if (it == tag_id_map.end()) { task->tag_id_ = TagId::GetNull(); task->SetModuleComplete(); return; @@ -237,8 +245,9 @@ class Server : public TaskLib { /** Get tag name */ void GetTagName(GetTagNameTask *task, RunContext &ctx) { - auto it = tag_map_.find(task->tag_id_); - if (it == tag_map_.end()) { + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + auto it = tag_map.find(task->tag_id_); + if (it == tag_map.end()) { task->SetModuleComplete(); return; } @@ -248,8 +257,9 @@ class Server : public TaskLib { /** Rename tag */ void RenameTag(RenameTagTask *task, RunContext &ctx) { - auto it = tag_map_.find(task->tag_id_); - if (it == tag_map_.end()) { + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + auto it = tag_map.find(task->tag_id_); + if (it == tag_map.end()) { task->SetModuleComplete(); return; } @@ -261,8 +271,10 @@ class Server : public TaskLib { void DestroyTag(DestroyTagTask *task, RunContext &ctx) { switch (task->phase_) { case DestroyTagPhase::kDestroyBlobs: { - TagInfo &tag = tag_map_[task->tag_id_]; - tag_id_map_.erase(tag.name_); + TAG_ID_MAP_T &tag_id_map = tag_id_map_[ctx.lane_id_]; + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + TagInfo &tag = tag_map[task->tag_id_]; + tag_id_map.erase(tag.name_); HSHM_MAKE_AR0(task->destroy_blob_tasks_, nullptr); std::vector blob_tasks = *task->destroy_blob_tasks_; blob_tasks.reserve(tag.blobs_.size()); @@ -284,8 +296,9 @@ class Server : public TaskLib { LABSTOR_CLIENT->DelTask(blob_task); blob_tasks.pop_back(); } + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; HSHM_DESTROY_AR(task->destroy_blob_tasks_); - tag_map_.erase(task->tag_id_); + tag_map.erase(task->tag_id_); task->SetModuleComplete(); } } @@ -293,8 +306,9 @@ class Server : public TaskLib { /** Add a blob to a tag */ void TagAddBlob(TagAddBlobTask *task, RunContext &ctx) { - auto it = tag_map_.find(task->tag_id_); - if (it == tag_map_.end()) { + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + auto it = tag_map.find(task->tag_id_); + if (it == tag_map.end()) { task->SetModuleComplete(); return; } @@ -305,8 +319,9 @@ class Server : public TaskLib { /** Remove a blob from a tag */ void TagRemoveBlob(TagRemoveBlobTask *task, RunContext &ctx) { - auto it = tag_map_.find(task->tag_id_); - if (it == tag_map_.end()) { + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + auto it = tag_map.find(task->tag_id_); + if (it == tag_map.end()) { task->SetModuleComplete(); return; } @@ -318,8 +333,9 @@ class Server : public TaskLib { /** Clear blobs from a tag */ void TagClearBlobs(TagClearBlobsTask *task, RunContext &ctx) { - auto it = tag_map_.find(task->tag_id_); - if (it == tag_map_.end()) { + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + auto it = tag_map.find(task->tag_id_); + if (it == tag_map.end()) { task->SetModuleComplete(); return; } @@ -331,8 +347,9 @@ class Server : public TaskLib { /** Get size of the bucket */ void GetSize(GetSizeTask *task, RunContext &ctx) { - auto it = tag_map_.find(task->tag_id_); - if (it == tag_map_.end()) { + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + auto it = tag_map.find(task->tag_id_); + if (it == tag_map.end()) { task->size_ = 0; task->SetModuleComplete(); return; diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue.h b/tasks_required/proc_queue/include/proc_queue/proc_queue.h index b6aa51aab..527bf3ec1 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue.h @@ -59,7 +59,7 @@ class Client : public TaskLibClient { void AsyncPushConstruct(labpq::TypedPushTask *task, const TaskNode &task_node, const DomainId &domain_id, - const hipc::Pointer &subtask) { + const hipc::LPointer &subtask) { LABSTOR_CLIENT->ConstructTask( task, task_node, domain_id, id_, subtask); } @@ -68,7 +68,7 @@ class Client : public TaskLibClient { LPointer> AsyncPush(const TaskNode &task_node, const DomainId &domain_id, - const hipc::Pointer &subtask) { + const hipc::LPointer &subtask) { LPointer> push_task = LABSTOR_CLIENT->AllocateTask>(); AsyncPushConstruct(push_task.ptr_, task_node, domain_id, subtask); diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index 002f42436..ee5bcf500 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -84,9 +84,8 @@ class PushTaskPhase { * */ template struct TypedPushTask : public Task, TaskFlags { - IN hipc::Pointer subtask_; //< SHM pointer to the subtask - TEMP TaskT *subtask_ptr_; //< Pointer to the subtask (client) - TEMP TaskT *ptr_; //< Pointer to the subtask (server) + IN LPointer sub_cli_; /**< Pointer to the subtask (client + SHM) */ + TEMP LPointer sub_run_; /**< Pointer to the subtask (runtime) */ TEMP int phase_ = PushTaskPhase::kSchedule; /** SHM default constructor */ @@ -99,7 +98,7 @@ struct TypedPushTask : public Task, TaskFlags { const TaskNode &task_node, const DomainId &domain_id, const TaskStateId &state_id, - const hipc::Pointer &subtask) : Task(alloc) { + const hipc::LPointer &subtask) : Task(alloc) { // Initialize task hshm::NodeThreadId tid; task_node_ = task_node; @@ -111,8 +110,16 @@ struct TypedPushTask : public Task, TaskFlags { domain_id_ = domain_id; // Custom params - subtask_ = subtask; - subtask_ptr_ = (TaskT*)LABSTOR_CLIENT->GetPrivatePointer(subtask_); + sub_cli_ = subtask; + } + + /** Destructor */ + ~TypedPushTask() { + if (!IsFireAndForget()) { + LABSTOR_CLIENT->DelTask(sub_cli_); + } else { + LABSTOR_CLIENT->DelTask(sub_run_); + } } /** Create group */ @@ -126,7 +133,7 @@ struct TypedPushTask : public Task, TaskFlags { /** Get the task address */ HSHM_ALWAYS_INLINE TaskT* get() { - return subtask_ptr_; + return sub_cli_.ptr_; } }; diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/tasks_required/proc_queue/src/proc_queue.cc index de580f9a7..7a74ce5d8 100644 --- a/tasks_required/proc_queue/src/proc_queue.cc +++ b/tasks_required/proc_queue/src/proc_queue.cc @@ -23,21 +23,24 @@ class Server : public TaskLib { void Push(PushTask *task, RunContext &ctx) { switch (task->phase_) { case PushTaskPhase::kSchedule: { - task->ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->subtask_); + task->sub_run_.shm_ = task->sub_cli_.shm_; + task->sub_run_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->sub_cli_.shm_); + Task *&ptr = task->sub_run_.ptr_; HILOG(kDebug, "Scheduling task {} on state {} tid {}", - task->ptr_->task_node_, task->ptr_->task_state_, GetLinuxTid()); - if (task->ptr_->IsFireAndForget()) { - task->ptr_->UnsetFireAndForget(); + ptr->task_node_, ptr->task_state_, GetLinuxTid()); + if (ptr->IsFireAndForget()) { + ptr->UnsetFireAndForget(); } - MultiQueue *real_queue = LABSTOR_CLIENT->GetQueue(QueueId(task->ptr_->task_state_)); - real_queue->Emplace(task->ptr_->prio_, task->ptr_->lane_hash_, task->subtask_); + MultiQueue *real_queue = LABSTOR_CLIENT->GetQueue(QueueId(ptr->task_state_)); + real_queue->Emplace(ptr->prio_, ptr->lane_hash_, task->sub_run_.shm_); task->phase_ = PushTaskPhase::kWaitSchedule; } case PushTaskPhase::kWaitSchedule: { - if (!task->ptr_->IsComplete()) { + Task *&ptr = task->sub_run_.ptr_; + if (!ptr->IsComplete()) { return; } - LABSTOR_CLIENT->DelTask(task->ptr_); + // TODO(llogan): handle fire & forget tasks gracefully task->SetModuleComplete(); } } diff --git a/tasks_required/small_message/include/small_message/small_message.h b/tasks_required/small_message/include/small_message/small_message.h index 6953a076e..bb5f8c1a2 100644 --- a/tasks_required/small_message/include/small_message/small_message.h +++ b/tasks_required/small_message/include/small_message/small_message.h @@ -70,7 +70,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncMdPushRoot(domain_id); push_task->Wait(); - MdPushTask *task = push_task->subtask_ptr_; + MdPushTask *task = push_task->get(); int ret = task->ret_[0]; LABSTOR_CLIENT->DelTask(push_task); return ret; @@ -85,8 +85,8 @@ class Client : public TaskLibClient { } int IoRoot(const DomainId &domain_id) { LPointer> push_task = AsyncIoRoot(domain_id); - push_task.ptr_->Wait(); - IoTask *task = push_task.ptr_->subtask_ptr_; + push_task->Wait(); + IoTask *task = push_task->get(); int ret = task->ret_; LABSTOR_CLIENT->DelTask(push_task); return ret; From c421806b983ef42a2166fd071005eca4f76b8278 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 22 Sep 2023 21:18:32 -0500 Subject: [PATCH 029/191] Small-scale, single-node concurrency seems to work again --- include/labstor/labstor_types.h | 8 +++++++- tasks/hermes/include/hermes/bucket.h | 1 + tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/labstor/labstor_types.h b/include/labstor/labstor_types.h index 7ecce655b..e4c8a1faf 100644 --- a/include/labstor/labstor_types.h +++ b/include/labstor/labstor_types.h @@ -243,7 +243,7 @@ struct UniqueId { /** Emplace constructor */ HSHM_ALWAYS_INLINE explicit UniqueId(u32 node_id, u64 unique) - : node_id_(node_id), unique_(unique) {} + : node_id_(node_id), hash_(0), unique_(unique) {} /** Emplace constructor (+hash) */ HSHM_ALWAYS_INLINE explicit @@ -254,6 +254,7 @@ struct UniqueId { HSHM_ALWAYS_INLINE UniqueId(const UniqueId &other) { node_id_ = other.node_id_; + hash_ = other.hash_; unique_ = other.unique_; } @@ -262,6 +263,7 @@ struct UniqueId { HSHM_ALWAYS_INLINE UniqueId(const UniqueId &other) { node_id_ = other.node_id_; + hash_ = other.hash_; unique_ = other.unique_; } @@ -270,6 +272,7 @@ struct UniqueId { UniqueId& operator=(const UniqueId &other) { if (this != &other) { node_id_ = other.node_id_; + hash_ = other.hash_; unique_ = other.unique_; } return *this; @@ -279,6 +282,7 @@ struct UniqueId { HSHM_ALWAYS_INLINE UniqueId(UniqueId &&other) noexcept { node_id_ = other.node_id_; + hash_ = other.hash_; unique_ = other.unique_; } @@ -287,6 +291,7 @@ struct UniqueId { UniqueId& operator=(UniqueId &&other) noexcept { if (this != &other) { node_id_ = other.node_id_; + hash_ = other.hash_; unique_ = other.unique_; } return *this; @@ -309,6 +314,7 @@ struct UniqueId { HSHM_ALWAYS_INLINE void SetNull() { node_id_ = 0; + hash_ = 0; unique_ = 0; } diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 241faf477..62879a53b 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -342,6 +342,7 @@ class Bucket { if (data_size == 0) { data_size = GetBlobSize(blob_id); } + HILOG(kInfo, "Data size: {}", data_size) LPointer data_p = LABSTOR_CLIENT->AllocateBuffer(data_size); data_size = blob_mdm_->GetBlobRoot(id_, blob_id, blob_off, data_size, data_p.shm_, ctx); char *data = data_p.ptr_; diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index b3a284023..ccd5dc763 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -531,6 +531,7 @@ class Server : public TaskLib { BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { + task->size_ = 0; task->SetModuleComplete(); return; } From f51aed41addbbb26ed93ed881f77303317420d5d Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 22 Sep 2023 22:14:59 -0500 Subject: [PATCH 030/191] Remove mem_ptr --- tasks/posix_bdev/src/posix_bdev.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/tasks/posix_bdev/src/posix_bdev.cc b/tasks/posix_bdev/src/posix_bdev.cc index 11fc832fe..71cce0b2e 100644 --- a/tasks/posix_bdev/src/posix_bdev.cc +++ b/tasks/posix_bdev/src/posix_bdev.cc @@ -17,7 +17,6 @@ namespace hermes::posix_bdev { class Server : public TaskLib { public: SlabAllocator alloc_; - char *mem_ptr_; int fd_; std::string path_; @@ -41,7 +40,6 @@ class Server : public TaskLib { } void Destruct(DestructTask *task, RunContext &ctx) { - free(mem_ptr_); task->SetModuleComplete(); } @@ -66,7 +64,6 @@ class Server : public TaskLib { } void Read(ReadTask *task, RunContext &ctx) { - memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); ssize_t count = pread(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { HELOG(kError, "BORG: read {} bytes, but expected {}", From 93049311ef680605324df5d35799956ff21c641d Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 22 Sep 2023 22:27:00 -0500 Subject: [PATCH 031/191] Use delete instead of free --- .../remote_queue/include/remote_queue/remote_queue_tasks.h | 1 - tasks_required/remote_queue/src/remote_queue.cc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index a3c120771..9aef218a4 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -94,7 +94,6 @@ struct PushTask : public Task, TaskFlags { IN TaskState *exec_; IN u32 exec_method_; IN std::vector xfer_; - // TEMP std::vector tl_future_; TEMP std::vector tl_future_; TEMP int phase_ = PushPhase::kStart; TEMP int replica_; diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 61f4001aa..5dd865054 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -187,7 +187,7 @@ class Server : public TaskLib { if (!tl_task->IsDone()) { return; } - free(tl_task); + delete tl_task; } HandlePushReplicaEnd(task); } From e869db558380273dcef62f9c309a0eee88371edc Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 22 Sep 2023 22:35:22 -0500 Subject: [PATCH 032/191] Add finished put log --- benchmark/hermes_api_bench.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index 18871c95a..15e7c8856 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -52,6 +52,7 @@ void PutTest(int nprocs, int rank, } } t.Pause(); + HILOG(kInfo, "Finished PUT") GatherTimes("Put", nprocs * blobs_per_rank * blob_size * repeat, t); } From fc2405f7d2b2fe2892e33bbb7aa95c7925d7c15f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 09:09:18 -0500 Subject: [PATCH 033/191] Add TestHermesConnect to ipc test --- test/unit/hermes/test_bucket.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index e9d131e96..164031069 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -9,6 +9,15 @@ #include "hermes/bucket.h" #include +TEST_CASE("TestHermesConnect") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + HERMES->ClientInit(); + MPI_Barrier(MPI_COMM_WORLD); +} + TEST_CASE("TestHermesPut1n") { int rank, nprocs; MPI_Barrier(MPI_COMM_WORLD); From 58072a95ba3369a719bbe24b10a0cd5fd7f9ecda Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 09:18:11 -0500 Subject: [PATCH 034/191] Try making it so only one RPC group at a time per-node --- .../remote_queue/include/remote_queue/remote_queue.h | 3 ++- .../remote_queue/include/remote_queue/remote_queue_tasks.h | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index e733c2b42..f0cf88119 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -35,7 +35,8 @@ class Client : public TaskLibClient { std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, - {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + // {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + {1, 1, qm.queue_depth_, QUEUE_LOW_LATENCY} }; return LABSTOR_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info); diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index 9aef218a4..63c904b0c 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -135,7 +135,10 @@ struct PushTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; + LocalSerialize srl(group); + srl << 0; + // return TASK_UNORDERED; + return 0; } }; From a4dd782f1039154febf88be0349c73d0b09b8022 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 09:24:37 -0500 Subject: [PATCH 035/191] Use a group other than 0? --- .../remote_queue/include/remote_queue/remote_queue_tasks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index 63c904b0c..a589a3652 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -136,7 +136,7 @@ struct PushTask : public Task, TaskFlags { HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { LocalSerialize srl(group); - srl << 0; + srl << 16; // return TASK_UNORDERED; return 0; } From ff3695364c4141997cdca8a574873b37fb157fe0 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 10:28:17 -0500 Subject: [PATCH 036/191] Add more debug logging to remote_queue --- tasks_required/remote_queue/src/remote_queue.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 5dd865054..b188d4f39 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -130,6 +130,13 @@ class Server : public TaskLib { task->exec_->id_, task->exec_method_, task->params_); + HILOG(kDebug, "(SM) Finished {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={})", + task->params_.size(), + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_); HandlePushReplicaOutput(replica, ret, task); tl_task->done_ = true; }, tl_task); @@ -173,6 +180,14 @@ class Server : public TaskLib { task->params_, tl_task->data_size_, io_type); + HILOG(kDebug, "(IO) Finished transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={}, type={})", + tl_task->data_size_, + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_, + static_cast(io_type)); HandlePushReplicaOutput(replica, ret, task); tl_task->done_ = true; }, tl_task); From 6dc06fcb6eb817699952217fe51762823ff80fb0 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 13:37:22 -0500 Subject: [PATCH 037/191] Unset long running --- include/labstor/task_registry/task.h | 5 +++++ tasks_required/labstor_admin/src/labstor_admin.cc | 7 ------- tasks_required/remote_queue/src/remote_queue.cc | 3 ++- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 94c4932ff..4d300be94 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -335,6 +335,11 @@ struct Task : public hipc::ShmContainer { return task_flags_.Any(TASK_MARKED); } + /** Set this task as started */ + HSHM_ALWAYS_INLINE void UnsetLongRunning() { + task_flags_.UnsetBits(TASK_LONG_RUNNING); + } + /** Wait for task to complete */ template void Wait() { diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index 53bc527c4..eb74f4acd 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -48,17 +48,10 @@ class Server : public TaskLib { } // Check global registry for task state if (task->id_.IsNull()) { -// if (task->domain_id_ == DomainId::GetLocal()) { -// HILOG(kDebug, "Domain ID is local for {} (task_node={})", state_name, task->task_node_); -// task->id_ = LABSTOR_TASK_REGISTRY->GetOrCreateTaskStateId(state_name); -// task->phase_ = CreateTaskStatePhase::kStateCreate; -// } else { - HILOG(kDebug, "Domain ID is global for {} (task_node={})", state_name, task->task_node_); DomainId domain = DomainId::GetNode(1); task->get_id_task_ = LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId( task->task_node_ + 1, domain, state_name).ptr_; task->phase_ = CreateTaskStatePhase::kIdAllocWait; -// } } else { HILOG(kDebug, "Domain ID is given as {} for {} (task_node={})", task->id_, state_name, task->task_node_); task->phase_ = CreateTaskStatePhase::kStateCreate; diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index b188d4f39..bbc59f793 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -323,6 +323,7 @@ class Server : public TaskLib { orig_task->UnsetStarted(); orig_task->UnsetMarked(); orig_task->UnsetDataOwner(); + orig_task->UnsetLongRunning(); queue->Emplace(orig_task->prio_, orig_task->lane_hash_, p); HILOG(kDebug, "(node {}) Executing task (task_node={}, task_state={}/{}, state_name={}, method={}, size={}, lane_hash={})", @@ -342,7 +343,6 @@ class Server : public TaskLib { TaskState *exec, TaskStateId state_id) { BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); std::vector out_xfer = exec->SaveEnd(method, ar, orig_task); - LABSTOR_CLIENT->DelTask(orig_task); HILOG(kDebug, "(node {}) Returning {} bytes of data (task_node={}, task_state={}/{}, method={})", LABSTOR_CLIENT->node_id_, out_xfer[0].data_size_, @@ -350,6 +350,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); + LABSTOR_CLIENT->DelTask(orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } From b42b0b2e53103616bc56e17b7bf7e3ac3355e947 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 15:33:34 -0500 Subject: [PATCH 038/191] Made ConstructTask blocking --- include/labstor/work_orchestrator/worker.h | 2 +- tasks/bdev/include/bdev/bdev_tasks.h | 6 ------ .../include/hermes_adapters/hermes_adapters_tasks.h | 6 ------ .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 6 ------ .../include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h | 6 ------ tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h | 6 ------ .../TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h | 6 ------ .../include/labstor_admin/labstor_admin_tasks.h | 4 +++- .../proc_queue/include/proc_queue/proc_queue_tasks.h | 6 ------ .../remote_queue/include/remote_queue/remote_queue_tasks.h | 6 ------ .../worch_proc_round_robin/worch_proc_round_robin_tasks.h | 6 ------ .../worch_queue_round_robin/worch_queue_round_robin_tasks.h | 6 ------ 12 files changed, 4 insertions(+), 62 deletions(-) diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 1894df6ad..16907ecc7 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -314,7 +314,7 @@ class Worker { return; } int ret = exec->GetGroup(task->method_, task, group_); - if (ret == TASK_UNORDERED || task->IsUnordered() || task->method_ < 2) { + if (ret == TASK_UNORDERED || task->IsUnordered()) { HILOG(kDebug, "(node {}) Decreasing depth of group remains 0 (task_node={} worker={})", LABSTOR_CLIENT->node_id_, task->task_node_, id_); return; diff --git a/tasks/bdev/include/bdev/bdev_tasks.h b/tasks/bdev/include/bdev/bdev_tasks.h index 6a502882d..c7f7be735 100644 --- a/tasks/bdev/include/bdev/bdev_tasks.h +++ b/tasks/bdev/include/bdev/bdev_tasks.h @@ -44,12 +44,6 @@ struct ConstructTask : public CreateTaskStateTask { // Custom params info_ = info; } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy hermes_mdm */ diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h index 6df2570a4..cfc03fe9a 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h @@ -45,12 +45,6 @@ struct ConstructTask : public CreateTaskStateTask { ~ConstructTask() { // Custom params } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy hermes_adapters */ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 015cbcc2b..0fa2e80c1 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -51,12 +51,6 @@ struct ConstructTask : public CreateTaskStateTask { : CreateTaskStateTask(alloc, task_node, domain_id, state_name, "hermes_blob_mdm", id, queue_info) { } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy hermes_mdm */ diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 2ec787bad..2dca04d1b 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -41,12 +41,6 @@ struct ConstructTask : public CreateTaskStateTask { : CreateTaskStateTask(alloc, task_node, domain_id, state_name, "hermes_bucket_mdm", id, queue_info) { } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy hermes_bucket_mdm */ diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h index 143f0b055..3a78e6de6 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h @@ -60,12 +60,6 @@ struct ConstructTask : public CreateTaskStateTask { /** (De)serialize message return */ template void SerializeEnd(u32 replica, Ar &ar) {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy hermes_mdm */ diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h index 52b4c6bce..0c464e258 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h @@ -45,12 +45,6 @@ struct ConstructTask : public CreateTaskStateTask { ~ConstructTask() { // Custom params } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy TASK_NAME */ diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index d236af213..5d524e16f 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -210,7 +210,9 @@ struct CreateTaskStateTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; + LocalSerialize srl(group); + srl << 16; + return 0; } }; diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index ee5bcf500..9372d1307 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -42,12 +42,6 @@ struct ConstructTask : public CreateTaskStateTask { ~ConstructTask() { // Custom params } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy proc_queue */ diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index a589a3652..5007ed42e 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -45,12 +45,6 @@ struct ConstructTask : public CreateTaskStateTask { "remote_queue", id, queue_info) { // Custom params } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy remote_queue */ diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h index d0926a34b..e9e6a27af 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h @@ -39,12 +39,6 @@ struct ConstructTask : public CreateTaskStateTask { : CreateTaskStateTask(alloc, task_node, domain_id, state_name, "worch_proc_round_robin", id, queue_info) { } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy worch_proc_round_robin */ diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h index a5acbbc01..68ebda34b 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h @@ -43,12 +43,6 @@ struct ConstructTask : public CreateTaskStateTask { /** Destructor */ HSHM_ALWAYS_INLINE ~ConstructTask() {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } }; /** A task to destroy worch_queue_round_robin */ From a1859ccf1aa088b96b94b1b44aad8e7f7c8ce958 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 16:45:12 -0500 Subject: [PATCH 039/191] Begin supporting blocking tasks --- .../labstor/queue_manager/queues/hshm_queue.h | 1 + include/labstor/task_registry/task.h | 12 ++ include/labstor/work_orchestrator/worker.h | 1 + src/worker.cc | 13 ++ tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 71 +++++----- .../labstor_admin/labstor_admin_tasks.h | 6 +- .../labstor_admin/src/labstor_admin.cc | 127 +++++++----------- 7 files changed, 109 insertions(+), 122 deletions(-) diff --git a/include/labstor/queue_manager/queues/hshm_queue.h b/include/labstor/queue_manager/queues/hshm_queue.h index d9c32ba31..a3ee79200 100644 --- a/include/labstor/queue_manager/queues/hshm_queue.h +++ b/include/labstor/queue_manager/queues/hshm_queue.h @@ -13,6 +13,7 @@ namespace labstor { struct LaneData { hipc::Pointer p_; bool complete_; + ABT_thread thread_; LaneData() = default; diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 4d300be94..3ff606607 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -43,6 +43,8 @@ namespace labstor { #define TASK_DATA_OWNER BIT_OPT(u32, 14) /** This task is marked */ #define TASK_MARKED BIT_OPT(u32, 15) +/** This task uses argobot wait */ +#define TASK_BLOCKING BIT_OPT(u32, 16) /** Used to define task methods */ #define TASK_METHOD_T static inline const u32 @@ -340,6 +342,16 @@ struct Task : public hipc::ShmContainer { task_flags_.UnsetBits(TASK_LONG_RUNNING); } + /** Set this task as blocking */ + HSHM_ALWAYS_INLINE void SetBlocking() { + task_flags_.SetBits(TASK_BLOCKING); + } + + /** Set this task as blocking */ + HSHM_ALWAYS_INLINE bool IsBlocking() { + return task_flags_.Any(TASK_BLOCKING); + } + /** Wait for task to complete */ template void Wait() { diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 16907ecc7..3a75d7bf0 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -364,6 +364,7 @@ class Worker { } void PollGrouped(WorkEntry &entry); + static void RunBlocking(void *data); }; } // namespace labstor diff --git a/src/worker.cc b/src/worker.cc index 37c696d27..4de72da55 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -75,6 +75,9 @@ void Worker::PollGrouped(WorkEntry &work_entry) { LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); task->DisableRun(); task->SetUnordered(); + } else if (task->IsBlocking()) { + task->SetStarted(); + entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunBlocking, task); } else { task->SetStarted(); exec->Run(task->method_, task, ctx); @@ -85,6 +88,9 @@ void Worker::PollGrouped(WorkEntry &work_entry) { // HILOG(kDebug, "(node {}) Ending task: task_node={} task_state={} lane={} queue={} worker={}", // LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); entry->complete_ = true; + if (task->IsBlocking()) { + ABT_thread_join(entry->thread_); + } RemoveTaskGroup(task, exec, work_entry.lane_id_, is_remote); EndTask(lane, task, off); } else { @@ -93,4 +99,11 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } } +void Worker::RunBlocking(void *data) { + Task *task = reinterpret_cast(data); + TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); + RunContext ctx(0); + exec->Run(task->method_, task, ctx); +} + } // namespace labstor \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index ccd5dc763..ba5a2268b 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -53,48 +53,37 @@ class Server : public TaskLib { void Construct(ConstructTask *task, RunContext &ctx) { id_alloc_ = 0; node_id_ = LABSTOR_CLIENT->node_id_; - switch (task->phase_) { - case ConstructTaskPhase::kCreateTaskStates: { - blob_id_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); - blob_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); - target_tasks_.reserve(HERMES_SERVER_CONF.devices_.size()); - for (DeviceInfo &dev : HERMES_SERVER_CONF.devices_) { - std::string dev_type; - if (dev.mount_dir_.empty()) { - dev_type = "ram_bdev"; - dev.mount_point_ = hshm::Formatter::format("{}/{}", dev.mount_dir_, dev.dev_name_); - } else { - dev_type = "posix_bdev"; - } - targets_.emplace_back(); - bdev::Client &client = targets_.back(); - bdev::ConstructTask *create_task = client.AsyncCreate( - task->task_node_ + 1, - DomainId::GetLocal(), - "hermes_" + dev.dev_name_, - dev_type, - dev).ptr_; - target_tasks_.emplace_back(create_task); - } - task->phase_ = ConstructTaskPhase::kWaitForTaskStates; - } - - case ConstructTaskPhase::kWaitForTaskStates: { - for (int i = (int)target_tasks_.size() - 1; i >= 0; --i) { - bdev::ConstructTask *tgt_task = target_tasks_[i]; - if (!tgt_task->IsComplete()) { - return; - } - bdev::Client &client = targets_[i]; - client.AsyncCreateComplete(tgt_task); - target_map_.emplace(client.id_, &client); - target_tasks_.pop_back(); - } - blob_mdm_.Init(id_); + // Initialize blob maps + blob_id_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + blob_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + // Initialize target tasks + target_tasks_.reserve(HERMES_SERVER_CONF.devices_.size()); + for (DeviceInfo &dev : HERMES_SERVER_CONF.devices_) { + std::string dev_type; + if (dev.mount_dir_.empty()) { + dev_type = "ram_bdev"; + dev.mount_point_ = hshm::Formatter::format("{}/{}", dev.mount_dir_, dev.dev_name_); + } else { + dev_type = "posix_bdev"; } - } - - // Create targets + targets_.emplace_back(); + bdev::Client &client = targets_.back(); + bdev::ConstructTask *create_task = client.AsyncCreate( + task->task_node_ + 1, + DomainId::GetLocal(), + "hermes_" + dev.dev_name_, + dev_type, + dev).ptr_; + target_tasks_.emplace_back(create_task); + } + for (int i = 0; i < target_tasks_.size(); ++i) { + bdev::ConstructTask *tgt_task = target_tasks_[i]; + tgt_task->Wait<1>(); + bdev::Client &client = targets_[i]; + client.AsyncCreateComplete(tgt_task); + target_map_.emplace(client.id_, &client); + } + blob_mdm_.Init(id_); HILOG(kInfo, "Created Blob MDM") task->SetModuleComplete(); } diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index 5d524e16f..7d9b11f42 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -147,8 +147,6 @@ struct CreateTaskStateTask : public Task, TaskFlags { IN hipc::ShmArchive state_name_; IN hipc::ShmArchive> queue_info_; INOUT TaskStateId id_; - TEMP int phase_ = 0; - TEMP GetOrCreateTaskStateIdTask *get_id_task_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -169,7 +167,7 @@ struct CreateTaskStateTask : public Task, TaskFlags { prio_ = TaskPrio::kAdmin; task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; method_ = Method::kCreateTaskState; - task_flags_.SetBits(0); + task_flags_.SetBits(TASK_BLOCKING); domain_id_ = domain_id; // Initialize @@ -198,7 +196,7 @@ struct CreateTaskStateTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); - ar(lib_name_, state_name_, id_, queue_info_, phase_); + ar(lib_name_, state_name_, id_, queue_info_); } /** (De)serialize message return */ diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index eb74f4acd..0b7b12264 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -35,84 +35,57 @@ class Server : public TaskLib { } void CreateTaskState(CreateTaskStateTask *task, RunContext &ctx) { - switch (task->phase_) { - case CreateTaskStatePhase::kIdAllocStart: { - std::string lib_name = task->lib_name_->str(); - std::string state_name = task->state_name_->str(); - // Check local registry for task state - TaskState *task_state = LABSTOR_TASK_REGISTRY->GetTaskState(state_name, task->id_); - if (task_state) { - task->id_ = task_state->id_; - task->SetModuleComplete(); - return; - } - // Check global registry for task state - if (task->id_.IsNull()) { - DomainId domain = DomainId::GetNode(1); - task->get_id_task_ = LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId( - task->task_node_ + 1, domain, state_name).ptr_; - task->phase_ = CreateTaskStatePhase::kIdAllocWait; - } else { - HILOG(kDebug, "Domain ID is given as {} for {} (task_node={})", task->id_, state_name, task->task_node_); - task->phase_ = CreateTaskStatePhase::kStateCreate; - } - return; - } - case CreateTaskStatePhase::kIdAllocWait: { - if (!task->get_id_task_->IsComplete()) { - return; - } - task->id_ = task->get_id_task_->id_; - task->phase_ = CreateTaskStatePhase::kStateCreate; - LABSTOR_CLIENT->DelTask(task->get_id_task_); - } - case CreateTaskStatePhase::kStateCreate: { - std::string lib_name = task->lib_name_->str(); - std::string state_name = task->state_name_->str(); - HILOG(kInfo, "(node {}) Creating task state {} with id {} (task_node={})", - LABSTOR_CLIENT->node_id_, state_name, task->id_, task->task_node_); - - // Verify the state isn't NULL - if (task->id_.IsNull()) { - HELOG(kError, "(node {}) The task state {} with id {} is NULL.", - LABSTOR_CLIENT->node_id_, state_name, task->id_); - task->SetModuleComplete(); - return; - } - - // Verify the state doesn't exist - if (LABSTOR_TASK_REGISTRY->TaskStateExists(task->id_)) { - HILOG(kInfo, "(node {}) The task state {} with id {} exists", - LABSTOR_CLIENT->node_id_, state_name, task->id_); - task->SetModuleComplete(); - return; - } - - // The state is being created - // NOTE(llogan): this does NOT return since task creations can have phases - task->method_ = Method::kConstruct; - - // Create the task queue for the state - QueueId qid(task->id_); - LABSTOR_QM_RUNTIME->CreateQueue( - qid, task->queue_info_->vec()); - - // Begin creating the task state - task->phase_ = 0; - task->task_state_ = task->id_; - bool ret = LABSTOR_TASK_REGISTRY->CreateTaskState( - lib_name.c_str(), - state_name.c_str(), - task->id_, - task); - if (!ret) { - task->SetModuleComplete(); - return; - } - HILOG(kInfo, "(node {}) Allocated task state {} with id {}", - LABSTOR_CLIENT->node_id_, state_name, task->task_state_); - } + std::string lib_name = task->lib_name_->str(); + std::string state_name = task->state_name_->str(); + // Check local registry for task state + TaskState *task_state = LABSTOR_TASK_REGISTRY->GetTaskState(state_name, task->id_); + if (task_state) { + task->id_ = task_state->id_; + task->SetModuleComplete(); + return; + } + // Check global registry for task state + if (task->id_.IsNull()) { + DomainId domain = DomainId::GetNode(1); + LPointer get_id = + LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId(task->task_node_ + 1, domain, state_name); + get_id->Wait<1>(); + task->id_ = get_id->id_; + LABSTOR_CLIENT->DelTask(get_id); + } + // Create the task state + HILOG(kInfo, "(node {}) Creating task state {} with id {} (task_node={})", + LABSTOR_CLIENT->node_id_, state_name, task->id_, task->task_node_); + if (task->id_.IsNull()) { + HELOG(kError, "(node {}) The task state {} with id {} is NULL.", + LABSTOR_CLIENT->node_id_, state_name, task->id_); + task->SetModuleComplete(); + return; + } + // Verify the state doesn't exist + if (LABSTOR_TASK_REGISTRY->TaskStateExists(task->id_)) { + HILOG(kInfo, "(node {}) The task state {} with id {} exists", + LABSTOR_CLIENT->node_id_, state_name, task->id_); + task->SetModuleComplete(); + return; + } + // Create the task queue for the state + QueueId qid(task->id_); + LABSTOR_QM_RUNTIME->CreateQueue( + qid, task->queue_info_->vec()); + // Run the task state's submethod + task->method_ = Method::kConstruct; + bool ret = LABSTOR_TASK_REGISTRY->CreateTaskState( + lib_name.c_str(), + state_name.c_str(), + task->id_, + task); + if (!ret) { + task->SetModuleComplete(); + return; } + HILOG(kInfo, "(node {}) Allocated task state {} with id {}", + LABSTOR_CLIENT->node_id_, state_name, task->task_state_); } void GetTaskStateId(GetTaskStateIdTask *task, RunContext &ctx) { From 4a8f3ebef8c3221010828cfe529e4b755899a493 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 16:57:30 -0500 Subject: [PATCH 040/191] Make remote queue push a blocking task --- .../include/remote_queue/remote_queue_tasks.h | 2 +- .../remote_queue/src/remote_queue.cc | 194 ++++++------------ 2 files changed, 62 insertions(+), 134 deletions(-) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index 5007ed42e..53b30677a 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -114,7 +114,7 @@ struct PushTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPush; - task_flags_.SetBits(TASK_LOW_LATENCY); + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_BLOCKING); domain_id_ = domain_id; // Custom params diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index bbc59f793..2ca1b70f5 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -15,39 +15,6 @@ SERIALIZE_ENUM(labstor::IoType); namespace labstor::remote_queue { -/** Parameters for spawning async thread for thallium */ -struct ThalliumTask { - int replica_; - PushTask *task_; - DomainId domain_id_; - IoType io_type_; - char *data_; - size_t data_size_; - ABT_thread thread_; - bool done_; - - /** Default constructor */ - ThalliumTask() : done_(false) {}; - - /** Emplace constructor Small */ - ThalliumTask(int replica, PushTask *task, DomainId domain_id) : - replica_(replica), task_(task), domain_id_(domain_id), done_(false) {} - - /** Emplace constructor I/O */ - ThalliumTask(int replica, PushTask *task, DomainId domain_id, - IoType io_type, void *data, size_t data_size) : - replica_(replica), task_(task), domain_id_(domain_id), - io_type_(io_type), data_((char*)data), data_size_(data_size), done_(false) {} - - /** Check if the thread is finished */ - bool IsDone() { - if (done_) { - ABT_thread_join(thread_); - } - return done_; - } -}; - class Server : public TaskLib { public: labstor::remote_queue::Client client_; @@ -112,35 +79,26 @@ class Server : public TaskLib { task->params_ = std::string((char *) xfer[0].data_, xfer[0].data_size_); for (int replica = 0; replica < task->domain_ids_.size(); ++replica) { DomainId domain_id = task->domain_ids_[replica]; - ThalliumTask *tl_task = new ThalliumTask(replica, task, domain_id); - tl_task->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread([](void *data) { - ThalliumTask *tl_task = (ThalliumTask *) data; - DomainId &domain_id = tl_task->domain_id_; - PushTask *task = tl_task->task_; - int replica = tl_task->replica_; - HILOG(kDebug, "(SM) Transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={})", - task->params_.size(), - task->orig_task_->task_node_, - task->orig_task_->task_state_, - task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, - domain_id.id_); - std::string ret = LABSTOR_THALLIUM->SyncCall(domain_id.id_, - "RpcPushSmall", - task->exec_->id_, - task->exec_method_, - task->params_); - HILOG(kDebug, "(SM) Finished {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={})", - task->params_.size(), - task->orig_task_->task_node_, - task->orig_task_->task_state_, - task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, - domain_id.id_); - HandlePushReplicaOutput(replica, ret, task); - tl_task->done_ = true; - }, tl_task); - task->tl_future_.emplace_back(tl_task); + HILOG(kDebug, "(SM) Transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={})", + task->params_.size(), + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_); + std::string ret = LABSTOR_THALLIUM->SyncCall(domain_id.id_, + "RpcPushSmall", + task->exec_->id_, + task->exec_method_, + task->params_); + HILOG(kDebug, "(SM) Finished {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={})", + task->params_.size(), + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_); + HandlePushReplicaOutput(replica, ret, task); } } @@ -153,85 +111,55 @@ class Server : public TaskLib { } for (int replica = 0; replica < task->domain_ids_.size(); ++replica) { DomainId domain_id = task->domain_ids_[replica]; - ThalliumTask *tl_task = new ThalliumTask( - replica, task, domain_id, io_type, - xfer[0].data_, xfer[0].data_size_); - tl_task->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread([](void *data) { - ThalliumTask *tl_task = (ThalliumTask *) data; - DomainId &domain_id = tl_task->domain_id_; - PushTask *task = tl_task->task_; - int replica = tl_task->replica_; - IoType &io_type = tl_task->io_type_; - HILOG(kDebug, "(IO) Transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={}, type={})", - tl_task->data_size_, - task->orig_task_->task_node_, - task->orig_task_->task_state_, - task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, - domain_id.id_, - static_cast(io_type)); - std::string ret = LABSTOR_THALLIUM->SyncIoCall(domain_id.id_, - "RpcPushBulk", - io_type, - tl_task->data_, - tl_task->data_size_, - task->exec_->id_, - task->exec_method_, - task->params_, - tl_task->data_size_, - io_type); - HILOG(kDebug, "(IO) Finished transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={}, type={})", - tl_task->data_size_, - task->orig_task_->task_node_, - task->orig_task_->task_state_, - task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, - domain_id.id_, - static_cast(io_type)); - HandlePushReplicaOutput(replica, ret, task); - tl_task->done_ = true; - }, tl_task); - task->tl_future_.emplace_back(tl_task); - } - } - - /** Wait for client to finish message */ - void ClientWaitForMessage(PushTask *task) { - for (; task->replica_ < task->tl_future_.size(); ++task->replica_) { - ThalliumTask *tl_task = (ThalliumTask *) task->tl_future_[task->replica_]; - if (!tl_task->IsDone()) { - return; - } - delete tl_task; + char *data = (char*)xfer[0].data_; + size_t data_size = xfer[0].data_size_; + HILOG(kDebug, "(IO) Transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={}, type={})", + data_size, + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_, + static_cast(io_type)); + std::string ret = LABSTOR_THALLIUM->SyncIoCall(domain_id.id_, + "RpcPushBulk", + io_type, + data, + data_size, + task->exec_->id_, + task->exec_method_, + task->params_, + data_size, + io_type); + HILOG(kDebug, "(IO) Finished transferring {} bytes of data (task_node={}, task_state={}, method={}, from={}, to={}, type={})", + data_size, + task->orig_task_->task_node_, + task->orig_task_->task_state_, + task->orig_task_->method_, + LABSTOR_CLIENT->node_id_, + domain_id.id_, + static_cast(io_type)); + HandlePushReplicaOutput(replica, ret, task); } - HandlePushReplicaEnd(task); } /** Push operation called on client */ void Push(PushTask *task, RunContext &ctx) { - switch (task->phase_) { - case PushPhase::kStart: { - std::vector &xfer = task->xfer_; - task->tl_future_.reserve(task->domain_ids_.size()); - switch (task->xfer_.size()) { - case 1: { - ClientSmallPush(xfer, task); - break; - } - case 2: { - ClientIoPush(xfer, task); - break; - } - default: { - HELOG(kFatal, "The task {}/{} does not support remote calls", task->task_state_, task->method_); - } - } - task->phase_ = PushPhase::kWait; + std::vector &xfer = task->xfer_; + switch (task->xfer_.size()) { + case 1: { + ClientSmallPush(xfer, task); + break; } - case PushPhase::kWait: { - ClientWaitForMessage(task); + case 2: { + ClientIoPush(xfer, task); + break; + } + default: { + HELOG(kFatal, "The task {}/{} does not support remote calls", task->task_state_, task->method_); } } + HandlePushReplicaEnd(task); } private: From 4717b8874f11db533dba76419f0c95d862679945 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 17:10:25 -0500 Subject: [PATCH 041/191] don't constantly spawn async threads --- src/worker.cc | 2 +- .../remote_queue/include/remote_queue/remote_queue_tasks.h | 4 ---- tasks_required/remote_queue/src/remote_queue.cc | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/worker.cc b/src/worker.cc index 4de72da55..54dce53a4 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -75,7 +75,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); task->DisableRun(); task->SetUnordered(); - } else if (task->IsBlocking()) { + } else if (task->IsBlocking() && !task->IsStarted()) { task->SetStarted(); entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunBlocking, task); } else { diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index 53b30677a..781b3e285 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -88,9 +88,6 @@ struct PushTask : public Task, TaskFlags { IN TaskState *exec_; IN u32 exec_method_; IN std::vector xfer_; - TEMP std::vector tl_future_; - TEMP int phase_ = PushPhase::kStart; - TEMP int replica_; TEMP std::string params_; /** SHM default constructor */ @@ -123,7 +120,6 @@ struct PushTask : public Task, TaskFlags { exec_ = exec; exec_method_ = exec_method; xfer_ = std::move(xfer); - replica_ = 0; } /** Create group */ diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 2ca1b70f5..096bcfe6a 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -146,7 +146,7 @@ class Server : public TaskLib { /** Push operation called on client */ void Push(PushTask *task, RunContext &ctx) { std::vector &xfer = task->xfer_; - switch (task->xfer_.size()) { + switch (xfer.size()) { case 1: { ClientSmallPush(xfer, task); break; From f67d2302c4185f1f40617358075dd627c0effcba Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 17:11:01 -0500 Subject: [PATCH 042/191] don't constantly spawn async threads --- src/worker.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/worker.cc b/src/worker.cc index 54dce53a4..1813d5a9e 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -75,7 +75,8 @@ void Worker::PollGrouped(WorkEntry &work_entry) { LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); task->DisableRun(); task->SetUnordered(); - } else if (task->IsBlocking() && !task->IsStarted()) { + } else if (task->IsBlocking()) { + task->DisableRun(); task->SetStarted(); entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunBlocking, task); } else { From c8038963e85e6e46c7e4773b843bfef48c8d03c5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 17:15:27 -0500 Subject: [PATCH 043/191] Make CreateTaskState unordered again --- .../labstor_admin/include/labstor_admin/labstor_admin_tasks.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index 7d9b11f42..61a50718a 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -208,9 +208,7 @@ struct CreateTaskStateTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - LocalSerialize srl(group); - srl << 16; - return 0; + return TASK_UNORDERED; } }; From 5e2d8e904fd38f61e9233c92f49869453d141641 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 20:42:34 -0500 Subject: [PATCH 044/191] Add cooperative tasking using boost --- CMakeLists.txt | 11 +- ci/hermes/packages/hermes_shm/package.py | 9 +- .../labstor_admin/labstor_admin_tasks.h | 4 +- test/unit/CMakeLists.txt | 3 +- test/unit/boost/CMakeLists.txt | 43 ++++++++ test/unit/boost/test_boost.cc | 100 ++++++++++++++++++ test/unit/boost/test_init.cc | 22 ++++ test/unit/boost/test_init.h | 19 ++++ 8 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 test/unit/boost/CMakeLists.txt create mode 100644 test/unit/boost/test_boost.cc create mode 100644 test/unit/boost/test_init.cc create mode 100644 test/unit/boost/test_init.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 481df1597..f2b98ba34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,6 +93,14 @@ if(thallium_FOUND) message(STATUS "found thallium at ${thallium_DIR}") endif() +# Boost +find_package(Boost REQUIRED COMPONENTS regex system filesystem fiber REQUIRED) +if (Boost_FOUND) + message(STATUS "found boost at ${Boost_INCLUDE_DIRS}") +endif() +include_directories(${Boost_INCLUDE_DIRS}) +message("Boost: ${Boost_LIBRARIES}") + #------------------------------------------------------------------------------ # Setup CMake Environment #------------------------------------------------------------------------------ @@ -158,7 +166,8 @@ set(Labstor_CLIENT_DEPS labstor_client) set(Labstor_RUNTIME_LIBRARIES ${Labstor_CLIENT_LIBRARIES} - labstor_runtime) + labstor_runtime + ${Boost_LIBRARIES}) set(Labstor_RUNTIME_DEPS labstor_client labstor_runtime) diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index 9225ceb29..76343ff00 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -5,11 +5,12 @@ class HermesShm(CMakePackage): git = "https://github.com/lukemartinlogan/hermes_shm.git" version('master', branch='master') depends_on('mochi-thallium~cereal@0.10.1') - # depends_on('catch2@3.0.1') - depends_on('catch2') - depends_on('mpi') - depends_on('boost@1.7:') + depends_on('catch2@3.0.1') + # depends_on('mpi') + depends_on('mpich@3.3.2') + depends_on('boost@1.7: +context +fiber') depends_on('cereal') + depends_on('yaml-cpp') depends_on('doxygen@1.9.3') variant('debug', default=False, description='Build shared libraries') diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index 61a50718a..7d9b11f42 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -208,7 +208,9 @@ struct CreateTaskStateTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; + LocalSerialize srl(group); + srl << 16; + return 0; } }; diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index b8b63529d..dbfd08a04 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -7,4 +7,5 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_SOURCE_DIR}/tasks/labstor_admin/include) add_subdirectory(ipc) add_subdirectory(hermes) -add_subdirectory(hermes_adapters) \ No newline at end of file +add_subdirectory(hermes_adapters) +add_subdirectory(boost) \ No newline at end of file diff --git a/test/unit/boost/CMakeLists.txt b/test/unit/boost/CMakeLists.txt new file mode 100644 index 000000000..81746e03f --- /dev/null +++ b/test/unit/boost/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.10) +project(labstor) + +set(CMAKE_CXX_STANDARD 17) + +#------------------------------------------------------------------------------ +# Build Tests +#------------------------------------------------------------------------------ + +add_executable(test_boost_exec + ${TEST_MAIN}/main.cc + test_init.cc + test_boost.cc +) +add_dependencies(test_boost_exec + ${Labstor_RUNTIME_DEPS} hermes) +target_link_libraries(test_boost_exec + ${Labstor_RUNTIME_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX) + +#------------------------------------------------------------------------------ +# Test Cases +#------------------------------------------------------------------------------ + +add_test(NAME test_boost COMMAND + ${CMAKE_BINARY_DIR}/bin/test_messages "TestBoost") + +#------------------------------------------------------------------------------ +# Install Targets +#------------------------------------------------------------------------------ +install(TARGETS + test_boost_exec + EXPORT + ${LABSTOR_EXPORTED_TARGETS} + LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(LABSTOR_ENABLE_COVERAGE) + set_coverage_flags(test_boost_exec) +endif() diff --git a/test/unit/boost/test_boost.cc b/test/unit/boost/test_boost.cc new file mode 100644 index 000000000..15ae5161b --- /dev/null +++ b/test/unit/boost/test_boost.cc @@ -0,0 +1,100 @@ +// +// Created by llogan on 7/1/23. +// + +#include "basic_test.h" +#include +#include "hermes_shm/util/timer.h" +#include "hermes_shm/util/logging.h" + +void function() { +} + +TEST_CASE("TestBoostFiber") { + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + + for (size_t i = 0; i < ops; ++i) { + int a; + boost::context::fiber source([&a](boost::context::fiber &&sink) { + function(); + return std::move(sink); + }); + } + + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +template< std::size_t Max, std::size_t Default, std::size_t Min > +class simple_stack_allocator +{ + public: + static std::size_t maximum_stacksize() + { return Max; } + + static std::size_t default_stacksize() + { return Default; } + + static std::size_t minimum_stacksize() + { return Min; } + + void * allocate( std::size_t size) const + { + BOOST_ASSERT( minimum_stacksize() <= size); + BOOST_ASSERT( maximum_stacksize() >= size); + + void * limit = malloc( size); + if ( ! limit) throw std::bad_alloc(); + + return static_cast< char * >( limit) + size; + } + + void deallocate( void * vp, std::size_t size) const + { + BOOST_ASSERT( vp); + BOOST_ASSERT( minimum_stacksize() <= size); + BOOST_ASSERT( maximum_stacksize() >= size); + + void * limit = static_cast< char * >( vp) - size; + free( limit); + } +}; + +typedef simple_stack_allocator< + 8 * 1024 * 1024, + 64 * 1024, + 32> stack_allocator; + +int value1; +namespace ctx = boost::context::detail; + +void f3( ctx::transfer_t t_) { + ++value1; + ctx::transfer_t t = ctx::jump_fcontext( t_.fctx, 0); + ++value1; + ctx::jump_fcontext( t.fctx, t.data); +} + + +TEST_CASE("TestBoostFcontext") { + value1 = 0; + stack_allocator alloc; + int size = 128; + + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + + void *sp = alloc.allocate(size); + for (size_t i = 0; i < ops; ++i) { + ctx::fcontext_t ctx = ctx::make_fcontext(sp, size, f3); + ctx::transfer_t t = ctx::jump_fcontext(ctx, 0); + ctx::jump_fcontext(t.fctx, 0); + } + alloc.deallocate(sp, size); + + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} \ No newline at end of file diff --git a/test/unit/boost/test_init.cc b/test/unit/boost/test_init.cc new file mode 100644 index 000000000..e101758d3 --- /dev/null +++ b/test/unit/boost/test_init.cc @@ -0,0 +1,22 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "labstor/api/labstor_client.h" +#include "basic_test.h" +#include "test_init.h" + +void MainPretest() { +} + +void MainPosttest() { +} diff --git a/test/unit/boost/test_init.h b/test/unit/boost/test_init.h new file mode 100644 index 000000000..a6c71f3ec --- /dev/null +++ b/test/unit/boost/test_init.h @@ -0,0 +1,19 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ + +#include "labstor/labstor_types.h" + +#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ From 576c9a655d77967287570e3d236854ffd35d5b23 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 23:14:10 -0500 Subject: [PATCH 045/191] Add high-performance co-routines for complex tasks --- include/labstor/labstor_types.h | 4 ++ include/labstor/task_registry/task.h | 62 ++++++++++++++----- include/labstor/work_orchestrator/worker.h | 7 ++- src/worker.cc | 41 +++++++++--- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 2 +- .../labstor_admin/labstor_admin_tasks.h | 2 +- .../labstor_admin/src/labstor_admin.cc | 2 +- .../include/remote_queue/remote_queue_tasks.h | 2 +- .../remote_queue/src/remote_queue.cc | 2 +- test/unit/boost/test_boost.cc | 38 +++++++++--- 10 files changed, 124 insertions(+), 38 deletions(-) diff --git a/include/labstor/labstor_types.h b/include/labstor/labstor_types.h index e4c8a1faf..9d01597b3 100644 --- a/include/labstor/labstor_types.h +++ b/include/labstor/labstor_types.h @@ -28,6 +28,10 @@ #include "hermes_shm/util/singleton.h" #include "hermes_shm/constants/macros.h" +#include + +namespace bctx = boost::context::detail; + typedef uint8_t u8; /**< 8-bit unsigned integer */ typedef uint16_t u16; /**< 16-bit unsigned integer */ typedef uint32_t u32; /**< 32-bit unsigned integer */ diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 3ff606607..2c6b14aa7 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -43,12 +43,19 @@ namespace labstor { #define TASK_DATA_OWNER BIT_OPT(u32, 14) /** This task is marked */ #define TASK_MARKED BIT_OPT(u32, 15) +/** This task uses co-routine wait */ +#define TASK_COROUTINE BIT_OPT(u32, 16) /** This task uses argobot wait */ -#define TASK_BLOCKING BIT_OPT(u32, 16) +#define TASK_PREEMPTIVE BIT_OPT(u32, 17) /** Used to define task methods */ #define TASK_METHOD_T static inline const u32 +/** Used to indicate Yield to use */ +#define TASK_YIELD_STD 0 +#define TASK_YIELD_CO 1 +#define TASK_YIELD_ABT 2 + /** The baseline set of tasks */ struct TaskMethod { TASK_METHOD_T kConstruct = 0; /**< The constructor of the task */ @@ -232,6 +239,9 @@ struct Task : public hipc::ShmContainer { u32 lane_hash_; /**< Determine the lane a task is keyed to */ u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ + bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ + size_t stack_size_ = KILOBYTES(256); /**< The size of the stack for the task (runtime) */ + void *stack_ptr_; /**< The pointer to the stack (runtime) */ /**==================================== * Task Helpers @@ -343,29 +353,51 @@ struct Task : public hipc::ShmContainer { } /** Set this task as blocking */ - HSHM_ALWAYS_INLINE void SetBlocking() { - task_flags_.SetBits(TASK_BLOCKING); + HSHM_ALWAYS_INLINE bool IsCoroutine() { + return task_flags_.Any(TASK_COROUTINE); } /** Set this task as blocking */ - HSHM_ALWAYS_INLINE bool IsBlocking() { - return task_flags_.Any(TASK_BLOCKING); + HSHM_ALWAYS_INLINE bool IsPreemptive() { + return task_flags_.Any(TASK_PREEMPTIVE); + } + + /** Yield the task */ + template + HSHM_ALWAYS_INLINE + void Yield() { + if constexpr (THREAD_MODEL == TASK_YIELD_STD) { + HERMES_THREAD_MODEL->Yield(); + } else if constexpr (THREAD_MODEL == TASK_YIELD_CO) { + jmp_ = bctx::jump_fcontext(jmp_.fctx, nullptr); + } else if constexpr (THREAD_MODEL == TASK_YIELD_ABT) { + ABT_thread_yield(); + } } /** Wait for task to complete */ template void Wait() { while (!IsComplete()) { - for (int i = 0; i < 100000; ++i) { - if (IsComplete()) { - return; - } - } - if constexpr(THREAD_MODEL == 0) { - HERMES_THREAD_MODEL->Yield(); - } else { - ABT_thread_yield(); - } +// for (int i = 0; i < 100000; ++i) { +// if (IsComplete()) { +// return; +// } +// } + Yield(); + } + } + + /** Wait for task to complete */ + template + void Wait(Task *yield_task) { + while (!IsComplete()) { +// for (int i = 0; i < 100000; ++i) { +// if (IsComplete()) { +// return; +// } +// } + yield_task->Yield(); } } diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 3a75d7bf0..1e46009a3 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -29,6 +29,10 @@ struct WorkEntry { LaneGroup *group_; MultiQueue *queue_; + TaskState *exec_; + Task *task_; + RunContext ctx_; + /** Default constructor */ HSHM_ALWAYS_INLINE WorkEntry() = default; @@ -364,7 +368,8 @@ class Worker { } void PollGrouped(WorkEntry &entry); - static void RunBlocking(void *data); + static void RunBlocking(bctx::transfer_t t); + static void RunPreemptive(void *data); }; } // namespace labstor diff --git a/src/worker.cc b/src/worker.cc index 1813d5a9e..575916f5b 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -41,12 +41,13 @@ void Worker::Run() { } void Worker::PollGrouped(WorkEntry &work_entry) { - Lane *lane = work_entry.lane_; - Task *task; - LaneData *entry; int off = 0; - RunContext ctx; + Lane *&lane = work_entry.lane_; + Task *&task = work_entry.task_; + TaskState *&exec = work_entry.exec_; + RunContext &ctx = work_entry.ctx_; ctx.lane_id_ = work_entry.lane_id_; + LaneData *entry; for (int i = 0; i < 1024; ++i) { // Get the task message if (lane->peek(entry, off).IsNull()) { @@ -58,7 +59,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } task = LABSTOR_CLIENT->GetPrivatePointer(entry->p_); // Get the task state - TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); + exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); if (!exec) { HELOG(kFatal, "(node {}) Could not find the task state: {}", LABSTOR_CLIENT->node_id_, task->task_state_); @@ -75,10 +76,19 @@ void Worker::PollGrouped(WorkEntry &work_entry) { LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); task->DisableRun(); task->SetUnordered(); - } else if (task->IsBlocking()) { + } else if (task->IsCoroutine()) { + if (!task->IsStarted()) { + task->stack_ptr_ = malloc(task->stack_size_); + task->jmp_.fctx = bctx::make_fcontext( + (char*)task->stack_ptr_ + task->stack_size_, + task->stack_size_, &RunBlocking); + task->SetStarted(); + } + task->jmp_ = bctx::jump_fcontext(task->jmp_.fctx, task); + HILOG(kInfo, "Jumping into function") + } else if (task->IsPreemptive()) { task->DisableRun(); - task->SetStarted(); - entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunBlocking, task); + entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunPreemptive, task); } else { task->SetStarted(); exec->Run(task->method_, task, ctx); @@ -89,7 +99,9 @@ void Worker::PollGrouped(WorkEntry &work_entry) { // HILOG(kDebug, "(node {}) Ending task: task_node={} task_state={} lane={} queue={} worker={}", // LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); entry->complete_ = true; - if (task->IsBlocking()) { + if (task->IsCoroutine()) { + free(task->stack_ptr_); + } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); } RemoveTaskGroup(task, exec, work_entry.lane_id_, is_remote); @@ -100,7 +112,16 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } } -void Worker::RunBlocking(void *data) { +void Worker::RunBlocking(bctx::transfer_t t) { + Task *task = reinterpret_cast(t.data); + task->jmp_ = t; + TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); + RunContext ctx(0); + exec->Run(task->method_, task, ctx); + task->Yield(); +} + +void Worker::RunPreemptive(void *data) { Task *task = reinterpret_cast(data); TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); RunContext ctx(0); diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index ba5a2268b..33cd20c70 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -78,7 +78,7 @@ class Server : public TaskLib { } for (int i = 0; i < target_tasks_.size(); ++i) { bdev::ConstructTask *tgt_task = target_tasks_[i]; - tgt_task->Wait<1>(); + tgt_task->Wait(task); bdev::Client &client = targets_[i]; client.AsyncCreateComplete(tgt_task); target_map_.emplace(client.id_, &client); diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index 7d9b11f42..dd28394e9 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -167,7 +167,7 @@ struct CreateTaskStateTask : public Task, TaskFlags { prio_ = TaskPrio::kAdmin; task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; method_ = Method::kCreateTaskState; - task_flags_.SetBits(TASK_BLOCKING); + task_flags_.SetBits(TASK_COROUTINE); domain_id_ = domain_id; // Initialize diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index 0b7b12264..d80e4998a 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -49,7 +49,7 @@ class Server : public TaskLib { DomainId domain = DomainId::GetNode(1); LPointer get_id = LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId(task->task_node_ + 1, domain, state_name); - get_id->Wait<1>(); + get_id->Wait(task); task->id_ = get_id->id_; LABSTOR_CLIENT->DelTask(get_id); } diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index 781b3e285..47e5ff634 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -111,7 +111,7 @@ struct PushTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPush; - task_flags_.SetBits(TASK_LOW_LATENCY | TASK_BLOCKING); + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_COROUTINE); domain_id_ = domain_id; // Custom params diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 096bcfe6a..b0718a7c7 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -263,7 +263,7 @@ class Server : public TaskLib { method, data_size, orig_task->lane_hash_); - orig_task->Wait<1>(); + orig_task->Wait(); } void RpcComplete(const tl::request &req, diff --git a/test/unit/boost/test_boost.cc b/test/unit/boost/test_boost.cc index 15ae5161b..78146590c 100644 --- a/test/unit/boost/test_boost.cc +++ b/test/unit/boost/test_boost.cc @@ -6,6 +6,8 @@ #include #include "hermes_shm/util/timer.h" #include "hermes_shm/util/logging.h" +#include + void function() { } @@ -68,13 +70,13 @@ typedef simple_stack_allocator< 32> stack_allocator; int value1; -namespace ctx = boost::context::detail; +namespace bctx = boost::context::detail; -void f3( ctx::transfer_t t_) { +void f3( bctx::transfer_t t_) { ++value1; - ctx::transfer_t t = ctx::jump_fcontext( t_.fctx, 0); + bctx::transfer_t t = bctx::jump_fcontext( t_.fctx, 0); ++value1; - ctx::jump_fcontext( t.fctx, t.data); + bctx::jump_fcontext( t.fctx, t.data); } @@ -89,12 +91,34 @@ TEST_CASE("TestBoostFcontext") { void *sp = alloc.allocate(size); for (size_t i = 0; i < ops; ++i) { - ctx::fcontext_t ctx = ctx::make_fcontext(sp, size, f3); - ctx::transfer_t t = ctx::jump_fcontext(ctx, 0); - ctx::jump_fcontext(t.fctx, 0); + bctx::fcontext_t ctx = bctx::make_fcontext(sp, size, f3); + bctx::transfer_t t = bctx::jump_fcontext(ctx, 0); + bctx::jump_fcontext(t.fctx, 0); } alloc.deallocate(sp, size); + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +using namespace boost::coroutines2; + +void myCoroutine(coroutine::push_type& yield) { + for (int i = 1; i <= 5; ++i) { + yield(); + } +} + +TEST_CASE("TestBoostCoroutine") { + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + + for (size_t i = 0; i < ops; ++i) { + coroutine::pull_type myCoroutineInstance(myCoroutine); + myCoroutineInstance(); + } + t.Pause(); HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } \ No newline at end of file From 57abb5b848c9bdfc54f97ada2654299e32c724ac Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 23 Sep 2023 23:19:02 -0500 Subject: [PATCH 046/191] Use a 16KB stack size --- include/labstor/task_registry/task.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 2c6b14aa7..d7b142043 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -240,7 +240,7 @@ struct Task : public hipc::ShmContainer { u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ - size_t stack_size_ = KILOBYTES(256); /**< The size of the stack for the task (runtime) */ + size_t stack_size_ = KILOBYTES(16); /**< The size of the stack for the task (runtime) */ void *stack_ptr_; /**< The pointer to the stack (runtime) */ /**==================================== From 75e3e14ef92473971237923a8fd0da05a7ec68be Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 01:29:38 -0500 Subject: [PATCH 047/191] Keep work entry when jumping to new context --- src/worker.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/worker.cc b/src/worker.cc index 575916f5b..146f6e9f5 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -84,7 +84,8 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->stack_size_, &RunBlocking); task->SetStarted(); } - task->jmp_ = bctx::jump_fcontext(task->jmp_.fctx, task); + task->jmp_.data = &work_entry; + task->jmp_ = bctx::jump_fcontext(task->jmp_.fctx, &work_entry); HILOG(kInfo, "Jumping into function") } else if (task->IsPreemptive()) { task->DisableRun(); @@ -113,10 +114,11 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } void Worker::RunBlocking(bctx::transfer_t t) { - Task *task = reinterpret_cast(t.data); + WorkEntry *work_entry = reinterpret_cast(t.data); + Task *&task = work_entry->task_; + TaskState *&exec = work_entry->exec_; + RunContext &ctx = work_entry->ctx_; task->jmp_ = t; - TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); - RunContext ctx(0); exec->Run(task->method_, task, ctx); task->Yield(); } From 530e17bd40267bbc8a7075e6516c344b94647c06 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 01:37:02 -0500 Subject: [PATCH 048/191] Make network unordered again --- .../remote_queue/include/remote_queue/remote_queue_tasks.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index 47e5ff634..c3e79057d 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -111,7 +111,7 @@ struct PushTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPush; - task_flags_.SetBits(TASK_LOW_LATENCY | TASK_COROUTINE); + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_PREEMPTIVE); domain_id_ = domain_id; // Custom params @@ -125,10 +125,7 @@ struct PushTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - LocalSerialize srl(group); - srl << 16; - // return TASK_UNORDERED; - return 0; + return TASK_UNORDERED; } }; From 8d8523587991ed83968e15f7612da23a0f96b54a Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 09:06:27 -0500 Subject: [PATCH 049/191] Print the nodes which finished blob_mdm --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 33cd20c70..43c0eb968 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -84,7 +84,7 @@ class Server : public TaskLib { target_map_.emplace(client.id_, &client); } blob_mdm_.Init(id_); - HILOG(kInfo, "Created Blob MDM") + HILOG(kInfo, "(node {}) Created Blob MDM") task->SetModuleComplete(); } From 35b0f5bec641ae4cc4b465d746378a527c033771 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 09:06:47 -0500 Subject: [PATCH 050/191] Print the nodes which finished blob_mdm --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 43c0eb968..4ee237633 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -84,7 +84,7 @@ class Server : public TaskLib { target_map_.emplace(client.id_, &client); } blob_mdm_.Init(id_); - HILOG(kInfo, "(node {}) Created Blob MDM") + HILOG(kInfo, "(node {}) Created Blob MDM", LABSTOR_CLIENT->node_id_); task->SetModuleComplete(); } From 954374a333e3786955a060d939182de4c1de4c1d Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 09:21:00 -0500 Subject: [PATCH 051/191] Use 64KB stack size --- include/labstor/task_registry/task.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index d7b142043..e28730080 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -240,7 +240,7 @@ struct Task : public hipc::ShmContainer { u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ - size_t stack_size_ = KILOBYTES(16); /**< The size of the stack for the task (runtime) */ + size_t stack_size_ = KILOBYTES(64); /**< The size of the stack for the task (runtime) */ void *stack_ptr_; /**< The pointer to the stack (runtime) */ /**==================================== From 192dbd98f5fc17af67ce67a25d792c53225abb73 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 09:48:45 -0500 Subject: [PATCH 052/191] Use 256KB stack size --- include/labstor/task_registry/task.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index e28730080..2c6b14aa7 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -240,7 +240,7 @@ struct Task : public hipc::ShmContainer { u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ - size_t stack_size_ = KILOBYTES(64); /**< The size of the stack for the task (runtime) */ + size_t stack_size_ = KILOBYTES(256); /**< The size of the stack for the task (runtime) */ void *stack_ptr_; /**< The pointer to the stack (runtime) */ /**==================================== From 0208c3d58a8c8f6f8ea11a785ec820e002cbc318 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 09:49:00 -0500 Subject: [PATCH 053/191] Use 256KB stack size --- src/worker.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/worker.cc b/src/worker.cc index 146f6e9f5..00955490b 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -84,7 +84,6 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->stack_size_, &RunBlocking); task->SetStarted(); } - task->jmp_.data = &work_entry; task->jmp_ = bctx::jump_fcontext(task->jmp_.fctx, &work_entry); HILOG(kInfo, "Jumping into function") } else if (task->IsPreemptive()) { From 719adabe9c1d719dc7a19b7efc7edc64e00ff8ea Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 09:51:46 -0500 Subject: [PATCH 054/191] Use 256KB stack size --- tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 75cae81b3..a39dc86a9 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -190,7 +190,7 @@ class Server : public TaskLib { /** Get or create a tag */ void GetOrCreateTag(GetOrCreateTagTask *task, RunContext &ctx) { TagId tag_id; - HILOG(kDebug, "Creating a tag") + HILOG(kDebug, "Creating a tag on lane {}", ctx.lane_id_); // Check if the tag exists TAG_ID_MAP_T &tag_id_map = tag_id_map_[ctx.lane_id_]; From d5e4c4572b7ced1a221787156ad7ee4a55856a96 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 10:15:26 -0500 Subject: [PATCH 055/191] Make RunContext apart of task for now --- include/labstor/task_registry/task.h | 23 ++++++++++++--- include/labstor/task_registry/task_lib.h | 11 -------- include/labstor/work_orchestrator/worker.h | 4 --- src/worker.cc | 33 ++++++++++++---------- test/unit/asan.supp | 10 +++++++ test/unit/boost/test_boost.cc | 22 +++++++++------ 6 files changed, 60 insertions(+), 43 deletions(-) create mode 100644 test/unit/asan.supp diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 2c6b14aa7..19a834109 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -11,6 +11,8 @@ namespace labstor { +class TaskLib; + /** This task reads a state */ #define TASK_READ BIT_OPT(u32, 0) /** This task writes to a state */ @@ -228,6 +230,21 @@ class TaskPrio { TASK_PRIO_T kLowLatency = 2; }; +/** Context passed to the Run method of a task */ +struct RunContext { + u32 lane_id_; /**< The lane id of the task */ + bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ + size_t stack_size_ = KILOBYTES(64); /**< The size of the stack for the task (runtime) */ + void *stack_ptr_; /**< The pointer to the stack (runtime) */ + TaskLib *exec_; + + /** Default constructor */ + RunContext() {} + + /** Emplace constructor */ + RunContext(u32 lane_id) : lane_id_(lane_id) {} +}; + /** A generic task base class */ struct Task : public hipc::ShmContainer { SHM_CONTAINER_TEMPLATE((Task), (Task)) @@ -239,9 +256,7 @@ struct Task : public hipc::ShmContainer { u32 lane_hash_; /**< Determine the lane a task is keyed to */ u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ - bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ - size_t stack_size_ = KILOBYTES(256); /**< The size of the stack for the task (runtime) */ - void *stack_ptr_; /**< The pointer to the stack (runtime) */ + RunContext ctx_; /**==================================== * Task Helpers @@ -369,7 +384,7 @@ struct Task : public hipc::ShmContainer { if constexpr (THREAD_MODEL == TASK_YIELD_STD) { HERMES_THREAD_MODEL->Yield(); } else if constexpr (THREAD_MODEL == TASK_YIELD_CO) { - jmp_ = bctx::jump_fcontext(jmp_.fctx, nullptr); + ctx_.jmp_ = bctx::jump_fcontext(ctx_.jmp_.fctx, nullptr); } else if constexpr (THREAD_MODEL == TASK_YIELD_ABT) { ABT_thread_yield(); } diff --git a/include/labstor/task_registry/task_lib.h b/include/labstor/task_registry/task_lib.h index 5dc4eda98..acf82efad 100644 --- a/include/labstor/task_registry/task_lib.h +++ b/include/labstor/task_registry/task_lib.h @@ -53,17 +53,6 @@ struct TaskPointer { } }; -/** Context passed to the Run method of a task */ -struct RunContext { - u32 lane_id_; /**< The lane id of the task */ - - /** Default constructor */ - RunContext() {} - - /** Emplace constructor */ - RunContext(u32 lane_id) : lane_id_(lane_id) {} -}; - /** * Represents a custom operation to perform. * Tasks are independent of Hermes. diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 1e46009a3..23555bd31 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -29,10 +29,6 @@ struct WorkEntry { LaneGroup *group_; MultiQueue *queue_; - TaskState *exec_; - Task *task_; - RunContext ctx_; - /** Default constructor */ HSHM_ALWAYS_INLINE WorkEntry() = default; diff --git a/src/worker.cc b/src/worker.cc index 00955490b..d4842c9b6 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -43,10 +43,7 @@ void Worker::Run() { void Worker::PollGrouped(WorkEntry &work_entry) { int off = 0; Lane *&lane = work_entry.lane_; - Task *&task = work_entry.task_; - TaskState *&exec = work_entry.exec_; - RunContext &ctx = work_entry.ctx_; - ctx.lane_id_ = work_entry.lane_id_; + Task *task; LaneData *entry; for (int i = 0; i < 1024; ++i) { // Get the task message @@ -58,7 +55,10 @@ void Worker::PollGrouped(WorkEntry &work_entry) { continue; } task = LABSTOR_CLIENT->GetPrivatePointer(entry->p_); + RunContext &ctx = task->ctx_; + ctx.lane_id_ = work_entry.lane_id_; // Get the task state + TaskState *&exec = ctx.exec_; exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); if (!exec) { HELOG(kFatal, "(node {}) Could not find the task state: {}", @@ -78,13 +78,17 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->SetUnordered(); } else if (task->IsCoroutine()) { if (!task->IsStarted()) { - task->stack_ptr_ = malloc(task->stack_size_); - task->jmp_.fctx = bctx::make_fcontext( - (char*)task->stack_ptr_ + task->stack_size_, - task->stack_size_, &RunBlocking); + ctx.stack_ptr_ = malloc(ctx.stack_size_); + if (ctx.stack_ptr_ == nullptr) { + HILOG(kFatal, "The stack pointer of size {} is NULL", + ctx.stack_size_, ctx.stack_ptr_); + } + ctx.jmp_.fctx = bctx::make_fcontext( + (char*)ctx.stack_ptr_ + ctx.stack_size_, + ctx.stack_size_, &RunBlocking); task->SetStarted(); } - task->jmp_ = bctx::jump_fcontext(task->jmp_.fctx, &work_entry); + ctx.jmp_ = bctx::jump_fcontext(ctx.jmp_.fctx, task); HILOG(kInfo, "Jumping into function") } else if (task->IsPreemptive()) { task->DisableRun(); @@ -100,7 +104,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { // LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); entry->complete_ = true; if (task->IsCoroutine()) { - free(task->stack_ptr_); + free(ctx.stack_ptr_); } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); } @@ -113,11 +117,10 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } void Worker::RunBlocking(bctx::transfer_t t) { - WorkEntry *work_entry = reinterpret_cast(t.data); - Task *&task = work_entry->task_; - TaskState *&exec = work_entry->exec_; - RunContext &ctx = work_entry->ctx_; - task->jmp_ = t; + Task *task = reinterpret_cast(t.data); + RunContext &ctx = task->ctx_; + TaskState *&exec = ctx.exec_; + ctx.jmp_ = t; exec->Run(task->method_, task, ctx); task->Yield(); } diff --git a/test/unit/asan.supp b/test/unit/asan.supp new file mode 100644 index 000000000..c17008e89 --- /dev/null +++ b/test/unit/asan.supp @@ -0,0 +1,10 @@ +# Ignore leaks from external libraries +leak:libfabric.so +leak:libabt.so +leak:libmargo.so +leak:libmpi.so +leak:librdmacm.so +leak:libhwloc.so +leak:libmpich.so +leak:aiori-HDF5.c +leak:ior.c \ No newline at end of file diff --git a/test/unit/boost/test_boost.cc b/test/unit/boost/test_boost.cc index 78146590c..6e53672b9 100644 --- a/test/unit/boost/test_boost.cc +++ b/test/unit/boost/test_boost.cc @@ -72,30 +72,34 @@ typedef simple_stack_allocator< int value1; namespace bctx = boost::context::detail; -void f3( bctx::transfer_t t_) { +bctx::transfer_t shared_xfer; + +void f3( bctx::transfer_t t) { ++value1; - bctx::transfer_t t = bctx::jump_fcontext( t_.fctx, 0); + shared_xfer = t; + HILOG(kInfo, "Aasfasfak;asdf {}", value1) + shared_xfer = bctx::jump_fcontext(shared_xfer.fctx, 0); ++value1; - bctx::jump_fcontext( t.fctx, t.data); + shared_xfer = bctx::jump_fcontext( shared_xfer.fctx, shared_xfer.data); } TEST_CASE("TestBoostFcontext") { value1 = 0; stack_allocator alloc; - int size = 128; + int size = KILOBYTES(64); hshm::Timer t; t.Resume(); size_t ops = (1 << 20); - void *sp = alloc.allocate(size); for (size_t i = 0; i < ops; ++i) { - bctx::fcontext_t ctx = bctx::make_fcontext(sp, size, f3); - bctx::transfer_t t = bctx::jump_fcontext(ctx, 0); - bctx::jump_fcontext(t.fctx, 0); + void *sp = alloc.allocate(size); + shared_xfer.fctx = bctx::make_fcontext(sp, size, f3); + shared_xfer = bctx::jump_fcontext(shared_xfer.fctx, 0); + shared_xfer = bctx::jump_fcontext(shared_xfer.fctx, 0); + alloc.deallocate(sp, size); } - alloc.deallocate(sp, size); t.Pause(); HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); From 41eb1a3830c13c3f944ee2822bff4927694e105f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 10:30:57 -0500 Subject: [PATCH 056/191] Irresponsibly destroy DelTask --- include/labstor/api/labstor_client.h | 7 +++++-- src/worker.cc | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index f3ef03473..54e9d9f99 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -139,7 +139,8 @@ class Client : public ConfigurationManager { template HSHM_ALWAYS_INLINE void DelTask(TaskT *task) { - main_alloc_->DelObj(task); + // TODO(llogan): verify leak + // main_alloc_->DelObj(task); } /** Destroy a task */ @@ -187,12 +188,14 @@ class Client : public ConfigurationManager { /** Free a buffer */ HSHM_ALWAYS_INLINE void FreeBuffer(hipc::Pointer &p) { - main_alloc_->Free(p); + // TODO(llogan): verify leak + // main_alloc_->Free(p); } /** Free a buffer */ HSHM_ALWAYS_INLINE void FreeBuffer(LPointer &p) { + // TODO(llogan): verify leak main_alloc_->FreeLocalPtr(p); } }; diff --git a/src/worker.cc b/src/worker.cc index d4842c9b6..1cabb5987 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -104,7 +104,8 @@ void Worker::PollGrouped(WorkEntry &work_entry) { // LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); entry->complete_ = true; if (task->IsCoroutine()) { - free(ctx.stack_ptr_); + // TODO(llogan): verify leak + // free(ctx.stack_ptr_); } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); } From 61d0d60bc988a12ff82048e3bcc8ce796f244ac5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 14:14:25 -0500 Subject: [PATCH 057/191] Make PUT and GET have async option --- tasks/hermes/include/hermes/bucket.h | 275 +++++++++++++----- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 16 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 41 ++- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 65 +++-- .../src/hermes_bucket_mdm.cc | 8 +- tasks/posix_bdev/src/posix_bdev.cc | 2 + tasks/ram_bdev/src/ram_bdev.cc | 2 + test/unit/hermes/test_bucket.cc | 7 +- 8 files changed, 298 insertions(+), 118 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 62879a53b..70438e28c 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -11,6 +11,10 @@ namespace hermes { +#include "labstor/labstor_namespace.h" +using hermes::blob_mdm::PutBlobTask; +using hermes::blob_mdm::GetBlobTask; + #define HERMES_BUCKET_IS_FILE BIT_OPT(u32, 1) class Bucket { @@ -211,6 +215,7 @@ class Bucket { const BlobId &orig_blob_id, Context &ctx) { BlobId blob_id = orig_blob_id; + bitfield32_t flags, task_flags(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY); // Copy data to shared memory LPointer p = LABSTOR_CLIENT->AllocateBuffer(blob.size()); char *data = p.ptr_; @@ -218,19 +223,25 @@ class Bucket { // Put to shared memory hshm::charbuf blob_name_buf = hshm::to_charbuf(blob_name); if (blob_id.IsNull()) { - blob_id = blob_mdm_->GetOrCreateBlobIdRoot(id_, blob_name_buf); + flags.SetBits(HERMES_GET_BLOB_ID); + task_flags.UnsetBits(TASK_FIRE_AND_FORGET); } - HILOG(kDebug, "The bucket's ID is: {}", blob_id); + LPointer> push_task; if constexpr(!PARTIAL) { - blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, - blob_id, 0, blob.size(), p.shm_, ctx.blob_score_, - bitfield32_t(HERMES_BLOB_REPLACE), - ctx); + flags.SetBits(HERMES_BLOB_REPLACE); + push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, + blob_id, 0, blob.size(), p.shm_, ctx.blob_score_, + flags, ctx, task_flags); } else { - blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, - blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, - bitfield32_t(0), - ctx); + push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, + blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, + flags, ctx, task_flags); + } + if (flags.Any(HERMES_GET_BLOB_ID)) { + push_task->Wait(); + PutBlobTask *task = push_task->get(); + blob_id = task->blob_id_; + LABSTOR_CLIENT->DelTask(push_task); } return blob_id; } @@ -238,24 +249,72 @@ class Bucket { /** * Put \a blob_name Blob into the bucket * */ + template + HSHM_ALWAYS_INLINE + BlobId SrlBasePut(const std::string &blob_name, + const T &data, + const BlobId &orig_blob_id, + Context &ctx) { + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar << data; + Blob blob(ss.str()); + return BasePut(blob_name, blob, 0, orig_blob_id, ctx); + } + + /** + * Put \a blob_name Blob into the bucket + * */ + template BlobId Put(const std::string &blob_name, - const Blob &blob, + const T &blob, Context &ctx) { - return BasePut(blob_name, blob, 0, BlobId::GetNull(), ctx); + if (std::is_same_v) { + return BasePut(blob_name, blob, 0, BlobId::GetNull(), ctx); + } else { + return SrlBasePut(blob_name, blob, BlobId::GetNull(), ctx); + } } /** * Put \a blob_id Blob into the bucket * */ + template BlobId Put(const BlobId &blob_id, - const Blob &blob, + const T &blob, Context &ctx) { - return BasePut("", blob, 0, blob_id, ctx); + if (std::is_same_v) { + return BasePut("", blob, 0, blob_id, ctx); + } else { + return SrlBasePut("", blob, blob_id, ctx); + } } /** * Put \a blob_name Blob into the bucket * */ + template + HSHM_ALWAYS_INLINE + void AsyncPut(const std::string &blob_name, + const T &blob, + Context &ctx) { + Put(blob_name, blob, ctx); + } + + /** + * Put \a blob_id Blob into the bucket + * */ + template + HSHM_ALWAYS_INLINE + void AsyncPut(const BlobId &blob_id, + const T &blob, + Context &ctx) { + Put(blob_id, blob, ctx); + } + + /** + * PartialPut \a blob_name Blob into the bucket + * */ BlobId PartialPut(const std::string &blob_name, const Blob &blob, size_t blob_off, @@ -264,7 +323,7 @@ class Bucket { } /** - * Put \a blob_id Blob into the bucket + * PartialPut \a blob_id Blob into the bucket * */ BlobId PartialPut(const BlobId &blob_id, const Blob &blob, @@ -274,28 +333,33 @@ class Bucket { } /** - * Serialized PUT + * AsyncPartialPut \a blob_name Blob into the bucket * */ - template - BlobId Put(const std::string &blob_name, - const T &data, - Context &ctx) { - std::stringstream ss; - cereal::BinaryOutputArchive ar(ss); - ar << data; - Blob blob(ss.str()); - return Put(blob_name, blob, ctx); + void AsyncPartialPut(const std::string &blob_name, + const Blob &blob, + size_t blob_off, + Context &ctx) { + BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); } /** - * Append \a blob_name Blob into the bucket + * AsyncPartialPut \a blob_id Blob into the bucket * */ - Status Append(const Blob &blob, size_t page_size, Context &ctx) { + void AsyncPartialPut(const BlobId &blob_id, + const Blob &blob, + size_t blob_off, + Context &ctx) { + BasePut("", blob, blob_off, blob_id, ctx); + } + + /** + * Append \a blob_name Blob into the bucket (fully asynchronous) + * */ + void Append(const Blob &blob, size_t page_size, Context &ctx) { LPointer p = LABSTOR_CLIENT->AllocateBuffer(blob.size()); char *data = p.ptr_; memcpy(data, blob.data(), blob.size()); bkt_mdm_->AppendBlobRoot(id_, blob.size(), p.shm_, page_size, ctx.blob_score_, ctx.node_id_, ctx); - return Status(); } /** @@ -316,51 +380,119 @@ class Bucket { } /** - * Get \a blob_id Blob from the bucket + * Get \a blob_id Blob from the bucket (async) * */ - BlobId BaseGet(const std::string &blob_name, - const BlobId &orig_blob_id, - Blob &blob, - size_t blob_off, - Context &ctx) { - BlobId blob_id = orig_blob_id; + LPointer> + HSHM_ALWAYS_INLINE + AsyncBaseGet(const std::string &blob_name, + const BlobId &blob_id, + Blob &blob, + size_t blob_off, + Context &ctx) { + bitfield32_t flags; // Get the blob ID if (blob_id.IsNull()) { - blob_id = blob_mdm_->GetBlobIdRoot(id_, hshm::to_charbuf(blob_name)); - } - if (blob_id.IsNull()) { - if (ctx.filename_.size() == 0) { - return blob_id; - } else { - // StageIn using PUT of an empty blob - hermes::Blob emtpy_blob; - blob_id = PartialPut(blob_name, emtpy_blob, 0, ctx); - } + flags.SetBits(HERMES_GET_BLOB_ID); } // Get from shared memory size_t data_size = blob.size(); - if (data_size == 0) { - data_size = GetBlobSize(blob_id); - } - HILOG(kInfo, "Data size: {}", data_size) LPointer data_p = LABSTOR_CLIENT->AllocateBuffer(data_size); - data_size = blob_mdm_->GetBlobRoot(id_, blob_id, blob_off, data_size, data_p.shm_, ctx); - char *data = data_p.ptr_; - // Copy data to blob + LPointer> push_task; + push_task = blob_mdm_->AsyncGetBlobRoot(id_, blob_name, blob_id, blob_off, + data_size, data_p.shm_, + ctx, flags); + return push_task; + } + + /** + * Get \a blob_id Blob from the bucket (sync) + * */ + BlobId BaseGet(const std::string &blob_name, + const BlobId &orig_blob_id, + Blob &blob, + size_t blob_off, + Context &ctx) { // TODO(llogan): intercept mmap to avoid copy - blob.resize(data_size); + // TODO(llogan): make GetBlobSize work with blob_name + size_t data_size = blob.size(); + if (blob.size() == 0) { + data_size = blob_mdm_->GetBlobSizeRoot(id_, orig_blob_id); + blob.resize(data_size); + } + BlobId blob_id; + LPointer> push_task; + push_task = AsyncBaseGet(blob_name, orig_blob_id, blob, blob_off, ctx); + push_task->Wait(); + GetBlobTask *task = push_task->get(); + blob_id = task->blob_id_; + char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); memcpy(blob.data(), data, data_size); - LABSTOR_CLIENT->FreeBuffer(data_p); + LABSTOR_CLIENT->FreeBuffer(task->data_); + LABSTOR_CLIENT->DelTask(task); return blob_id; } + /** + * Get \a blob_id Blob from the bucket (sync) + * */ + template + BlobId SrlBaseGet(const std::string &blob_name, + const BlobId &orig_blob_id, + T &data, + Context &ctx) { + std::stringstream ss; + cereal::BinaryInputArchive ar(ss); + ar >> data; + Blob blob(ss.str()); + return BaseGet(blob_name, orig_blob_id, blob, 0, ctx); + } + /** * Get \a blob_id Blob from the bucket * */ + template BlobId Get(const std::string &blob_name, - Blob &blob, + T &blob, Context &ctx) { - return BaseGet(blob_name, BlobId::GetNull(), blob, 0, ctx); + if (std::is_same_v) { + return BaseGet(blob_name, BlobId::GetNull(), blob, 0, ctx); + } else { + return SrlBaseGet(blob_name, BlobId::GetNull(), blob, ctx); + } + } + + /** + * Get \a blob_id Blob from the bucket + * */ + template + BlobId Get(const BlobId &blob_id, + T &blob, + Context &ctx) { + if (std::is_same_v) { + return BaseGet("", blob_id, blob, 0, ctx); + } else { + return SrlBaseGet("", blob_id, blob, ctx); + } + } + + /** + * AsyncGet \a blob_name Blob from the bucket + * */ + LPointer> + AsyncGet(const std::string &blob_name, + Blob &blob, + Context &ctx) { + return AsyncBaseGet(blob_name, BlobId::GetNull(), blob, 0, ctx); + } + + /** + * AsyncGet \a blob_id Blob from the bucket + * */ + LPointer> + AsyncGet(const BlobId &blob_id, + Blob &blob, + Context &ctx) { + return AsyncBaseGet("", blob_id, blob, 0, ctx); } /** @@ -373,15 +505,6 @@ class Bucket { return BaseGet(blob_name, BlobId::GetNull(), blob, blob_off, ctx); } - /** - * Get \a blob_id Blob from the bucket - * */ - BlobId Get(const BlobId &blob_id, - Blob &blob, - Context &ctx) { - return BaseGet("", blob_id, blob, 0, ctx); - } - /** * Put \a blob_name Blob into the bucket * */ @@ -392,6 +515,28 @@ class Bucket { return BaseGet("", blob_id, blob, blob_off, ctx); } + /** + * AsyncGet \a blob_name Blob from the bucket + * */ + LPointer> + AsyncPartialGet(const std::string &blob_name, + Blob &blob, + size_t blob_off, + Context &ctx) { + return AsyncBaseGet(blob_name, BlobId::GetNull(), blob, blob_off, ctx); + } + + /** + * AsyncGet \a blob_id Blob from the bucket + * */ + LPointer> + AsyncPartialGet(const BlobId &blob_id, + Blob &blob, + size_t blob_off, + Context &ctx) { + return AsyncBaseGet("", blob_id, blob, blob_off, ctx); + } + /** * Determine if the bucket contains \a blob_id BLOB * */ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index abb5be9cf..16c8c3f70 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -127,14 +127,13 @@ class Client : public TaskLibClient { const hipc::Pointer &blob, float score, bitfield32_t flags, Context ctx = Context(), - bitfield32_t task_flags = bitfield32_t(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER)) { + bitfield32_t task_flags = bitfield32_t(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY)) { HILOG(kDebug, "Beginning PUT (task_node={})", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id, blob_off, blob_size, - blob, score, flags, ctx); - task->task_flags_ = task_flags; + blob, score, flags, ctx, task_flags); HILOG(kDebug, "Constructed PUT (task_node={})", task_node); } LABSTOR_TASK_NODE_PUSH_ROOT(PutBlob); @@ -143,24 +142,27 @@ class Client : public TaskLibClient { void AsyncGetBlobConstruct(GetBlobTask *task, const TaskNode &task_node, const TagId &tag_id, + const std::string &blob_name, const BlobId &blob_id, size_t off, ssize_t data_size, hipc::Pointer &data, - Context ctx = Context()) { + Context ctx = Context(), + bitfield32_t flags = bitfield32_t(0)) { HILOG(kDebug, "Beginning GET (task_node={})", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, - tag_id, blob_id, off, data_size, data, ctx); + tag_id, blob_name, blob_id, off, data_size, data, ctx, flags); } size_t GetBlobRoot(const TagId &tag_id, const BlobId &blob_id, size_t off, ssize_t data_size, hipc::Pointer &data, - Context ctx = Context()) { + Context ctx = Context(), + bitfield32_t flags = bitfield32_t(0)) { LPointer> push_task = - AsyncGetBlobRoot(tag_id, blob_id, off, data_size, data); + AsyncGetBlobRoot(tag_id, "", blob_id, off, data_size, data, ctx, flags); push_task->Wait(); GetBlobTask *task = push_task->get(); data = task->data_; diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 0fa2e80c1..1f9213847 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -209,7 +209,7 @@ class PutBlobPhase { #define HERMES_DID_STAGE_IN BIT_OPT(u32, 2) #define HERMES_IS_FILE BIT_OPT(u32, 3) #define HERMES_BLOB_DID_CREATE BIT_OPT(u32, 4) - +#define HERMES_GET_BLOB_ID BIT_OPT(u32, 5) /** A task to put data in a blob */ struct PutBlobTask : public Task, TaskFlags { @@ -251,15 +251,20 @@ struct PutBlobTask : public Task, TaskFlags const hipc::Pointer &data, float score, bitfield32_t flags, - const Context &ctx) : Task(alloc) { + const Context &ctx, + bitfield32_t task_flags) : Task(alloc) { // Initialize task HILOG(kDebug, "Beginning PUT task constructor") task_node_ = task_node; - lane_hash_ = blob_id.hash_; + if (!blob_id.IsNull()) { + lane_hash_ = blob_id.hash_; + } else { + lane_hash_ = std::hash{}(blob_name); + } prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPutBlob; - task_flags_.SetBits(TASK_LOW_LATENCY); + task_flags_ = task_flags; domain_id_ = domain_id; // Custom params @@ -309,7 +314,11 @@ struct PutBlobTask : public Task, TaskFlags /** (De)serialize message return */ template - void SerializeEnd(u32 replica, Ar &ar) {} + void SerializeEnd(u32 replica, Ar &ar) { + if (flags_.Any(HERMES_GET_BLOB_ID)) { + ar(blob_id_); + } + } /** Create group */ HSHM_ALWAYS_INLINE @@ -331,12 +340,14 @@ class GetBlobPhase { /** A task to get data from a blob */ struct GetBlobTask : public Task, TaskFlags { IN TagId tag_id_; - IN BlobId blob_id_; + IN hipc::ShmArchive blob_name_; + INOUT BlobId blob_id_; IN size_t blob_off_; IN hipc::Pointer data_; IN hipc::ShmArchive filename_; IN size_t page_size_; INOUT ssize_t data_size_; + IN bitfield32_t flags_; TEMP int phase_ = GetBlobPhase::kStart; TEMP hipc::ShmArchive> bdev_reads_; TEMP PutBlobTask *stage_task_ = nullptr; @@ -352,11 +363,13 @@ struct GetBlobTask : public Task, TaskFlags const DomainId &domain_id, const TaskStateId &state_id, const TagId &tag_id, + const std::string &blob_name, const BlobId &blob_id, size_t off, ssize_t data_size, hipc::Pointer &data, - const Context &ctx) : Task(alloc) { + const Context &ctx, + bitfield32_t flags) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = blob_id.hash_; @@ -372,25 +385,27 @@ struct GetBlobTask : public Task, TaskFlags blob_off_ = off; data_size_ = data_size; data_ = data; + flags_ = flags; + HSHM_MAKE_AR(blob_name_, alloc, blob_name); HSHM_MAKE_AR(filename_, alloc, ctx.filename_); page_size_ = ctx.page_size_; } /** Destructor */ ~GetBlobTask() { + HSHM_DESTROY_AR(blob_name_); HSHM_DESTROY_AR(filename_); } /** (De)serialize message call */ template void SaveStart(Ar &ar) { - // TODO(llogan): Make it so Get takes as input a buffer, instead of returning one DataTransfer xfer(DT_RECEIVER_WRITE, HERMES_MEMORY_MANAGER->Convert(data_), data_size_, domain_id_); task_serialize(ar); ar & xfer; - ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_); + ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_, flags_); } /** Deserialize message call */ @@ -400,12 +415,16 @@ struct GetBlobTask : public Task, TaskFlags task_serialize(ar); ar & xfer; data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); - ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_); + ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_, flags_); } /** (De)serialize message return */ template - void SerializeEnd(u32 replica, Ar &ar) {} + void SerializeEnd(u32 replica, Ar &ar) { + if (flags_.Any(HERMES_GET_BLOB_ID)) { + ar(blob_id_); + } + } /** Create group */ HSHM_ALWAYS_INLINE diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 4ee237633..b324db3b3 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -142,16 +142,14 @@ class Server : public TaskLib { /** Create blob / update metadata for the PUT */ void PutBlobCreatePhase(PutBlobTask *task, RunContext &ctx) { HILOG(kDebug, "PutBlobPhase::kCreate {}", task->blob_id_); + // Get the blob info data structure hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; - auto it = blob_map.find(task->blob_id_); - if (it == blob_map.end()) { - task->flags_.SetBits(HERMES_BLOB_DID_CREATE); - } - if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { - blob_map.emplace(task->blob_id_, BlobInfo()); + if (task->flags_.Any(HERMES_GET_BLOB_ID)) { + task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, + blob_name, ctx, task->flags_); } + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; // Update the blob info @@ -333,13 +331,13 @@ class Server : public TaskLib { /** Wait for the update to complete */ void PutBlobWaitModifyPhase(PutBlobTask *task, RunContext &ctx) { std::vector &write_tasks = *task->bdev_writes_; - for (int i = (int)write_tasks.size() - 1; i >= 0; --i) { - bdev::WriteTask *write_task = write_tasks[i]; + for (bdev::WriteTask *&write_task : write_tasks) { if (!write_task->IsComplete()) { return; } + } + for (bdev::WriteTask *&write_task : write_tasks) { LABSTOR_CLIENT->DelTask(write_task); - write_tasks.pop_back(); } HILOG(kDebug, "PutBlobTask complete"); HSHM_DESTROY_AR(task->schema_); @@ -377,14 +375,16 @@ class Server : public TaskLib { } void GetBlobGetPhase(GetBlobTask *task, RunContext &ctx) { + if (task->flags_.Any(HERMES_GET_BLOB_ID)) { + hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); + task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, + blob_name, ctx, task->flags_); + } BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; HSHM_MAKE_AR0(task->bdev_reads_, nullptr); std::vector &read_tasks = *task->bdev_reads_; read_tasks.reserve(blob_info.buffers_.size()); - if (task->data_size_ < 0) { - task->data_size_ = (ssize_t)(blob_info.blob_size_ - task->blob_off_); - } HILOG(kDebug, "Getting blob {} of size {} starting at offset {} (total_blob_size={}, buffers={})", task->blob_id_, task->data_size_, task->blob_off_, blob_info.blob_size_, blob_info.buffers_.size()); size_t blob_off = 0, buf_off = 0; @@ -415,13 +415,13 @@ class Server : public TaskLib { void GetBlobWaitPhase(GetBlobTask *task, RunContext &ctx) { std::vector &read_tasks = *task->bdev_reads_; - for (auto it = read_tasks.rbegin(); it != read_tasks.rend(); ++it) { - bdev::ReadTask *read_task = *it; + for (bdev::ReadTask *&read_task : read_tasks) { if (!read_task->IsComplete()) { return; } + } + for (bdev::ReadTask *&read_task : read_tasks) { LABSTOR_CLIENT->DelTask(read_task); - read_tasks.pop_back(); } HSHM_DESTROY_AR(task->bdev_reads_); HILOG(kDebug, "GetBlobTask complete"); @@ -463,18 +463,26 @@ class Server : public TaskLib { /** * Create \a blob_id BLOB ID * */ - void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &ctx) { - hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); + BlobId GetOrCreateBlobId(TagId &tag_id, u32 lane_hash, + const hshm::charbuf &blob_name, RunContext &ctx, + bitfield32_t &flags) { + hshm::charbuf blob_name_unique = GetBlobNameWithBucket(tag_id, blob_name); BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; auto it = blob_id_map.find(blob_name_unique); if (it == blob_id_map.end()) { - task->blob_id_ = BlobId(node_id_, task->lane_hash_, id_alloc_.fetch_add(1)); - blob_id_map.emplace(blob_name_unique, task->blob_id_); - task->SetModuleComplete(); - return; - } - task->blob_id_ = it->second; + BlobId blob_id = BlobId(node_id_, lane_hash, id_alloc_.fetch_add(1)); + blob_id_map.emplace(blob_name_unique, blob_id); + flags.SetBits(HERMES_BLOB_DID_CREATE); + BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + blob_map.emplace(blob_id, BlobInfo()); + return blob_id; + } + return it->second; + } + void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &ctx) { + hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); + bitfield32_t flags; + task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, blob_name, ctx, flags); task->SetModuleComplete(); } @@ -622,13 +630,13 @@ class Server : public TaskLib { } case DestroyBlobPhase::kWaitFreeBuffers: { std::vector &free_tasks = *task->free_tasks_; - for (auto it = free_tasks.rbegin(); it != free_tasks.rend(); ++it) { - bdev::FreeTask *free_task = *it; + for (bdev::FreeTask *&free_task : free_tasks) { if (!free_task->IsComplete()) { return; } + } + for (bdev::FreeTask *&free_task : free_tasks) { LABSTOR_CLIENT->DelTask(free_task); - free_tasks.pop_back(); } BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; @@ -660,6 +668,7 @@ class Server : public TaskLib { task->data_size_ = blob_info.blob_size_; task->get_task_ = blob_mdm_.AsyncGetBlob(task->task_node_ + 1, task->tag_id_, + "", task->blob_id_, 0, task->data_size_, diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index a39dc86a9..5d1a96fc0 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -288,16 +288,16 @@ class Server : public TaskLib { } case DestroyTagPhase::kWaitDestroyBlobs: { std::vector blob_tasks = *task->destroy_blob_tasks_; - for (auto it = blob_tasks.rbegin(); it != blob_tasks.rend(); ++it) { - blob_mdm::DestroyBlobTask *blob_task = *it; + for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { if (!blob_task->IsComplete()) { return; } + } + for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { LABSTOR_CLIENT->DelTask(blob_task); - blob_tasks.pop_back(); } - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; HSHM_DESTROY_AR(task->destroy_blob_tasks_); + TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; tag_map.erase(task->tag_id_); task->SetModuleComplete(); } diff --git a/tasks/posix_bdev/src/posix_bdev.cc b/tasks/posix_bdev/src/posix_bdev.cc index 71cce0b2e..038debf86 100644 --- a/tasks/posix_bdev/src/posix_bdev.cc +++ b/tasks/posix_bdev/src/posix_bdev.cc @@ -55,6 +55,7 @@ class Server : public TaskLib { } void Write(WriteTask *task, RunContext &ctx) { + HILOG(kDebug, "Writing {} bytes to {}", task->size_, path_); ssize_t count = pwrite(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { HELOG(kError, "BORG: wrote {} bytes, but expected {}: {}", @@ -64,6 +65,7 @@ class Server : public TaskLib { } void Read(ReadTask *task, RunContext &ctx) { + HILOG(kDebug, "Reading {} bytes from {}", task->size_, path_); ssize_t count = pread(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { HELOG(kError, "BORG: read {} bytes, but expected {}", diff --git a/tasks/ram_bdev/src/ram_bdev.cc b/tasks/ram_bdev/src/ram_bdev.cc index d5f2a042a..ec1ae7107 100644 --- a/tasks/ram_bdev/src/ram_bdev.cc +++ b/tasks/ram_bdev/src/ram_bdev.cc @@ -42,11 +42,13 @@ class Server : public TaskLib { } void Write(WriteTask *task, RunContext &ctx) { + HILOG(kDebug, "Writing {} bytes to RAM", task->size_); memcpy(mem_ptr_ + task->disk_off_, task->buf_, task->size_); task->SetModuleComplete(); } void Read(ReadTask *task, RunContext &ctx) { + HILOG(kDebug, "Reading {} bytes from RAM", task->size_); memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); task->SetModuleComplete(); } diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 164031069..a074e884d 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -109,12 +109,13 @@ TEST_CASE("TestHermesPutGet") { memset(blob.data(), i % 256, blob.size()); hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + } + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {} with blob name {}", i, std::to_string(i)); // Get a blob hermes::Blob blob2; - bkt.Get(blob_id, blob2, ctx); - REQUIRE(blob.size() == blob2.size()); - REQUIRE(blob == blob2); + bkt.Get(std::to_string(i), blob2, ctx); } } } From 49a3e2356594dd0ad4b4b9d8501e4fe5376f186a Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 14:49:05 -0500 Subject: [PATCH 058/191] Use kDebug for jumping into function --- src/worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.cc b/src/worker.cc index 1cabb5987..7a943839a 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -89,7 +89,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->SetStarted(); } ctx.jmp_ = bctx::jump_fcontext(ctx.jmp_.fctx, task); - HILOG(kInfo, "Jumping into function") + HILOG(kDebug, "Jumping into function") } else if (task->IsPreemptive()) { task->DisableRun(); entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunPreemptive, task); From 3d263192500b4a5eb9d652ba0ace73c8e5599cd5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 14:51:08 -0500 Subject: [PATCH 059/191] Add flags_ to putblob --- .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 1f9213847..8cbc0be86 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -299,7 +299,7 @@ struct PutBlobTask : public Task, TaskFlags data_size_, domain_id_); task_serialize(ar); ar & xfer; - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_); + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_, flags_); } /** Deserialize message call */ @@ -309,7 +309,7 @@ struct PutBlobTask : public Task, TaskFlags task_serialize(ar); ar & xfer; data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_); + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_, flags_); } /** (De)serialize message return */ From c22b4abdde2ec51a7230e48d4a77b26273468c6e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 14:54:20 -0500 Subject: [PATCH 060/191] Lower debug logging --- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 2 -- .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 9 ++++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 16c8c3f70..9d82c52a6 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -128,13 +128,11 @@ class Client : public TaskLibClient { bitfield32_t flags, Context ctx = Context(), bitfield32_t task_flags = bitfield32_t(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY)) { - HILOG(kDebug, "Beginning PUT (task_node={})", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id, blob_off, blob_size, blob, score, flags, ctx, task_flags); - HILOG(kDebug, "Constructed PUT (task_node={})", task_node); } LABSTOR_TASK_NODE_PUSH_ROOT(PutBlob); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 8cbc0be86..6e1e11717 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -254,7 +254,6 @@ struct PutBlobTask : public Task, TaskFlags const Context &ctx, bitfield32_t task_flags) : Task(alloc) { // Initialize task - HILOG(kDebug, "Beginning PUT task constructor") task_node_ = task_node; if (!blob_id.IsNull()) { lane_hash_ = blob_id.hash_; @@ -269,7 +268,6 @@ struct PutBlobTask : public Task, TaskFlags // Custom params tag_id_ = tag_id; - HILOG(kDebug, "Setting blob name {}", blob_name.str()); HSHM_MAKE_AR(blob_name_, alloc, blob_name); blob_id_ = blob_id; blob_off_ = blob_off; @@ -279,7 +277,8 @@ struct PutBlobTask : public Task, TaskFlags flags_ = flags; HSHM_MAKE_AR(filename_, alloc, ctx.filename_); page_size_ = ctx.page_size_; - HILOG(kDebug, "Finished setting blob name {}", blob_name.str()); + HILOG(kDebug, "Construct PUT task for {}, while getting BlobId is {}", + blob_name.str(), flags_.Any(HERMES_GET_BLOB_ID)); } /** Destructor */ @@ -299,7 +298,7 @@ struct PutBlobTask : public Task, TaskFlags data_size_, domain_id_); task_serialize(ar); ar & xfer; - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_, flags_); + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_); } /** Deserialize message call */ @@ -309,7 +308,7 @@ struct PutBlobTask : public Task, TaskFlags task_serialize(ar); ar & xfer; data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); - ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_, flags_); + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, score_, flags_, filename_, page_size_); } /** (De)serialize message return */ From 8a1b8f0e6d7eb1c583e30aa02a11345ee19bfa33 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 14:58:55 -0500 Subject: [PATCH 061/191] Support ASYNC --- tasks/hermes/include/hermes/bucket.h | 34 +++++++++++++++------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 70438e28c..9e23c1fd9 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -207,7 +207,7 @@ class Bucket { /** * Put \a blob_name Blob into the bucket * */ - template + template HSHM_ALWAYS_INLINE BlobId BasePut(const std::string &blob_name, const Blob &blob, @@ -222,9 +222,11 @@ class Bucket { memcpy(data, blob.data(), blob.size()); // Put to shared memory hshm::charbuf blob_name_buf = hshm::to_charbuf(blob_name); - if (blob_id.IsNull()) { - flags.SetBits(HERMES_GET_BLOB_ID); - task_flags.UnsetBits(TASK_FIRE_AND_FORGET); + if constexpr (!ASYNC) { + if (blob_id.IsNull()) { + flags.SetBits(HERMES_GET_BLOB_ID); + task_flags.UnsetBits(TASK_FIRE_AND_FORGET); + } } LPointer> push_task; if constexpr(!PARTIAL) { @@ -249,7 +251,7 @@ class Bucket { /** * Put \a blob_name Blob into the bucket * */ - template + template HSHM_ALWAYS_INLINE BlobId SrlBasePut(const std::string &blob_name, const T &data, @@ -259,7 +261,7 @@ class Bucket { cereal::BinaryOutputArchive ar(ss); ar << data; Blob blob(ss.str()); - return BasePut(blob_name, blob, 0, orig_blob_id, ctx); + return BasePut(blob_name, blob, 0, orig_blob_id, ctx); } /** @@ -270,9 +272,9 @@ class Bucket { const T &blob, Context &ctx) { if (std::is_same_v) { - return BasePut(blob_name, blob, 0, BlobId::GetNull(), ctx); + return BasePut(blob_name, blob, 0, BlobId::GetNull(), ctx); } else { - return SrlBasePut(blob_name, blob, BlobId::GetNull(), ctx); + return SrlBasePut(blob_name, blob, BlobId::GetNull(), ctx); } } @@ -284,9 +286,9 @@ class Bucket { const T &blob, Context &ctx) { if (std::is_same_v) { - return BasePut("", blob, 0, blob_id, ctx); + return BasePut("", blob, 0, blob_id, ctx); } else { - return SrlBasePut("", blob, blob_id, ctx); + return SrlBasePut("", blob, blob_id, ctx); } } @@ -298,7 +300,7 @@ class Bucket { void AsyncPut(const std::string &blob_name, const T &blob, Context &ctx) { - Put(blob_name, blob, ctx); + Put(blob_name, blob, ctx); } /** @@ -309,7 +311,7 @@ class Bucket { void AsyncPut(const BlobId &blob_id, const T &blob, Context &ctx) { - Put(blob_id, blob, ctx); + Put(blob_id, blob, ctx); } /** @@ -319,7 +321,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - return BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); + return BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); } /** @@ -329,7 +331,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - return BasePut("", blob, blob_off, blob_id, ctx); + return BasePut("", blob, blob_off, blob_id, ctx); } /** @@ -339,7 +341,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); + BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); } /** @@ -349,7 +351,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - BasePut("", blob, blob_off, blob_id, ctx); + BasePut("", blob, blob_off, blob_id, ctx); } /** From c47e91b36c24f389a359cfd973de5ee69d031208 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 15:05:04 -0500 Subject: [PATCH 062/191] Re-merge putget test --- tasks/hermes/include/hermes/bucket.h | 12 ++++-------- test/unit/hermes/test_bucket.cc | 6 +----- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 9e23c1fd9..3c796c2a4 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -228,17 +228,13 @@ class Bucket { task_flags.UnsetBits(TASK_FIRE_AND_FORGET); } } - LPointer> push_task; if constexpr(!PARTIAL) { flags.SetBits(HERMES_BLOB_REPLACE); - push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, - blob_id, 0, blob.size(), p.shm_, ctx.blob_score_, - flags, ctx, task_flags); - } else { - push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, - blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, - flags, ctx, task_flags); } + LPointer> push_task; + push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, + blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, + flags, ctx, task_flags); if (flags.Any(HERMES_GET_BLOB_ID)) { push_task->Wait(); PutBlobTask *task = push_task->get(); diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index a074e884d..2ce662919 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -109,13 +109,9 @@ TEST_CASE("TestHermesPutGet") { memset(blob.data(), i % 256, blob.size()); hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); - } - - for (size_t i = off; i < proc_count; ++i) { - HILOG(kInfo, "Iteration: {} with blob name {}", i, std::to_string(i)); // Get a blob hermes::Blob blob2; - bkt.Get(std::to_string(i), blob2, ctx); + bkt.Get(blob_id, blob2, ctx); } } } From 8edff15dfe114d302ecae85ce164f79448d54bbb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 15:05:20 -0500 Subject: [PATCH 063/191] Re-merge putget test --- test/unit/hermes/test_bucket.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 2ce662919..b084edb29 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -112,6 +112,8 @@ TEST_CASE("TestHermesPutGet") { // Get a blob hermes::Blob blob2; bkt.Get(blob_id, blob2, ctx); + REQUIRE(blob.size() == blob2.size()); + REQUIRE(blob == blob2); } } } From 289dd564d22387943a3f226038acb5cbf8e818b5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 15:31:39 -0500 Subject: [PATCH 064/191] Make it so task can be hashed using ID or name --- tasks/hermes/include/hermes/bucket.h | 4 +++- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 4 ++-- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 14 ++++++++++---- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 3c796c2a4..908fb463a 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -396,7 +396,8 @@ class Bucket { size_t data_size = blob.size(); LPointer data_p = LABSTOR_CLIENT->AllocateBuffer(data_size); LPointer> push_task; - push_task = blob_mdm_->AsyncGetBlobRoot(id_, blob_name, blob_id, blob_off, + push_task = blob_mdm_->AsyncGetBlobRoot(id_, hshm::to_charbuf(blob_name), + blob_id, blob_off, data_size, data_p.shm_, ctx, flags); return push_task; @@ -417,6 +418,7 @@ class Bucket { data_size = blob_mdm_->GetBlobSizeRoot(id_, orig_blob_id); blob.resize(data_size); } + HILOG(kInfo, "Getting blob of size {}", data_size); BlobId blob_id; LPointer> push_task; push_task = AsyncBaseGet(blob_name, orig_blob_id, blob, blob_off, ctx); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 9d82c52a6..02cdd90f6 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -140,7 +140,7 @@ class Client : public TaskLibClient { void AsyncGetBlobConstruct(GetBlobTask *task, const TaskNode &task_node, const TagId &tag_id, - const std::string &blob_name, + const hshm::charbuf &blob_name, const BlobId &blob_id, size_t off, ssize_t data_size, @@ -160,7 +160,7 @@ class Client : public TaskLibClient { Context ctx = Context(), bitfield32_t flags = bitfield32_t(0)) { LPointer> push_task = - AsyncGetBlobRoot(tag_id, "", blob_id, off, data_size, data, ctx, flags); + AsyncGetBlobRoot(tag_id, hshm::charbuf(""), blob_id, off, data_size, data, ctx, flags); push_task->Wait(); GetBlobTask *task = push_task->get(); data = task->data_; diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 6e1e11717..b043127a1 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -257,14 +257,15 @@ struct PutBlobTask : public Task, TaskFlags task_node_ = task_node; if (!blob_id.IsNull()) { lane_hash_ = blob_id.hash_; + domain_id_ = domain_id; } else { lane_hash_ = std::hash{}(blob_name); + domain_id_ = DomainId::GetNode(HASH_TO_NODE_ID(lane_hash_)); } prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPutBlob; task_flags_ = task_flags; - domain_id_ = domain_id; // Custom params tag_id_ = tag_id; @@ -362,7 +363,7 @@ struct GetBlobTask : public Task, TaskFlags const DomainId &domain_id, const TaskStateId &state_id, const TagId &tag_id, - const std::string &blob_name, + const hshm::charbuf &blob_name, const BlobId &blob_id, size_t off, ssize_t data_size, @@ -371,12 +372,17 @@ struct GetBlobTask : public Task, TaskFlags bitfield32_t flags) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetBlob; task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; + if (!blob_id.IsNull()) { + lane_hash_ = blob_id.hash_; + domain_id_ = domain_id; + } else { + lane_hash_ = std::hash{}(blob_name); + domain_id_ = DomainId::GetNode(HASH_TO_NODE_ID(lane_hash_)); + } // Custom params tag_id_ = tag_id; diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index b324db3b3..0f2d5c604 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -668,7 +668,7 @@ class Server : public TaskLib { task->data_size_ = blob_info.blob_size_; task->get_task_ = blob_mdm_.AsyncGetBlob(task->task_node_ + 1, task->tag_id_, - "", + hshm::charbuf(""), task->blob_id_, 0, task->data_size_, From aff44a6072c2bfadba000fcc92c70155debaefc9 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 15:53:12 -0500 Subject: [PATCH 065/191] Remove some unneeded info logs --- include/labstor/network/rpc_thallium.h | 4 ++-- tasks/hermes/include/hermes/bucket.h | 2 +- test/unit/boost/test_boost.cc | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/labstor/network/rpc_thallium.h b/include/labstor/network/rpc_thallium.h index bc504fede..667ddf278 100644 --- a/include/labstor/network/rpc_thallium.h +++ b/include/labstor/network/rpc_thallium.h @@ -235,7 +235,7 @@ class ThalliumRpc { // The "local_bulk" object will only be read from flag = tl::bulk_mode::read_only; // flag = tl::bulk_mode::read_write; - HILOG(kInfo, "(node {}) Reading {} bytes from the server", + HILOG(kDebug, "(node {}) Reading {} bytes from the server", rpc_->node_id_, size) break; } @@ -243,7 +243,7 @@ class ThalliumRpc { // The "local_bulk" object will only be written to flag = tl::bulk_mode::write_only; // flag = tl::bulk_mode::read_write; - HILOG(kInfo, "(node {}) Writing {} bytes to the server", + HILOG(kDebug, "(node {}) Writing {} bytes to the server", rpc_->node_id_, size) break; } diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 908fb463a..e799f0660 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -418,7 +418,7 @@ class Bucket { data_size = blob_mdm_->GetBlobSizeRoot(id_, orig_blob_id); blob.resize(data_size); } - HILOG(kInfo, "Getting blob of size {}", data_size); + HILOG(kDebug, "Getting blob of size {}", data_size); BlobId blob_id; LPointer> push_task; push_task = AsyncBaseGet(blob_name, orig_blob_id, blob, blob_off, ctx); diff --git a/test/unit/boost/test_boost.cc b/test/unit/boost/test_boost.cc index 6e53672b9..84c517341 100644 --- a/test/unit/boost/test_boost.cc +++ b/test/unit/boost/test_boost.cc @@ -77,7 +77,6 @@ bctx::transfer_t shared_xfer; void f3( bctx::transfer_t t) { ++value1; shared_xfer = t; - HILOG(kInfo, "Aasfasfak;asdf {}", value1) shared_xfer = bctx::jump_fcontext(shared_xfer.fctx, 0); ++value1; shared_xfer = bctx::jump_fcontext( shared_xfer.fctx, shared_xfer.data); From fc128f3ed099f0f7a0b3a5ffdd137780f7d759a4 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 16:11:49 -0500 Subject: [PATCH 066/191] Allow GetBlobSize to be done using the blob name directly --- tasks/hermes/include/hermes/bucket.h | 11 ++++++-- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 6 +++-- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 25 ++++++++++++++----- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 6 +++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index e799f0660..1a97f7bd5 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -374,7 +374,14 @@ class Bucket { * Get the current size of the blob in the bucket * */ size_t GetBlobSize(const BlobId &blob_id) { - return blob_mdm_->GetBlobSizeRoot(id_, blob_id); + return blob_mdm_->GetBlobSizeRoot(id_, hshm::charbuf(""), blob_id); + } + + /** + * Get the current size of the blob in the bucket + * */ + size_t GetBlobSize(const std::string &name) { + return blob_mdm_->GetBlobSizeRoot(id_, hshm::charbuf(name), BlobId::GetNull()); } /** @@ -415,7 +422,7 @@ class Bucket { // TODO(llogan): make GetBlobSize work with blob_name size_t data_size = blob.size(); if (blob.size() == 0) { - data_size = blob_mdm_->GetBlobSizeRoot(id_, orig_blob_id); + data_size = blob_mdm_->GetBlobSizeRoot(id_, hshm::charbuf(blob_name), orig_blob_id); blob.resize(data_size); } HILOG(kDebug, "Getting blob of size {}", data_size); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 02cdd90f6..9b82d620f 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -293,16 +293,18 @@ class Client : public TaskLibClient { void AsyncGetBlobSizeConstruct(GetBlobSizeTask *task, const TaskNode &task_node, const TagId &tag_id, + const hshm::charbuf &blob_name, const BlobId &blob_id) { HILOG(kDebug, "Getting blob size {}", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, - tag_id, blob_id); + tag_id, blob_name, blob_id); } size_t GetBlobSizeRoot(const TagId &tag_id, + const hshm::charbuf &blob_name, const BlobId &blob_id) { LPointer> push_task = - AsyncGetBlobSizeRoot(tag_id, blob_id); + AsyncGetBlobSizeRoot(tag_id, blob_name, blob_id); push_task->Wait(); GetBlobSizeTask *task = push_task->get(); size_t size = task->size_; diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index b043127a1..b3638ee85 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -255,6 +255,10 @@ struct PutBlobTask : public Task, TaskFlags bitfield32_t task_flags) : Task(alloc) { // Initialize task task_node_ = task_node; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kPutBlob; + task_flags_ = task_flags; if (!blob_id.IsNull()) { lane_hash_ = blob_id.hash_; domain_id_ = domain_id; @@ -262,10 +266,6 @@ struct PutBlobTask : public Task, TaskFlags lane_hash_ = std::hash{}(blob_name); domain_id_ = DomainId::GetNode(HASH_TO_NODE_ID(lane_hash_)); } - prio_ = TaskPrio::kLowLatency; - task_state_ = state_id; - method_ = Method::kPutBlob; - task_flags_ = task_flags; // Custom params tag_id_ = tag_id; @@ -686,6 +686,7 @@ struct GetBlobNameTask : public Task, TaskFlags { /** Get \a score from \a blob_id BLOB id */ struct GetBlobSizeTask : public Task, TaskFlags { IN TagId tag_id_; + IN hipc::ShmArchive blob_name_; IN BlobId blob_id_; OUT size_t size_; @@ -700,21 +701,33 @@ struct GetBlobSizeTask : public Task, TaskFlags { const DomainId &domain_id, const TaskStateId &state_id, const TagId &tag_id, + const hshm::charbuf &blob_name, const BlobId &blob_id) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = blob_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kGetBlobSize; task_flags_.SetBits(TASK_LOW_LATENCY); - domain_id_ = domain_id; + if (!blob_id.IsNull()) { + lane_hash_ = blob_id.hash_; + domain_id_ = domain_id; + } else { + lane_hash_ = std::hash{}(blob_name); + domain_id_ = DomainId::GetNode(HASH_TO_NODE_ID(lane_hash_)); + } // Custom tag_id_ = tag_id; + HSHM_MAKE_AR(blob_name_, alloc, blob_name) blob_id_ = blob_id; } + /** Destructor */ + ~GetBlobSizeTask() { + HSHM_DESTROY_AR(blob_name_) + } + /** (De)serialize message call */ template void SerializeStart(Ar &ar) { diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 0f2d5c604..1d3778fea 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -525,6 +525,12 @@ class Server : public TaskLib { * Get \a score from \a blob_id BLOB id * */ void GetBlobSize(GetBlobSizeTask *task, RunContext &ctx) { + if (task->blob_id_.IsNull()) { + bitfield32_t flags; + task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, + hshm::to_charbuf(*task->blob_name_), + ctx, flags); + } BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { From 26efc0f076da00a3d88f267349e9fa8aa7b24d89 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 16:33:40 -0500 Subject: [PATCH 067/191] Add blob_name to GetBlob and GetBlobSize --- .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index b3638ee85..e487d23c3 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -410,7 +410,7 @@ struct GetBlobTask : public Task, TaskFlags data_size_, domain_id_); task_serialize(ar); ar & xfer; - ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_, flags_); + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, filename_, page_size_, flags_); } /** Deserialize message call */ @@ -420,7 +420,7 @@ struct GetBlobTask : public Task, TaskFlags task_serialize(ar); ar & xfer; data_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); - ar(tag_id_, blob_id_, blob_off_, data_size_, filename_, page_size_, flags_); + ar(tag_id_, blob_name_, blob_id_, blob_off_, data_size_, filename_, page_size_, flags_); } /** (De)serialize message return */ @@ -732,7 +732,7 @@ struct GetBlobSizeTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); - ar(tag_id_, blob_id_); + ar(tag_id_, blob_name_, blob_id_); } /** (De)serialize message return */ From 5e9b6d7b8fa6fe1bdef180e59c7f912ca719cc92 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 17:22:29 -0500 Subject: [PATCH 068/191] Try asynchronous push --- benchmark/hermes_api_bench.cc | 2 +- tasks/hermes/include/hermes/bucket.h | 44 ++++++++++++++-------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index 15e7c8856..a97311a10 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -48,7 +48,7 @@ void PutTest(int nprocs, int rank, for (size_t i = 0; i < blobs_per_rank; ++i) { size_t blob_name_int = rank * blobs_per_rank + i; std::string name = std::to_string(blob_name_int); - bkt.Put(name, blob, ctx); + bkt.AsyncPut(name, blob, ctx); } } t.Pause(); diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 1a97f7bd5..67ade3690 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -210,9 +210,9 @@ class Bucket { template HSHM_ALWAYS_INLINE BlobId BasePut(const std::string &blob_name, + const BlobId &orig_blob_id, const Blob &blob, size_t blob_off, - const BlobId &orig_blob_id, Context &ctx) { BlobId blob_id = orig_blob_id; bitfield32_t flags, task_flags(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY); @@ -235,11 +235,13 @@ class Bucket { push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, flags, ctx, task_flags); - if (flags.Any(HERMES_GET_BLOB_ID)) { - push_task->Wait(); - PutBlobTask *task = push_task->get(); - blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); + if constexpr (!ASYNC) { + if (flags.Any(HERMES_GET_BLOB_ID)) { + push_task->Wait(); + PutBlobTask *task = push_task->get(); + blob_id = task->blob_id_; + LABSTOR_CLIENT->DelTask(push_task); + } } return blob_id; } @@ -250,14 +252,14 @@ class Bucket { template HSHM_ALWAYS_INLINE BlobId SrlBasePut(const std::string &blob_name, - const T &data, const BlobId &orig_blob_id, + const T &data, Context &ctx) { std::stringstream ss; cereal::BinaryOutputArchive ar(ss); ar << data; Blob blob(ss.str()); - return BasePut(blob_name, blob, 0, orig_blob_id, ctx); + return BasePut(blob_name, orig_blob_id, blob, 0, ctx); } /** @@ -268,9 +270,9 @@ class Bucket { const T &blob, Context &ctx) { if (std::is_same_v) { - return BasePut(blob_name, blob, 0, BlobId::GetNull(), ctx); + return BasePut(blob_name, BlobId::GetNull(), blob, 0, ctx); } else { - return SrlBasePut(blob_name, blob, BlobId::GetNull(), ctx); + return SrlBasePut(blob_name, BlobId::GetNull(), blob, ctx); } } @@ -282,32 +284,30 @@ class Bucket { const T &blob, Context &ctx) { if (std::is_same_v) { - return BasePut("", blob, 0, blob_id, ctx); + return BasePut("", blob_id, blob, 0, ctx); } else { - return SrlBasePut("", blob, blob_id, ctx); + return SrlBasePut("", blob_id, blob, ctx); } } /** * Put \a blob_name Blob into the bucket * */ - template HSHM_ALWAYS_INLINE void AsyncPut(const std::string &blob_name, - const T &blob, + const Blob &blob, Context &ctx) { - Put(blob_name, blob, ctx); + BasePut(blob_name, BlobId::GetNull(), blob, 0, ctx); } /** * Put \a blob_id Blob into the bucket * */ - template HSHM_ALWAYS_INLINE void AsyncPut(const BlobId &blob_id, - const T &blob, + const Blob &blob, Context &ctx) { - Put(blob_id, blob, ctx); + BasePut("", BlobId::GetNull(), blob, 0, ctx); } /** @@ -317,7 +317,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - return BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); + return BasePut(blob_name, BlobId::GetNull(), blob, blob_off, ctx); } /** @@ -327,7 +327,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - return BasePut("", blob, blob_off, blob_id, ctx); + return BasePut("", blob_id, blob, blob_off, ctx); } /** @@ -337,7 +337,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - BasePut(blob_name, blob, blob_off, BlobId::GetNull(), ctx); + BasePut(blob_name, BlobId::GetNull(), blob, blob_off, ctx); } /** @@ -347,7 +347,7 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - BasePut("", blob, blob_off, blob_id, ctx); + BasePut("", blob_id, blob, blob_off, ctx); } /** From 78bf9ac318fdb64e363a14ea6a04b6a2e72ab46c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 17:43:21 -0500 Subject: [PATCH 069/191] dont rely on HERMES_GET_BLOB_ID flag --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 1d3778fea..df2077126 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -145,7 +145,7 @@ class Server : public TaskLib { // Get the blob info data structure hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); - if (task->flags_.Any(HERMES_GET_BLOB_ID)) { + if (task->blob_id_.IsNull()) { task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, blob_name, ctx, task->flags_); } @@ -375,7 +375,7 @@ class Server : public TaskLib { } void GetBlobGetPhase(GetBlobTask *task, RunContext &ctx) { - if (task->flags_.Any(HERMES_GET_BLOB_ID)) { + if (task->blob_id_.IsNull()) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, blob_name, ctx, task->flags_); From b1d8f8ebffdd412b2e704861498d0f988850b0f1 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 24 Sep 2023 23:58:28 -0500 Subject: [PATCH 070/191] Include blob size --- benchmark/hermes_api_bench.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index a97311a10..018ac7c11 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -68,7 +68,7 @@ void GetTest(int nprocs, int rank, t.Resume(); for (int j = 0; j < repeat; ++j) { for (size_t i = 0; i < blobs_per_rank; ++i) { - hermes::Blob ret; + hermes::Blob ret(blob_size); size_t blob_name_int = rank * blobs_per_rank + i; std::string name = std::to_string(blob_name_int); bkt.Get(name, ret, ctx); From 2eb22d1952cd45f324f1587ba5b0a06c1ac6c58c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 25 Sep 2023 00:25:13 -0500 Subject: [PATCH 071/191] Use async partial put in filesystem adapter --- tasks/hermes_adapters/filesystem/filesystem.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/hermes_adapters/filesystem/filesystem.cc b/tasks/hermes_adapters/filesystem/filesystem.cc index 44d3a0a45..00f2656f8 100644 --- a/tasks/hermes_adapters/filesystem/filesystem.cc +++ b/tasks/hermes_adapters/filesystem/filesystem.cc @@ -130,7 +130,7 @@ size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr, const Blob page((const char*)ptr + data_offset, p.blob_size_); if (!is_append) { std::string blob_name(p.CreateBlobName().str()); - bkt.PartialPut(blob_name, page, p.blob_off_, ctx); + bkt.AsyncPartialPut(blob_name, page, p.blob_off_, ctx); } else { bkt.Append(page, stat.page_size_, ctx); } From 528f82651cb27e151f8cb7c66d9b46735142bea2 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 25 Sep 2023 12:18:15 -0500 Subject: [PATCH 072/191] Add TimespecLatency --- benchmark/test_init.cc | 1 - benchmark/test_latency.cc | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/benchmark/test_init.cc b/benchmark/test_init.cc index 63132b16a..e101758d3 100644 --- a/benchmark/test_init.cc +++ b/benchmark/test_init.cc @@ -16,7 +16,6 @@ #include "test_init.h" void MainPretest() { - TRANSPARENT_LABSTOR(); } void MainPosttest() { diff --git a/benchmark/test_latency.cc b/benchmark/test_latency.cc index ea8f14c34..c6b8410bc 100644 --- a/benchmark/test_latency.cc +++ b/benchmark/test_latency.cc @@ -243,6 +243,7 @@ void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) { /** Time for worker to process a request */ TEST_CASE("TestWorkerLatency") { + TRANSPARENT_LABSTOR(); TestWorkerIterationLatency(1, 16); TestWorkerIterationLatency(5, 16); TestWorkerIterationLatency(10, 16); @@ -251,6 +252,7 @@ TEST_CASE("TestWorkerLatency") { /** Time to process a request */ TEST_CASE("TestRoundTripLatency") { + TRANSPARENT_LABSTOR(); HERMES->ClientInit(); labstor::small_message::Client client; LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetLocal(), "small_message"); @@ -276,6 +278,20 @@ TEST_CASE("TestRoundTripLatency") { HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } +TEST_CASE("TestTimespecLatency") { + size_t ops = (1 << 20); + hshm::Timer t; + + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + struct timespec ts; + timespec_get(&ts, TIME_UTC); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + /** Time to process a request */ //TEST_CASE("TestHermesGetBlobIdLatency") { // HERMES->ClientInit(); From 7c1f3f1fc568e309ee7e9fe3d88f8a08bf90483d Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:00:23 -0500 Subject: [PATCH 073/191] re-enable task frees --- include/labstor/api/labstor_client.h | 4 ++-- src/worker.cc | 2 +- tasks/hermes/include/hermes/bucket.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 54e9d9f99..f23f0ca66 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -140,7 +140,7 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void DelTask(TaskT *task) { // TODO(llogan): verify leak - // main_alloc_->DelObj(task); + main_alloc_->DelObj(task); } /** Destroy a task */ @@ -189,7 +189,7 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void FreeBuffer(hipc::Pointer &p) { // TODO(llogan): verify leak - // main_alloc_->Free(p); + main_alloc_->Free(p); } /** Free a buffer */ diff --git a/src/worker.cc b/src/worker.cc index 7a943839a..491fdd29f 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -105,7 +105,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { entry->complete_ = true; if (task->IsCoroutine()) { // TODO(llogan): verify leak - // free(ctx.stack_ptr_); + free(ctx.stack_ptr_); } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); } diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 67ade3690..66da72dc1 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -435,7 +435,7 @@ class Bucket { char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); memcpy(blob.data(), data, data_size); LABSTOR_CLIENT->FreeBuffer(task->data_); - LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(push_task); return blob_id; } From 534fde9143ad8b23218b53530587ff77eb7bf111 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:09:20 -0500 Subject: [PATCH 074/191] Comment out a few deltasks --- .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 10 +++++----- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 10 +++++----- tasks_required/remote_queue/src/remote_queue.cc | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index e487d23c3..037abaa9c 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -284,11 +284,11 @@ struct PutBlobTask : public Task, TaskFlags /** Destructor */ ~PutBlobTask() { - HSHM_DESTROY_AR(blob_name_); - HSHM_DESTROY_AR(filename_); - if (IsDataOwner()) { - LABSTOR_CLIENT->FreeBuffer(data_); - } +// HSHM_DESTROY_AR(blob_name_); +// HSHM_DESTROY_AR(filename_); +// if (IsDataOwner()) { +// LABSTOR_CLIENT->FreeBuffer(data_); +// } } /** (De)serialize message call */ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index df2077126..d77f3516b 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -340,11 +340,11 @@ class Server : public TaskLib { LABSTOR_CLIENT->DelTask(write_task); } HILOG(kDebug, "PutBlobTask complete"); - HSHM_DESTROY_AR(task->schema_); - HSHM_DESTROY_AR(task->bdev_writes_); - if (task->flags_.Any(HERMES_DID_STAGE_IN)) { - LABSTOR_CLIENT->FreeBuffer(task->data_ptr_); - } +// HSHM_DESTROY_AR(task->schema_); +// HSHM_DESTROY_AR(task->bdev_writes_); +// if (task->flags_.Any(HERMES_DID_STAGE_IN)) { +// LABSTOR_CLIENT->FreeBuffer(task->data_ptr_); +// } // Update the bucket statistics int update_mode = bucket_mdm::UpdateSizeMode::kAdd; if (task->flags_.Any(HERMES_IS_FILE)) { diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index b0718a7c7..f8ecbbadc 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - LABSTOR_CLIENT->DelTask(orig_task); +// LABSTOR_CLIENT->DelTask(orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } From da4bcef0dadecb3304b57eccf1b0639a726db039 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:18:39 -0500 Subject: [PATCH 075/191] Remove all DelTasks from cc files --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 10 +++++----- tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index d77f3516b..08cba96e8 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -282,7 +282,7 @@ class Server : public TaskLib { sub_plcmnt.size_ += diff; HILOG(kDebug, "Passing {} bytes to target {}", diff, sub_plcmnt.tid_); } - LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); +// LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); if (task->plcmnt_idx_ < (*task->schema_).size()) { task->phase_ = PutBlobPhase::kAllocate; return; @@ -337,7 +337,7 @@ class Server : public TaskLib { } } for (bdev::WriteTask *&write_task : write_tasks) { - LABSTOR_CLIENT->DelTask(write_task); +// LABSTOR_CLIENT->DelTask(write_task); } HILOG(kDebug, "PutBlobTask complete"); // HSHM_DESTROY_AR(task->schema_); @@ -421,7 +421,7 @@ class Server : public TaskLib { } } for (bdev::ReadTask *&read_task : read_tasks) { - LABSTOR_CLIENT->DelTask(read_task); +// LABSTOR_CLIENT->DelTask(read_task); } HSHM_DESTROY_AR(task->bdev_reads_); HILOG(kDebug, "GetBlobTask complete"); @@ -642,7 +642,7 @@ class Server : public TaskLib { } } for (bdev::FreeTask *&free_task : free_tasks) { - LABSTOR_CLIENT->DelTask(free_task); +// LABSTOR_CLIENT->DelTask(free_task); } BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; @@ -686,7 +686,7 @@ class Server : public TaskLib { if (!task->get_task_->IsComplete()) { return; } - LABSTOR_CLIENT->DelTask(task->get_task_); +// LABSTOR_CLIENT->DelTask(task->get_task_); task->phase_ = ReorganizeBlobPhase::kPut; } case ReorganizeBlobPhase::kPut: { diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 5d1a96fc0..5b2768925 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -115,7 +115,7 @@ class Server : public TaskLib { LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) for (AppendInfo &append : append_info) { append.blob_id_ = append.blob_id_task_->blob_id_; - LABSTOR_CLIENT->DelTask(append.blob_id_task_); +// LABSTOR_CLIENT->DelTask(append.blob_id_task_); } task->SetModuleComplete(); } @@ -178,10 +178,10 @@ class Server : public TaskLib { HILOG(kDebug, "(node {}) PUT blobs for tag {} (task_node={})", LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) for (AppendInfo &append : append_info) { - LABSTOR_CLIENT->DelTask(append.put_task_); +// LABSTOR_CLIENT->DelTask(append.put_task_); } HSHM_DESTROY_AR(task->schema_->append_info_); - LABSTOR_CLIENT->DelTask(task->schema_); +// LABSTOR_CLIENT->DelTask(task->schema_); task->SetModuleComplete(); } } @@ -294,7 +294,7 @@ class Server : public TaskLib { } } for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { - LABSTOR_CLIENT->DelTask(blob_task); +// LABSTOR_CLIENT->DelTask(blob_task); } HSHM_DESTROY_AR(task->destroy_blob_tasks_); TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; From f1727e83acfa620e88cf39a3fec19632dcf6c42b Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:20:56 -0500 Subject: [PATCH 076/191] Only DelTask --- include/labstor/api/labstor_client.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index f23f0ca66..5d2984799 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -140,14 +140,14 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void DelTask(TaskT *task) { // TODO(llogan): verify leak - main_alloc_->DelObj(task); + // main_alloc_->DelObj(task); } /** Destroy a task */ template HSHM_ALWAYS_INLINE void DelTask(LPointer &task) { - main_alloc_->DelObjLocal(task); + // main_alloc_->DelObjLocal(task); } /** Get a queue by its ID */ From 87cc20ee12733159b11ce5170d4e51f99e3a6965 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:26:56 -0500 Subject: [PATCH 077/191] Remove FreeBuffer and DelTask --- include/labstor/api/labstor_client.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 5d2984799..43500b752 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -189,14 +189,14 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void FreeBuffer(hipc::Pointer &p) { // TODO(llogan): verify leak - main_alloc_->Free(p); + // main_alloc_->Free(p); } /** Free a buffer */ HSHM_ALWAYS_INLINE void FreeBuffer(LPointer &p) { // TODO(llogan): verify leak - main_alloc_->FreeLocalPtr(p); + // main_alloc_->FreeLocalPtr(p); } }; From 1398fdb7e44ed7c3cccae01c7836113e8cb32e13 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:36:05 -0500 Subject: [PATCH 078/191] Bring FreeBuffer back --- include/labstor/api/labstor_client.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 43500b752..a1daf9546 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -189,14 +189,14 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void FreeBuffer(hipc::Pointer &p) { // TODO(llogan): verify leak - // main_alloc_->Free(p); + main_alloc_->Free(p); } /** Free a buffer */ HSHM_ALWAYS_INLINE void FreeBuffer(LPointer &p) { // TODO(llogan): verify leak - // main_alloc_->FreeLocalPtr(p); + main_alloc_->FreeLocalPtr(p); } }; From 77e33dc9ee13c768a4ed84c57816dc3b110a9bfd Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:42:04 -0500 Subject: [PATCH 079/191] Comment out all DelTasks in tasks --- include/labstor/api/labstor_client.h | 4 +-- tasks/bdev/include/bdev/bdev.h | 2 +- tasks/hermes/include/hermes/bucket.h | 4 +-- .../include/hermes_adapters/hermes_adapters.h | 2 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 28 +++++++++---------- .../hermes_bucket_mdm/hermes_bucket_mdm.h | 20 ++++++------- .../remote_queue/src/remote_queue.cc | 2 +- 7 files changed, 31 insertions(+), 31 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index a1daf9546..a7d98f267 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -140,14 +140,14 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void DelTask(TaskT *task) { // TODO(llogan): verify leak - // main_alloc_->DelObj(task); + main_alloc_->DelObj(task); } /** Destroy a task */ template HSHM_ALWAYS_INLINE void DelTask(LPointer &task) { - // main_alloc_->DelObjLocal(task); + main_alloc_->DelObjLocal(task); } /** Get a queue by its ID */ diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index 17b130b87..b69f56c07 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -55,7 +55,7 @@ class Client : public TaskLibClient { id_ = task->id_; queue_id_ = QueueId(id_); monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 66da72dc1..317d1d82d 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -240,7 +240,7 @@ class Bucket { push_task->Wait(); PutBlobTask *task = push_task->get(); blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } } return blob_id; @@ -435,7 +435,7 @@ class Bucket { char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); memcpy(blob.data(), data, data_size); LABSTOR_CLIENT->FreeBuffer(task->data_); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return blob_id; } diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h index 1817cdf75..25a8e4f9e 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h @@ -42,7 +42,7 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } /** Destroy task state + queue */ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 9b82d620f..d3cce712b 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -44,7 +44,7 @@ class Client : public TaskLibClient { if (task->IsComplete()) { id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreate); @@ -79,7 +79,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncSetBucketMdmRoot(domain_id, blob_mdm_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(SetBucketMdm); @@ -101,7 +101,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetOrCreateBlobIdTask *task = push_task->get(); BlobId blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return blob_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateBlobId); @@ -165,7 +165,7 @@ class Client : public TaskLibClient { GetBlobTask *task = push_task->get(); data = task->data_; size_t true_size = task->data_size_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return true_size; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlob); @@ -211,7 +211,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagBlobRoot(tag_id, blob_id, tag); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagBlob); @@ -235,7 +235,7 @@ class Client : public TaskLibClient { push_task->Wait(); BlobHasTagTask *task = push_task->get(); bool has_tag = task->has_tag_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return has_tag; } LABSTOR_TASK_NODE_PUSH_ROOT(BlobHasTag); @@ -259,7 +259,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobIdTask *task = push_task->get(); BlobId blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return blob_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobId); @@ -282,7 +282,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobNameTask *task = push_task->get(); std::string blob_name = task->blob_name_->str(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return blob_name; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobName); @@ -308,7 +308,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobSizeTask *task = push_task->get(); size_t size = task->size_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return size; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobSize); @@ -331,7 +331,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobScoreTask *task = push_task->get(); float score = task->score_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return score; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobScore); @@ -354,7 +354,7 @@ class Client : public TaskLibClient { GetBlobBuffersTask *task = push_task->get(); std::vector buffers = hshm::to_stl_vector(*task->buffers_); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return buffers; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobBuffers) @@ -378,7 +378,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncRenameBlobRoot(tag_id, blob_id, new_blob_name); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(RenameBlob); @@ -400,7 +400,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTruncateBlobRoot(tag_id, blob_id, new_size); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TruncateBlob); @@ -420,7 +420,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncDestroyBlobRoot(tag_id, blob_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(DestroyBlob); }; diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index 479085856..8c7d980a8 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -57,7 +57,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncSetBlobMdmRoot(domain_id, blob_mdm_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(SetBlobMdm); @@ -141,7 +141,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetOrCreateTagTask *task = push_task->get(); TagId tag_id = task->tag_id_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return tag_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateTag); @@ -161,7 +161,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetTagIdTask *task = push_task->get(); TagId tag_id = task->tag_id_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return tag_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetTagId); @@ -181,7 +181,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetTagNameTask *task = push_task->get(); hshm::string tag_name = hshm::to_charbuf(*task->tag_name_.get()); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return tag_name; } LABSTOR_TASK_NODE_PUSH_ROOT(GetTagName); @@ -200,7 +200,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncRenameTagRoot(tag_id, new_tag_name); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(RenameTag); @@ -217,7 +217,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncDestroyTagRoot(tag_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(DestroyTag); @@ -235,7 +235,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagAddBlobRoot(tag_id, blob_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagAddBlob); @@ -252,7 +252,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagRemoveBlobRoot(tag_id, blob_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagRemoveBlob); @@ -269,7 +269,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagClearBlobsRoot(tag_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagClearBlobs); @@ -288,7 +288,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetSizeTask *task = push_task->get(); size_t size = task->size_; - LABSTOR_CLIENT->DelTask(push_task); + // LABSTOR_CLIENT->DelTask(push_task); return size; } LABSTOR_TASK_NODE_PUSH_ROOT(GetSize); diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index f8ecbbadc..b0718a7c7 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); -// LABSTOR_CLIENT->DelTask(orig_task); + LABSTOR_CLIENT->DelTask(orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } From a2c75b19627ee38c57f9a386b637de929c2bfaca Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:46:10 -0500 Subject: [PATCH 080/191] Don't free proc_queue --- .../proc_queue/include/proc_queue/proc_queue_tasks.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index 9372d1307..dd50a3d5b 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -109,11 +109,11 @@ struct TypedPushTask : public Task, TaskFlags { /** Destructor */ ~TypedPushTask() { - if (!IsFireAndForget()) { - LABSTOR_CLIENT->DelTask(sub_cli_); - } else { - LABSTOR_CLIENT->DelTask(sub_run_); - } +// if (!IsFireAndForget()) { +// LABSTOR_CLIENT->DelTask(sub_cli_); +// } else { +// LABSTOR_CLIENT->DelTask(sub_run_); +// } } /** Create group */ From 1ec1f3926847e9f561acb5eae1e741a328b5c75f Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:50:02 -0500 Subject: [PATCH 081/191] Comment out free buffer --- tasks/hermes/include/hermes/bucket.h | 2 +- .../include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 317d1d82d..2869a2816 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -434,7 +434,7 @@ class Bucket { blob_id = task->blob_id_; char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); memcpy(blob.data(), data, data_size); - LABSTOR_CLIENT->FreeBuffer(task->data_); + // LABSTOR_CLIENT->FreeBuffer(task->data_); // LABSTOR_CLIENT->DelTask(push_task); return blob_id; } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 2dca04d1b..bfb9c7060 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -317,9 +317,9 @@ struct AppendBlobTask : public Task, TaskFlags { /** Destructor */ ~AppendBlobTask() { - if (IsDataOwner()) { - LABSTOR_CLIENT->FreeBuffer(data_); - } +// if (IsDataOwner()) { +// LABSTOR_CLIENT->FreeBuffer(data_); +// } } /** Create group */ From 00278ef7332c875a1c23d923e13e3eb408f8de68 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 11:58:46 -0500 Subject: [PATCH 082/191] Comment out admin frees --- .../include/labstor_admin/labstor_admin.h | 16 ++++++++-------- .../labstor_admin/src/labstor_admin.cc | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h index 8fc0ab097..2ded473d5 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h @@ -35,7 +35,7 @@ class Client : public TaskLibClient { const std::string &lib_name) { LPointer task = AsyncRegisterTaskLibRoot(domain_id, lib_name); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(RegisterTaskLib) @@ -55,7 +55,7 @@ class Client : public TaskLibClient { LPointer task = AsyncDestroyTaskLibRoot(domain_id, lib_name); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskLib) @@ -77,7 +77,7 @@ class Client : public TaskLibClient { domain_id, std::forward(args)...); task->Wait(); TaskStateId new_id = task->id_; - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); if (new_id.IsNull()) { HELOG(kWarning, "Failed to create task state"); } @@ -130,7 +130,7 @@ class Client : public TaskLibClient { AsyncGetOrCreateTaskStateIdRoot(domain_id, state_name); task->Wait(); TaskStateId new_id = task->id_; - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); return new_id; } LABSTOR_TASK_NODE_ADMIN_ROOT(GetOrCreateTaskStateId) @@ -149,7 +149,7 @@ class Client : public TaskLibClient { AsyncGetTaskStateIdRoot(domain_id, state_name); task->Wait(); TaskStateId new_id = task->id_; - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); return new_id; } LABSTOR_TASK_NODE_ADMIN_ROOT(GetTaskStateId) @@ -169,7 +169,7 @@ class Client : public TaskLibClient { LPointer task = AsyncDestroyTaskStateRoot(domain_id, id); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskState) @@ -195,7 +195,7 @@ class Client : public TaskLibClient { LPointer task = AsyncSetWorkOrchQueuePolicyRoot(domain_id, policy); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchQueuePolicy); @@ -212,7 +212,7 @@ class Client : public TaskLibClient { LPointer task = AsyncSetWorkOrchProcPolicyRoot(domain_id, policy); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchProcPolicy); }; diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index d80e4998a..d400d06e4 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -51,7 +51,7 @@ class Server : public TaskLib { LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId(task->task_node_ + 1, domain, state_name); get_id->Wait(task); task->id_ = get_id->id_; - LABSTOR_CLIENT->DelTask(get_id); + // LABSTOR_CLIENT->DelTask(get_id); } // Create the task state HILOG(kInfo, "(node {}) Creating task state {} with id {} (task_node={})", From 5c7cdd969472c4fcbd26c84047858d62e511bd1b Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 12:02:40 -0500 Subject: [PATCH 083/191] Comment out remote queue deltasks --- tasks_required/remote_queue/include/remote_queue/remote_queue.h | 2 +- tasks_required/remote_queue/src/remote_queue.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index f0cf88119..b736b71c1 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -50,7 +50,7 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } /** Destroy task state + queue */ diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index b0718a7c7..02cca5b92 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - LABSTOR_CLIENT->DelTask(orig_task); + // LABSTOR_CLIENT->DelTask(orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } From 403d7cc4813ccb4352f758031badf32573072afa Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 12:05:18 -0500 Subject: [PATCH 084/191] Enable proc_queue fress --- .../include/labstor_admin/labstor_admin.h | 16 ++++++++-------- .../include/proc_queue/proc_queue_tasks.h | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h index 2ded473d5..8fc0ab097 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h @@ -35,7 +35,7 @@ class Client : public TaskLibClient { const std::string &lib_name) { LPointer task = AsyncRegisterTaskLibRoot(domain_id, lib_name); task->Wait(); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(RegisterTaskLib) @@ -55,7 +55,7 @@ class Client : public TaskLibClient { LPointer task = AsyncDestroyTaskLibRoot(domain_id, lib_name); task->Wait(); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskLib) @@ -77,7 +77,7 @@ class Client : public TaskLibClient { domain_id, std::forward(args)...); task->Wait(); TaskStateId new_id = task->id_; - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); if (new_id.IsNull()) { HELOG(kWarning, "Failed to create task state"); } @@ -130,7 +130,7 @@ class Client : public TaskLibClient { AsyncGetOrCreateTaskStateIdRoot(domain_id, state_name); task->Wait(); TaskStateId new_id = task->id_; - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); return new_id; } LABSTOR_TASK_NODE_ADMIN_ROOT(GetOrCreateTaskStateId) @@ -149,7 +149,7 @@ class Client : public TaskLibClient { AsyncGetTaskStateIdRoot(domain_id, state_name); task->Wait(); TaskStateId new_id = task->id_; - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); return new_id; } LABSTOR_TASK_NODE_ADMIN_ROOT(GetTaskStateId) @@ -169,7 +169,7 @@ class Client : public TaskLibClient { LPointer task = AsyncDestroyTaskStateRoot(domain_id, id); task->Wait(); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskState) @@ -195,7 +195,7 @@ class Client : public TaskLibClient { LPointer task = AsyncSetWorkOrchQueuePolicyRoot(domain_id, policy); task->Wait(); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchQueuePolicy); @@ -212,7 +212,7 @@ class Client : public TaskLibClient { LPointer task = AsyncSetWorkOrchProcPolicyRoot(domain_id, policy); task->Wait(); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchProcPolicy); }; diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index dd50a3d5b..9372d1307 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -109,11 +109,11 @@ struct TypedPushTask : public Task, TaskFlags { /** Destructor */ ~TypedPushTask() { -// if (!IsFireAndForget()) { -// LABSTOR_CLIENT->DelTask(sub_cli_); -// } else { -// LABSTOR_CLIENT->DelTask(sub_run_); -// } + if (!IsFireAndForget()) { + LABSTOR_CLIENT->DelTask(sub_cli_); + } else { + LABSTOR_CLIENT->DelTask(sub_run_); + } } /** Create group */ From 1fe00243d6a28473e82fc6cc53283f4b1cdb06d1 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 12:08:55 -0500 Subject: [PATCH 085/191] Disable proc queue free --- .../proc_queue/include/proc_queue/proc_queue_tasks.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index 9372d1307..dd50a3d5b 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -109,11 +109,11 @@ struct TypedPushTask : public Task, TaskFlags { /** Destructor */ ~TypedPushTask() { - if (!IsFireAndForget()) { - LABSTOR_CLIENT->DelTask(sub_cli_); - } else { - LABSTOR_CLIENT->DelTask(sub_run_); - } +// if (!IsFireAndForget()) { +// LABSTOR_CLIENT->DelTask(sub_cli_); +// } else { +// LABSTOR_CLIENT->DelTask(sub_run_); +// } } /** Create group */ From 7c2178b84e242723132f78ab21db4210e69c5751 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 12:27:13 -0500 Subject: [PATCH 086/191] Add back all other deltasks --- tasks/bdev/include/bdev/bdev.h | 2 +- tasks/hermes/include/hermes/bucket.h | 4 +-- .../include/hermes_adapters/hermes_adapters.h | 2 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 28 +++++++++---------- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 20 ++++++------- .../hermes_bucket_mdm/hermes_bucket_mdm.h | 20 ++++++------- .../src/hermes_bucket_mdm.cc | 8 +++--- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index b69f56c07..17b130b87 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -55,7 +55,7 @@ class Client : public TaskLibClient { id_ = task->id_; queue_id_ = QueueId(id_); monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 2869a2816..3c13ffe30 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -240,7 +240,7 @@ class Bucket { push_task->Wait(); PutBlobTask *task = push_task->get(); blob_id = task->blob_id_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } } return blob_id; @@ -435,7 +435,7 @@ class Bucket { char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); memcpy(blob.data(), data, data_size); // LABSTOR_CLIENT->FreeBuffer(task->data_); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return blob_id; } diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h index 25a8e4f9e..1817cdf75 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h @@ -42,7 +42,7 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } /** Destroy task state + queue */ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index d3cce712b..9b82d620f 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -44,7 +44,7 @@ class Client : public TaskLibClient { if (task->IsComplete()) { id_ = task->id_; queue_id_ = QueueId(id_); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreate); @@ -79,7 +79,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncSetBucketMdmRoot(domain_id, blob_mdm_id); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(SetBucketMdm); @@ -101,7 +101,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetOrCreateBlobIdTask *task = push_task->get(); BlobId blob_id = task->blob_id_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return blob_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateBlobId); @@ -165,7 +165,7 @@ class Client : public TaskLibClient { GetBlobTask *task = push_task->get(); data = task->data_; size_t true_size = task->data_size_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return true_size; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlob); @@ -211,7 +211,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagBlobRoot(tag_id, blob_id, tag); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagBlob); @@ -235,7 +235,7 @@ class Client : public TaskLibClient { push_task->Wait(); BlobHasTagTask *task = push_task->get(); bool has_tag = task->has_tag_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return has_tag; } LABSTOR_TASK_NODE_PUSH_ROOT(BlobHasTag); @@ -259,7 +259,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobIdTask *task = push_task->get(); BlobId blob_id = task->blob_id_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return blob_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobId); @@ -282,7 +282,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobNameTask *task = push_task->get(); std::string blob_name = task->blob_name_->str(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return blob_name; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobName); @@ -308,7 +308,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobSizeTask *task = push_task->get(); size_t size = task->size_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return size; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobSize); @@ -331,7 +331,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetBlobScoreTask *task = push_task->get(); float score = task->score_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return score; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobScore); @@ -354,7 +354,7 @@ class Client : public TaskLibClient { GetBlobBuffersTask *task = push_task->get(); std::vector buffers = hshm::to_stl_vector(*task->buffers_); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return buffers; } LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobBuffers) @@ -378,7 +378,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncRenameBlobRoot(tag_id, blob_id, new_blob_name); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(RenameBlob); @@ -400,7 +400,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTruncateBlobRoot(tag_id, blob_id, new_size); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TruncateBlob); @@ -420,7 +420,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncDestroyBlobRoot(tag_id, blob_id); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(DestroyBlob); }; diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 08cba96e8..df2077126 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -282,7 +282,7 @@ class Server : public TaskLib { sub_plcmnt.size_ += diff; HILOG(kDebug, "Passing {} bytes to target {}", diff, sub_plcmnt.tid_); } -// LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); + LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); if (task->plcmnt_idx_ < (*task->schema_).size()) { task->phase_ = PutBlobPhase::kAllocate; return; @@ -337,14 +337,14 @@ class Server : public TaskLib { } } for (bdev::WriteTask *&write_task : write_tasks) { -// LABSTOR_CLIENT->DelTask(write_task); + LABSTOR_CLIENT->DelTask(write_task); } HILOG(kDebug, "PutBlobTask complete"); -// HSHM_DESTROY_AR(task->schema_); -// HSHM_DESTROY_AR(task->bdev_writes_); -// if (task->flags_.Any(HERMES_DID_STAGE_IN)) { -// LABSTOR_CLIENT->FreeBuffer(task->data_ptr_); -// } + HSHM_DESTROY_AR(task->schema_); + HSHM_DESTROY_AR(task->bdev_writes_); + if (task->flags_.Any(HERMES_DID_STAGE_IN)) { + LABSTOR_CLIENT->FreeBuffer(task->data_ptr_); + } // Update the bucket statistics int update_mode = bucket_mdm::UpdateSizeMode::kAdd; if (task->flags_.Any(HERMES_IS_FILE)) { @@ -421,7 +421,7 @@ class Server : public TaskLib { } } for (bdev::ReadTask *&read_task : read_tasks) { -// LABSTOR_CLIENT->DelTask(read_task); + LABSTOR_CLIENT->DelTask(read_task); } HSHM_DESTROY_AR(task->bdev_reads_); HILOG(kDebug, "GetBlobTask complete"); @@ -642,7 +642,7 @@ class Server : public TaskLib { } } for (bdev::FreeTask *&free_task : free_tasks) { -// LABSTOR_CLIENT->DelTask(free_task); + LABSTOR_CLIENT->DelTask(free_task); } BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; @@ -686,7 +686,7 @@ class Server : public TaskLib { if (!task->get_task_->IsComplete()) { return; } -// LABSTOR_CLIENT->DelTask(task->get_task_); + LABSTOR_CLIENT->DelTask(task->get_task_); task->phase_ = ReorganizeBlobPhase::kPut; } case ReorganizeBlobPhase::kPut: { diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index 8c7d980a8..479085856 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -57,7 +57,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncSetBlobMdmRoot(domain_id, blob_mdm_id); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(SetBlobMdm); @@ -141,7 +141,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetOrCreateTagTask *task = push_task->get(); TagId tag_id = task->tag_id_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return tag_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateTag); @@ -161,7 +161,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetTagIdTask *task = push_task->get(); TagId tag_id = task->tag_id_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return tag_id; } LABSTOR_TASK_NODE_PUSH_ROOT(GetTagId); @@ -181,7 +181,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetTagNameTask *task = push_task->get(); hshm::string tag_name = hshm::to_charbuf(*task->tag_name_.get()); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return tag_name; } LABSTOR_TASK_NODE_PUSH_ROOT(GetTagName); @@ -200,7 +200,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncRenameTagRoot(tag_id, new_tag_name); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(RenameTag); @@ -217,7 +217,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncDestroyTagRoot(tag_id); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(DestroyTag); @@ -235,7 +235,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagAddBlobRoot(tag_id, blob_id); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagAddBlob); @@ -252,7 +252,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagRemoveBlobRoot(tag_id, blob_id); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagRemoveBlob); @@ -269,7 +269,7 @@ class Client : public TaskLibClient { LPointer> push_task = AsyncTagClearBlobsRoot(tag_id); push_task->Wait(); - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(TagClearBlobs); @@ -288,7 +288,7 @@ class Client : public TaskLibClient { push_task->Wait(); GetSizeTask *task = push_task->get(); size_t size = task->size_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); return size; } LABSTOR_TASK_NODE_PUSH_ROOT(GetSize); diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 5b2768925..5d1a96fc0 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -115,7 +115,7 @@ class Server : public TaskLib { LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) for (AppendInfo &append : append_info) { append.blob_id_ = append.blob_id_task_->blob_id_; -// LABSTOR_CLIENT->DelTask(append.blob_id_task_); + LABSTOR_CLIENT->DelTask(append.blob_id_task_); } task->SetModuleComplete(); } @@ -178,10 +178,10 @@ class Server : public TaskLib { HILOG(kDebug, "(node {}) PUT blobs for tag {} (task_node={})", LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) for (AppendInfo &append : append_info) { -// LABSTOR_CLIENT->DelTask(append.put_task_); + LABSTOR_CLIENT->DelTask(append.put_task_); } HSHM_DESTROY_AR(task->schema_->append_info_); -// LABSTOR_CLIENT->DelTask(task->schema_); + LABSTOR_CLIENT->DelTask(task->schema_); task->SetModuleComplete(); } } @@ -294,7 +294,7 @@ class Server : public TaskLib { } } for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { -// LABSTOR_CLIENT->DelTask(blob_task); + LABSTOR_CLIENT->DelTask(blob_task); } HSHM_DESTROY_AR(task->destroy_blob_tasks_); TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; From 0ec596487bcf112df4fa4b3124ffcc9d42195b75 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 18:06:11 -0500 Subject: [PATCH 087/191] Remove all deltasks from cc files --- tasks/bdev/include/bdev/bdev.h | 2 +- .../hermes_adapters/include/hermes_adapters/hermes_adapters.h | 2 +- .../hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h | 2 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 4 ++-- tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index 17b130b87..b69f56c07 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -55,7 +55,7 @@ class Client : public TaskLibClient { id_ = task->id_; queue_id_ = QueueId(id_); monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h index 1817cdf75..25a8e4f9e 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h @@ -42,7 +42,7 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } /** Destroy task state + queue */ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 9b82d620f..80b5e1ec6 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -44,7 +44,7 @@ class Client : public TaskLibClient { if (task->IsComplete()) { id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreate); diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index df2077126..17f78a331 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -282,7 +282,7 @@ class Server : public TaskLib { sub_plcmnt.size_ += diff; HILOG(kDebug, "Passing {} bytes to target {}", diff, sub_plcmnt.tid_); } - LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); + // LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); if (task->plcmnt_idx_ < (*task->schema_).size()) { task->phase_ = PutBlobPhase::kAllocate; return; @@ -686,7 +686,7 @@ class Server : public TaskLib { if (!task->get_task_->IsComplete()) { return; } - LABSTOR_CLIENT->DelTask(task->get_task_); + // LABSTOR_CLIENT->DelTask(task->get_task_); task->phase_ = ReorganizeBlobPhase::kPut; } case ReorganizeBlobPhase::kPut: { diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 5d1a96fc0..14d9b32e9 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -181,7 +181,7 @@ class Server : public TaskLib { LABSTOR_CLIENT->DelTask(append.put_task_); } HSHM_DESTROY_AR(task->schema_->append_info_); - LABSTOR_CLIENT->DelTask(task->schema_); + // LABSTOR_CLIENT->DelTask(task->schema_); task->SetModuleComplete(); } } From 61833578b50a1abbfb23de4a68ccebeb48ea47f2 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 19:02:15 -0500 Subject: [PATCH 088/191] Uncomment asynccreatecomplete --- tasks/bdev/include/bdev/bdev.h | 2 +- tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h | 2 +- tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index b69f56c07..17b130b87 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -55,7 +55,7 @@ class Client : public TaskLibClient { id_ = task->id_; queue_id_ = QueueId(id_); monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h index 25a8e4f9e..1817cdf75 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h @@ -42,7 +42,7 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } /** Destroy task state + queue */ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 80b5e1ec6..9b82d620f 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -44,7 +44,7 @@ class Client : public TaskLibClient { if (task->IsComplete()) { id_ = task->id_; queue_id_ = QueueId(id_); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreate); From 92da3106673183f27d7ffe08ed4f1512c1991124 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 19:06:21 -0500 Subject: [PATCH 089/191] Comment out asynccreaetcomplete --- tasks/bdev/include/bdev/bdev.h | 2 +- tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index 17b130b87..b69f56c07 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -55,7 +55,7 @@ class Client : public TaskLibClient { id_ = task->id_; queue_id_ = QueueId(id_); monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 9b82d620f..80b5e1ec6 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -44,7 +44,7 @@ class Client : public TaskLibClient { if (task->IsComplete()) { id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + // LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreate); From e3d8c040ec4b49a30838c0344a5b5c5ebb6ae170 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 20:21:17 -0500 Subject: [PATCH 090/191] Use stack allocator instead of page allocator --- include/labstor/api/labstor_runtime.h | 3 ++- tasks/bdev/include/bdev/bdev.h | 2 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 2 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 4 ++-- tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc | 2 +- tasks_required/labstor_admin/src/labstor_admin.cc | 2 +- .../proc_queue/include/proc_queue/proc_queue_tasks.h | 10 +++++----- .../remote_queue/include/remote_queue/remote_queue.h | 2 +- tasks_required/remote_queue/src/remote_queue.cc | 2 +- 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/include/labstor/api/labstor_runtime.h b/include/labstor/api/labstor_runtime.h index 76f4525e6..2465b9d4c 100644 --- a/include/labstor/api/labstor_runtime.h +++ b/include/labstor/api/labstor_runtime.h @@ -135,8 +135,9 @@ class Runtime : public ConfigurationManager { mem_mngr->CreateBackend( server_config_.queue_manager_.shm_size_, server_config_.queue_manager_.shm_name_); + // hipc::ScalablePageAllocator main_alloc_ = - mem_mngr->CreateAllocator( + mem_mngr->CreateAllocator( server_config_.queue_manager_.shm_name_, main_alloc_id_, sizeof(LabstorShm)); diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index b69f56c07..17b130b87 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -55,7 +55,7 @@ class Client : public TaskLibClient { id_ = task->id_; queue_id_ = QueueId(id_); monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 80b5e1ec6..9b82d620f 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -44,7 +44,7 @@ class Client : public TaskLibClient { if (task->IsComplete()) { id_ = task->id_; queue_id_ = QueueId(id_); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } } LABSTOR_TASK_NODE_ROOT(AsyncCreate); diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 17f78a331..df2077126 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -282,7 +282,7 @@ class Server : public TaskLib { sub_plcmnt.size_ += diff; HILOG(kDebug, "Passing {} bytes to target {}", diff, sub_plcmnt.tid_); } - // LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); + LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); if (task->plcmnt_idx_ < (*task->schema_).size()) { task->phase_ = PutBlobPhase::kAllocate; return; @@ -686,7 +686,7 @@ class Server : public TaskLib { if (!task->get_task_->IsComplete()) { return; } - // LABSTOR_CLIENT->DelTask(task->get_task_); + LABSTOR_CLIENT->DelTask(task->get_task_); task->phase_ = ReorganizeBlobPhase::kPut; } case ReorganizeBlobPhase::kPut: { diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 14d9b32e9..5d1a96fc0 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -181,7 +181,7 @@ class Server : public TaskLib { LABSTOR_CLIENT->DelTask(append.put_task_); } HSHM_DESTROY_AR(task->schema_->append_info_); - // LABSTOR_CLIENT->DelTask(task->schema_); + LABSTOR_CLIENT->DelTask(task->schema_); task->SetModuleComplete(); } } diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index d400d06e4..d80e4998a 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -51,7 +51,7 @@ class Server : public TaskLib { LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId(task->task_node_ + 1, domain, state_name); get_id->Wait(task); task->id_ = get_id->id_; - // LABSTOR_CLIENT->DelTask(get_id); + LABSTOR_CLIENT->DelTask(get_id); } // Create the task state HILOG(kInfo, "(node {}) Creating task state {} with id {} (task_node={})", diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index dd50a3d5b..9372d1307 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -109,11 +109,11 @@ struct TypedPushTask : public Task, TaskFlags { /** Destructor */ ~TypedPushTask() { -// if (!IsFireAndForget()) { -// LABSTOR_CLIENT->DelTask(sub_cli_); -// } else { -// LABSTOR_CLIENT->DelTask(sub_run_); -// } + if (!IsFireAndForget()) { + LABSTOR_CLIENT->DelTask(sub_cli_); + } else { + LABSTOR_CLIENT->DelTask(sub_run_); + } } /** Create group */ diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index b736b71c1..f0cf88119 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -50,7 +50,7 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - // LABSTOR_CLIENT->DelTask(task); + LABSTOR_CLIENT->DelTask(task); } /** Destroy task state + queue */ diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 02cca5b92..b0718a7c7 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - // LABSTOR_CLIENT->DelTask(orig_task); + LABSTOR_CLIENT->DelTask(orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } From cecea758c717fdd56b5573e66b1df9c8b2dc1cf7 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 20:43:03 -0500 Subject: [PATCH 091/191] Deltask increments --- include/labstor/api/labstor_client.h | 14 ++++++++++++-- include/labstor/api/labstor_runtime.h | 3 +-- include/labstor/task_registry/task.h | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index a7d98f267..e2adff3fb 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -140,14 +140,24 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void DelTask(TaskT *task) { // TODO(llogan): verify leak - main_alloc_->DelObj(task); + task->delcnt_++; + if (task->delcnt_ > 0) { + HELOG(kFatal, "Freed task twice: node={}, state={}. method={}", + task->task_node_, task->task_state_, task->method_) + } + // main_alloc_->DelObj(task); } /** Destroy a task */ template HSHM_ALWAYS_INLINE void DelTask(LPointer &task) { - main_alloc_->DelObjLocal(task); + task->delcnt_++; + if (task->delcnt_ > 0) { + HELOG(kFatal, "Freed task twice: node={}, state={}. method={}", + task->task_node_, task->task_state_, task->method_) + } + // main_alloc_->DelObjLocal(task); } /** Get a queue by its ID */ diff --git a/include/labstor/api/labstor_runtime.h b/include/labstor/api/labstor_runtime.h index 2465b9d4c..76f4525e6 100644 --- a/include/labstor/api/labstor_runtime.h +++ b/include/labstor/api/labstor_runtime.h @@ -135,9 +135,8 @@ class Runtime : public ConfigurationManager { mem_mngr->CreateBackend( server_config_.queue_manager_.shm_size_, server_config_.queue_manager_.shm_name_); - // hipc::ScalablePageAllocator main_alloc_ = - mem_mngr->CreateAllocator( + mem_mngr->CreateAllocator( server_config_.queue_manager_.shm_name_, main_alloc_id_, sizeof(LabstorShm)); diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 19a834109..d9871b876 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -256,6 +256,7 @@ struct Task : public hipc::ShmContainer { u32 lane_hash_; /**< Determine the lane a task is keyed to */ u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ + std::atomic delcnt_ = 0; /**< # of times deltask called */ RunContext ctx_; /**==================================== From 6639d7eca0fdd82ffc69318f21c73e19f804ce0b Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 21:21:45 -0500 Subject: [PATCH 092/191] Make delete counter more accurate --- include/labstor/api/labstor_client.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index e2adff3fb..518bcae26 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -141,9 +141,9 @@ class Client : public ConfigurationManager { void DelTask(TaskT *task) { // TODO(llogan): verify leak task->delcnt_++; - if (task->delcnt_ > 0) { - HELOG(kFatal, "Freed task twice: node={}, state={}. method={}", - task->task_node_, task->task_state_, task->method_) + if (task->delcnt_ > 1) { + HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", + task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } // main_alloc_->DelObj(task); } @@ -153,9 +153,9 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void DelTask(LPointer &task) { task->delcnt_++; - if (task->delcnt_ > 0) { - HELOG(kFatal, "Freed task twice: node={}, state={}. method={}", - task->task_node_, task->task_state_, task->method_) + if (task->delcnt_ > 1) { + HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", + task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } // main_alloc_->DelObjLocal(task); } From e77c8a45313af61403510b531e18f7dc3a149309 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 27 Sep 2023 23:51:51 -0500 Subject: [PATCH 093/191] Add Del method to TaskState --- codegen/refresh_methods | 14 ++++ include/labstor/api/labstor_client.h | 14 ++++ include/labstor/task_registry/task_lib.h | 3 + include/labstor/work_orchestrator/worker.h | 6 +- src/worker.cc | 4 +- tasks/bdev/include/bdev/bdev_lib_exec.h | 37 ++++++++++ .../hermes_adapters_lib_exec.h | 17 +++++ .../hermes_blob_mdm_lib_exec.h | 73 +++++++++++++++++++ .../hermes_bucket_mdm_lib_exec.h | 65 +++++++++++++++++ .../include/hermes_mdm/hermes_mdm_lib_exec.h | 13 ++++ .../include/TASK_NAME/TASK_NAME_lib_exec.h | 17 +++++ .../labstor_admin/labstor_admin_lib_exec.h | 41 +++++++++++ .../include/proc_queue/proc_queue_lib_exec.h | 17 +++++ .../remote_queue/remote_queue_lib_exec.h | 17 +++++ .../remote_queue/src/remote_queue.cc | 2 +- .../small_message/small_message_lib_exec.h | 25 +++++++ .../worch_proc_round_robin_lib_exec.h | 17 +++++ .../worch_queue_round_robin_lib_exec.h | 17 +++++ 18 files changed, 393 insertions(+), 6 deletions(-) diff --git a/codegen/refresh_methods b/codegen/refresh_methods index 404f6a705..5082431da 100755 --- a/codegen/refresh_methods +++ b/codegen/refresh_methods @@ -67,6 +67,20 @@ def refresh_methods(TASK_ROOT): lines += [' }'] lines += ['}'] + ## Create the Del method + lines += ['/** Delete a task */', + 'void Del(u32 method, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' LABSTOR_CLIENT->DelTask(reinterpret_cast<{task_name} *>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + ## Create the ReplicateStart method lines += ['/** Ensure there is space to store replicated outputs */', 'void ReplicateStart(u32 method, u32 count, Task *task) override {', diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 518bcae26..1ecec78ac 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -160,6 +160,20 @@ class Client : public ConfigurationManager { // main_alloc_->DelObjLocal(task); } + /** Destroy a task */ + template + HSHM_ALWAYS_INLINE + void DelTask(TaskStateT *exec, TaskT *task) { + exec->Del(task->method_, task); + } + + /** Destroy a task */ + template + HSHM_ALWAYS_INLINE + void DelTask(TaskStateT *exec, LPointer &task) { + exec->Del(task->method_, task); + } + /** Get a queue by its ID */ HSHM_ALWAYS_INLINE MultiQueue* GetQueue(const QueueId &queue_id) { diff --git a/include/labstor/task_registry/task_lib.h b/include/labstor/task_registry/task_lib.h index acf82efad..38166c1bf 100644 --- a/include/labstor/task_registry/task_lib.h +++ b/include/labstor/task_registry/task_lib.h @@ -79,6 +79,9 @@ class TaskLib { /** Run a method of the task */ virtual void Run(u32 method, Task *task, RunContext &ctx) = 0; + /** Delete a task */ + virtual void Del(u32 method, Task *task) = 0; + /** Allow task to store replicas of completion */ virtual void ReplicateStart(u32 method, u32 count, Task *task) = 0; diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 23555bd31..4f60cfd24 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -345,10 +345,10 @@ class Worker { } HSHM_ALWAYS_INLINE - void EndTask(Lane *lane, Task *task, int &off) { + void EndTask(Lane *lane, TaskState *exec, Task *task, int &off) { PopTask(lane, off); - if (task->IsFireAndForget()) { - LABSTOR_CLIENT->DelTask(task); + if (exec && task->IsFireAndForget()) { + LABSTOR_CLIENT->DelTask(exec, task); } else { task->SetComplete(); } diff --git a/src/worker.cc b/src/worker.cc index 491fdd29f..83eba8c7a 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -64,7 +64,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { HELOG(kFatal, "(node {}) Could not find the task state: {}", LABSTOR_CLIENT->node_id_, task->task_state_); entry->complete_ = true; - EndTask(lane, task, off); + EndTask(lane, exec, task, off); continue; } // Attempt to run the task if it's ready and runnable @@ -110,7 +110,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { ABT_thread_join(entry->thread_); } RemoveTaskGroup(task, exec, work_entry.lane_id_, is_remote); - EndTask(lane, task, off); + EndTask(lane, exec, task, off); } else { off += 1; } diff --git a/tasks/bdev/include/bdev/bdev_lib_exec.h b/tasks/bdev/include/bdev/bdev_lib_exec.h index 943c61ed2..09d254b63 100644 --- a/tasks/bdev/include/bdev/bdev_lib_exec.h +++ b/tasks/bdev/include/bdev/bdev_lib_exec.h @@ -38,6 +38,43 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kWrite: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kRead: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kAllocate: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kFree: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kMonitor: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kUpdateCapacity: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h index 3ca8e4e03..14487c641 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h @@ -18,6 +18,23 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index 55f37fe6d..82d11fb4a 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -74,6 +74,79 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kPutBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kTruncateBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestroyBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kTagBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kBlobHasTag: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetBlobId: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateBlobId: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetBlobName: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetBlobSize: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetBlobScore: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetBlobBuffers: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kRenameBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kReorganizeBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kSetBucketMdm: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index def2f8430..e6049800e 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -66,6 +66,71 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTag: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetTagId: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetTagName: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kRenameTag: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestroyTag: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kTagAddBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kTagRemoveBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kTagClearBlobs: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kUpdateSize: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kAppendBlobSchema: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kAppendBlob: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetSize: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kSetBlobMdm: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h index 2d5c8cd7b..3e63ce550 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h @@ -14,6 +14,19 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h index 95ba64a49..3d5dec399 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h @@ -18,6 +18,23 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h index 0bcdc2d02..67ef8580a 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h @@ -42,6 +42,47 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kCreateTaskState: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskState: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kRegisterTaskLib: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestroyTaskLib: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetOrCreateTaskStateId: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kGetTaskStateId: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kStopRuntime: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kSetWorkOrchProcPolicy: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h index f7f4ef23f..d1e9d9f76 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h @@ -18,6 +18,23 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kPush: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h index 1bb6e92a9..d7cf4da16 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h @@ -18,6 +18,23 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kPush: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index b0718a7c7..1b7b03f3c 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - LABSTOR_CLIENT->DelTask(orig_task); + LABSTOR_CLIENT->DelTask(exec, orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/tasks_required/small_message/include/small_message/small_message_lib_exec.h index 68aab730b..d8506c79c 100644 --- a/tasks_required/small_message/include/small_message/small_message_lib_exec.h +++ b/tasks_required/small_message/include/small_message/small_message_lib_exec.h @@ -26,6 +26,31 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kMd: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kIo: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kMdPush: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h index 21d3493ee..da2bbf2ad 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h @@ -18,6 +18,23 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h index 9223e7f1e..badbb309e 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h @@ -18,6 +18,23 @@ void Run(u32 method, Task *task, RunContext &ctx) override { } } } +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kSchedule: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { From 69763139f3311bc73bbbd7fceab2044bec9cbdc5 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 00:04:06 -0500 Subject: [PATCH 094/191] Inequal one? --- include/labstor/api/labstor_client.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 1ecec78ac..d70845639 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -141,7 +141,7 @@ class Client : public ConfigurationManager { void DelTask(TaskT *task) { // TODO(llogan): verify leak task->delcnt_++; - if (task->delcnt_ > 1) { + if (task->delcnt_ != 1) { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } @@ -153,7 +153,7 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void DelTask(LPointer &task) { task->delcnt_++; - if (task->delcnt_ > 1) { + if (task->delcnt_ != 1) { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } From 073266e8a00fe90639760d015c4319fe0a344188 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 00:06:21 -0500 Subject: [PATCH 095/191] Uncomment DelObj for task ptr --- include/labstor/api/labstor_client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index d70845639..b24e5fdf2 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -145,7 +145,7 @@ class Client : public ConfigurationManager { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } - // main_alloc_->DelObj(task); + main_alloc_->DelObj(task); } /** Destroy a task */ From 029b79b89319aac6f631a03913784f0a9f2b55ac Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 00:14:29 -0500 Subject: [PATCH 096/191] Change TaskPointer to LPointer --- codegen/refresh_methods | 4 +- include/labstor/task_registry/task_lib.h | 40 +---------- tasks/bdev/include/bdev/bdev_lib_exec.h | 32 ++++----- .../hermes_adapters_lib_exec.h | 12 ++-- .../hermes_blob_mdm_lib_exec.h | 68 +++++++++---------- .../hermes_bucket_mdm_lib_exec.h | 60 ++++++++-------- .../include/hermes_mdm/hermes_mdm_lib_exec.h | 8 +-- .../include/TASK_NAME/TASK_NAME_lib_exec.h | 12 ++-- .../labstor_admin/labstor_admin_lib_exec.h | 36 +++++----- .../include/proc_queue/proc_queue_lib_exec.h | 12 ++-- .../remote_queue/remote_queue_lib_exec.h | 12 ++-- .../remote_queue/src/remote_queue.cc | 4 +- .../small_message/small_message_lib_exec.h | 20 +++--- .../worch_proc_round_robin_lib_exec.h | 12 ++-- .../worch_queue_round_robin_lib_exec.h | 12 ++-- 15 files changed, 153 insertions(+), 191 deletions(-) diff --git a/codegen/refresh_methods b/codegen/refresh_methods index 5082431da..b10e94ed5 100755 --- a/codegen/refresh_methods +++ b/codegen/refresh_methods @@ -133,8 +133,8 @@ def refresh_methods(TASK_ROOT): method_name = method_enum_name.replace('k', '', 1) task_name = method_name + "Task" lines += [f' case Method::{method_enum_name}: {{', - f' task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask<{task_name}>(task_ptr.p_);', - f' ar >> *reinterpret_cast<{task_name}*>(task_ptr.task_);', + f' task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask<{task_name}>(task_ptr.shm_);', + f' ar >> *reinterpret_cast<{task_name}*>(task_ptr.ptr_);', f' break;', f' }}'] lines += [' }'] diff --git a/include/labstor/task_registry/task_lib.h b/include/labstor/task_registry/task_lib.h index 38166c1bf..53caedaf6 100644 --- a/include/labstor/task_registry/task_lib.h +++ b/include/labstor/task_registry/task_lib.h @@ -13,45 +13,7 @@ namespace labstor { -struct TaskPointer { - Task *task_; - hipc::Pointer p_; - - /** Default constructor */ - TaskPointer() : task_(nullptr) {} - - /** Task-only constructor */ - TaskPointer(Task *task) : task_(task) {} - - /** Emplace constructor */ - TaskPointer(Task *task, hipc::Pointer p) : task_(task), p_(p) {} - - /** Copy constructor */ - TaskPointer(const TaskPointer &other) : task_(other.task_), p_(other.p_) {} - - /** Copy operator */ - TaskPointer &operator=(const TaskPointer &other) { - task_ = other.task_; - p_ = other.p_; - return *this; - } - - /** Move constructor */ - TaskPointer(TaskPointer &&other) noexcept - : task_(other.task_), p_(other.p_) { - other.task_ = nullptr; - other.p_ = hipc::Pointer(); - } - - /** Move operator */ - TaskPointer &operator=(TaskPointer &&other) noexcept { - task_ = other.task_; - p_ = other.p_; - other.task_ = nullptr; - other.p_ = hipc::Pointer(); - return *this; - } -}; +typedef LPointer TaskPointer; /** * Represents a custom operation to perform. diff --git a/tasks/bdev/include/bdev/bdev_lib_exec.h b/tasks/bdev/include/bdev/bdev_lib_exec.h index 09d254b63..d130e634e 100644 --- a/tasks/bdev/include/bdev/bdev_lib_exec.h +++ b/tasks/bdev/include/bdev/bdev_lib_exec.h @@ -192,43 +192,43 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kWrite: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRead: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kAllocate: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kFree: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kMonitor: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kUpdateCapacity: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h index 14487c641..49e23ba6b 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h @@ -92,18 +92,18 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kCustom: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index 82d11fb4a..49b80c9ac 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -372,88 +372,88 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPutBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTruncateBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kBlobHasTag: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobId: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetOrCreateBlobId: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobName: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobSize: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobScore: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobBuffers: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRenameBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kReorganizeBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetBucketMdm: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index e6049800e..8e2a1d03e 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -332,78 +332,78 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetOrCreateTag: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetTagId: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetTagName: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRenameTag: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyTag: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagAddBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagRemoveBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagClearBlobs: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kUpdateSize: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kAppendBlobSchema: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kAppendBlob: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetSize: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetBlobMdm: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h index 3e63ce550..105572c86 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h @@ -72,13 +72,13 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h index 3d5dec399..5b9af6f98 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h @@ -92,18 +92,18 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kCustom: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h index 67ef8580a..53fac0830 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h @@ -212,48 +212,48 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kCreateTaskState: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyTaskState: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRegisterTaskLib: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyTaskLib: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetOrCreateTaskStateId: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetTaskStateId: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kStopRuntime: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetWorkOrchQueuePolicy: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetWorkOrchProcPolicy: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h index d1e9d9f76..c3fffff4b 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h @@ -92,18 +92,18 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPush: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h index d7cf4da16..6d5ef8721 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h @@ -92,18 +92,18 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPush: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 1b7b03f3c..917a6d58a 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -241,8 +241,8 @@ class Server : public TaskLib { state_id); } TaskPointer task_ptr = exec->LoadStart(method, ar); - orig_task = task_ptr.task_; - hipc::Pointer &p = task_ptr.p_; + orig_task = task_ptr.ptr_; + hipc::Pointer &p = task_ptr.shm_; orig_task->domain_id_ = DomainId::GetNode(LABSTOR_CLIENT->node_id_); // Execute task diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/tasks_required/small_message/include/small_message/small_message_lib_exec.h index d8506c79c..ae08efc87 100644 --- a/tasks_required/small_message/include/small_message/small_message_lib_exec.h +++ b/tasks_required/small_message/include/small_message/small_message_lib_exec.h @@ -132,28 +132,28 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kMd: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kIo: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kMdPush: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h index da2bbf2ad..5d47d3c7d 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h @@ -92,18 +92,18 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSchedule: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h index badbb309e..2e6a1aec9 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h @@ -92,18 +92,18 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSchedule: { - task_ptr.task_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.p_); - ar >> *reinterpret_cast(task_ptr.task_); + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } From cd773655cc713623a481f6499d4854ec6c7d3f93 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 00:57:14 -0500 Subject: [PATCH 097/191] Try running it using PutTask as coroutingg --- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 10 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 189 ++++++------------ 2 files changed, 62 insertions(+), 137 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 037abaa9c..82f9f2a08 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -224,15 +224,6 @@ struct PutBlobTask : public Task, TaskFlags IN hipc::ShmArchive filename_; IN size_t page_size_; - TEMP int phase_ = PutBlobPhase::kCreate; - TEMP int plcmnt_idx_ = 0; - TEMP int sub_plcmnt_idx_ = 0; - TEMP LPointer data_ptr_; - TEMP size_t data_off_; - TEMP hermes::bdev::AllocateTask *cur_bdev_alloc_; - TEMP hipc::ShmArchive> schema_; - TEMP hipc::ShmArchive> bdev_writes_; - /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit PutBlobTask(hipc::Allocator *alloc) : Task(alloc) {} @@ -259,6 +250,7 @@ struct PutBlobTask : public Task, TaskFlags task_state_ = state_id; method_ = Method::kPutBlob; task_flags_ = task_flags; + task_flags_.SetBits(TASK_COROUTINE); if (!blob_id.IsNull()) { lane_hash_ = blob_id.hash_; domain_id_ = domain_id; diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index df2077126..9ad9b3c4b 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -116,33 +116,6 @@ class Server : public TaskLib { * Create a blob's metadata * */ void PutBlob(PutBlobTask *task, RunContext &ctx) { - if (task->phase_ == PutBlobPhase::kCreate) { - PutBlobCreatePhase(task, ctx); - } - if (task->phase_ == PutBlobPhase::kAllocate) { - PutBlobAllocatePhase(task, ctx); - } - if (task->phase_ == PutBlobPhase::kWaitAllocate) { - if (!task->cur_bdev_alloc_->IsComplete()){ - return; - } - PutBlobWaitAllocatePhase(task, ctx); - } - if (task->phase_ == PutBlobPhase::kModify) { - PutBlobModifyPhase(task, ctx); - } - if (task->phase_ == PutBlobPhase::kWaitModify) { - PutBlobWaitModifyPhase(task, ctx); - if (task->phase_ == PutBlobPhase::kWaitModify) { - return; - } - } - } - - /** Create blob / update metadata for the PUT */ - void PutBlobCreatePhase(PutBlobTask *task, RunContext &ctx) { - HILOG(kDebug, "PutBlobPhase::kCreate {}", task->blob_id_); - // Get the blob info data structure hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); if (task->blob_id_.IsNull()) { @@ -174,13 +147,15 @@ class Server : public TaskLib { } // Stage in blob data from FS - task->data_ptr_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->data_); - task->data_ptr_.shm_ = task->data_; + LPointer data_ptr; + data_ptr.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->data_); + data_ptr.shm_ = task->data_; + size_t data_off = 0; if (task->filename_->size() > 0) { adapter::BlobPlacement plcmnt; plcmnt.DecodeBlobName(*task->blob_name_); task->flags_.SetBits(HERMES_IS_FILE); - task->data_off_ = plcmnt.bucket_off_ + task->blob_off_ + task->data_size_; + data_off = plcmnt.bucket_off_ + task->blob_off_ + task->data_size_; if (blob_info.blob_size_ == 0 && task->blob_off_ == 0 && task->data_size_ < task->page_size_) { @@ -197,8 +172,8 @@ class Server : public TaskLib { HELOG(kError, "Failed to stage in {} bytes from {}", task->page_size_, task->filename_->str()); } HERMES_POSIX_API->close(fd); - memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, task->data_ptr_.ptr_, task->data_size_); - task->data_ptr_ = new_data_ptr; + memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, data_ptr.ptr_, task->data_size_); + data_ptr = new_data_ptr; task->blob_off_ = 0; if (ret < task->blob_off_ + task->data_size_) { task->data_size_ = task->blob_off_ + task->data_size_; @@ -218,87 +193,44 @@ class Server : public TaskLib { size_diff = needed_space - blob_info.max_blob_size_; } if (!task->flags_.Any(HERMES_IS_FILE)) { - task->data_off_ = size_diff; + data_off = size_diff; } blob_info.blob_size_ += size_diff; HILOG(kDebug, "The size diff is {} bytes", size_diff) - - // Initialize archives - HSHM_MAKE_AR0(task->schema_, nullptr); - HSHM_MAKE_AR0(task->bdev_writes_, nullptr); - // Use DPE + std::vector schema_vec; if (size_diff > 0) { Context ctx; auto *dpe = DpeFactory::Get(ctx.dpe_); - dpe->Placement({size_diff}, targets_, ctx, *task->schema_); - task->phase_ = PutBlobPhase::kAllocate; - } else { - task->phase_ = PutBlobPhase::kModify; - } - } - - /** Release buffers */ - void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task, RunContext &ctx) { - for (BufferInfo &buf : blob_info.buffers_) { - TargetInfo &target = *target_map_[buf.tid_]; - std::vector buf_vec = {buf}; - target.AsyncFree(task->task_node_ + 1, std::move(buf_vec), true); - } - blob_info.buffers_.clear(); - blob_info.max_blob_size_ = 0; - blob_info.blob_size_ = 0; - } - - /** Resolve the current sub-placement using BPM */ - void PutBlobAllocatePhase(PutBlobTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; - BlobInfo &blob_info = blob_map[task->blob_id_]; - PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; - SubPlacement &placement = schema.plcmnts_[task->sub_plcmnt_idx_]; - TargetInfo &bdev = *target_map_[placement.tid_]; - HILOG(kDebug, "Allocating {} bytes of blob {}", placement.size_, task->blob_id_); - task->cur_bdev_alloc_ = bdev.AsyncAllocate(task->task_node_ + 1, - placement.size_, - blob_info.buffers_).ptr_; - task->phase_ = PutBlobPhase::kWaitAllocate; - } - - /** Wait for the current-subplacement to complete */ - void PutBlobWaitAllocatePhase(PutBlobTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; - BlobInfo &blob_info = blob_map[task->blob_id_]; - PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; - ++task->sub_plcmnt_idx_; - if (task->sub_plcmnt_idx_ >= schema.plcmnts_.size()) { - ++task->plcmnt_idx_; - task->sub_plcmnt_idx_ = 0; - } - if (task->cur_bdev_alloc_->alloc_size_ < task->cur_bdev_alloc_->size_) { - size_t diff = task->cur_bdev_alloc_->size_ - task->cur_bdev_alloc_->alloc_size_; - PlacementSchema &schema = (*task->schema_)[task->plcmnt_idx_]; - SubPlacement &sub_plcmnt = schema.plcmnts_[task->sub_plcmnt_idx_]; - sub_plcmnt.size_ += diff; - HILOG(kDebug, "Passing {} bytes to target {}", diff, sub_plcmnt.tid_); - } - LABSTOR_CLIENT->DelTask(task->cur_bdev_alloc_); - if (task->plcmnt_idx_ < (*task->schema_).size()) { - task->phase_ = PutBlobPhase::kAllocate; - return; - } else { - task->phase_ = PutBlobPhase::kModify; - task->bdev_writes_->reserve(blob_info.buffers_.size()); + dpe->Placement({size_diff}, targets_, ctx, schema_vec); + } + + // Allocate blob buffers + for (PlacementSchema &schema : schema_vec) { + for (size_t sub_idx = 0; sub_idx < schema.plcmnts_.size(); ++sub_idx) { + SubPlacement &placement = schema.plcmnts_[sub_idx]; + TargetInfo &bdev = *target_map_[placement.tid_]; + LPointer alloc_task = + bdev.AsyncAllocate(task->task_node_ + 1, + placement.size_, + blob_info.buffers_); + alloc_task->Wait(task); + if (alloc_task->alloc_size_ < alloc_task->size_) { + // SubPlacement &next_placement = schema.plcmnts_[sub_idx + 1]; + // size_t diff = alloc_task->size_ - alloc_task->alloc_size_; + // next_placement.size_ += diff; + HILOG(kFatal, "Ran outta space in this tier -- will fix soon") + } + LABSTOR_CLIENT->DelTask(alloc_task); + } } - } - /** Update the data on storage */ - void PutBlobModifyPhase(PutBlobTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; - BlobInfo &blob_info = blob_map[task->blob_id_]; - char *blob_buf = task->data_ptr_.ptr_; - std::vector &write_tasks = *task->bdev_writes_; + // Place blob in buffers + std::vector> write_tasks; + write_tasks.reserve(blob_info.buffers_.size()); size_t blob_off = 0, buf_off = 0; + char *blob_buf = data_ptr.ptr_; HILOG(kDebug, "Number of buffers {}", blob_info.buffers_.size()); for (BufferInfo &buf : blob_info.buffers_) { size_t blob_left = blob_off; @@ -312,56 +244,57 @@ class Server : public TaskLib { } HILOG(kDebug, "Writing {} bytes at off {} from target {}", buf_size, tgt_off, buf.tid_) TargetInfo &target = *target_map_[buf.tid_]; - bdev::WriteTask *write_task = target.AsyncWrite(task->task_node_ + 1, - blob_buf + buf_off, - tgt_off, buf_size).ptr_; + LPointer write_task = + target.AsyncWrite(task->task_node_ + 1, + blob_buf + buf_off, + tgt_off, buf_size); write_tasks.emplace_back(write_task); buf_off += buf_size; } blob_off += buf.t_size_; } - if (blob_off < task->data_size_) { - HELOG(kFatal, "Something annoying happened"); - } blob_info.max_blob_size_ = blob_off; - task->phase_ = PutBlobPhase::kWaitModify; - HILOG(kDebug, "Modified {} bytes of blob {}", blob_off, task->blob_id_); - } - /** Wait for the update to complete */ - void PutBlobWaitModifyPhase(PutBlobTask *task, RunContext &ctx) { - std::vector &write_tasks = *task->bdev_writes_; - for (bdev::WriteTask *&write_task : write_tasks) { - if (!write_task->IsComplete()) { - return; - } - } - for (bdev::WriteTask *&write_task : write_tasks) { + // Wait for the placements to complete + for (LPointer &write_task : write_tasks) { + write_task->Wait(task); LABSTOR_CLIENT->DelTask(write_task); } - HILOG(kDebug, "PutBlobTask complete"); - HSHM_DESTROY_AR(task->schema_); - HSHM_DESTROY_AR(task->bdev_writes_); - if (task->flags_.Any(HERMES_DID_STAGE_IN)) { - LABSTOR_CLIENT->FreeBuffer(task->data_ptr_); - } - // Update the bucket statistics + + // Update information int update_mode = bucket_mdm::UpdateSizeMode::kAdd; if (task->flags_.Any(HERMES_IS_FILE)) { update_mode = bucket_mdm::UpdateSizeMode::kCap; } bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, task->tag_id_, - task->data_off_, + data_off, update_mode); if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, task->tag_id_, task->blob_id_); } + + // Free data + if (task->flags_.Any(HERMES_DID_STAGE_IN)) { + LABSTOR_CLIENT->FreeBuffer(data_ptr); + } task->SetModuleComplete(); } + /** Release buffers */ + void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task, RunContext &ctx) { + for (BufferInfo &buf : blob_info.buffers_) { + TargetInfo &target = *target_map_[buf.tid_]; + std::vector buf_vec = {buf}; + target.AsyncFree(task->task_node_ + 1, std::move(buf_vec), true); + } + blob_info.buffers_.clear(); + blob_info.max_blob_size_ = 0; + blob_info.blob_size_ = 0; + } + /** Get a blob's data */ void GetBlob(GetBlobTask *task, RunContext &ctx) { switch (task->phase_) { From 3acb1b87d1679ba1851e92dc729f6933810918e3 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:16:03 -0500 Subject: [PATCH 098/191] Don't free sync put blob --- tasks/hermes/include/hermes/bucket.h | 12 ++++++------ .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 3c13ffe30..60b21bea2 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -236,12 +236,12 @@ class Bucket { blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, flags, ctx, task_flags); if constexpr (!ASYNC) { - if (flags.Any(HERMES_GET_BLOB_ID)) { - push_task->Wait(); - PutBlobTask *task = push_task->get(); - blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); - } +// if (flags.Any(HERMES_GET_BLOB_ID)) { +// push_task->Wait(); +// PutBlobTask *task = push_task->get(); +// blob_id = task->blob_id_; +// LABSTOR_CLIENT->DelTask(push_task); +// } } return blob_id; } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 82f9f2a08..35740abb9 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -276,11 +276,11 @@ struct PutBlobTask : public Task, TaskFlags /** Destructor */ ~PutBlobTask() { -// HSHM_DESTROY_AR(blob_name_); -// HSHM_DESTROY_AR(filename_); -// if (IsDataOwner()) { -// LABSTOR_CLIENT->FreeBuffer(data_); -// } + HSHM_DESTROY_AR(blob_name_); + HSHM_DESTROY_AR(filename_); + if (IsDataOwner()) { + LABSTOR_CLIENT->FreeBuffer(data_); + } } /** (De)serialize message call */ From 8dc94690606e6104936c9ba3bfbaccc78844a851 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:20:27 -0500 Subject: [PATCH 099/191] PushTask wait --- tasks/hermes/include/hermes/bucket.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 60b21bea2..bbdded1b4 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -236,12 +236,12 @@ class Bucket { blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, flags, ctx, task_flags); if constexpr (!ASYNC) { -// if (flags.Any(HERMES_GET_BLOB_ID)) { -// push_task->Wait(); -// PutBlobTask *task = push_task->get(); -// blob_id = task->blob_id_; -// LABSTOR_CLIENT->DelTask(push_task); -// } + if (flags.Any(HERMES_GET_BLOB_ID)) { + push_task->Wait(); + PutBlobTask *task = push_task->get(); + blob_id = task->blob_id_; + // LABSTOR_CLIENT->DelTask(push_task); + } } return blob_id; } From 353ef38c50b93cab236fe7d31249566007de94d4 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:29:32 -0500 Subject: [PATCH 100/191] Freeing task print --- include/labstor/api/labstor_client.h | 2 ++ tasks/hermes/include/hermes/bucket.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index b24e5fdf2..a30254ca2 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -145,6 +145,8 @@ class Client : public ConfigurationManager { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } + HILOG(kInfo, "Freeing task_node={} task_state={} task_method={}", + task->task_node_, task->task_state_, task->method_) main_alloc_->DelObj(task); } diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index bbdded1b4..3c13ffe30 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -240,7 +240,7 @@ class Bucket { push_task->Wait(); PutBlobTask *task = push_task->get(); blob_id = task->blob_id_; - // LABSTOR_CLIENT->DelTask(push_task); + LABSTOR_CLIENT->DelTask(push_task); } } return blob_id; From 01e970a9364e5684d5cb0d3ad28dad64c364920f Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:37:08 -0500 Subject: [PATCH 101/191] Comment out put blob task --- .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 35740abb9..82f9f2a08 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -276,11 +276,11 @@ struct PutBlobTask : public Task, TaskFlags /** Destructor */ ~PutBlobTask() { - HSHM_DESTROY_AR(blob_name_); - HSHM_DESTROY_AR(filename_); - if (IsDataOwner()) { - LABSTOR_CLIENT->FreeBuffer(data_); - } +// HSHM_DESTROY_AR(blob_name_); +// HSHM_DESTROY_AR(filename_); +// if (IsDataOwner()) { +// LABSTOR_CLIENT->FreeBuffer(data_); +// } } /** (De)serialize message call */ From d60beacb43e04371a6da4a0cee96cef817c1414c Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:44:14 -0500 Subject: [PATCH 102/191] Don't do fire & forget frees --- include/labstor/work_orchestrator/worker.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 4f60cfd24..c56525368 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -348,7 +348,7 @@ class Worker { void EndTask(Lane *lane, TaskState *exec, Task *task, int &off) { PopTask(lane, off); if (exec && task->IsFireAndForget()) { - LABSTOR_CLIENT->DelTask(exec, task); + // LABSTOR_CLIENT->DelTask(exec, task); } else { task->SetComplete(); } From 68b2a4633141b3967efdfe084a14b020cd1abec7 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:49:12 -0500 Subject: [PATCH 103/191] Remove a few deltasks from hermes_blob_mdm --- include/labstor/work_orchestrator/worker.h | 2 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index c56525368..4f60cfd24 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -348,7 +348,7 @@ class Worker { void EndTask(Lane *lane, TaskState *exec, Task *task, int &off) { PopTask(lane, off); if (exec && task->IsFireAndForget()) { - // LABSTOR_CLIENT->DelTask(exec, task); + LABSTOR_CLIENT->DelTask(exec, task); } else { task->SetComplete(); } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 9ad9b3c4b..668fe461d 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -258,7 +258,7 @@ class Server : public TaskLib { // Wait for the placements to complete for (LPointer &write_task : write_tasks) { write_task->Wait(task); - LABSTOR_CLIENT->DelTask(write_task); + // LABSTOR_CLIENT->DelTask(write_task); } // Update information @@ -278,7 +278,7 @@ class Server : public TaskLib { // Free data if (task->flags_.Any(HERMES_DID_STAGE_IN)) { - LABSTOR_CLIENT->FreeBuffer(data_ptr); + // LABSTOR_CLIENT->FreeBuffer(data_ptr); } task->SetModuleComplete(); } @@ -288,7 +288,7 @@ class Server : public TaskLib { for (BufferInfo &buf : blob_info.buffers_) { TargetInfo &target = *target_map_[buf.tid_]; std::vector buf_vec = {buf}; - target.AsyncFree(task->task_node_ + 1, std::move(buf_vec), true); + // target.AsyncFree(task->task_node_ + 1, std::move(buf_vec), true); } blob_info.buffers_.clear(); blob_info.max_blob_size_ = 0; From efaf2ce056bcae5e225f1b8e1db4b499bb2ad72d Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:51:27 -0500 Subject: [PATCH 104/191] Remove blob registration and bucket updates --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 668fe461d..d3cb3f667 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -266,15 +266,15 @@ class Server : public TaskLib { if (task->flags_.Any(HERMES_IS_FILE)) { update_mode = bucket_mdm::UpdateSizeMode::kCap; } - bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, - task->tag_id_, - data_off, - update_mode); - if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { - bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, - task->tag_id_, - task->blob_id_); - } +// bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, +// task->tag_id_, +// data_off, +// update_mode); +// if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { +// bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, +// task->tag_id_, +// task->blob_id_); +// } // Free data if (task->flags_.Any(HERMES_DID_STAGE_IN)) { From bfb1531344eedb5512b368eafbfaac404a1e8051 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:53:11 -0500 Subject: [PATCH 105/191] Remove deltsk from remote queue --- tasks_required/remote_queue/src/remote_queue.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 917a6d58a..ac6c4768a 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - LABSTOR_CLIENT->DelTask(exec, orig_task); + // LABSTOR_CLIENT->DelTask(exec, orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } From 40bf8f1af6008c0c36d3f9e6fd203c3d42e8a9b4 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 01:58:22 -0500 Subject: [PATCH 106/191] Remove del of allocts --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index d3cb3f667..69ba7b882 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -222,7 +222,7 @@ class Server : public TaskLib { // next_placement.size_ += diff; HILOG(kFatal, "Ran outta space in this tier -- will fix soon") } - LABSTOR_CLIENT->DelTask(alloc_task); + // LABSTOR_CLIENT->DelTask(alloc_task); } } From 50fe5902d833bccc6a7f0d669b237c30c9d23191 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 02:01:00 -0500 Subject: [PATCH 107/191] ensure its not stack overflow --- include/labstor/task_registry/task.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index d9871b876..ed532b88c 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -234,7 +234,7 @@ class TaskPrio { struct RunContext { u32 lane_id_; /**< The lane id of the task */ bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ - size_t stack_size_ = KILOBYTES(64); /**< The size of the stack for the task (runtime) */ + size_t stack_size_ = KILOBYTES(256); /**< The size of the stack for the task (runtime) */ void *stack_ptr_; /**< The pointer to the stack (runtime) */ TaskLib *exec_; From 7f93491d263d622f5e2387431e029b162c4787b4 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 12:27:57 -0500 Subject: [PATCH 108/191] Place blob in buffers --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 69ba7b882..b532ed9d5 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -222,7 +222,7 @@ class Server : public TaskLib { // next_placement.size_ += diff; HILOG(kFatal, "Ran outta space in this tier -- will fix soon") } - // LABSTOR_CLIENT->DelTask(alloc_task); + LABSTOR_CLIENT->DelTask(alloc_task); } } @@ -258,7 +258,7 @@ class Server : public TaskLib { // Wait for the placements to complete for (LPointer &write_task : write_tasks) { write_task->Wait(task); - // LABSTOR_CLIENT->DelTask(write_task); + LABSTOR_CLIENT->DelTask(write_task); } // Update information @@ -266,19 +266,19 @@ class Server : public TaskLib { if (task->flags_.Any(HERMES_IS_FILE)) { update_mode = bucket_mdm::UpdateSizeMode::kCap; } -// bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, -// task->tag_id_, -// data_off, -// update_mode); -// if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { -// bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, -// task->tag_id_, -// task->blob_id_); -// } + bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, + task->tag_id_, + data_off, + update_mode); + if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { + bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, + task->tag_id_, + task->blob_id_); + } // Free data if (task->flags_.Any(HERMES_DID_STAGE_IN)) { - // LABSTOR_CLIENT->FreeBuffer(data_ptr); + LABSTOR_CLIENT->FreeBuffer(data_ptr); } task->SetModuleComplete(); } From 2560369b6cb9d66cbe069c0f5546726e54f7c182 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 12:30:25 -0500 Subject: [PATCH 109/191] Support all deltasks --- include/labstor/api/labstor_client.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index a30254ca2..03be87336 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -145,8 +145,6 @@ class Client : public ConfigurationManager { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } - HILOG(kInfo, "Freeing task_node={} task_state={} task_method={}", - task->task_node_, task->task_state_, task->method_) main_alloc_->DelObj(task); } @@ -159,7 +157,7 @@ class Client : public ConfigurationManager { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } - // main_alloc_->DelObjLocal(task); + main_alloc_->DelObjLocal(task); } /** Destroy a task */ From 6568e4e5a3d75b6de8df9cb326e834bac564b10d Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 12:59:23 -0500 Subject: [PATCH 110/191] Slightly change access order in remote queue --- .../remote_queue/include/remote_queue/remote_queue.h | 2 +- tasks_required/remote_queue/src/remote_queue.cc | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index f0cf88119..fe9b31c3d 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -72,7 +72,7 @@ class Client : public TaskLibClient { // Create subtasks exec->ReplicateStart(orig_task->method_, domain_ids.size(), orig_task); - auto push_task = LABSTOR_CLIENT->NewTask( + LPointer push_task = LABSTOR_CLIENT->NewTask( orig_task->task_node_ + 1, DomainId::GetLocal(), id_, domain_ids, orig_task, exec, orig_task->method_, xfer); MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index ac6c4768a..6c0a26a8c 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -50,7 +50,7 @@ class Server : public TaskLib { } /** Handle output from replica PUSH */ - static void HandlePushReplicaOutput(int replica, std::string &ret, PushTask *task) { + static void ClientHandlePushReplicaOutput(int replica, std::string &ret, PushTask *task) { std::vector xfer(1); xfer[0].data_ = ret.data(); xfer[0].data_size_ = ret.size(); @@ -64,13 +64,13 @@ class Server : public TaskLib { } /** Handle finalization of PUSH replicate */ - static void HandlePushReplicaEnd(PushTask *task) { + static void ClientHandlePushReplicaEnd(PushTask *task) { task->exec_->ReplicateEnd(task->orig_task_->method_, task->orig_task_); - task->orig_task_->SetModuleComplete(); HILOG(kDebug, "Completing task (task_node={}, task_state={}, method={})", task->orig_task_->task_node_, task->orig_task_->task_state_, task->orig_task_->method_); + task->orig_task_->SetModuleComplete(); task->SetModuleComplete(); } @@ -98,7 +98,7 @@ class Server : public TaskLib { task->orig_task_->method_, LABSTOR_CLIENT->node_id_, domain_id.id_); - HandlePushReplicaOutput(replica, ret, task); + ClientHandlePushReplicaOutput(replica, ret, task); } } @@ -139,7 +139,7 @@ class Server : public TaskLib { LABSTOR_CLIENT->node_id_, domain_id.id_, static_cast(io_type)); - HandlePushReplicaOutput(replica, ret, task); + ClientHandlePushReplicaOutput(replica, ret, task); } } @@ -159,7 +159,7 @@ class Server : public TaskLib { HELOG(kFatal, "The task {}/{} does not support remote calls", task->task_state_, task->method_); } } - HandlePushReplicaEnd(task); + ClientHandlePushReplicaEnd(task); } private: From 2d96735f52682ee41607b426f79456a8c9d6dece Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 13:01:47 -0500 Subject: [PATCH 111/191] Comment out all HSHM_DESTROY_AR --- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 26 +++++++++---------- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 4 +-- .../hermes_bucket_mdm_tasks.h | 10 +++---- .../src/hermes_bucket_mdm.cc | 4 +-- .../include/hermes_mdm/hermes_mdm_tasks.h | 2 +- .../labstor_admin/labstor_admin_tasks.h | 12 ++++----- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 82f9f2a08..be5e50f36 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -171,7 +171,7 @@ struct GetOrCreateBlobIdTask : public Task, TaskFlags { /** Destructor */ ~GetOrCreateBlobIdTask() { - HSHM_DESTROY_AR(blob_name_) + // HSHM_DESTROY_AR(blob_name_) } /** (De)serialize message call */ @@ -276,11 +276,11 @@ struct PutBlobTask : public Task, TaskFlags /** Destructor */ ~PutBlobTask() { -// HSHM_DESTROY_AR(blob_name_); -// HSHM_DESTROY_AR(filename_); -// if (IsDataOwner()) { -// LABSTOR_CLIENT->FreeBuffer(data_); -// } + // HSHM_DESTROY_AR(blob_name_); + // HSHM_DESTROY_AR(filename_); + if (IsDataOwner()) { + LABSTOR_CLIENT->FreeBuffer(data_); + } } /** (De)serialize message call */ @@ -390,8 +390,8 @@ struct GetBlobTask : public Task, TaskFlags /** Destructor */ ~GetBlobTask() { - HSHM_DESTROY_AR(blob_name_); - HSHM_DESTROY_AR(filename_); + // HSHM_DESTROY_AR(blob_name_); + // HSHM_DESTROY_AR(filename_); } /** (De)serialize message call */ @@ -586,7 +586,7 @@ struct GetBlobIdTask : public Task, TaskFlags { /** Destructor */ ~GetBlobIdTask() { - HSHM_DESTROY_AR(blob_name_) + // HSHM_DESTROY_AR(blob_name_) } /** (De)serialize message call */ @@ -649,7 +649,7 @@ struct GetBlobNameTask : public Task, TaskFlags { /** Destructor */ ~GetBlobNameTask() { - HSHM_DESTROY_AR(blob_name_) + // HSHM_DESTROY_AR(blob_name_) }; /** (De)serialize message call */ @@ -717,7 +717,7 @@ struct GetBlobSizeTask : public Task, TaskFlags { /** Destructor */ ~GetBlobSizeTask() { - HSHM_DESTROY_AR(blob_name_) + // HSHM_DESTROY_AR(blob_name_) } /** (De)serialize message call */ @@ -833,7 +833,7 @@ struct GetBlobBuffersTask : public Task, TaskFlags { /** Destructor */ ~GetBlobBuffersTask() { - HSHM_DESTROY_AR(buffers_) + // HSHM_DESTROY_AR(buffers_) } /** (De)serialize message call */ @@ -898,7 +898,7 @@ struct RenameBlobTask : public Task, TaskFlags { /** Destructor */ ~RenameBlobTask() { - HSHM_DESTROY_AR(new_blob_name_) + // HSHM_DESTROY_AR(new_blob_name_) } /** (De)serialize message call */ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index b532ed9d5..f0275a061 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -356,7 +356,7 @@ class Server : public TaskLib { for (bdev::ReadTask *&read_task : read_tasks) { LABSTOR_CLIENT->DelTask(read_task); } - HSHM_DESTROY_AR(task->bdev_reads_); + // HSHM_DESTROY_AR(task->bdev_reads_); HILOG(kDebug, "GetBlobTask complete"); task->SetModuleComplete(); } @@ -583,7 +583,7 @@ class Server : public TaskLib { task->tag_id_, -(ssize_t)blob_info.blob_size_, bucket_mdm::UpdateSizeMode::kAdd); - HSHM_DESTROY_AR(task->free_tasks_); + // HSHM_DESTROY_AR(task->free_tasks_); blob_map.erase(task->blob_id_); task->SetModuleComplete(); } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index bfb9c7060..f19aa2edb 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -375,8 +375,8 @@ struct GetOrCreateTagTask : public Task, TaskFlags { /** Destructor */ ~GetOrCreateTagTask() { - HSHM_DESTROY_AR(tag_name_) - HSHM_DESTROY_AR(traits_) + // HSHM_DESTROY_AR(tag_name_) + // HSHM_DESTROY_AR(traits_) } /** (De)serialize message call */ @@ -432,7 +432,7 @@ struct GetTagIdTask : public Task, TaskFlags { /** Destructor */ ~GetTagIdTask() { - HSHM_DESTROY_AR(tag_name_) + // HSHM_DESTROY_AR(tag_name_) } /** (De)serialize message call */ @@ -488,7 +488,7 @@ struct GetTagNameTask : public Task, TaskFlags { /** Destructor */ ~GetTagNameTask() { - HSHM_DESTROY_AR(tag_name_) + // HSHM_DESTROY_AR(tag_name_) } /** (De)serialize message call */ @@ -547,7 +547,7 @@ struct RenameTagTask : public Task, TaskFlags { /** Destructor */ ~RenameTagTask() { - HSHM_DESTROY_AR(tag_name_) + // HSHM_DESTROY_AR(tag_name_) } /** (De)serialize message call */ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 5d1a96fc0..eca36dc36 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -180,7 +180,7 @@ class Server : public TaskLib { for (AppendInfo &append : append_info) { LABSTOR_CLIENT->DelTask(append.put_task_); } - HSHM_DESTROY_AR(task->schema_->append_info_); + // HSHM_DESTROY_AR(task->schema_->append_info_); LABSTOR_CLIENT->DelTask(task->schema_); task->SetModuleComplete(); } @@ -296,7 +296,7 @@ class Server : public TaskLib { for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { LABSTOR_CLIENT->DelTask(blob_task); } - HSHM_DESTROY_AR(task->destroy_blob_tasks_); + // HSHM_DESTROY_AR(task->destroy_blob_tasks_); TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; tag_map.erase(task->tag_id_); task->SetModuleComplete(); diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h index 3a78e6de6..6d161e3be 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h @@ -47,7 +47,7 @@ struct ConstructTask : public CreateTaskStateTask { /** Destructor */ HSHM_ALWAYS_INLINE ~ConstructTask() { - HSHM_DESTROY_AR(server_config_path_); + // HSHM_DESTROY_AR(server_config_path_); } /** (De)serialize message call */ diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index dd28394e9..90a428e74 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -50,7 +50,7 @@ struct RegisterTaskLibTaskTempl : public Task, TaskFlags { /** Destructor */ ~RegisterTaskLibTaskTempl() { - HSHM_DESTROY_AR(lib_name_); + // HSHM_DESTROY_AR(lib_name_); } /** (De)serialize message call */ @@ -118,7 +118,7 @@ struct GetOrCreateTaskStateIdTask : public Task, TaskFlags { } ~GetOrCreateTaskStateIdTask() { - HSHM_DESTROY_AR(state_name_); + // HSHM_DESTROY_AR(state_name_); } /** (De)serialize message call */ @@ -179,9 +179,9 @@ struct CreateTaskStateTask : public Task, TaskFlags { /** Destructor */ ~CreateTaskStateTask() { - HSHM_DESTROY_AR(state_name_); - HSHM_DESTROY_AR(lib_name_); - HSHM_DESTROY_AR(queue_info_); + // HSHM_DESTROY_AR(state_name_); + // HSHM_DESTROY_AR(lib_name_); + // HSHM_DESTROY_AR(queue_info_); } /** Replication (does nothing) */ @@ -245,7 +245,7 @@ struct GetTaskStateIdTask : public Task, TaskFlags { } ~GetTaskStateIdTask() { - HSHM_DESTROY_AR(state_name_); + // HSHM_DESTROY_AR(state_name_); } /** (De)serialize message call */ From 0c7bb45fe329147495d2045faa2f30a0e42b46ac Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 13:16:42 -0500 Subject: [PATCH 112/191] Re-enable remote queue --- tasks_required/remote_queue/src/remote_queue.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 6c0a26a8c..2186a7a67 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - // LABSTOR_CLIENT->DelTask(exec, orig_task); + LABSTOR_CLIENT->DelTask(exec, orig_task); req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } From cb4a685a6be768a6023ab4857217a399df0e688e Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 13:24:41 -0500 Subject: [PATCH 113/191] Null check in empty task --- include/labstor/api/labstor_client.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 03be87336..e66b79179 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -92,14 +92,22 @@ class Client : public ConfigurationManager { template HSHM_ALWAYS_INLINE TaskT* NewEmptyTask(hipc::Pointer &p) { - return main_alloc_->NewObj(p, main_alloc_); + TaskT *task = main_alloc_->NewObj(p, main_alloc_); + if (task == nullptr) { + throw std::runtime_error("Could not allocate buffer"); + } + return task; } /** Allocate task */ template HSHM_ALWAYS_INLINE hipc::LPointer AllocateTask() { - return main_alloc_->AllocateLocalPtr(sizeof(TaskT)); + hipc::LPointer task = main_alloc_->AllocateLocalPtr(sizeof(TaskT)); + if (task.ptr_ == nullptr) { + throw std::runtime_error("Could not allocate buffer"); + } + return task; } /** Construct task */ From 87f7b3b246dca6c0ce4e3fa8de0d8a6b37481ba1 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 13:31:15 -0500 Subject: [PATCH 114/191] Req respond --- tasks_required/remote_queue/src/remote_queue.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 2186a7a67..2c6992641 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -279,7 +279,11 @@ class Server : public TaskLib { state_id, method); LABSTOR_CLIENT->DelTask(exec, orig_task); - req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); + if (out_xfer.size() > 0) { + req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); + } else { + req.respond(std::string()); + } } public: From e3b9f664f8f9ba0aa2721c9f0c6d461cd9aaa194 Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 28 Sep 2023 13:32:54 -0500 Subject: [PATCH 115/191] Change to debug --- tasks_required/remote_queue/src/remote_queue.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 2c6992641..28960fafa 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -253,7 +253,7 @@ class Server : public TaskLib { orig_task->UnsetDataOwner(); orig_task->UnsetLongRunning(); queue->Emplace(orig_task->prio_, orig_task->lane_hash_, p); - HILOG(kDebug, + HILOG(kInfo, "(node {}) Executing task (task_node={}, task_state={}/{}, state_name={}, method={}, size={}, lane_hash={})", LABSTOR_CLIENT->node_id_, orig_task->task_node_, @@ -271,7 +271,7 @@ class Server : public TaskLib { TaskState *exec, TaskStateId state_id) { BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); std::vector out_xfer = exec->SaveEnd(method, ar, orig_task); - HILOG(kDebug, "(node {}) Returning {} bytes of data (task_node={}, task_state={}/{}, method={})", + HILOG(kInfo, "(node {}) Returning {} bytes of data (task_node={}, task_state={}/{}, method={})", LABSTOR_CLIENT->node_id_, out_xfer[0].data_size_, orig_task->task_node_, From 22ab17c1f596714712a68e24a435c8deb59deb0b Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Fri, 29 Sep 2023 00:55:52 -0500 Subject: [PATCH 116/191] Begin remote task debug --- include/labstor/task_registry/task.h | 2 ++ src/worker.cc | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index ed532b88c..eb390a7f2 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -49,6 +49,8 @@ class TaskLib; #define TASK_COROUTINE BIT_OPT(u32, 16) /** This task uses argobot wait */ #define TASK_PREEMPTIVE BIT_OPT(u32, 17) +/** This task is apart of remote debugging */ +#define TASK_REMOTE_DEBUG_MARK BIT_OPT(u32, 18) /** Used to define task methods */ #define TASK_METHOD_T static inline const u32 diff --git a/src/worker.cc b/src/worker.cc index 83eba8c7a..c1dcfe19f 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -70,6 +70,12 @@ void Worker::PollGrouped(WorkEntry &work_entry) { // Attempt to run the task if it's ready and runnable bool is_remote = task->domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote)) { + // TODO(llogan): Make a remote debug macro + if (task->task_state_ != LABSTOR_QM_CLIENT->admin_task_state_ && + !task->task_flags_.Any(TASK_REMOTE_DEBUG_MARK)) { + is_remote = true; + task->task_flags_.SetBits(TASK_REMOTE_DEBUG_MARK); + } // Execute or schedule task if (is_remote) { auto ids = LABSTOR_RUNTIME->ResolveDomainId(task->domain_id_); From 9c71182febef5b87d9beb4c2b235140e9f3bfdec Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 14:27:40 -0500 Subject: [PATCH 117/191] Fix remote coroutine double-free problem --- include/labstor/api/labstor_runtime.h | 2 ++ include/labstor/labstor_types.h | 13 ++++++++++++ include/labstor/task_registry/task.h | 6 ++++++ include/labstor/work_orchestrator/scheduler.h | 2 +- src/worker.cc | 11 +++++++--- tasks/bdev/include/bdev/bdev_tasks.h | 12 +++++------ .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 20 +++++++++---------- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 4 ++-- .../hermes_bucket_mdm_tasks.h | 12 +++++------ .../src/hermes_bucket_mdm.cc | 4 ++-- .../include/hermes_mdm/hermes_mdm_tasks.h | 2 +- .../labstor_admin/labstor_admin_tasks.h | 12 +++++------ .../include/proc_queue/proc_queue_tasks.h | 2 +- .../include/remote_queue/remote_queue_tasks.h | 2 +- .../remote_queue/src/remote_queue.cc | 4 ++-- 15 files changed, 67 insertions(+), 41 deletions(-) diff --git a/include/labstor/api/labstor_runtime.h b/include/labstor/api/labstor_runtime.h index 76f4525e6..4feb03bab 100644 --- a/include/labstor/api/labstor_runtime.h +++ b/include/labstor/api/labstor_runtime.h @@ -33,6 +33,7 @@ class Runtime : public ConfigurationManager { remote_queue::Client remote_queue_; RpcContext rpc_; ThalliumRpc thallium_; + bool remote_created_ = false; public: /** Default constructor */ @@ -121,6 +122,7 @@ class Runtime : public ConfigurationManager { task_registry_.RegisterTaskLib("remote_queue"); remote_queue_.CreateRoot(DomainId::GetLocal(), "remote_queue", LABSTOR_CLIENT->MakeTaskStateId()); + remote_created_ = true; } public: diff --git a/include/labstor/labstor_types.h b/include/labstor/labstor_types.h index 9d01597b3..26f9dae54 100644 --- a/include/labstor/labstor_types.h +++ b/include/labstor/labstor_types.h @@ -340,6 +340,19 @@ struct UniqueId { } }; +/** A sized data pointer */ +template +struct DataPointer { + hipc::Pointer ptr_; + size_t size_; + + /** Serialization */ + template + void serialize(Ar &ar) { + ar(ptr_, size_); + } +}; + /** Uniquely identify a task state */ using TaskStateId = UniqueId<1>; /** Uniquely identify a queue */ diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index eb390a7f2..23a683a1b 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -215,6 +215,7 @@ constexpr inline void CALL_REPLICA_END(T *task) { template struct TaskFlags : public IsTask { public: + TASK_FLAG_T IS_LOCAL = FLAGS & TF_LOCAL; TASK_FLAG_T SUPPORTS_SRL = FLAGS & (TF_SRL_SYM | TF_SRL_ASYM); TASK_FLAG_T SRL_SYM_START = FLAGS & TF_SRL_SYM_START; TASK_FLAG_T SRL_SYM_END = FLAGS & TF_SRL_SYM_END; @@ -375,6 +376,11 @@ struct Task : public hipc::ShmContainer { return task_flags_.Any(TASK_COROUTINE); } + /** Set this task as blocking */ + HSHM_ALWAYS_INLINE void UnsetCoroutine() { + task_flags_.UnsetBits(TASK_COROUTINE); + } + /** Set this task as blocking */ HSHM_ALWAYS_INLINE bool IsPreemptive() { return task_flags_.Any(TASK_PREEMPTIVE); diff --git a/include/labstor/work_orchestrator/scheduler.h b/include/labstor/work_orchestrator/scheduler.h index bcd6d9889..e8a7ad8e4 100644 --- a/include/labstor/work_orchestrator/scheduler.h +++ b/include/labstor/work_orchestrator/scheduler.h @@ -34,7 +34,7 @@ struct ScheduleTask : public Task, TaskFlags { prio_ = TaskPrio::kLongRunning; task_state_ = state_id; method_ = SchedulerMethod::kSchedule; - task_flags_.SetBits(TASK_LONG_RUNNING); + task_flags_.SetBits(TASK_LONG_RUNNING | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Custom params diff --git a/src/worker.cc b/src/worker.cc index c1dcfe19f..ecbd6eea2 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -72,16 +72,19 @@ void Worker::PollGrouped(WorkEntry &work_entry) { if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote)) { // TODO(llogan): Make a remote debug macro if (task->task_state_ != LABSTOR_QM_CLIENT->admin_task_state_ && - !task->task_flags_.Any(TASK_REMOTE_DEBUG_MARK)) { + !task->task_flags_.Any(TASK_REMOTE_DEBUG_MARK) && + task->method_ != TaskMethod::kConstruct && + LABSTOR_RUNTIME->remote_created_) { is_remote = true; - task->task_flags_.SetBits(TASK_REMOTE_DEBUG_MARK); } + task->task_flags_.SetBits(TASK_REMOTE_DEBUG_MARK); // Execute or schedule task if (is_remote) { auto ids = LABSTOR_RUNTIME->ResolveDomainId(task->domain_id_); LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); task->DisableRun(); task->SetUnordered(); + task->UnsetCoroutine(); } else if (task->IsCoroutine()) { if (!task->IsStarted()) { ctx.stack_ptr_ = malloc(ctx.stack_size_); @@ -89,6 +92,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { HILOG(kFatal, "The stack pointer of size {} is NULL", ctx.stack_size_, ctx.stack_ptr_); } + HILOG(kInfo, "Allocated {}", (size_t)ctx.stack_ptr_) ctx.jmp_.fctx = bctx::make_fcontext( (char*)ctx.stack_ptr_ + ctx.stack_size_, ctx.stack_size_, &RunBlocking); @@ -111,7 +115,8 @@ void Worker::PollGrouped(WorkEntry &work_entry) { entry->complete_ = true; if (task->IsCoroutine()) { // TODO(llogan): verify leak - free(ctx.stack_ptr_); + HILOG(kInfo, "Freeing {}", (size_t)ctx.stack_ptr_) + // free(ctx.stack_ptr_); } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); } diff --git a/tasks/bdev/include/bdev/bdev_tasks.h b/tasks/bdev/include/bdev/bdev_tasks.h index c7f7be735..fc9965732 100644 --- a/tasks/bdev/include/bdev/bdev_tasks.h +++ b/tasks/bdev/include/bdev/bdev_tasks.h @@ -94,7 +94,7 @@ struct AllocateTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kAllocate; - task_flags_.SetBits(TASK_UNORDERED); + task_flags_.SetBits(TASK_UNORDERED | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Free params @@ -135,7 +135,7 @@ struct FreeTask : public Task, TaskFlags { method_ = Method::kFree; task_flags_.SetBits(0); if (fire_and_forget) { - task_flags_.SetBits(TASK_FIRE_AND_FORGET | TASK_UNORDERED); + task_flags_.SetBits(TASK_FIRE_AND_FORGET | TASK_UNORDERED | TASK_REMOTE_DEBUG_MARK); } domain_id_ = domain_id; @@ -177,7 +177,7 @@ struct WriteTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kWrite; - task_flags_.SetBits(TASK_UNORDERED); + task_flags_.SetBits(TASK_UNORDERED | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Free params @@ -220,7 +220,7 @@ struct ReadTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kRead; - task_flags_.SetBits(TASK_UNORDERED); + task_flags_.SetBits(TASK_UNORDERED | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Free params @@ -259,7 +259,7 @@ struct MonitorTask : public Task, TaskFlags { prio_ = TaskPrio::kLongRunning; task_state_ = state_id; method_ = Method::kMonitor; - task_flags_.SetBits(TASK_LONG_RUNNING); + task_flags_.SetBits(TASK_LONG_RUNNING | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Custom @@ -294,7 +294,7 @@ struct UpdateCapacityTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kUpdateCapacity; - task_flags_.SetBits(TASK_FIRE_AND_FORGET | TASK_UNORDERED); + task_flags_.SetBits(TASK_FIRE_AND_FORGET | TASK_UNORDERED | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Custom diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index be5e50f36..35740abb9 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -171,7 +171,7 @@ struct GetOrCreateBlobIdTask : public Task, TaskFlags { /** Destructor */ ~GetOrCreateBlobIdTask() { - // HSHM_DESTROY_AR(blob_name_) + HSHM_DESTROY_AR(blob_name_) } /** (De)serialize message call */ @@ -276,8 +276,8 @@ struct PutBlobTask : public Task, TaskFlags /** Destructor */ ~PutBlobTask() { - // HSHM_DESTROY_AR(blob_name_); - // HSHM_DESTROY_AR(filename_); + HSHM_DESTROY_AR(blob_name_); + HSHM_DESTROY_AR(filename_); if (IsDataOwner()) { LABSTOR_CLIENT->FreeBuffer(data_); } @@ -390,8 +390,8 @@ struct GetBlobTask : public Task, TaskFlags /** Destructor */ ~GetBlobTask() { - // HSHM_DESTROY_AR(blob_name_); - // HSHM_DESTROY_AR(filename_); + HSHM_DESTROY_AR(blob_name_); + HSHM_DESTROY_AR(filename_); } /** (De)serialize message call */ @@ -586,7 +586,7 @@ struct GetBlobIdTask : public Task, TaskFlags { /** Destructor */ ~GetBlobIdTask() { - // HSHM_DESTROY_AR(blob_name_) + HSHM_DESTROY_AR(blob_name_) } /** (De)serialize message call */ @@ -649,7 +649,7 @@ struct GetBlobNameTask : public Task, TaskFlags { /** Destructor */ ~GetBlobNameTask() { - // HSHM_DESTROY_AR(blob_name_) + HSHM_DESTROY_AR(blob_name_) }; /** (De)serialize message call */ @@ -717,7 +717,7 @@ struct GetBlobSizeTask : public Task, TaskFlags { /** Destructor */ ~GetBlobSizeTask() { - // HSHM_DESTROY_AR(blob_name_) + HSHM_DESTROY_AR(blob_name_) } /** (De)serialize message call */ @@ -833,7 +833,7 @@ struct GetBlobBuffersTask : public Task, TaskFlags { /** Destructor */ ~GetBlobBuffersTask() { - // HSHM_DESTROY_AR(buffers_) + HSHM_DESTROY_AR(buffers_) } /** (De)serialize message call */ @@ -898,7 +898,7 @@ struct RenameBlobTask : public Task, TaskFlags { /** Destructor */ ~RenameBlobTask() { - // HSHM_DESTROY_AR(new_blob_name_) + HSHM_DESTROY_AR(new_blob_name_) } /** (De)serialize message call */ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index f0275a061..b532ed9d5 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -356,7 +356,7 @@ class Server : public TaskLib { for (bdev::ReadTask *&read_task : read_tasks) { LABSTOR_CLIENT->DelTask(read_task); } - // HSHM_DESTROY_AR(task->bdev_reads_); + HSHM_DESTROY_AR(task->bdev_reads_); HILOG(kDebug, "GetBlobTask complete"); task->SetModuleComplete(); } @@ -583,7 +583,7 @@ class Server : public TaskLib { task->tag_id_, -(ssize_t)blob_info.blob_size_, bucket_mdm::UpdateSizeMode::kAdd); - // HSHM_DESTROY_AR(task->free_tasks_); + HSHM_DESTROY_AR(task->free_tasks_); blob_map.erase(task->blob_id_); task->SetModuleComplete(); } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index f19aa2edb..5c8460287 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -303,7 +303,7 @@ struct AppendBlobTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kAppendBlob; - task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_UNORDERED); + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_UNORDERED | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Custom params @@ -375,8 +375,8 @@ struct GetOrCreateTagTask : public Task, TaskFlags { /** Destructor */ ~GetOrCreateTagTask() { - // HSHM_DESTROY_AR(tag_name_) - // HSHM_DESTROY_AR(traits_) + HSHM_DESTROY_AR(tag_name_) + HSHM_DESTROY_AR(traits_) } /** (De)serialize message call */ @@ -432,7 +432,7 @@ struct GetTagIdTask : public Task, TaskFlags { /** Destructor */ ~GetTagIdTask() { - // HSHM_DESTROY_AR(tag_name_) + HSHM_DESTROY_AR(tag_name_) } /** (De)serialize message call */ @@ -488,7 +488,7 @@ struct GetTagNameTask : public Task, TaskFlags { /** Destructor */ ~GetTagNameTask() { - // HSHM_DESTROY_AR(tag_name_) + HSHM_DESTROY_AR(tag_name_) } /** (De)serialize message call */ @@ -547,7 +547,7 @@ struct RenameTagTask : public Task, TaskFlags { /** Destructor */ ~RenameTagTask() { - // HSHM_DESTROY_AR(tag_name_) + HSHM_DESTROY_AR(tag_name_) } /** (De)serialize message call */ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index eca36dc36..5d1a96fc0 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -180,7 +180,7 @@ class Server : public TaskLib { for (AppendInfo &append : append_info) { LABSTOR_CLIENT->DelTask(append.put_task_); } - // HSHM_DESTROY_AR(task->schema_->append_info_); + HSHM_DESTROY_AR(task->schema_->append_info_); LABSTOR_CLIENT->DelTask(task->schema_); task->SetModuleComplete(); } @@ -296,7 +296,7 @@ class Server : public TaskLib { for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { LABSTOR_CLIENT->DelTask(blob_task); } - // HSHM_DESTROY_AR(task->destroy_blob_tasks_); + HSHM_DESTROY_AR(task->destroy_blob_tasks_); TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; tag_map.erase(task->tag_id_); task->SetModuleComplete(); diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h index 6d161e3be..3a78e6de6 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h @@ -47,7 +47,7 @@ struct ConstructTask : public CreateTaskStateTask { /** Destructor */ HSHM_ALWAYS_INLINE ~ConstructTask() { - // HSHM_DESTROY_AR(server_config_path_); + HSHM_DESTROY_AR(server_config_path_); } /** (De)serialize message call */ diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index 90a428e74..dd28394e9 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -50,7 +50,7 @@ struct RegisterTaskLibTaskTempl : public Task, TaskFlags { /** Destructor */ ~RegisterTaskLibTaskTempl() { - // HSHM_DESTROY_AR(lib_name_); + HSHM_DESTROY_AR(lib_name_); } /** (De)serialize message call */ @@ -118,7 +118,7 @@ struct GetOrCreateTaskStateIdTask : public Task, TaskFlags { } ~GetOrCreateTaskStateIdTask() { - // HSHM_DESTROY_AR(state_name_); + HSHM_DESTROY_AR(state_name_); } /** (De)serialize message call */ @@ -179,9 +179,9 @@ struct CreateTaskStateTask : public Task, TaskFlags { /** Destructor */ ~CreateTaskStateTask() { - // HSHM_DESTROY_AR(state_name_); - // HSHM_DESTROY_AR(lib_name_); - // HSHM_DESTROY_AR(queue_info_); + HSHM_DESTROY_AR(state_name_); + HSHM_DESTROY_AR(lib_name_); + HSHM_DESTROY_AR(queue_info_); } /** Replication (does nothing) */ @@ -245,7 +245,7 @@ struct GetTaskStateIdTask : public Task, TaskFlags { } ~GetTaskStateIdTask() { - // HSHM_DESTROY_AR(state_name_); + HSHM_DESTROY_AR(state_name_); } /** (De)serialize message call */ diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index 9372d1307..08adb8457 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -100,7 +100,7 @@ struct TypedPushTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPush; - task_flags_.SetBits(TASK_DATA_OWNER | TASK_LOW_LATENCY); + task_flags_.SetBits(TASK_DATA_OWNER | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Custom params diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index c3e79057d..5c146dcdc 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -111,7 +111,7 @@ struct PushTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPush; - task_flags_.SetBits(TASK_LOW_LATENCY | TASK_PREEMPTIVE); + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_PREEMPTIVE | TASK_REMOTE_DEBUG_MARK); domain_id_ = domain_id; // Custom params diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 28960fafa..b018b4956 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,8 +278,8 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - LABSTOR_CLIENT->DelTask(exec, orig_task); - if (out_xfer.size() > 0) { + // LABSTOR_CLIENT->DelTask(exec, orig_task); + if (out_xfer.size() > 0 && out_xfer[0].data_size_ > 0) { req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } else { req.respond(std::string()); From 7b13e29e9ee658c0fed396fc63b8e5c719938ee3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 14:38:14 -0500 Subject: [PATCH 118/191] Don't do remote debug by default --- src/worker.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/worker.cc b/src/worker.cc index ecbd6eea2..b164d7d54 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -71,6 +71,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { bool is_remote = task->domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote)) { // TODO(llogan): Make a remote debug macro +#ifdef REMOTE_DEBUG if (task->task_state_ != LABSTOR_QM_CLIENT->admin_task_state_ && !task->task_flags_.Any(TASK_REMOTE_DEBUG_MARK) && task->method_ != TaskMethod::kConstruct && @@ -78,6 +79,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { is_remote = true; } task->task_flags_.SetBits(TASK_REMOTE_DEBUG_MARK); +#endif // Execute or schedule task if (is_remote) { auto ids = LABSTOR_RUNTIME->ResolveDomainId(task->domain_id_); @@ -92,7 +94,6 @@ void Worker::PollGrouped(WorkEntry &work_entry) { HILOG(kFatal, "The stack pointer of size {} is NULL", ctx.stack_size_, ctx.stack_ptr_); } - HILOG(kInfo, "Allocated {}", (size_t)ctx.stack_ptr_) ctx.jmp_.fctx = bctx::make_fcontext( (char*)ctx.stack_ptr_ + ctx.stack_size_, ctx.stack_size_, &RunBlocking); @@ -115,8 +116,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { entry->complete_ = true; if (task->IsCoroutine()) { // TODO(llogan): verify leak - HILOG(kInfo, "Freeing {}", (size_t)ctx.stack_ptr_) - // free(ctx.stack_ptr_); + free(ctx.stack_ptr_); } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); } From d51d6f5128370297270bc21384261aeabaccc971 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 14:39:54 -0500 Subject: [PATCH 119/191] Use debug for prints in RPC --- tasks_required/remote_queue/src/remote_queue.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index b018b4956..3afeee499 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -253,7 +253,7 @@ class Server : public TaskLib { orig_task->UnsetDataOwner(); orig_task->UnsetLongRunning(); queue->Emplace(orig_task->prio_, orig_task->lane_hash_, p); - HILOG(kInfo, + HILOG(kDebug, "(node {}) Executing task (task_node={}, task_state={}/{}, state_name={}, method={}, size={}, lane_hash={})", LABSTOR_CLIENT->node_id_, orig_task->task_node_, @@ -271,7 +271,7 @@ class Server : public TaskLib { TaskState *exec, TaskStateId state_id) { BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); std::vector out_xfer = exec->SaveEnd(method, ar, orig_task); - HILOG(kInfo, "(node {}) Returning {} bytes of data (task_node={}, task_state={}/{}, method={})", + HILOG(kDebug, "(node {}) Returning {} bytes of data (task_node={}, task_state={}/{}, method={})", LABSTOR_CLIENT->node_id_, out_xfer[0].data_size_, orig_task->task_node_, From 3c1645759699985a09a5153ebe3f667fdc1daca5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 14:45:24 -0500 Subject: [PATCH 120/191] Re-enable remote deltask --- tasks_required/remote_queue/src/remote_queue.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 3afeee499..acd6b5c73 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -278,7 +278,7 @@ class Server : public TaskLib { orig_task->task_state_, state_id, method); - // LABSTOR_CLIENT->DelTask(exec, orig_task); + LABSTOR_CLIENT->DelTask(exec, orig_task); if (out_xfer.size() > 0 && out_xfer[0].data_size_ > 0) { req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } else { From d7a34f0eecbcaf62d07454cada833aa9d9b68ac7 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 14:52:07 -0500 Subject: [PATCH 121/191] Lower the stack size for tasks --- include/labstor/task_registry/task.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 23a683a1b..b42b0b94b 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -237,7 +237,7 @@ class TaskPrio { struct RunContext { u32 lane_id_; /**< The lane id of the task */ bctx::transfer_t jmp_; /**< Current execution state of the task (runtime) */ - size_t stack_size_ = KILOBYTES(256); /**< The size of the stack for the task (runtime) */ + size_t stack_size_ = KILOBYTES(64); /**< The size of the stack for the task (runtime) */ void *stack_ptr_; /**< The pointer to the stack (runtime) */ TaskLib *exec_; From 1285edd54c99f4e86506b1f21df75e28de0ce427 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 14:54:14 -0500 Subject: [PATCH 122/191] Free buffer data for get and append --- tasks/hermes/include/hermes/bucket.h | 2 +- .../include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 3c13ffe30..66da72dc1 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -434,7 +434,7 @@ class Bucket { blob_id = task->blob_id_; char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); memcpy(blob.data(), data, data_size); - // LABSTOR_CLIENT->FreeBuffer(task->data_); + LABSTOR_CLIENT->FreeBuffer(task->data_); LABSTOR_CLIENT->DelTask(push_task); return blob_id; } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 5c8460287..11303272f 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -317,9 +317,9 @@ struct AppendBlobTask : public Task, TaskFlags { /** Destructor */ ~AppendBlobTask() { -// if (IsDataOwner()) { -// LABSTOR_CLIENT->FreeBuffer(data_); -// } + if (IsDataOwner()) { + LABSTOR_CLIENT->FreeBuffer(data_); + } } /** Create group */ From e5ce5f42ce122c0541642d9f039c23d62f317988 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 16:46:05 -0500 Subject: [PATCH 123/191] Made it HermesConfig.cmake and change LABSTOR -> HERMES in CMake --- ...LabstorConfig.cmake => HermesConfig.cmake} | 65 ++++++++++--------- CMakeLists.txt | 42 ++++++------ benchmark/CMakeLists.txt | 24 +++---- ci/install_docs.sh | 4 +- ci/install_hshm.sh | 4 +- src/CMakeLists.txt | 42 ++++++------ tasks/hermes/src/CMakeLists.txt | 30 ++++----- .../hermes_adapters/filesystem/CMakeLists.txt | 22 +++---- tasks/hermes_adapters/mpiio/CMakeLists.txt | 10 +-- tasks/hermes_adapters/posix/CMakeLists.txt | 20 +++--- tasks/hermes_adapters/src/CMakeLists.txt | 30 ++++----- tasks/hermes_adapters/stdio/CMakeLists.txt | 20 +++--- tasks/hermes_adapters/vfd/CMakeLists.txt | 18 ++--- tasks/hermes_blob_mdm/src/CMakeLists.txt | 30 ++++----- tasks/hermes_bucket_mdm/src/CMakeLists.txt | 30 ++++----- tasks/hermes_mdm/src/CMakeLists.txt | 30 ++++----- tasks/posix_bdev/src/CMakeLists.txt | 30 ++++----- tasks/ram_bdev/src/CMakeLists.txt | 30 ++++----- tasks_required/TASK_NAME/src/CMakeLists.txt | 30 ++++----- .../labstor_admin/src/CMakeLists.txt | 30 ++++----- tasks_required/proc_queue/src/CMakeLists.txt | 30 ++++----- .../remote_queue/src/CMakeLists.txt | 28 ++++---- .../small_message/src/CMakeLists.txt | 30 ++++----- .../worch_proc_round_robin/src/CMakeLists.txt | 30 ++++----- .../src/CMakeLists.txt | 30 ++++----- test/CMakeLists.txt | 2 +- test/unit/CMakeLists.txt | 2 +- test/unit/boost/CMakeLists.txt | 16 ++--- test/unit/hermes/CMakeLists.txt | 16 ++--- test/unit/ipc/CMakeLists.txt | 16 ++--- 30 files changed, 371 insertions(+), 370 deletions(-) rename CMake/{LabstorConfig.cmake => HermesConfig.cmake} (50%) diff --git a/CMake/LabstorConfig.cmake b/CMake/HermesConfig.cmake similarity index 50% rename from CMake/LabstorConfig.cmake rename to CMake/HermesConfig.cmake index 221f7ca09..9ca777914 100644 --- a/CMake/LabstorConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -1,26 +1,26 @@ -# Find labstor header and library. +# Find Hermes header and library. # # This module defines the following uncached variables: -# Labstor_FOUND, if false, do not try to use labstor. -# Labstor_INCLUDE_DIRS, where to find labstor.h. -# Labstor_LIBRARIES, the libraries to link against to use the labstor library -# Labstor_LIBRARY_DIRS, the directory where the labstor library is found. +# Hermes_FOUND, if false, do not try to use Hermes. +# Hermes_INCLUDE_DIRS, where to find Hermes.h. +# Hermes_LIBRARIES, the libraries to link against to use the Hermes library +# Hermes_LIBRARY_DIRS, the directory where the Hermes library is found. find_path( - Labstor_INCLUDE_DIR - labstor/labstor_types.h + Hermes_INCLUDE_DIR + Hermes/Hermes_types.h ) -if( Labstor_INCLUDE_DIR ) - get_filename_component(Labstor_DIR ${Labstor_INCLUDE_DIR} PATH) +if( Hermes_INCLUDE_DIR ) + get_filename_component(Hermes_DIR ${Hermes_INCLUDE_DIR} PATH) #----------------------------------------------------------------------------- - # Find all packages needed by labstor + # Find all packages needed by Hermes #----------------------------------------------------------------------------- find_library( - Labstor_LIBRARY - NAMES labstor_client labstor_runtime + Hermes_LIBRARY + NAMES Hermes_client Hermes_runtime ) # HermesShm @@ -56,29 +56,30 @@ if( Labstor_INCLUDE_DIR ) #----------------------------------------------------------------------------- # Mark hermes as found and set all needed packages #----------------------------------------------------------------------------- - if( Labstor_LIBRARY ) - set(Labstor_LIBRARY_DIR "") - get_filename_component(Labstor_LIBRARY_DIRS ${Labstor_LIBRARY} PATH) + if( Hermes_LIBRARY ) + set(Hermes_LIBRARY_DIR "") + get_filename_component(Hermes_LIBRARY_DIRS ${Hermes_LIBRARY} PATH) # Set uncached variables as per standard. - set(Labstor_FOUND ON) - set(Labstor_INCLUDE_DIRS ${Labstor_INCLUDE_DIR}) - set(Labstor_LIBRARIES + set(Hermes_FOUND ON) + set(Hermes_INCLUDE_DIRS ${Hermes_INCLUDE_DIR}) + set(Hermes_LIBRARIES ${HermesShm_LIBRARIES} yaml-cpp cereal::cereal - -ldl -lrt -lc -pthread ${Labstor_LIBRARY}) - endif(Labstor_LIBRARY) + -ldl -lrt -lc -pthread ${Hermes_LIBRARY}) + set(Hermes_CLIENT_LIBRARIES ${Hermes_LIBRARIES}) + endif(Hermes_LIBRARY) -else(Labstor_INCLUDE_DIR) - message(STATUS "FindLabstor: Could not find labstor.h") -endif(Labstor_INCLUDE_DIR) +else(Hermes_INCLUDE_DIR) + message(STATUS "FindHermes: Could not find Hermes.h") +endif(Hermes_INCLUDE_DIR) -if(Labstor_FOUND) - if(NOT Labstor_FIND_QUIETLY) - message(STATUS "FindLabstor: Found both labstor.h and liblabstor_client.so") - endif(NOT Labstor_FIND_QUIETLY) -else(Labstor_FOUND) - if(Labstor_FIND_REQUIRED) - message(STATUS "FindLabstor: Could not find labstor.h and/or liblabstor_client.so") - endif(Labstor_FIND_REQUIRED) -endif(Labstor_FOUND) +if(Hermes_FOUND) + if(NOT Hermes_FIND_QUIETLY) + message(STATUS "FindHermes: Found both Hermes.h and libHermes_client.so") + endif(NOT Hermes_FIND_QUIETLY) +else(Hermes_FOUND) + if(Hermes_FIND_REQUIRED) + message(STATUS "FindHermes: Could not find Hermes.h and/or libHermes_client.so") + endif(Hermes_FIND_REQUIRED) +endif(Hermes_FOUND) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2b98ba34..56f245755 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,8 @@ project(hermes) option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" ON) option(BUILD_MPI_TESTS "Build tests which depend on MPI" ON) option(BUILD_OpenMP_TESTS "Build tests which depend on OpenMP" ON) -option(LABSTOR_ENABLE_COVERAGE "Check how well tests cover code" OFF) -option(LABSTOR_ENABLE_DOXYGEN "Check how well the code is documented" OFF) +option(HERMES_ENABLE_COVERAGE "Check how well tests cover code" OFF) +option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" OFF) option(HERMES_ENABLE_POSIX_ADAPTER "Build the Hermes POSIX adapter." ON) option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." ON) @@ -38,8 +38,8 @@ add_compile_options(-march=native -fomit-frame-pointer) # Targets built within this project are exported at Install time for use # by other projects. #----------------------------------------------------------------------------- -if(NOT LABSTOR_EXPORTED_TARGETS) - set(LABSTOR_EXPORTED_TARGETS "labstor-targets") +if(NOT HERMES_EXPORTED_TARGETS) + set(HERMES_EXPORTED_TARGETS "labstor-targets") endif() #----------------------------------------------------------------------------- @@ -115,21 +115,21 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY #------------------------------------------------------------------------------ # Setup install and output Directories #------------------------------------------------------------------------------ -if(NOT LABSTOR_INSTALL_BIN_DIR) - set(LABSTOR_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) +if(NOT HERMES_INSTALL_BIN_DIR) + set(HERMES_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) endif() -if(NOT LABSTOR_INSTALL_LIB_DIR) - set(LABSTOR_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) +if(NOT HERMES_INSTALL_LIB_DIR) + set(HERMES_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) endif() -if(NOT LABSTOR_INSTALL_INCLUDE_DIR) - set(LABSTOR_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) +if(NOT HERMES_INSTALL_INCLUDE_DIR) + set(HERMES_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) endif() -if(NOT LABSTOR_INSTALL_DATA_DIR) - set(LABSTOR_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) +if(NOT HERMES_INSTALL_DATA_DIR) + set(HERMES_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) endif() #----------------------------------------------------------------------------- -# Build Labstor Main Packages +# Build Hermes Main Packages #----------------------------------------------------------------------------- # Main includes include_directories(${HermesShm_INCLUDE_DIRS}) @@ -157,18 +157,18 @@ include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_adapters/include) # Test includes include_directories(${CMAKE_SOURCE_DIR}/test/unit) -set(Labstor_CLIENT_LIBRARIES +set(Hermes_CLIENT_LIBRARIES ${HermesShm_LIBRARIES} yaml-cpp cereal::cereal -ldl -lrt -lc -pthread labstor_client) -set(Labstor_CLIENT_DEPS +set(Hermes_CLIENT_DEPS labstor_client) -set(Labstor_RUNTIME_LIBRARIES - ${Labstor_CLIENT_LIBRARIES} +set(Hermes_RUNTIME_LIBRARIES + ${Hermes_CLIENT_LIBRARIES} labstor_runtime ${Boost_LIBRARIES}) -set(Labstor_RUNTIME_DEPS +set(Hermes_RUNTIME_DEPS labstor_client labstor_runtime) set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) @@ -197,13 +197,13 @@ add_subdirectory(test) install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/CMake/LabstorConfig.cmake - ${PROJECT_BINARY_DIR}/CMakeFiles/LabstorConfig.cmake @ONLY + ${CMAKE_CURRENT_SOURCE_DIR}/CMake/HermesConfig.cmake + ${PROJECT_BINARY_DIR}/CMakeFiles/HermesConfig.cmake @ONLY ) install( FILES - ${PROJECT_BINARY_DIR}/CMakeFiles/LabstorConfig.cmake + ${PROJECT_BINARY_DIR}/CMakeFiles/HermesConfig.cmake DESTINATION ${CMAKE_INSTALL_PREFIX}/cmake ) \ No newline at end of file diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 28c07cf82..6055ce288 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10) -project(labstor) +project(hermes) set(CMAKE_CXX_STANDARD 17) @@ -12,9 +12,9 @@ add_executable(test_performance_exec test_init.cc test_latency.cc) add_dependencies(test_performance_exec - ${Labstor_RUNTIME_DEPS} hermes) + ${Hermes_RUNTIME_DEPS} hermes) target_link_libraries(test_performance_exec - ${Labstor_RUNTIME_LIBRARIES} hermes Catch2::Catch2 + ${Hermes_RUNTIME_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) #add_executable(test_performance_exec @@ -24,17 +24,17 @@ target_link_libraries(test_performance_exec # test_zmq.cc #) #add_dependencies(test_performance_exec -# ${Labstor_CLIENT_DEPS} hermes) +# ${Hermes_CLIENT_DEPS} hermes) #target_link_libraries(test_performance_exec -# ${Labstor_CLIENT_LIBRARIES} hermes Catch2::Catch2 +# ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 # MPI::MPI_CXX ${ZMQ_LIBRARIES}) add_executable(hermes_api_bench hermes_api_bench.cc) add_dependencies(hermes_api_bench - ${Labstor_CLIENT_DEPS} hermes) + ${Hermes_CLIENT_DEPS} hermes) target_link_libraries(hermes_api_bench - ${Labstor_CLIENT_LIBRARIES} hermes + ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) #------------------------------------------------------------------------------ @@ -52,15 +52,15 @@ install(TARGETS test_performance_exec hermes_api_bench EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) #----------------------------------------------------------------------------- # Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(test_performance_exec) set_coverage_flags(test_hermes_api) endif() diff --git a/ci/install_docs.sh b/ci/install_docs.sh index f7d9189e7..63f912bd8 100755 --- a/ci/install_docs.sh +++ b/ci/install_docs.sh @@ -17,8 +17,8 @@ cmake \ -DCMAKE_INSTALL_RPATH=${INSTALL_PREFIX}/lib \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DBUILD_SHARED_LIBS=ON \ - -DLABSTOR_ENABLE_DOXYGEN=ON \ - -DLABSTOR_ENABLE_COVERAGE=ON \ + -DHERMES_ENABLE_DOXYGEN=ON \ + -DHERMES_ENABLE_COVERAGE=ON \ .. make dox >& out.txt # cat out.txt | grep warning | grep -v "ignoring unsupported tag" diff --git a/ci/install_hshm.sh b/ci/install_hshm.sh index 4f1445d91..3a0b2be1d 100755 --- a/ci/install_hshm.sh +++ b/ci/install_hshm.sh @@ -18,8 +18,8 @@ cd build spack load --only dependencies hermes_shm cmake ../ \ -DCMAKE_BUILD_TYPE=Debug \ --DLABSTOR_ENABLE_COVERAGE=ON \ --DLABSTOR_ENABLE_DOXYGEN=ON \ +-DHERMES_ENABLE_COVERAGE=ON \ +-DHERMES_ENABLE_DOXYGEN=ON \ -DBUILD_HSHM_BENCHMARKS=ON \ -DBUILD_HSHM_TESTS=ON \ -DCMAKE_INSTALL_PREFIX=${HOME}/install diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 04bfaf302..cdfd0b46e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,32 +23,32 @@ target_link_libraries(labstor_client add_library(labstor_runtime worker.cc labstor_runtime.cc) -add_dependencies(labstor_runtime ${Labstor_CLIENT_DEPS}) -target_link_libraries(labstor_runtime thallium ${Labstor_CLIENT_LIBRARIES}) +add_dependencies(labstor_runtime ${Hermes_CLIENT_DEPS}) +target_link_libraries(labstor_runtime thallium ${Hermes_CLIENT_LIBRARIES}) #------------------------------------------------------------------------------ # Build Labstor Runtime Start Function #------------------------------------------------------------------------------ add_executable(labstor_start_runtime labstor_start_runtime.cc) -add_dependencies(labstor_start_runtime ${Labstor_RUNTIME_DEPS}) -target_link_libraries(labstor_start_runtime ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(labstor_start_runtime ${Hermes_RUNTIME_DEPS}) +target_link_libraries(labstor_start_runtime ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Build LabStor Runtime Stop Function #------------------------------------------------------------------------------ add_executable(labstor_stop_runtime labstor_stop_runtime.cc) -add_dependencies(labstor_stop_runtime ${Labstor_CLIENT_DEPS}) -target_link_libraries(labstor_stop_runtime ${Labstor_CLIENT_LIBRARIES}) +add_dependencies(labstor_stop_runtime ${Hermes_CLIENT_DEPS}) +target_link_libraries(labstor_stop_runtime ${Hermes_CLIENT_LIBRARIES}) #----------------------------------------------------------------------------- # Add file(s) to CMake Install #----------------------------------------------------------------------------- install( FILES - ${LABSTOR_HEADERS} + ${HERMES_HEADERS} DESTINATION - ${LABSTOR_INSTALL_INCLUDE_DIR} + ${HERMES_INSTALL_INCLUDE_DIR} COMPONENT headers ) @@ -63,10 +63,10 @@ install( labstor_start_runtime labstor_stop_runtime EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -74,35 +74,35 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS labstor_client labstor_runtime labstor_start_runtime labstor_stop_runtime - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(labstor_client) set_coverage_flags(labstor_runtime) set_coverage_flags(labstor_start_runtime) diff --git a/tasks/hermes/src/CMakeLists.txt b/tasks/hermes/src/CMakeLists.txt index 306b950d8..f9ebf9a1c 100644 --- a/tasks/hermes/src/CMakeLists.txt +++ b/tasks/hermes/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(hermes SHARED config_manager.cc) -add_dependencies(hermes ${Labstor_RUNTIME_DEPS}) -target_link_libraries(hermes ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(hermes ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hermes ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS hermes EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes) endif() diff --git a/tasks/hermes_adapters/filesystem/CMakeLists.txt b/tasks/hermes_adapters/filesystem/CMakeLists.txt index 7014b106b..c9564a624 100644 --- a/tasks/hermes_adapters/filesystem/CMakeLists.txt +++ b/tasks/hermes_adapters/filesystem/CMakeLists.txt @@ -23,16 +23,16 @@ install( TARGETS hermes_fs_base EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) install( FILES filesystem_io_client.h DESTINATION - ${LABSTOR_INSTALL_INCLUDE_DIR} + ${HERMES_INSTALL_INCLUDE_DIR} COMPONENT headers ) @@ -40,21 +40,21 @@ install( #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes_fs_base - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_fs_base) endif() diff --git a/tasks/hermes_adapters/mpiio/CMakeLists.txt b/tasks/hermes_adapters/mpiio/CMakeLists.txt index 4cfd54d0e..595994051 100644 --- a/tasks/hermes_adapters/mpiio/CMakeLists.txt +++ b/tasks/hermes_adapters/mpiio/CMakeLists.txt @@ -29,15 +29,15 @@ install( hermes_mpiio_io_client hermes_mpiio EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_mpiio_io_client) #set_coverage_flags(hermes_mpiio) endif() \ No newline at end of file diff --git a/tasks/hermes_adapters/posix/CMakeLists.txt b/tasks/hermes_adapters/posix/CMakeLists.txt index 0553be94a..d5e9eadad 100644 --- a/tasks/hermes_adapters/posix/CMakeLists.txt +++ b/tasks/hermes_adapters/posix/CMakeLists.txt @@ -30,32 +30,32 @@ install( hermes_posix_io_client hermes_posix EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes_posix_io_client hermes_posix - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_posix_io_client) #set_coverage_flags(hermes_posix) endif() diff --git a/tasks/hermes_adapters/src/CMakeLists.txt b/tasks/hermes_adapters/src/CMakeLists.txt index 3382fd743..8c5d06d8e 100644 --- a/tasks/hermes_adapters/src/CMakeLists.txt +++ b/tasks/hermes_adapters/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(hermes_adapters SHARED hermes_adapters.cc) -add_dependencies(hermes_adapters ${Labstor_RUNTIME_DEPS}) -target_link_libraries(hermes_adapters ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(hermes_adapters ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hermes_adapters ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS hermes_adapters EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes_adapters - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_adapters) endif() diff --git a/tasks/hermes_adapters/stdio/CMakeLists.txt b/tasks/hermes_adapters/stdio/CMakeLists.txt index 142f395c7..f4a9e7743 100644 --- a/tasks/hermes_adapters/stdio/CMakeLists.txt +++ b/tasks/hermes_adapters/stdio/CMakeLists.txt @@ -30,32 +30,32 @@ install( hermes_stdio_io_client hermes_stdio EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes_stdio_io_client hermes_stdio - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_stdio_io_client) #set_coverage_flags(hermes_stdio) endif() diff --git a/tasks/hermes_adapters/vfd/CMakeLists.txt b/tasks/hermes_adapters/vfd/CMakeLists.txt index 1ef53b42d..0836028c9 100644 --- a/tasks/hermes_adapters/vfd/CMakeLists.txt +++ b/tasks/hermes_adapters/vfd/CMakeLists.txt @@ -52,7 +52,7 @@ install( FILES ${HDF5_HERMES_VFD_HEADERS} DESTINATION - ${LABSTOR_INSTALL_INCLUDE_DIR} + ${HERMES_INSTALL_INCLUDE_DIR} COMPONENT headers ) @@ -64,10 +64,10 @@ install( TARGETS hdf5_hermes_vfd EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR}/${HERMES_VFD_DIR_NAME} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -75,11 +75,11 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hdf5_hermes_vfd + ${HERMES_INSTALL_DATA_DIR}/cmake/hdf5_hermes_vfd FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- @@ -90,7 +90,7 @@ if(NOT HDF5_HERMES_VFD_EXTERNALLY_CONFIGURED) TARGETS ${HDF5_HERMES_VFD_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() diff --git a/tasks/hermes_blob_mdm/src/CMakeLists.txt b/tasks/hermes_blob_mdm/src/CMakeLists.txt index 4f91d37a3..bf38a7084 100644 --- a/tasks/hermes_blob_mdm/src/CMakeLists.txt +++ b/tasks/hermes_blob_mdm/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(hermes_blob_mdm SHARED hermes_blob_mdm.cc) -add_dependencies(hermes_blob_mdm ${Labstor_RUNTIME_DEPS}) -target_link_libraries(hermes_blob_mdm ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(hermes_blob_mdm ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hermes_blob_mdm ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS hermes_blob_mdm EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes_blob_mdm - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_blob_mdm) endif() diff --git a/tasks/hermes_bucket_mdm/src/CMakeLists.txt b/tasks/hermes_bucket_mdm/src/CMakeLists.txt index 03b889366..fb7484699 100644 --- a/tasks/hermes_bucket_mdm/src/CMakeLists.txt +++ b/tasks/hermes_bucket_mdm/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(hermes_bucket_mdm SHARED hermes_bucket_mdm.cc) -add_dependencies(hermes_bucket_mdm ${Labstor_RUNTIME_DEPS}) -target_link_libraries(hermes_bucket_mdm ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(hermes_bucket_mdm ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hermes_bucket_mdm ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS hermes_bucket_mdm EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes_bucket_mdm - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_bucket_mdm) endif() diff --git a/tasks/hermes_mdm/src/CMakeLists.txt b/tasks/hermes_mdm/src/CMakeLists.txt index 07107874a..da27855f2 100644 --- a/tasks/hermes_mdm/src/CMakeLists.txt +++ b/tasks/hermes_mdm/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(hermes_mdm SHARED hermes_mdm.cc) -add_dependencies(hermes_mdm ${Labstor_RUNTIME_DEPS} hermes) -target_link_libraries(hermes_mdm ${Labstor_RUNTIME_LIBRARIES} hermes) +add_dependencies(hermes_mdm ${Hermes_RUNTIME_DEPS} hermes) +target_link_libraries(hermes_mdm ${Hermes_RUNTIME_LIBRARIES} hermes) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS hermes_mdm EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS hermes_mdm - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(hermes_mdm) endif() diff --git a/tasks/posix_bdev/src/CMakeLists.txt b/tasks/posix_bdev/src/CMakeLists.txt index b8c763fb3..75695cf98 100644 --- a/tasks/posix_bdev/src/CMakeLists.txt +++ b/tasks/posix_bdev/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(posix_bdev SHARED posix_bdev.cc) -add_dependencies(posix_bdev ${Labstor_RUNTIME_DEPS}) -target_link_libraries(posix_bdev ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(posix_bdev ${Hermes_RUNTIME_DEPS}) +target_link_libraries(posix_bdev ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS posix_bdev EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS posix_bdev - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(posix_bdev) endif() diff --git a/tasks/ram_bdev/src/CMakeLists.txt b/tasks/ram_bdev/src/CMakeLists.txt index a0a22e62f..1bf7dd79e 100644 --- a/tasks/ram_bdev/src/CMakeLists.txt +++ b/tasks/ram_bdev/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(ram_bdev SHARED ram_bdev.cc) -add_dependencies(ram_bdev ${Labstor_RUNTIME_DEPS}) -target_link_libraries(ram_bdev ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(ram_bdev ${Hermes_RUNTIME_DEPS}) +target_link_libraries(ram_bdev ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS ram_bdev EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS ram_bdev - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(ram_bdev) endif() diff --git a/tasks_required/TASK_NAME/src/CMakeLists.txt b/tasks_required/TASK_NAME/src/CMakeLists.txt index b021e33ec..c4339eea4 100644 --- a/tasks_required/TASK_NAME/src/CMakeLists.txt +++ b/tasks_required/TASK_NAME/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(TASK_NAME SHARED TASK_NAME.cc) -add_dependencies(TASK_NAME ${Labstor_RUNTIME_DEPS}) -target_link_libraries(TASK_NAME ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(TASK_NAME ${Hermes_RUNTIME_DEPS}) +target_link_libraries(TASK_NAME ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS TASK_NAME EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS TASK_NAME - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(TASK_NAME) endif() diff --git a/tasks_required/labstor_admin/src/CMakeLists.txt b/tasks_required/labstor_admin/src/CMakeLists.txt index bf3c06c61..ad727d992 100644 --- a/tasks_required/labstor_admin/src/CMakeLists.txt +++ b/tasks_required/labstor_admin/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(labstor_admin SHARED ${CMAKE_CURRENT_SOURCE_DIR}/labstor_admin.cc) -add_dependencies(labstor_admin ${Labstor_RUNTIME_DEPS}) -target_link_libraries(labstor_admin ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(labstor_admin ${Hermes_RUNTIME_DEPS}) +target_link_libraries(labstor_admin ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Labstor Admin Task Library @@ -13,10 +13,10 @@ install( TARGETS labstor_admin EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS labstor_admin - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(labstor_admin) endif() diff --git a/tasks_required/proc_queue/src/CMakeLists.txt b/tasks_required/proc_queue/src/CMakeLists.txt index 1d4946b6a..79d810195 100644 --- a/tasks_required/proc_queue/src/CMakeLists.txt +++ b/tasks_required/proc_queue/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(proc_queue SHARED proc_queue.cc) -add_dependencies(proc_queue ${Labstor_RUNTIME_DEPS}) -target_link_libraries(proc_queue ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(proc_queue ${Hermes_RUNTIME_DEPS}) +target_link_libraries(proc_queue ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS proc_queue EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS proc_queue - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(proc_queue) endif() diff --git a/tasks_required/remote_queue/src/CMakeLists.txt b/tasks_required/remote_queue/src/CMakeLists.txt index ead8373f0..dce89518d 100644 --- a/tasks_required/remote_queue/src/CMakeLists.txt +++ b/tasks_required/remote_queue/src/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories(myapp PRIVATE ${libfabric_INCLUDE_DIRS}) link_directories(${libfabric_LIBRARY_DIRS}) add_library(remote_queue SHARED remote_queue.cc) -add_dependencies(remote_queue ${Labstor_RUNTIME_DEPS}) +add_dependencies(remote_queue ${Hermes_RUNTIME_DEPS}) target_link_libraries(remote_queue labstor_runtime ${libfabric_LIBRARIES}) @@ -16,10 +16,10 @@ install( TARGETS remote_queue EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -27,31 +27,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS remote_queue - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(remote_queue) endif() diff --git a/tasks_required/small_message/src/CMakeLists.txt b/tasks_required/small_message/src/CMakeLists.txt index 8bff4c3f7..2ff5b6185 100644 --- a/tasks_required/small_message/src/CMakeLists.txt +++ b/tasks_required/small_message/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(small_message SHARED small_message.cc) -add_dependencies(small_message ${Labstor_RUNTIME_DEPS}) -target_link_libraries(small_message ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(small_message ${Hermes_RUNTIME_DEPS}) +target_link_libraries(small_message ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS small_message EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS small_message - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(small_message) endif() diff --git a/tasks_required/worch_proc_round_robin/src/CMakeLists.txt b/tasks_required/worch_proc_round_robin/src/CMakeLists.txt index 8188f116d..8537ffd9e 100644 --- a/tasks_required/worch_proc_round_robin/src/CMakeLists.txt +++ b/tasks_required/worch_proc_round_robin/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(worch_proc_round_robin SHARED worch_proc_round_robin.cc) -add_dependencies(worch_proc_round_robin ${Labstor_RUNTIME_DEPS}) -target_link_libraries(worch_proc_round_robin ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(worch_proc_round_robin ${Hermes_RUNTIME_DEPS}) +target_link_libraries(worch_proc_round_robin ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS worch_proc_round_robin EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS worch_proc_round_robin - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(worch_proc_round_robin) endif() diff --git a/tasks_required/worch_queue_round_robin/src/CMakeLists.txt b/tasks_required/worch_queue_round_robin/src/CMakeLists.txt index ec0fcbd6a..838c8566a 100644 --- a/tasks_required/worch_queue_round_robin/src/CMakeLists.txt +++ b/tasks_required/worch_queue_round_robin/src/CMakeLists.txt @@ -3,8 +3,8 @@ #------------------------------------------------------------------------------ add_library(worch_queue_round_robin SHARED worch_queue_round_robin.cc) -add_dependencies(worch_queue_round_robin ${Labstor_RUNTIME_DEPS}) -target_link_libraries(worch_queue_round_robin ${Labstor_RUNTIME_LIBRARIES}) +add_dependencies(worch_queue_round_robin ${Hermes_RUNTIME_DEPS}) +target_link_libraries(worch_queue_round_robin ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library @@ -13,10 +13,10 @@ install( TARGETS worch_queue_round_robin EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR} + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -24,31 +24,31 @@ install( #----------------------------------------------------------------------------- install( EXPORT - ${LABSTOR_EXPORTED_TARGETS} + ${HERMES_EXPORTED_TARGETS} DESTINATION - ${LABSTOR_INSTALL_DATA_DIR}/cmake/hermes + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) #----------------------------------------------------------------------------- # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- -set(LABSTOR_EXPORTED_LIBS +set(HERMES_EXPORTED_LIBS worch_queue_round_robin - ${LABSTOR_EXPORTED_LIBS}) -if(NOT LABSTOR_EXTERNALLY_CONFIGURED) + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( TARGETS - ${LABSTOR_EXPORTED_LIBS} + ${HERMES_EXPORTED_LIBS} FILE - ${LABSTOR_EXPORTED_TARGETS}.cmake + ${HERMES_EXPORTED_TARGETS}.cmake ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(worch_queue_round_robin) endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 670a02ac7..ca96d7b40 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,4 @@ cmake_minimum_required(VERSION 3.10) -project(labstor) +project(hermes) add_subdirectory(unit) \ No newline at end of file diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index dbfd08a04..ced63696c 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10) -project(labstor) +project(hermes) set(CMAKE_CXX_STANDARD 17) diff --git a/test/unit/boost/CMakeLists.txt b/test/unit/boost/CMakeLists.txt index 81746e03f..b3b17fcde 100644 --- a/test/unit/boost/CMakeLists.txt +++ b/test/unit/boost/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10) -project(labstor) +project(hermes) set(CMAKE_CXX_STANDARD 17) @@ -13,9 +13,9 @@ add_executable(test_boost_exec test_boost.cc ) add_dependencies(test_boost_exec - ${Labstor_RUNTIME_DEPS} hermes) + ${Hermes_RUNTIME_DEPS} hermes) target_link_libraries(test_boost_exec - ${Labstor_RUNTIME_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX) + ${Hermes_RUNTIME_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX) #------------------------------------------------------------------------------ # Test Cases @@ -30,14 +30,14 @@ add_test(NAME test_boost COMMAND install(TARGETS test_boost_exec EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) #----------------------------------------------------------------------------- # Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(test_boost_exec) endif() diff --git a/test/unit/hermes/CMakeLists.txt b/test/unit/hermes/CMakeLists.txt index 7230f31f6..06e1418b4 100644 --- a/test/unit/hermes/CMakeLists.txt +++ b/test/unit/hermes/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10) -project(labstor) +project(hermes) set(CMAKE_CXX_STANDARD 17) @@ -13,9 +13,9 @@ add_executable(test_hermes_exec test_bucket.cc ) add_dependencies(test_hermes_exec - ${Labstor_CLIENT_DEPS} hermes) + ${Hermes_CLIENT_DEPS} hermes) target_link_libraries(test_hermes_exec - ${Labstor_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) + ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) #------------------------------------------------------------------------------ # Test Cases @@ -31,14 +31,14 @@ add_test(NAME test_ipc COMMAND install(TARGETS test_hermes_exec EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) #----------------------------------------------------------------------------- # Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(test_messages) endif() diff --git a/test/unit/ipc/CMakeLists.txt b/test/unit/ipc/CMakeLists.txt index 27f886489..f3cafba1e 100644 --- a/test/unit/ipc/CMakeLists.txt +++ b/test/unit/ipc/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10) -project(labstor) +project(hermes) set(CMAKE_CXX_STANDARD 17) @@ -15,9 +15,9 @@ add_executable(test_ipc_exec test_serialize.cc ) -add_dependencies(test_ipc_exec ${Labstor_CLIENT_DEPS}) +add_dependencies(test_ipc_exec ${Hermes_CLIENT_DEPS}) target_link_libraries(test_ipc_exec - ${Labstor_CLIENT_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) + ${Hermes_CLIENT_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) #------------------------------------------------------------------------------ # Test Cases @@ -33,14 +33,14 @@ add_test(NAME test_ipc COMMAND install(TARGETS test_ipc_exec EXPORT - ${LABSTOR_EXPORTED_TARGETS} - LIBRARY DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${LABSTOR_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${LABSTOR_INSTALL_BIN_DIR}) + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) #----------------------------------------------------------------------------- # Coverage #----------------------------------------------------------------------------- -if(LABSTOR_ENABLE_COVERAGE) +if(HERMES_ENABLE_COVERAGE) set_coverage_flags(test_messages) endif() From e860ec770d36d0b90767c0a46edaad94c6268434 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 16:49:00 -0500 Subject: [PATCH 124/191] Fix cmake slightly --- CMake/HermesConfig.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index 9ca777914..4541888b0 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -20,7 +20,7 @@ if( Hermes_INCLUDE_DIR ) #----------------------------------------------------------------------------- find_library( Hermes_LIBRARY - NAMES Hermes_client Hermes_runtime + NAMES labstor_client labstor_runtime ) # HermesShm From b14d937646f03c40daa9b11f56547d605916feac Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 17:02:50 -0500 Subject: [PATCH 125/191] Test serialized put + get --- tasks/hermes/include/hermes/bucket.h | 15 +++++++------- test/unit/hermes/test_bucket.cc | 31 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 66da72dc1..c24b73025 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -269,7 +269,7 @@ class Bucket { BlobId Put(const std::string &blob_name, const T &blob, Context &ctx) { - if (std::is_same_v) { + if constexpr(std::is_same_v) { return BasePut(blob_name, BlobId::GetNull(), blob, 0, ctx); } else { return SrlBasePut(blob_name, BlobId::GetNull(), blob, ctx); @@ -283,7 +283,7 @@ class Bucket { BlobId Put(const BlobId &blob_id, const T &blob, Context &ctx) { - if (std::is_same_v) { + if constexpr(std::is_same_v) { return BasePut("", blob_id, blob, 0, ctx); } else { return SrlBasePut("", blob_id, blob, ctx); @@ -447,11 +447,12 @@ class Bucket { const BlobId &orig_blob_id, T &data, Context &ctx) { - std::stringstream ss; + Blob blob; + BlobId blob_id = BaseGet(blob_name, orig_blob_id, blob, 0, ctx); + std::stringstream ss(std::string(blob.data(), blob.size())); cereal::BinaryInputArchive ar(ss); ar >> data; - Blob blob(ss.str()); - return BaseGet(blob_name, orig_blob_id, blob, 0, ctx); + return blob_id; } /** @@ -461,7 +462,7 @@ class Bucket { BlobId Get(const std::string &blob_name, T &blob, Context &ctx) { - if (std::is_same_v) { + if constexpr(std::is_same_v) { return BaseGet(blob_name, BlobId::GetNull(), blob, 0, ctx); } else { return SrlBaseGet(blob_name, BlobId::GetNull(), blob, ctx); @@ -475,7 +476,7 @@ class Bucket { BlobId Get(const BlobId &blob_id, T &blob, Context &ctx) { - if (std::is_same_v) { + if constexpr(std::is_same_v) { return BaseGet("", blob_id, blob, 0, ctx); } else { return SrlBaseGet("", blob_id, blob, ctx); diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index b084edb29..21fb128e9 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -163,6 +163,37 @@ TEST_CASE("TestHermesPartialPutGet") { } } +TEST_CASE("TestHermesSerializedPutGet") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("hello"); + + size_t count_per_proc = 4; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (int rep = 0; rep < 4; ++rep) { + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {} with blob name {}", i, std::to_string(i)); + // Put a blob + std::vector data(1024, i); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), data, ctx); + HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + // Get a blob + std::vector data2(1024, i); + bkt.Get(blob_id, data2, ctx); + REQUIRE(data == data2); + } + } +} + TEST_CASE("TestHermesBlobDestroy") { int rank, nprocs; MPI_Barrier(MPI_COMM_WORLD); From 3a331156c3a7c2bd09da5253ea677bdb25e3451f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 17:12:23 -0500 Subject: [PATCH 126/191] Add async serialized put + get --- tasks/hermes/include/hermes/bucket.h | 14 ++++++++++++-- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index c24b73025..77c899975 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -293,21 +293,31 @@ class Bucket { /** * Put \a blob_name Blob into the bucket * */ + template HSHM_ALWAYS_INLINE void AsyncPut(const std::string &blob_name, const Blob &blob, Context &ctx) { - BasePut(blob_name, BlobId::GetNull(), blob, 0, ctx); + if constexpr(std::is_same_v) { + BasePut(blob_name, BlobId::GetNull(), blob, 0, ctx); + } else { + SrlBasePut(blob_name, BlobId::GetNull(), blob, ctx); + } } /** * Put \a blob_id Blob into the bucket * */ + template HSHM_ALWAYS_INLINE void AsyncPut(const BlobId &blob_id, const Blob &blob, Context &ctx) { - BasePut("", BlobId::GetNull(), blob, 0, ctx); + if constexpr(std::is_same_v) { + BasePut("", blob_id, blob, 0, ctx); + } else { + SrlBasePut("", blob_id, blob, ctx); + } } /** diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 35740abb9..0705260b9 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -388,6 +388,24 @@ struct GetBlobTask : public Task, TaskFlags page_size_ = ctx.page_size_; } + /** Convert data to a data structure */ + template + HSHM_ALWAYS_INLINE + void Get(T &obj) { + char *data = LABSTOR_CLIENT->GetPrivatePointer(data_); + std::stringstream ss(std::string(data, data_size_)); + cereal::BinaryInputArchive ar(ss); + ar >> obj; + } + + /** Convert data to a data structure */ + template + HSHM_ALWAYS_INLINE + T Get() { + T obj; + return Get(obj); + } + /** Destructor */ ~GetBlobTask() { HSHM_DESTROY_AR(blob_name_); From c1053adf6879bc0c4c27a49be540b49ec9e5a365 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 19:47:06 -0500 Subject: [PATCH 127/191] Add boost --- CMake/HermesConfig.cmake | 12 +++++++++--- tasks/hermes/include/hermes/bucket.h | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index 4541888b0..a84375aad 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -9,7 +9,7 @@ find_path( Hermes_INCLUDE_DIR - Hermes/Hermes_types.h + hermes/hermes_types.h ) if( Hermes_INCLUDE_DIR ) @@ -53,6 +53,12 @@ if( Hermes_INCLUDE_DIR ) message(STATUS "found cereal") endif() + # Boost + find_package(Boost REQUIRED COMPONENTS regex system filesystem fiber REQUIRED) + if (Boost_FOUND) + message(STATUS "found boost at ${Boost_INCLUDE_DIRS}") + endif() + #----------------------------------------------------------------------------- # Mark hermes as found and set all needed packages #----------------------------------------------------------------------------- @@ -61,12 +67,12 @@ if( Hermes_INCLUDE_DIR ) get_filename_component(Hermes_LIBRARY_DIRS ${Hermes_LIBRARY} PATH) # Set uncached variables as per standard. set(Hermes_FOUND ON) - set(Hermes_INCLUDE_DIRS ${Hermes_INCLUDE_DIR}) + set(Hermes_INCLUDE_DIRS ${Boost_INCLUDE_DIRS} ${Hermes_INCLUDE_DIR}) set(Hermes_LIBRARIES ${HermesShm_LIBRARIES} yaml-cpp cereal::cereal - -ldl -lrt -lc -pthread ${Hermes_LIBRARY}) + -ldl -lrt -lc -pthread ${Boost_LIBRARIES} ${Hermes_LIBRARY}) set(Hermes_CLIENT_LIBRARIES ${Hermes_LIBRARIES}) endif(Hermes_LIBRARY) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 77c899975..d5a96032c 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -370,6 +370,23 @@ class Bucket { bkt_mdm_->AppendBlobRoot(id_, blob.size(), p.shm_, page_size, ctx.blob_score_, ctx.node_id_, ctx); } + /** + * Reorganize a blob to a new score or node + * */ + void ReorganizeBlob(const BlobId &blob_id, + float score) { + blob_mdm_->AsyncReorganizeBlobRoot(id_, blob_id, score, 0); + } + + /** + * Reorganize a blob to a new score or node + * */ + void ReorganizeBlob(const BlobId &blob_id, + float score, + Context &ctx) { + blob_mdm_->AsyncReorganizeBlobRoot(id_, blob_id, score, 0); + } + /** * Reorganize a blob to a new score or node * */ From 9d9c7400b46b2e81c1bd37126b10c5cee836ee7d Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 19:55:23 -0500 Subject: [PATCH 128/191] Fix the straggling header in hshm_queue --- include/labstor/queue_manager/queues/hshm_queue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/labstor/queue_manager/queues/hshm_queue.h b/include/labstor/queue_manager/queues/hshm_queue.h index a3ee79200..bc7efad5b 100644 --- a/include/labstor/queue_manager/queues/hshm_queue.h +++ b/include/labstor/queue_manager/queues/hshm_queue.h @@ -5,7 +5,7 @@ #ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ #define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ -#include "include/labstor/queue_manager/queue.h" +#include "labstor/queue_manager/queue.h" namespace labstor { From 772dae79b3fceb2ba4abaafbc4a331a3a7b6f39e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 20:00:58 -0500 Subject: [PATCH 129/191] Add thallium --- CMake/HermesConfig.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index a84375aad..9407a4842 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -72,7 +72,9 @@ if( Hermes_INCLUDE_DIR ) ${HermesShm_LIBRARIES} yaml-cpp cereal::cereal - -ldl -lrt -lc -pthread ${Boost_LIBRARIES} ${Hermes_LIBRARY}) + -ldl -lrt -lc -pthread + thallium + ${Boost_LIBRARIES} ${Hermes_LIBRARY}) set(Hermes_CLIENT_LIBRARIES ${Hermes_LIBRARIES}) endif(Hermes_LIBRARY) From 5745dff42b3b1c16314086e7fee95b0edc9722f9 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 20:15:53 -0500 Subject: [PATCH 130/191] Add compile tester --- CMake/HermesConfig.cmake | 6 ++++++ test/unit/CMakeLists.txt | 2 +- test/unit/external/CMakeLists.txt | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 test/unit/external/CMakeLists.txt diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index 9407a4842..5b13104c1 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -59,6 +59,12 @@ if( Hermes_INCLUDE_DIR ) message(STATUS "found boost at ${Boost_INCLUDE_DIRS}") endif() + # Thallium + find_package(thallium CONFIG REQUIRED) + if(thallium_FOUND) + message(STATUS "found thallium at ${thallium_DIR}") + endif() + #----------------------------------------------------------------------------- # Mark hermes as found and set all needed packages #----------------------------------------------------------------------------- diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index ced63696c..da3d58de3 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -8,4 +8,4 @@ include_directories(${CMAKE_SOURCE_DIR}/tasks/labstor_admin/include) add_subdirectory(ipc) add_subdirectory(hermes) add_subdirectory(hermes_adapters) -add_subdirectory(boost) \ No newline at end of file +add_subdirectory(boost) diff --git a/test/unit/external/CMakeLists.txt b/test/unit/external/CMakeLists.txt new file mode 100644 index 000000000..9e8d56c68 --- /dev/null +++ b/test/unit/external/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.10) +project(hermes) + +set(CMAKE_CXX_STANDARD 17) +find_package(Hermes REQUIRED) +find_package(MPI REQUIRED COMPONENTS C CXX) +include_directories(Hermes_INCLUDE_DIRS) +add_executable(test_hermes_external_compile + ../../main_mpi.cc + ../hermes/test_init.cc + ../hermes/test_bucket.cc +) +target_link_libraries(test_hermes_external_compile + ${Hermes_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX) +message("${Hermes_LIBRARIES}") + From 9479ae09ec289e586c0195f6a35aff62e7c7cfeb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 20:18:45 -0500 Subject: [PATCH 131/191] Fix some headers --- tasks/hermes_adapters/include/hermes_adapters/adapter_types.h | 2 +- tasks/hermes_adapters/posix/posix_api.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h b/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h index 5e4cabaeb..2bc971455 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h +++ b/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h @@ -13,7 +13,7 @@ #ifndef HERMES_ADAPTER_ADAPTER_TYPES_H_ #define HERMES_ADAPTER_ADAPTER_TYPES_H_ -#include "../../posix/posix_api.h" +#include "hermes_adapters/posix/posix_api.h" namespace hermes::adapter { diff --git a/tasks/hermes_adapters/posix/posix_api.h b/tasks/hermes_adapters/posix/posix_api.h index 932ca5953..3134df732 100644 --- a/tasks/hermes_adapters/posix/posix_api.h +++ b/tasks/hermes_adapters/posix/posix_api.h @@ -19,7 +19,7 @@ #include #include #include -#include "../include/hermes_adapters/real_api.h" +#include "hermes_adapters/real_api.h" #ifndef O_TMPFILE #define O_TMPFILE 0 From 6580ff04e95e01dded13769ecd28f889ce608e1b Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 21:03:11 -0500 Subject: [PATCH 132/191] Install adapter headers in hermes_adapters dir --- tasks/hermes_adapters/filesystem/CMakeLists.txt | 13 +++++++++++++ tasks/hermes_adapters/mpiio/CMakeLists.txt | 14 ++++++++++++++ tasks/hermes_adapters/posix/CMakeLists.txt | 13 +++++++++++++ tasks/hermes_adapters/stdio/CMakeLists.txt | 13 +++++++++++++ tasks/hermes_adapters/vfd/CMakeLists.txt | 13 +++++++++++++ test/unit/external/CMakeLists.txt | 3 ++- 6 files changed, 68 insertions(+), 1 deletion(-) diff --git a/tasks/hermes_adapters/filesystem/CMakeLists.txt b/tasks/hermes_adapters/filesystem/CMakeLists.txt index c9564a624..8713f8080 100644 --- a/tasks/hermes_adapters/filesystem/CMakeLists.txt +++ b/tasks/hermes_adapters/filesystem/CMakeLists.txt @@ -52,6 +52,19 @@ EXPORT ( ) endif() +#----------------------------------------------------------------------------- +# Install headers +#----------------------------------------------------------------------------- +file(GLOB_RECURSE HERMES_HEADERS "*.h") +install( + FILES + ${HERMES_HEADERS} + DESTINATION + ${CMAKE_INSTALL_PREFIX}/hermes_adapters/filesystem + COMPONENT + headers +) + #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- diff --git a/tasks/hermes_adapters/mpiio/CMakeLists.txt b/tasks/hermes_adapters/mpiio/CMakeLists.txt index 595994051..ceefc7c4c 100644 --- a/tasks/hermes_adapters/mpiio/CMakeLists.txt +++ b/tasks/hermes_adapters/mpiio/CMakeLists.txt @@ -34,6 +34,20 @@ install( ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) + +#----------------------------------------------------------------------------- +# Install headers +#----------------------------------------------------------------------------- +file(GLOB_RECURSE HERMES_HEADERS "*.h") +install( + FILES + ${HERMES_HEADERS} + DESTINATION + ${CMAKE_INSTALL_PREFIX}/hermes_adapters/stdio + COMPONENT + headers +) + #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- diff --git a/tasks/hermes_adapters/posix/CMakeLists.txt b/tasks/hermes_adapters/posix/CMakeLists.txt index d5e9eadad..d0de57890 100644 --- a/tasks/hermes_adapters/posix/CMakeLists.txt +++ b/tasks/hermes_adapters/posix/CMakeLists.txt @@ -52,6 +52,19 @@ EXPORT ( ) endif() +#----------------------------------------------------------------------------- +# Install headers +#----------------------------------------------------------------------------- +file(GLOB_RECURSE HERMES_HEADERS "*.h") +install( + FILES + ${HERMES_HEADERS} + DESTINATION + ${CMAKE_INSTALL_PREFIX}/hermes_adapters/posix + COMPONENT + headers +) + #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- diff --git a/tasks/hermes_adapters/stdio/CMakeLists.txt b/tasks/hermes_adapters/stdio/CMakeLists.txt index f4a9e7743..5af45322f 100644 --- a/tasks/hermes_adapters/stdio/CMakeLists.txt +++ b/tasks/hermes_adapters/stdio/CMakeLists.txt @@ -52,6 +52,19 @@ if(NOT HERMES_EXTERNALLY_CONFIGURED) ) endif() +#----------------------------------------------------------------------------- +# Install headers +#----------------------------------------------------------------------------- +file(GLOB_RECURSE HERMES_HEADERS "*.h") +install( + FILES + ${HERMES_HEADERS} + DESTINATION + ${CMAKE_INSTALL_PREFIX}/hermes_adapters/stdio + COMPONENT + headers +) + #----------------------------------------------------------------------------- # Add Target(s) to Coverage #----------------------------------------------------------------------------- diff --git a/tasks/hermes_adapters/vfd/CMakeLists.txt b/tasks/hermes_adapters/vfd/CMakeLists.txt index 0836028c9..5addc892a 100644 --- a/tasks/hermes_adapters/vfd/CMakeLists.txt +++ b/tasks/hermes_adapters/vfd/CMakeLists.txt @@ -173,3 +173,16 @@ set(HDF5_HERMES_VFD_INCLUDES_INSTALL_TIME ${CMAKE_CURRENT_BINARY_DIR} ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} PARENT_SCOPE) + +#----------------------------------------------------------------------------- +# Install headers +#----------------------------------------------------------------------------- +file(GLOB_RECURSE HERMES_HEADERS "*.h") +install( + FILES + ${HERMES_HEADERS} + DESTINATION + ${CMAKE_INSTALL_PREFIX}/hermes_adapters/vfd + COMPONENT + headers +) diff --git a/test/unit/external/CMakeLists.txt b/test/unit/external/CMakeLists.txt index 9e8d56c68..5e9284ba1 100644 --- a/test/unit/external/CMakeLists.txt +++ b/test/unit/external/CMakeLists.txt @@ -1,10 +1,11 @@ cmake_minimum_required(VERSION 3.10) project(hermes) +include_directories(/home/lukemartinlogan/Documents/Projects/PhD/hermes/test/unit) set(CMAKE_CXX_STANDARD 17) find_package(Hermes REQUIRED) find_package(MPI REQUIRED COMPONENTS C CXX) -include_directories(Hermes_INCLUDE_DIRS) +include_directories(${Hermes_INCLUDE_DIRS}) add_executable(test_hermes_external_compile ../../main_mpi.cc ../hermes/test_init.cc From bb61ef9ff2964b18a0e2634343fb785c546e413d Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 21:10:37 -0500 Subject: [PATCH 133/191] POSIX adapter enabled by default --- tasks/hermes_adapters/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tasks/hermes_adapters/CMakeLists.txt b/tasks/hermes_adapters/CMakeLists.txt index e0549ae9f..ad7a2441d 100644 --- a/tasks/hermes_adapters/CMakeLists.txt +++ b/tasks/hermes_adapters/CMakeLists.txt @@ -3,9 +3,7 @@ #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(filesystem) -if (HERMES_ENABLE_POSIX_ADAPTER) - add_subdirectory(posix) -endif() +add_subdirectory(posix) if (HERMES_ENABLE_STDIO_ADAPTER) add_subdirectory(stdio) endif() From 13e212ec1237f0f3061e4acb7c8b4ad3ad0ece02 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 21:16:32 -0500 Subject: [PATCH 134/191] include --- tasks/hermes_adapters/filesystem/CMakeLists.txt | 2 +- tasks/hermes_adapters/mpiio/CMakeLists.txt | 2 +- tasks/hermes_adapters/posix/CMakeLists.txt | 2 +- tasks/hermes_adapters/stdio/CMakeLists.txt | 2 +- tasks/hermes_adapters/vfd/CMakeLists.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks/hermes_adapters/filesystem/CMakeLists.txt b/tasks/hermes_adapters/filesystem/CMakeLists.txt index 8713f8080..bdcd61ac0 100644 --- a/tasks/hermes_adapters/filesystem/CMakeLists.txt +++ b/tasks/hermes_adapters/filesystem/CMakeLists.txt @@ -60,7 +60,7 @@ install( FILES ${HERMES_HEADERS} DESTINATION - ${CMAKE_INSTALL_PREFIX}/hermes_adapters/filesystem + ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/filesystem COMPONENT headers ) diff --git a/tasks/hermes_adapters/mpiio/CMakeLists.txt b/tasks/hermes_adapters/mpiio/CMakeLists.txt index ceefc7c4c..448ba7593 100644 --- a/tasks/hermes_adapters/mpiio/CMakeLists.txt +++ b/tasks/hermes_adapters/mpiio/CMakeLists.txt @@ -43,7 +43,7 @@ install( FILES ${HERMES_HEADERS} DESTINATION - ${CMAKE_INSTALL_PREFIX}/hermes_adapters/stdio + ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/stdio COMPONENT headers ) diff --git a/tasks/hermes_adapters/posix/CMakeLists.txt b/tasks/hermes_adapters/posix/CMakeLists.txt index d0de57890..7d3aea406 100644 --- a/tasks/hermes_adapters/posix/CMakeLists.txt +++ b/tasks/hermes_adapters/posix/CMakeLists.txt @@ -60,7 +60,7 @@ install( FILES ${HERMES_HEADERS} DESTINATION - ${CMAKE_INSTALL_PREFIX}/hermes_adapters/posix + ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/posix COMPONENT headers ) diff --git a/tasks/hermes_adapters/stdio/CMakeLists.txt b/tasks/hermes_adapters/stdio/CMakeLists.txt index 5af45322f..3cadb6c7b 100644 --- a/tasks/hermes_adapters/stdio/CMakeLists.txt +++ b/tasks/hermes_adapters/stdio/CMakeLists.txt @@ -60,7 +60,7 @@ install( FILES ${HERMES_HEADERS} DESTINATION - ${CMAKE_INSTALL_PREFIX}/hermes_adapters/stdio + ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/stdio COMPONENT headers ) diff --git a/tasks/hermes_adapters/vfd/CMakeLists.txt b/tasks/hermes_adapters/vfd/CMakeLists.txt index 5addc892a..24cc3effa 100644 --- a/tasks/hermes_adapters/vfd/CMakeLists.txt +++ b/tasks/hermes_adapters/vfd/CMakeLists.txt @@ -182,7 +182,7 @@ install( FILES ${HERMES_HEADERS} DESTINATION - ${CMAKE_INSTALL_PREFIX}/hermes_adapters/vfd + ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/vfd COMPONENT headers ) From 76f70cdec0c2a033b975f4d45b4a74d388890ef6 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 21:32:58 -0500 Subject: [PATCH 135/191] Add Hermes to the hermesconfig.cmake --- CMake/HermesConfig.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index 5b13104c1..a5e08b26f 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -80,6 +80,7 @@ if( Hermes_INCLUDE_DIR ) cereal::cereal -ldl -lrt -lc -pthread thallium + hermes ${Boost_LIBRARIES} ${Hermes_LIBRARY}) set(Hermes_CLIENT_LIBRARIES ${Hermes_LIBRARIES}) endif(Hermes_LIBRARY) From ad40a3844baf94afa8fdefc68f2d55790cfa7316 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 29 Sep 2023 23:12:28 -0500 Subject: [PATCH 136/191] Begin making staging trait --- tasks/data_stager/CMakeLists.txt | 10 ++ .../include/data_stager/data_stager.h | 72 ++++++++ .../data_stager/data_stager_lib_exec.h | 163 ++++++++++++++++++ .../include/data_stager/data_stager_methods.h | 9 + .../data_stager/data_stager_methods.yaml | 1 + .../include/data_stager/data_stager_tasks.h | 119 +++++++++++++ tasks/data_stager/src/CMakeLists.txt | 54 ++++++ tasks/data_stager/src/data_stager.cc | 33 ++++ tasks/hermes/include/hermes/hermes_types.h | 6 + 9 files changed, 467 insertions(+) create mode 100644 tasks/data_stager/CMakeLists.txt create mode 100644 tasks/data_stager/include/data_stager/data_stager.h create mode 100644 tasks/data_stager/include/data_stager/data_stager_lib_exec.h create mode 100644 tasks/data_stager/include/data_stager/data_stager_methods.h create mode 100644 tasks/data_stager/include/data_stager/data_stager_methods.yaml create mode 100644 tasks/data_stager/include/data_stager/data_stager_tasks.h create mode 100644 tasks/data_stager/src/CMakeLists.txt create mode 100644 tasks/data_stager/src/data_stager.cc diff --git a/tasks/data_stager/CMakeLists.txt b/tasks/data_stager/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks/data_stager/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h new file mode 100644 index 000000000..201b76123 --- /dev/null +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -0,0 +1,72 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_data_stager_H_ +#define LABSTOR_data_stager_H_ + +#include "data_stager_tasks.h" + +namespace labstor::data_stager { + +/** Create data_stager requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Async create a task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info); + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate) + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + auto *task = AsyncCreateRoot(std::forward(args)...); + task->Wait(); + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Call a custom method */ + HSHM_ALWAYS_INLINE + void AsyncCustomConstruct(CustomTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_); + } + HSHM_ALWAYS_INLINE + void CustomRoot(const DomainId &domain_id) { + LPointer> task = AsyncCustomRoot(domain_id); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Custom); +}; + +} // namespace labstor + +#endif // LABSTOR_data_stager_H_ diff --git a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h new file mode 100644 index 000000000..a74b663a3 --- /dev/null +++ b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h @@ -0,0 +1,163 @@ +#ifndef LABSTOR_data_stager_LIB_EXEC_H_ +#define LABSTOR_data_stager_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task, RunContext &ctx) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task), ctx); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task), ctx); + break; + } + case Method::kCustom: { + Custom(reinterpret_cast(task), ctx); + break; + } + } +} +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kDestruct: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kCustom: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kCustom: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kCustom: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_data_stager_METHODS_H_ \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_methods.h b/tasks/data_stager/include/data_stager/data_stager_methods.h new file mode 100644 index 000000000..583bb358a --- /dev/null +++ b/tasks/data_stager/include/data_stager/data_stager_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_data_stager_METHODS_H_ +#define LABSTOR_data_stager_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kCustom = kLast + 0; +}; + +#endif // LABSTOR_data_stager_METHODS_H_ \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_methods.yaml b/tasks/data_stager/include/data_stager/data_stager_methods.yaml new file mode 100644 index 000000000..b1b54e2a4 --- /dev/null +++ b/tasks/data_stager/include/data_stager/data_stager_methods.yaml @@ -0,0 +1 @@ +kCustom: 0 \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_tasks.h b/tasks/data_stager/include/data_stager/data_stager_tasks.h new file mode 100644 index 000000000..0883f1c36 --- /dev/null +++ b/tasks/data_stager/include/data_stager/data_stager_tasks.h @@ -0,0 +1,119 @@ +// +// Created by lukemartinlogan on 8/11/23. +// + +#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ +#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::data_stager { + +#include "data_stager_methods.h" +#include "labstor/labstor_namespace.h" +using labstor::proc_queue::TypedPushTask; +using labstor::proc_queue::PushTask; + +/** + * A task to create data_stager + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) + : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "data_stager", id, queue_info) { + // Custom params + } + + HSHM_ALWAYS_INLINE + ~ConstructTask() { + // Custom params + } +}; + +/** A task to destroy data_stager */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) + : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in data_stager + * */ +struct CustomTask : public Task, TaskFlags { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kCustom; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Custom params + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor::data_stager + +#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ diff --git a/tasks/data_stager/src/CMakeLists.txt b/tasks/data_stager/src/CMakeLists.txt new file mode 100644 index 000000000..07eca417a --- /dev/null +++ b/tasks/data_stager/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(data_stager SHARED + data_stager.cc) +add_dependencies(data_stager ${Hermes_RUNTIME_DEPS}) +target_link_libraries(data_stager ${Hermes_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + data_stager + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${HERMES_EXPORTED_TARGETS} + DESTINATION + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${HERMES_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(HERMES_EXPORTED_LIBS + data_stager + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${HERMES_EXPORTED_LIBS} + FILE + ${HERMES_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(data_stager) +endif() diff --git a/tasks/data_stager/src/data_stager.cc b/tasks/data_stager/src/data_stager.cc new file mode 100644 index 000000000..90356d474 --- /dev/null +++ b/tasks/data_stager/src/data_stager.cc @@ -0,0 +1,33 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "data_stager/data_stager.h" + +namespace labstor::data_stager { + +class Server : public TaskLib { + public: + Server() = default; + + void Construct(ConstructTask *task, RunContext &ctx) { + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task, RunContext &ctx) { + task->SetModuleComplete(); + } + + void Custom(CustomTask *task, RunContext &ctx) { + task->SetModuleComplete(); + } + + public: +#include "data_stager/data_stager_lib_exec.h" +}; + +} // namespace labstor::data_stager + +LABSTOR_TASK_CC(labstor::data_stager::Server, "data_stager"); diff --git a/tasks/hermes/include/hermes/hermes_types.h b/tasks/hermes/include/hermes/hermes_types.h index ab29f27fe..1de093c97 100644 --- a/tasks/hermes/include/hermes/hermes_types.h +++ b/tasks/hermes/include/hermes/hermes_types.h @@ -46,6 +46,12 @@ typedef TaskStateId TargetId; /** Represents a trait */ typedef TaskStateId TraitId; +/** Different categories of traits */ +enum class TraitType { + kStagingTrait, + kProducerOpTrait +}; + /** Represents a blob */ typedef hshm::charbuf Blob; From f9027969f53a1af28a3bb89d5798af48a2d255b7 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 30 Sep 2023 04:29:56 -0500 Subject: [PATCH 137/191] Continue staging --- benchmark/test_latency.cc | 6 +- codegen/refresh_methods | 4 +- include/labstor/labstor_namespace.h | 5 +- include/labstor/task_registry/task_lib.h | 6 +- src/worker.cc | 36 ++--- tasks/CMakeLists.txt | 1 + tasks/bdev/include/bdev/bdev_lib_exec.h | 18 +-- .../include/data_stager/data_stager.h | 34 +++-- .../data_stager/data_stager_lib_exec.h | 122 ++++++++++++---- .../include/data_stager/data_stager_methods.h | 10 +- .../data_stager/data_stager_methods.yaml | 4 +- .../include/data_stager/data_stager_tasks.h | 135 ++++++++++++++++-- tasks/data_stager/src/data_stager.cc | 91 +++++++++++- tasks/hermes/include/hermes/config_manager.h | 2 +- tasks/hermes/include/hermes/hermes.h | 2 +- .../filesystem/filesystem_io_client.h | 2 +- .../filesystem/filesystem_mdm.h | 2 +- .../hermes_adapters_lib_exec.h | 8 +- tasks/hermes_adapters/mpiio/mpiio_api.h | 2 +- tasks/hermes_adapters/mpiio/mpiio_fs_api.h | 2 +- tasks/hermes_adapters/mpiio/mpiio_io_client.h | 2 +- tasks/hermes_adapters/posix/posix_api.h | 2 +- tasks/hermes_adapters/posix/posix_fs_api.h | 2 +- tasks/hermes_adapters/posix/posix_io_client.h | 2 +- tasks/hermes_adapters/src/hermes_adapters.cc | 6 +- tasks/hermes_adapters/stdio/stdio_api.h | 2 +- tasks/hermes_adapters/stdio/stdio_fs_api.h | 2 +- tasks/hermes_adapters/stdio/stdio_io_client.h | 2 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 2 +- .../hermes_blob_mdm_lib_exec.h | 36 ++--- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 92 ++++++------ .../hermes_bucket_mdm_lib_exec.h | 32 ++--- .../src/hermes_bucket_mdm.cc | 60 ++++---- .../include/hermes_mdm/hermes_mdm_lib_exec.h | 6 +- tasks/hermes_mdm/src/hermes_mdm.cc | 4 +- tasks/posix_bdev/src/posix_bdev.cc | 16 +-- tasks/ram_bdev/src/ram_bdev.cc | 16 +-- .../include/TASK_NAME/TASK_NAME_lib_exec.h | 8 +- tasks_required/TASK_NAME/src/TASK_NAME.cc | 6 +- .../labstor_admin/labstor_admin_lib_exec.h | 20 +-- .../labstor_admin/src/labstor_admin.cc | 18 +-- .../include/proc_queue/proc_queue_lib_exec.h | 8 +- tasks_required/proc_queue/src/proc_queue.cc | 6 +- .../remote_queue/remote_queue_lib_exec.h | 8 +- .../remote_queue/src/remote_queue.cc | 6 +- .../small_message/small_message_lib_exec.h | 12 +- .../small_message/src/small_message.cc | 10 +- .../worch_proc_round_robin_lib_exec.h | 8 +- .../src/worch_proc_round_robin.cc | 6 +- .../worch_queue_round_robin_lib_exec.h | 8 +- .../src/worch_queue_round_robin.cc | 6 +- 51 files changed, 596 insertions(+), 310 deletions(-) diff --git a/benchmark/test_latency.cc b/benchmark/test_latency.cc index c6b8410bc..57a536861 100644 --- a/benchmark/test_latency.cc +++ b/benchmark/test_latency.cc @@ -328,7 +328,7 @@ TEST_CASE("TestTimespecLatency") { // ctx.page_size_ = 4096; // std::string data(ctx.page_size_, 0); // for (size_t i = 0; i < ops; ++i) { -// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, ctx); +// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, rctx); // } // t.Pause(); // @@ -349,12 +349,12 @@ TEST_CASE("TestTimespecLatency") { // ctx.page_size_ = 4096; // std::string data(ctx.page_size_, 0); // for (size_t i = 0; i < ops; ++i) { -// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, ctx); +// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, rctx); // } // // t.Resume(); // for (size_t i = 0; i < ops; ++i) { -// HERMES_FILESYSTEM_API->Read(bkt, data.data(), i * ctx.page_size_, data.size(), false, ctx); +// HERMES_FILESYSTEM_API->Read(bkt, data.data(), i * ctx.page_size_, data.size(), false, rctx); // } // t.Pause(); // diff --git a/codegen/refresh_methods b/codegen/refresh_methods index b10e94ed5..7929025d6 100755 --- a/codegen/refresh_methods +++ b/codegen/refresh_methods @@ -55,13 +55,13 @@ def refresh_methods(TASK_ROOT): ''] ## Create the Run method lines += ['/** Execute a task */', - 'void Run(u32 method, Task *task, RunContext &ctx) override {', + 'void Run(u32 method, Task *task, RunContext &rctx) override {', ' switch (method) {'] for method_enum_name, method_off in methods: method_name = method_enum_name.replace('k', '', 1) task_name = method_name + "Task" lines += [f' case Method::{method_enum_name}: {{', - f' {method_name}(reinterpret_cast<{task_name} *>(task), ctx);', + f' {method_name}(reinterpret_cast<{task_name} *>(task), rctx);', f' break;', f' }}'] lines += [' }'] diff --git a/include/labstor/labstor_namespace.h b/include/labstor/labstor_namespace.h index 230834691..da84cfd71 100644 --- a/include/labstor/labstor_namespace.h +++ b/include/labstor/labstor_namespace.h @@ -15,7 +15,10 @@ using labstor::Task; using labstor::TaskPointer; using labstor::MultiQueue; using labstor::PriorityInfo; -using labstor::Task; +using labstor::TaskNode; +using labstor::DomainId; +using labstor::TaskStateId; +using labstor::QueueId; using labstor::TaskFlags; using labstor::DataTransfer; using labstor::TaskLib; diff --git a/include/labstor/task_registry/task_lib.h b/include/labstor/task_registry/task_lib.h index 53caedaf6..e30d9b70b 100644 --- a/include/labstor/task_registry/task_lib.h +++ b/include/labstor/task_registry/task_lib.h @@ -39,7 +39,7 @@ class TaskLib { virtual ~TaskLib() = default; /** Run a method of the task */ - virtual void Run(u32 method, Task *task, RunContext &ctx) = 0; + virtual void Run(u32 method, Task *task, RunContext &rctx) = 0; /** Delete a task */ virtual void Del(u32 method, Task *task) = 0; @@ -97,8 +97,8 @@ typedef const char* (*get_task_lib_name_t)(void); labstor::TaskState *exec = reinterpret_cast( \ new TYPE_UNWRAP(TRAIT_CLASS)()); \ exec->Init(task->id_, state_name); \ - RunContext ctx(0); \ - exec->Run(labstor::TaskMethod::kConstruct, task, ctx); \ + RunContext rctx(0); \ + exec->Run(labstor::TaskMethod::kConstruct, task, rctx); \ return exec; \ } \ const char* get_task_lib_name(void) { return TASK_NAME; } \ diff --git a/src/worker.cc b/src/worker.cc index b164d7d54..9cdaa8f4d 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -55,10 +55,10 @@ void Worker::PollGrouped(WorkEntry &work_entry) { continue; } task = LABSTOR_CLIENT->GetPrivatePointer(entry->p_); - RunContext &ctx = task->ctx_; - ctx.lane_id_ = work_entry.lane_id_; + RunContext &rctx = task->ctx_; + rctx.lane_id_ = work_entry.lane_id_; // Get the task state - TaskState *&exec = ctx.exec_; + TaskState *&exec = rctx.exec_; exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); if (!exec) { HELOG(kFatal, "(node {}) Could not find the task state: {}", @@ -89,24 +89,24 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->UnsetCoroutine(); } else if (task->IsCoroutine()) { if (!task->IsStarted()) { - ctx.stack_ptr_ = malloc(ctx.stack_size_); - if (ctx.stack_ptr_ == nullptr) { + rctx.stack_ptr_ = malloc(rctx.stack_size_); + if (rctx.stack_ptr_ == nullptr) { HILOG(kFatal, "The stack pointer of size {} is NULL", - ctx.stack_size_, ctx.stack_ptr_); + rctx.stack_size_, rctx.stack_ptr_); } - ctx.jmp_.fctx = bctx::make_fcontext( - (char*)ctx.stack_ptr_ + ctx.stack_size_, - ctx.stack_size_, &RunBlocking); + rctx.jmp_.fctx = bctx::make_fcontext( + (char*)rctx.stack_ptr_ + rctx.stack_size_, + rctx.stack_size_, &RunBlocking); task->SetStarted(); } - ctx.jmp_ = bctx::jump_fcontext(ctx.jmp_.fctx, task); + rctx.jmp_ = bctx::jump_fcontext(rctx.jmp_.fctx, task); HILOG(kDebug, "Jumping into function") } else if (task->IsPreemptive()) { task->DisableRun(); entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunPreemptive, task); } else { task->SetStarted(); - exec->Run(task->method_, task, ctx); + exec->Run(task->method_, task, rctx); } } // Cleanup on task completion @@ -116,7 +116,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { entry->complete_ = true; if (task->IsCoroutine()) { // TODO(llogan): verify leak - free(ctx.stack_ptr_); + free(rctx.stack_ptr_); } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); } @@ -130,18 +130,18 @@ void Worker::PollGrouped(WorkEntry &work_entry) { void Worker::RunBlocking(bctx::transfer_t t) { Task *task = reinterpret_cast(t.data); - RunContext &ctx = task->ctx_; - TaskState *&exec = ctx.exec_; - ctx.jmp_ = t; - exec->Run(task->method_, task, ctx); + RunContext &rctx = task->ctx_; + TaskState *&exec = rctx.exec_; + rctx.jmp_ = t; + exec->Run(task->method_, task, rctx); task->Yield(); } void Worker::RunPreemptive(void *data) { Task *task = reinterpret_cast(data); TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); - RunContext ctx(0); - exec->Run(task->method_, task, ctx); + RunContext rctx(0); + exec->Run(task->method_, task, rctx); } } // namespace labstor \ No newline at end of file diff --git a/tasks/CMakeLists.txt b/tasks/CMakeLists.txt index 1c73b9be4..f91850e92 100644 --- a/tasks/CMakeLists.txt +++ b/tasks/CMakeLists.txt @@ -6,3 +6,4 @@ add_subdirectory(hermes_mdm) add_subdirectory(hermes_blob_mdm) add_subdirectory(hermes_bucket_mdm) add_subdirectory(hermes_adapters) +add_subdirectory(data_stager) diff --git a/tasks/bdev/include/bdev/bdev_lib_exec.h b/tasks/bdev/include/bdev/bdev_lib_exec.h index d130e634e..88d49653d 100644 --- a/tasks/bdev/include/bdev/bdev_lib_exec.h +++ b/tasks/bdev/include/bdev/bdev_lib_exec.h @@ -2,38 +2,38 @@ #define LABSTOR_BDEV_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kWrite: { - Write(reinterpret_cast(task), ctx); + Write(reinterpret_cast(task), rctx); break; } case Method::kRead: { - Read(reinterpret_cast(task), ctx); + Read(reinterpret_cast(task), rctx); break; } case Method::kAllocate: { - Allocate(reinterpret_cast(task), ctx); + Allocate(reinterpret_cast(task), rctx); break; } case Method::kFree: { - Free(reinterpret_cast(task), ctx); + Free(reinterpret_cast(task), rctx); break; } case Method::kMonitor: { - Monitor(reinterpret_cast(task), ctx); + Monitor(reinterpret_cast(task), rctx); break; } case Method::kUpdateCapacity: { - UpdateCapacity(reinterpret_cast(task), ctx); + UpdateCapacity(reinterpret_cast(task), rctx); break; } } diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h index 201b76123..d65aa5b58 100644 --- a/tasks/data_stager/include/data_stager/data_stager.h +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -7,7 +7,7 @@ #include "data_stager_tasks.h" -namespace labstor::data_stager { +namespace hermes::data_stager { /** Create data_stager requests */ class Client : public TaskLibClient { @@ -23,7 +23,8 @@ class Client : public TaskLibClient { HSHM_ALWAYS_INLINE LPointer AsyncCreate(const TaskNode &task_node, const DomainId &domain_id, - const std::string &state_name) { + const std::string &state_name, + const TaskStateId &blob_mdm) { id_ = TaskStateId::GetNull(); QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; std::vector queue_info = { @@ -32,7 +33,7 @@ class Client : public TaskLibClient { {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; return LABSTOR_ADMIN->AsyncCreateTaskState( - task_node, domain_id, state_name, id_, queue_info); + task_node, domain_id, state_name, id_, queue_info, blob_mdm); } LABSTOR_TASK_NODE_ROOT(AsyncCreate) template @@ -51,20 +52,35 @@ class Client : public TaskLibClient { LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); } - /** Call a custom method */ + /** Stage in data from a remote source */ HSHM_ALWAYS_INLINE - void AsyncCustomConstruct(CustomTask *task, + void AsyncStageInConstruct(StageInTask *task, const TaskNode &task_node, const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( + LABSTOR_CLIENT->ConstructTask( task, task_node, domain_id, id_); } HSHM_ALWAYS_INLINE - void CustomRoot(const DomainId &domain_id) { - LPointer> task = AsyncCustomRoot(domain_id); + void StageIn(const DomainId &domain_id) { + LPointer> task = AsyncStageInRoot(domain_id); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(Custom); + LABSTOR_TASK_NODE_PUSH_ROOT(StageIn); + + /** Stage out data to a remote source */ + HSHM_ALWAYS_INLINE + void AsyncStageOutConstruct(StageInTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_); + } + HSHM_ALWAYS_INLINE + void StageOut(const DomainId &domain_id) { + LPointer> task = AsyncStageInRoot(domain_id); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(StageOut); }; } // namespace labstor diff --git a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h index a74b663a3..b41a89215 100644 --- a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h +++ b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h @@ -1,19 +1,27 @@ -#ifndef LABSTOR_data_stager_LIB_EXEC_H_ -#define LABSTOR_data_stager_LIB_EXEC_H_ +#ifndef LABSTOR_DATA_STAGER_LIB_EXEC_H_ +#define LABSTOR_DATA_STAGER_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } - case Method::kCustom: { - Custom(reinterpret_cast(task), ctx); + case Method::kRegisterStager: { + RegisterStager(reinterpret_cast(task), rctx); + break; + } + case Method::kStageIn: { + StageIn(reinterpret_cast(task), rctx); + break; + } + case Method::kStageOut: { + StageOut(reinterpret_cast(task), rctx); break; } } @@ -29,8 +37,16 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } - case Method::kCustom: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + case Method::kRegisterStager: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kStageIn: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kStageOut: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -46,8 +62,16 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } - case Method::kCustom: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + case Method::kRegisterStager: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kStageIn: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kStageOut: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -63,8 +87,16 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } - case Method::kCustom: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + case Method::kRegisterStager: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kStageIn: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kStageOut: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -80,8 +112,16 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } - case Method::kCustom: { - ar << *reinterpret_cast(task); + case Method::kRegisterStager: { + ar << *reinterpret_cast(task); + break; + } + case Method::kStageIn: { + ar << *reinterpret_cast(task); + break; + } + case Method::kStageOut: { + ar << *reinterpret_cast(task); break; } } @@ -101,9 +141,19 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } - case Method::kCustom: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); + case Method::kRegisterStager: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kStageIn: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kStageOut: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); break; } } @@ -120,8 +170,16 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } - case Method::kCustom: { - ar << *reinterpret_cast(task); + case Method::kRegisterStager: { + ar << *reinterpret_cast(task); + break; + } + case Method::kStageIn: { + ar << *reinterpret_cast(task); + break; + } + case Method::kStageOut: { + ar << *reinterpret_cast(task); break; } } @@ -138,8 +196,16 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } - case Method::kCustom: { - ar.Deserialize(replica, *reinterpret_cast(task)); + case Method::kRegisterStager: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kStageIn: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kStageOut: { + ar.Deserialize(replica, *reinterpret_cast(task)); break; } } @@ -153,11 +219,17 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kDestruct: { return reinterpret_cast(task)->GetGroup(group); } - case Method::kCustom: { - return reinterpret_cast(task)->GetGroup(group); + case Method::kRegisterStager: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kStageIn: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kStageOut: { + return reinterpret_cast(task)->GetGroup(group); } } return -1; } -#endif // LABSTOR_data_stager_METHODS_H_ \ No newline at end of file +#endif // LABSTOR_DATA_STAGER_METHODS_H_ \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_methods.h b/tasks/data_stager/include/data_stager/data_stager_methods.h index 583bb358a..36700a1a2 100644 --- a/tasks/data_stager/include/data_stager/data_stager_methods.h +++ b/tasks/data_stager/include/data_stager/data_stager_methods.h @@ -1,9 +1,11 @@ -#ifndef LABSTOR_data_stager_METHODS_H_ -#define LABSTOR_data_stager_METHODS_H_ +#ifndef LABSTOR_DATA_STAGER_METHODS_H_ +#define LABSTOR_DATA_STAGER_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { - TASK_METHOD_T kCustom = kLast + 0; + TASK_METHOD_T kRegisterStager = kLast + 0; + TASK_METHOD_T kStageIn = kLast + 1; + TASK_METHOD_T kStageOut = kLast + 2; }; -#endif // LABSTOR_data_stager_METHODS_H_ \ No newline at end of file +#endif // LABSTOR_DATA_STAGER_METHODS_H_ \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_methods.yaml b/tasks/data_stager/include/data_stager/data_stager_methods.yaml index b1b54e2a4..27aa4b7e2 100644 --- a/tasks/data_stager/include/data_stager/data_stager_methods.yaml +++ b/tasks/data_stager/include/data_stager/data_stager_methods.yaml @@ -1 +1,3 @@ -kCustom: 0 \ No newline at end of file +kRegisterStager: 0 +kStageIn: 1 +kStageOut: 2 \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_tasks.h b/tasks/data_stager/include/data_stager/data_stager_tasks.h index 0883f1c36..54aff1314 100644 --- a/tasks/data_stager/include/data_stager/data_stager_tasks.h +++ b/tasks/data_stager/include/data_stager/data_stager_tasks.h @@ -10,8 +10,9 @@ #include "labstor_admin/labstor_admin.h" #include "labstor/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" +#include "hermes/hermes_types.h" -namespace labstor::data_stager { +namespace hermes::data_stager { #include "data_stager_methods.h" #include "labstor/labstor_namespace.h" @@ -23,10 +24,12 @@ using labstor::proc_queue::PushTask; * */ using labstor::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { + TaskStateId blob_mdm_; + /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit ConstructTask(hipc::Allocator *alloc) - : CreateTaskStateTask(alloc) {} + : CreateTaskStateTask(alloc) {} /** Emplace constructor */ HSHM_ALWAYS_INLINE explicit @@ -35,10 +38,12 @@ struct ConstructTask : public CreateTaskStateTask { const DomainId &domain_id, const std::string &state_name, const TaskStateId &id, - const std::vector &queue_info) + const std::vector &queue_info, + const TaskStateId &blob_mdm) : CreateTaskStateTask(alloc, task_node, domain_id, state_name, "data_stager", id, queue_info) { // Custom params + blob_mdm_ = blob_mdm; } HSHM_ALWAYS_INLINE @@ -53,7 +58,7 @@ struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit DestructTask(hipc::Allocator *alloc) - : DestroyTaskStateTask(alloc) {} + : DestroyTaskStateTask(alloc) {} /** Emplace constructor */ HSHM_ALWAYS_INLINE explicit @@ -61,7 +66,106 @@ struct DestructTask : public DestroyTaskStateTask { const TaskNode &task_node, const DomainId &domain_id, TaskStateId &state_id) - : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * Register a new stager + * */ +struct RegisterStagerTask : public Task, TaskFlags { + hermes::BucketId bkt_id_; + hipc::ShmArchive url_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterStagerTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterStagerTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + hermes::BucketId bkt_id, + hshm::charbuf &url) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kStageIn; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Custom params + bkt_id_ = bkt_id; + HSHM_MAKE_AR(url_, alloc, url); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A task to stage in data from a remote source + * */ +struct StageInTask : public Task, TaskFlags { + hermes::BucketId bkt_id_; + hipc::ShmArchive blob_name_; + float score_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + StageInTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + StageInTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kStageIn; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Custom params + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } /** Create group */ HSHM_ALWAYS_INLINE @@ -71,25 +175,30 @@ struct DestructTask : public DestroyTaskStateTask { }; /** - * A custom task in data_stager + * A task to stage data out of a hermes to a remote source * */ -struct CustomTask : public Task, TaskFlags { +struct StageOutTask : public Task, TaskFlags { + hermes::BucketId bkt_id_; + hipc::ShmArchive blob_name_; + hipc::Pointer data_; + size_t data_size_; + /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit - CustomTask(hipc::Allocator *alloc) : Task(alloc) {} + StageOutTask(hipc::Allocator *alloc) : Task(alloc) {} /** Emplace constructor */ HSHM_ALWAYS_INLINE explicit - CustomTask(hipc::Allocator *alloc, - const TaskNode &task_node, - const DomainId &domain_id, - const TaskStateId &state_id) : Task(alloc) { + StageOutTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; - method_ = Method::kCustom; + method_ = Method::kStageOut; task_flags_.SetBits(0); domain_id_ = domain_id; diff --git a/tasks/data_stager/src/data_stager.cc b/tasks/data_stager/src/data_stager.cc index 90356d474..2b631797e 100644 --- a/tasks/data_stager/src/data_stager.cc +++ b/tasks/data_stager/src/data_stager.cc @@ -5,29 +5,110 @@ #include "labstor_admin/labstor_admin.h" #include "labstor/api/labstor_runtime.h" #include "data_stager/data_stager.h" +#include "hermes_adapters/mapper/abstract_mapper.h" +#include "hermes_adapters/posix/posix_api.h" +#include "hermes_blob_mdm/hermes_blob_mdm.h" -namespace labstor::data_stager { +namespace hermes::data_stager { + +struct StagerInfo { + std::string url_; + int fd_; + size_t page_size_; +}; class Server : public TaskLib { + public: + std::vector> url_map_; + blob_mdm::Client blob_mdm_; + public: Server() = default; - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { + url_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + blob_mdm_.Init(task->blob_mdm_); + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void StageIn(StageInTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Custom(CustomTask *task, RunContext &ctx) { + void StageOut(StageOutTask *task, RunContext &rctx) { task->SetModuleComplete(); } + void RegisterStager(RegisterStagerTask *task, RunContext &rctx) { + BinaryRegisterStager(task, rctx); + } + + void BinaryRegisterStager(RegisterStagerTask *task, RunContext &rctx) { + url_map_[rctx.lane_id_].emplace(task->bkt_id_, StagerInfo()); + StagerInfo &stage_data = url_map_[rctx.lane_id_][task->bkt_id_]; + stage_data.url_ = task->url_->str(); + stage_data.fd_ = HERMES_POSIX_API->open(stage_data.url_.c_str(), O_RDWR); + if (stage_data.fd_ < 0) { + HELOG(kError, "Failed to open file {}", stage_data.url_); + } + } + + void BinaryStageIn(StageInTask *task, RunContext &rctx) { + StagerInfo &stage_data = url_map_[rctx.lane_id_][task->bkt_id_]; + adapter::BlobPlacement plcmnt; + plcmnt.DecodeBlobName(*task->blob_name_); + HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", + stage_data.page_size_, stage_data.url_, plcmnt.bucket_off_); + LPointer blob = LABSTOR_CLIENT->AllocateBuffer(stage_data.page_size_); + ssize_t real_size = HERMES_POSIX_API->pread(stage_data.fd_, + blob.ptr_, + stage_data.page_size_, + (off_t)plcmnt.bucket_off_); + if (real_size < 0) { + HELOG(kError, "Failed to stage in {} bytes from {}", + stage_data.page_size_, stage_data.url_); + } + memcpy(blob.ptr_ + plcmnt.blob_off_, blob.ptr_, real_size); + HILOG(kDebug, "Staged {} bytes from the backend file {}", + real_size, stage_data.url_); + hapi::Context ctx; + LPointer put_task = + blob_mdm_.AsyncPutBlob(task->task_node_ + 1, + task->bkt_id_, + hshm::to_charbuf(*task->blob_name_), + hermes::BlobId::GetNull(), + 0, real_size, blob.shm_, task->score_, bitfield32_t(0), + ctx, bitfield32_t(TASK_DATA_OWNER | TASK_LOW_LATENCY)); + put_task->Wait(task); + LABSTOR_CLIENT->DelTask(put_task); + } + + void BinaryStageOut(StageOutTask *task, RunContext &rctx) { + StagerInfo &stage_data = url_map_[rctx.lane_id_][task->bkt_id_]; + adapter::BlobPlacement plcmnt; + plcmnt.DecodeBlobName(*task->blob_name_); + HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", + stage_data.page_size_, stage_data.url_, plcmnt.bucket_off_); + char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); + ssize_t real_size = HERMES_POSIX_API->pwrite(stage_data.fd_, + data, + task->data_size_, + (off_t)plcmnt.bucket_off_); + if (real_size < 0) { + HELOG(kError, "Failed to stage out {} bytes from {}", + task->data_size_, stage_data.url_); + } + HILOG(kDebug, "Staged out {} bytes to the backend file {}", + real_size, stage_data.url_); + } public: #include "data_stager/data_stager_lib_exec.h" }; } // namespace labstor::data_stager -LABSTOR_TASK_CC(labstor::data_stager::Server, "data_stager"); +LABSTOR_TASK_CC(hermes::data_stager::Server, "data_stager"); diff --git a/tasks/hermes/include/hermes/config_manager.h b/tasks/hermes/include/hermes/config_manager.h index 0a4edf314..7e3c23c83 100644 --- a/tasks/hermes/include/hermes/config_manager.h +++ b/tasks/hermes/include/hermes/config_manager.h @@ -64,7 +64,7 @@ class ConfigurationManager { } // namespace hermes -#define HERMES_CONF hshm::Singleton::GetInstance() +#define HERMES_CONF hshm::Singleton<::hermes::ConfigurationManager>::GetInstance() #define HERMES_CLIENT_CONF HERMES_CONF->client_config_ #define HERMES_SERVER_CONF HERMES_CONF->server_config_ diff --git a/tasks/hermes/include/hermes/hermes.h b/tasks/hermes/include/hermes/hermes.h index 3c57b2311..874eabd05 100644 --- a/tasks/hermes/include/hermes/hermes.h +++ b/tasks/hermes/include/hermes/hermes.h @@ -40,7 +40,7 @@ class Hermes { } }; -#define HERMES hshm::EasySingleton::GetInstance() +#define HERMES hshm::EasySingleton<::hermes::Hermes>::GetInstance() } // namespace hermes diff --git a/tasks/hermes_adapters/filesystem/filesystem_io_client.h b/tasks/hermes_adapters/filesystem/filesystem_io_client.h index 9c7edd1f9..96a4a92ef 100644 --- a/tasks/hermes_adapters/filesystem/filesystem_io_client.h +++ b/tasks/hermes_adapters/filesystem/filesystem_io_client.h @@ -364,7 +364,7 @@ class FilesystemIoClient { namespace std { /** A structure to represent hash */ template <> -struct hash { +struct hash<::hermes::adapter::fs::File> { /** hash creator functor */ std::size_t operator()(const hermes::adapter::fs::File &key) const { return key.hash(); diff --git a/tasks/hermes_adapters/filesystem/filesystem_mdm.h b/tasks/hermes_adapters/filesystem/filesystem_mdm.h index ac2b6d0ec..a4f8a6430 100644 --- a/tasks/hermes_adapters/filesystem/filesystem_mdm.h +++ b/tasks/hermes_adapters/filesystem/filesystem_mdm.h @@ -105,7 +105,7 @@ class MetadataManager { #include "hermes_shm/util/singleton.h" #define HERMES_FS_METADATA_MANAGER \ - hshm::Singleton::GetInstance() + hshm::Singleton<::hermes::adapter::fs::MetadataManager>::GetInstance() #define HERMES_FS_METADATA_MANAGER_T hermes::adapter::fs::MetadataManager* diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h index 49e23ba6b..c7fbb171e 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_HERMES_ADAPTERS_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kCustom: { - Custom(reinterpret_cast(task), ctx); + Custom(reinterpret_cast(task), rctx); break; } } diff --git a/tasks/hermes_adapters/mpiio/mpiio_api.h b/tasks/hermes_adapters/mpiio/mpiio_api.h index efdd2a101..1328447a4 100644 --- a/tasks/hermes_adapters/mpiio/mpiio_api.h +++ b/tasks/hermes_adapters/mpiio/mpiio_api.h @@ -182,7 +182,7 @@ class MpiioApi : public RealApi { /** Simplify access to the stateless MpiioFs Singleton */ #define HERMES_MPIIO_API \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::MpiioApi>::GetInstance() #define HERMES_MPIIO_API_T hermes::adapter::fs::MpiioApi* #endif // HERMES_ADAPTER_MPIIO_H diff --git a/tasks/hermes_adapters/mpiio/mpiio_fs_api.h b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h index 53a8d3a07..55775c86f 100644 --- a/tasks/hermes_adapters/mpiio/mpiio_fs_api.h +++ b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h @@ -478,7 +478,7 @@ class MpiioFs : public Filesystem { /** Simplify access to the stateless StdioFs Singleton */ #define HERMES_MPIIO_FS \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::MpiioFs>::GetInstance() #define HERMES_STDIO_FS_T hermes::adapter::fs::MpiioFs* #endif // HERMES_ADAPTER_MPIIO_MPIIO_FS_API_H_ diff --git a/tasks/hermes_adapters/mpiio/mpiio_io_client.h b/tasks/hermes_adapters/mpiio/mpiio_io_client.h index 0b7ad7008..e9f7dd2b3 100644 --- a/tasks/hermes_adapters/mpiio/mpiio_io_client.h +++ b/tasks/hermes_adapters/mpiio/mpiio_io_client.h @@ -102,7 +102,7 @@ class MpiioIoClient : public hermes::adapter::fs::FilesystemIoClient { /** Simplify access to the stateless StdioIoClient Singleton */ #define HERMES_MPIIO_IO_CLIENT \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::MpiioIoClient>::GetInstance() #define HERMES_MPIIO_IO_CLIENT_T hermes::adapter::fs::MpiioIoClient* #endif // HERMES_ADAPTER_MPIIO_MPIIO_IO_CLIENT_H_ diff --git a/tasks/hermes_adapters/posix/posix_api.h b/tasks/hermes_adapters/posix/posix_api.h index 3134df732..8bd62f133 100644 --- a/tasks/hermes_adapters/posix/posix_api.h +++ b/tasks/hermes_adapters/posix/posix_api.h @@ -231,7 +231,7 @@ class PosixApi : public RealApi { #include "hermes_shm/util/singleton.h" #define HERMES_POSIX_API \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::PosixApi>::GetInstance() #define HERMES_POSIX_API_T hermes::adapter::fs::PosixApi* namespace hermes::adapter::fs { diff --git a/tasks/hermes_adapters/posix/posix_fs_api.h b/tasks/hermes_adapters/posix/posix_fs_api.h index af6d0f74a..1aa5f473d 100644 --- a/tasks/hermes_adapters/posix/posix_fs_api.h +++ b/tasks/hermes_adapters/posix/posix_fs_api.h @@ -105,7 +105,7 @@ class PosixFs : public hermes::adapter::fs::Filesystem { /** Simplify access to the stateless PosixFs Singleton */ #define HERMES_POSIX_FS \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::PosixFs>::GetInstance() #define HERMES_POSIX_FS_T hermes::adapter::fs::PosixFs* } // namespace hermes::adapter::fs diff --git a/tasks/hermes_adapters/posix/posix_io_client.h b/tasks/hermes_adapters/posix/posix_io_client.h index 2b4849bb8..907443144 100644 --- a/tasks/hermes_adapters/posix/posix_io_client.h +++ b/tasks/hermes_adapters/posix/posix_io_client.h @@ -95,7 +95,7 @@ class PosixIoClient : public hermes::adapter::fs::FilesystemIoClient { /** Simplify access to the stateless PosixIoClient Singleton */ #define HERMES_POSIX_IO_CLIENT \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::PosixIoClient>::GetInstance() #define HERMES_POSIX_IO_CLIENT_T hermes::adapter::fs::PosixIoClient* #endif // HERMES_ADAPTER_POSIX_POSIX_IO_CLIENT_H_ diff --git a/tasks/hermes_adapters/src/hermes_adapters.cc b/tasks/hermes_adapters/src/hermes_adapters.cc index 03d166b76..d87424bd6 100644 --- a/tasks/hermes_adapters/src/hermes_adapters.cc +++ b/tasks/hermes_adapters/src/hermes_adapters.cc @@ -12,15 +12,15 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Custom(CustomTask *task, RunContext &ctx) { + void Custom(CustomTask *task, RunContext &rctx) { task->SetModuleComplete(); } diff --git a/tasks/hermes_adapters/stdio/stdio_api.h b/tasks/hermes_adapters/stdio/stdio_api.h index 0ed8cd871..12de1ad83 100644 --- a/tasks/hermes_adapters/stdio/stdio_api.h +++ b/tasks/hermes_adapters/stdio/stdio_api.h @@ -167,7 +167,7 @@ class StdioApi : public RealApi { // Singleton macros #define HERMES_STDIO_API \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::StdioApi>::GetInstance() #define HERMES_STDIO_API_T hermes::adapter::fs::StdioApi* #endif // HERMES_ADAPTER_STDIO_H diff --git a/tasks/hermes_adapters/stdio/stdio_fs_api.h b/tasks/hermes_adapters/stdio/stdio_fs_api.h index 1af7fb154..5fb5531f6 100644 --- a/tasks/hermes_adapters/stdio/stdio_fs_api.h +++ b/tasks/hermes_adapters/stdio/stdio_fs_api.h @@ -98,7 +98,7 @@ class StdioFs : public hermes::adapter::fs::Filesystem { /** Simplify access to the stateless StdioFs Singleton */ #define HERMES_STDIO_FS \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::StdioFs>::GetInstance() #define HERMES_STDIO_FS_T hermes::adapter::fs::StdioFs* } // namespace hermes::adapter::fs diff --git a/tasks/hermes_adapters/stdio/stdio_io_client.h b/tasks/hermes_adapters/stdio/stdio_io_client.h index 9ebca5add..be5d2077c 100644 --- a/tasks/hermes_adapters/stdio/stdio_io_client.h +++ b/tasks/hermes_adapters/stdio/stdio_io_client.h @@ -94,7 +94,7 @@ class StdioIoClient : public hermes::adapter::fs::FilesystemIoClient { /** Simplify access to the stateless StdioIoClient Singleton */ #define HERMES_STDIO_IO_CLIENT \ - hshm::EasySingleton::GetInstance() + hshm::EasySingleton<::hermes::adapter::fs::StdioIoClient>::GetInstance() #define HERMES_STDIO_IO_CLIENT_T hermes::adapter::fs::StdioIoClient* #endif // HERMES_ADAPTER_STDIO_STDIO_IO_CLIENT_H_ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 9b82d620f..c5f86a2ac 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -123,7 +123,7 @@ class Client : public TaskLibClient { PutBlobTask *task, const TaskNode &task_node, TagId tag_id, const hshm::charbuf &blob_name, - BlobId &blob_id, size_t blob_off, size_t blob_size, + const BlobId &blob_id, size_t blob_off, size_t blob_size, const hipc::Pointer &blob, float score, bitfield32_t flags, Context ctx = Context(), diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index 49b80c9ac..58b52ac96 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -2,74 +2,74 @@ #define LABSTOR_HERMES_BLOB_MDM_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kPutBlob: { - PutBlob(reinterpret_cast(task), ctx); + PutBlob(reinterpret_cast(task), rctx); break; } case Method::kGetBlob: { - GetBlob(reinterpret_cast(task), ctx); + GetBlob(reinterpret_cast(task), rctx); break; } case Method::kTruncateBlob: { - TruncateBlob(reinterpret_cast(task), ctx); + TruncateBlob(reinterpret_cast(task), rctx); break; } case Method::kDestroyBlob: { - DestroyBlob(reinterpret_cast(task), ctx); + DestroyBlob(reinterpret_cast(task), rctx); break; } case Method::kTagBlob: { - TagBlob(reinterpret_cast(task), ctx); + TagBlob(reinterpret_cast(task), rctx); break; } case Method::kBlobHasTag: { - BlobHasTag(reinterpret_cast(task), ctx); + BlobHasTag(reinterpret_cast(task), rctx); break; } case Method::kGetBlobId: { - GetBlobId(reinterpret_cast(task), ctx); + GetBlobId(reinterpret_cast(task), rctx); break; } case Method::kGetOrCreateBlobId: { - GetOrCreateBlobId(reinterpret_cast(task), ctx); + GetOrCreateBlobId(reinterpret_cast(task), rctx); break; } case Method::kGetBlobName: { - GetBlobName(reinterpret_cast(task), ctx); + GetBlobName(reinterpret_cast(task), rctx); break; } case Method::kGetBlobSize: { - GetBlobSize(reinterpret_cast(task), ctx); + GetBlobSize(reinterpret_cast(task), rctx); break; } case Method::kGetBlobScore: { - GetBlobScore(reinterpret_cast(task), ctx); + GetBlobScore(reinterpret_cast(task), rctx); break; } case Method::kGetBlobBuffers: { - GetBlobBuffers(reinterpret_cast(task), ctx); + GetBlobBuffers(reinterpret_cast(task), rctx); break; } case Method::kRenameBlob: { - RenameBlob(reinterpret_cast(task), ctx); + RenameBlob(reinterpret_cast(task), rctx); break; } case Method::kReorganizeBlob: { - ReorganizeBlob(reinterpret_cast(task), ctx); + ReorganizeBlob(reinterpret_cast(task), rctx); break; } case Method::kSetBucketMdm: { - SetBucketMdm(reinterpret_cast(task), ctx); + SetBucketMdm(reinterpret_cast(task), rctx); break; } } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index b532ed9d5..6840acb03 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -50,7 +50,7 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { id_alloc_ = 0; node_id_ = LABSTOR_CLIENT->node_id_; // Initialize blob maps @@ -88,7 +88,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } @@ -107,7 +107,7 @@ class Server : public TaskLib { /** * Set the Bucket MDM * */ - void SetBucketMdm(SetBucketMdmTask *task, RunContext &ctx) { + void SetBucketMdm(SetBucketMdmTask *task, RunContext &rctx) { bkt_mdm_.Init(task->bkt_mdm_); task->SetModuleComplete(); } @@ -115,14 +115,14 @@ class Server : public TaskLib { /** * Create a blob's metadata * */ - void PutBlob(PutBlobTask *task, RunContext &ctx) { + void PutBlob(PutBlobTask *task, RunContext &rctx) { // Get the blob info data structure hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); if (task->blob_id_.IsNull()) { task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, - blob_name, ctx, task->flags_); + blob_name, rctx, task->flags_); } - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; // Update the blob info @@ -143,7 +143,7 @@ class Server : public TaskLib { blob_info.UpdateWriteStats(); } if (task->flags_.Any(HERMES_BLOB_REPLACE)) { - PutBlobFreeBuffersPhase(blob_info, task, ctx); + PutBlobFreeBuffersPhase(blob_info, task, rctx); } // Stage in blob data from FS @@ -284,7 +284,7 @@ class Server : public TaskLib { } /** Release buffers */ - void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task, RunContext &ctx) { + void PutBlobFreeBuffersPhase(BlobInfo &blob_info, PutBlobTask *task, RunContext &rctx) { for (BufferInfo &buf : blob_info.buffers_) { TargetInfo &target = *target_map_[buf.tid_]; std::vector buf_vec = {buf}; @@ -296,24 +296,24 @@ class Server : public TaskLib { } /** Get a blob's data */ - void GetBlob(GetBlobTask *task, RunContext &ctx) { + void GetBlob(GetBlobTask *task, RunContext &rctx) { switch (task->phase_) { case GetBlobPhase::kStart: { - GetBlobGetPhase(task, ctx); + GetBlobGetPhase(task, rctx); } case GetBlobPhase::kWait: { - GetBlobWaitPhase(task, ctx); + GetBlobWaitPhase(task, rctx); } } } - void GetBlobGetPhase(GetBlobTask *task, RunContext &ctx) { + void GetBlobGetPhase(GetBlobTask *task, RunContext &rctx) { if (task->blob_id_.IsNull()) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, - blob_name, ctx, task->flags_); + blob_name, rctx, task->flags_); } - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; HSHM_MAKE_AR0(task->bdev_reads_, nullptr); std::vector &read_tasks = *task->bdev_reads_; @@ -346,7 +346,7 @@ class Server : public TaskLib { task->phase_ = GetBlobPhase::kWait; } - void GetBlobWaitPhase(GetBlobTask *task, RunContext &ctx) { + void GetBlobWaitPhase(GetBlobTask *task, RunContext &rctx) { std::vector &read_tasks = *task->bdev_reads_; for (bdev::ReadTask *&read_task : read_tasks) { if (!read_task->IsComplete()) { @@ -364,8 +364,8 @@ class Server : public TaskLib { /** * Tag a blob * */ - void TagBlob(TagBlobTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + void TagBlob(TagBlobTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); @@ -379,8 +379,8 @@ class Server : public TaskLib { /** * Check if blob has a tag * */ - void BlobHasTag(BlobHasTagTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + void BlobHasTag(BlobHasTagTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); @@ -397,25 +397,25 @@ class Server : public TaskLib { * Create \a blob_id BLOB ID * */ BlobId GetOrCreateBlobId(TagId &tag_id, u32 lane_hash, - const hshm::charbuf &blob_name, RunContext &ctx, + const hshm::charbuf &blob_name, RunContext &rctx, bitfield32_t &flags) { hshm::charbuf blob_name_unique = GetBlobNameWithBucket(tag_id, blob_name); - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; auto it = blob_id_map.find(blob_name_unique); if (it == blob_id_map.end()) { BlobId blob_id = BlobId(node_id_, lane_hash, id_alloc_.fetch_add(1)); blob_id_map.emplace(blob_name_unique, blob_id); flags.SetBits(HERMES_BLOB_DID_CREATE); - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; blob_map.emplace(blob_id, BlobInfo()); return blob_id; } return it->second; } - void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &ctx) { + void GetOrCreateBlobId(GetOrCreateBlobIdTask *task, RunContext &rctx) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); bitfield32_t flags; - task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, blob_name, ctx, flags); + task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, blob_name, rctx, flags); task->SetModuleComplete(); } @@ -423,10 +423,10 @@ class Server : public TaskLib { * Get \a blob_name BLOB from \a bkt_id bucket * */ HSHM_ALWAYS_INLINE - void GetBlobId(GetBlobIdTask *task, RunContext &ctx) { + void GetBlobId(GetBlobIdTask *task, RunContext &rctx) { hshm::charbuf blob_name = hshm::to_charbuf(*task->blob_name_); hshm::charbuf blob_name_unique = GetBlobNameWithBucket(task->tag_id_, blob_name); - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; auto it = blob_id_map.find(blob_name_unique); if (it == blob_id_map.end()) { task->blob_id_ = BlobId::GetNull(); @@ -442,8 +442,8 @@ class Server : public TaskLib { /** * Get \a blob_name BLOB name from \a blob_id BLOB id * */ - void GetBlobName(GetBlobNameTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + void GetBlobName(GetBlobNameTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); @@ -457,14 +457,14 @@ class Server : public TaskLib { /** * Get \a score from \a blob_id BLOB id * */ - void GetBlobSize(GetBlobSizeTask *task, RunContext &ctx) { + void GetBlobSize(GetBlobSizeTask *task, RunContext &rctx) { if (task->blob_id_.IsNull()) { bitfield32_t flags; task->blob_id_ = GetOrCreateBlobId(task->tag_id_, task->lane_hash_, hshm::to_charbuf(*task->blob_name_), - ctx, flags); + rctx, flags); } - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->size_ = 0; @@ -479,8 +479,8 @@ class Server : public TaskLib { /** * Get \a score from \a blob_id BLOB id * */ - void GetBlobScore(GetBlobScoreTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + void GetBlobScore(GetBlobScoreTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); @@ -494,8 +494,8 @@ class Server : public TaskLib { /** * Get \a blob_id blob's buffers * */ - void GetBlobBuffers(GetBlobBuffersTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + void GetBlobBuffers(GetBlobBuffersTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); @@ -510,14 +510,14 @@ class Server : public TaskLib { * Rename \a blob_id blob to \a new_blob_name new blob name * in \a bkt_id bucket. * */ - void RenameBlob(RenameBlobTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + void RenameBlob(RenameBlobTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); return; } - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; BlobInfo &blob = it->second; blob_id_map.erase(blob.name_); blob_id_map[blob.name_] = task->blob_id_; @@ -528,8 +528,8 @@ class Server : public TaskLib { /** * Truncate a blob to a new size * */ - void TruncateBlob(TruncateBlobTask *task, RunContext &ctx) { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + void TruncateBlob(TruncateBlobTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); @@ -543,16 +543,16 @@ class Server : public TaskLib { /** * Destroy \a blob_id blob in \a bkt_id bucket * */ - void DestroyBlob(DestroyBlobTask *task, RunContext &ctx) { + void DestroyBlob(DestroyBlobTask *task, RunContext &rctx) { switch (task->phase_) { case DestroyBlobPhase::kFreeBuffers: { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); return; } - BLOB_ID_MAP_T &blob_id_map = blob_id_map_[ctx.lane_id_]; + BLOB_ID_MAP_T &blob_id_map = blob_id_map_[rctx.lane_id_]; BlobInfo &blob_info = it->second; hshm::charbuf unique_name = GetBlobNameWithBucket(blob_info.tag_id_, blob_info.name_); blob_id_map.erase(unique_name); @@ -577,7 +577,7 @@ class Server : public TaskLib { for (bdev::FreeTask *&free_task : free_tasks) { LABSTOR_CLIENT->DelTask(free_task); } - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, task->tag_id_, @@ -593,10 +593,10 @@ class Server : public TaskLib { /** * Reorganize \a blob_id blob in \a bkt_id bucket * */ - void ReorganizeBlob(ReorganizeBlobTask *task, RunContext &ctx) { + void ReorganizeBlob(ReorganizeBlobTask *task, RunContext &rctx) { switch (task->phase_) { case ReorganizeBlobPhase::kGet: { - BLOB_MAP_T &blob_map = blob_map_[ctx.lane_id_]; + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; auto it = blob_map.find(task->blob_id_); if (it == blob_map.end()) { task->SetModuleComplete(); diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index 8e2a1d03e..1c5127c07 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -2,66 +2,66 @@ #define LABSTOR_HERMES_BUCKET_MDM_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kGetOrCreateTag: { - GetOrCreateTag(reinterpret_cast(task), ctx); + GetOrCreateTag(reinterpret_cast(task), rctx); break; } case Method::kGetTagId: { - GetTagId(reinterpret_cast(task), ctx); + GetTagId(reinterpret_cast(task), rctx); break; } case Method::kGetTagName: { - GetTagName(reinterpret_cast(task), ctx); + GetTagName(reinterpret_cast(task), rctx); break; } case Method::kRenameTag: { - RenameTag(reinterpret_cast(task), ctx); + RenameTag(reinterpret_cast(task), rctx); break; } case Method::kDestroyTag: { - DestroyTag(reinterpret_cast(task), ctx); + DestroyTag(reinterpret_cast(task), rctx); break; } case Method::kTagAddBlob: { - TagAddBlob(reinterpret_cast(task), ctx); + TagAddBlob(reinterpret_cast(task), rctx); break; } case Method::kTagRemoveBlob: { - TagRemoveBlob(reinterpret_cast(task), ctx); + TagRemoveBlob(reinterpret_cast(task), rctx); break; } case Method::kTagClearBlobs: { - TagClearBlobs(reinterpret_cast(task), ctx); + TagClearBlobs(reinterpret_cast(task), rctx); break; } case Method::kUpdateSize: { - UpdateSize(reinterpret_cast(task), ctx); + UpdateSize(reinterpret_cast(task), rctx); break; } case Method::kAppendBlobSchema: { - AppendBlobSchema(reinterpret_cast(task), ctx); + AppendBlobSchema(reinterpret_cast(task), rctx); break; } case Method::kAppendBlob: { - AppendBlob(reinterpret_cast(task), ctx); + AppendBlob(reinterpret_cast(task), rctx); break; } case Method::kGetSize: { - GetSize(reinterpret_cast(task), ctx); + GetSize(reinterpret_cast(task), rctx); break; } case Method::kSetBlobMdm: { - SetBlobMdm(reinterpret_cast(task), ctx); + SetBlobMdm(reinterpret_cast(task), rctx); break; } } diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 5d1a96fc0..38fb5116c 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -27,7 +27,7 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { id_alloc_ = 0; node_id_ = LABSTOR_CLIENT->node_id_; bkt_mdm_.Init(id_); @@ -36,21 +36,21 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } /** * Set the Blob MDM * */ - void SetBlobMdm(SetBlobMdmTask *task, RunContext &ctx) { + void SetBlobMdm(SetBlobMdmTask *task, RunContext &rctx) { blob_mdm_.Init(task->blob_mdm_); task->SetModuleComplete(); } /** Update the size of the bucket */ - void UpdateSize(UpdateSizeTask *task, RunContext &ctx) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + void UpdateSize(UpdateSizeTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; TagInfo &tag_info = tag_map[task->tag_id_]; ssize_t internal_size = (ssize_t) tag_info.internal_size_; if (task->mode_ == UpdateSizeMode::kAdd) { @@ -67,12 +67,12 @@ class Server : public TaskLib { /** * Create the PartialPuts for append operations. * */ - void AppendBlobSchema(AppendBlobSchemaTask *task, RunContext &ctx) { + void AppendBlobSchema(AppendBlobSchemaTask *task, RunContext &rctx) { switch (task->phase_) { case AppendBlobPhase::kGetBlobIds: { HILOG(kDebug, "(node {}) Getting blob IDs for tag {} (task_node={})", LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; TagInfo &tag_info = tag_map[task->tag_id_]; size_t bucket_size = tag_info.internal_size_; size_t cur_page = bucket_size / task->page_size_; @@ -127,7 +127,7 @@ class Server : public TaskLib { * are named 0 ... N. Each blob is assumed to have a certain * fixed page size. * */ - void AppendBlob(AppendBlobTask *task, RunContext &ctx) { + void AppendBlob(AppendBlobTask *task, RunContext &rctx) { switch (task->phase_) { case AppendBlobPhase::kGetBlobIds: { HILOG(kDebug, "(node {}) Appending {} bytes to bucket {} (task_node={})", @@ -188,12 +188,12 @@ class Server : public TaskLib { } /** Get or create a tag */ - void GetOrCreateTag(GetOrCreateTagTask *task, RunContext &ctx) { + void GetOrCreateTag(GetOrCreateTagTask *task, RunContext &rctx) { TagId tag_id; - HILOG(kDebug, "Creating a tag on lane {}", ctx.lane_id_); + HILOG(kDebug, "Creating a tag on lane {}", rctx.lane_id_); // Check if the tag exists - TAG_ID_MAP_T &tag_id_map = tag_id_map_[ctx.lane_id_]; + TAG_ID_MAP_T &tag_id_map = tag_id_map_[rctx.lane_id_]; hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); bool did_create = false; if (tag_name.size() > 0) { @@ -202,7 +202,7 @@ class Server : public TaskLib { // Emplace bucket if it does not already exist if (did_create) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; tag_id.unique_ = id_alloc_.fetch_add(1); tag_id.hash_ = task->lane_hash_; tag_id.node_id_ = LABSTOR_RUNTIME->rpc_.node_id_; @@ -230,8 +230,8 @@ class Server : public TaskLib { } /** Get tag ID */ - void GetTagId(GetTagIdTask *task, RunContext &ctx) { - TAG_ID_MAP_T &tag_id_map = tag_id_map_[ctx.lane_id_]; + void GetTagId(GetTagIdTask *task, RunContext &rctx) { + TAG_ID_MAP_T &tag_id_map = tag_id_map_[rctx.lane_id_]; hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); auto it = tag_id_map.find(tag_name); if (it == tag_id_map.end()) { @@ -244,8 +244,8 @@ class Server : public TaskLib { } /** Get tag name */ - void GetTagName(GetTagNameTask *task, RunContext &ctx) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + void GetTagName(GetTagNameTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; auto it = tag_map.find(task->tag_id_); if (it == tag_map.end()) { task->SetModuleComplete(); @@ -256,8 +256,8 @@ class Server : public TaskLib { } /** Rename tag */ - void RenameTag(RenameTagTask *task, RunContext &ctx) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + void RenameTag(RenameTagTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; auto it = tag_map.find(task->tag_id_); if (it == tag_map.end()) { task->SetModuleComplete(); @@ -268,11 +268,11 @@ class Server : public TaskLib { } /** Destroy tag */ - void DestroyTag(DestroyTagTask *task, RunContext &ctx) { + void DestroyTag(DestroyTagTask *task, RunContext &rctx) { switch (task->phase_) { case DestroyTagPhase::kDestroyBlobs: { - TAG_ID_MAP_T &tag_id_map = tag_id_map_[ctx.lane_id_]; - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + TAG_ID_MAP_T &tag_id_map = tag_id_map_[rctx.lane_id_]; + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; TagInfo &tag = tag_map[task->tag_id_]; tag_id_map.erase(tag.name_); HSHM_MAKE_AR0(task->destroy_blob_tasks_, nullptr); @@ -297,7 +297,7 @@ class Server : public TaskLib { LABSTOR_CLIENT->DelTask(blob_task); } HSHM_DESTROY_AR(task->destroy_blob_tasks_); - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; tag_map.erase(task->tag_id_); task->SetModuleComplete(); } @@ -305,8 +305,8 @@ class Server : public TaskLib { } /** Add a blob to a tag */ - void TagAddBlob(TagAddBlobTask *task, RunContext &ctx) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + void TagAddBlob(TagAddBlobTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; auto it = tag_map.find(task->tag_id_); if (it == tag_map.end()) { task->SetModuleComplete(); @@ -318,8 +318,8 @@ class Server : public TaskLib { } /** Remove a blob from a tag */ - void TagRemoveBlob(TagRemoveBlobTask *task, RunContext &ctx) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + void TagRemoveBlob(TagRemoveBlobTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; auto it = tag_map.find(task->tag_id_); if (it == tag_map.end()) { task->SetModuleComplete(); @@ -332,8 +332,8 @@ class Server : public TaskLib { } /** Clear blobs from a tag */ - void TagClearBlobs(TagClearBlobsTask *task, RunContext &ctx) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + void TagClearBlobs(TagClearBlobsTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; auto it = tag_map.find(task->tag_id_); if (it == tag_map.end()) { task->SetModuleComplete(); @@ -346,8 +346,8 @@ class Server : public TaskLib { } /** Get size of the bucket */ - void GetSize(GetSizeTask *task, RunContext &ctx) { - TAG_MAP_T &tag_map = tag_map_[ctx.lane_id_]; + void GetSize(GetSizeTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; auto it = tag_map.find(task->tag_id_); if (it == tag_map.end()) { task->size_ = 0; diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h index 105572c86..17c62ad64 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h @@ -2,14 +2,14 @@ #define LABSTOR_HERMES_MDM_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } } diff --git a/tasks/hermes_mdm/src/hermes_mdm.cc b/tasks/hermes_mdm/src/hermes_mdm.cc index 9c5557b88..6a870cf8b 100644 --- a/tasks/hermes_mdm/src/hermes_mdm.cc +++ b/tasks/hermes_mdm/src/hermes_mdm.cc @@ -25,7 +25,7 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { HILOG(kDebug, "ConstructTaskPhase::kLoadConfig") std::string config_path = task->server_config_path_->str(); HERMES_CONF->LoadServerConfig(config_path); @@ -33,7 +33,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } diff --git a/tasks/posix_bdev/src/posix_bdev.cc b/tasks/posix_bdev/src/posix_bdev.cc index 038debf86..bf0bba084 100644 --- a/tasks/posix_bdev/src/posix_bdev.cc +++ b/tasks/posix_bdev/src/posix_bdev.cc @@ -21,7 +21,7 @@ class Server : public TaskLib { std::string path_; public: - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { DeviceInfo &dev_info = task->info_; alloc_.Init(id_, dev_info.capacity_, dev_info.slab_sizes_); std::string text = dev_info.mount_dir_ + @@ -39,22 +39,22 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Allocate(AllocateTask *task, RunContext &ctx) { + void Allocate(AllocateTask *task, RunContext &rctx) { alloc_.Allocate(task->size_, *task->buffers_, task->alloc_size_); HILOG(kDebug, "Allocated {}/{} bytes ({})", task->alloc_size_, task->size_, path_); task->SetModuleComplete(); } - void Free(FreeTask *task, RunContext &ctx) { + void Free(FreeTask *task, RunContext &rctx) { alloc_.Free(task->buffers_); task->SetModuleComplete(); } - void Write(WriteTask *task, RunContext &ctx) { + void Write(WriteTask *task, RunContext &rctx) { HILOG(kDebug, "Writing {} bytes to {}", task->size_, path_); ssize_t count = pwrite(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { @@ -64,7 +64,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Read(ReadTask *task, RunContext &ctx) { + void Read(ReadTask *task, RunContext &rctx) { HILOG(kDebug, "Reading {} bytes from {}", task->size_, path_); ssize_t count = pread(fd_, task->buf_, task->size_, (off_t)task->disk_off_); if (count != task->size_) { @@ -74,10 +74,10 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Monitor(MonitorTask *task, RunContext &ctx) { + void Monitor(MonitorTask *task, RunContext &rctx) { } - void UpdateCapacity(UpdateCapacityTask *task, RunContext &ctx) { + void UpdateCapacity(UpdateCapacityTask *task, RunContext &rctx) { task->SetModuleComplete(); } diff --git a/tasks/ram_bdev/src/ram_bdev.cc b/tasks/ram_bdev/src/ram_bdev.cc index ec1ae7107..638b10633 100644 --- a/tasks/ram_bdev/src/ram_bdev.cc +++ b/tasks/ram_bdev/src/ram_bdev.cc @@ -15,7 +15,7 @@ class Server : public TaskLib { char *mem_ptr_; public: - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { DeviceInfo &dev_info = task->info_; alloc_.Init(id_, dev_info.capacity_, dev_info.slab_sizes_); mem_ptr_ = (char*)malloc(dev_info.capacity_); @@ -24,39 +24,39 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { free(mem_ptr_); task->SetModuleComplete(); } - void Allocate(AllocateTask *task, RunContext &ctx) { + void Allocate(AllocateTask *task, RunContext &rctx) { HILOG(kDebug, "Allocating {} bytes (RAM)", task->size_); alloc_.Allocate(task->size_, *task->buffers_, task->alloc_size_); HILOG(kDebug, "Allocated {} bytes (RAM)", task->alloc_size_); task->SetModuleComplete(); } - void Free(FreeTask *task, RunContext &ctx) { + void Free(FreeTask *task, RunContext &rctx) { alloc_.Free(task->buffers_); task->SetModuleComplete(); } - void Write(WriteTask *task, RunContext &ctx) { + void Write(WriteTask *task, RunContext &rctx) { HILOG(kDebug, "Writing {} bytes to RAM", task->size_); memcpy(mem_ptr_ + task->disk_off_, task->buf_, task->size_); task->SetModuleComplete(); } - void Read(ReadTask *task, RunContext &ctx) { + void Read(ReadTask *task, RunContext &rctx) { HILOG(kDebug, "Reading {} bytes from RAM", task->size_); memcpy(task->buf_, mem_ptr_ + task->disk_off_, task->size_); task->SetModuleComplete(); } - void Monitor(MonitorTask *task, RunContext &ctx) { + void Monitor(MonitorTask *task, RunContext &rctx) { } - void UpdateCapacity(UpdateCapacityTask *task, RunContext &ctx) { + void UpdateCapacity(UpdateCapacityTask *task, RunContext &rctx) { task->SetModuleComplete(); } diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h index 5b9af6f98..5adb82c2c 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_TASK_NAME_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kCustom: { - Custom(reinterpret_cast(task), ctx); + Custom(reinterpret_cast(task), rctx); break; } } diff --git a/tasks_required/TASK_NAME/src/TASK_NAME.cc b/tasks_required/TASK_NAME/src/TASK_NAME.cc index 03ff393dd..5912dba1f 100644 --- a/tasks_required/TASK_NAME/src/TASK_NAME.cc +++ b/tasks_required/TASK_NAME/src/TASK_NAME.cc @@ -12,15 +12,15 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Custom(CustomTask *task, RunContext &ctx) { + void Custom(CustomTask *task, RunContext &rctx) { task->SetModuleComplete(); } diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h index 53fac0830..b25c7d1ac 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h @@ -2,42 +2,42 @@ #define LABSTOR_LABSTOR_ADMIN_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kCreateTaskState: { - CreateTaskState(reinterpret_cast(task), ctx); + CreateTaskState(reinterpret_cast(task), rctx); break; } case Method::kDestroyTaskState: { - DestroyTaskState(reinterpret_cast(task), ctx); + DestroyTaskState(reinterpret_cast(task), rctx); break; } case Method::kRegisterTaskLib: { - RegisterTaskLib(reinterpret_cast(task), ctx); + RegisterTaskLib(reinterpret_cast(task), rctx); break; } case Method::kDestroyTaskLib: { - DestroyTaskLib(reinterpret_cast(task), ctx); + DestroyTaskLib(reinterpret_cast(task), rctx); break; } case Method::kGetOrCreateTaskStateId: { - GetOrCreateTaskStateId(reinterpret_cast(task), ctx); + GetOrCreateTaskStateId(reinterpret_cast(task), rctx); break; } case Method::kGetTaskStateId: { - GetTaskStateId(reinterpret_cast(task), ctx); + GetTaskStateId(reinterpret_cast(task), rctx); break; } case Method::kStopRuntime: { - StopRuntime(reinterpret_cast(task), ctx); + StopRuntime(reinterpret_cast(task), rctx); break; } case Method::kSetWorkOrchQueuePolicy: { - SetWorkOrchQueuePolicy(reinterpret_cast(task), ctx); + SetWorkOrchQueuePolicy(reinterpret_cast(task), rctx); break; } case Method::kSetWorkOrchProcPolicy: { - SetWorkOrchProcPolicy(reinterpret_cast(task), ctx); + SetWorkOrchProcPolicy(reinterpret_cast(task), rctx); break; } } diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index d80e4998a..760a77eb4 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -16,25 +16,25 @@ class Server : public TaskLib { public: Server() : queue_sched_(nullptr), proc_sched_(nullptr) {} - void RegisterTaskLib(RegisterTaskLibTask *task, RunContext &ctx) { + void RegisterTaskLib(RegisterTaskLibTask *task, RunContext &rctx) { std::string lib_name = task->lib_name_->str(); LABSTOR_TASK_REGISTRY->RegisterTaskLib(lib_name); task->SetModuleComplete(); } - void DestroyTaskLib(DestroyTaskLibTask *task, RunContext &ctx) { + void DestroyTaskLib(DestroyTaskLibTask *task, RunContext &rctx) { std::string lib_name = task->lib_name_->str(); LABSTOR_TASK_REGISTRY->DestroyTaskLib(lib_name); task->SetModuleComplete(); } - void GetOrCreateTaskStateId(GetOrCreateTaskStateIdTask *task, RunContext &ctx) { + void GetOrCreateTaskStateId(GetOrCreateTaskStateIdTask *task, RunContext &rctx) { std::string state_name = task->state_name_->str(); task->id_ = LABSTOR_TASK_REGISTRY->GetOrCreateTaskStateId(state_name); task->SetModuleComplete(); } - void CreateTaskState(CreateTaskStateTask *task, RunContext &ctx) { + void CreateTaskState(CreateTaskStateTask *task, RunContext &rctx) { std::string lib_name = task->lib_name_->str(); std::string state_name = task->state_name_->str(); // Check local registry for task state @@ -88,25 +88,25 @@ class Server : public TaskLib { LABSTOR_CLIENT->node_id_, state_name, task->task_state_); } - void GetTaskStateId(GetTaskStateIdTask *task, RunContext &ctx) { + void GetTaskStateId(GetTaskStateIdTask *task, RunContext &rctx) { std::string state_name = task->state_name_->str(); task->id_ = LABSTOR_TASK_REGISTRY->GetTaskStateId(state_name); task->SetModuleComplete(); } - void DestroyTaskState(DestroyTaskStateTask *task, RunContext &ctx) { + void DestroyTaskState(DestroyTaskStateTask *task, RunContext &rctx) { LABSTOR_TASK_REGISTRY->DestroyTaskState(task->id_); task->SetModuleComplete(); } - void StopRuntime(StopRuntimeTask *task, RunContext &ctx) { + void StopRuntime(StopRuntimeTask *task, RunContext &rctx) { HILOG(kInfo, "Stopping (server mode)"); LABSTOR_WORK_ORCHESTRATOR->FinalizeRuntime(); LABSTOR_THALLIUM->StopThisDaemon(); task->SetModuleComplete(); } - void SetWorkOrchQueuePolicy(SetWorkOrchQueuePolicyTask *task, RunContext &ctx) { + void SetWorkOrchQueuePolicy(SetWorkOrchQueuePolicyTask *task, RunContext &rctx) { if (queue_sched_) { queue_sched_->SetModuleComplete(); } @@ -121,7 +121,7 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void SetWorkOrchProcPolicy(SetWorkOrchProcPolicyTask *task, RunContext &ctx) { + void SetWorkOrchProcPolicy(SetWorkOrchProcPolicyTask *task, RunContext &rctx) { if (proc_sched_) { proc_sched_->SetModuleComplete(); } diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h index c3fffff4b..a4a9eaf9a 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_PROC_QUEUE_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kPush: { - Push(reinterpret_cast(task), ctx); + Push(reinterpret_cast(task), rctx); break; } } diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/tasks_required/proc_queue/src/proc_queue.cc index 7a74ce5d8..2c2463378 100644 --- a/tasks_required/proc_queue/src/proc_queue.cc +++ b/tasks_required/proc_queue/src/proc_queue.cc @@ -12,15 +12,15 @@ class Server : public TaskLib { public: Server() = default; - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Push(PushTask *task, RunContext &ctx) { + void Push(PushTask *task, RunContext &rctx) { switch (task->phase_) { case PushTaskPhase::kSchedule: { task->sub_run_.shm_ = task->sub_cli_.shm_; diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h index 6d5ef8721..0166190f4 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_REMOTE_QUEUE_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kPush: { - Push(reinterpret_cast(task), ctx); + Push(reinterpret_cast(task), rctx); break; } } diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index acd6b5c73..1b8075f54 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -23,7 +23,7 @@ class Server : public TaskLib { Server() = default; /** Construct remote queue */ - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { HILOG(kInfo, "(node {}) Constructing remote queue (task_node={}, task_state={}, method={})", LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, task->method_); LABSTOR_THALLIUM->RegisterRpc("RpcPushSmall", [this](const tl::request &req, @@ -45,7 +45,7 @@ class Server : public TaskLib { } /** Destroy remote queue */ - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } @@ -144,7 +144,7 @@ class Server : public TaskLib { } /** Push operation called on client */ - void Push(PushTask *task, RunContext &ctx) { + void Push(PushTask *task, RunContext &rctx) { std::vector &xfer = task->xfer_; switch (xfer.size()) { case 1: { diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/tasks_required/small_message/include/small_message/small_message_lib_exec.h index ae08efc87..31d55bc73 100644 --- a/tasks_required/small_message/include/small_message/small_message_lib_exec.h +++ b/tasks_required/small_message/include/small_message/small_message_lib_exec.h @@ -2,26 +2,26 @@ #define LABSTOR_SMALL_MESSAGE_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kMd: { - Md(reinterpret_cast(task), ctx); + Md(reinterpret_cast(task), rctx); break; } case Method::kIo: { - Io(reinterpret_cast(task), ctx); + Io(reinterpret_cast(task), rctx); break; } case Method::kMdPush: { - MdPush(reinterpret_cast(task), ctx); + MdPush(reinterpret_cast(task), rctx); break; } } diff --git a/tasks_required/small_message/src/small_message.cc b/tasks_required/small_message/src/small_message.cc index 16282873d..05ed670e2 100644 --- a/tasks_required/small_message/src/small_message.cc +++ b/tasks_required/small_message/src/small_message.cc @@ -13,25 +13,25 @@ class Server : public TaskLib { int count_ = 0; public: - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Md(MdTask *task, RunContext &ctx) { + void Md(MdTask *task, RunContext &rctx) { task->ret_[0] = 1; task->SetModuleComplete(); } - void MdPush(MdPushTask *task, RunContext &ctx) { + void MdPush(MdPushTask *task, RunContext &rctx) { task->ret_[0] = 1; task->SetModuleComplete(); } - void Io(IoTask *task, RunContext &ctx) { + void Io(IoTask *task, RunContext &rctx) { task->ret_ = 1; for (int i = 0; i < 256; ++i) { if (task->data_[i] != 10) { diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h index 5d47d3c7d..ca8ce7fc6 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kSchedule: { - Schedule(reinterpret_cast(task), ctx); + Schedule(reinterpret_cast(task), rctx); break; } } diff --git a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc index 238143a27..81411bf8f 100644 --- a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc +++ b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc @@ -10,15 +10,15 @@ namespace labstor::worch_proc_round_robin { class Server : public TaskLib { public: - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Schedule(ScheduleTask *task, RunContext &ctx) { + void Schedule(ScheduleTask *task, RunContext &rctx) { int rr = 0; for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { worker.SetCpuAffinity(rr % HERMES_SYSTEM_INFO->ncpu_); diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h index 2e6a1aec9..0d95305e6 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h @@ -2,18 +2,18 @@ #define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ /** Execute a task */ -void Run(u32 method, Task *task, RunContext &ctx) override { +void Run(u32 method, Task *task, RunContext &rctx) override { switch (method) { case Method::kConstruct: { - Construct(reinterpret_cast(task), ctx); + Construct(reinterpret_cast(task), rctx); break; } case Method::kDestruct: { - Destruct(reinterpret_cast(task), ctx); + Destruct(reinterpret_cast(task), rctx); break; } case Method::kSchedule: { - Schedule(reinterpret_cast(task), ctx); + Schedule(reinterpret_cast(task), rctx); break; } } diff --git a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc index 10f53ed33..a73ba20ae 100644 --- a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc +++ b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc @@ -13,16 +13,16 @@ class Server : public TaskLib { u32 count_; public: - void Construct(ConstructTask *task, RunContext &ctx) { + void Construct(ConstructTask *task, RunContext &rctx) { count_ = 0; task->SetModuleComplete(); } - void Destruct(DestructTask *task, RunContext &ctx) { + void Destruct(DestructTask *task, RunContext &rctx) { task->SetModuleComplete(); } - void Schedule(ScheduleTask *task, RunContext &ctx) { + void Schedule(ScheduleTask *task, RunContext &rctx) { // Check if any new queues need to be scheduled for (MultiQueue &queue : *LABSTOR_QM_RUNTIME->queue_map_) { if (queue.id_.IsNull()) { From 3659452aa326cea8a737af715e7f832ca6ba39d6 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 30 Sep 2023 16:37:16 -0500 Subject: [PATCH 138/191] Make stager factory --- .../data_stager/data_stager_lib_exec.h | 36 ++++++++ .../include/data_stager/data_stager_methods.h | 5 +- .../data_stager/data_stager_methods.yaml | 5 +- .../include/data_stager/data_stager_tasks.h | 76 ++++++++++++++-- tasks/data_stager/src/abstract_stager.h | 26 ++++++ tasks/data_stager/src/binary_stager.h | 86 +++++++++++++++++++ tasks/data_stager/src/data_stager.cc | 85 ++++-------------- tasks/data_stager/src/stager_factory.h | 31 +++++++ 8 files changed, 271 insertions(+), 79 deletions(-) create mode 100644 tasks/data_stager/src/abstract_stager.h create mode 100644 tasks/data_stager/src/binary_stager.h create mode 100644 tasks/data_stager/src/stager_factory.h diff --git a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h index b41a89215..8e157a7af 100644 --- a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h +++ b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h @@ -16,6 +16,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { RegisterStager(reinterpret_cast(task), rctx); break; } + case Method::kUnregisterStager: { + UnregisterStager(reinterpret_cast(task), rctx); + break; + } case Method::kStageIn: { StageIn(reinterpret_cast(task), rctx); break; @@ -41,6 +45,10 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kUnregisterStager: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } case Method::kStageIn: { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; @@ -66,6 +74,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kUnregisterStager: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } case Method::kStageIn: { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; @@ -91,6 +103,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kUnregisterStager: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } case Method::kStageIn: { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; @@ -116,6 +132,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kUnregisterStager: { + ar << *reinterpret_cast(task); + break; + } case Method::kStageIn: { ar << *reinterpret_cast(task); break; @@ -146,6 +166,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kUnregisterStager: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } case Method::kStageIn: { task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); @@ -174,6 +199,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kUnregisterStager: { + ar << *reinterpret_cast(task); + break; + } case Method::kStageIn: { ar << *reinterpret_cast(task); break; @@ -200,6 +229,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kUnregisterStager: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } case Method::kStageIn: { ar.Deserialize(replica, *reinterpret_cast(task)); break; @@ -222,6 +255,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kRegisterStager: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kUnregisterStager: { + return reinterpret_cast(task)->GetGroup(group); + } case Method::kStageIn: { return reinterpret_cast(task)->GetGroup(group); } diff --git a/tasks/data_stager/include/data_stager/data_stager_methods.h b/tasks/data_stager/include/data_stager/data_stager_methods.h index 36700a1a2..4b1b9d8f3 100644 --- a/tasks/data_stager/include/data_stager/data_stager_methods.h +++ b/tasks/data_stager/include/data_stager/data_stager_methods.h @@ -4,8 +4,9 @@ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kRegisterStager = kLast + 0; - TASK_METHOD_T kStageIn = kLast + 1; - TASK_METHOD_T kStageOut = kLast + 2; + TASK_METHOD_T kUnregisterStager = kLast + 1; + TASK_METHOD_T kStageIn = kLast + 2; + TASK_METHOD_T kStageOut = kLast + 3; }; #endif // LABSTOR_DATA_STAGER_METHODS_H_ \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_methods.yaml b/tasks/data_stager/include/data_stager/data_stager_methods.yaml index 27aa4b7e2..a2cac4953 100644 --- a/tasks/data_stager/include/data_stager/data_stager_methods.yaml +++ b/tasks/data_stager/include/data_stager/data_stager_methods.yaml @@ -1,3 +1,4 @@ kRegisterStager: 0 -kStageIn: 1 -kStageOut: 2 \ No newline at end of file +kUnregisterStager: 1 +kStageIn: 2 +kStageOut: 3 \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_tasks.h b/tasks/data_stager/include/data_stager/data_stager_tasks.h index 54aff1314..8762b82b8 100644 --- a/tasks/data_stager/include/data_stager/data_stager_tasks.h +++ b/tasks/data_stager/include/data_stager/data_stager_tasks.h @@ -78,7 +78,7 @@ struct DestructTask : public DestroyTaskStateTask { /** * Register a new stager * */ -struct RegisterStagerTask : public Task, TaskFlags { +struct RegisterStagerTask : public Task, TaskFlags { hermes::BucketId bkt_id_; hipc::ShmArchive url_; @@ -96,11 +96,11 @@ struct RegisterStagerTask : public Task, TaskFlags { hshm::charbuf &url) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = 0; + lane_hash_ = bkt_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; - method_ = Method::kStageIn; - task_flags_.SetBits(0); + method_ = Method::kRegisterStager; + task_flags_.SetBits(TASK_FIRE_AND_FORGET); domain_id_ = domain_id; // Custom params @@ -126,6 +126,54 @@ struct RegisterStagerTask : public Task, TaskFlags { } }; +/** + * Unregister a new stager + * */ +struct UnregisterStagerTask : public Task, TaskFlags { + hermes::BucketId bkt_id_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + UnregisterStagerTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + UnregisterStagerTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + hermes::BucketId bkt_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = bkt_id.hash_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kUnregisterStager; + task_flags_.SetBits(TASK_FIRE_AND_FORGET); + domain_id_ = domain_id; + + // Custom params + bkt_id_ = bkt_id; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + /** * A task to stage in data from a remote source * */ @@ -143,7 +191,10 @@ struct StageInTask : public Task, TaskFlags { StageInTask(hipc::Allocator *alloc, const TaskNode &task_node, const DomainId &domain_id, - const TaskStateId &state_id) : Task(alloc) { + const TaskStateId &state_id, + const BucketId &bkt_id, + const hshm::charbuf &blob_name, + float score) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = 0; @@ -154,6 +205,9 @@ struct StageInTask : public Task, TaskFlags { domain_id_ = domain_id; // Custom params + bkt_id_ = bkt_id; + HSHM_MAKE_AR(blob_name_, alloc, blob_name); + score_ = score; } /** (De)serialize message call */ @@ -192,10 +246,14 @@ struct StageOutTask : public Task, TaskFlags { StageOutTask(hipc::Allocator *alloc, const TaskNode &task_node, const DomainId &domain_id, - const TaskStateId &state_id) : Task(alloc) { + const TaskStateId &state_id, + const BucketId &bkt_id, + const hshm::charbuf &blob_name, + const hipc::Pointer &data, + size_t data_size) : Task(alloc) { // Initialize task task_node_ = task_node; - lane_hash_ = 0; + lane_hash_ = bkt_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kStageOut; @@ -203,6 +261,10 @@ struct StageOutTask : public Task, TaskFlags { domain_id_ = domain_id; // Custom params + bkt_id_ = bkt_id; + HSHM_MAKE_AR(blob_name_, alloc, blob_name); + data_ = data; + data_size_ = data_size; } /** (De)serialize message call */ diff --git a/tasks/data_stager/src/abstract_stager.h b/tasks/data_stager/src/abstract_stager.h new file mode 100644 index 000000000..cf178fee2 --- /dev/null +++ b/tasks/data_stager/src/abstract_stager.h @@ -0,0 +1,26 @@ +// +// Created by lukemartinlogan on 9/30/23. +// + +#ifndef HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ +#define HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ + +#include "data_stager/data_stager.h" + +namespace hermes::data_stager { + +class AbstractStager { + public: + std::string url_; + + AbstractStager() = default; + ~AbstractStager() = default; + + virtual void RegisterStager(RegisterStagerTask *task, RunContext &rctx) = 0; + virtual void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) = 0; + virtual void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) = 0; +}; + +} // namespace hermes + +#endif // HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ diff --git a/tasks/data_stager/src/binary_stager.h b/tasks/data_stager/src/binary_stager.h new file mode 100644 index 000000000..4dc45f793 --- /dev/null +++ b/tasks/data_stager/src/binary_stager.h @@ -0,0 +1,86 @@ +// +// Created by lukemartinlogan on 9/30/23. +// + +#ifndef HERMES_TASKS_DATA_STAGER_SRC_BINARY_STAGER_H_ +#define HERMES_TASKS_DATA_STAGER_SRC_BINARY_STAGER_H_ + +#include "abstract_stager.h" + +namespace hermes::data_stager { + +class BinaryFileStager : public AbstractStager { + public: + int fd_; + size_t page_size_; + + public: + /** Default constructor */ + BinaryFileStager() = default; + + /** Destructor */ + ~BinaryFileStager() { + HERMES_POSIX_API->close(fd_); + } + + /** Create the data stager payload */ + void RegisterStager(RegisterStagerTask *task, RunContext &rctx) override { + fd_ = HERMES_POSIX_API->open(url_.c_str(), O_RDWR); + if (fd_ < 0) { + HELOG(kError, "Failed to open file {}", url_); + } + } + + /** Stage data in from remote source */ + void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) override { + adapter::BlobPlacement plcmnt; + plcmnt.DecodeBlobName(*task->blob_name_); + HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", + page_size_, url_, plcmnt.bucket_off_); + LPointer blob = LABSTOR_CLIENT->AllocateBuffer(page_size_); + ssize_t real_size = HERMES_POSIX_API->pread(fd_, + blob.ptr_, + page_size_, + (off_t)plcmnt.bucket_off_); + if (real_size < 0) { + HELOG(kError, "Failed to stage in {} bytes from {}", + page_size_, url_); + } + memcpy(blob.ptr_ + plcmnt.blob_off_, blob.ptr_, real_size); + HILOG(kDebug, "Staged {} bytes from the backend file {}", + real_size, url_); + hapi::Context ctx; + LPointer put_task = + blob_mdm.AsyncPutBlob(task->task_node_ + 1, + task->bkt_id_, + hshm::to_charbuf(*task->blob_name_), + hermes::BlobId::GetNull(), + 0, real_size, blob.shm_, task->score_, bitfield32_t(0), + ctx, bitfield32_t(TASK_DATA_OWNER | TASK_LOW_LATENCY)); + put_task->Wait(task); + LABSTOR_CLIENT->DelTask(put_task); + } + + /** Stage data out to remote source */ + void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) override { + adapter::BlobPlacement plcmnt; + plcmnt.DecodeBlobName(*task->blob_name_); + HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", + page_size_, url_, plcmnt.bucket_off_); + char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); + ssize_t real_size = HERMES_POSIX_API->pwrite(fd_, + data, + task->data_size_, + (off_t)plcmnt.bucket_off_); + if (real_size < 0) { + HELOG(kError, "Failed to stage out {} bytes from {}", + task->data_size_, url_); + } + HILOG(kDebug, "Staged out {} bytes to the backend file {}", + real_size, url_); + } +}; + +} // namespace hermes::data_stager + +#endif // HERMES_TASKS_DATA_STAGER_SRC_BINARY_STAGER_H_ diff --git a/tasks/data_stager/src/data_stager.cc b/tasks/data_stager/src/data_stager.cc index 2b631797e..c821eefa7 100644 --- a/tasks/data_stager/src/data_stager.cc +++ b/tasks/data_stager/src/data_stager.cc @@ -8,18 +8,13 @@ #include "hermes_adapters/mapper/abstract_mapper.h" #include "hermes_adapters/posix/posix_api.h" #include "hermes_blob_mdm/hermes_blob_mdm.h" +#include "stager_factory.h" namespace hermes::data_stager { -struct StagerInfo { - std::string url_; - int fd_; - size_t page_size_; -}; - class Server : public TaskLib { public: - std::vector> url_map_; + std::vector>> url_map_; blob_mdm::Client blob_mdm_; public: @@ -35,75 +30,29 @@ class Server : public TaskLib { task->SetModuleComplete(); } - void StageIn(StageInTask *task, RunContext &rctx) { + void RegisterStager(RegisterStagerTask *task, RunContext &rctx) { + std::string url = task->url_->str(); + std::unique_ptr stager = StagerFactory::Get(url); + stager->RegisterStager(task, rctx); + url_map_[rctx.lane_id_].emplace(task->bkt_id_, std::move(stager)); task->SetModuleComplete(); } - void StageOut(StageOutTask *task, RunContext &rctx) { + void UnregisterStager(UnregisterStagerTask *task, RunContext &rctx) { + url_map_[rctx.lane_id_].erase(task->bkt_id_); task->SetModuleComplete(); } - void RegisterStager(RegisterStagerTask *task, RunContext &rctx) { - BinaryRegisterStager(task, rctx); - } - - void BinaryRegisterStager(RegisterStagerTask *task, RunContext &rctx) { - url_map_[rctx.lane_id_].emplace(task->bkt_id_, StagerInfo()); - StagerInfo &stage_data = url_map_[rctx.lane_id_][task->bkt_id_]; - stage_data.url_ = task->url_->str(); - stage_data.fd_ = HERMES_POSIX_API->open(stage_data.url_.c_str(), O_RDWR); - if (stage_data.fd_ < 0) { - HELOG(kError, "Failed to open file {}", stage_data.url_); - } - } - - void BinaryStageIn(StageInTask *task, RunContext &rctx) { - StagerInfo &stage_data = url_map_[rctx.lane_id_][task->bkt_id_]; - adapter::BlobPlacement plcmnt; - plcmnt.DecodeBlobName(*task->blob_name_); - HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", - stage_data.page_size_, stage_data.url_, plcmnt.bucket_off_); - LPointer blob = LABSTOR_CLIENT->AllocateBuffer(stage_data.page_size_); - ssize_t real_size = HERMES_POSIX_API->pread(stage_data.fd_, - blob.ptr_, - stage_data.page_size_, - (off_t)plcmnt.bucket_off_); - if (real_size < 0) { - HELOG(kError, "Failed to stage in {} bytes from {}", - stage_data.page_size_, stage_data.url_); - } - memcpy(blob.ptr_ + plcmnt.blob_off_, blob.ptr_, real_size); - HILOG(kDebug, "Staged {} bytes from the backend file {}", - real_size, stage_data.url_); - hapi::Context ctx; - LPointer put_task = - blob_mdm_.AsyncPutBlob(task->task_node_ + 1, - task->bkt_id_, - hshm::to_charbuf(*task->blob_name_), - hermes::BlobId::GetNull(), - 0, real_size, blob.shm_, task->score_, bitfield32_t(0), - ctx, bitfield32_t(TASK_DATA_OWNER | TASK_LOW_LATENCY)); - put_task->Wait(task); - LABSTOR_CLIENT->DelTask(put_task); + void StageIn(StageInTask *task, RunContext &rctx) { + AbstractStager &stager = *url_map_[rctx.lane_id_][task->bkt_id_]; + stager.StageIn(blob_mdm_, task, rctx); + task->SetModuleComplete(); } - void BinaryStageOut(StageOutTask *task, RunContext &rctx) { - StagerInfo &stage_data = url_map_[rctx.lane_id_][task->bkt_id_]; - adapter::BlobPlacement plcmnt; - plcmnt.DecodeBlobName(*task->blob_name_); - HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", - stage_data.page_size_, stage_data.url_, plcmnt.bucket_off_); - char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); - ssize_t real_size = HERMES_POSIX_API->pwrite(stage_data.fd_, - data, - task->data_size_, - (off_t)plcmnt.bucket_off_); - if (real_size < 0) { - HELOG(kError, "Failed to stage out {} bytes from {}", - task->data_size_, stage_data.url_); - } - HILOG(kDebug, "Staged out {} bytes to the backend file {}", - real_size, stage_data.url_); + void StageOut(StageOutTask *task, RunContext &rctx) { + AbstractStager &stager = *url_map_[rctx.lane_id_][task->bkt_id_]; + stager.StageOut(blob_mdm_, task, rctx); + task->SetModuleComplete(); } public: #include "data_stager/data_stager_lib_exec.h" diff --git a/tasks/data_stager/src/stager_factory.h b/tasks/data_stager/src/stager_factory.h new file mode 100644 index 000000000..4adf0270b --- /dev/null +++ b/tasks/data_stager/src/stager_factory.h @@ -0,0 +1,31 @@ +// +// Created by lukemartinlogan on 9/30/23. +// + +#ifndef HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ +#define HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ + +#include "data_stager/data_stager.h" +#include "abstract_stager.h" +#include "binary_stager.h" + +namespace hermes::data_stager { + +class StagerFactory { + public: + static std::unique_ptr Get(const std::string &url) { + std::unique_ptr stager; + if (url.find("file://") == 0) { + stager = std::make_unique(); + } else if (url.find("parquet://")) { + } else if (url.find("hdf5://")) { + } else { + throw std::runtime_error("Unknown stager type"); + } + stager->url_ = url; + } +}; + +} // namespace hermes + +#endif // HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ From 14ff6013db1372119632e6db3dc3962f5c96dac4 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 30 Sep 2023 20:31:10 -0500 Subject: [PATCH 139/191] Initial stager --- .../include/data_stager/data_stager.h | 38 ++++++++---- .../include/data_stager/data_stager_tasks.h | 61 ++++++++++--------- tasks/data_stager/src/stager_factory.h | 1 + .../include/TASK_NAME/TASK_NAME_tasks.h | 7 +++ 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h index d65aa5b58..cae2c3d97 100644 --- a/tasks/data_stager/include/data_stager/data_stager.h +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -56,28 +56,44 @@ class Client : public TaskLibClient { HSHM_ALWAYS_INLINE void AsyncStageInConstruct(StageInTask *task, const TaskNode &task_node, - const DomainId &domain_id) { + const BucketId &bkt_id, + const hshm::charbuf &blob_name, + float score, + u32 node_id) { LABSTOR_CLIENT->ConstructTask( - task, task_node, domain_id, id_); + task, task_node, id_, bkt_id, + blob_name, score, node_id); } HSHM_ALWAYS_INLINE - void StageIn(const DomainId &domain_id) { - LPointer> task = AsyncStageInRoot(domain_id); + void StageInRoot(const BucketId &bkt_id, + const hshm::charbuf &blob_name, + float score, + u32 node_id) { + LPointer> task = + AsyncStageInRoot(bkt_id, blob_name, score, node_id); task.ptr_->Wait(); } LABSTOR_TASK_NODE_PUSH_ROOT(StageIn); /** Stage out data to a remote source */ HSHM_ALWAYS_INLINE - void AsyncStageOutConstruct(StageInTask *task, - const TaskNode &task_node, - const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( - task, task_node, domain_id, id_); + void AsyncStageOutConstruct(StageOutTask *task, + const TaskNode &task_node, + const BucketId &bkt_id, + const hshm::charbuf &blob_name, + const hipc::Pointer &data, + size_t data_size) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_, bkt_id, + blob_name, data, data_size); } HSHM_ALWAYS_INLINE - void StageOut(const DomainId &domain_id) { - LPointer> task = AsyncStageInRoot(domain_id); + void StageOutRoot(const BucketId &bkt_id, + const hshm::charbuf &blob_name, + const hipc::Pointer &data, + size_t data_size) { + LPointer> task = + AsyncStageOutRoot(bkt_id, blob_name, data, data_size); task.ptr_->Wait(); } LABSTOR_TASK_NODE_PUSH_ROOT(StageOut); diff --git a/tasks/data_stager/include/data_stager/data_stager_tasks.h b/tasks/data_stager/include/data_stager/data_stager_tasks.h index 8762b82b8..fcaea7284 100644 --- a/tasks/data_stager/include/data_stager/data_stager_tasks.h +++ b/tasks/data_stager/include/data_stager/data_stager_tasks.h @@ -112,6 +112,7 @@ struct RegisterStagerTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); + ar(bkt_id_, url_); } /** (De)serialize message return */ @@ -119,6 +120,12 @@ struct RegisterStagerTask : public Task, TaskFlags { void SerializeEnd(u32 replica, Ar &ar) { } + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} + /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { @@ -129,7 +136,7 @@ struct RegisterStagerTask : public Task, TaskFlags { /** * Unregister a new stager * */ -struct UnregisterStagerTask : public Task, TaskFlags { +struct UnregisterStagerTask : public Task, TaskFlags { hermes::BucketId bkt_id_; /** SHM default constructor */ @@ -160,6 +167,7 @@ struct UnregisterStagerTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); + ar(bkt_id_); } /** (De)serialize message return */ @@ -167,6 +175,12 @@ struct UnregisterStagerTask : public Task, TaskFlags { void SerializeEnd(u32 replica, Ar &ar) { } + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} + /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { @@ -177,10 +191,11 @@ struct UnregisterStagerTask : public Task, TaskFlags { /** * A task to stage in data from a remote source * */ -struct StageInTask : public Task, TaskFlags { +struct StageInTask : public Task, TaskFlags { hermes::BucketId bkt_id_; hipc::ShmArchive blob_name_; float score_; + u32 node_id_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -190,35 +205,31 @@ struct StageInTask : public Task, TaskFlags { HSHM_ALWAYS_INLINE explicit StageInTask(hipc::Allocator *alloc, const TaskNode &task_node, - const DomainId &domain_id, const TaskStateId &state_id, const BucketId &bkt_id, const hshm::charbuf &blob_name, - float score) : Task(alloc) { + float score, + u32 node_id) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kStageIn; - task_flags_.SetBits(0); - domain_id_ = domain_id; + task_flags_.SetBits(TASK_COROUTINE | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); + domain_id_ = DomainId::GetLocal(); // Custom params bkt_id_ = bkt_id; HSHM_MAKE_AR(blob_name_, alloc, blob_name); score_ = score; + node_id_ = node_id; } - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { + /** Destructor */ + HSHM_ALWAYS_INLINE + ~StageInTask() { + HSHM_DESTROY_AR(blob_name_) } /** Create group */ @@ -231,7 +242,7 @@ struct StageInTask : public Task, TaskFlags { /** * A task to stage data out of a hermes to a remote source * */ -struct StageOutTask : public Task, TaskFlags { +struct StageOutTask : public Task, TaskFlags { hermes::BucketId bkt_id_; hipc::ShmArchive blob_name_; hipc::Pointer data_; @@ -245,7 +256,6 @@ struct StageOutTask : public Task, TaskFlags { HSHM_ALWAYS_INLINE explicit StageOutTask(hipc::Allocator *alloc, const TaskNode &task_node, - const DomainId &domain_id, const TaskStateId &state_id, const BucketId &bkt_id, const hshm::charbuf &blob_name, @@ -257,8 +267,8 @@ struct StageOutTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kStageOut; - task_flags_.SetBits(0); - domain_id_ = domain_id; + task_flags_.SetBits(TASK_COROUTINE | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); + domain_id_ = DomainId::GetLocal(); // Custom params bkt_id_ = bkt_id; @@ -267,15 +277,10 @@ struct StageOutTask : public Task, TaskFlags { data_size_ = data_size; } - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { + /** Destructor */ + HSHM_ALWAYS_INLINE + ~StageOutTask() { + HSHM_DESTROY_AR(blob_name_) } /** Create group */ diff --git a/tasks/data_stager/src/stager_factory.h b/tasks/data_stager/src/stager_factory.h index 4adf0270b..4bc015259 100644 --- a/tasks/data_stager/src/stager_factory.h +++ b/tasks/data_stager/src/stager_factory.h @@ -23,6 +23,7 @@ class StagerFactory { throw std::runtime_error("Unknown stager type"); } stager->url_ = url; + return stager; } }; diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h index 0c464e258..39b87c5e7 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h @@ -107,6 +107,13 @@ struct CustomTask : public Task, TaskFlags { void SerializeEnd(u32 replica, Ar &ar) { } + /** Begin replication */ + void ReplicateStart(u32 count) { + } + + /** Finalize replication */ + void ReplicateEnd() {} + /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { From 561decf26d757a7f076f9c52fed505acd5c917d4 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 30 Sep 2023 22:52:27 -0500 Subject: [PATCH 140/191] Create stager --- CMakeLists.txt | 1 + ci/external/CMakeLists.txt | 18 ------- ci/external/test.cc | 16 ------ ci/py_hermes_ci/bin/run_test | 2 +- tasks/bdev/include/bdev/bdev.h | 5 +- .../include/data_stager/data_stager.h | 46 +++++++++++++++- .../include/data_stager/data_stager_tasks.h | 12 ++--- tasks/hermes/include/hermes/config_manager.h | 7 ++- .../include/hermes_adapters/hermes_adapters.h | 3 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 10 ++-- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 7 ++- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 54 ++----------------- .../hermes_bucket_mdm/hermes_bucket_mdm.h | 10 ++-- .../hermes_bucket_mdm_tasks.h | 8 ++- .../src/hermes_bucket_mdm.cc | 13 ++++- .../TASK_NAME/include/TASK_NAME/TASK_NAME.h | 3 +- .../include/proc_queue/proc_queue.h | 3 +- 17 files changed, 105 insertions(+), 113 deletions(-) delete mode 100644 ci/external/CMakeLists.txt delete mode 100644 ci/external/test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 56f245755..89e163d9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,6 +153,7 @@ include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_mdm/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_blob_mdm/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_bucket_mdm/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_adapters/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/data_stager/include) # Test includes include_directories(${CMAKE_SOURCE_DIR}/test/unit) diff --git a/ci/external/CMakeLists.txt b/ci/external/CMakeLists.txt deleted file mode 100644 index 32d44c9a3..000000000 --- a/ci/external/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(hermes_shm) - -set(CMAKE_CXX_STANDARD 17) - -#------------------------------------------------------------------------------ -# External libraries -#------------------------------------------------------------------------------ - -find_package(HermesShm CONFIG REQUIRED) -message(STATUS "found hermes_shm at ${HermesShm_DIR}") - -#------------------------------------------------------------------------------ -# Build -#------------------------------------------------------------------------------ - -add_executable(test test.cc) -target_link_libraries(test ${HermesShm_LIBRARIES}) \ No newline at end of file diff --git a/ci/external/test.cc b/ci/external/test.cc deleted file mode 100644 index cbf628b8c..000000000 --- a/ci/external/test.cc +++ /dev/null @@ -1,16 +0,0 @@ -// -// Created by lukemartinlogan on 6/6/23. -// - -#include "hermes_shm/hermes_shm.h" - -int main() { - std::string shm_url = "test_serializers"; - hipc::allocator_id_t alloc_id(0, 1); - auto mem_mngr = HERMES_MEMORY_MANAGER; - mem_mngr->UnregisterAllocator(alloc_id); - mem_mngr->UnregisterBackend(shm_url); - mem_mngr->CreateBackend( - hipc::MemoryManager::GetDefaultBackendSize(), shm_url); - mem_mngr->CreateAllocator(shm_url, alloc_id, 0); -} \ No newline at end of file diff --git a/ci/py_hermes_ci/bin/run_test b/ci/py_hermes_ci/bin/run_test index 7d1f04027..51383167f 100755 --- a/ci/py_hermes_ci/bin/run_test +++ b/ci/py_hermes_ci/bin/run_test @@ -33,7 +33,7 @@ if __name__ == '__main__': elif test_type == 'native': pkg_dir = f"{HERMES_ROOT}/test" elif test_type == 'data_stager': - pkg_dir = f"{HERMES_ROOT}/data_stager/test" + pkg_dir = f"{HERMES_ROOT}data_stager/test" elif test_type == 'kvstore': pkg_dir = f"{HERMES_ROOT}/hermes_adapters/kvstore" elif test_type == 'java_wrapper': diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index 17b130b87..2df064cee 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -62,9 +62,10 @@ class Client : public TaskLibClient { template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { - auto *task = AsyncCreateRoot(std::forward(args)...); + LPointer task = + AsyncCreateRoot(std::forward(args)...); task->Wait(); - AsyncCreateComplete(task); + AsyncCreateComplete(task.ptr_); } /** Destroy task state + queue */ diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h index cae2c3d97..9652b6132 100644 --- a/tasks/data_stager/include/data_stager/data_stager.h +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -11,7 +11,6 @@ namespace hermes::data_stager { /** Create data_stager requests */ class Client : public TaskLibClient { - public: /** Default constructor */ Client() = default; @@ -39,7 +38,8 @@ class Client : public TaskLibClient { template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { - auto *task = AsyncCreateRoot(std::forward(args)...); + LPointer task = + AsyncCreateRoot(std::forward(args)...); task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); @@ -52,6 +52,40 @@ class Client : public TaskLibClient { LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); } + /** Register task state */ + HSHM_ALWAYS_INLINE + void AsyncRegisterStagerConstruct(RegisterStagerTask *task, + const TaskNode &task_node, + const BucketId &bkt_id, + const hshm::charbuf &url) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_, bkt_id, url); + } + HSHM_ALWAYS_INLINE + void RegisterStagerRoot(const BucketId &bkt_id, + const hshm::charbuf &url) { + LPointer> task = + AsyncRegisterStagerRoot(bkt_id, url); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(RegisterStager); + + /** Unregister task state */ + HSHM_ALWAYS_INLINE + void AsyncUnregisterStagerConstruct(UnregisterStagerTask *task, + const TaskNode &task_node, + const BucketId &bkt_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_, bkt_id); + } + HSHM_ALWAYS_INLINE + void UnregisterStagerRoot(const BucketId &bkt_id) { + LPointer> task = + AsyncUnregisterStagerRoot(bkt_id); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(UnregisterStager); + /** Stage in data from a remote source */ HSHM_ALWAYS_INLINE void AsyncStageInConstruct(StageInTask *task, @@ -97,6 +131,14 @@ class Client : public TaskLibClient { task.ptr_->Wait(); } LABSTOR_TASK_NODE_PUSH_ROOT(StageOut); + + /** URL parser */ + static hshm::charbuf ParseUrl(const hshm::string &url) { + hshm::charbuf ret; + ret.resize(url.size() + 1); + memcpy(ret.data(), url.data(), url.size() + 1); + return ret; + } }; } // namespace labstor diff --git a/tasks/data_stager/include/data_stager/data_stager_tasks.h b/tasks/data_stager/include/data_stager/data_stager_tasks.h index fcaea7284..30a2970bc 100644 --- a/tasks/data_stager/include/data_stager/data_stager_tasks.h +++ b/tasks/data_stager/include/data_stager/data_stager_tasks.h @@ -90,18 +90,17 @@ struct RegisterStagerTask : public Task, TaskFlags { HSHM_ALWAYS_INLINE explicit RegisterStagerTask(hipc::Allocator *alloc, const TaskNode &task_node, - const DomainId &domain_id, const TaskStateId &state_id, hermes::BucketId bkt_id, - hshm::charbuf &url) : Task(alloc) { + const hshm::charbuf &url) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = bkt_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kRegisterStager; - task_flags_.SetBits(TASK_FIRE_AND_FORGET); - domain_id_ = domain_id; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = DomainId::GetGlobal(); // Custom params bkt_id_ = bkt_id; @@ -147,9 +146,8 @@ struct UnregisterStagerTask : public Task, TaskFlags { HSHM_ALWAYS_INLINE explicit UnregisterStagerTask(hipc::Allocator *alloc, const TaskNode &task_node, - const DomainId &domain_id, const TaskStateId &state_id, - hermes::BucketId bkt_id) : Task(alloc) { + const hermes::BucketId &bkt_id) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = bkt_id.hash_; @@ -157,7 +155,7 @@ struct UnregisterStagerTask : public Task, TaskFlags { task_state_ = state_id; method_ = Method::kUnregisterStager; task_flags_.SetBits(TASK_FIRE_AND_FORGET); - domain_id_ = domain_id; + domain_id_ = DomainId::GetGlobal(); // Custom params bkt_id_ = bkt_id; diff --git a/tasks/hermes/include/hermes/config_manager.h b/tasks/hermes/include/hermes/config_manager.h index 7e3c23c83..f88eb6ba6 100644 --- a/tasks/hermes/include/hermes/config_manager.h +++ b/tasks/hermes/include/hermes/config_manager.h @@ -12,6 +12,7 @@ #include "hermes_blob_mdm/hermes_blob_mdm.h" #include "hermes/config_client.h" #include "hermes/config_server.h" +#include "data_stager/data_stager.h" namespace hermes { @@ -20,6 +21,7 @@ class ConfigurationManager { mdm::Client mdm_; bucket_mdm::Client bkt_mdm_; blob_mdm::Client blob_mdm_; + data_stager::Client stager_mdm_; ServerConfig server_config_; ClientConfig client_config_; bool is_initialized_ = false; @@ -38,8 +40,9 @@ class ConfigurationManager { mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_mdm"); blob_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_blob_mdm"); bkt_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_bkt_mdm"); - blob_mdm_.SetBucketMdmRoot(DomainId::GetGlobal(), bkt_mdm_.id_); - bkt_mdm_.SetBlobMdmRoot(DomainId::GetGlobal(), blob_mdm_.id_); + stager_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_stager_mdm", blob_mdm_.id_); + blob_mdm_.SetBucketMdmRoot(DomainId::GetGlobal(), bkt_mdm_.id_, stager_mdm_.id_); + bkt_mdm_.SetBlobMdmRoot(DomainId::GetGlobal(), blob_mdm_.id_, stager_mdm_.id_); is_initialized_ = true; } diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h index 1817cdf75..463166e26 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h @@ -38,7 +38,8 @@ class Client : public TaskLibClient { template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { - auto *task = AsyncCreateRoot(std::forward(args)...); + LPointer task = + AsyncCreateRoot(std::forward(args)...); task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index c5f86a2ac..f332a5bce 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -70,14 +70,16 @@ class Client : public TaskLibClient { void AsyncSetBucketMdmConstruct(SetBucketMdmTask *task, const TaskNode &task_node, const DomainId &domain_id, - const TaskStateId &blob_mdm_id) { + const TaskStateId &blob_mdm_id, + const TaskStateId &stager_mdm) { LABSTOR_CLIENT->ConstructTask( - task, task_node, domain_id, id_, blob_mdm_id); + task, task_node, domain_id, id_, blob_mdm_id, stager_mdm); } void SetBucketMdmRoot(const DomainId &domain_id, - const TaskStateId &blob_mdm_id) { + const TaskStateId &blob_mdm, + const TaskStateId &stager_mdm) { LPointer> push_task = - AsyncSetBucketMdmRoot(domain_id, blob_mdm_id); + AsyncSetBucketMdmRoot(domain_id, blob_mdm, stager_mdm); push_task->Wait(); LABSTOR_CLIENT->DelTask(push_task); } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 0705260b9..78b0fffd3 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -78,6 +78,7 @@ struct DestructTask : public DestroyTaskStateTask { /** Set the BUCKET MDM ID */ struct SetBucketMdmTask : public Task, TaskFlags { IN TaskStateId bkt_mdm_; + IN TaskStateId stager_mdm_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -89,7 +90,8 @@ struct SetBucketMdmTask : public Task, TaskFlags { const TaskNode &task_node, const DomainId &domain_id, const TaskStateId &state_id, - const TaskStateId &bkt_mdm) : Task(alloc) { + const TaskStateId &bkt_mdm, + const TaskStateId &stager_mdm) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = 0; @@ -101,6 +103,7 @@ struct SetBucketMdmTask : public Task, TaskFlags { // Custom params bkt_mdm_ = bkt_mdm; + stager_mdm_ = stager_mdm; } /** Destructor */ @@ -110,7 +113,7 @@ struct SetBucketMdmTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); - ar(bkt_mdm_); + ar(bkt_mdm_, stager_mdm_); } /** (De)serialize message return */ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 6840acb03..27d4d349c 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -9,6 +9,7 @@ #include "hermes/dpe/dpe_factory.h" #include "hermes_adapters/posix/posix_api.h" #include "bdev/bdev.h" +#include "data_stager/data_stager.h" namespace hermes::blob_mdm { @@ -46,6 +47,7 @@ class Server : public TaskLib { std::unordered_map target_map_; Client blob_mdm_; bucket_mdm::Client bkt_mdm_; + data_stager::Client stager_mdm_; public: Server() = default; @@ -109,6 +111,7 @@ class Server : public TaskLib { * */ void SetBucketMdm(SetBucketMdmTask *task, RunContext &rctx) { bkt_mdm_.Init(task->bkt_mdm_); + stager_mdm_.Init(task->stager_mdm_); task->SetModuleComplete(); } @@ -127,7 +130,6 @@ class Server : public TaskLib { // Update the blob info if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { - // Update blob info blob_info.name_ = std::move(blob_name); blob_info.blob_id_ = task->blob_id_; blob_info.tag_id_ = task->tag_id_; @@ -146,55 +148,12 @@ class Server : public TaskLib { PutBlobFreeBuffersPhase(blob_info, task, rctx); } - // Stage in blob data from FS - LPointer data_ptr; - data_ptr.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->data_); - data_ptr.shm_ = task->data_; - size_t data_off = 0; - if (task->filename_->size() > 0) { - adapter::BlobPlacement plcmnt; - plcmnt.DecodeBlobName(*task->blob_name_); - task->flags_.SetBits(HERMES_IS_FILE); - data_off = plcmnt.bucket_off_ + task->blob_off_ + task->data_size_; - if (blob_info.blob_size_ == 0 && - task->blob_off_ == 0 && - task->data_size_ < task->page_size_) { - HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", - task->page_size_, task->filename_->str(), plcmnt.bucket_off_); - LPointer new_data_ptr = LABSTOR_CLIENT->AllocateBuffer(task->page_size_); - int fd = HERMES_POSIX_API->open(task->filename_->c_str(), O_RDONLY); - if (fd < 0) { - HELOG(kError, "Failed to open file {}", task->filename_->str()); - } - int ret = HERMES_POSIX_API->pread(fd, new_data_ptr.ptr_, task->page_size_, (off_t)plcmnt.bucket_off_); - if (ret < 0) { - // TODO(llogan): ret != page_size_ will require knowing file size before-hand - HELOG(kError, "Failed to stage in {} bytes from {}", task->page_size_, task->filename_->str()); - } - HERMES_POSIX_API->close(fd); - memcpy(new_data_ptr.ptr_ + plcmnt.blob_off_, data_ptr.ptr_, task->data_size_); - data_ptr = new_data_ptr; - task->blob_off_ = 0; - if (ret < task->blob_off_ + task->data_size_) { - task->data_size_ = task->blob_off_ + task->data_size_; - } else { - task->data_size_ = ret; - } - task->flags_.SetBits(HERMES_DID_STAGE_IN); - HILOG(kDebug, "Staged {} bytes from the backend file {}", - task->data_size_, task->filename_->str()); - } - } - // Determine amount of additional buffering space needed size_t needed_space = task->blob_off_ + task->data_size_; size_t size_diff = 0; if (needed_space > blob_info.max_blob_size_) { size_diff = needed_space - blob_info.max_blob_size_; } - if (!task->flags_.Any(HERMES_IS_FILE)) { - data_off = size_diff; - } blob_info.blob_size_ += size_diff; HILOG(kDebug, "The size diff is {} bytes", size_diff) @@ -230,7 +189,7 @@ class Server : public TaskLib { std::vector> write_tasks; write_tasks.reserve(blob_info.buffers_.size()); size_t blob_off = 0, buf_off = 0; - char *blob_buf = data_ptr.ptr_; + char *blob_buf = LABSTOR_CLIENT->GetPrivatePointer(task->data_); HILOG(kDebug, "Number of buffers {}", blob_info.buffers_.size()); for (BufferInfo &buf : blob_info.buffers_) { size_t blob_left = blob_off; @@ -268,7 +227,7 @@ class Server : public TaskLib { } bkt_mdm_.AsyncUpdateSize(task->task_node_ + 1, task->tag_id_, - data_off, + task->blob_off_ + task->data_size_, update_mode); if (task->flags_.Any(HERMES_BLOB_DID_CREATE)) { bkt_mdm_.AsyncTagAddBlob(task->task_node_ + 1, @@ -277,9 +236,6 @@ class Server : public TaskLib { } // Free data - if (task->flags_.Any(HERMES_DID_STAGE_IN)) { - LABSTOR_CLIENT->FreeBuffer(data_ptr); - } task->SetModuleComplete(); } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index 479085856..ed515cea4 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -48,14 +48,16 @@ class Client : public TaskLibClient { void AsyncSetBlobMdmConstruct(SetBlobMdmTask *task, const TaskNode &task_node, const DomainId &domain_id, - const TaskStateId &blob_mdm_id) { + const TaskStateId &blob_mdm, + const TaskStateId &stager_mdm) { LABSTOR_CLIENT->ConstructTask( - task, task_node, domain_id, id_, blob_mdm_id); + task, task_node, domain_id, id_, blob_mdm, stager_mdm); } void SetBlobMdmRoot(const DomainId &domain_id, - const TaskStateId &blob_mdm_id) { + const TaskStateId &blob_mdm, + const TaskStateId &stager_mdm) { LPointer> push_task = - AsyncSetBlobMdmRoot(domain_id, blob_mdm_id); + AsyncSetBlobMdmRoot(domain_id, blob_mdm, stager_mdm); push_task->Wait(); LABSTOR_CLIENT->DelTask(push_task); } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 11303272f..e253079ca 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -68,6 +68,7 @@ struct DestructTask : public DestroyTaskStateTask { /** Set the BLOB MDM ID */ struct SetBlobMdmTask : public Task, TaskFlags { IN TaskStateId blob_mdm_; + IN TaskStateId stager_mdm_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -79,7 +80,8 @@ struct SetBlobMdmTask : public Task, TaskFlags { const TaskNode &task_node, const DomainId &domain_id, const TaskStateId &state_id, - const TaskStateId &blob_mdm) : Task(alloc) { + const TaskStateId &blob_mdm, + const TaskStateId &stager_mdm) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = 0; @@ -91,6 +93,7 @@ struct SetBlobMdmTask : public Task, TaskFlags { // Custom params blob_mdm_ = blob_mdm; + stager_mdm_ = stager_mdm; } /** Destructor */ @@ -100,7 +103,7 @@ struct SetBlobMdmTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); - ar(blob_mdm_); + ar(blob_mdm_, stager_mdm_); } /** (De)serialize message return */ @@ -335,6 +338,7 @@ struct AppendBlobTask : public Task, TaskFlags { /** A task to get or create a tag */ struct GetOrCreateTagTask : public Task, TaskFlags { IN hipc::ShmArchive tag_name_; + IN hipc::ShmArchive url_; IN bool blob_owner_; IN hipc::ShmArchive> traits_; IN size_t backend_size_; diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 38fb5116c..369902745 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -9,6 +9,7 @@ #include "hermes_adapters/mapper/abstract_mapper.h" #include "hermes/dpe/dpe_factory.h" #include "bdev/bdev.h" +#include "data_stager/data_stager.h" namespace hermes::bucket_mdm { @@ -23,6 +24,7 @@ class Server : public TaskLib { std::atomic id_alloc_; Client bkt_mdm_; blob_mdm::Client blob_mdm_; + data_stager::Client stager_mdm_; public: Server() = default; @@ -45,6 +47,7 @@ class Server : public TaskLib { * */ void SetBlobMdm(SetBlobMdmTask *task, RunContext &rctx) { blob_mdm_.Init(task->blob_mdm_); + stager_mdm_.Init(task->stager_mdm_); task->SetModuleComplete(); } @@ -194,7 +197,8 @@ class Server : public TaskLib { // Check if the tag exists TAG_ID_MAP_T &tag_id_map = tag_id_map_[rctx.lane_id_]; - hshm::charbuf tag_name = hshm::to_charbuf(*task->tag_name_); + hshm::string url = hshm::to_charbuf(*task->url_); + hshm::charbuf tag_name = data_stager::Client::ParseUrl(url); bool did_create = false; if (tag_name.size() > 0) { did_create = tag_id_map.find(tag_name) == tag_id_map.end(); @@ -214,6 +218,13 @@ class Server : public TaskLib { tag_info.tag_id_ = tag_id; tag_info.owner_ = task->blob_owner_; tag_info.internal_size_ = task->backend_size_; + + LPointer register_task = + stager_mdm_.AsyncRegisterStager(task->task_node_ + 1, + tag_id, + hshm::charbuf(task->url_->str())); + register_task->Wait(task); + LABSTOR_CLIENT->DelTask(register_task); } else { if (tag_name.size()) { HILOG(kDebug, "Found existing tag: {}", tag_name.str()) diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h index 2495f6594..1c1e71fd7 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h @@ -38,7 +38,8 @@ class Client : public TaskLibClient { template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { - auto *task = AsyncCreateRoot(std::forward(args)...); + LPointer task = + AsyncCreateRoot(std::forward(args)...); task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue.h b/tasks_required/proc_queue/include/proc_queue/proc_queue.h index 527bf3ec1..170307a68 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue.h @@ -40,7 +40,8 @@ class Client : public TaskLibClient { template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { - auto *task = AsyncCreateRoot(std::forward(args)...); + LPointer task = + AsyncCreateRoot(std::forward(args)...); task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); From 35a52b70f5c63b30bf55451876ca1290b5bbef9e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 2 Oct 2023 00:25:43 -0500 Subject: [PATCH 141/191] First draft of TASK_LANE_ALL --- codegen/refresh_methods | 29 +++ include/labstor/api/labstor_client.h | 45 +++++ .../labstor/queue_manager/queues/hshm_queue.h | 5 + include/labstor/task_registry/task.h | 56 ++++-- include/labstor/task_registry/task_lib.h | 6 + src/worker.cc | 6 + tasks/bdev/include/bdev/bdev_lib_exec.h | 74 +++++++ .../include/data_stager/data_stager.h | 10 +- .../data_stager/data_stager_lib_exec.h | 58 ++++++ .../include/data_stager/data_stager_tasks.h | 28 ++- .../hermes_adapters_lib_exec.h | 34 ++++ .../hermes_blob_mdm_lib_exec.h | 190 ++++++++++++++++++ .../hermes_blob_mdm/hermes_blob_mdm_methods.h | 1 + .../hermes_blob_mdm_methods.yaml | 3 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 63 ++++++ tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 48 ++++- .../hermes_bucket_mdm_lib_exec.h | 130 ++++++++++++ .../hermes_bucket_mdm_tasks.h | 9 + .../src/hermes_bucket_mdm.cc | 12 +- .../include/hermes_mdm/hermes_mdm_lib_exec.h | 26 +++ .../include/TASK_NAME/TASK_NAME_lib_exec.h | 34 ++++ .../include/TASK_NAME/TASK_NAME_tasks.h | 9 + .../labstor_admin/labstor_admin_lib_exec.h | 82 ++++++++ .../labstor_admin/labstor_admin_tasks.h | 11 + .../include/proc_queue/proc_queue_lib_exec.h | 34 ++++ .../include/remote_queue/remote_queue.h | 30 ++- .../remote_queue/remote_queue_lib_exec.h | 78 +++++++ .../remote_queue/remote_queue_methods.h | 1 + .../remote_queue/remote_queue_methods.yaml | 3 +- .../include/remote_queue/remote_queue_tasks.h | 47 ++++- .../remote_queue/src/remote_queue.cc | 13 ++ .../small_message/small_message_lib_exec.h | 50 +++++ .../small_message/small_message_tasks.h | 18 ++ .../worch_proc_round_robin_lib_exec.h | 34 ++++ .../worch_queue_round_robin_lib_exec.h | 34 ++++ 35 files changed, 1268 insertions(+), 43 deletions(-) diff --git a/codegen/refresh_methods b/codegen/refresh_methods index 7929025d6..1e9484641 100755 --- a/codegen/refresh_methods +++ b/codegen/refresh_methods @@ -81,6 +81,35 @@ def refresh_methods(TASK_ROOT): lines += [' }'] lines += ['}'] + ## Create the Dup method + lines += ['/** Duplicate a task */', + 'void Dup(u32 method, Task *orig_task, std::vector> &dups) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' labstor::CALL_DUPLICATE(reinterpret_cast<{task_name}*>(orig_task), dups);', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Register a duplicate's output + lines += ['/** Register the duplicate output with the origin task */', + 'void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + task_cast = f'reinterpret_cast<{task_name}*>' + lines += [f' case Method::{method_enum_name}: {{', + f' labstor::CALL_DUPLICATE_END(replica, {task_cast}(orig_task), {task_cast}(dup_task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + ## Create the ReplicateStart method lines += ['/** Ensure there is space to store replicated outputs */', 'void ReplicateStart(u32 method, u32 count, Task *task) override {', diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index e66b79179..eba69bf27 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -99,6 +99,17 @@ class Client : public ConfigurationManager { return task; } + /** Create a default-constructed task */ + template + HSHM_ALWAYS_INLINE + LPointer NewEmptyTask() { + LPointer task = main_alloc_->NewObjLocal(main_alloc_); + if (task.ptr_ == nullptr) { + throw std::runtime_error("Could not allocate buffer"); + } + return task; + } + /** Allocate task */ template HSHM_ALWAYS_INLINE @@ -308,6 +319,40 @@ class Client : public ConfigurationManager { return push_task;\ } +/** Call duplicate if applicable */ +template +constexpr inline void CALL_DUPLICATE(TaskT *orig_task, std::vector> &dups) { + if constexpr(TaskT::REPLICA) { + for (LPointer &dup : dups) { + LPointer task = LABSTOR_CLIENT->NewEmptyTask(); + orig_task->Dup(LABSTOR_CLIENT->main_alloc_, *task.ptr_); + dup.ptr_ = task.ptr_; + dup.shm_ = task.shm_; + } + } +} +/** Call duplicate if applicable */ +template +constexpr inline void CALL_DUPLICATE_END(u32 replica, TaskT *orig_task, TaskT *dup_task) { + if constexpr(TaskT::REPLICA) { + orig_task->DupEnd(replica, *dup_task); + } +} +/** Call replica start if applicable */ +template +constexpr inline void CALL_REPLICA_START(u32 count, TaskT *task) { + if constexpr(TaskT::REPLICA) { + task->ReplicateStart(count); + } +} +/** Call replica end if applicable */ +template +constexpr inline void CALL_REPLICA_END(TaskT *task) { + if constexpr(TaskT::REPLICA) { + task->ReplicateEnd(); + } +} + } // namespace labstor static inline bool TRANSPARENT_LABSTOR() { diff --git a/include/labstor/queue_manager/queues/hshm_queue.h b/include/labstor/queue_manager/queues/hshm_queue.h index bc7efad5b..d1472b96c 100644 --- a/include/labstor/queue_manager/queues/hshm_queue.h +++ b/include/labstor/queue_manager/queues/hshm_queue.h @@ -74,6 +74,11 @@ struct LaneGroup : public PriorityInfo { bool IsLowPriority() { return flags_.Any(QUEUE_LONG_RUNNING) || prio_ == 0; } + + /** Get lane */ + Lane& GetLane(u32 lane_id) { + return (*lanes_)[lane_id]; + } }; /** Represents the HSHM queue type */ diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index b42b0b94b..2ec37aa37 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -51,6 +51,12 @@ class TaskLib; #define TASK_PREEMPTIVE BIT_OPT(u32, 17) /** This task is apart of remote debugging */ #define TASK_REMOTE_DEBUG_MARK BIT_OPT(u32, 18) +/** This task will match all groups */ +#define TASK_GROUP_ANY BIT_OPT(u32, 19) +/** This task can be scheduled on any lane */ +#define TASK_LANE_ANY BIT_OPT(u32, 20) +/** This task should be scheduled on all lanes */ +#define TASK_LANE_ALL BIT_OPT(u32, 21) /** Used to define task methods */ #define TASK_METHOD_T static inline const u32 @@ -177,6 +183,7 @@ static inline std::ostream &operator<<(std::ostream &os, const TaskNode &obj) { #define TF_SRL_ASYM (TF_SRL_ASYM_START | TF_SRL_ASYM_END) /** This task uses replication */ #define TF_REPLICA BIT_OPT(u32, 31) +/** This task uses duplication */ /** This task is intended to be used only locally */ #define TF_LOCAL BIT_OPT(u32, 5) @@ -196,20 +203,6 @@ class IsTask {}; /** Determine this task uses SerializeEnd */ #define USES_SRL_END(T) \ T::SRL_SYM_END -/** Call replica start if applicable */ -template -constexpr inline void CALL_REPLICA_START(u32 count, T *task) { - if constexpr(T::REPLICA) { - task->ReplicateStart(count); - } -} -/** Call replica end if applicable */ -template -constexpr inline void CALL_REPLICA_END(T *task) { - if constexpr(T::REPLICA) { - task->ReplicateEnd(); - } -} /** Compile-time flags indicating task methods and operation support */ template @@ -386,6 +379,16 @@ struct Task : public hipc::ShmContainer { return task_flags_.Any(TASK_PREEMPTIVE); } + /** This task should be dispersed across all lanes */ + HSHM_ALWAYS_INLINE bool IsLaneAll() { + return task_flags_.Any(TASK_LANE_ALL); + } + + /** Unset this task as lane-dispersable */ + HSHM_ALWAYS_INLINE void UnsetLaneAll() { + task_flags_.UnsetBits(TASK_LANE_ALL); + } + /** Yield the task */ template HSHM_ALWAYS_INLINE @@ -499,13 +502,24 @@ struct Task : public hipc::ShmContainer { /** Sets this Task as empty */ HSHM_ALWAYS_INLINE void SetNull() {} - /**==================================== - * Serialization - * ===================================*/ - template - void task_serialize(Ar &ar) { - ar(task_state_, task_node_, domain_id_, lane_hash_, prio_, method_, task_flags_); - } + /**==================================== + * Serialization + * ===================================*/ + template + void task_serialize(Ar &ar) { + ar(task_state_, task_node_, domain_id_, lane_hash_, prio_, method_, task_flags_); + } + + template + void task_dup(TaskT &other) { + task_state_ = other.task_state_; + task_node_ = other.task_node_; + domain_id_ = other.domain_id_; + lane_hash_ = other.lane_hash_; + prio_ = other.prio_; + method_ = other.method_; + task_flags_ = other.task_flags_; + } /**==================================== * Grouping diff --git a/include/labstor/task_registry/task_lib.h b/include/labstor/task_registry/task_lib.h index e30d9b70b..053ff5ac9 100644 --- a/include/labstor/task_registry/task_lib.h +++ b/include/labstor/task_registry/task_lib.h @@ -44,6 +44,12 @@ class TaskLib { /** Delete a task */ virtual void Del(u32 method, Task *task) = 0; + /** Duplicate a task */ + virtual void Dup(u32 method, Task *orig_task, std::vector> &dups) = 0; + + /** Register end of duplicate */ + virtual void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) = 0; + /** Allow task to store replicas of completion */ virtual void ReplicateStart(u32 method, u32 count, Task *task) = 0; diff --git a/src/worker.cc b/src/worker.cc index 9cdaa8f4d..a9553b089 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -87,6 +87,12 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->DisableRun(); task->SetUnordered(); task->UnsetCoroutine(); + } else if (task->IsLaneAll()) { + LABSTOR_REMOTE_QUEUE->DisperseLocal(task, exec, work_entry.queue_, work_entry.group_); + task->DisableRun(); + task->SetUnordered(); + task->UnsetCoroutine(); + task->UnsetLaneAll(); } else if (task->IsCoroutine()) { if (!task->IsStarted()) { rctx.stack_ptr_ = malloc(rctx.stack_size_); diff --git a/tasks/bdev/include/bdev/bdev_lib_exec.h b/tasks/bdev/include/bdev/bdev_lib_exec.h index 88d49653d..a1cf7beed 100644 --- a/tasks/bdev/include/bdev/bdev_lib_exec.h +++ b/tasks/bdev/include/bdev/bdev_lib_exec.h @@ -75,6 +75,80 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kWrite: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRead: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kAllocate: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kFree: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kMonitor: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kUpdateCapacity: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kWrite: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRead: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kAllocate: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kFree: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kMonitor: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kUpdateCapacity: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h index 9652b6132..64ce90f5c 100644 --- a/tasks/data_stager/include/data_stager/data_stager.h +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -116,18 +116,20 @@ class Client : public TaskLibClient { const BucketId &bkt_id, const hshm::charbuf &blob_name, const hipc::Pointer &data, - size_t data_size) { + size_t data_size, + u32 task_flags) { LABSTOR_CLIENT->ConstructTask( task, task_node, id_, bkt_id, - blob_name, data, data_size); + blob_name, data, data_size, task_flags); } HSHM_ALWAYS_INLINE void StageOutRoot(const BucketId &bkt_id, const hshm::charbuf &blob_name, const hipc::Pointer &data, - size_t data_size) { + size_t data_size, + u32 task_flags) { LPointer> task = - AsyncStageOutRoot(bkt_id, blob_name, data, data_size); + AsyncStageOutRoot(bkt_id, blob_name, data, data_size, task_flags); task.ptr_->Wait(); } LABSTOR_TASK_NODE_PUSH_ROOT(StageOut); diff --git a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h index 8e157a7af..3a12864da 100644 --- a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h +++ b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h @@ -59,6 +59,64 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRegisterStager: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kUnregisterStager: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kStageIn: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kStageOut: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRegisterStager: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kUnregisterStager: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kStageIn: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kStageOut: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/data_stager/include/data_stager/data_stager_tasks.h b/tasks/data_stager/include/data_stager/data_stager_tasks.h index 30a2970bc..da7e142ed 100644 --- a/tasks/data_stager/include/data_stager/data_stager_tasks.h +++ b/tasks/data_stager/include/data_stager/data_stager_tasks.h @@ -99,7 +99,7 @@ struct RegisterStagerTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kRegisterStager; - task_flags_.SetBits(TASK_LOW_LATENCY); + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_FIRE_AND_FORGET); domain_id_ = DomainId::GetGlobal(); // Custom params @@ -107,6 +107,15 @@ struct RegisterStagerTask : public Task, TaskFlags { HSHM_MAKE_AR(url_, alloc, url); } + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, RegisterStagerTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, RegisterStagerTask &dup_task) { + } + /** (De)serialize message call */ template void SerializeStart(Ar &ar) { @@ -161,6 +170,15 @@ struct UnregisterStagerTask : public Task, TaskFlags { bkt_id_ = bkt_id; } + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, UnregisterStagerTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, UnregisterStagerTask &dup_task) { + } + /** (De)serialize message call */ template void SerializeStart(Ar &ar) { @@ -258,14 +276,15 @@ struct StageOutTask : public Task, TaskFlags { const BucketId &bkt_id, const hshm::charbuf &blob_name, const hipc::Pointer &data, - size_t data_size) : Task(alloc) { + size_t data_size, + u32 task_flags): Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = bkt_id.hash_; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kStageOut; - task_flags_.SetBits(TASK_COROUTINE | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); + task_flags_.SetBits(task_flags | TASK_COROUTINE | TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK); domain_id_ = DomainId::GetLocal(); // Custom params @@ -279,6 +298,9 @@ struct StageOutTask : public Task, TaskFlags { HSHM_ALWAYS_INLINE ~StageOutTask() { HSHM_DESTROY_AR(blob_name_) + if (IsDataOwner()) { + LABSTOR_CLIENT->FreeBuffer(data_); + } } /** Create group */ diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h index c7fbb171e..7ff739e1c 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h @@ -35,6 +35,40 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kCustom: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kCustom: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index 58b52ac96..b1ac48623 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -72,6 +72,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { SetBucketMdm(reinterpret_cast(task), rctx); break; } + case Method::kFlushData: { + FlushData(reinterpret_cast(task), rctx); + break; + } } } /** Delete a task */ @@ -145,6 +149,164 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kFlushData: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kPutBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kTruncateBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestroyBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kTagBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kBlobHasTag: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetBlobId: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetOrCreateBlobId: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetBlobName: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetBlobSize: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetBlobScore: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetBlobBuffers: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRenameBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kReorganizeBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kSetBucketMdm: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kFlushData: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kPutBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kTruncateBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestroyBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kTagBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kBlobHasTag: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetBlobId: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetOrCreateBlobId: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetBlobName: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetBlobSize: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetBlobScore: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetBlobBuffers: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRenameBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kReorganizeBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kSetBucketMdm: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kFlushData: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } } } /** Ensure there is space to store replicated outputs */ @@ -218,6 +380,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kFlushData: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } } } /** Determine success and handle failures */ @@ -291,6 +457,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kFlushData: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } } } /** Serialize a task when initially pushing into remote */ @@ -364,6 +534,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kFlushData: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -456,6 +630,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kFlushData: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } } return task_ptr; } @@ -530,6 +709,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kFlushData: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -604,6 +787,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kFlushData: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } } } /** Get the grouping of the task */ @@ -660,6 +847,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kSetBucketMdm: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kFlushData: { + return reinterpret_cast(task)->GetGroup(group); + } } return -1; } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h index 2cbd3cb79..3ee9d30f6 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h @@ -18,6 +18,7 @@ struct Method : public TaskMethod { TASK_METHOD_T kRenameBlob = kLast + 14; TASK_METHOD_T kReorganizeBlob = kLast + 15; TASK_METHOD_T kSetBucketMdm = kLast + 16; + TASK_METHOD_T kFlushData = kLast + 17; }; #endif // LABSTOR_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml index d6c7b93db..b53850223 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml @@ -14,4 +14,5 @@ kGetBlobScore: 12 kGetBlobBuffers: 13 kRenameBlob: 14 kReorganizeBlob: 15 -kSetBucketMdm: 16 \ No newline at end of file +kSetBucketMdm: 16 +kFlushData: 17 \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 78b0fffd3..a00ccba4f 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -109,6 +109,15 @@ struct SetBucketMdmTask : public Task, TaskFlags { /** Destructor */ ~SetBucketMdmTask() {} + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, SetBucketMdmTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, SetBucketMdmTask &dup_task) { + } + /** (De)serialize message call */ template void SerializeStart(Ar &ar) { @@ -1133,6 +1142,60 @@ struct ReorganizeBlobTask : public Task, TaskFlags { } }; +/** A task to reorganize a blob's composition in the hierarchy */ +struct FlushDataTask : public Task, TaskFlags { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + FlushDataTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + FlushDataTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kReorganizeBlob; + task_flags_.SetBits(TASK_LANE_ALL | TASK_FIRE_AND_FORGET | TASK_LONG_RUNNING | TASK_COROUTINE); + domain_id_ = DomainId::GetGlobal(); + } + + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, FlushDataTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, FlushDataTask &dup_task) { + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_GROUP_ANY; + } +}; + } // namespace hermes::blob_mdm #endif //LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 27d4d349c..ad71d77df 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -115,6 +115,39 @@ class Server : public TaskLib { task->SetModuleComplete(); } + /** + * Long-running task to stage out data periodically + * */ + void FlushData(FlushDataTask *task, RunContext &rctx) { + // Get the blob info data structure + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; + for (auto &it : blob_map) { + BlobInfo &blob_info = it.second; + if (blob_info.last_flush_ > 0 && + blob_info.mod_count_ > blob_info.last_flush_) { + blob_info.last_flush_ = 1; + blob_info.mod_count_ = 0; + blob_info.access_freq_ = 0; + blob_info.UpdateWriteStats(); + LPointer data = LABSTOR_CLIENT->AllocateBuffer(blob_info.blob_size_); + LPointer get_blob = + blob_mdm_.AsyncGetBlob(task->task_node_ + 1, + blob_info.tag_id_, + blob_info.name_, + blob_info.blob_id_, + 0, blob_info.blob_size_, + data.shm_); + get_blob->Wait(task); + stager_mdm_.AsyncStageOut(task->task_node_ + 1, + blob_info.tag_id_, + blob_info.name_, + data.shm_, blob_info.blob_size_, + TASK_DATA_OWNER | TASK_FIRE_AND_FORGET); + } + } + task->SetModuleComplete(); + } + /** * Create a blob's metadata * */ @@ -136,10 +169,19 @@ class Server : public TaskLib { blob_info.blob_size_ = 0; blob_info.max_blob_size_ = 0; blob_info.score_ = task->score_; - blob_info.mod_count_ = 0; + blob_info.mod_count_ = 1; blob_info.access_freq_ = 0; - blob_info.last_flush_ = 0; + blob_info.last_flush_ = 1; blob_info.UpdateWriteStats(); + if (task->flags_.Any(HERMES_IS_FILE)) { + LPointer stage_task = + stager_mdm_.AsyncStageIn(task->task_node_ + 1, + task->tag_id_, + blob_info.name_, + task->score_, 0); + stage_task->Wait(task); + LABSTOR_CLIENT->DelTask(stage_task); + } } else { // Modify existing blob blob_info.UpdateWriteStats(); @@ -179,7 +221,7 @@ class Server : public TaskLib { // SubPlacement &next_placement = schema.plcmnts_[sub_idx + 1]; // size_t diff = alloc_task->size_ - alloc_task->alloc_size_; // next_placement.size_ += diff; - HILOG(kFatal, "Ran outta space in this tier -- will fix soon") + HELOG(kFatal, "Ran outta space in this tier -- will fix soon") } LABSTOR_CLIENT->DelTask(alloc_task); } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index 1c5127c07..51d0f77b4 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -131,6 +131,136 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetOrCreateTag: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetTagId: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetTagName: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRenameTag: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestroyTag: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kTagAddBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kTagRemoveBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kTagClearBlobs: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kUpdateSize: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kAppendBlobSchema: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kAppendBlob: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetSize: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kSetBlobMdm: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetOrCreateTag: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetTagId: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetTagName: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRenameTag: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestroyTag: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kTagAddBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kTagRemoveBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kTagClearBlobs: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kUpdateSize: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kAppendBlobSchema: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kAppendBlob: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetSize: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kSetBlobMdm: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index e253079ca..6a372e915 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -99,6 +99,15 @@ struct SetBlobMdmTask : public Task, TaskFlags { /** Destructor */ ~SetBlobMdmTask() {} + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, SetBlobMdmTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, SetBlobMdmTask &dup_task) { + } + /** (De)serialize message call */ template void SerializeStart(Ar &ar) { diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 369902745..3f005dcbc 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -218,13 +218,9 @@ class Server : public TaskLib { tag_info.tag_id_ = tag_id; tag_info.owner_ = task->blob_owner_; tag_info.internal_size_ = task->backend_size_; - - LPointer register_task = - stager_mdm_.AsyncRegisterStager(task->task_node_ + 1, - tag_id, - hshm::charbuf(task->url_->str())); - register_task->Wait(task); - LABSTOR_CLIENT->DelTask(register_task); + stager_mdm_.AsyncRegisterStager(task->task_node_ + 1, + tag_id, + hshm::charbuf(task->url_->str())); } else { if (tag_name.size()) { HILOG(kDebug, "Found existing tag: {}", tag_name.str()) @@ -294,6 +290,8 @@ class Server : public TaskLib { blob_mdm_.AsyncDestroyBlob(task->task_node_ + 1, task->tag_id_, blob_id).ptr_; blob_tasks.emplace_back(blob_task); } + stager_mdm_.AsyncUnregisterStager(task->task_node_ + 1, + task->tag_id_); task->phase_ = DestroyTagPhase::kWaitDestroyBlobs; return; } diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h index 17c62ad64..411a6065a 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h @@ -27,6 +27,32 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h index 5adb82c2c..f315a229d 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h @@ -35,6 +35,40 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kCustom: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kCustom: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h index 39b87c5e7..0c714be8a 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h @@ -107,6 +107,15 @@ struct CustomTask : public Task, TaskFlags { void SerializeEnd(u32 replica, Ar &ar) { } + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, CustomTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, CustomTask &dup_task) { + } + /** Begin replication */ void ReplicateStart(u32 count) { } diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h index b25c7d1ac..1b26dae2c 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h @@ -83,6 +83,88 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kCreateTaskState: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestroyTaskState: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRegisterTaskLib: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestroyTaskLib: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetOrCreateTaskStateId: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kGetTaskStateId: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kStopRuntime: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kSetWorkOrchProcPolicy: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kCreateTaskState: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestroyTaskState: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRegisterTaskLib: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestroyTaskLib: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetOrCreateTaskStateId: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kGetTaskStateId: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kStopRuntime: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kSetWorkOrchQueuePolicy: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kSetWorkOrchProcPolicy: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index dd28394e9..5c85a3aab 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -184,6 +184,17 @@ struct CreateTaskStateTask : public Task, TaskFlags { HSHM_DESTROY_AR(queue_info_); } + /** Duplicate message */ + template + void Dup(hipc::Allocator *alloc, TaskT &other) { + task_dup(other); + } + + /** Process duplicate message output */ + template + void DupEnd(u32 replica, TaskT &dup_task) { + } + /** Replication (does nothing) */ void ReplicateStart(u32 count) { } diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h index a4a9eaf9a..a3357d770 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h @@ -35,6 +35,40 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kPush: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kPush: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index fe9b31c3d..88f8de085 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -59,7 +59,7 @@ class Client : public TaskLibClient { LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); } - /** Call a custom method */ + /** Disperse a task among a domain of nodes */ HSHM_ALWAYS_INLINE void Disperse(Task *orig_task, TaskState *exec, @@ -68,7 +68,8 @@ class Client : public TaskLibClient { HILOG(kDebug, "Beginning dispersion for (task_node={}, task_state={}, method={})", orig_task->task_node_ + 1, orig_task->task_state_, orig_task->method_) BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); - auto xfer = exec->SaveStart(orig_task->method_, ar, orig_task); + std::vector xfer = + exec->SaveStart(orig_task->method_, ar, orig_task); // Create subtasks exec->ReplicateStart(orig_task->method_, domain_ids.size(), orig_task); @@ -79,6 +80,31 @@ class Client : public TaskLibClient { queue->Emplace(orig_task->prio_, orig_task->lane_hash_, push_task.shm_); } + /** Disperse a task among each lane of this node */ + HSHM_ALWAYS_INLINE + void DisperseLocal(Task *orig_task, TaskState *exec, + MultiQueue *orig_queue, LaneGroup *lane_group) { + // Duplicate task + HILOG(kDebug, "Beginning duplication for (task_node={}, task_state={}, method={})", + orig_task->task_node_ + 1, orig_task->task_state_, orig_task->method_); + std::vector> dups(lane_group->num_lanes_); + exec->Dup(orig_task->method_, orig_task, dups); + for (size_t i = 0; i < dups.size(); ++i) { + LPointer &task = dups[i]; + task->UnsetFireAndForget(); + task->lane_hash_ = i; + orig_queue->Emplace(task->prio_, task->lane_hash_, task.shm_); + } + + // Create duplicate task + exec->ReplicateStart(orig_task->method_, lane_group->num_lanes_, orig_task); + LPointer dup_task = LABSTOR_CLIENT->NewTask( + orig_task->task_node_ + 1, id_, + orig_task, exec, orig_task->method_, dups); + MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + queue->Emplace(orig_task->prio_, orig_task->lane_hash_, dup_task.shm_); + } + /** Spawn task to accept new connections */ // HSHM_ALWAYS_INLINE // AcceptTask* AsyncAcceptThread() { diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h index 0166190f4..8fb284193 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h @@ -16,6 +16,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { Push(reinterpret_cast(task), rctx); break; } + case Method::kDup: { + Dup(reinterpret_cast(task), rctx); + break; + } } } /** Delete a task */ @@ -33,6 +37,52 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kDup: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kPush: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDup: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kPush: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDup: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } } } /** Ensure there is space to store replicated outputs */ @@ -50,6 +100,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kDup: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } } } /** Determine success and handle failures */ @@ -67,6 +121,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kDup: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } } } /** Serialize a task when initially pushing into remote */ @@ -84,6 +142,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kDup: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -106,6 +168,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kDup: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } } return task_ptr; } @@ -124,6 +191,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kDup: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -142,6 +213,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kDup: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } } } /** Get the grouping of the task */ @@ -156,6 +231,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kPush: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kDup: { + return reinterpret_cast(task)->GetGroup(group); + } } return -1; } diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h index d0a029c43..001659a9a 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h @@ -4,6 +4,7 @@ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kPush = kLast + 0; + TASK_METHOD_T kDup = kLast + 1; }; #endif // LABSTOR_REMOTE_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml index fb48aa97f..fc6f10549 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml @@ -1 +1,2 @@ -kPush: 0 \ No newline at end of file +kPush: 0 +kDup: 1 \ No newline at end of file diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index 5c146dcdc..e3c8f2d87 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -111,7 +111,7 @@ struct PushTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPush; - task_flags_.SetBits(TASK_LOW_LATENCY | TASK_PREEMPTIVE | TASK_REMOTE_DEBUG_MARK); + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_PREEMPTIVE | TASK_REMOTE_DEBUG_MARK | TASK_FIRE_AND_FORGET); domain_id_ = domain_id; // Custom params @@ -129,6 +129,51 @@ struct PushTask : public Task, TaskFlags { } }; +/** + * A task to push a serialized task onto the remote queue + * */ +struct DupTask : public Task, TaskFlags { + IN Task *orig_task_; + IN TaskState *exec_; + IN u32 exec_method_; + IN std::vector> dups_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DupTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DupTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const TaskStateId &state_id, + Task *orig_task, + TaskState *exec, + u32 exec_method, + std::vector> &dups) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kDup; + task_flags_.SetBits(TASK_LOW_LATENCY | TASK_REMOTE_DEBUG_MARK | TASK_FIRE_AND_FORGET | TASK_COROUTINE); + domain_id_ = DomainId::GetLocal(); + + // Custom params + orig_task_ = orig_task; + exec_ = exec; + exec_method_ = exec_method; + dups_ = std::move(dups); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + /** * A task to push a serialized task onto the remote queue * */ diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 1b8075f54..e42087a0e 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -162,6 +162,19 @@ class Server : public TaskLib { ClientHandlePushReplicaEnd(task); } + /** Duplicate operations */ + void Dup(DupTask *task, RunContext &ctx) { + for (size_t i = 0; i < task->dups_.size(); ++i) { + LPointer &dup = task->dups_[i]; + dup->Wait(task); + task->exec_->DupEnd(dup->method_, i, task->orig_task_, dup.ptr_); + LABSTOR_CLIENT->DelTask(dup); + } + task->exec_->ReplicateEnd(task->exec_method_, task->orig_task_); + task->orig_task_->SetModuleComplete(); + task->SetModuleComplete(); + } + private: /** The RPC for processing a small message */ void RpcPushSmall(const tl::request &req, diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/tasks_required/small_message/include/small_message/small_message_lib_exec.h index 31d55bc73..3dbb98838 100644 --- a/tasks_required/small_message/include/small_message/small_message_lib_exec.h +++ b/tasks_required/small_message/include/small_message/small_message_lib_exec.h @@ -51,6 +51,56 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kMd: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kIo: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kMdPush: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kMd: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kIo: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kMdPush: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/small_message/include/small_message/small_message_tasks.h b/tasks_required/small_message/include/small_message/small_message_tasks.h index 2ee7004b4..1503e9f96 100644 --- a/tasks_required/small_message/include/small_message/small_message_tasks.h +++ b/tasks_required/small_message/include/small_message/small_message_tasks.h @@ -83,6 +83,15 @@ struct MdTask : public Task, TaskFlags { ret_.construct(alloc, 1); } + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, MdTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, MdTask &dup_task) { + } + /** Begin replication */ void ReplicateStart(u32 count) { ret_.resize(count); @@ -139,6 +148,15 @@ struct MdPushTask : public Task, TaskFlags { ret_.construct(alloc, 1); } + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, MdPushTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, MdPushTask &dup_task) { + } + /** Begin replication */ void ReplicateStart(u32 count) { ret_.resize(count); diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h index ca8ce7fc6..ed3c310cd 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h @@ -35,6 +35,40 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kSchedule: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kSchedule: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h index 0d95305e6..51b17b71d 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h @@ -35,6 +35,40 @@ void Del(u32 method, Task *task) override { } } } +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kSchedule: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kSchedule: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} /** Ensure there is space to store replicated outputs */ void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { From 1c2774177e788c65089bb08f0bb8d3747db4a567 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 2 Oct 2023 19:01:48 -0500 Subject: [PATCH 142/191] Change stager --- include/labstor/work_orchestrator/worker.h | 2 +- .../include/data_stager/data_stager.h | 8 --- .../data_stager/factory/abstract_stager.h | 60 +++++++++++++++++++ .../data_stager/factory}/binary_stager.h | 23 +++++++ .../data_stager/factory}/stager_factory.h | 2 +- tasks/data_stager/src/abstract_stager.h | 26 -------- tasks/data_stager/src/data_stager.cc | 2 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 2 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 6 +- .../hermes_bucket_mdm_tasks.h | 1 - test/unit/hermes/test_bucket.cc | 49 +++++++++++++++ 11 files changed, 140 insertions(+), 41 deletions(-) create mode 100644 tasks/data_stager/include/data_stager/factory/abstract_stager.h rename tasks/data_stager/{src => include/data_stager/factory}/binary_stager.h (81%) rename tasks/data_stager/{src => include/data_stager/factory}/stager_factory.h (95%) delete mode 100644 tasks/data_stager/src/abstract_stager.h diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 4f60cfd24..02587af30 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -267,7 +267,7 @@ class Worker { bool CheckTaskGroup(Task *task, TaskState *exec, u32 lane_id, TaskNode node, const bool &is_remote) { - if (is_remote || task->IsStarted()) { + if (is_remote || task->IsStarted() || task->IsLaneAll()) { return true; } int ret = exec->GetGroup(task->method_, task, group_); diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h index 64ce90f5c..1965c90a3 100644 --- a/tasks/data_stager/include/data_stager/data_stager.h +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -133,14 +133,6 @@ class Client : public TaskLibClient { task.ptr_->Wait(); } LABSTOR_TASK_NODE_PUSH_ROOT(StageOut); - - /** URL parser */ - static hshm::charbuf ParseUrl(const hshm::string &url) { - hshm::charbuf ret; - ret.resize(url.size() + 1); - memcpy(ret.data(), url.data(), url.size() + 1); - return ret; - } }; } // namespace labstor diff --git a/tasks/data_stager/include/data_stager/factory/abstract_stager.h b/tasks/data_stager/include/data_stager/factory/abstract_stager.h new file mode 100644 index 000000000..a2a9ef818 --- /dev/null +++ b/tasks/data_stager/include/data_stager/factory/abstract_stager.h @@ -0,0 +1,60 @@ +// +// Created by lukemartinlogan on 9/30/23. +// + +#ifndef HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ +#define HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ + +#include "../data_stager.h" + +namespace hermes::data_stager { + +class AbstractStager { + public: + std::string url_; + + AbstractStager() = default; + ~AbstractStager() = default; + + virtual void RegisterStager(RegisterStagerTask *task, RunContext &rctx) = 0; + virtual void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) = 0; + virtual void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) = 0; + + /** Parse url */ + static void GetUrlProtocolAndAction(const std::string &url, + std::string &protocol, + std::string &action, + std::vector &tokens) { + // Determine the protocol + action + std::string token; + size_t found = url.find("://"); + if (found != std::string::npos) { + protocol = url.substr(0, found); + action = url.substr(found + 3); + } + + // Divide action into tokens + std::stringstream ss(action); + while (std::getline(ss, token, ':')) { + tokens.push_back(token); + } + } + + /** Bucket name from url */ + static hshm::charbuf GetBucketNameFromUrl(const hshm::string &url) { + // Parse url + std::string protocol, action; + std::vector tokens; + GetUrlProtocolAndAction(url, protocol, action, tokens); + // file://[path]:[page_size] + if (protocol == "file") { + std::string path = tokens[0]; + size_t page_size = std::stoul(tokens[1]); + return hshm::charbuf(path); + } + } +}; + +} // namespace hermes + +#endif // HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ diff --git a/tasks/data_stager/src/binary_stager.h b/tasks/data_stager/include/data_stager/factory/binary_stager.h similarity index 81% rename from tasks/data_stager/src/binary_stager.h rename to tasks/data_stager/include/data_stager/factory/binary_stager.h index 4dc45f793..e80f1d7fd 100644 --- a/tasks/data_stager/src/binary_stager.h +++ b/tasks/data_stager/include/data_stager/factory/binary_stager.h @@ -13,6 +13,7 @@ class BinaryFileStager : public AbstractStager { public: int fd_; size_t page_size_; + std::string path_; public: /** Default constructor */ @@ -23,8 +24,30 @@ class BinaryFileStager : public AbstractStager { HERMES_POSIX_API->close(fd_); } + /** Build file url */ + static hshm::charbuf BuildFileUrl(const std::string &path, size_t page_size) { + std::stringstream ss; + ss << "file://" << path << ":" << page_size; + return hshm::charbuf(ss.str()); + } + + /** Parse file url */ + static void ParseFileUrl(const std::string &url, std::string &path, size_t &page_size) { + // Parse url + std::string protocol, action; + std::vector tokens; + GetUrlProtocolAndAction(url, protocol, action, tokens); + // file://[path]:[page_size] + if (protocol == "file") { + path = tokens[0]; + page_size = std::stoul(tokens[1]); + } + } + /** Create the data stager payload */ void RegisterStager(RegisterStagerTask *task, RunContext &rctx) override { + std::string path; + ParseFileUrl(task->url_->str(), path_, page_size_); fd_ = HERMES_POSIX_API->open(url_.c_str(), O_RDWR); if (fd_ < 0) { HELOG(kError, "Failed to open file {}", url_); diff --git a/tasks/data_stager/src/stager_factory.h b/tasks/data_stager/include/data_stager/factory/stager_factory.h similarity index 95% rename from tasks/data_stager/src/stager_factory.h rename to tasks/data_stager/include/data_stager/factory/stager_factory.h index 4bc015259..341b55964 100644 --- a/tasks/data_stager/src/stager_factory.h +++ b/tasks/data_stager/include/data_stager/factory/stager_factory.h @@ -5,7 +5,7 @@ #ifndef HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ #define HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ -#include "data_stager/data_stager.h" +#include "../data_stager.h" #include "abstract_stager.h" #include "binary_stager.h" diff --git a/tasks/data_stager/src/abstract_stager.h b/tasks/data_stager/src/abstract_stager.h deleted file mode 100644 index cf178fee2..000000000 --- a/tasks/data_stager/src/abstract_stager.h +++ /dev/null @@ -1,26 +0,0 @@ -// -// Created by lukemartinlogan on 9/30/23. -// - -#ifndef HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ -#define HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ - -#include "data_stager/data_stager.h" - -namespace hermes::data_stager { - -class AbstractStager { - public: - std::string url_; - - AbstractStager() = default; - ~AbstractStager() = default; - - virtual void RegisterStager(RegisterStagerTask *task, RunContext &rctx) = 0; - virtual void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) = 0; - virtual void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) = 0; -}; - -} // namespace hermes - -#endif // HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ diff --git a/tasks/data_stager/src/data_stager.cc b/tasks/data_stager/src/data_stager.cc index c821eefa7..6a816c161 100644 --- a/tasks/data_stager/src/data_stager.cc +++ b/tasks/data_stager/src/data_stager.cc @@ -8,7 +8,7 @@ #include "hermes_adapters/mapper/abstract_mapper.h" #include "hermes_adapters/posix/posix_api.h" #include "hermes_blob_mdm/hermes_blob_mdm.h" -#include "stager_factory.h" +#include "data_stager/factory/stager_factory.h" namespace hermes::data_stager { diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index a00ccba4f..7ff54e7a4 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -1192,7 +1192,7 @@ struct FlushDataTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - return TASK_GROUP_ANY; + return TASK_UNORDERED; } }; diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index ad71d77df..0d353f79d 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -169,11 +169,13 @@ class Server : public TaskLib { blob_info.blob_size_ = 0; blob_info.max_blob_size_ = 0; blob_info.score_ = task->score_; - blob_info.mod_count_ = 1; + blob_info.mod_count_ = 0; blob_info.access_freq_ = 0; - blob_info.last_flush_ = 1; + blob_info.last_flush_ = 0; blob_info.UpdateWriteStats(); if (task->flags_.Any(HERMES_IS_FILE)) { + blob_info.mod_count_ = 1; + blob_info.last_flush_ = 1; LPointer stage_task = stager_mdm_.AsyncStageIn(task->task_node_ + 1, task->tag_id_, diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 6a372e915..86df93a7f 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -347,7 +347,6 @@ struct AppendBlobTask : public Task, TaskFlags { /** A task to get or create a tag */ struct GetOrCreateTagTask : public Task, TaskFlags { IN hipc::ShmArchive tag_name_; - IN hipc::ShmArchive url_; IN bool blob_owner_; IN hipc::ShmArchive> traits_; IN size_t backend_size_; diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 21fb128e9..d4f118eb5 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -362,4 +362,53 @@ TEST_CASE("TestHermesBucketAppend1n") { } } MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_CASE("TestDataStager") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + if (rank == 0) { + std::vector data(KILOBYTES(256)); + FILE *file = fopen("/tmp/test.txt", "w"); + fwrite(data.data(), sizeof(char), data.size(), file); + fclose(file); + } + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 0) { + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + std::stringstream ss; + ss << "file::/tmp/test.txt" << ":"; + ss << KILOBYTES(4); + hermes::Bucket bkt(ss.str(), KILOBYTES(256), HERMES_IS_FILE); + + // Put a few blobs in the bucket + size_t page_size = KILOBYTES(4); + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + bkt.Append(blob, page_size, ctx); + hermes::Blob blob2; + bkt.Get(std::to_string(i), blob2, ctx); + REQUIRE(blob.size() == blob2.size()); + REQUIRE(blob == blob2); + } + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "ContainsBlob Iteration: {}", i); + REQUIRE(bkt.ContainsBlob(std::to_string(i))); + } + } + MPI_Barrier(MPI_COMM_WORLD); } \ No newline at end of file From 8d7ff6090f05ac0d6bab05e4ed24e16be85ca87e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 2 Oct 2023 19:07:35 -0500 Subject: [PATCH 143/191] Use -DCMAKE_BUILD_TYPE=Release --- ci/hermes/packages/hermes/package.py | 2 ++ ci/hermes/packages/hermes_shm/package.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ci/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py index 184922e1b..607150dd9 100644 --- a/ci/hermes/packages/hermes/package.py +++ b/ci/hermes/packages/hermes/package.py @@ -30,6 +30,8 @@ def cmake_args(self): args = ['-DCMAKE_INSTALL_PREFIX={}'.format(self.prefix)] if '+debug' in self.spec: args.append('-DCMAKE_BUILD_TYPE=Debug') + else: + args.append('-DCMAKE_BUILD_TYPE=Release') return args def set_include(self, env, path): diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index 76343ff00..9ad520eda 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -19,6 +19,8 @@ def cmake_args(self): args = [] if '+debug' in self.spec: args.append('-DCMAKE_BUILD_TYPE=Debug') + else: + args.append('-DCMAKE_BUILD_TYPE=Release') return args def set_include(self, env, path): From 6fa97b73b33bd0fa5d2582c4f535007aec392e24 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 3 Oct 2023 03:03:52 -0500 Subject: [PATCH 144/191] GetContainedBlobIds works again --- tasks/hermes/include/hermes/bucket.h | 3 +- .../hermes_bucket_mdm/hermes_bucket_mdm.h | 20 +++++++ .../hermes_bucket_mdm_lib_exec.h | 36 ++++++++++++ .../hermes_bucket_mdm_methods.h | 1 + .../hermes_bucket_mdm_methods.yaml | 3 +- .../hermes_bucket_mdm_tasks.h | 58 +++++++++++++++++++ .../src/hermes_bucket_mdm.cc | 17 +++++- test/unit/hermes/test_bucket.cc | 27 +++++++++ 8 files changed, 161 insertions(+), 4 deletions(-) diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index d5a96032c..0e7ebee42 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -599,8 +599,7 @@ class Bucket { * Get the set of blob IDs contained in the bucket * */ std::vector GetContainedBlobIds() { - // TODO(llogan) - return {}; + return bkt_mdm_->GetContainedBlobIdsRoot(id_); } }; diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index 479085856..bacf69b1f 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -292,6 +292,26 @@ class Client : public TaskLibClient { return size; } LABSTOR_TASK_NODE_PUSH_ROOT(GetSize); + + /** Get contained blob ids */ + void AsyncGetContainedBlobIdsConstruct(GetContainedBlobIdsTask *task, + const TaskNode &task_node, + const TagId &tag_id) { + u32 hash = tag_id.unique_; + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, + tag_id); + } + std::vector GetContainedBlobIdsRoot(const TagId &tag_id) { + LPointer> push_task = + AsyncGetContainedBlobIdsRoot(tag_id); + push_task->Wait(); + GetContainedBlobIdsTask *task = push_task->get(); + std::vector blob_ids = task->blob_ids_->vec(); + LABSTOR_CLIENT->DelTask(push_task); + return blob_ids; + } + LABSTOR_TASK_NODE_PUSH_ROOT(GetContainedBlobIds); }; } // namespace labstor diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index 1c5127c07..5ca7567d3 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -64,6 +64,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { SetBlobMdm(reinterpret_cast(task), rctx); break; } + case Method::kGetContainedBlobIds: { + GetContainedBlobIds(reinterpret_cast(task), rctx); + break; + } } } /** Delete a task */ @@ -129,6 +133,10 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kGetContainedBlobIds: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } } } /** Ensure there is space to store replicated outputs */ @@ -194,6 +202,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kGetContainedBlobIds: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } } } /** Determine success and handle failures */ @@ -259,6 +271,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kGetContainedBlobIds: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } } } /** Serialize a task when initially pushing into remote */ @@ -324,6 +340,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kGetContainedBlobIds: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -406,6 +426,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kGetContainedBlobIds: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } } return task_ptr; } @@ -472,6 +497,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kGetContainedBlobIds: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -538,6 +567,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kGetContainedBlobIds: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } } } /** Get the grouping of the task */ @@ -588,6 +621,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kSetBlobMdm: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kGetContainedBlobIds: { + return reinterpret_cast(task)->GetGroup(group); + } } return -1; } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h index bed26dd43..a7650139e 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h @@ -16,6 +16,7 @@ struct Method : public TaskMethod { TASK_METHOD_T kAppendBlob = kLast + 13; TASK_METHOD_T kGetSize = kLast + 14; TASK_METHOD_T kSetBlobMdm = kLast + 15; + TASK_METHOD_T kGetContainedBlobIds = kLast + 16; }; #endif // LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml index d56feaa39..b200fcc0a 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml @@ -13,4 +13,5 @@ kUpdateSize: 11 kAppendBlobSchema: 12 kAppendBlob: 13 kGetSize: 14 -kSetBlobMdm: 15 \ No newline at end of file +kSetBlobMdm: 15 +kGetContainedBlobIds: 16 \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 11303272f..046d60c0e 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -844,6 +844,64 @@ struct GetSizeTask : public Task, TaskFlags { } }; +/** A task to destroy all blobs in the tag */ +struct GetContainedBlobIdsTask : public Task, TaskFlags { + IN TagId tag_id_; + OUT hipc::ShmArchive> blob_ids_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + GetContainedBlobIdsTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + GetContainedBlobIdsTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const TagId &tag_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = tag_id.hash_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kGetContainedBlobIds; + task_flags_.SetBits(TASK_LOW_LATENCY); + domain_id_ = domain_id; + + // Custom params + tag_id_ = tag_id; + HSHM_MAKE_AR0(blob_ids_, alloc) + } + + /** Destructor */ + ~GetContainedBlobIdsTask() { + HSHM_DESTROY_AR(blob_ids_) + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(tag_id_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(blob_ids_); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + labstor::LocalSerialize srl(group); + srl << tag_id_.unique_; + srl << tag_id_.node_id_; + return 0; + } +}; + } // namespace hermes::bucket_mdm #endif // LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 38fb5116c..27a1048ca 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -359,7 +359,22 @@ class Server : public TaskLib { task->SetModuleComplete(); } - + /** Get contained blob IDs */ + void GetContainedBlobIds(GetContainedBlobIdsTask *task, RunContext &rctx) { + TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; + auto it = tag_map.find(task->tag_id_); + if (it == tag_map.end()) { + task->SetModuleComplete(); + return; + } + TagInfo &tag = it->second; + hipc::vector &blobs = (*task->blob_ids_); + blobs.reserve(tag.blobs_.size()); + for (BlobId &blob_id : tag.blobs_) { + blobs.emplace_back(blob_id); + } + task->SetModuleComplete(); + } public: #include "hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h" diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 21fb128e9..af1a22aee 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -362,4 +362,31 @@ TEST_CASE("TestHermesBucketAppend1n") { } } MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_CASE("TestHermesGetContainedBlobIds") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("append_test" + std::to_string(rank)); + + // Put a few blobs in the bucket + for (int i = 0; i < 10; ++i) { + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + bkt.Put(std::to_string(i), blob, ctx); + } + + // Get contained blob ids + std::vector blob_ids; + blob_ids = bkt.GetContainedBlobIds(); + REQUIRE(blob_ids.size() == 10); + MPI_Barrier(MPI_COMM_WORLD); } \ No newline at end of file From 532196ef40270420b98d44f745ee8e90079faed3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 3 Oct 2023 18:24:39 -0500 Subject: [PATCH 145/191] Make it so bkt is hashed using bkt.hash --- .../hermes_bucket_mdm/hermes_bucket_mdm.h | 16 ++++++++-------- test/unit/hermes/test_bucket.cc | 5 +++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index bacf69b1f..eb189b5cb 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -170,7 +170,7 @@ class Client : public TaskLibClient { void AsyncGetTagNameConstruct(GetTagNameTask *task, const TaskNode &task_node, const TagId &tag_id) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); @@ -191,7 +191,7 @@ class Client : public TaskLibClient { const TaskNode &task_node, const TagId &tag_id, const hshm::charbuf &new_tag_name) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, new_tag_name); @@ -208,7 +208,7 @@ class Client : public TaskLibClient { void AsyncDestroyTagConstruct(DestroyTagTask *task, const TaskNode &task_node, const TagId &tag_id) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); @@ -226,7 +226,7 @@ class Client : public TaskLibClient { const TaskNode &task_node, const TagId &tag_id, const BlobId &blob_id) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, blob_id); @@ -243,7 +243,7 @@ class Client : public TaskLibClient { void AsyncTagRemoveBlobConstruct(TagRemoveBlobTask *task, const TaskNode &task_node, const TagId &tag_id, const BlobId &blob_id) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, blob_id); @@ -260,7 +260,7 @@ class Client : public TaskLibClient { void AsyncTagClearBlobsConstruct(TagClearBlobsTask *task, const TaskNode &task_node, const TagId &tag_id) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); @@ -277,7 +277,7 @@ class Client : public TaskLibClient { void AsyncGetSizeConstruct(GetSizeTask *task, const TaskNode &task_node, const TagId &tag_id) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); @@ -297,7 +297,7 @@ class Client : public TaskLibClient { void AsyncGetContainedBlobIdsConstruct(GetContainedBlobIdsTask *task, const TaskNode &task_node, const TagId &tag_id) { - u32 hash = tag_id.unique_; + u32 hash = tag_id.hash_; LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index af1a22aee..735bcca90 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -376,9 +376,10 @@ TEST_CASE("TestHermesGetContainedBlobIds") { // Create a bucket hermes::Context ctx; hermes::Bucket bkt("append_test" + std::to_string(rank)); + u32 num_blobs = 1024; // Put a few blobs in the bucket - for (int i = 0; i < 10; ++i) { + for (int i = 0; i < num_blobs; ++i) { hermes::Blob blob(KILOBYTES(4)); memset(blob.data(), i % 256, blob.size()); bkt.Put(std::to_string(i), blob, ctx); @@ -387,6 +388,6 @@ TEST_CASE("TestHermesGetContainedBlobIds") { // Get contained blob ids std::vector blob_ids; blob_ids = bkt.GetContainedBlobIds(); - REQUIRE(blob_ids.size() == 10); + REQUIRE(blob_ids.size() == num_blobs); MPI_Barrier(MPI_COMM_WORLD); } \ No newline at end of file From 11c9acdaf00730a0e44b94e40799471ee9916f69 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 3 Oct 2023 18:37:44 -0500 Subject: [PATCH 146/191] Create many buckets --- test/unit/hermes/test_bucket.cc | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 735bcca90..e00172b99 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -364,6 +364,37 @@ TEST_CASE("TestHermesBucketAppend1n") { MPI_Barrier(MPI_COMM_WORLD); } +TEST_CASE("TestHermesMultiGetBucket") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("append_test" + std::to_string(rank)); + u32 num_blobs = 1024; + + // Put a few blobs in the bucket + for (int i = 0; i < num_blobs; ++i) { + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + // Get a blob + hermes::Blob blob2; + bkt.Get(blob_id, blob2, ctx); + REQUIRE(blob.size() == blob2.size()); + REQUIRE(blob == blob2); + } + + // Get contained blob ids + MPI_Barrier(MPI_COMM_WORLD); +} + TEST_CASE("TestHermesGetContainedBlobIds") { int rank, nprocs; MPI_Barrier(MPI_COMM_WORLD); @@ -382,7 +413,8 @@ TEST_CASE("TestHermesGetContainedBlobIds") { for (int i = 0; i < num_blobs; ++i) { hermes::Blob blob(KILOBYTES(4)); memset(blob.data(), i % 256, blob.size()); - bkt.Put(std::to_string(i), blob, ctx); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); } // Get contained blob ids From 39f34d39a0f655a833dd93c7b74ff832e4d4a357 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 3 Oct 2023 19:15:37 -0500 Subject: [PATCH 147/191] Serialize blob_ids --- .../include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 046d60c0e..ac361f483 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -883,7 +883,7 @@ struct GetContainedBlobIdsTask : public Task, TaskFlags { template void SerializeStart(Ar &ar) { task_serialize(ar); - ar(tag_id_); + ar(tag_id_, blob_ids_); } /** (De)serialize message return */ From e91a2a5854375cbe876743b0e082cdbb5b40a2fd Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 3 Oct 2023 19:19:29 -0500 Subject: [PATCH 148/191] Barrier --- test/unit/hermes/test_bucket.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index e00172b99..e4a26fc67 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -416,6 +416,7 @@ TEST_CASE("TestHermesGetContainedBlobIds") { hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); } + MPI_Barrier(MPI_COMM_WORLD); // Get contained blob ids std::vector blob_ids; From ee2cfd150eb56ea0ede5d888861013c939655e13 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 4 Oct 2023 03:19:32 -0500 Subject: [PATCH 149/191] enable MPI adapter by default --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 56f245755..fed5cedaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" OFF) option(HERMES_ENABLE_POSIX_ADAPTER "Build the Hermes POSIX adapter." ON) option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." ON) -option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." OFF) +option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON) option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF) option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF) option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) From 025ecbe929e3543da734801a31aa8ac83a6117a7 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 4 Oct 2023 03:24:50 -0500 Subject: [PATCH 150/191] Fix compile --- tasks/hermes_adapters/mpiio/mpiio_fs_api.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/hermes_adapters/mpiio/mpiio_fs_api.h b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h index 55775c86f..6e98b7b41 100644 --- a/tasks/hermes_adapters/mpiio/mpiio_fs_api.h +++ b/tasks/hermes_adapters/mpiio/mpiio_fs_api.h @@ -178,8 +178,9 @@ class MpiioFs : public Filesystem { mdm->request_map.erase(iter); delete (hreq); return MPI_SUCCESS; - }*/ - return real_api->MPI_Wait(req, status); + } + return real_api->MPI_Wait(req, status);*/ + return 0; } int WaitAll(int count, MPI_Request *req, MPI_Status *status) { From cb45bc9416551ec2f60e35ee19c6939570c6b94f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 4 Oct 2023 03:26:26 -0500 Subject: [PATCH 151/191] Fix compile --- test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp b/test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp index 0e0402a6b..8c94d3cdb 100644 --- a/test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp +++ b/test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cpp @@ -18,7 +18,7 @@ #include "adapter_test_utils.h" #include "catch_config.h" #if HERMES_INTERCEPT == 1 -#include "filesystem/filesystem.h" +#include "hermes_adapters/filesystem/filesystem.h" #endif #include "adapter_test_utils.h" From d6690f76aaa5ad2ce268f5335a3d1f5530aead81 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 8 Oct 2023 01:34:09 -0500 Subject: [PATCH 152/191] Add latency tests for different time types --- benchmark/hermes_api_bench.cc | 35 ++++++++----------- benchmark/test_latency.cc | 29 +++++++++++++++ include/labstor/api/labstor_client.h | 2 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 8 +++++ .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 2 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 15 +++++--- .../include/remote_queue/remote_queue.h | 1 + test/unit/hermes/test_bucket.cc | 17 +++++---- 8 files changed, 76 insertions(+), 33 deletions(-) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index 018ac7c11..196b772c2 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -11,35 +11,30 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include -#include +#include #include #include "hermes/hermes.h" #include "hermes/bucket.h" #include "labstor/work_orchestrator/affinity.h" namespace hapi = hermes; -using Timer = hshm::HighResMonotonicTimer; +using hshm::MpiTimer; /** Gather times per-process */ -void GatherTimes(std::string test_name, size_t io_size, Timer &t) { - MPI_Barrier(MPI_COMM_WORLD); - int rank, nprocs; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - double time = t.GetSec(), max; - MPI_Reduce(&time, &max, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - if (rank == 0) { - double mbps = io_size / (max * 1000000); +void GatherTimes(std::string test_name, size_t io_size, MpiTimer &t) { + t.Collect(); + if (t.rank_ == 0) { + double max = t.GetUsec(); + double mbps = io_size / t.GetUsec(); HIPRINT("{}: Time: {} sec, MBps (or MOps): {}, Count: {}, Nprocs: {}\n", - test_name, max, mbps, io_size, nprocs); + test_name, max, mbps, io_size, t.nprocs_); } } /** Each process PUTS into the same bucket, but with different blob names */ void PutTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size) { - Timer t; + MpiTimer t(MPI_COMM_WORLD); hermes::Context ctx; hermes::Bucket bkt("hello", ctx); hermes::Blob blob(blob_size); @@ -62,7 +57,7 @@ void PutTest(int nprocs, int rank, * */ void GetTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size) { - Timer t; + MpiTimer t(MPI_COMM_WORLD); hermes::Context ctx; hermes::Bucket bkt("hello", ctx); t.Resume(); @@ -90,7 +85,7 @@ void PutGetTest(int nprocs, int rank, int repeat, void PartialPutTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size, size_t part_size) { - Timer t; + MpiTimer t(MPI_COMM_WORLD); hermes::Context ctx; hermes::Bucket bkt("hello", ctx); hermes::Blob blob(blob_size); @@ -115,7 +110,7 @@ void PartialPutTest(int nprocs, int rank, void PartialGetTest(int nprocs, int rank, int repeat, size_t blobs_per_rank, size_t blob_size, size_t part_size) { - Timer t; + MpiTimer t(MPI_COMM_WORLD); hermes::Context ctx; hermes::Bucket bkt("hello", ctx); t.Resume(); @@ -145,7 +140,7 @@ void PartialPutGetTest(int nprocs, int rank, int repeat, /** Each process creates a set of buckets */ void CreateBucketTest(int nprocs, int rank, size_t bkts_per_rank) { - Timer t; + MpiTimer t(MPI_COMM_WORLD); t.Resume(); hapi::Context ctx; std::unordered_map mdm_; @@ -169,7 +164,7 @@ void GetBucketTest(int nprocs, int rank, } // Get existing buckets - Timer t; + MpiTimer t(MPI_COMM_WORLD); t.Resume(); for (size_t i = 0; i < bkts_per_rank; ++i) { int bkt_name = rank * bkts_per_rank + i; @@ -183,7 +178,7 @@ void GetBucketTest(int nprocs, int rank, void DeleteBucketTest(int nprocs, int rank, size_t bkt_per_rank, size_t blobs_per_bucket) { - Timer t; + MpiTimer t(MPI_COMM_WORLD); hapi::Context ctx; // Create the buckets diff --git a/benchmark/test_latency.cc b/benchmark/test_latency.cc index 57a536861..80abe788e 100644 --- a/benchmark/test_latency.cc +++ b/benchmark/test_latency.cc @@ -292,6 +292,35 @@ TEST_CASE("TestTimespecLatency") { HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } +TEST_CASE("TestTimerLatency") { + size_t ops = (1 << 20); + hshm::Timer t, tmp; + + t.Resume(); + double usec; + for (size_t i = 0; i < ops; ++i) { + usec = tmp.GetUsecFromStart(); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps (usec={})", ops / t.GetUsec(), usec); +} + +TEST_CASE("TestTimepointLatency") { + size_t ops = (1 << 20); + hshm::Timer t; + hshm::Timepoint tmp; + + t.Resume(); + double usec; + for (size_t i = 0; i < ops; ++i) { + usec = tmp.GetUsecFromStart(); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps (usec={})", ops / t.GetUsec(), usec); +} + /** Time to process a request */ //TEST_CASE("TestHermesGetBlobIdLatency") { // HERMES->ClientInit(); diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index eba69bf27..069cfb241 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -325,7 +325,7 @@ constexpr inline void CALL_DUPLICATE(TaskT *orig_task, std::vector &dup : dups) { LPointer task = LABSTOR_CLIENT->NewEmptyTask(); - orig_task->Dup(LABSTOR_CLIENT->main_alloc_, *task.ptr_); + task.ptr_->Dup(LABSTOR_CLIENT->main_alloc_, *orig_task); dup.ptr_ = task.ptr_; dup.shm_ = task.shm_; } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index f332a5bce..00dd83c49 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -425,6 +425,14 @@ class Client : public TaskLibClient { LABSTOR_CLIENT->DelTask(push_task); } LABSTOR_TASK_NODE_PUSH_ROOT(DestroyBlob); + + /** Initialize automatic flushing */ + void AsyncFlushDataConstruct(FlushDataTask *task, + const TaskNode &task_node) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_); + } + LABSTOR_TASK_NODE_PUSH_ROOT(FlushData); }; } // namespace labstor diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 3a4d34556..b298c5f0a 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -1151,7 +1151,7 @@ struct FlushDataTask : public Task, TaskFlags { lane_hash_ = 0; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; - method_ = Method::kReorganizeBlob; + method_ = Method::kFlushData; task_flags_.SetBits(TASK_LANE_ALL | TASK_FIRE_AND_FORGET | TASK_LONG_RUNNING | TASK_COROUTINE); domain_id_ = DomainId::GetGlobal(); } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 0d353f79d..e07c35e0a 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -48,6 +48,7 @@ class Server : public TaskLib { Client blob_mdm_; bucket_mdm::Client bkt_mdm_; data_stager::Client stager_mdm_; + LPointer flush_task_; public: Server() = default; @@ -58,7 +59,7 @@ class Server : public TaskLib { // Initialize blob maps blob_id_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); blob_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); - // Initialize target tasks + // Initialize targets target_tasks_.reserve(HERMES_SERVER_CONF.devices_.size()); for (DeviceInfo &dev : HERMES_SERVER_CONF.devices_) { std::string dev_type; @@ -110,8 +111,11 @@ class Server : public TaskLib { * Set the Bucket MDM * */ void SetBucketMdm(SetBucketMdmTask *task, RunContext &rctx) { - bkt_mdm_.Init(task->bkt_mdm_); - stager_mdm_.Init(task->stager_mdm_); + if (bkt_mdm_.id_.IsNull()) { + bkt_mdm_.Init(task->bkt_mdm_); + stager_mdm_.Init(task->stager_mdm_); + flush_task_ = blob_mdm_.AsyncFlushData(task->task_node_ + 1); + } task->SetModuleComplete(); } @@ -121,6 +125,9 @@ class Server : public TaskLib { void FlushData(FlushDataTask *task, RunContext &rctx) { // Get the blob info data structure BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; + if (stager_mdm_.id_.IsNull()) { + return; + } for (auto &it : blob_map) { BlobInfo &blob_info = it.second; if (blob_info.last_flush_ > 0 && @@ -145,7 +152,7 @@ class Server : public TaskLib { TASK_DATA_OWNER | TASK_FIRE_AND_FORGET); } } - task->SetModuleComplete(); + // task->SetModuleComplete(); } /** diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index 88f8de085..c55acfbe8 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -93,6 +93,7 @@ class Client : public TaskLibClient { LPointer &task = dups[i]; task->UnsetFireAndForget(); task->lane_hash_ = i; + task->UnsetLaneAll(); orig_queue->Emplace(task->prio_, task->lane_hash_, task.shm_); } diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 03da9caf2..242d7f76d 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -371,8 +371,8 @@ TEST_CASE("TestHermesDataStager") { MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + std::vector data(KILOBYTES(256), 0); if (rank == 0) { - std::vector data(KILOBYTES(256)); FILE *file = fopen("/tmp/test.txt", "w"); fwrite(data.data(), sizeof(char), data.size(), file); fclose(file); @@ -383,13 +383,13 @@ TEST_CASE("TestHermesDataStager") { HERMES->ClientInit(); // Create a stageable bucket - size_t stage_size = KILOBYTES(256); + size_t file_size = KILOBYTES(256); size_t page_size = KILOBYTES(4); using hermes::data_stager::BinaryFileStager; hermes::Context ctx; ctx.flags_.SetBits(HERMES_IS_FILE); hshm::charbuf url = BinaryFileStager::BuildFileUrl("/tmp/test.txt", page_size); - hermes::Bucket bkt(url.str(), stage_size, HERMES_IS_FILE); + hermes::Bucket bkt(url.str(), file_size, HERMES_IS_FILE); // Put a few blobs in the bucket size_t count_per_proc = 16; @@ -398,13 +398,16 @@ TEST_CASE("TestHermesDataStager") { for (size_t i = off; i < proc_count; ++i) { HILOG(kInfo, "Iteration: {}", i); // Put a blob - hermes::Blob blob(page_size); + hermes::Blob blob(page_size / 2); memset(blob.data(), i % 256, blob.size()); - bkt.Put(std::to_string(i), blob, ctx); + bkt.PartialPut(std::to_string(i), blob, 0, ctx); hermes::Blob blob2; bkt.Get(std::to_string(i), blob2, ctx); - REQUIRE(blob2.size() == stage_size); - REQUIRE(blob == blob2); + REQUIRE(blob2.size() == page_size); + hermes::Blob full_blob(page_size); + memcpy(full_blob.data(), blob.data(), blob.size()); + memcpy(full_blob.data() + blob.size(), data.data(), page_size / 2); + REQUIRE(full_blob == blob2); } for (size_t i = off; i < proc_count; ++i) { HILOG(kInfo, "ContainsBlob Iteration: {}", i); From 3d2146e1de16f516c190266e6d4ab63f20e9fa89 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 9 Oct 2023 06:48:18 -0500 Subject: [PATCH 153/191] Staging in and out work again --- include/labstor/task_registry/task.h | 80 +++++++++++++------ include/labstor/work_orchestrator/scheduler.h | 1 + include/labstor/work_orchestrator/worker.h | 5 +- src/worker.cc | 28 +++++-- tasks/bdev/include/bdev/bdev_tasks.h | 3 +- .../include/data_stager/data_stager.h | 29 ++++--- .../data_stager/factory/binary_stager.h | 5 +- .../filesystem/filesystem_io_client.h | 9 --- .../hermes_adapters/mapper/abstract_mapper.h | 18 +++-- .../hermes_adapters/mapper/balanced_mapper.h | 1 - .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 3 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 5 +- .../src/hermes_bucket_mdm.cc | 8 +- .../labstor_admin/src/labstor_admin.cc | 5 +- .../remote_queue/src/remote_queue.cc | 1 - test/unit/hermes/test_bucket.cc | 21 ++--- 16 files changed, 134 insertions(+), 88 deletions(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 2ec37aa37..04b1a1e8a 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -43,20 +43,14 @@ class TaskLib; #define TASK_DISABLE_RUN BIT_OPT(u32, 13) /** This task owns the data in the task */ #define TASK_DATA_OWNER BIT_OPT(u32, 14) -/** This task is marked */ -#define TASK_MARKED BIT_OPT(u32, 15) /** This task uses co-routine wait */ -#define TASK_COROUTINE BIT_OPT(u32, 16) +#define TASK_COROUTINE BIT_OPT(u32, 15) /** This task uses argobot wait */ #define TASK_PREEMPTIVE BIT_OPT(u32, 17) -/** This task is apart of remote debugging */ -#define TASK_REMOTE_DEBUG_MARK BIT_OPT(u32, 18) -/** This task will match all groups */ -#define TASK_GROUP_ANY BIT_OPT(u32, 19) -/** This task can be scheduled on any lane */ -#define TASK_LANE_ANY BIT_OPT(u32, 20) /** This task should be scheduled on all lanes */ -#define TASK_LANE_ALL BIT_OPT(u32, 21) +#define TASK_LANE_ALL BIT_OPT(u32, 19) +/** This task is apart of remote debugging */ +#define TASK_REMOTE_DEBUG_MARK BIT_OPT(u32, 31) /** Used to define task methods */ #define TASK_METHOD_T static inline const u32 @@ -253,6 +247,8 @@ struct Task : public hipc::ShmContainer { u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ std::atomic delcnt_ = 0; /**< # of times deltask called */ + double period_ns_; /**< The period of the task */ + hshm::Timepoint start_; /**< The time the task started */ RunContext ctx_; /**==================================== @@ -344,21 +340,6 @@ struct Task : public hipc::ShmContainer { return task_flags_.Any(TASK_HAS_STARTED); } - /** Set this task as started */ - HSHM_ALWAYS_INLINE void SetMarked() { - task_flags_.SetBits(TASK_MARKED); - } - - /** Set this task as started */ - HSHM_ALWAYS_INLINE void UnsetMarked() { - task_flags_.UnsetBits(TASK_MARKED); - } - - /** Check if task is marked */ - HSHM_ALWAYS_INLINE bool IsMarked() { - return task_flags_.Any(TASK_MARKED); - } - /** Set this task as started */ HSHM_ALWAYS_INLINE void UnsetLongRunning() { task_flags_.UnsetBits(TASK_LONG_RUNNING); @@ -389,6 +370,49 @@ struct Task : public hipc::ShmContainer { task_flags_.UnsetBits(TASK_LANE_ALL); } + /** Set period in nanoseconds */ + HSHM_ALWAYS_INLINE void SetPeriodNs(double ns) { + period_ns_ = ns; + } + + /** Set period in microseconds */ + HSHM_ALWAYS_INLINE void SetPeriodUs(double us) { + period_ns_ = us * 1000; + } + + /** Set period in milliseconds */ + HSHM_ALWAYS_INLINE void SetPeriodMs(double ms) { + period_ns_ = ms * 1000000; + } + + /** Set period in seconds */ + HSHM_ALWAYS_INLINE void SetPeriodSec(double sec) { + period_ns_ = sec * 1000000000; + } + + /** Set period in minutes */ + HSHM_ALWAYS_INLINE void SetPeriodMin(double min) { + period_ns_ = min * 60000000000; + } + + /** Determine if time has elapsed */ + HSHM_ALWAYS_INLINE bool ShouldRun(hshm::Timepoint &cur_time) { + if (!IsStarted()) { + start_ = cur_time; + return true; + } + if (IsLongRunning()) { + return start_.GetNsecFromStart(cur_time) >= period_ns_; + } else { + return true; + } + } + + /** Mark this task as having been run */ + HSHM_ALWAYS_INLINE void DidRun(hshm::Timepoint &cur_time) { + start_ = cur_time; + } + /** Yield the task */ template HSHM_ALWAYS_INLINE @@ -507,7 +531,9 @@ struct Task : public hipc::ShmContainer { * ===================================*/ template void task_serialize(Ar &ar) { - ar(task_state_, task_node_, domain_id_, lane_hash_, prio_, method_, task_flags_); + // NOTE(llogan): don't serialize start_ because of clock drift + ar(task_state_, task_node_, domain_id_, lane_hash_, prio_, method_, + task_flags_, period_ns_); } template @@ -519,6 +545,8 @@ struct Task : public hipc::ShmContainer { prio_ = other.prio_; method_ = other.method_; task_flags_ = other.task_flags_; + period_ns_ = other.period_ns_; + start_ = other.start_; } /**==================================== diff --git a/include/labstor/work_orchestrator/scheduler.h b/include/labstor/work_orchestrator/scheduler.h index e8a7ad8e4..4b9bad446 100644 --- a/include/labstor/work_orchestrator/scheduler.h +++ b/include/labstor/work_orchestrator/scheduler.h @@ -35,6 +35,7 @@ struct ScheduleTask : public Task, TaskFlags { task_state_ = state_id; method_ = SchedulerMethod::kSchedule; task_flags_.SetBits(TASK_LONG_RUNNING | TASK_REMOTE_DEBUG_MARK); + SetPeriodSec(1); domain_id_ = domain_id; // Custom params diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 02587af30..4c40c9530 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -28,6 +28,9 @@ struct WorkEntry { Lane *lane_; LaneGroup *group_; MultiQueue *queue_; + hshm::Timepoint last_monitor_; + hshm::Timepoint cur_time_; + double sample_epoch_; /** Default constructor */ HSHM_ALWAYS_INLINE @@ -364,7 +367,7 @@ class Worker { } void PollGrouped(WorkEntry &entry); - static void RunBlocking(bctx::transfer_t t); + static void RunCoroutine(bctx::transfer_t t); static void RunPreemptive(void *data); }; diff --git a/src/worker.cc b/src/worker.cc index a9553b089..2b021eec3 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -36,6 +36,7 @@ void Worker::Run() { continue; } } + work_entry.cur_time_.Now(); PollGrouped(work_entry); } } @@ -69,7 +70,9 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } // Attempt to run the task if it's ready and runnable bool is_remote = task->domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); - if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote)) { + if (!task->IsRunDisabled() && + CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote) && + task->ShouldRun(work_entry.cur_time_)) { // TODO(llogan): Make a remote debug macro #ifdef REMOTE_DEBUG if (task->task_state_ != LABSTOR_QM_CLIENT->admin_task_state_ && @@ -102,11 +105,16 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } rctx.jmp_.fctx = bctx::make_fcontext( (char*)rctx.stack_ptr_ + rctx.stack_size_, - rctx.stack_size_, &RunBlocking); + rctx.stack_size_, &RunCoroutine); task->SetStarted(); } rctx.jmp_ = bctx::jump_fcontext(rctx.jmp_.fctx, task); - HILOG(kDebug, "Jumping into function") + if (!task->IsStarted()) { + rctx.jmp_.fctx = bctx::make_fcontext( + (char*)rctx.stack_ptr_ + rctx.stack_size_, + rctx.stack_size_, &RunCoroutine); + task->SetStarted(); + } } else if (task->IsPreemptive()) { task->DisableRun(); entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunPreemptive, task); @@ -114,6 +122,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->SetStarted(); exec->Run(task->method_, task, rctx); } + task->DidRun(work_entry.cur_time_); } // Cleanup on task completion if (task->IsModuleComplete()) { @@ -134,12 +143,13 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } } -void Worker::RunBlocking(bctx::transfer_t t) { +void Worker::RunCoroutine(bctx::transfer_t t) { Task *task = reinterpret_cast(t.data); RunContext &rctx = task->ctx_; TaskState *&exec = rctx.exec_; rctx.jmp_ = t; exec->Run(task->method_, task, rctx); + task->UnsetStarted(); task->Yield(); } @@ -147,7 +157,15 @@ void Worker::RunPreemptive(void *data) { Task *task = reinterpret_cast(data); TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); RunContext rctx(0); - exec->Run(task->method_, task, rctx); + do { + hshm::Timepoint now; + now.Now(); + if (task->ShouldRun(now)) { + exec->Run(task->method_, task, rctx); + task->DidRun(now); + } + task->Yield(); + } while(!task->IsModuleComplete()); } } // namespace labstor \ No newline at end of file diff --git a/tasks/bdev/include/bdev/bdev_tasks.h b/tasks/bdev/include/bdev/bdev_tasks.h index fc9965732..22548f180 100644 --- a/tasks/bdev/include/bdev/bdev_tasks.h +++ b/tasks/bdev/include/bdev/bdev_tasks.h @@ -238,7 +238,6 @@ struct ReadTask : public Task, TaskFlags { /** A task to monitor bdev statistics */ struct MonitorTask : public Task, TaskFlags { - IN size_t freq_ms_; /**< Frequency in ms */ OUT size_t rem_cap_; /**< Remaining capacity of the target */ /** SHM default constructor */ @@ -260,10 +259,10 @@ struct MonitorTask : public Task, TaskFlags { task_state_ = state_id; method_ = Method::kMonitor; task_flags_.SetBits(TASK_LONG_RUNNING | TASK_REMOTE_DEBUG_MARK); + SetPeriodMs(freq_ms); domain_id_ = domain_id; // Custom - freq_ms_ = freq_ms; rem_cap_ = rem_cap; } diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h index 68057a818..7c0094d7d 100644 --- a/tasks/data_stager/include/data_stager/data_stager.h +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -135,37 +135,40 @@ class Client : public TaskLibClient { LABSTOR_TASK_NODE_PUSH_ROOT(StageOut); /** Parse url */ - static void GetUrlProtocolAndAction(const std::string &url, - std::string &protocol, - std::string &action, - std::vector &tokens) { + static inline + bool GetUrlProtocolAndAction(const std::string &url, + std::string &protocol, + std::string &action, + std::vector &tokens) { // Determine the protocol + action std::string token; size_t found = url.find("://"); - if (found != std::string::npos) { - protocol = url.substr(0, found); - action = url.substr(found + 3); + if (found == std::string::npos) { + return false; } + protocol = url.substr(0, found); + action = url.substr(found + 3); // Divide action into tokens std::stringstream ss(action); while (std::getline(ss, token, ':')) { tokens.push_back(token); } + return true; } /** Bucket name from url */ - static hshm::charbuf GetTagNameFromUrl(const hshm::string &url) { + static inline hshm::charbuf GetTagNameFromUrl(const hshm::string &url) { // Parse url std::string protocol, action; std::vector tokens; - GetUrlProtocolAndAction(url.str(), protocol, action, tokens); - if (protocol == "file") { - // file://[path]:[page_size] - std::string path = tokens[0]; + bool ret = GetUrlProtocolAndAction(url.str(), protocol, action, tokens); + if (ret) { + std::string &path = tokens[0]; return hshm::charbuf(path); + } else { + return url; } - return url; } }; diff --git a/tasks/data_stager/include/data_stager/factory/binary_stager.h b/tasks/data_stager/include/data_stager/factory/binary_stager.h index 2b1017692..9287cbce2 100644 --- a/tasks/data_stager/include/data_stager/factory/binary_stager.h +++ b/tasks/data_stager/include/data_stager/factory/binary_stager.h @@ -57,7 +57,7 @@ class BinaryFileStager : public AbstractStager { /** Stage data in from remote source */ void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) override { adapter::BlobPlacement plcmnt; - plcmnt.DecodeBlobName(*task->blob_name_); + plcmnt.DecodeBlobName(*task->blob_name_, page_size_); HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", page_size_, url_, plcmnt.bucket_off_); LPointer blob = LABSTOR_CLIENT->AllocateBuffer(page_size_); @@ -68,6 +68,7 @@ class BinaryFileStager : public AbstractStager { if (real_size < 0) { HELOG(kError, "Failed to stage in {} bytes from {}", page_size_, url_); + return; } memcpy(blob.ptr_ + plcmnt.blob_off_, blob.ptr_, real_size); HILOG(kDebug, "Staged {} bytes from the backend file {}", @@ -87,7 +88,7 @@ class BinaryFileStager : public AbstractStager { /** Stage data out to remote source */ void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) override { adapter::BlobPlacement plcmnt; - plcmnt.DecodeBlobName(*task->blob_name_); + plcmnt.DecodeBlobName(*task->blob_name_, page_size_); HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", page_size_, url_, plcmnt.bucket_off_); char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); diff --git a/tasks/hermes_adapters/filesystem/filesystem_io_client.h b/tasks/hermes_adapters/filesystem/filesystem_io_client.h index 96a4a92ef..dab208fcd 100644 --- a/tasks/hermes_adapters/filesystem/filesystem_io_client.h +++ b/tasks/hermes_adapters/filesystem/filesystem_io_client.h @@ -297,15 +297,6 @@ struct FilesystemIoClientState { * */ class FilesystemIoClient { public: - /** Decode I/O client context from the original blob name */ - FsIoOptions DecodeBlobName(const std::string &blob_name) { - FsIoOptions decode_opts; - BlobPlacement p; - p.DecodeBlobName(blob_name); - decode_opts.backend_off_ = p.page_ * p.page_size_; - return decode_opts; - } - /** virtual destructor */ virtual ~FilesystemIoClient() = default; diff --git a/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h b/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h index dd0858099..8ca20026f 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h +++ b/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h @@ -30,28 +30,32 @@ enum class MapperType { */ struct BlobPlacement { size_t page_; /**< The index in the array placements */ - size_t page_size_; /**< The size of a page */ size_t bucket_off_; /**< Offset from file start (for FS) */ size_t blob_off_; /**< Offset from BLOB start */ size_t blob_size_; /**< Size after offset to read */ - int time_; /**< The order of the blob in a list of blobs */ + + /** create a BLOB name from index. */ + static hshm::charbuf CreateBlobName(size_t page) { + hshm::charbuf buf(sizeof(page)); + labstor::LocalSerialize srl(buf); + srl << page; + return buf; + } /** create a BLOB name from index. */ hshm::charbuf CreateBlobName() const { - hshm::charbuf buf(sizeof(page_) + sizeof(blob_off_)); + hshm::charbuf buf(sizeof(page_)); labstor::LocalSerialize srl(buf); srl << page_; - srl << page_size_; return buf; } /** decode \a blob_name BLOB name to index. */ template - void DecodeBlobName(const StringT &blob_name) { + void DecodeBlobName(const StringT &blob_name, size_t page_size) { labstor::LocalDeserialize srl(blob_name); srl >> page_; - srl >> page_size_; - bucket_off_ = page_ * page_size_; + bucket_off_ = page_ * page_size; blob_off_ = 0; } }; diff --git a/tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h b/tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h index 331901ba7..fba1c849a 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h +++ b/tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h @@ -35,7 +35,6 @@ class BalancedMapper : public AbstractMapper { BlobPlacement p; p.bucket_off_ = off + size_mapped; p.page_ = p.bucket_off_ / kPageSize; - p.page_size_ = page_size; p.blob_off_ = p.bucket_off_ % kPageSize; auto left_size_page = kPageSize - p.blob_off_; p.blob_size_ = left_size_page < size - size_mapped ? left_size_page diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index b298c5f0a..12925e2b1 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -281,8 +281,6 @@ struct PutBlobTask : public Task, TaskFlags if (ctx.flags_.Any(HERMES_IS_FILE)) { flags_.SetBits(HERMES_IS_FILE); } - HILOG(kDebug, "Construct PUT task for {}, while getting BlobId is {}", - blob_name.str(), flags_.Any(HERMES_GET_BLOB_ID)); } /** Destructor */ @@ -1153,6 +1151,7 @@ struct FlushDataTask : public Task, TaskFlags { task_state_ = state_id; method_ = Method::kFlushData; task_flags_.SetBits(TASK_LANE_ALL | TASK_FIRE_AND_FORGET | TASK_LONG_RUNNING | TASK_COROUTINE); + SetPeriodSec(10); // TODO(llogan): don't hardcode this domain_id_ = DomainId::GetGlobal(); } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index e07c35e0a..887317518 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -125,13 +125,12 @@ class Server : public TaskLib { void FlushData(FlushDataTask *task, RunContext &rctx) { // Get the blob info data structure BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; - if (stager_mdm_.id_.IsNull()) { - return; - } for (auto &it : blob_map) { BlobInfo &blob_info = it.second; if (blob_info.last_flush_ > 0 && blob_info.mod_count_ > blob_info.last_flush_) { + HILOG(kDebug, "Flushing blob {} (mod_count={}, last_flush={})", + blob_info.blob_id_, blob_info.mod_count_, blob_info.last_flush_); blob_info.last_flush_ = 1; blob_info.mod_count_ = 0; blob_info.access_freq_ = 0; diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index d413cacdf..1faf3e347 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -218,9 +218,11 @@ class Server : public TaskLib { tag_info.tag_id_ = tag_id; tag_info.owner_ = task->blob_owner_; tag_info.internal_size_ = task->backend_size_; - stager_mdm_.AsyncRegisterStager(task->task_node_ + 1, - tag_id, - hshm::charbuf(task->tag_name_->str())); + if (task->flags_.Any(HERMES_IS_FILE)) { + stager_mdm_.AsyncRegisterStager(task->task_node_ + 1, + tag_id, + hshm::charbuf(task->tag_name_->str())); + } } else { if (tag_name.size()) { HILOG(kDebug, "Found existing tag: {}", tag_name.str()) diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index 760a77eb4..145a8b8c3 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -80,10 +80,7 @@ class Server : public TaskLib { state_name.c_str(), task->id_, task); - if (!ret) { - task->SetModuleComplete(); - return; - } + task->SetModuleComplete(); HILOG(kInfo, "(node {}) Allocated task state {} with id {}", LABSTOR_CLIENT->node_id_, state_name, task->task_state_); } diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index e42087a0e..5db2458f1 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -262,7 +262,6 @@ class Server : public TaskLib { MultiQueue *queue = LABSTOR_CLIENT->GetQueue(QueueId(state_id)); orig_task->UnsetFireAndForget(); orig_task->UnsetStarted(); - orig_task->UnsetMarked(); orig_task->UnsetDataOwner(); orig_task->UnsetLongRunning(); queue->Emplace(orig_task->prio_, orig_task->lane_hash_, p); diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 242d7f76d..6c87caa56 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -371,7 +371,13 @@ TEST_CASE("TestHermesDataStager") { MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - std::vector data(KILOBYTES(256), 0); + // create dataset + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + size_t page_size = KILOBYTES(4); + size_t file_size = nprocs * page_size * 16; + std::vector data(file_size, 0); if (rank == 0) { FILE *file = fopen("/tmp/test.txt", "w"); fwrite(data.data(), sizeof(char), data.size(), file); @@ -383,8 +389,6 @@ TEST_CASE("TestHermesDataStager") { HERMES->ClientInit(); // Create a stageable bucket - size_t file_size = KILOBYTES(256); - size_t page_size = KILOBYTES(4); using hermes::data_stager::BinaryFileStager; hermes::Context ctx; ctx.flags_.SetBits(HERMES_IS_FILE); @@ -392,17 +396,15 @@ TEST_CASE("TestHermesDataStager") { hermes::Bucket bkt(url.str(), file_size, HERMES_IS_FILE); // Put a few blobs in the bucket - size_t count_per_proc = 16; - size_t off = rank * count_per_proc; - size_t proc_count = off + count_per_proc; for (size_t i = off; i < proc_count; ++i) { HILOG(kInfo, "Iteration: {}", i); // Put a blob hermes::Blob blob(page_size / 2); memset(blob.data(), i % 256, blob.size()); - bkt.PartialPut(std::to_string(i), blob, 0, ctx); + hshm::charbuf blob_name = hermes::adapter::BlobPlacement::CreateBlobName(i); + bkt.PartialPut(blob_name.str(), blob, 0, ctx); hermes::Blob blob2; - bkt.Get(std::to_string(i), blob2, ctx); + bkt.Get(blob_name.str(), blob2, ctx); REQUIRE(blob2.size() == page_size); hermes::Blob full_blob(page_size); memcpy(full_blob.data(), blob.data(), blob.size()); @@ -410,8 +412,9 @@ TEST_CASE("TestHermesDataStager") { REQUIRE(full_blob == blob2); } for (size_t i = off; i < proc_count; ++i) { + hshm::charbuf blob_name = hermes::adapter::BlobPlacement::CreateBlobName(i); HILOG(kInfo, "ContainsBlob Iteration: {}", i); - REQUIRE(bkt.ContainsBlob(std::to_string(i))); + REQUIRE(bkt.ContainsBlob(blob_name.str())); } MPI_Barrier(MPI_COMM_WORLD); From 712b034767e278354b7c35b7ff7544a68c494891 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 9 Oct 2023 07:41:41 -0500 Subject: [PATCH 154/191] Untested long-running remote tasks --- .../queue_manager/queue_manager_runtime.h | 6 +++--- include/labstor/task_registry/task.h | 7 ++++++- include/labstor/task_registry/task_registry.h | 6 +++--- include/labstor/work_orchestrator/worker.h | 20 +++++++++---------- src/worker.cc | 7 +++---- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 2 +- .../remote_queue/src/remote_queue.cc | 7 ++++++- .../src/worch_queue_round_robin.cc | 4 ++-- 8 files changed, 34 insertions(+), 25 deletions(-) diff --git a/include/labstor/queue_manager/queue_manager_runtime.h b/include/labstor/queue_manager/queue_manager_runtime.h index a489e6d9f..5b62031e7 100644 --- a/include/labstor/queue_manager/queue_manager_runtime.h +++ b/include/labstor/queue_manager/queue_manager_runtime.h @@ -60,14 +60,14 @@ class QueueManagerRuntime : public QueueManager { void CreateQueue(const QueueId &id, const std::vector &queue_info) { MultiQueue *queue = GetQueue(id); if (id.IsNull()) { - HILOG(kDebug, "Cannot create null queue {}", id); + HELOG(kError, "Cannot create null queue {}", id); return; } if (!queue->id_.IsNull()) { - HILOG(kDebug, "Queue {} already exists", id); + HELOG(kError, "Queue {} already exists", id); return; } - HILOG(kDebug, "Creating queue {}", id); + // HILOG(kDebug, "Creating queue {}", id); queue_map_->replace(queue_map_->begin() + id.unique_, id, queue_info); } diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 04b1a1e8a..318e5249f 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -301,10 +301,15 @@ struct Task : public hipc::ShmContainer { } /** Disable the running of a task */ - HSHM_ALWAYS_INLINE void DisableRun() { + HSHM_ALWAYS_INLINE void SetDisableRun() { task_flags_.SetBits(TASK_DISABLE_RUN); } + /** Enable the running of a task */ + HSHM_ALWAYS_INLINE void UnsetDisableRun() { + task_flags_.UnsetBits(TASK_DISABLE_RUN); + } + /** Check if running task is disable */ HSHM_ALWAYS_INLINE bool IsRunDisabled() { return task_flags_.Any(TASK_DISABLE_RUN); diff --git a/include/labstor/task_registry/task_registry.h b/include/labstor/task_registry/task_registry.h index c294faf08..7bf7ef987 100644 --- a/include/labstor/task_registry/task_registry.h +++ b/include/labstor/task_registry/task_registry.h @@ -201,8 +201,8 @@ class TaskRegistry { task->SetModuleComplete(); return false; } - HILOG(kInfo, "(node {}) Creating an instance of {} with name {}", - LABSTOR_CLIENT->node_id_, lib_name, state_name) +// HILOG(kInfo, "(node {}) Creating an instance of {} with name {}", +// LABSTOR_CLIENT->node_id_, lib_name, state_name) // Find the task library to instantiate auto it = libs_.find(lib_name); @@ -234,7 +234,7 @@ class TaskRegistry { task_state->name_ = state_name; task_state_ids_.emplace(state_name, state_id); task_states_.emplace(state_id, task_state); - HILOG(kInfo, "(node {}) Allocated an instance of {} with name {} and ID {}", + HILOG(kInfo, "(node {}) Created an instance of {} with name {} and ID {}", LABSTOR_CLIENT->node_id_, lib_name, state_name, state_id) return true; } diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 4c40c9530..090243ee8 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -275,8 +275,8 @@ class Worker { } int ret = exec->GetGroup(task->method_, task, group_); if (ret == TASK_UNORDERED || task->IsUnordered()) { - HILOG(kDebug, "(node {}) Task {} is unordered, so count remains 0 worker={}", - LABSTOR_CLIENT->node_id_, task->task_node_, id_); +// HILOG(kDebug, "(node {}) Task {} is unordered, so count remains 0 worker={}", +// LABSTOR_CLIENT->node_id_, task->task_node_, id_); return true; } @@ -296,15 +296,15 @@ class Worker { if (it == group_map_.end()) { node.node_depth_ = 1; group_map_.emplace(group_, node); - HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", - LABSTOR_CLIENT->node_id_, node.node_depth_, id_); +// HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", +// LABSTOR_CLIENT->node_id_, node.node_depth_, id_); return true; } TaskNode &node_cmp = it->second; if (node_cmp.root_ == node.root_) { node_cmp.node_depth_ += 1; - HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", - LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, id_); +// HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", +// LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, id_); return true; } return false; @@ -318,8 +318,8 @@ class Worker { } int ret = exec->GetGroup(task->method_, task, group_); if (ret == TASK_UNORDERED || task->IsUnordered()) { - HILOG(kDebug, "(node {}) Decreasing depth of group remains 0 (task_node={} worker={})", - LABSTOR_CLIENT->node_id_, task->task_node_, id_); +// HILOG(kDebug, "(node {}) Decreasing depth of group remains 0 (task_node={} worker={})", +// LABSTOR_CLIENT->node_id_, task->task_node_, id_); return; } @@ -340,8 +340,8 @@ class Worker { LABSTOR_CLIENT->node_id_, task->task_node_, id_); } node_cmp.node_depth_ -= 1; - HILOG(kDebug, "(node {}) Decreasing depth of to {} (task_node={} worker={})", - LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, task->task_node_, id_); +// HILOG(kDebug, "(node {}) Decreasing depth of to {} (task_node={} worker={})", +// LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, task->task_node_, id_); if (node_cmp.node_depth_ == 0) { group_map_.erase(group_); } diff --git a/src/worker.cc b/src/worker.cc index 2b021eec3..e134f8995 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -73,7 +73,6 @@ void Worker::PollGrouped(WorkEntry &work_entry) { if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote) && task->ShouldRun(work_entry.cur_time_)) { - // TODO(llogan): Make a remote debug macro #ifdef REMOTE_DEBUG if (task->task_state_ != LABSTOR_QM_CLIENT->admin_task_state_ && !task->task_flags_.Any(TASK_REMOTE_DEBUG_MARK) && @@ -87,12 +86,12 @@ void Worker::PollGrouped(WorkEntry &work_entry) { if (is_remote) { auto ids = LABSTOR_RUNTIME->ResolveDomainId(task->domain_id_); LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); - task->DisableRun(); + task->SetDisableRun(); task->SetUnordered(); task->UnsetCoroutine(); } else if (task->IsLaneAll()) { LABSTOR_REMOTE_QUEUE->DisperseLocal(task, exec, work_entry.queue_, work_entry.group_); - task->DisableRun(); + task->SetDisableRun(); task->SetUnordered(); task->UnsetCoroutine(); task->UnsetLaneAll(); @@ -116,7 +115,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task->SetStarted(); } } else if (task->IsPreemptive()) { - task->DisableRun(); + task->SetDisableRun(); entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunPreemptive, task); } else { task->SetStarted(); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 12925e2b1..b4bf4d405 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -1152,7 +1152,7 @@ struct FlushDataTask : public Task, TaskFlags { method_ = Method::kFlushData; task_flags_.SetBits(TASK_LANE_ALL | TASK_FIRE_AND_FORGET | TASK_LONG_RUNNING | TASK_COROUTINE); SetPeriodSec(10); // TODO(llogan): don't hardcode this - domain_id_ = DomainId::GetGlobal(); + domain_id_ = DomainId::GetLocal(); } /** Duplicate message */ diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 5db2458f1..50190e29d 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -70,7 +70,12 @@ class Server : public TaskLib { task->orig_task_->task_node_, task->orig_task_->task_state_, task->orig_task_->method_); - task->orig_task_->SetModuleComplete(); + if (!task->orig_task_->IsLongRunning()) { + task->orig_task_->SetModuleComplete(); + } else { + task->orig_task_->UnsetStarted(); + task->orig_task_->UnsetDisableRun(); + } task->SetModuleComplete(); } diff --git a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc index a73ba20ae..a50e01ffe 100644 --- a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc +++ b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc @@ -32,14 +32,14 @@ class Server : public TaskLib { // NOTE(llogan): Assumes a minimum of two workers, admin on worker 0. if (lane_group.IsLowPriority()) { for (u32 lane_id = lane_group.num_scheduled_; lane_id < lane_group.num_lanes_; ++lane_id) { - HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); + // HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); Worker &worker = LABSTOR_WORK_ORCHESTRATOR->workers_[0]; worker.PollQueues({WorkEntry(lane_group.prio_, lane_id, &queue)}); } lane_group.num_scheduled_ = lane_group.num_lanes_; } else { for (u32 lane_id = lane_group.num_scheduled_; lane_id < lane_group.num_lanes_; ++lane_id) { - HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); + // HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); u32 worker_id = (count_ % (LABSTOR_WORK_ORCHESTRATOR->workers_.size() - 1)) + 1; Worker &worker = LABSTOR_WORK_ORCHESTRATOR->workers_[worker_id]; worker.PollQueues({WorkEntry(lane_group.prio_, lane_id, &queue)}); From a751703e61d399daba3aef2352ab1c65081a6507 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 9 Oct 2023 07:49:28 -0500 Subject: [PATCH 155/191] Used remote debug macro for testing current staging in Hermes --- .../include/data_stager/factory/binary_stager.h | 2 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 6 +++--- .../include/hermes_blob_mdm/hermes_blob_mdm_tasks.h | 9 +++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tasks/data_stager/include/data_stager/factory/binary_stager.h b/tasks/data_stager/include/data_stager/factory/binary_stager.h index 9287cbce2..5b0c3dcd1 100644 --- a/tasks/data_stager/include/data_stager/factory/binary_stager.h +++ b/tasks/data_stager/include/data_stager/factory/binary_stager.h @@ -89,7 +89,7 @@ class BinaryFileStager : public AbstractStager { void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) override { adapter::BlobPlacement plcmnt; plcmnt.DecodeBlobName(*task->blob_name_, page_size_); - HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", + HILOG(kDebug, "Attempting to stage {} bytes to the backend file {} at offset {}", page_size_, url_, plcmnt.bucket_off_); char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); ssize_t real_size = HERMES_POSIX_API->pwrite(fd_, diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 00dd83c49..c09e56aaf 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -149,7 +149,7 @@ class Client : public TaskLibClient { hipc::Pointer &data, Context ctx = Context(), bitfield32_t flags = bitfield32_t(0)) { - HILOG(kDebug, "Beginning GET (task_node={})", task_node); + // HILOG(kDebug, "Beginning GET (task_node={})", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id, off, data_size, data, ctx, flags); @@ -185,7 +185,7 @@ class Client : public TaskLibClient { const BlobId &blob_id, float score, u32 node_id) { - HILOG(kDebug, "Beginning REORGANIZE (task_node={})", task_node); + // HILOG(kDebug, "Beginning REORGANIZE (task_node={})", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id, score, node_id); @@ -297,7 +297,7 @@ class Client : public TaskLibClient { const TagId &tag_id, const hshm::charbuf &blob_name, const BlobId &blob_id) { - HILOG(kDebug, "Getting blob size {}", task_node); + // HILOG(kDebug, "Getting blob size {}", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index b4bf4d405..302773199 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -1150,8 +1150,13 @@ struct FlushDataTask : public Task, TaskFlags { prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kFlushData; - task_flags_.SetBits(TASK_LANE_ALL | TASK_FIRE_AND_FORGET | TASK_LONG_RUNNING | TASK_COROUTINE); - SetPeriodSec(10); // TODO(llogan): don't hardcode this + task_flags_.SetBits( + TASK_LANE_ALL | + TASK_FIRE_AND_FORGET | + TASK_LONG_RUNNING | + TASK_COROUTINE | + TASK_REMOTE_DEBUG_MARK); + SetPeriodSec(2); // TODO(llogan): don't hardcode this domain_id_ = DomainId::GetLocal(); } From 91e64bfe089b7d573b4a464eae66020085112ca1 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 10 Oct 2023 02:54:04 -0500 Subject: [PATCH 156/191] Use getsec --- benchmark/hermes_api_bench.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index 196b772c2..12b9a1f63 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -24,7 +24,7 @@ using hshm::MpiTimer; void GatherTimes(std::string test_name, size_t io_size, MpiTimer &t) { t.Collect(); if (t.rank_ == 0) { - double max = t.GetUsec(); + double max = t.GetSec(); double mbps = io_size / t.GetUsec(); HIPRINT("{}: Time: {} sec, MBps (or MOps): {}, Count: {}, Nprocs: {}\n", test_name, max, mbps, io_size, t.nprocs_); From b6caaf302a689a682f8ef38a298c6d3f1246e877 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 10 Oct 2023 03:03:22 -0500 Subject: [PATCH 157/191] Remove timer for this second --- include/labstor/work_orchestrator/worker.h | 5 +++++ src/worker.cc | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 090243ee8..493c89db7 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -43,6 +43,7 @@ struct WorkEntry { group_ = &queue->GetGroup(prio); lane_ = &queue->GetLane(*group_, lane_id); count_ = 0; + cur_time_.Now(); } /** Copy constructor */ @@ -53,6 +54,7 @@ struct WorkEntry { lane_ = other.lane_; group_ = other.group_; queue_ = other.queue_; + cur_time_.Now(); } /** Copy assignment */ @@ -64,6 +66,7 @@ struct WorkEntry { lane_ = other.lane_; group_ = other.group_; queue_ = other.queue_; + cur_time_.Now(); } return *this; } @@ -76,6 +79,7 @@ struct WorkEntry { lane_ = other.lane_; group_ = other.group_; queue_ = other.queue_; + cur_time_.Now(); } /** Move assignment */ @@ -87,6 +91,7 @@ struct WorkEntry { lane_ = other.lane_; group_ = other.group_; queue_ = other.queue_; + cur_time_.Now(); } return *this; } diff --git a/src/worker.cc b/src/worker.cc index e134f8995..dab7919db 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -36,7 +36,6 @@ void Worker::Run() { continue; } } - work_entry.cur_time_.Now(); PollGrouped(work_entry); } } From c5eff8784c8ff78982b0eafe950b7b453885e82c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 10 Oct 2023 03:08:21 -0500 Subject: [PATCH 158/191] Long running tasks always run --- include/labstor/task_registry/task.h | 1 + tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 318e5249f..96d00a2e2 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -402,6 +402,7 @@ struct Task : public hipc::ShmContainer { /** Determine if time has elapsed */ HSHM_ALWAYS_INLINE bool ShouldRun(hshm::Timepoint &cur_time) { + return true; if (!IsStarted()) { start_ = cur_time; return true; diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 887317518..e49b86efd 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -114,7 +114,7 @@ class Server : public TaskLib { if (bkt_mdm_.id_.IsNull()) { bkt_mdm_.Init(task->bkt_mdm_); stager_mdm_.Init(task->stager_mdm_); - flush_task_ = blob_mdm_.AsyncFlushData(task->task_node_ + 1); + // flush_task_ = blob_mdm_.AsyncFlushData(task->task_node_ + 1); } task->SetModuleComplete(); } From b35a43eb5f1ca93cb8e453e071cd3cb674950d36 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 10 Oct 2023 10:48:56 -0500 Subject: [PATCH 159/191] Timer before worker iteration --- src/worker.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/worker.cc b/src/worker.cc index dab7919db..6243457af 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -29,6 +29,8 @@ void Worker::Run() { if (relinquish_queues_.size() > 0) { _RelinquishQueues(); } + hshm::Timepoint now; + now.Now(); for (WorkEntry &work_entry : work_queue_) { if (!work_entry.lane_->flags_.Any(QUEUE_LOW_LATENCY)) { work_entry.count_ += 1; @@ -36,6 +38,7 @@ void Worker::Run() { continue; } } + work_entry.cur_time_ = now; PollGrouped(work_entry); } } From b275a67dedd455b6db69c7cd24c05e83a8911ef3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 10 Oct 2023 10:51:58 -0500 Subject: [PATCH 160/191] Add timer todo --- include/labstor/api/labstor_client.h | 2 -- src/worker.cc | 1 - tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 1 + 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 069cfb241..8a42302ce 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -231,14 +231,12 @@ class Client : public ConfigurationManager { /** Free a buffer */ HSHM_ALWAYS_INLINE void FreeBuffer(hipc::Pointer &p) { - // TODO(llogan): verify leak main_alloc_->Free(p); } /** Free a buffer */ HSHM_ALWAYS_INLINE void FreeBuffer(LPointer &p) { - // TODO(llogan): verify leak main_alloc_->FreeLocalPtr(p); } }; diff --git a/src/worker.cc b/src/worker.cc index 6243457af..559ffef58 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -131,7 +131,6 @@ void Worker::PollGrouped(WorkEntry &work_entry) { // LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); entry->complete_ = true; if (task->IsCoroutine()) { - // TODO(llogan): verify leak free(rctx.stack_ptr_); } else if (task->IsPreemptive()) { ABT_thread_join(entry->thread_); diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index e49b86efd..8edd7d5ed 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -114,6 +114,7 @@ class Server : public TaskLib { if (bkt_mdm_.id_.IsNull()) { bkt_mdm_.Init(task->bkt_mdm_); stager_mdm_.Init(task->stager_mdm_); + // TODO(llogan): Add back // flush_task_ = blob_mdm_.AsyncFlushData(task->task_node_ + 1); } task->SetModuleComplete(); From 22a527fc9f596456bc27d667497ea80724733ae8 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 11 Oct 2023 06:47:28 -0500 Subject: [PATCH 161/191] Add data op tasks --- CMakeLists.txt | 1 + include/labstor/network/serialize.h | 1 + op_min_max/CMakeLists.txt | 10 + op_min_max/include/op_min_max/op_min_max.h | 73 +++++ .../include/op_min_max/op_min_max_lib_exec.h | 197 ++++++++++++ .../include/op_min_max/op_min_max_methods.h | 9 + .../op_min_max/op_min_max_methods.yaml | 1 + .../include/op_min_max/op_min_max_tasks.h | 135 ++++++++ op_min_max/src/CMakeLists.txt | 54 ++++ op_min_max/src/op_min_max.cc | 33 ++ tasks/CMakeLists.txt | 3 +- .../data_stager/factory/binary_stager.h | 4 +- tasks/hermes/include/hermes/bucket.h | 4 +- tasks/hermes/include/hermes/config_manager.h | 5 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 20 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 18 +- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 13 +- .../hermes_bucket_mdm_lib_exec.h | 8 + .../src/hermes_bucket_mdm.cc | 6 +- tasks/hermes_data_op/CMakeLists.txt | 10 + .../include/hermes_data_op/hermes_data_op.h | 101 ++++++ .../hermes_data_op/hermes_data_op_lib_exec.h | 285 +++++++++++++++++ .../hermes_data_op/hermes_data_op_methods.h | 11 + .../hermes_data_op_methods.yaml | 3 + .../hermes_data_op/hermes_data_op_tasks.h | 290 ++++++++++++++++++ tasks/hermes_data_op/src/CMakeLists.txt | 54 ++++ tasks/hermes_data_op/src/hermes_data_op.cc | 208 +++++++++++++ .../labstor_admin/src/labstor_admin.cc | 8 +- 28 files changed, 1534 insertions(+), 31 deletions(-) create mode 100644 op_min_max/CMakeLists.txt create mode 100644 op_min_max/include/op_min_max/op_min_max.h create mode 100644 op_min_max/include/op_min_max/op_min_max_lib_exec.h create mode 100644 op_min_max/include/op_min_max/op_min_max_methods.h create mode 100644 op_min_max/include/op_min_max/op_min_max_methods.yaml create mode 100644 op_min_max/include/op_min_max/op_min_max_tasks.h create mode 100644 op_min_max/src/CMakeLists.txt create mode 100644 op_min_max/src/op_min_max.cc create mode 100644 tasks/hermes_data_op/CMakeLists.txt create mode 100644 tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h create mode 100644 tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h create mode 100644 tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h create mode 100644 tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml create mode 100644 tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h create mode 100644 tasks/hermes_data_op/src/CMakeLists.txt create mode 100644 tasks/hermes_data_op/src/hermes_data_op.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b499e24c..71e56e847 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,6 +154,7 @@ include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_blob_mdm/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_bucket_mdm/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_adapters/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/data_stager/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_data_op/include) # Test includes include_directories(${CMAKE_SOURCE_DIR}/test/unit) diff --git a/include/labstor/network/serialize.h b/include/labstor/network/serialize.h index 70a44241d..ccdaf76da 100644 --- a/include/labstor/network/serialize.h +++ b/include/labstor/network/serialize.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/op_min_max/CMakeLists.txt b/op_min_max/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/op_min_max/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/op_min_max/include/op_min_max/op_min_max.h b/op_min_max/include/op_min_max/op_min_max.h new file mode 100644 index 000000000..e85e8b4fd --- /dev/null +++ b/op_min_max/include/op_min_max/op_min_max.h @@ -0,0 +1,73 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_op_min_max_H_ +#define LABSTOR_op_min_max_H_ + +#include "op_min_max_tasks.h" + +namespace labstor::op_min_max { + +/** Create op_min_max requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Async create a task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info); + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate) + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + LPointer task = + AsyncCreateRoot(std::forward(args)...); + task->Wait(); + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Call a custom method */ + HSHM_ALWAYS_INLINE + void AsyncCustomConstruct(CustomTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id, id_); + } + HSHM_ALWAYS_INLINE + void CustomRoot(const DomainId &domain_id) { + LPointer> task = AsyncCustomRoot(domain_id); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(Custom); +}; + +} // namespace labstor + +#endif // LABSTOR_op_min_max_H_ diff --git a/op_min_max/include/op_min_max/op_min_max_lib_exec.h b/op_min_max/include/op_min_max/op_min_max_lib_exec.h new file mode 100644 index 000000000..d84dd08a7 --- /dev/null +++ b/op_min_max/include/op_min_max/op_min_max_lib_exec.h @@ -0,0 +1,197 @@ +#ifndef LABSTOR_op_min_max_LIB_EXEC_H_ +#define LABSTOR_op_min_max_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task, RunContext &rctx) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task), rctx); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task), rctx); + break; + } + case Method::kCustom: { + Custom(reinterpret_cast(task), rctx); + break; + } + } +} +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kCustom: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kCustom: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kCustom: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kDestruct: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kCustom: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kCustom: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kCustom: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kCustom: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_op_min_max_METHODS_H_ \ No newline at end of file diff --git a/op_min_max/include/op_min_max/op_min_max_methods.h b/op_min_max/include/op_min_max/op_min_max_methods.h new file mode 100644 index 000000000..4bed8afcd --- /dev/null +++ b/op_min_max/include/op_min_max/op_min_max_methods.h @@ -0,0 +1,9 @@ +#ifndef LABSTOR_op_min_max_METHODS_H_ +#define LABSTOR_op_min_max_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kCustom = kLast + 0; +}; + +#endif // LABSTOR_op_min_max_METHODS_H_ \ No newline at end of file diff --git a/op_min_max/include/op_min_max/op_min_max_methods.yaml b/op_min_max/include/op_min_max/op_min_max_methods.yaml new file mode 100644 index 000000000..b1b54e2a4 --- /dev/null +++ b/op_min_max/include/op_min_max/op_min_max_methods.yaml @@ -0,0 +1 @@ +kCustom: 0 \ No newline at end of file diff --git a/op_min_max/include/op_min_max/op_min_max_tasks.h b/op_min_max/include/op_min_max/op_min_max_tasks.h new file mode 100644 index 000000000..97afcb865 --- /dev/null +++ b/op_min_max/include/op_min_max/op_min_max_tasks.h @@ -0,0 +1,135 @@ +// +// Created by lukemartinlogan on 8/11/23. +// + +#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_op_min_max_op_min_max_TASKS_H_ +#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_op_min_max_op_min_max_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" + +namespace labstor::op_min_max { + +#include "op_min_max_methods.h" +#include "labstor/labstor_namespace.h" +using labstor::proc_queue::TypedPushTask; +using labstor::proc_queue::PushTask; + +/** + * A task to create op_min_max + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) + : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "op_min_max", id, queue_info) { + // Custom params + } + + HSHM_ALWAYS_INLINE + ~ConstructTask() { + // Custom params + } +}; + +/** A task to destroy op_min_max */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) + : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * A custom task in op_min_max + * */ +struct CustomTask : public Task, TaskFlags { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + CustomTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kCustom; + task_flags_.SetBits(0); + domain_id_ = domain_id; + + // Custom params + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, CustomTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, CustomTask &dup_task) { + } + + /** Begin replication */ + void ReplicateStart(u32 count) { + } + + /** Finalize replication */ + void ReplicateEnd() {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace labstor::op_min_max + +#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_op_min_max_op_min_max_TASKS_H_ diff --git a/op_min_max/src/CMakeLists.txt b/op_min_max/src/CMakeLists.txt new file mode 100644 index 000000000..dcd2cf9d5 --- /dev/null +++ b/op_min_max/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(op_min_max SHARED + op_min_max.cc) +add_dependencies(op_min_max ${Hermes_RUNTIME_DEPS}) +target_link_libraries(op_min_max ${Hermes_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + op_min_max + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${HERMES_EXPORTED_TARGETS} + DESTINATION + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${HERMES_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(HERMES_EXPORTED_LIBS + op_min_max + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${HERMES_EXPORTED_LIBS} + FILE + ${HERMES_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(op_min_max) +endif() diff --git a/op_min_max/src/op_min_max.cc b/op_min_max/src/op_min_max.cc new file mode 100644 index 000000000..51bf94d33 --- /dev/null +++ b/op_min_max/src/op_min_max.cc @@ -0,0 +1,33 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "op_min_max/op_min_max.h" + +namespace labstor::op_min_max { + +class Server : public TaskLib { + public: + Server() = default; + + void Construct(ConstructTask *task, RunContext &rctx) { + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task, RunContext &rctx) { + task->SetModuleComplete(); + } + + void Custom(CustomTask *task, RunContext &rctx) { + task->SetModuleComplete(); + } + + public: +#include "op_min_max/op_min_max_lib_exec.h" +}; + +} // namespace labstor::op_min_max + +LABSTOR_TASK_CC(labstor::op_min_max::Server, "op_min_max"); diff --git a/tasks/CMakeLists.txt b/tasks/CMakeLists.txt index f91850e92..52b374e69 100644 --- a/tasks/CMakeLists.txt +++ b/tasks/CMakeLists.txt @@ -6,4 +6,5 @@ add_subdirectory(hermes_mdm) add_subdirectory(hermes_blob_mdm) add_subdirectory(hermes_bucket_mdm) add_subdirectory(hermes_adapters) -add_subdirectory(data_stager) +add_subdirectory(hermes_data_op) +add_subdirectory(data_stager) \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/factory/binary_stager.h b/tasks/data_stager/include/data_stager/factory/binary_stager.h index 5b0c3dcd1..981eb5d83 100644 --- a/tasks/data_stager/include/data_stager/factory/binary_stager.h +++ b/tasks/data_stager/include/data_stager/factory/binary_stager.h @@ -79,8 +79,8 @@ class BinaryFileStager : public AbstractStager { task->bkt_id_, hshm::to_charbuf(*task->blob_name_), hermes::BlobId::GetNull(), - 0, real_size, blob.shm_, task->score_, bitfield32_t(0), - ctx, bitfield32_t(TASK_DATA_OWNER | TASK_LOW_LATENCY)); + 0, real_size, blob.shm_, task->score_, 0, + ctx, TASK_DATA_OWNER | TASK_LOW_LATENCY); put_task->Wait(task); LABSTOR_CLIENT->DelTask(put_task); } diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 0e7ebee42..9dd50c61d 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -234,7 +234,7 @@ class Bucket { LPointer> push_task; push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, - flags, ctx, task_flags); + flags.bits_, ctx, task_flags.bits_); if constexpr (!ASYNC) { if (flags.Any(HERMES_GET_BLOB_ID)) { push_task->Wait(); @@ -433,7 +433,7 @@ class Bucket { push_task = blob_mdm_->AsyncGetBlobRoot(id_, hshm::to_charbuf(blob_name), blob_id, blob_off, data_size, data_p.shm_, - ctx, flags); + ctx, flags.bits_); return push_task; } diff --git a/tasks/hermes/include/hermes/config_manager.h b/tasks/hermes/include/hermes/config_manager.h index f88eb6ba6..e29172953 100644 --- a/tasks/hermes/include/hermes/config_manager.h +++ b/tasks/hermes/include/hermes/config_manager.h @@ -10,6 +10,7 @@ #include "hermes_mdm/hermes_mdm.h" #include "hermes_bucket_mdm/hermes_bucket_mdm.h" #include "hermes_blob_mdm/hermes_blob_mdm.h" +#include "hermes_data_op/hermes_data_op.h" #include "hermes/config_client.h" #include "hermes/config_server.h" #include "data_stager/data_stager.h" @@ -22,6 +23,7 @@ class ConfigurationManager { bucket_mdm::Client bkt_mdm_; blob_mdm::Client blob_mdm_; data_stager::Client stager_mdm_; + data_op::Client op_mdm_; ServerConfig server_config_; ClientConfig client_config_; bool is_initialized_ = false; @@ -40,8 +42,9 @@ class ConfigurationManager { mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_mdm"); blob_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_blob_mdm"); bkt_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_bkt_mdm"); + op_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_op_mdm", bkt_mdm_.id_, blob_mdm_.id_); stager_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_stager_mdm", blob_mdm_.id_); - blob_mdm_.SetBucketMdmRoot(DomainId::GetGlobal(), bkt_mdm_.id_, stager_mdm_.id_); + blob_mdm_.SetBucketMdmRoot(DomainId::GetGlobal(), bkt_mdm_.id_, stager_mdm_.id_, op_mdm_.id_); bkt_mdm_.SetBlobMdmRoot(DomainId::GetGlobal(), blob_mdm_.id_, stager_mdm_.id_); is_initialized_ = true; } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index c09e56aaf..088b94531 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -70,16 +70,18 @@ class Client : public TaskLibClient { void AsyncSetBucketMdmConstruct(SetBucketMdmTask *task, const TaskNode &task_node, const DomainId &domain_id, - const TaskStateId &blob_mdm_id, - const TaskStateId &stager_mdm) { + const TaskStateId &blob_mdm, + const TaskStateId &stager_mdm, + const TaskStateId &op_mdm) { LABSTOR_CLIENT->ConstructTask( - task, task_node, domain_id, id_, blob_mdm_id, stager_mdm); + task, task_node, domain_id, id_, blob_mdm, stager_mdm, op_mdm); } void SetBucketMdmRoot(const DomainId &domain_id, const TaskStateId &blob_mdm, - const TaskStateId &stager_mdm) { + const TaskStateId &stager_mdm, + const TaskStateId &op_mdm) { LPointer> push_task = - AsyncSetBucketMdmRoot(domain_id, blob_mdm, stager_mdm); + AsyncSetBucketMdmRoot(domain_id, blob_mdm, stager_mdm, op_mdm); push_task->Wait(); LABSTOR_CLIENT->DelTask(push_task); } @@ -127,9 +129,9 @@ class Client : public TaskLibClient { TagId tag_id, const hshm::charbuf &blob_name, const BlobId &blob_id, size_t blob_off, size_t blob_size, const hipc::Pointer &blob, float score, - bitfield32_t flags, + u32 flags, Context ctx = Context(), - bitfield32_t task_flags = bitfield32_t(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY)) { + u32 task_flags = TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY) { LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id, @@ -148,7 +150,7 @@ class Client : public TaskLibClient { ssize_t data_size, hipc::Pointer &data, Context ctx = Context(), - bitfield32_t flags = bitfield32_t(0)) { + u32 flags = 0) { // HILOG(kDebug, "Beginning GET (task_node={})", task_node); LABSTOR_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, @@ -160,7 +162,7 @@ class Client : public TaskLibClient { ssize_t data_size, hipc::Pointer &data, Context ctx = Context(), - bitfield32_t flags = bitfield32_t(0)) { + u32 flags = 0) { LPointer> push_task = AsyncGetBlobRoot(tag_id, hshm::charbuf(""), blob_id, off, data_size, data, ctx, flags); push_task->Wait(); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 302773199..bb9a50396 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -79,6 +79,7 @@ struct DestructTask : public DestroyTaskStateTask { struct SetBucketMdmTask : public Task, TaskFlags { IN TaskStateId bkt_mdm_; IN TaskStateId stager_mdm_; + IN TaskStateId op_mdm_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -91,7 +92,8 @@ struct SetBucketMdmTask : public Task, TaskFlags { const DomainId &domain_id, const TaskStateId &state_id, const TaskStateId &bkt_mdm, - const TaskStateId &stager_mdm) : Task(alloc) { + const TaskStateId &stager_mdm, + const TaskStateId &op_mdm) : Task(alloc) { // Initialize task task_node_ = task_node; lane_hash_ = 0; @@ -104,6 +106,7 @@ struct SetBucketMdmTask : public Task, TaskFlags { // Custom params bkt_mdm_ = bkt_mdm; stager_mdm_ = stager_mdm; + op_mdm_ = op_mdm; } /** Destructor */ @@ -222,6 +225,7 @@ class PutBlobPhase { #define HERMES_IS_FILE BIT_OPT(u32, 3) #define HERMES_BLOB_DID_CREATE BIT_OPT(u32, 4) #define HERMES_GET_BLOB_ID BIT_OPT(u32, 5) +#define HERMES_HAS_DERIVED BIT_OPT(u32, 6) /** A task to put data in a blob */ struct PutBlobTask : public Task, TaskFlags { @@ -251,15 +255,15 @@ struct PutBlobTask : public Task, TaskFlags size_t data_size, const hipc::Pointer &data, float score, - bitfield32_t flags, + u32 flags, const Context &ctx, - bitfield32_t task_flags) : Task(alloc) { + u32 task_flags) : Task(alloc) { // Initialize task task_node_ = task_node; prio_ = TaskPrio::kLowLatency; task_state_ = state_id; method_ = Method::kPutBlob; - task_flags_ = task_flags; + task_flags_ = bitfield32_t(task_flags); task_flags_.SetBits(TASK_COROUTINE); if (!blob_id.IsNull()) { lane_hash_ = blob_id.hash_; @@ -277,7 +281,7 @@ struct PutBlobTask : public Task, TaskFlags data_size_ = data_size; data_ = data; score_ = score; - flags_ = flags; + flags_ = bitfield32_t(flags); if (ctx.flags_.Any(HERMES_IS_FILE)) { flags_.SetBits(HERMES_IS_FILE); } @@ -367,7 +371,7 @@ struct GetBlobTask : public Task, TaskFlags ssize_t data_size, hipc::Pointer &data, const Context &ctx, - bitfield32_t flags) : Task(alloc) { + u32 flags) : Task(alloc) { // Initialize task task_node_ = task_node; prio_ = TaskPrio::kLowLatency; @@ -388,7 +392,7 @@ struct GetBlobTask : public Task, TaskFlags blob_off_ = off; data_size_ = data_size; data_ = data; - flags_ = flags; + flags_ = bitfield32_t(flags); HSHM_MAKE_AR(blob_name_, alloc, blob_name); } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 8edd7d5ed..302f9cca8 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -10,6 +10,7 @@ #include "hermes_adapters/posix/posix_api.h" #include "bdev/bdev.h" #include "data_stager/data_stager.h" +#include "hermes_data_op/hermes_data_op.h" namespace hermes::blob_mdm { @@ -48,6 +49,7 @@ class Server : public TaskLib { Client blob_mdm_; bucket_mdm::Client bkt_mdm_; data_stager::Client stager_mdm_; + data_op::Client op_mdm_; LPointer flush_task_; public: @@ -114,6 +116,7 @@ class Server : public TaskLib { if (bkt_mdm_.id_.IsNull()) { bkt_mdm_.Init(task->bkt_mdm_); stager_mdm_.Init(task->stager_mdm_); + op_mdm_.Init(task->op_mdm_); // TODO(llogan): Add back // flush_task_ = blob_mdm_.AsyncFlushData(task->task_node_ + 1); } @@ -285,6 +288,14 @@ class Server : public TaskLib { task->tag_id_, task->blob_id_); } + if (task->flags_.Any(HERMES_HAS_DERIVED)) { + op_mdm_.AsyncRegisterData(task->task_node_ + 1, + task->tag_id_, + task->blob_name_->str(), + task->blob_id_, + task->blob_off_, + task->data_size_); + } // Free data task->SetModuleComplete(); @@ -636,7 +647,7 @@ class Server : public TaskLib { task->data_size_, task->data_, task->score_, - bitfield32_t(HERMES_BLOB_REPLACE)).ptr_; + HERMES_BLOB_REPLACE).ptr_; task->SetModuleComplete(); } } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index 78b9043ee..1a4f10e2c 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -202,6 +202,10 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } + case Method::kGetContainedBlobIds: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } } } /** Register the duplicate output with the origin task */ @@ -267,6 +271,10 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } + case Method::kGetContainedBlobIds: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } } } /** Ensure there is space to store replicated outputs */ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 1faf3e347..37f12c0cd 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -160,10 +160,8 @@ class Server : public TaskLib { append.blob_off_, append.data_size_, task->data_ + buf_off, - task->score_, - bitfield32_t(0), - Context(), - bitfield32_t(0)).ptr_; + task->score_, 0, + Context(), 0).ptr_; HILOG(kDebug, "(node {}) Finished spawning blob {} of size {} for tag {} (task_node={} blob_mdm={})", LABSTOR_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, task->tag_id_, task->task_node_, blob_mdm_.id_); diff --git a/tasks/hermes_data_op/CMakeLists.txt b/tasks/hermes_data_op/CMakeLists.txt new file mode 100644 index 000000000..5b409c224 --- /dev/null +++ b/tasks/hermes_data_op/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Labstor Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install LabStor Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h new file mode 100644 index 000000000..61439b0a5 --- /dev/null +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h @@ -0,0 +1,101 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#ifndef LABSTOR_hermes_data_op_H_ +#define LABSTOR_hermes_data_op_H_ + +#include "hermes_data_op_tasks.h" + +namespace hermes::data_op { + +/** Create hermes_data_op requests */ +class Client : public TaskLibClient { + + public: + /** Default constructor */ + Client() = default; + + /** Destructor */ + ~Client() = default; + + /** Async create a task state */ + HSHM_ALWAYS_INLINE + LPointer AsyncCreate(const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + TaskStateId &bkt_mdm_id, + TaskStateId &blob_mdm_id) { + id_ = TaskStateId::GetNull(); + QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + std::vector queue_info = { + {1, 1, qm.queue_depth_, 0}, + {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, + {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} + }; + return LABSTOR_ADMIN->AsyncCreateTaskState( + task_node, domain_id, state_name, id_, queue_info, + bkt_mdm_id, blob_mdm_id); + } + LABSTOR_TASK_NODE_ROOT(AsyncCreate) + template + HSHM_ALWAYS_INLINE + void CreateRoot(Args&& ...args) { + LPointer task = + AsyncCreateRoot(std::forward(args)...); + task->Wait(); + id_ = task->id_; + queue_id_ = QueueId(id_); + LABSTOR_CLIENT->DelTask(task); + } + + /** Destroy task state + queue */ + HSHM_ALWAYS_INLINE + void DestroyRoot(const DomainId &domain_id) { + LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + } + + /** Register the OpGraph to perform on data */ + HSHM_ALWAYS_INLINE + void AsyncRegisterOpConstruct(RegisterOpTask *task, + const TaskNode &task_node, + const OpGraph &op_graph) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, DomainId::GetGlobal(), id_, op_graph); + } + HSHM_ALWAYS_INLINE + void RegisterOpRoot(const OpGraph &op_graph) { + LPointer> task = + AsyncRegisterOpRoot(op_graph); + task.ptr_->Wait(); + } + LABSTOR_TASK_NODE_PUSH_ROOT(RegisterOp); + + /** Register data as ready for operations to be performed */ + HSHM_ALWAYS_INLINE + void AsyncRegisterDataConstruct(RegisterDataTask *task, + const TaskNode &task_node, + const BucketId &bkt_id, + const std::string &blob_name, + const BlobId &blob_id, + size_t off, + size_t size) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_, bkt_id, + blob_name, blob_id, off, size); + } + LABSTOR_TASK_NODE_PUSH_ROOT(RegisterData); + + /** Async task to run operators */ + HSHM_ALWAYS_INLINE + void AsyncRunOpConstruct(RunOpTask *task, + const TaskNode &task_node) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_); + } + LABSTOR_TASK_NODE_PUSH_ROOT(RunOp); +}; + +} // namespace labstor + +#endif // LABSTOR_hermes_data_op_H_ diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h new file mode 100644 index 000000000..ce08374ad --- /dev/null +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h @@ -0,0 +1,285 @@ +#ifndef LABSTOR_HERMES_DATA_OP_LIB_EXEC_H_ +#define LABSTOR_HERMES_DATA_OP_LIB_EXEC_H_ + +/** Execute a task */ +void Run(u32 method, Task *task, RunContext &rctx) override { + switch (method) { + case Method::kConstruct: { + Construct(reinterpret_cast(task), rctx); + break; + } + case Method::kDestruct: { + Destruct(reinterpret_cast(task), rctx); + break; + } + case Method::kRegisterOp: { + RegisterOp(reinterpret_cast(task), rctx); + break; + } + case Method::kRegisterData: { + RegisterData(reinterpret_cast(task), rctx); + break; + } + case Method::kRunOp: { + RunOp(reinterpret_cast(task), rctx); + break; + } + } +} +/** Delete a task */ +void Del(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kRegisterOp: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kRegisterData: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + case Method::kRunOp: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } + } +} +/** Duplicate a task */ +void Dup(u32 method, Task *orig_task, std::vector> &dups) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRegisterOp: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRegisterData: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + case Method::kRunOp: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } + } +} +/** Register the duplicate output with the origin task */ +void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kDestruct: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRegisterOp: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRegisterData: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + case Method::kRunOp: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } + } +} +/** Ensure there is space to store replicated outputs */ +void ReplicateStart(u32 method, u32 count, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kRegisterOp: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kRegisterData: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + case Method::kRunOp: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } + } +} +/** Determine success and handle failures */ +void ReplicateEnd(u32 method, Task *task) override { + switch (method) { + case Method::kConstruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kRegisterOp: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kRegisterData: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + case Method::kRunOp: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } + } +} +/** Serialize a task when initially pushing into remote */ +std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRegisterOp: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRegisterData: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRunOp: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when popping from remote queue */ +TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { + TaskPointer task_ptr; + switch (method) { + case Method::kConstruct: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kDestruct: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kRegisterOp: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kRegisterData: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + case Method::kRunOp: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } + } + return task_ptr; +} +/** Serialize a task when returning from remote queue */ +std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kDestruct: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRegisterOp: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRegisterData: { + ar << *reinterpret_cast(task); + break; + } + case Method::kRunOp: { + ar << *reinterpret_cast(task); + break; + } + } + return ar.Get(); +} +/** Deserialize a task when returning from remote queue */ +void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { + switch (method) { + case Method::kConstruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kDestruct: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kRegisterOp: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kRegisterData: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + case Method::kRunOp: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } + } +} +/** Get the grouping of the task */ +u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { + switch (method) { + case Method::kConstruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kDestruct: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kRegisterOp: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kRegisterData: { + return reinterpret_cast(task)->GetGroup(group); + } + case Method::kRunOp: { + return reinterpret_cast(task)->GetGroup(group); + } + } + return -1; +} + +#endif // LABSTOR_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h new file mode 100644 index 000000000..2e04c31ca --- /dev/null +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h @@ -0,0 +1,11 @@ +#ifndef LABSTOR_HERMES_DATA_OP_METHODS_H_ +#define LABSTOR_HERMES_DATA_OP_METHODS_H_ + +/** The set of methods in the admin task */ +struct Method : public TaskMethod { + TASK_METHOD_T kRegisterOp = kLast + 0; + TASK_METHOD_T kRegisterData = kLast + 1; + TASK_METHOD_T kRunOp = kLast + 2; +}; + +#endif // LABSTOR_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml new file mode 100644 index 000000000..372caa08e --- /dev/null +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml @@ -0,0 +1,3 @@ +kRegisterOp: 0 +kRegisterData: 1 +kRunOp: 2 \ No newline at end of file diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h new file mode 100644 index 000000000..74c6ffb9c --- /dev/null +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h @@ -0,0 +1,290 @@ +// +// Created by lukemartinlogan on 8/11/23. +// + +#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ +#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ + +#include "labstor/api/labstor_client.h" +#include "labstor/task_registry/task_lib.h" +#include "labstor_admin/labstor_admin.h" +#include "labstor/queue_manager/queue_manager_client.h" +#include "proc_queue/proc_queue.h" +#include "hermes/hermes_types.h" +#include "hermes_data_op/hermes_data_op.h" +#include "hermes_bucket_mdm/hermes_bucket_mdm.h" + +namespace hermes::data_op { + +#include "hermes_data_op_methods.h" +#include "labstor/labstor_namespace.h" +using labstor::proc_queue::TypedPushTask; +using labstor::proc_queue::PushTask; + +/** The bucket to use for op data */ +struct OpBucketName { + std::string url_; // URL of bucket + BucketId bkt_id_; // Bucket ID (internal) + LPointer + bkt_id_task_; // Task to get bucket ID (internal) + + template + void serialize(Ar &ar) { + ar(url_); + } +}; + +/** An operation to perform on a set of inputs */ +struct Op { + std::vector in_; // Input URLs, indicates data format as well + OpBucketName var_name_; // Output URL + std::string op_name_; // Operation name + u32 op_id_; // Operation ID (internal) + + template + void serialize(Ar &ar) { + ar(in_, var_name_, op_name_); + } +}; + +/** The graph of operations to perform on data */ +struct OpGraph { + std::vector ops_; + + template + void serialize(Ar &ar) { + ar(ops_); + } +}; + +/** + * A task to create hermes_data_op + * */ +using labstor::Admin::CreateTaskStateTask; +struct ConstructTask : public CreateTaskStateTask { + IN TaskStateId bkt_mdm_; + IN TaskStateId blob_mdm_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc) + : CreateTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + ConstructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const std::string &state_name, + const TaskStateId &id, + const std::vector &queue_info, + const TaskStateId &bkt_mdm_id, + const TaskStateId &blob_mdm_id) + : CreateTaskStateTask(alloc, task_node, domain_id, state_name, + "hermes_data_op", id, queue_info) { + // Custom params + bkt_mdm_ = bkt_mdm_id; + blob_mdm_ = blob_mdm_id; + } + + HSHM_ALWAYS_INLINE + ~ConstructTask() { + // Custom params + } +}; + +/** A task to destroy hermes_data_op */ +using labstor::Admin::DestroyTaskStateTask; +struct DestructTask : public DestroyTaskStateTask { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc) + : DestroyTaskStateTask(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + DestructTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + TaskStateId &state_id) + : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * Register an operation to perform on data + * */ +struct RegisterOpTask : public Task, TaskFlags { + IN hipc::ShmArchive op_graph_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterOpTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterOpTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id, + const TaskStateId &state_id, + const OpGraph &graph) : Task(alloc) { + // Store OpGraph + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar << graph; + std::string op_graph_str = ss.str(); + HSHM_MAKE_AR(op_graph_, alloc, op_graph_str); + + // Initialize task + task_node_ = task_node; + lane_hash_ = std::hash{}(op_graph_str); + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kRegisterOp; + task_flags_.SetBits(TASK_COROUTINE); + domain_id_ = domain_id; + } + + /** Get opgraph */ + OpGraph GetOpGraph() { + OpGraph graph; + std::stringstream ss; + cereal::BinaryInputArchive ar(ss); + ar >> graph; + return graph; + } + + /** Destructor */ + ~RegisterOpTask() { + HSHM_DESTROY_AR(op_graph_); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(op_graph_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + } + + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, RegisterOpTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, RegisterOpTask &dup_task) { + } + + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * Register some data as being available for derived + * quantity calculation + * */ +struct OpData { + BucketId bkt_id_; + BlobId blob_id_; + size_t off_; + size_t size_; + u64 data_id_; + std::string blob_name_; + int refcnt_; + + OpData() : refcnt_(0) {} +}; +struct RegisterDataTask : public Task, TaskFlags { + IN OpData data_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterDataTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + RegisterDataTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const TaskStateId &state_id, + const BucketId &bkt_id, + const std::string &blob_name, + const BlobId &blob_id, + size_t off, + size_t size) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = bkt_id.hash_; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kRegisterData; + task_flags_.SetBits(TASK_COROUTINE | TASK_FIRE_AND_FORGET); + domain_id_ = DomainId::GetLocal(); + + // Custom params + data_.bkt_id_ = bkt_id; + data_.blob_name_ = blob_name; + data_.blob_id_ = blob_id; + data_.off_ = off; + data_.size_ = size; + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +/** + * Run an operation over a piece of data + * */ +struct RunOpTask : public Task, TaskFlags { + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + RunOpTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + RunOpTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kRunOp; + task_flags_.SetBits(TASK_LONG_RUNNING | TASK_COROUTINE | TASK_LANE_ALL); + SetPeriodSec(1); // TODO(llogan): don't hardcode this + domain_id_ = DomainId::GetLocal(); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + +} // namespace hermes::data_op + +#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ diff --git a/tasks/hermes_data_op/src/CMakeLists.txt b/tasks/hermes_data_op/src/CMakeLists.txt new file mode 100644 index 000000000..96ca3eac9 --- /dev/null +++ b/tasks/hermes_data_op/src/CMakeLists.txt @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------------ +# Build Small Message Task Library +#------------------------------------------------------------------------------ +add_library(hermes_data_op SHARED + hermes_data_op.cc) +add_dependencies(hermes_data_op ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hermes_data_op ${Hermes_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Install Small Message Task Library +#------------------------------------------------------------------------------ +install( + TARGETS + hermes_data_op + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${HERMES_EXPORTED_TARGETS} + DESTINATION + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${HERMES_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(HERMES_EXPORTED_LIBS + hermes_data_op + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${HERMES_EXPORTED_LIBS} + FILE + ${HERMES_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(hermes_data_op) +endif() diff --git a/tasks/hermes_data_op/src/hermes_data_op.cc b/tasks/hermes_data_op/src/hermes_data_op.cc new file mode 100644 index 000000000..4655903a8 --- /dev/null +++ b/tasks/hermes_data_op/src/hermes_data_op.cc @@ -0,0 +1,208 @@ +// +// Created by lukemartinlogan on 6/29/23. +// + +#include "labstor_admin/labstor_admin.h" +#include "labstor/api/labstor_runtime.h" +#include "hermes_data_op/hermes_data_op.h" +#include "hermes_bucket_mdm/hermes_bucket_mdm.h" + +namespace hermes::data_op { + +struct OpPendingData { + std::list pending_; + u32 num_refs_; + u64 data_id_; + + OpPendingData() : num_refs_(0) {} +}; + +class Server : public TaskLib { + public: + std::vector> op_graphs_; + std::unordered_map op_id_map_; + Mutex op_data_lock_; + std::unordered_map op_data_map_; + hermes::bucket_mdm::Client bkt_mdm_; + hermes::blob_mdm::Client blob_mdm_; + Client client_; + LPointer run_task_; + + public: + Server() = default; + + void Construct(ConstructTask *task, RunContext &rctx) { + bkt_mdm_.Init(task->bkt_mdm_); + blob_mdm_.Init(task->blob_mdm_); + client_.Init(id_); + op_id_map_["min"] = 0; + op_id_map_["max"] = 1; + run_task_ = client_.AsyncRunOp(task->task_node_ + 1); + task->SetModuleComplete(); + } + + void Destruct(DestructTask *task, RunContext &rctx) { + task->SetModuleComplete(); + } + + void RegisterOp(RegisterOpTask *task, RunContext &rctx) { + // Load OpGraph + op_graphs_[rctx.lane_id_].push_back(task->GetOpGraph()); + OpGraph &op_graph = op_graphs_[rctx.lane_id_].back(); + + // Get or create all needed bucket IDs + std::vector traits; + for (Op &op : op_graph.ops_) { + // Spawn bucket ID task for each input + for (OpBucketName &bkt_name : op.in_) { + bkt_name.bkt_id_task_ = + bkt_mdm_.AsyncGetOrCreateTag(task->task_node_ + 1, + hshm::charbuf(bkt_name.url_), + true, + traits, 0, 0); + } + // Spawn bucket ID task for the output + op.var_name_.bkt_id_task_ = + bkt_mdm_.AsyncGetOrCreateTag(task->task_node_ + 1, + hshm::charbuf(op.var_name_.url_), + true, + traits, 0, 0); + // Set the op ID + op.op_id_ = op_id_map_[op.op_name_]; + } + + // Complete all bucket ID tasks + for (Op &op : op_graph.ops_) { + // Spawn bucket ID task for each input + for (OpBucketName &bkt_name : op.in_) { + bkt_name.bkt_id_task_->Wait(task); + bkt_name.bkt_id_ = bkt_name.bkt_id_task_->tag_id_; + op_data_map_.emplace(bkt_name.bkt_id_, OpPendingData()); + LABSTOR_CLIENT->DelTask(bkt_name.bkt_id_task_); + } + // Spawn bucket ID task for the output + op.var_name_.bkt_id_task_->Wait(task); + op.var_name_.bkt_id_ = op.var_name_.bkt_id_task_->tag_id_; + op_data_map_.emplace(op.var_name_.bkt_id_, OpPendingData()); + LABSTOR_CLIENT->DelTask(op.var_name_.bkt_id_task_); + } + + // Get number of operations that depend on each data object + for (Op &op : op_graph.ops_) { + for (OpBucketName &bkt_name : op.in_) { + op_data_map_[bkt_name.bkt_id_].num_refs_++; + } + } + + // Store the operator to perform + task->SetModuleComplete(); + } + + void RegisterData(RegisterDataTask *task, RunContext &rctx) { + if (!op_data_lock_.TryLock(0)) { + return; + } + OpPendingData &op_data = op_data_map_[task->data_.bkt_id_]; + task->data_.data_id_ = op_data.data_id_++; + op_data.pending_.emplace_back(task->data_); + op_data_lock_.Unlock(); + task->SetModuleComplete(); + } + + void RunOp(RunOpTask *task, RunContext &rctx) { + for (OpGraph &op_graph : op_graphs_[rctx.lane_id_]) { + for (Op &op : op_graph.ops_) { + switch(op.op_id_) { + case 0: + RunMin(task, op); + break; + case 1: + RunMax(task, op); + break; + } + } + } + task->SetModuleComplete(); + } + + std::list GetPendingData(Op &op) { + std::list pending; + if (!op_data_lock_.TryLock(0)) { + return pending; + } + for (OpBucketName &bkt_name : op.in_) { + OpPendingData &op_data = op_data_map_[bkt_name.bkt_id_]; + pending = op_data.pending_; + for (auto iter = pending.begin(); iter != pending.end(); ++iter) { + OpData &data = *iter; + ++data.refcnt_; + if (data.refcnt_ == op_data.num_refs_) { + op_data.pending_.erase(iter); + } + } + } + op_data_lock_.Unlock(); + return pending; + } + + void RunMin(RunOpTask *task, Op &op) { + // Get the pending data from Hermes + std::list op_data = GetPendingData(op); + if (op_data.empty()) { + return; + } + + // Get the input data from Hermes + typedef std::tuple, + LPointer> DataPair; + std::vector in_tasks; + for (OpData &data : op_data) { + // Get the input data + LPointer data_ptr = LABSTOR_CLIENT->AllocateBuffer(data.size_); + LPointer in_task = + blob_mdm_.AsyncGetBlob(task->task_node_ + 1, + data.bkt_id_, + hshm::charbuf(""), data.blob_id_, + data.off_, data.size_, + data_ptr.shm_); + in_tasks.emplace_back(data, data_ptr, in_task); + } + + // Calculate the min of input data + for (DataPair &data_pair : in_tasks) { + // Wait for data to be available + OpData &data = std::get<0>(data_pair); + LPointer &data_ptr = std::get<1>(data_pair); + LPointer &in_task = std::get<2>(data_pair); + in_task->Wait(task); + + // Calaculate the minimum + LPointer min_ptr = LABSTOR_CLIENT->AllocateBuffer(sizeof(float)); + float &min = *((float*)min_ptr.ptr_); + min = std::numeric_limits::max(); + for (size_t i = 0; i < in_task->data_size_; i += sizeof(float)) { + min = std::min(min, *(float*)(data_ptr.ptr_ + i)); + } + + // Store the minimum in Hermes + std::string min_blob_name = data.blob_name_ + "_min"; + blob_mdm_.AsyncPutBlob(task->task_node_ + 1, + data.bkt_id_, + hshm::charbuf(min_blob_name), + data.blob_id_, + 0, sizeof(float), + min_ptr.shm_, 0, 0); + } + } + + void RunMax(RunOpTask *task, Op &op) { + } + + public: +#include "hermes_data_op/hermes_data_op_lib_exec.h" +}; + +} // namespace hermes::data_op + +LABSTOR_TASK_CC(hermes::data_op::Server, "hermes_data_op"); diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index 145a8b8c3..b4ff009f5 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -69,10 +69,6 @@ class Server : public TaskLib { task->SetModuleComplete(); return; } - // Create the task queue for the state - QueueId qid(task->id_); - LABSTOR_QM_RUNTIME->CreateQueue( - qid, task->queue_info_->vec()); // Run the task state's submethod task->method_ = Method::kConstruct; bool ret = LABSTOR_TASK_REGISTRY->CreateTaskState( @@ -80,6 +76,10 @@ class Server : public TaskLib { state_name.c_str(), task->id_, task); + // Create the task queue for the state + QueueId qid(task->id_); + LABSTOR_QM_RUNTIME->CreateQueue( + qid, task->queue_info_->vec()); task->SetModuleComplete(); HILOG(kInfo, "(node {}) Allocated task state {} with id {}", LABSTOR_CLIENT->node_id_, state_name, task->task_state_); From 01855e8f4d63e6a4f64f0c6c5525816a77be0032 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 11 Oct 2023 07:37:19 -0500 Subject: [PATCH 162/191] Testing the bucket --- tasks/hermes/include/hermes/hermes.h | 5 + .../hermes_data_op/hermes_data_op_tasks.h | 6 + tasks/hermes_data_op/src/hermes_data_op.cc | 6 +- test/unit/hermes/test_bucket.cc | 132 ++++++++++++------ 4 files changed, 102 insertions(+), 47 deletions(-) diff --git a/tasks/hermes/include/hermes/hermes.h b/tasks/hermes/include/hermes/hermes.h index 874eabd05..b4c628004 100644 --- a/tasks/hermes/include/hermes/hermes.h +++ b/tasks/hermes/include/hermes/hermes.h @@ -34,6 +34,11 @@ class Hermes { return hermes::Bucket(path, ctx, backend_size, flags); } + /** Register an operation graph */ + void RegisterOp(hermes::data_op::OpGraph &op_graph) { + HERMES_CONF->op_mdm_.RegisterOpRoot(op_graph); + } + /** Clear all data from hermes */ void Clear() { // TODO(llogan) diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h index 74c6ffb9c..d796b2d04 100644 --- a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h @@ -28,6 +28,12 @@ struct OpBucketName { LPointer bkt_id_task_; // Task to get bucket ID (internal) + /** Default constructor */ + OpBucketName() = default; + + /** Emplace constructor */ + OpBucketName(const std::string &url) : url_(url) {} + template void serialize(Ar &ar) { ar(url_); diff --git a/tasks/hermes_data_op/src/hermes_data_op.cc b/tasks/hermes_data_op/src/hermes_data_op.cc index 4655903a8..d90fac73e 100644 --- a/tasks/hermes_data_op/src/hermes_data_op.cc +++ b/tasks/hermes_data_op/src/hermes_data_op.cc @@ -186,11 +186,11 @@ class Server : public TaskLib { } // Store the minimum in Hermes - std::string min_blob_name = data.blob_name_ + "_min"; + std::string min_blob_name = data.blob_name_; blob_mdm_.AsyncPutBlob(task->task_node_ + 1, - data.bkt_id_, + op.var_name_.bkt_id_, hshm::charbuf(min_blob_name), - data.blob_id_, + BlobId::GetNull(), 0, sizeof(float), min_ptr.shm_, 0, 0); } diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 6c87caa56..cd2d7ce0a 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -365,6 +365,67 @@ TEST_CASE("TestHermesBucketAppend1n") { MPI_Barrier(MPI_COMM_WORLD); } +TEST_CASE("TestHermesMultiGetBucket") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("append_test" + std::to_string(rank)); + u32 num_blobs = 1024; + + // Put a few blobs in the bucket + for (int i = 0; i < num_blobs; ++i) { + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + // Get a blob + hermes::Blob blob2; + bkt.Get(blob_id, blob2, ctx); + REQUIRE(blob.size() == blob2.size()); + REQUIRE(blob == blob2); + } + + // Get contained blob ids + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_CASE("TestHermesGetContainedBlobIds") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("append_test" + std::to_string(rank)); + u32 num_blobs = 1024; + + // Put a few blobs in the bucket + for (int i = 0; i < num_blobs; ++i) { + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + } + MPI_Barrier(MPI_COMM_WORLD); + + // Get contained blob ids + std::vector blob_ids; + blob_ids = bkt.GetContainedBlobIds(); + REQUIRE(blob_ids.size() == num_blobs); + MPI_Barrier(MPI_COMM_WORLD); +} + TEST_CASE("TestHermesDataStager") { int rank, nprocs; MPI_Barrier(MPI_COMM_WORLD); @@ -421,38 +482,8 @@ TEST_CASE("TestHermesDataStager") { // Verify staging happened } -TEST_CASE("TestHermesMultiGetBucket") { - int rank, nprocs; - MPI_Barrier(MPI_COMM_WORLD); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - // Initialize Hermes on all nodes - HERMES->ClientInit(); - // Create a bucket - hermes::Context ctx; - hermes::Bucket bkt("append_test" + std::to_string(rank)); - u32 num_blobs = 1024; - - // Put a few blobs in the bucket - for (int i = 0; i < num_blobs; ++i) { - hermes::Blob blob(KILOBYTES(4)); - memset(blob.data(), i % 256, blob.size()); - hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); - HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); - // Get a blob - hermes::Blob blob2; - bkt.Get(blob_id, blob2, ctx); - REQUIRE(blob.size() == blob2.size()); - REQUIRE(blob == blob2); - } - - // Get contained blob ids - MPI_Barrier(MPI_COMM_WORLD); -} - -TEST_CASE("TestHermesGetContainedBlobIds") { +TEST_CASE("TestHermesDataOp") { int rank, nprocs; MPI_Barrier(MPI_COMM_WORLD); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -461,23 +492,36 @@ TEST_CASE("TestHermesGetContainedBlobIds") { // Initialize Hermes on all nodes HERMES->ClientInit(); - // Create a bucket + // Create a bucket that supports derived quantities + using hermes::data_stager::BinaryFileStager; hermes::Context ctx; - hermes::Bucket bkt("append_test" + std::to_string(rank)); - u32 num_blobs = 1024; + std::string url = "data_bkt"; + hermes::Bucket bkt(url, 0, HERMES_HAS_DERIVED); + + // Create the derived quantity with + hermes::data_op::OpGraph op_graph; + hermes::data_op::Op op; + op.in_.emplace_back(url); + op.var_name_.url_ = url + "_min"; + op.op_name_ = "min"; + op_graph.ops_.emplace_back(op); + HERMES->RegisterOp(op_graph); + + size_t count_per_proc = 16; + size_t off = rank * count_per_proc; + size_t proc_count = off + count_per_proc; + size_t page_size = KILOBYTES(4); // Put a few blobs in the bucket - for (int i = 0; i < num_blobs; ++i) { - hermes::Blob blob(KILOBYTES(4)); + for (size_t i = off; i < proc_count; ++i) { + HILOG(kInfo, "Iteration: {}", i); + // Put a blob + hermes::Blob blob(page_size); memset(blob.data(), i % 256, blob.size()); - hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); - HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + std::string blob_name = std::to_string(i); + bkt.Put(blob_name, blob, ctx); } MPI_Barrier(MPI_COMM_WORLD); - // Get contained blob ids - std::vector blob_ids; - blob_ids = bkt.GetContainedBlobIds(); - REQUIRE(blob_ids.size() == num_blobs); - MPI_Barrier(MPI_COMM_WORLD); -} \ No newline at end of file + // Verify derived operator happens +} From 7d4938cbb53d7077cae2a2e0fb247d28b64d626f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 11 Oct 2023 08:13:11 -0500 Subject: [PATCH 163/191] Segfault at GetPendingData --- config/labstor_server_default.yaml | 1 + .../labstor/config/config_server_default.h | 1 + include/labstor/queue_manager/queue.h | 12 +++++++----- .../queue_manager/queue_manager_runtime.h | 14 +++++++++----- include/labstor/task_registry/task.h | 1 - .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 5 +---- .../hermes_data_op/hermes_data_op_tasks.h | 19 +++++++++++++++++-- tasks/hermes_data_op/src/hermes_data_op.cc | 2 +- .../labstor_admin/src/labstor_admin.cc | 9 +++++---- .../src/worch_queue_round_robin.cc | 2 +- test/unit/hermes/config/labstor_server.yaml | 1 + test/unit/hermes/test_bucket.cc | 1 + 12 files changed, 45 insertions(+), 23 deletions(-) diff --git a/config/labstor_server_default.yaml b/config/labstor_server_default.yaml index d3e12c4ed..977ea6c0c 100644 --- a/config/labstor_server_default.yaml +++ b/config/labstor_server_default.yaml @@ -50,6 +50,7 @@ task_registry: [ 'hermes_mdm', 'hermes_blob_mdm', 'hermes_bucket_mdm', + 'hermes_data_op', 'data_stager', 'posix_bdev', 'ram_bdev' diff --git a/include/labstor/config/config_server_default.h b/include/labstor/config/config_server_default.h index 08770e810..009f7e1f2 100644 --- a/include/labstor/config/config_server_default.h +++ b/include/labstor/config/config_server_default.h @@ -53,6 +53,7 @@ const char* kLabstorServerDefaultConfigStr = " \'hermes_mdm\',\n" " \'hermes_blob_mdm\',\n" " \'hermes_bucket_mdm\',\n" +" \'hermes_data_op\',\n" " \'data_stager\',\n" " \'posix_bdev\',\n" " \'ram_bdev\'\n" diff --git a/include/labstor/queue_manager/queue.h b/include/labstor/queue_manager/queue.h index 4f284ffd8..647ba69a7 100644 --- a/include/labstor/queue_manager/queue.h +++ b/include/labstor/queue_manager/queue.h @@ -9,16 +9,18 @@ #include "labstor/task_registry/task.h" #include +/** Requests in this queue can be processed in any order */ +#define QUEUE_READY BIT_OPT(u32, 0) /** This queue contains only latency-sensitive tasks */ -#define QUEUE_LOW_LATENCY BIT_OPT(u32, 0) +#define QUEUE_LOW_LATENCY BIT_OPT(u32, 1) /** This queue is currently being resized */ -#define QUEUE_RESIZE BIT_OPT(u32, 1) +#define QUEUE_RESIZE BIT_OPT(u32, 2) /** This queue is currently processing updates */ -#define QUEUE_UPDATE BIT_OPT(u32, 2) +#define QUEUE_UPDATE BIT_OPT(u32, 3) /** Requests in this queue can be processed in any order */ -#define QUEUE_UNORDERED BIT_OPT(u32, 3) +#define QUEUE_UNORDERED BIT_OPT(u32, 4) /** Requests in this queue are long-running */ -#define QUEUE_LONG_RUNNING BIT_OPT(u32, 3) +#define QUEUE_LONG_RUNNING BIT_OPT(u32, 5) namespace labstor { diff --git a/include/labstor/queue_manager/queue_manager_runtime.h b/include/labstor/queue_manager/queue_manager_runtime.h index 5b62031e7..aa43b50c3 100644 --- a/include/labstor/queue_manager/queue_manager_runtime.h +++ b/include/labstor/queue_manager/queue_manager_runtime.h @@ -46,29 +46,33 @@ class QueueManagerRuntime : public QueueManager { queue_map_ = shm.queue_map_.get(); queue_map_->resize(max_queues_); // Create the admin queue - CreateQueue(admin_queue_, { + MultiQueue *queue; + queue = CreateQueue(admin_queue_, { {1, 1, qm.queue_depth_, QUEUE_UNORDERED} }); - CreateQueue(process_queue_, { + queue->flags_.SetBits(QUEUE_READY); + queue = CreateQueue(process_queue_, { {1, 1, qm.queue_depth_, QUEUE_UNORDERED}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }); + queue->flags_.SetBits(QUEUE_READY); } /** Create a new queue (with pre-allocated ID) in the map */ - void CreateQueue(const QueueId &id, const std::vector &queue_info) { + MultiQueue* CreateQueue(const QueueId &id, const std::vector &queue_info) { MultiQueue *queue = GetQueue(id); if (id.IsNull()) { HELOG(kError, "Cannot create null queue {}", id); - return; + return nullptr; } if (!queue->id_.IsNull()) { HELOG(kError, "Queue {} already exists", id); - return; + return nullptr; } // HILOG(kDebug, "Creating queue {}", id); queue_map_->replace(queue_map_->begin() + id.unique_, id, queue_info); + return queue; } /** diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 96d00a2e2..318e5249f 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -402,7 +402,6 @@ struct Task : public hipc::ShmContainer { /** Determine if time has elapsed */ HSHM_ALWAYS_INLINE bool ShouldRun(hshm::Timepoint &cur_time) { - return true; if (!IsStarted()) { start_ = cur_time; return true; diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index bb9a50396..6bdb8af35 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -281,10 +281,7 @@ struct PutBlobTask : public Task, TaskFlags data_size_ = data_size; data_ = data; score_ = score; - flags_ = bitfield32_t(flags); - if (ctx.flags_.Any(HERMES_IS_FILE)) { - flags_.SetBits(HERMES_IS_FILE); - } + flags_ = bitfield32_t(flags | ctx.flags_.bits_); } /** Destructor */ diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h index d796b2d04..c159f8a61 100644 --- a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h @@ -159,7 +159,7 @@ struct RegisterOpTask : public Task, TaskFlags { /** Get opgraph */ OpGraph GetOpGraph() { OpGraph graph; - std::stringstream ss; + std::stringstream ss(op_graph_->str()); cereal::BinaryInputArchive ar(ss); ar >> graph; return graph; @@ -185,6 +185,7 @@ struct RegisterOpTask : public Task, TaskFlags { /** Duplicate message */ void Dup(hipc::Allocator *alloc, RegisterOpTask &other) { task_dup(other); + HSHM_MAKE_AR(op_graph_, alloc, *other.op_graph_); } /** Process duplicate message output */ @@ -263,7 +264,7 @@ struct RegisterDataTask : public Task, TaskFlags { /** * Run an operation over a piece of data * */ -struct RunOpTask : public Task, TaskFlags { +struct RunOpTask : public Task, TaskFlags { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit RunOpTask(hipc::Allocator *alloc) : Task(alloc) {} @@ -284,6 +285,20 @@ struct RunOpTask : public Task, TaskFlags { domain_id_ = DomainId::GetLocal(); } + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, RunOpTask &other) { + task_dup(other); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, RunOpTask &dup_task) {} + + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} + /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { diff --git a/tasks/hermes_data_op/src/hermes_data_op.cc b/tasks/hermes_data_op/src/hermes_data_op.cc index d90fac73e..b429672a6 100644 --- a/tasks/hermes_data_op/src/hermes_data_op.cc +++ b/tasks/hermes_data_op/src/hermes_data_op.cc @@ -38,6 +38,7 @@ class Server : public TaskLib { op_id_map_["min"] = 0; op_id_map_["max"] = 1; run_task_ = client_.AsyncRunOp(task->task_node_ + 1); + op_graphs_.resize(LABSTOR_QM_RUNTIME->max_lanes_); task->SetModuleComplete(); } @@ -122,7 +123,6 @@ class Server : public TaskLib { } } } - task->SetModuleComplete(); } std::list GetPendingData(Op &op) { diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index b4ff009f5..1d93d9605 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -69,6 +69,10 @@ class Server : public TaskLib { task->SetModuleComplete(); return; } + // Create the task queue for the state + QueueId qid(task->id_); + MultiQueue *queue = LABSTOR_QM_RUNTIME->CreateQueue( + qid, task->queue_info_->vec()); // Run the task state's submethod task->method_ = Method::kConstruct; bool ret = LABSTOR_TASK_REGISTRY->CreateTaskState( @@ -76,10 +80,7 @@ class Server : public TaskLib { state_name.c_str(), task->id_, task); - // Create the task queue for the state - QueueId qid(task->id_); - LABSTOR_QM_RUNTIME->CreateQueue( - qid, task->queue_info_->vec()); + queue->flags_.SetBits(QUEUE_READY); task->SetModuleComplete(); HILOG(kInfo, "(node {}) Allocated task state {} with id {}", LABSTOR_CLIENT->node_id_, state_name, task->task_state_); diff --git a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc index a50e01ffe..e1b484b11 100644 --- a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc +++ b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc @@ -25,7 +25,7 @@ class Server : public TaskLib { void Schedule(ScheduleTask *task, RunContext &rctx) { // Check if any new queues need to be scheduled for (MultiQueue &queue : *LABSTOR_QM_RUNTIME->queue_map_) { - if (queue.id_.IsNull()) { + if (queue.id_.IsNull() || !queue.flags_.Any(QUEUE_READY)) { continue; } for (LaneGroup &lane_group : *queue.groups_) { diff --git a/test/unit/hermes/config/labstor_server.yaml b/test/unit/hermes/config/labstor_server.yaml index e8d613c7c..cb4051d8d 100644 --- a/test/unit/hermes/config/labstor_server.yaml +++ b/test/unit/hermes/config/labstor_server.yaml @@ -49,6 +49,7 @@ task_registry: [ 'hermes_mdm', 'hermes_blob_mdm', 'hermes_bucket_mdm', + 'hermes_data_op', 'data_stager', 'posix_bdev', 'ram_bdev' diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index cd2d7ce0a..95cbea9b2 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -495,6 +495,7 @@ TEST_CASE("TestHermesDataOp") { // Create a bucket that supports derived quantities using hermes::data_stager::BinaryFileStager; hermes::Context ctx; + ctx.flags_.SetBits(HERMES_HAS_DERIVED); std::string url = "data_bkt"; hermes::Bucket bkt(url, 0, HERMES_HAS_DERIVED); From 849309166d2130c8e5b4e0f5ba11fc1693a088ce Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Wed, 11 Oct 2023 10:41:08 -0500 Subject: [PATCH 164/191] Min derived quantity is available --- tasks/hermes_data_op/src/hermes_data_op.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tasks/hermes_data_op/src/hermes_data_op.cc b/tasks/hermes_data_op/src/hermes_data_op.cc index b429672a6..a411394c6 100644 --- a/tasks/hermes_data_op/src/hermes_data_op.cc +++ b/tasks/hermes_data_op/src/hermes_data_op.cc @@ -133,12 +133,14 @@ class Server : public TaskLib { for (OpBucketName &bkt_name : op.in_) { OpPendingData &op_data = op_data_map_[bkt_name.bkt_id_]; pending = op_data.pending_; + std::list pruned; for (auto iter = pending.begin(); iter != pending.end(); ++iter) { OpData &data = *iter; ++data.refcnt_; - if (data.refcnt_ == op_data.num_refs_) { - op_data.pending_.erase(iter); + if (data.refcnt_ < op_data.num_refs_) { + pruned.push_back(data); } + op_data.pending_ = pruned; } } op_data_lock_.Unlock(); From 760f3ceac65a1a867214173c30d06d6a4f49a57a Mon Sep 17 00:00:00 2001 From: Luke Logan Date: Thu, 12 Oct 2023 08:06:52 -0500 Subject: [PATCH 165/191] Add flush task --- include/labstor/task_registry/task.h | 1 + include/labstor/work_orchestrator/worker.h | 1 + .../labstor_admin/labstor_admin_lib_exec.h | 44 +++++++++++++++ .../labstor_admin/labstor_admin_methods.h | 1 + .../labstor_admin/labstor_admin_methods.yaml | 3 +- .../labstor_admin/labstor_admin_tasks.h | 54 +++++++++++++++++++ .../labstor_admin/src/labstor_admin.cc | 4 ++ 7 files changed, 107 insertions(+), 1 deletion(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 318e5249f..609ee2f52 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -227,6 +227,7 @@ struct RunContext { size_t stack_size_ = KILOBYTES(64); /**< The size of the stack for the task (runtime) */ void *stack_ptr_; /**< The pointer to the stack (runtime) */ TaskLib *exec_; + int *flush_count_; /** Default constructor */ RunContext() {} diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index 493c89db7..e2f26dd4f 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -145,6 +145,7 @@ class Worker { bitfield32_t flags_; /** Worker metadata flags */ std::unordered_map group_map_; /** Determine if a task can be executed right now */ hshm::charbuf group_; /** The current group */ + int flush_count_ = 0; public: diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h index 1b26dae2c..9ac96d764 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h @@ -40,6 +40,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { SetWorkOrchProcPolicy(reinterpret_cast(task), rctx); break; } + case Method::kFlush: { + Flush(reinterpret_cast(task), rctx); + break; + } } } /** Delete a task */ @@ -81,6 +85,10 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kFlush: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } } } /** Duplicate a task */ @@ -122,6 +130,10 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } + case Method::kFlush: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } } } /** Register the duplicate output with the origin task */ @@ -163,6 +175,10 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } + case Method::kFlush: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } } } /** Ensure there is space to store replicated outputs */ @@ -204,6 +220,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kFlush: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } } } /** Determine success and handle failures */ @@ -245,6 +265,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kFlush: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } } } /** Serialize a task when initially pushing into remote */ @@ -286,6 +310,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kFlush: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -338,6 +366,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kFlush: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } } return task_ptr; } @@ -380,6 +413,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kFlush: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -422,6 +459,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kFlush: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } } } /** Get the grouping of the task */ @@ -454,6 +495,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kSetWorkOrchProcPolicy: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kFlush: { + return reinterpret_cast(task)->GetGroup(group); + } } return -1; } diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h index 6887eda8b..8921a31c3 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h @@ -12,6 +12,7 @@ struct Method : public TaskMethod { TASK_METHOD_T kStopRuntime = kLast + 6; TASK_METHOD_T kSetWorkOrchQueuePolicy = kLast + 7; TASK_METHOD_T kSetWorkOrchProcPolicy = kLast + 8; + TASK_METHOD_T kFlush = kLast + 9; }; #endif // LABSTOR_LABSTOR_ADMIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml index 016aa0455..a314374b5 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml @@ -6,4 +6,5 @@ kGetOrCreateTaskStateId: 4 kGetTaskStateId: 5 kStopRuntime: 6 kSetWorkOrchQueuePolicy: 7 -kSetWorkOrchProcPolicy: 8 \ No newline at end of file +kSetWorkOrchProcPolicy: 8 +kFlush: 9 \ No newline at end of file diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index 5c85a3aab..2e00206fb 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -416,6 +416,60 @@ struct SetWorkOrchestratorPolicyTask : public Task, TaskFlags { using SetWorkOrchQueuePolicyTask = SetWorkOrchestratorPolicyTask<0>; using SetWorkOrchProcPolicyTask = SetWorkOrchestratorPolicyTask<1>; +/** A task to destroy a Task state */ +struct FlushTask : public Task, TaskFlags { + /** SHM default constructor */ + FlushTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + FlushTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const DomainId &domain_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kAdmin; + task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + method_ = Method::kFlush; + task_flags_.SetBits(TASK_LANE_ALL); + domain_id_ = domain_id; + } + + /** Duplicate message */ + template + void Dup(hipc::Allocator *alloc, TaskT &other) { + task_dup(other); + } + + /** Process duplicate message output */ + template + void DupEnd(u32 replica, TaskT &dup_task) {} + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) {} + + /** Begin replication */ + void ReplicateStart(u32 count) {} + + /** Finalize replication */ + void ReplicateEnd() {} + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + + } // namespace labstor::Admin #endif // LABSTOR_TASKS_LABSTOR_ADMIN_INCLUDE_LABSTOR_ADMIN_LABSTOR_ADMIN_TASKS_H_ diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index 1d93d9605..755fee317 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -134,6 +134,10 @@ class Server : public TaskLib { task->SetModuleComplete(); } + void Flush(FlushTask *task, RunContext &rctx) { + + } + public: #include "labstor_admin/labstor_admin_lib_exec.h" }; From e3603de731503024c73965f99f791ec37798f896 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 12 Oct 2023 10:13:56 -0500 Subject: [PATCH 166/191] Flush didn't deadlock at least --- include/labstor/task_registry/task.h | 28 +++++++++++++++-- include/labstor/work_orchestrator/worker.h | 9 +++--- src/worker.cc | 31 ++++++++++++++----- .../include/labstor_admin/labstor_admin.h | 15 +++++++++ .../labstor_admin/labstor_admin_tasks.h | 2 +- .../labstor_admin/src/labstor_admin.cc | 21 ++++++++++++- .../include/remote_queue/remote_queue.h | 2 +- test/unit/ipc/test_ipc.cc | 29 +++++++++++++++++ 8 files changed, 119 insertions(+), 18 deletions(-) diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 609ee2f52..8eb39b003 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -49,6 +49,8 @@ class TaskLib; #define TASK_PREEMPTIVE BIT_OPT(u32, 17) /** This task should be scheduled on all lanes */ #define TASK_LANE_ALL BIT_OPT(u32, 19) +/** This task flushes the runtime */ +#define TASK_FLUSH BIT_OPT(u32, 20) /** This task is apart of remote debugging */ #define TASK_REMOTE_DEBUG_MARK BIT_OPT(u32, 31) @@ -220,6 +222,21 @@ class TaskPrio { TASK_PRIO_T kLowLatency = 2; }; + +/** Used to indicate the amount of work remaining to do when flushing */ +struct WorkPending { + bool flushing_; + std::atomic pending_; + + /** Default constructor */ + WorkPending() + : flushing_(false), pending_(0) {} + + /** Copy constructor */ + WorkPending(const WorkPending &other) + : flushing_(other.flushing_), pending_(other.pending_.load()) {} +}; + /** Context passed to the Run method of a task */ struct RunContext { u32 lane_id_; /**< The lane id of the task */ @@ -227,7 +244,7 @@ struct RunContext { size_t stack_size_ = KILOBYTES(64); /**< The size of the stack for the task (runtime) */ void *stack_ptr_; /**< The pointer to the stack (runtime) */ TaskLib *exec_; - int *flush_count_; + WorkPending *flush_; /** Default constructor */ RunContext() {} @@ -401,9 +418,14 @@ struct Task : public hipc::ShmContainer { period_ns_ = min * 60000000000; } + /** This task flushes the runtime */ + HSHM_ALWAYS_INLINE bool IsFlush() { + return task_flags_.Any(TASK_FLUSH); + } + /** Determine if time has elapsed */ - HSHM_ALWAYS_INLINE bool ShouldRun(hshm::Timepoint &cur_time) { - if (!IsStarted()) { + HSHM_ALWAYS_INLINE bool ShouldRun(hshm::Timepoint &cur_time, bool flushing) { + if (!IsStarted() || flushing) { start_ = cur_time; return true; } diff --git a/include/labstor/work_orchestrator/worker.h b/include/labstor/work_orchestrator/worker.h index e2f26dd4f..f1aaf7476 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/labstor/work_orchestrator/worker.h @@ -140,12 +140,13 @@ class Worker { hshm::spsc_queue> poll_queues_; /** A set of queues to stop polling in a worker */ hshm::spsc_queue> relinquish_queues_; - size_t sleep_us_; /** Time the worker should sleep after a run */ - u32 retries_; /** The number of times to repeat the internal run loop before sleeping */ + size_t sleep_us_; /** Time the worker should sleep after a run */ + u32 retries_; /** The number of times to repeat the internal run loop before sleeping */ bitfield32_t flags_; /** Worker metadata flags */ - std::unordered_map group_map_; /** Determine if a task can be executed right now */ + std::unordered_map + group_map_; /** Determine if a task can be executed right now */ hshm::charbuf group_; /** The current group */ - int flush_count_ = 0; + WorkPending flush_; /** Info needed for flushing ops */ public: diff --git a/src/worker.cc b/src/worker.cc index 559ffef58..e57aac0ad 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -13,7 +13,11 @@ void Worker::Loop() { WorkOrchestrator *orchestrator = LABSTOR_WORK_ORCHESTRATOR; while (orchestrator->IsAlive()) { try { + flush_.pending_ = 0; Run(); + if (flush_.flushing_ && flush_.pending_ == 0) { + flush_.flushing_ = false; + } } catch (hshm::Error &e) { HELOG(kFatal, "(node {}) Worker {} caught an error: {}", LABSTOR_CLIENT->node_id_, id_, e.what()); } @@ -32,12 +36,12 @@ void Worker::Run() { hshm::Timepoint now; now.Now(); for (WorkEntry &work_entry : work_queue_) { - if (!work_entry.lane_->flags_.Any(QUEUE_LOW_LATENCY)) { - work_entry.count_ += 1; - if (work_entry.count_ % 4096 != 0) { - continue; - } - } +// if (!work_entry.lane_->flags_.Any(QUEUE_LOW_LATENCY)) { +// work_entry.count_ += 1; +// if (work_entry.count_ % 4096 != 0) { +// continue; +// } +// } work_entry.cur_time_ = now; PollGrouped(work_entry); } @@ -60,6 +64,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { task = LABSTOR_CLIENT->GetPrivatePointer(entry->p_); RunContext &rctx = task->ctx_; rctx.lane_id_ = work_entry.lane_id_; + rctx.flush_ = &flush_; // Get the task state TaskState *&exec = rctx.exec_; exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); @@ -74,7 +79,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { bool is_remote = task->domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote) && - task->ShouldRun(work_entry.cur_time_)) { + task->ShouldRun(work_entry.cur_time_, flush_.flushing_)) { #ifdef REMOTE_DEBUG if (task->task_state_ != LABSTOR_QM_CLIENT->admin_task_state_ && !task->task_flags_.Any(TASK_REMOTE_DEBUG_MARK) && @@ -124,6 +129,9 @@ void Worker::PollGrouped(WorkEntry &work_entry) { exec->Run(task->method_, task, rctx); } task->DidRun(work_entry.cur_time_); + if (!task->IsModuleComplete() && !task->IsFlush()) { + flush_.pending_ += 1; + } } // Cleanup on task completion if (task->IsModuleComplete()) { @@ -150,20 +158,27 @@ void Worker::RunCoroutine(bctx::transfer_t t) { rctx.jmp_ = t; exec->Run(task->method_, task, rctx); task->UnsetStarted(); + if (task->IsLongRunning()) { + rctx.flush_->pending_ -= 1; + } task->Yield(); } void Worker::RunPreemptive(void *data) { Task *task = reinterpret_cast(data); TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); + RunContext &real_rctx = task->ctx_; RunContext rctx(0); do { hshm::Timepoint now; now.Now(); - if (task->ShouldRun(now)) { + if (task->ShouldRun(now, real_rctx.flush_->flushing_)) { exec->Run(task->method_, task, rctx); task->DidRun(now); } + if (task->IsLongRunning()) { + real_rctx.flush_->pending_ -= 1; + } task->Yield(); } while(!task->IsModuleComplete()); } diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h index 8fc0ab097..151880126 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h @@ -215,6 +215,21 @@ class Client : public TaskLibClient { LABSTOR_CLIENT->DelTask(task); } LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchProcPolicy); + + /** Flush the runtime */ + void AsyncFlushConstruct(FlushTask *task, + const TaskNode &task_node, + const DomainId &domain_id) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, domain_id); + } + void FlushRoot(const DomainId &domain_id) { + LPointer task = + AsyncFlushRoot(domain_id); + task->Wait(); + LABSTOR_CLIENT->DelTask(task); + } + LABSTOR_TASK_NODE_ADMIN_ROOT(Flush); }; } // namespace labstor::Admin diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h index 2e00206fb..a64b112a3 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h @@ -432,7 +432,7 @@ struct FlushTask : public Task, TaskFlags { prio_ = TaskPrio::kAdmin; task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; method_ = Method::kFlush; - task_flags_.SetBits(TASK_LANE_ALL); + task_flags_.SetBits(TASK_FLUSH | TASK_COROUTINE); domain_id_ = domain_id; } diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/labstor_admin/src/labstor_admin.cc index 755fee317..bd91dac4a 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/labstor_admin/src/labstor_admin.cc @@ -135,7 +135,26 @@ class Server : public TaskLib { } void Flush(FlushTask *task, RunContext &rctx) { - + HILOG(kDebug, "Beginning to flush runtime"); + for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { + worker.flush_.flushing_ = true; + } + while (true) { + int count = 0; + for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { + if (worker.flush_.flushing_) { + count += 1; + break; + } + } + if (count) { + task->Yield(); + } else { + break; + } + } + HILOG(kDebug, "Flushing completed"); + task->SetModuleComplete(); } public: diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index c55acfbe8..a8d3d1bc8 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -77,7 +77,7 @@ class Client : public TaskLibClient { orig_task->task_node_ + 1, DomainId::GetLocal(), id_, domain_ids, orig_task, exec, orig_task->method_, xfer); MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); - queue->Emplace(orig_task->prio_, orig_task->lane_hash_, push_task.shm_); + queue->Emplace(TaskPrio::kLowLatency, orig_task->lane_hash_, push_task.shm_); } /** Disperse a task among each lane of this node */ diff --git a/test/unit/ipc/test_ipc.cc b/test/unit/ipc/test_ipc.cc index 41ecaf5ee..643af0c59 100644 --- a/test/unit/ipc/test_ipc.cc +++ b/test/unit/ipc/test_ipc.cc @@ -40,6 +40,35 @@ TEST_CASE("TestIpc") { HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } +TEST_CASE("TestFlush") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + labstor::small_message::Client client; + LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + MPI_Barrier(MPI_COMM_WORLD); + hshm::Timer t; + + int pid = getpid(); + ProcessAffiner::SetCpuAffinity(pid, 8); + + t.Resume(); + size_t ops = 256; + for (size_t i = 0; i < ops; ++i) { + int ret; + HILOG(kInfo, "Sending message {}", i); + int node_id = 1 + ((rank + 1) % nprocs); + LPointer task = + client.AsyncMdRoot(labstor::DomainId::GetNode(node_id)); + } + LABSTOR_ADMIN->FlushRoot(DomainId::GetGlobal()); + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + void TestIpcMultithread(int nprocs) { labstor::small_message::Client client; LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); From 85a0b8af945daa48f3129fa0b1d3df88d78b5a9c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 12 Oct 2023 13:24:22 -0500 Subject: [PATCH 167/191] Add check for the derived op --- test/unit/hermes/test_bucket.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 95cbea9b2..e2de3ef9c 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -524,5 +524,9 @@ TEST_CASE("TestHermesDataOp") { } MPI_Barrier(MPI_COMM_WORLD); + // LABSTOR_ADMIN->FlushRoot(DomainId::GetGlobal()); // Verify derived operator happens + hermes::Bucket bkt_min("data_bkt_min", 0, 0); + size_t size = bkt_min.GetSize(); + REQUIRE(size == sizeof(float) * count_per_proc * nprocs); } From 5ef9b25121272fd5b0317396b82d9d27c6b7db3e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 13 Oct 2023 14:29:54 -0500 Subject: [PATCH 168/191] Add full libfabric to hermes_shm --- ci/hermes/packages/hermes_shm/package.py | 4 + op_min_max/CMakeLists.txt | 10 - op_min_max/include/op_min_max/op_min_max.h | 73 ------- .../include/op_min_max/op_min_max_lib_exec.h | 197 ------------------ .../include/op_min_max/op_min_max_methods.h | 9 - .../op_min_max/op_min_max_methods.yaml | 1 - .../include/op_min_max/op_min_max_tasks.h | 135 ------------ op_min_max/src/CMakeLists.txt | 54 ----- op_min_max/src/op_min_max.cc | 33 --- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 12 ++ 10 files changed, 16 insertions(+), 512 deletions(-) delete mode 100644 op_min_max/CMakeLists.txt delete mode 100644 op_min_max/include/op_min_max/op_min_max.h delete mode 100644 op_min_max/include/op_min_max/op_min_max_lib_exec.h delete mode 100644 op_min_max/include/op_min_max/op_min_max_methods.h delete mode 100644 op_min_max/include/op_min_max/op_min_max_methods.yaml delete mode 100644 op_min_max/include/op_min_max/op_min_max_tasks.h delete mode 100644 op_min_max/src/CMakeLists.txt delete mode 100644 op_min_max/src/op_min_max.cc diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index 9ad520eda..6b93443fb 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -4,6 +4,8 @@ class HermesShm(CMakePackage): homepage = "https://github.com/lukemartinlogan/hermes_shm/wiki" git = "https://github.com/lukemartinlogan/hermes_shm.git" version('master', branch='master') + variant('ares', default=False, description='Enable full libfabric install') + depends_on('mochi-thallium~cereal@0.10.1') depends_on('catch2@3.0.1') # depends_on('mpi') @@ -12,6 +14,8 @@ class HermesShm(CMakePackage): depends_on('cereal') depends_on('yaml-cpp') depends_on('doxygen@1.9.3') + depends_on('libfabric fabrics=sockets,tcp,udp,efa,gni,mlx,mrail,psm,psm2,psm3,rxm,rxd,shm,usnix,verbs,xpmem', + when='+ares') variant('debug', default=False, description='Build shared libraries') diff --git a/op_min_max/CMakeLists.txt b/op_min_max/CMakeLists.txt deleted file mode 100644 index 5b409c224..000000000 --- a/op_min_max/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Labstor Admin Task Library -#------------------------------------------------------------------------------ -include_directories(include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/op_min_max/include/op_min_max/op_min_max.h b/op_min_max/include/op_min_max/op_min_max.h deleted file mode 100644 index e85e8b4fd..000000000 --- a/op_min_max/include/op_min_max/op_min_max.h +++ /dev/null @@ -1,73 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#ifndef LABSTOR_op_min_max_H_ -#define LABSTOR_op_min_max_H_ - -#include "op_min_max_tasks.h" - -namespace labstor::op_min_max { - -/** Create op_min_max requests */ -class Client : public TaskLibClient { - - public: - /** Default constructor */ - Client() = default; - - /** Destructor */ - ~Client() = default; - - /** Async create a task state */ - HSHM_ALWAYS_INLINE - LPointer AsyncCreate(const TaskNode &task_node, - const DomainId &domain_id, - const std::string &state_name) { - id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; - std::vector queue_info = { - {1, 1, qm.queue_depth_, 0}, - {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, - {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} - }; - return LABSTOR_ADMIN->AsyncCreateTaskState( - task_node, domain_id, state_name, id_, queue_info); - } - LABSTOR_TASK_NODE_ROOT(AsyncCreate) - template - HSHM_ALWAYS_INLINE - void CreateRoot(Args&& ...args) { - LPointer task = - AsyncCreateRoot(std::forward(args)...); - task->Wait(); - id_ = task->id_; - queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); - } - - /** Destroy task state + queue */ - HSHM_ALWAYS_INLINE - void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); - } - - /** Call a custom method */ - HSHM_ALWAYS_INLINE - void AsyncCustomConstruct(CustomTask *task, - const TaskNode &task_node, - const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( - task, task_node, domain_id, id_); - } - HSHM_ALWAYS_INLINE - void CustomRoot(const DomainId &domain_id) { - LPointer> task = AsyncCustomRoot(domain_id); - task.ptr_->Wait(); - } - LABSTOR_TASK_NODE_PUSH_ROOT(Custom); -}; - -} // namespace labstor - -#endif // LABSTOR_op_min_max_H_ diff --git a/op_min_max/include/op_min_max/op_min_max_lib_exec.h b/op_min_max/include/op_min_max/op_min_max_lib_exec.h deleted file mode 100644 index d84dd08a7..000000000 --- a/op_min_max/include/op_min_max/op_min_max_lib_exec.h +++ /dev/null @@ -1,197 +0,0 @@ -#ifndef LABSTOR_op_min_max_LIB_EXEC_H_ -#define LABSTOR_op_min_max_LIB_EXEC_H_ - -/** Execute a task */ -void Run(u32 method, Task *task, RunContext &rctx) override { - switch (method) { - case Method::kConstruct: { - Construct(reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - Destruct(reinterpret_cast(task), rctx); - break; - } - case Method::kCustom: { - Custom(reinterpret_cast(task), rctx); - break; - } - } -} -/** Delete a task */ -void Del(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kCustom: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - } -} -/** Duplicate a task */ -void Dup(u32 method, Task *orig_task, std::vector> &dups) override { - switch (method) { - case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kCustom: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - } -} -/** Register the duplicate output with the origin task */ -void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { - switch (method) { - case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kCustom: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - } -} -/** Ensure there is space to store replicated outputs */ -void ReplicateStart(u32 method, u32 count, Task *task) override { - switch (method) { - case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kCustom: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - } -} -/** Determine success and handle failures */ -void ReplicateEnd(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kCustom: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - } -} -/** Serialize a task when initially pushing into remote */ -std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kCustom: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when popping from remote queue */ -TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { - TaskPointer task_ptr; - switch (method) { - case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kCustom: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - } - return task_ptr; -} -/** Serialize a task when returning from remote queue */ -std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kCustom: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when returning from remote queue */ -void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kCustom: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - } -} -/** Get the grouping of the task */ -u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { - switch (method) { - case Method::kConstruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kCustom: { - return reinterpret_cast(task)->GetGroup(group); - } - } - return -1; -} - -#endif // LABSTOR_op_min_max_METHODS_H_ \ No newline at end of file diff --git a/op_min_max/include/op_min_max/op_min_max_methods.h b/op_min_max/include/op_min_max/op_min_max_methods.h deleted file mode 100644 index 4bed8afcd..000000000 --- a/op_min_max/include/op_min_max/op_min_max_methods.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef LABSTOR_op_min_max_METHODS_H_ -#define LABSTOR_op_min_max_METHODS_H_ - -/** The set of methods in the admin task */ -struct Method : public TaskMethod { - TASK_METHOD_T kCustom = kLast + 0; -}; - -#endif // LABSTOR_op_min_max_METHODS_H_ \ No newline at end of file diff --git a/op_min_max/include/op_min_max/op_min_max_methods.yaml b/op_min_max/include/op_min_max/op_min_max_methods.yaml deleted file mode 100644 index b1b54e2a4..000000000 --- a/op_min_max/include/op_min_max/op_min_max_methods.yaml +++ /dev/null @@ -1 +0,0 @@ -kCustom: 0 \ No newline at end of file diff --git a/op_min_max/include/op_min_max/op_min_max_tasks.h b/op_min_max/include/op_min_max/op_min_max_tasks.h deleted file mode 100644 index 97afcb865..000000000 --- a/op_min_max/include/op_min_max/op_min_max_tasks.h +++ /dev/null @@ -1,135 +0,0 @@ -// -// Created by lukemartinlogan on 8/11/23. -// - -#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_op_min_max_op_min_max_TASKS_H_ -#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_op_min_max_op_min_max_TASKS_H_ - -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" -#include "proc_queue/proc_queue.h" - -namespace labstor::op_min_max { - -#include "op_min_max_methods.h" -#include "labstor/labstor_namespace.h" -using labstor::proc_queue::TypedPushTask; -using labstor::proc_queue::PushTask; - -/** - * A task to create op_min_max - * */ -using labstor::Admin::CreateTaskStateTask; -struct ConstructTask : public CreateTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(hipc::Allocator *alloc) - : CreateTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(hipc::Allocator *alloc, - const TaskNode &task_node, - const DomainId &domain_id, - const std::string &state_name, - const TaskStateId &id, - const std::vector &queue_info) - : CreateTaskStateTask(alloc, task_node, domain_id, state_name, - "op_min_max", id, queue_info) { - // Custom params - } - - HSHM_ALWAYS_INLINE - ~ConstructTask() { - // Custom params - } -}; - -/** A task to destroy op_min_max */ -using labstor::Admin::DestroyTaskStateTask; -struct DestructTask : public DestroyTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(hipc::Allocator *alloc) - : DestroyTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(hipc::Allocator *alloc, - const TaskNode &task_node, - const DomainId &domain_id, - TaskStateId &state_id) - : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * A custom task in op_min_max - * */ -struct CustomTask : public Task, TaskFlags { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - CustomTask(hipc::Allocator *alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - CustomTask(hipc::Allocator *alloc, - const TaskNode &task_node, - const DomainId &domain_id, - const TaskStateId &state_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kLowLatency; - task_state_ = state_id; - method_ = Method::kCustom; - task_flags_.SetBits(0); - domain_id_ = domain_id; - - // Custom params - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Duplicate message */ - void Dup(hipc::Allocator *alloc, CustomTask &other) { - task_dup(other); - } - - /** Process duplicate message output */ - void DupEnd(u32 replica, CustomTask &dup_task) { - } - - /** Begin replication */ - void ReplicateStart(u32 count) { - } - - /** Finalize replication */ - void ReplicateEnd() {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } -}; - -} // namespace labstor::op_min_max - -#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_op_min_max_op_min_max_TASKS_H_ diff --git a/op_min_max/src/CMakeLists.txt b/op_min_max/src/CMakeLists.txt deleted file mode 100644 index dcd2cf9d5..000000000 --- a/op_min_max/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(op_min_max SHARED - op_min_max.cc) -add_dependencies(op_min_max ${Hermes_RUNTIME_DEPS}) -target_link_libraries(op_min_max ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - op_min_max - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - op_min_max - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(op_min_max) -endif() diff --git a/op_min_max/src/op_min_max.cc b/op_min_max/src/op_min_max.cc deleted file mode 100644 index 51bf94d33..000000000 --- a/op_min_max/src/op_min_max.cc +++ /dev/null @@ -1,33 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" -#include "op_min_max/op_min_max.h" - -namespace labstor::op_min_max { - -class Server : public TaskLib { - public: - Server() = default; - - void Construct(ConstructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - - void Destruct(DestructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - - void Custom(CustomTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - - public: -#include "op_min_max/op_min_max_lib_exec.h" -}; - -} // namespace labstor::op_min_max - -LABSTOR_TASK_CC(labstor::op_min_max::Server, "op_min_max"); diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 302f9cca8..6a5d056e3 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -653,6 +653,18 @@ class Server : public TaskLib { } } + /** + * Get \a blob_name BLOB from \a bkt_id bucket + * */ + HSHM_ALWAYS_INLINE + void PollBlobMetadata(GetBlobIdTask *task, RunContext &rctx) { + BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; + for (const std::pair &blob_part : blob_map) { + const BlobInfo &blob_info = blob_part.second; + } + task->SetModuleComplete(); + } + public: #include "hermes_blob_mdm/hermes_blob_mdm_lib_exec.h" }; From ec3ac69261ea07c26fd0e8164986016018a461d7 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 13 Oct 2023 14:34:42 -0500 Subject: [PATCH 169/191] Fix fabric +ares --- ci/hermes/packages/hermes_shm/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index 6b93443fb..4517d6ebc 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -14,7 +14,7 @@ class HermesShm(CMakePackage): depends_on('cereal') depends_on('yaml-cpp') depends_on('doxygen@1.9.3') - depends_on('libfabric fabrics=sockets,tcp,udp,efa,gni,mlx,mrail,psm,psm2,psm3,rxm,rxd,shm,usnix,verbs,xpmem', + depends_on('libfabric fabrics=sockets,tcp,udp,efa,gni,mlx,mrail,psm,psm2,psm3,rxm,rxd,shm,usnic,verbs,xpmem', when='+ares') variant('debug', default=False, description='Build shared libraries') From 35be57b43a4e8c1376c468d667f51be7f1b230a0 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 13 Oct 2023 14:39:37 -0500 Subject: [PATCH 170/191] Remove some stuff from libfabric --- ci/hermes/packages/hermes_shm/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index 4517d6ebc..d5efd8410 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -14,7 +14,7 @@ class HermesShm(CMakePackage): depends_on('cereal') depends_on('yaml-cpp') depends_on('doxygen@1.9.3') - depends_on('libfabric fabrics=sockets,tcp,udp,efa,gni,mlx,mrail,psm,psm2,psm3,rxm,rxd,shm,usnic,verbs,xpmem', + depends_on('libfabric fabrics=sockets,tcp,udp,rxm,rxd,shm,verbs', when='+ares') variant('debug', default=False, description='Build shared libraries') From 7d8f27abea378ba89d8649de1422771442d44438 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 13 Oct 2023 14:47:38 -0500 Subject: [PATCH 171/191] Simplify fabric install --- ci/hermes/packages/hermes_shm/package.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index d5efd8410..c70722f52 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -5,8 +5,10 @@ class HermesShm(CMakePackage): git = "https://github.com/lukemartinlogan/hermes_shm.git" version('master', branch='master') variant('ares', default=False, description='Enable full libfabric install') + variant('only_verbs', default=False, description='Only verbs') - depends_on('mochi-thallium~cereal@0.10.1') + depends_on('mochi-thallium~cereal@0.10.1 +ofi') + depends_on('mercury+ofi') depends_on('catch2@3.0.1') # depends_on('mpi') depends_on('mpich@3.3.2') @@ -14,8 +16,10 @@ class HermesShm(CMakePackage): depends_on('cereal') depends_on('yaml-cpp') depends_on('doxygen@1.9.3') - depends_on('libfabric fabrics=sockets,tcp,udp,rxm,rxd,shm,verbs', + depends_on('libfabric fabrics=sockets,tcp,udp,rxm,rxd,verbs', when='+ares') + depends_on('libfabric fabrics=verbs', + when='+only_verbs') variant('debug', default=False, description='Build shared libraries') From 0a1d49db1a8663621f9ab08305c18a20ed3a14c5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Fri, 13 Oct 2023 14:48:50 -0500 Subject: [PATCH 172/191] Simplify fabric install --- ci/hermes/packages/hermes_shm/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index c70722f52..0ba3d2b1c 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -7,8 +7,7 @@ class HermesShm(CMakePackage): variant('ares', default=False, description='Enable full libfabric install') variant('only_verbs', default=False, description='Only verbs') - depends_on('mochi-thallium~cereal@0.10.1 +ofi') - depends_on('mercury+ofi') + depends_on('mochi-thallium~cereal@0.10.1') depends_on('catch2@3.0.1') # depends_on('mpi') depends_on('mpich@3.3.2') From c09bcabcab1493c1873456b1cd0e10cb28b7ba26 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 17:34:48 -0500 Subject: [PATCH 173/191] First draft of poll blob info query --- README.md | 24 ++--- include/labstor/labstor_types.h | 13 --- include/labstor/network/serialize.h | 21 +++- include/labstor/task_registry/task.h | 2 + tasks/hermes/include/hermes/hermes.h | 11 ++ tasks/hermes/include/hermes/hermes_types.h | 20 +++- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 20 ++++ .../hermes_blob_mdm_lib_exec.h | 44 ++++++++ .../hermes_blob_mdm/hermes_blob_mdm_methods.h | 1 + .../hermes_blob_mdm_methods.yaml | 3 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 101 ++++++++++++++++++ tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 10 +- test/unit/hermes/test_bucket.cc | 30 +++++- 13 files changed, 264 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index d96b02b9b..dec668244 100644 --- a/README.md +++ b/README.md @@ -36,22 +36,14 @@ spack install hermes@dev-1.1 Hermes makes use of the CMake build system and requires an out of source build. - cd /path/to/hermes - mkdir build - cd build - ccmake .. - -Type 'c' to configure until there are no errors, then generate the makefile with 'g'. The default options should suffice for most use cases. In addition, we recommend the following options. - - -DCMAKE_INSTALL_PREFIX=/installation/prefix - -DCMAKE_PREFIX_PATH=/path/to/dependencies - -DCMAKE_BUILD_RPATH=/path/to/dependencies/lib - -DCMAKE_INSTALL_RPATH=/path/to/dependencies/lib - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_COMPILER=`which mpicxx` - -DBUILD_SHARED_LIBS=ON - -After the makefile has been generated, you can type `make -j 4` or `cmake --build . -- -j 4`. Add `VERBOSE=1` to see detailed compiler output. +``` +cd /path/to/hermes +mkdir build +cd build +cmake ../ -DCMAKE_BUILD_TYPE=Relase -DCMAKE_INSTALL_PREFIX=... +make -j8 +make install +``` ### Testing and Installation diff --git a/include/labstor/labstor_types.h b/include/labstor/labstor_types.h index 26f9dae54..9d01597b3 100644 --- a/include/labstor/labstor_types.h +++ b/include/labstor/labstor_types.h @@ -340,19 +340,6 @@ struct UniqueId { } }; -/** A sized data pointer */ -template -struct DataPointer { - hipc::Pointer ptr_; - size_t size_; - - /** Serialization */ - template - void serialize(Ar &ar) { - ar(ptr_, size_); - } -}; - /** Uniquely identify a task state */ using TaskStateId = UniqueId<1>; /** Uniquely identify a queue */ diff --git a/include/labstor/network/serialize.h b/include/labstor/network/serialize.h index ccdaf76da..1336df3c8 100644 --- a/include/labstor/network/serialize.h +++ b/include/labstor/network/serialize.h @@ -14,6 +14,7 @@ #include #include #include +#include namespace labstor { @@ -76,6 +77,13 @@ struct DataTransferBase { } }; +/** A sized data pointer indicating data I/O direction */ +struct DataPointer { + hshm::bitfield32_t flags_; /**< Indicates how data will be accessed */ + hipc::Pointer data_; /**< The SHM address of data on the node */ + size_t data_size_; /**< The amount of data to transfer */ +}; + using DataTransfer = DataTransferBase; using PassDataTransfer = DataTransferBase; @@ -152,6 +160,12 @@ class BinaryOutputArchive { } else if constexpr (std::is_same_v){ var.node_id_ = node_id_; xfer_.emplace_back(var); + } else if constexpr(std::is_same_v) { + DataTransfer xfer; + xfer.flags_ = var.flags_; + xfer.data_ = HERMES_MEMORY_MANAGER->Convert(var.data_); + xfer.data_size_ = var.size_; + xfer_.emplace_back(var); } else { ar_ << var; } @@ -244,7 +258,12 @@ class BinaryInputArchive { } } else if constexpr (std::is_same_v) { var = xfer_[xfer_off_++]; - } else { + } else if constexpr(std::is_same_v) { + DataTransfer &xfer = xfer_[xfer_off_++]; + var.shm_ = HERMES_MEMORY_MANAGER->Convert(xfer.data_); + var.size_ = xfer.data_size_; + var.flags_ = xfer.flags_; + } else { ar_ >> var; } return Deserialize(replica, std::forward(args)...); diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index 8eb39b003..f44b10b9a 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -47,6 +47,8 @@ class TaskLib; #define TASK_COROUTINE BIT_OPT(u32, 15) /** This task uses argobot wait */ #define TASK_PREEMPTIVE BIT_OPT(u32, 17) +/** This task can be scheduled on any lane */ +#define TASK_LANE_ANY BIT_OPT(u32, 18) /** This task should be scheduled on all lanes */ #define TASK_LANE_ALL BIT_OPT(u32, 19) /** This task flushes the runtime */ diff --git a/tasks/hermes/include/hermes/hermes.h b/tasks/hermes/include/hermes/hermes.h index b4c628004..5318d7a6b 100644 --- a/tasks/hermes/include/hermes/hermes.h +++ b/tasks/hermes/include/hermes/hermes.h @@ -9,6 +9,10 @@ namespace hermes { +struct MetadataTable { + std::vector blob_info_; +}; + class Hermes { public: /** Init hermes client */ @@ -26,6 +30,13 @@ class Hermes { return HERMES_CONF->bkt_mdm_.GetTagIdRoot(hshm::to_charbuf(tag_name)); } + /** Collect a snapshot of all metadata in Hermes */ + MetadataTable CollectMetadataSnapshot() { + MetadataTable table; + table.blob_info_ = HERMES_CONF->blob_mdm_.PollBlobMetadataRoot(); + return table; + } + /** Get or create a bucket */ hermes::Bucket GetBucket(const std::string &path, Context ctx = Context(), diff --git a/tasks/hermes/include/hermes/hermes_types.h b/tasks/hermes/include/hermes/hermes_types.h index 12da20ed5..e39fa5935 100644 --- a/tasks/hermes/include/hermes/hermes_types.h +++ b/tasks/hermes/include/hermes/hermes_types.h @@ -275,27 +275,45 @@ struct BlobInfo { std::atomic mod_count_; /**< The number of times blob modified */ std::atomic last_flush_; /**< The last mod that was flushed */ + /** Serialization */ + template + void serialize(Ar &ar) { + ar(tag_id_, blob_id_, name_, buffers_, tags_, blob_size_, max_blob_size_, + score_, access_freq_, last_access_, mod_count_, last_flush_); + } + + /** Default constructor */ BlobInfo() = default; + /** Copy constructor */ BlobInfo(const BlobInfo &other) { - blob_id_ = other.blob_id_; tag_id_ = other.tag_id_; + blob_id_ = other.blob_id_; + name_ = other.name_; + buffers_ = other.buffers_; + tags_ = other.tags_; blob_size_ = other.blob_size_; + max_blob_size_ = other.max_blob_size_; score_ = other.score_; + access_freq_ = other.access_freq_.load(); + last_access_ = other.last_access_; mod_count_ = other.mod_count_.load(); last_flush_ = other.last_flush_.load(); } + /** Update modify stats */ void UpdateWriteStats() { mod_count_.fetch_add(1); UpdateReadStats(); } + /** Update read stats */ void UpdateReadStats() { last_access_ = GetTimeFromStartNs(); access_freq_.fetch_add(1); } + /** Get the time from start in nanoseconds */ static u64 GetTimeFromStartNs() { struct timespec currentTime; clock_gettime(CLOCK_MONOTONIC, ¤tTime); diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 088b94531..1442540dd 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -435,6 +435,26 @@ class Client : public TaskLibClient { task, task_node, id_); } LABSTOR_TASK_NODE_PUSH_ROOT(FlushData); + + /** + * Get all blob metadata + * */ + void AsyncPollBlobMetadataConstruct(PollBlobMetadataTask *task, + const TaskNode &task_node) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_); + } + std::vector PollBlobMetadataRoot() { + LPointer> push_task = + AsyncPollBlobMetadataRoot(); + push_task->Wait(); + PollBlobMetadataTask *task = push_task->get(); + std::vector blob_mdms = + task->DeserializeBlobMetadata(); + LABSTOR_CLIENT->DelTask(push_task); + return blob_mdms; + } + LABSTOR_TASK_NODE_PUSH_ROOT(PollBlobMetadata); }; } // namespace labstor diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index b1ac48623..250b2f27c 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -76,6 +76,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { FlushData(reinterpret_cast(task), rctx); break; } + case Method::kPollBlobMetadata: { + PollBlobMetadata(reinterpret_cast(task), rctx); + break; + } } } /** Delete a task */ @@ -153,6 +157,10 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kPollBlobMetadata: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } } } /** Duplicate a task */ @@ -230,6 +238,10 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } + case Method::kPollBlobMetadata: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } } } /** Register the duplicate output with the origin task */ @@ -307,6 +319,10 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } + case Method::kPollBlobMetadata: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } } } /** Ensure there is space to store replicated outputs */ @@ -384,6 +400,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kPollBlobMetadata: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } } } /** Determine success and handle failures */ @@ -461,6 +481,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kPollBlobMetadata: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } } } /** Serialize a task when initially pushing into remote */ @@ -538,6 +562,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kPollBlobMetadata: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -635,6 +663,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kPollBlobMetadata: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } } return task_ptr; } @@ -713,6 +746,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kPollBlobMetadata: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -791,6 +828,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kPollBlobMetadata: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } } } /** Get the grouping of the task */ @@ -850,6 +891,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kFlushData: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kPollBlobMetadata: { + return reinterpret_cast(task)->GetGroup(group); + } } return -1; } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h index 3ee9d30f6..2af72faea 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h @@ -19,6 +19,7 @@ struct Method : public TaskMethod { TASK_METHOD_T kReorganizeBlob = kLast + 15; TASK_METHOD_T kSetBucketMdm = kLast + 16; TASK_METHOD_T kFlushData = kLast + 17; + TASK_METHOD_T kPollBlobMetadata = kLast + 18; }; #endif // LABSTOR_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml index b53850223..ab4524951 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml @@ -15,4 +15,5 @@ kGetBlobBuffers: 13 kRenameBlob: 14 kReorganizeBlob: 15 kSetBucketMdm: 16 -kFlushData: 17 \ No newline at end of file +kFlushData: 17 +kPollBlobMetadata: 18 \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 6bdb8af35..c4a3120d6 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -1194,6 +1194,107 @@ struct FlushDataTask : public Task, TaskFlags { } }; +/** A task to collect blob metadata */ +struct PollBlobMetadataTask : public Task, TaskFlags { + OUT hipc::ShmArchive> blob_mdms_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + PollBlobMetadataTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + PollBlobMetadataTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kPollBlobMetadata; + task_flags_.SetBits(TASK_LANE_ALL); + domain_id_ = DomainId::GetGlobal(); + + // Custom params + HSHM_MAKE_AR0(blob_mdms_, alloc) + blob_mdms_->reserve(1); + } + + /** Serialize blob info */ + void SerializeBlobMetadata(const std::vector &blob_info) { + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar << blob_info; + blob_mdms_->emplace_back(ss.str()); + } + + /** Deserialize blob info */ + void DeserializeBlobMetadata(const std::string &srl, std::vector &blob_mdms) { + std::vector tmp_blob_mdms; + std::stringstream ss(srl); + cereal::BinaryInputArchive ar(ss); + ar >> tmp_blob_mdms; + for (BlobInfo &blob_info : tmp_blob_mdms) { + blob_mdms.emplace_back(blob_info); + } + } + + std::vector DeserializeBlobMetadata() { + std::vector blob_mdms; + for (const hipc::string &srl : *blob_mdms_) { + DeserializeBlobMetadata(srl.str(), blob_mdms); + } + return blob_mdms; + } + + /** Destructor */ + ~PollBlobMetadataTask() { + HSHM_DESTROY_AR(blob_mdms_) + } + + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, PollBlobMetadataTask &other) { + task_dup(other); + HSHM_MAKE_AR(blob_mdms_, alloc, *other.blob_mdms_); + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, PollBlobMetadataTask &dup_task) { + (*blob_mdms_)[replica] = (*dup_task.blob_mdms_)[0]; + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(blob_mdms_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar(blob_mdms_); + } + + /** Begin replication */ + void ReplicateStart(u32 count) { + blob_mdms_->resize(count); + } + + /** Finalize replication */ + void ReplicateEnd() { + std::vector blob_mdms = DeserializeBlobMetadata(); + SerializeBlobMetadata(blob_mdms); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + } // namespace hermes::blob_mdm #endif //LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 6a5d056e3..1e9065c3a 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -654,14 +654,18 @@ class Server : public TaskLib { } /** - * Get \a blob_name BLOB from \a bkt_id bucket - * */ + * Get all metadata about a blob + * */ HSHM_ALWAYS_INLINE - void PollBlobMetadata(GetBlobIdTask *task, RunContext &rctx) { + void PollBlobMetadata(PollBlobMetadataTask *task, RunContext &rctx) { BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; + std::vector blob_mdms; + blob_mdms.reserve(blob_map.size()); for (const std::pair &blob_part : blob_map) { const BlobInfo &blob_info = blob_part.second; + blob_mdms.emplace_back(blob_info); } + task->SerializeBlobMetadata(blob_mdms); task->SetModuleComplete(); } diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index e2de3ef9c..7ab2f80c3 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -482,7 +482,6 @@ TEST_CASE("TestHermesDataStager") { // Verify staging happened } - TEST_CASE("TestHermesDataOp") { int rank, nprocs; MPI_Barrier(MPI_COMM_WORLD); @@ -530,3 +529,32 @@ TEST_CASE("TestHermesDataOp") { size_t size = bkt_min.GetSize(); REQUIRE(size == sizeof(float) * count_per_proc * nprocs); } + +TEST_CASE("TestHermesCollectMetadata") { + int rank, nprocs; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + // Initialize Hermes on all nodes + HERMES->ClientInit(); + + // Create a bucket + hermes::Context ctx; + hermes::Bucket bkt("append_test" + std::to_string(rank)); + u32 num_blobs = 1024; + + // Put a few blobs in the bucket + for (int i = 0; i < num_blobs; ++i) { + hermes::Blob blob(KILOBYTES(4)); + memset(blob.data(), i % 256, blob.size()); + hermes::BlobId blob_id = bkt.Put(std::to_string(i), blob, ctx); + HILOG(kInfo, "(iteration {}) Using BlobID: {}", i, blob_id); + } + MPI_Barrier(MPI_COMM_WORLD); + + // Get contained blob ids + hermes::MetadataTable table = HERMES->CollectMetadataSnapshot(); + REQUIRE(table.blob_info_.size() == 1024 * nprocs); + MPI_Barrier(MPI_COMM_WORLD); +} \ No newline at end of file From 9e5845209a4fdbb05ab00eeed98e25585b3a8513 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 19:40:37 -0500 Subject: [PATCH 174/191] Begin target metadata collection --- .../hermes_blob_mdm_lib_exec.h | 44 ++++++ .../hermes_blob_mdm/hermes_blob_mdm_methods.h | 1 + .../hermes_blob_mdm_methods.yaml | 3 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 141 ++++++++++++++++-- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 10 ++ 5 files changed, 189 insertions(+), 10 deletions(-) diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index 250b2f27c..3e54512d9 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -80,6 +80,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { PollBlobMetadata(reinterpret_cast(task), rctx); break; } + case Method::kPollTargetMetadata: { + PollTargetMetadata(reinterpret_cast(task), rctx); + break; + } } } /** Delete a task */ @@ -161,6 +165,10 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kPollTargetMetadata: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } } } /** Duplicate a task */ @@ -242,6 +250,10 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } + case Method::kPollTargetMetadata: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } } } /** Register the duplicate output with the origin task */ @@ -323,6 +335,10 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } + case Method::kPollTargetMetadata: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } } } /** Ensure there is space to store replicated outputs */ @@ -404,6 +420,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kPollTargetMetadata: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } } } /** Determine success and handle failures */ @@ -485,6 +505,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kPollTargetMetadata: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } } } /** Serialize a task when initially pushing into remote */ @@ -566,6 +590,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kPollTargetMetadata: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -668,6 +696,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kPollTargetMetadata: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } } return task_ptr; } @@ -750,6 +783,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kPollTargetMetadata: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -832,6 +869,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kPollTargetMetadata: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } } } /** Get the grouping of the task */ @@ -894,6 +935,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kPollBlobMetadata: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kPollTargetMetadata: { + return reinterpret_cast(task)->GetGroup(group); + } } return -1; } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h index 2af72faea..3cbb59a0b 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h @@ -20,6 +20,7 @@ struct Method : public TaskMethod { TASK_METHOD_T kSetBucketMdm = kLast + 16; TASK_METHOD_T kFlushData = kLast + 17; TASK_METHOD_T kPollBlobMetadata = kLast + 18; + TASK_METHOD_T kPollTargetMetadata = kLast + 19; }; #endif // LABSTOR_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml index ab4524951..a886bf876 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml @@ -16,4 +16,5 @@ kRenameBlob: 14 kReorganizeBlob: 15 kSetBucketMdm: 16 kFlushData: 17 -kPollBlobMetadata: 18 \ No newline at end of file +kPollBlobMetadata: 18 +kPollTargetMetadata: 19 \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index c4a3120d6..18f79412d 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -1196,7 +1196,8 @@ struct FlushDataTask : public Task, TaskFlags { /** A task to collect blob metadata */ struct PollBlobMetadataTask : public Task, TaskFlags { - OUT hipc::ShmArchive> blob_mdms_; + OUT hipc::ShmArchive my_blob_mdms_; + TEMP hipc::ShmArchive> blob_mdms_; /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -1217,8 +1218,8 @@ struct PollBlobMetadataTask : public Task, TaskFlags { domain_id_ = DomainId::GetGlobal(); // Custom params + HSHM_MAKE_AR0(my_blob_mdms_, alloc) HSHM_MAKE_AR0(blob_mdms_, alloc) - blob_mdms_->reserve(1); } /** Serialize blob info */ @@ -1226,7 +1227,7 @@ struct PollBlobMetadataTask : public Task, TaskFlags { std::stringstream ss; cereal::BinaryOutputArchive ar(ss); ar << blob_info; - blob_mdms_->emplace_back(ss.str()); + (*my_blob_mdms_) = ss.str(); } /** Deserialize blob info */ @@ -1240,7 +1241,8 @@ struct PollBlobMetadataTask : public Task, TaskFlags { } } - std::vector DeserializeBlobMetadata() { + /** Get combined output of all replicas */ + std::vector MergeBlobMetadata() { std::vector blob_mdms; for (const hipc::string &srl : *blob_mdms_) { DeserializeBlobMetadata(srl.str(), blob_mdms); @@ -1248,33 +1250,42 @@ struct PollBlobMetadataTask : public Task, TaskFlags { return blob_mdms; } + /** Deserialize final query output */ + std::vector DeserializeBlobMetadata() { + std::vector blob_mdms; + DeserializeBlobMetadata(my_blob_mdms_->str(), blob_mdms); + return blob_mdms; + } + /** Destructor */ ~PollBlobMetadataTask() { + HSHM_DESTROY_AR(my_blob_mdms_) HSHM_DESTROY_AR(blob_mdms_) } /** Duplicate message */ void Dup(hipc::Allocator *alloc, PollBlobMetadataTask &other) { task_dup(other); - HSHM_MAKE_AR(blob_mdms_, alloc, *other.blob_mdms_); + HSHM_MAKE_AR(blob_mdms_, alloc, *other.blob_mdms_) + HSHM_MAKE_AR(my_blob_mdms_, alloc, *other.my_blob_mdms_) } /** Process duplicate message output */ void DupEnd(u32 replica, PollBlobMetadataTask &dup_task) { - (*blob_mdms_)[replica] = (*dup_task.blob_mdms_)[0]; + (*blob_mdms_)[replica] = (*dup_task.my_blob_mdms_); } /** (De)serialize message call */ template void SerializeStart(Ar &ar) { task_serialize(ar); - ar(blob_mdms_); + ar(my_blob_mdms_); } /** (De)serialize message return */ template void SerializeEnd(u32 replica, Ar &ar) { - ar(blob_mdms_); + ar((*blob_mdms_)[replica]); } /** Begin replication */ @@ -1284,7 +1295,7 @@ struct PollBlobMetadataTask : public Task, TaskFlags { /** Finalize replication */ void ReplicateEnd() { - std::vector blob_mdms = DeserializeBlobMetadata(); + std::vector blob_mdms = MergeBlobMetadata(); SerializeBlobMetadata(blob_mdms); } @@ -1295,6 +1306,118 @@ struct PollBlobMetadataTask : public Task, TaskFlags { } }; +/** A task to collect blob metadata */ +struct PollTargetMetadataTask : public Task, TaskFlags { + OUT hipc::ShmArchive my_target_mdms_; + TEMP hipc::ShmArchive> target_mdms_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + PollTargetMetadataTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + PollTargetMetadataTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kPollTargetMetadata; + task_flags_.SetBits(TASK_COROUTINE); + domain_id_ = DomainId::GetGlobal(); + + // Custom params + HSHM_MAKE_AR0(my_target_mdms_, alloc) + HSHM_MAKE_AR0(target_mdms_, alloc) + } + + /** Serialize target info */ + void SerializeTargetMetadata(const std::vector &target_info) { + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar << target_info; + (*my_target_mdms_) = ss.str(); + } + + /** Deserialize target info */ + void DeserializeTargetMetadata(const std::string &srl, std::vector &target_mdms) { + std::vector tmp_target_mdms; + std::stringstream ss(srl); + cereal::BinaryInputArchive ar(ss); + ar >> tmp_target_mdms; + for (BlobInfo &target_info : tmp_target_mdms) { + target_mdms.emplace_back(target_info); + } + } + + /** Get combined output of all replicas */ + std::vector MergeTargetMetadata() { + std::vector target_mdms; + for (const hipc::string &srl : *target_mdms_) { + DeserializeTargetMetadata(srl.str(), target_mdms); + } + return target_mdms; + } + + /** Deserialize final query output */ + std::vector DeserializeTargetMetadata() { + std::vector target_mdms; + DeserializeTargetMetadata(my_target_mdms_->str(), target_mdms); + return target_mdms; + } + + /** Destructor */ + ~PollBlobMetadataTask() { + HSHM_DESTROY_AR(my_target_mdms_) + HSHM_DESTROY_AR(target_mdms_) + } + + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, PollTargetMetadataTask &other) { + task_dup(other); + HSHM_MAKE_AR(target_mdms_, alloc, *other.target_mdms_) + HSHM_MAKE_AR(my_target_mdms_, alloc, *other.my_target_mdms_) + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, PollTargetMetadataTask &dup_task) { + (*target_mdms_)[replica] = (*dup_task.my_target_mdms_); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(my_target_mdms_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar((*target_mdms_)[replica]); + } + + /** Begin replication */ + void ReplicateStart(u32 count) { + target_mdms_->resize(count); + } + + /** Finalize replication */ + void ReplicateEnd() { + std::vector target_mdms = MergeTargetMetadata(); + SerializeTargetMetadata(target_mdms); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + } // namespace hermes::blob_mdm #endif //LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 1e9065c3a..03a621269 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -669,6 +669,16 @@ class Server : public TaskLib { task->SetModuleComplete(); } + /** + * Get all metadata about a blob + * */ + HSHM_ALWAYS_INLINE + void PollTargetMetadata(PollBlobMetadataTask *task, RunContext &rctx) { + for (const bdev::Client &bdev_client : targets_) { + } + task->SetModuleComplete(); + } + public: #include "hermes_blob_mdm/hermes_blob_mdm_lib_exec.h" }; From f43409606367b1d33b35eec065fadd350e1e8f80 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 21:02:02 -0500 Subject: [PATCH 175/191] Target info task compiles --- include/labstor/api/labstor_client.h | 4 ++++ include/labstor/task_registry/task.h | 6 ++++-- tasks/bdev/include/bdev/bdev.h | 18 +++++++++++++++++ tasks/hermes/include/hermes/hermes.h | 4 ++++ .../include/hermes_blob_mdm/hermes_blob_mdm.h | 20 +++++++++++++++++++ .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 20 +++++++++---------- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 19 ++++++++++++++++-- 7 files changed, 77 insertions(+), 14 deletions(-) diff --git a/include/labstor/api/labstor_client.h b/include/labstor/api/labstor_client.h index 8a42302ce..35523ab5e 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/labstor/api/labstor_client.h @@ -159,11 +159,13 @@ class Client : public ConfigurationManager { HSHM_ALWAYS_INLINE void DelTask(TaskT *task) { // TODO(llogan): verify leak +#ifdef TASK_DEBUG task->delcnt_++; if (task->delcnt_ != 1) { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } +#endif main_alloc_->DelObj(task); } @@ -171,11 +173,13 @@ class Client : public ConfigurationManager { template HSHM_ALWAYS_INLINE void DelTask(LPointer &task) { +#ifdef TASK_DEBUG task->delcnt_++; if (task->delcnt_ != 1) { HELOG(kFatal, "Freed task {} times: node={}, state={}. method={}", task->delcnt_.load(), task->task_node_, task->task_state_, task->method_) } +#endif main_alloc_->DelObjLocal(task); } diff --git a/include/labstor/task_registry/task.h b/include/labstor/task_registry/task.h index f44b10b9a..229698ec9 100644 --- a/include/labstor/task_registry/task.h +++ b/include/labstor/task_registry/task.h @@ -266,10 +266,12 @@ struct Task : public hipc::ShmContainer { u32 lane_hash_; /**< Determine the lane a task is keyed to */ u32 method_; /**< The method to call in the state */ bitfield32_t task_flags_; /**< Properties of the task */ - std::atomic delcnt_ = 0; /**< # of times deltask called */ - double period_ns_; /**< The period of the task */ + double period_ns_; /**< The period of the task */ hshm::Timepoint start_; /**< The time the task started */ RunContext ctx_; +#ifdef TASK_DEBUG + std::atomic delcnt_ = 0; /**< # of times deltask called */ +#endif /**==================================== * Task Helpers diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index 2df064cee..a454b46b5 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -161,6 +161,24 @@ class Server { namespace hermes { typedef bdev::Client TargetInfo; + +struct TargetStats { + public: + TargetId tgt_id_; + size_t rem_cap_; /**< Current remaining capacity */ + size_t max_cap_; /**< maximum capacity of the target */ + double bandwidth_; /**< the bandwidth of the device */ + double latency_; /**< the latency of the device */ + float score_; /**< Relative importance of this tier */ + + public: + /** Serialize */ + template + void serialize(Ar &ar) { + ar(tgt_id_, max_cap_, bandwidth_, + latency_, score_, rem_cap_); + } +}; } // namespace hermes #endif // LABSTOR_bdev_H_ diff --git a/tasks/hermes/include/hermes/hermes.h b/tasks/hermes/include/hermes/hermes.h index 5318d7a6b..00a30da19 100644 --- a/tasks/hermes/include/hermes/hermes.h +++ b/tasks/hermes/include/hermes/hermes.h @@ -11,6 +11,8 @@ namespace hermes { struct MetadataTable { std::vector blob_info_; + std::vector target_info_; + std::vector bkt_info_; }; class Hermes { @@ -34,6 +36,8 @@ class Hermes { MetadataTable CollectMetadataSnapshot() { MetadataTable table; table.blob_info_ = HERMES_CONF->blob_mdm_.PollBlobMetadataRoot(); + table.target_info_ = HERMES_CONF->blob_mdm_.PollTargetMetadataRoot(); + // table.bkt_info_ = HERMES_CONF->bkt_mdm_.PollTagMetadataRoot(); return table; } diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 1442540dd..047366a62 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -455,6 +455,26 @@ class Client : public TaskLibClient { return blob_mdms; } LABSTOR_TASK_NODE_PUSH_ROOT(PollBlobMetadata); + + /** + * Get all target metadata + * */ + void AsyncPollTargetMetadataConstruct(PollTargetMetadataTask *task, + const TaskNode &task_node) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_); + } + std::vector PollTargetMetadataRoot() { + LPointer> push_task = + AsyncPollTargetMetadataRoot(); + push_task->Wait(); + PollTargetMetadataTask *task = push_task->get(); + std::vector target_mdms = + task->DeserializeTargetMetadata(); + LABSTOR_CLIENT->DelTask(push_task); + return target_mdms; + } + LABSTOR_TASK_NODE_PUSH_ROOT(PollTargetMetadata); }; } // namespace labstor diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 18f79412d..56a2ad7f9 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -1335,7 +1335,7 @@ struct PollTargetMetadataTask : public Task, TaskFlags } /** Serialize target info */ - void SerializeTargetMetadata(const std::vector &target_info) { + void SerializeTargetMetadata(const std::vector &target_info) { std::stringstream ss; cereal::BinaryOutputArchive ar(ss); ar << target_info; @@ -1343,19 +1343,19 @@ struct PollTargetMetadataTask : public Task, TaskFlags } /** Deserialize target info */ - void DeserializeTargetMetadata(const std::string &srl, std::vector &target_mdms) { - std::vector tmp_target_mdms; + void DeserializeTargetMetadata(const std::string &srl, std::vector &target_mdms) { + std::vector tmp_target_mdms; std::stringstream ss(srl); cereal::BinaryInputArchive ar(ss); ar >> tmp_target_mdms; - for (BlobInfo &target_info : tmp_target_mdms) { + for (TargetStats &target_info : tmp_target_mdms) { target_mdms.emplace_back(target_info); } } /** Get combined output of all replicas */ - std::vector MergeTargetMetadata() { - std::vector target_mdms; + std::vector MergeTargetMetadata() { + std::vector target_mdms; for (const hipc::string &srl : *target_mdms_) { DeserializeTargetMetadata(srl.str(), target_mdms); } @@ -1363,14 +1363,14 @@ struct PollTargetMetadataTask : public Task, TaskFlags } /** Deserialize final query output */ - std::vector DeserializeTargetMetadata() { - std::vector target_mdms; + std::vector DeserializeTargetMetadata() { + std::vector target_mdms; DeserializeTargetMetadata(my_target_mdms_->str(), target_mdms); return target_mdms; } /** Destructor */ - ~PollBlobMetadataTask() { + ~PollTargetMetadataTask() { HSHM_DESTROY_AR(my_target_mdms_) HSHM_DESTROY_AR(target_mdms_) } @@ -1407,7 +1407,7 @@ struct PollTargetMetadataTask : public Task, TaskFlags /** Finalize replication */ void ReplicateEnd() { - std::vector target_mdms = MergeTargetMetadata(); + std::vector target_mdms = MergeTargetMetadata(); SerializeTargetMetadata(target_mdms); } diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 03a621269..582fd9d6f 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -673,9 +673,24 @@ class Server : public TaskLib { * Get all metadata about a blob * */ HSHM_ALWAYS_INLINE - void PollTargetMetadata(PollBlobMetadataTask *task, RunContext &rctx) { + void PollTargetMetadata(PollTargetMetadataTask *task, RunContext &rctx) { + std::vector target_mdms; + target_mdms.reserve(targets_.size()); for (const bdev::Client &bdev_client : targets_) { - } + bool is_remote = bdev_client.domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); + if (is_remote) { + continue; + } + TargetStats stats; + stats.tgt_id_ = bdev_client.id_; + stats.rem_cap_ = bdev_client.monitor_task_->rem_cap_; + stats.max_cap_ = bdev_client.max_cap_; + stats.bandwidth_ = bdev_client.bandwidth_; + stats.latency_ = bdev_client.latency_; + stats.score_ = bdev_client.score_; + target_mdms.emplace_back(stats); + } + task->SerializeTargetMetadata(target_mdms); task->SetModuleComplete(); } From cbcc7e67c060ccb2ecce5cf983d7d2d81849ef81 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 21:15:53 -0500 Subject: [PATCH 176/191] Single-node metadata queries work --- tasks/hermes/include/hermes/hermes.h | 2 +- tasks/hermes/include/hermes/hermes_types.h | 6 + .../hermes_bucket_mdm/hermes_bucket_mdm.h | 20 ++++ .../hermes_bucket_mdm_lib_exec.h | 44 +++++++ .../hermes_bucket_mdm_methods.h | 1 + .../hermes_bucket_mdm_methods.yaml | 3 +- .../hermes_bucket_mdm_tasks.h | 112 ++++++++++++++++++ .../src/hermes_bucket_mdm.cc | 16 +++ tasks_required/proc_queue/src/proc_queue.cc | 4 +- .../include/remote_queue/remote_queue.h | 4 +- test/unit/hermes/test_bucket.cc | 2 + 11 files changed, 208 insertions(+), 6 deletions(-) diff --git a/tasks/hermes/include/hermes/hermes.h b/tasks/hermes/include/hermes/hermes.h index 00a30da19..0645a580e 100644 --- a/tasks/hermes/include/hermes/hermes.h +++ b/tasks/hermes/include/hermes/hermes.h @@ -37,7 +37,7 @@ class Hermes { MetadataTable table; table.blob_info_ = HERMES_CONF->blob_mdm_.PollBlobMetadataRoot(); table.target_info_ = HERMES_CONF->blob_mdm_.PollTargetMetadataRoot(); - // table.bkt_info_ = HERMES_CONF->bkt_mdm_.PollTagMetadataRoot(); + table.bkt_info_ = HERMES_CONF->bkt_mdm_.PollTagMetadataRoot(); return table; } diff --git a/tasks/hermes/include/hermes/hermes_types.h b/tasks/hermes/include/hermes/hermes_types.h index e39fa5935..acc12ea08 100644 --- a/tasks/hermes/include/hermes/hermes_types.h +++ b/tasks/hermes/include/hermes/hermes_types.h @@ -332,6 +332,12 @@ struct TagInfo { size_t internal_size_; size_t page_size_; bool owner_; + + /** Serialization */ + template + void serialize(Ar &ar) { + ar(tag_id_, name_, internal_size_, page_size_, owner_); + } }; /** The types of I/O that can be performed (for IoCall RPC) */ diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index 56723d5d1..0da9260d9 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -314,6 +314,26 @@ class Client : public TaskLibClient { return blob_ids; } LABSTOR_TASK_NODE_PUSH_ROOT(GetContainedBlobIds); + + /** + * Get all tag metadata + * */ + void AsyncPollTagMetadataConstruct(PollTagMetadataTask *task, + const TaskNode &task_node) { + LABSTOR_CLIENT->ConstructTask( + task, task_node, id_); + } + std::vector PollTagMetadataRoot() { + LPointer> push_task = + AsyncPollTagMetadataRoot(); + push_task->Wait(); + PollTagMetadataTask *task = push_task->get(); + std::vector target_mdms = + task->DeserializeTagMetadata(); + LABSTOR_CLIENT->DelTask(push_task); + return target_mdms; + } + LABSTOR_TASK_NODE_PUSH_ROOT(PollTagMetadata); }; } // namespace labstor diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index 1a4f10e2c..560014830 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -68,6 +68,10 @@ void Run(u32 method, Task *task, RunContext &rctx) override { GetContainedBlobIds(reinterpret_cast(task), rctx); break; } + case Method::kPollTagMetadata: { + PollTagMetadata(reinterpret_cast(task), rctx); + break; + } } } /** Delete a task */ @@ -137,6 +141,10 @@ void Del(u32 method, Task *task) override { LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); break; } + case Method::kPollTagMetadata: { + LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + break; + } } } /** Duplicate a task */ @@ -206,6 +214,10 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } + case Method::kPollTagMetadata: { + labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + break; + } } } /** Register the duplicate output with the origin task */ @@ -275,6 +287,10 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } + case Method::kPollTagMetadata: { + labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + break; + } } } /** Ensure there is space to store replicated outputs */ @@ -344,6 +360,10 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } + case Method::kPollTagMetadata: { + labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + break; + } } } /** Determine success and handle failures */ @@ -413,6 +433,10 @@ void ReplicateEnd(u32 method, Task *task) override { labstor::CALL_REPLICA_END(reinterpret_cast(task)); break; } + case Method::kPollTagMetadata: { + labstor::CALL_REPLICA_END(reinterpret_cast(task)); + break; + } } } /** Serialize a task when initially pushing into remote */ @@ -482,6 +506,10 @@ std::vector SaveStart(u32 method, BinaryOutputArchive &ar, T ar << *reinterpret_cast(task); break; } + case Method::kPollTagMetadata: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -569,6 +597,11 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { ar >> *reinterpret_cast(task_ptr.ptr_); break; } + case Method::kPollTagMetadata: { + task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + ar >> *reinterpret_cast(task_ptr.ptr_); + break; + } } return task_ptr; } @@ -639,6 +672,10 @@ std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Ta ar << *reinterpret_cast(task); break; } + case Method::kPollTagMetadata: { + ar << *reinterpret_cast(task); + break; + } } return ar.Get(); } @@ -709,6 +746,10 @@ void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) ar.Deserialize(replica, *reinterpret_cast(task)); break; } + case Method::kPollTagMetadata: { + ar.Deserialize(replica, *reinterpret_cast(task)); + break; + } } } /** Get the grouping of the task */ @@ -762,6 +803,9 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { case Method::kGetContainedBlobIds: { return reinterpret_cast(task)->GetGroup(group); } + case Method::kPollTagMetadata: { + return reinterpret_cast(task)->GetGroup(group); + } } return -1; } diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h index a7650139e..f797d2b18 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h @@ -17,6 +17,7 @@ struct Method : public TaskMethod { TASK_METHOD_T kGetSize = kLast + 14; TASK_METHOD_T kSetBlobMdm = kLast + 15; TASK_METHOD_T kGetContainedBlobIds = kLast + 16; + TASK_METHOD_T kPollTagMetadata = kLast + 17; }; #endif // LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml index b200fcc0a..527f10a58 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml @@ -14,4 +14,5 @@ kAppendBlobSchema: 12 kAppendBlob: 13 kGetSize: 14 kSetBlobMdm: 15 -kGetContainedBlobIds: 16 \ No newline at end of file +kGetContainedBlobIds: 16 +kPollTagMetadata: 17 \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index dc0ed42bb..764a168a3 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -914,6 +914,118 @@ struct GetContainedBlobIdsTask : public Task, TaskFlags { } }; +/** A task to collect blob metadata */ +struct PollTagMetadataTask : public Task, TaskFlags { + OUT hipc::ShmArchive my_tag_mdms_; + TEMP hipc::ShmArchive> tag_mdms_; + + /** SHM default constructor */ + HSHM_ALWAYS_INLINE explicit + PollTagMetadataTask(hipc::Allocator *alloc) : Task(alloc) {} + + /** Emplace constructor */ + HSHM_ALWAYS_INLINE explicit + PollTagMetadataTask(hipc::Allocator *alloc, + const TaskNode &task_node, + const TaskStateId &state_id) : Task(alloc) { + // Initialize task + task_node_ = task_node; + lane_hash_ = 0; + prio_ = TaskPrio::kLowLatency; + task_state_ = state_id; + method_ = Method::kPollTagMetadata; + task_flags_.SetBits(TASK_LANE_ALL); + domain_id_ = DomainId::GetGlobal(); + + // Custom params + HSHM_MAKE_AR0(my_tag_mdms_, alloc) + HSHM_MAKE_AR0(tag_mdms_, alloc) + } + + /** Serialize tag info */ + void SerializeTagMetadata(const std::vector &tag_info) { + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar << tag_info; + (*my_tag_mdms_) = ss.str(); + } + + /** Deserialize tag info */ + void DeserializeTagMetadata(const std::string &srl, std::vector &tag_mdms) { + std::vector tmp_tag_mdms; + std::stringstream ss(srl); + cereal::BinaryInputArchive ar(ss); + ar >> tmp_tag_mdms; + for (TagInfo &tag_info : tmp_tag_mdms) { + tag_mdms.emplace_back(tag_info); + } + } + + /** Get combined output of all replicas */ + std::vector MergeTagMetadata() { + std::vector tag_mdms; + for (const hipc::string &srl : *tag_mdms_) { + DeserializeTagMetadata(srl.str(), tag_mdms); + } + return tag_mdms; + } + + /** Deserialize final query output */ + std::vector DeserializeTagMetadata() { + std::vector tag_mdms; + DeserializeTagMetadata(my_tag_mdms_->str(), tag_mdms); + return tag_mdms; + } + + /** Destructor */ + ~PollTagMetadataTask() { + HSHM_DESTROY_AR(my_tag_mdms_) + HSHM_DESTROY_AR(tag_mdms_) + } + + /** Duplicate message */ + void Dup(hipc::Allocator *alloc, PollTagMetadataTask &other) { + task_dup(other); + HSHM_MAKE_AR(tag_mdms_, alloc, *other.tag_mdms_) + HSHM_MAKE_AR(my_tag_mdms_, alloc, *other.my_tag_mdms_) + } + + /** Process duplicate message output */ + void DupEnd(u32 replica, PollTagMetadataTask &dup_task) { + (*tag_mdms_)[replica] = (*dup_task.my_tag_mdms_); + } + + /** (De)serialize message call */ + template + void SerializeStart(Ar &ar) { + task_serialize(ar); + ar(my_tag_mdms_); + } + + /** (De)serialize message return */ + template + void SerializeEnd(u32 replica, Ar &ar) { + ar((*tag_mdms_)[replica]); + } + + /** Begin replication */ + void ReplicateStart(u32 count) { + tag_mdms_->resize(count); + } + + /** Finalize replication */ + void ReplicateEnd() { + std::vector tag_mdms = MergeTagMetadata(); + SerializeTagMetadata(tag_mdms); + } + + /** Create group */ + HSHM_ALWAYS_INLINE + u32 GetGroup(hshm::charbuf &group) { + return TASK_UNORDERED; + } +}; + } // namespace hermes::bucket_mdm #endif // LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 37f12c0cd..486c5070a 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -385,6 +385,22 @@ class Server : public TaskLib { task->SetModuleComplete(); } + /** + * Get all metadata about a blob + * */ + HSHM_ALWAYS_INLINE + void PollTagMetadata(PollTagMetadataTask *task, RunContext &rctx) { + TAG_MAP_T &blob_map = tag_map_[rctx.lane_id_]; + std::vector tag_mdms; + tag_mdms.reserve(blob_map.size()); + for (const std::pair &tag_part : blob_map) { + const TagInfo &tag_info = tag_part.second; + tag_mdms.emplace_back(tag_info); + } + task->SerializeTagMetadata(tag_mdms); + task->SetModuleComplete(); + } + public: #include "hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h" }; diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/tasks_required/proc_queue/src/proc_queue.cc index 2c2463378..c911bb65a 100644 --- a/tasks_required/proc_queue/src/proc_queue.cc +++ b/tasks_required/proc_queue/src/proc_queue.cc @@ -26,8 +26,8 @@ class Server : public TaskLib { task->sub_run_.shm_ = task->sub_cli_.shm_; task->sub_run_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->sub_cli_.shm_); Task *&ptr = task->sub_run_.ptr_; - HILOG(kDebug, "Scheduling task {} on state {} tid {}", - ptr->task_node_, ptr->task_state_, GetLinuxTid()); +// HILOG(kDebug, "Scheduling task {} on state {} tid {}", +// ptr->task_node_, ptr->task_state_, GetLinuxTid()); if (ptr->IsFireAndForget()) { ptr->UnsetFireAndForget(); } diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index a8d3d1bc8..8c304b1da 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -85,8 +85,8 @@ class Client : public TaskLibClient { void DisperseLocal(Task *orig_task, TaskState *exec, MultiQueue *orig_queue, LaneGroup *lane_group) { // Duplicate task - HILOG(kDebug, "Beginning duplication for (task_node={}, task_state={}, method={})", - orig_task->task_node_ + 1, orig_task->task_state_, orig_task->method_); +// HILOG(kDebug, "Beginning duplication for (task_node={}, task_state={}, method={})", +// orig_task->task_node_ + 1, orig_task->task_state_, orig_task->method_); std::vector> dups(lane_group->num_lanes_); exec->Dup(orig_task->method_, orig_task, dups); for (size_t i = 0; i < dups.size(); ++i) { diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 7ab2f80c3..b4dd27838 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -556,5 +556,7 @@ TEST_CASE("TestHermesCollectMetadata") { // Get contained blob ids hermes::MetadataTable table = HERMES->CollectMetadataSnapshot(); REQUIRE(table.blob_info_.size() == 1024 * nprocs); + REQUIRE(table.bkt_info_.size() == 1); + REQUIRE(table.target_info_.size() >= 4); MPI_Barrier(MPI_COMM_WORLD); } \ No newline at end of file From ce4c4c7fa89f839ffb464411f42758bb2f9ccf6b Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 21:26:34 -0500 Subject: [PATCH 177/191] labstor -> hrun --- CMake/HermesConfig.cmake | 2 +- CMakeLists.txt | 12 +- benchmark/hermes_api_bench.cc | 4 +- benchmark/test_init.cc | 2 +- benchmark/test_init.h | 8 +- benchmark/test_latency.cc | 94 +++---- benchmark/test_zmq.cc | 12 +- codegen/codegen/labstor_config/generator.py | 6 +- codegen/codegen/util/paths.py | 2 +- codegen/hermes_config | 6 +- codegen/labstor_config | 18 +- codegen/make_macro | 2 +- codegen/make_task | 4 +- codegen/preamble.py | 6 +- codegen/refresh_methods | 20 +- codegen/update_task | 4 +- config/labstor_server_default.yaml | 2 +- .../api/hrun_client.h} | 60 ++--- .../api/hrun_runtime.h} | 66 ++--- include/{labstor => hrun}/api/manager.h | 20 +- .../hrun_task_node_admin_root.template} | 6 +- .../hrun_task_node_push_root.template} | 12 +- include/{labstor => hrun}/config/config.h | 10 +- .../{labstor => hrun}/config/config_client.h | 14 +- include/hrun/config/config_client_default.h | 5 + .../{labstor => hrun}/config/config_server.h | 16 +- .../config/config_server_default.h | 8 +- include/hrun/hrun_constants.h | 29 +++ include/hrun/hrun_namespace.h | 39 +++ .../labstor_types.h => hrun/hrun_types.h} | 18 +- .../network/local_serialize.h | 10 +- include/{labstor => hrun}/network/rpc.h | 12 +- .../{labstor => hrun}/network/rpc_thallium.h | 2 +- include/{labstor => hrun}/network/serialize.h | 14 +- .../{labstor => hrun}/queue_manager/queue.h | 14 +- include/hrun/queue_manager/queue_factory.h | 19 ++ .../queue_manager/queue_manager.h | 14 +- .../queue_manager/queue_manager_client.h | 14 +- .../queue_manager/queue_manager_runtime.h | 14 +- .../queue_manager/queues/hshm_queue.h | 12 +- .../{labstor => hrun}/task_registry/task.h | 16 +- .../task_registry/task_lib.h | 26 +- .../task_registry/task_registry.h | 28 +- .../work_orchestrator/affinity.h | 6 +- .../work_orchestrator/scheduler.h | 12 +- .../work_orchestrator/work_orchestrator.h | 22 +- .../work_orchestrator/worker.h | 40 +-- .../labstor/config/config_client_default.h | 5 - include/labstor/labstor_constants.h | 29 --- include/labstor/labstor_namespace.h | 39 --- include/labstor/queue_manager/queue_factory.h | 19 -- scripts/lint.sh | 4 +- src/CMakeLists.txt | 52 ++-- src/config_client.cc | 8 +- src/config_server.cc | 10 +- src/{labstor_client.cc => hrun_client.cc} | 4 +- src/{labstor_runtime.cc => hrun_runtime.cc} | 4 +- ...start_runtime.cc => hrun_start_runtime.cc} | 8 +- src/hrun_stop_runtime.cc | 10 + src/labstor_stop_runtime.cc | 10 - src/worker.cc | 38 +-- tasks/bdev/include/bdev/bdev.h | 40 +-- tasks/bdev/include/bdev/bdev_lib_exec.h | 102 ++++---- tasks/bdev/include/bdev/bdev_methods.h | 6 +- tasks/bdev/include/bdev/bdev_namespace.h | 8 +- tasks/bdev/include/bdev/bdev_tasks.h | 20 +- .../include/data_stager/data_stager.h | 42 +-- .../data_stager/data_stager_lib_exec.h | 78 +++--- .../include/data_stager/data_stager_methods.h | 6 +- .../include/data_stager/data_stager_tasks.h | 28 +- .../data_stager/factory/binary_stager.h | 6 +- tasks/data_stager/src/data_stager.cc | 10 +- tasks/hermes/include/hermes/bucket.h | 38 +-- .../include/hermes/config_client_default.h | 6 +- tasks/hermes/include/hermes/config_manager.h | 10 +- .../include/hermes/config_server_default.h | 6 +- tasks/hermes/include/hermes/hermes.h | 6 +- tasks/hermes/include/hermes/hermes_types.h | 32 +-- tasks/hermes/include/hermes/slab_allocator.h | 8 +- tasks/hermes/include/hermes/statuses.h | 6 +- .../include/hermes_adapters/hermes_adapters.h | 26 +- .../hermes_adapters_lib_exec.h | 42 +-- .../hermes_adapters/hermes_adapters_methods.h | 6 +- .../hermes_adapters/hermes_adapters_tasks.h | 28 +- .../hermes_adapters/mapper/abstract_mapper.h | 6 +- tasks/hermes_adapters/src/hermes_adapters.cc | 10 +- tasks/hermes_adapters/vfd/README.md | 6 +- .../include/hermes_blob_mdm/hermes_blob_mdm.h | 150 +++++------ .../hermes_blob_mdm_lib_exec.h | 246 +++++++++--------- .../hermes_blob_mdm/hermes_blob_mdm_methods.h | 6 +- .../hermes_blob_mdm/hermes_blob_mdm_tasks.h | 60 ++--- tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc | 38 +-- .../hermes_bucket_mdm/hermes_bucket_mdm.h | 122 ++++----- .../hermes_bucket_mdm_lib_exec.h | 210 +++++++-------- .../hermes_bucket_mdm_methods.h | 6 +- .../hermes_bucket_mdm_tasks.h | 48 ++-- .../src/hermes_bucket_mdm.cc | 40 +-- .../include/hermes_data_op/hermes_data_op.h | 32 +-- .../hermes_data_op/hermes_data_op_lib_exec.h | 66 ++--- .../hermes_data_op/hermes_data_op_methods.h | 6 +- .../hermes_data_op/hermes_data_op_tasks.h | 24 +- tasks/hermes_data_op/src/hermes_data_op.cc | 16 +- .../include/hermes_mdm/hermes_mdm.h | 12 +- .../include/hermes_mdm/hermes_mdm_lib_exec.h | 30 +-- .../include/hermes_mdm/hermes_mdm_methods.h | 6 +- .../include/hermes_mdm/hermes_mdm_tasks.h | 20 +- tasks/hermes_mdm/src/hermes_mdm.cc | 10 +- .../include/posix_bdev/posix_bdev.h | 18 +- tasks/posix_bdev/src/posix_bdev.cc | 8 +- tasks/ram_bdev/include/ram_bdev/ram_bdev.h | 18 +- tasks/ram_bdev/src/ram_bdev.cc | 8 +- tasks_required/CMakeLists.txt | 2 +- .../TASK_NAME/include/TASK_NAME/TASK_NAME.h | 26 +- .../include/TASK_NAME/TASK_NAME_lib_exec.h | 42 +-- .../include/TASK_NAME/TASK_NAME_methods.h | 6 +- .../include/TASK_NAME/TASK_NAME_tasks.h | 28 +- tasks_required/TASK_NAME/src/TASK_NAME.cc | 10 +- .../CMakeLists.txt | 0 .../include/hrun_admin/hrun_admin.h} | 82 +++--- .../include/hrun_admin/hrun_admin_lib_exec.h} | 126 ++++----- .../include/hrun_admin/hrun_admin_methods.h} | 6 +- .../hrun_admin/hrun_admin_methods.yaml} | 0 .../include/hrun_admin/hrun_admin_tasks.h} | 36 +-- .../src/CMakeLists.txt | 14 +- .../src/hrun_admin.cc} | 60 ++--- .../include/proc_queue/proc_queue.h | 42 +-- .../include/proc_queue/proc_queue_lib_exec.h | 42 +-- .../include/proc_queue/proc_queue_methods.h | 6 +- .../include/proc_queue/proc_queue_tasks.h | 32 +-- tasks_required/proc_queue/src/proc_queue.cc | 14 +- .../include/remote_queue/remote_queue.h | 34 +-- .../remote_queue/remote_queue_lib_exec.h | 54 ++-- .../remote_queue/remote_queue_methods.h | 6 +- .../include/remote_queue/remote_queue_tasks.h | 24 +- .../remote_queue/src/CMakeLists.txt | 2 +- .../remote_queue/src/remote_queue.cc | 60 ++--- .../include/small_message/small_message.h | 40 +-- .../small_message/small_message_lib_exec.h | 66 ++--- .../small_message/small_message_methods.h | 6 +- .../small_message/small_message_tasks.h | 24 +- .../small_message/src/small_message.cc | 10 +- .../worch_proc_round_robin.h | 14 +- .../worch_proc_round_robin_lib_exec.h | 42 +-- .../worch_proc_round_robin_methods.h | 6 +- .../worch_proc_round_robin_tasks.h | 26 +- .../src/worch_proc_round_robin.cc | 12 +- .../worch_queue_round_robin.h | 14 +- .../worch_queue_round_robin_lib_exec.h | 42 +-- .../worch_queue_round_robin_methods.h | 6 +- .../worch_queue_round_robin_tasks.h | 26 +- .../src/worch_queue_round_robin.cc | 18 +- test/unit/CMakeLists.txt | 2 +- test/unit/basic_test.h | 6 +- test/unit/boost/test_init.cc | 2 +- test/unit/boost/test_init.h | 8 +- test/unit/hermes/config/labstor_server.yaml | 2 +- test/unit/hermes/test_bucket.cc | 6 +- test/unit/hermes/test_init.cc | 4 +- test/unit/hermes/test_init.h | 8 +- test/unit/ipc/test_finalize.cc | 6 +- test/unit/ipc/test_init.cc | 4 +- test/unit/ipc/test_init.h | 8 +- test/unit/ipc/test_ipc.cc | 46 ++-- test/unit/ipc/test_serialize.cc | 18 +- 164 files changed, 2015 insertions(+), 2015 deletions(-) rename include/{labstor/api/labstor_client.h => hrun/api/hrun_client.h} (86%) rename include/{labstor/api/labstor_runtime.h => hrun/api/hrun_runtime.h} (74%) rename include/{labstor => hrun}/api/manager.h (83%) rename include/{labstor/api/template/labstor_task_node_admin_root.template => hrun/api/template/hrun_task_node_admin_root.template} (86%) rename include/{labstor/api/template/labstor_task_node_push_root.template => hrun/api/template/hrun_task_node_push_root.template} (76%) rename include/{labstor => hrun}/config/config.h (91%) rename include/{labstor => hrun}/config/config_client.h (85%) create mode 100644 include/hrun/config/config_client_default.h rename include/{labstor => hrun}/config/config_server.h (91%) rename include/{labstor => hrun}/config/config_server_default.h (92%) create mode 100644 include/hrun/hrun_constants.h create mode 100644 include/hrun/hrun_namespace.h rename include/{labstor/labstor_types.h => hrun/hrun_types.h} (96%) rename include/{labstor => hrun}/network/local_serialize.h (89%) rename include/{labstor => hrun}/network/rpc.h (97%) rename include/{labstor => hrun}/network/rpc_thallium.h (99%) rename include/{labstor => hrun}/network/serialize.h (96%) rename include/{labstor => hrun}/queue_manager/queue.h (90%) create mode 100644 include/hrun/queue_manager/queue_factory.h rename include/{labstor => hrun}/queue_manager/queue_manager.h (79%) rename include/{labstor => hrun}/queue_manager/queue_manager_client.h (62%) rename include/{labstor => hrun}/queue_manager/queue_manager_runtime.h (88%) rename include/{labstor => hrun}/queue_manager/queues/hshm_queue.h (96%) rename include/{labstor => hrun}/task_registry/task.h (98%) rename include/{labstor => hrun}/task_registry/task_lib.h (82%) rename include/{labstor => hrun}/task_registry/task_registry.h (93%) rename include/{labstor => hrun}/work_orchestrator/affinity.h (74%) rename include/{labstor => hrun}/work_orchestrator/scheduler.h (80%) rename include/{labstor => hrun}/work_orchestrator/work_orchestrator.h (85%) rename include/{labstor => hrun}/work_orchestrator/worker.h (90%) delete mode 100644 include/labstor/config/config_client_default.h delete mode 100644 include/labstor/labstor_constants.h delete mode 100644 include/labstor/labstor_namespace.h delete mode 100644 include/labstor/queue_manager/queue_factory.h rename src/{labstor_client.cc => hrun_client.cc} (60%) rename src/{labstor_runtime.cc => hrun_runtime.cc} (59%) rename src/{labstor_start_runtime.cc => hrun_start_runtime.cc} (50%) create mode 100644 src/hrun_stop_runtime.cc delete mode 100644 src/labstor_stop_runtime.cc rename tasks_required/{labstor_admin => hrun_admin}/CMakeLists.txt (100%) rename tasks_required/{labstor_admin/include/labstor_admin/labstor_admin.h => hrun_admin/include/hrun_admin/hrun_admin.h} (79%) rename tasks_required/{labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h => hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h} (67%) rename tasks_required/{labstor_admin/include/labstor_admin/labstor_admin_methods.h => hrun_admin/include/hrun_admin/hrun_admin_methods.h} (81%) rename tasks_required/{labstor_admin/include/labstor_admin/labstor_admin_methods.yaml => hrun_admin/include/hrun_admin/hrun_admin_methods.yaml} (100%) rename tasks_required/{labstor_admin/include/labstor_admin/labstor_admin_tasks.h => hrun_admin/include/hrun_admin/hrun_admin_tasks.h} (92%) rename tasks_required/{labstor_admin => hrun_admin}/src/CMakeLists.txt (85%) rename tasks_required/{labstor_admin/src/labstor_admin.cc => hrun_admin/src/hrun_admin.cc} (68%) diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index a5e08b26f..961ad449a 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -20,7 +20,7 @@ if( Hermes_INCLUDE_DIR ) #----------------------------------------------------------------------------- find_library( Hermes_LIBRARY - NAMES labstor_client labstor_runtime + NAMES hrun_client hrun_runtime ) # HermesShm diff --git a/CMakeLists.txt b/CMakeLists.txt index 71e56e847..f316ce7ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ add_compile_options(-march=native -fomit-frame-pointer) # by other projects. #----------------------------------------------------------------------------- if(NOT HERMES_EXPORTED_TARGETS) - set(HERMES_EXPORTED_TARGETS "labstor-targets") + set(HERMES_EXPORTED_TARGETS "hrun-targets") endif() #----------------------------------------------------------------------------- @@ -137,7 +137,7 @@ include_directories(${CMAKE_SOURCE_DIR}) include_directories(${CMAKE_SOURCE_DIR}/include) # Task includes include_directories(${CMAKE_SOURCE_DIR}/tasks_required) -include_directories(${CMAKE_SOURCE_DIR}/tasks_required/labstor_admin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/hrun_admin/include) include_directories(${CMAKE_SOURCE_DIR}/tasks_required/small_message/include) include_directories(${CMAKE_SOURCE_DIR}/tasks_required/remote_queue/include) include_directories(${CMAKE_SOURCE_DIR}/tasks_required/worch_proc_round_robin/include) @@ -163,15 +163,15 @@ set(Hermes_CLIENT_LIBRARIES ${HermesShm_LIBRARIES} yaml-cpp cereal::cereal - -ldl -lrt -lc -pthread labstor_client) + -ldl -lrt -lc -pthread hrun_client) set(Hermes_CLIENT_DEPS - labstor_client) + hrun_client) set(Hermes_RUNTIME_LIBRARIES ${Hermes_CLIENT_LIBRARIES} - labstor_runtime + hrun_runtime ${Boost_LIBRARIES}) set(Hermes_RUNTIME_DEPS - labstor_client labstor_runtime) + hrun_client hrun_runtime) set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) add_subdirectory(src) diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc index 12b9a1f63..19ba38889 100644 --- a/benchmark/hermes_api_bench.cc +++ b/benchmark/hermes_api_bench.cc @@ -15,7 +15,7 @@ #include #include "hermes/hermes.h" #include "hermes/bucket.h" -#include "labstor/work_orchestrator/affinity.h" +#include "hrun/work_orchestrator/affinity.h" namespace hapi = hermes; using hshm::MpiTimer; @@ -238,7 +238,7 @@ int main(int argc, char **argv) { MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - TRANSPARENT_LABSTOR(); + TRANSPARENT_HRUN(); HERMES->ClientInit(); // Get mode diff --git a/benchmark/test_init.cc b/benchmark/test_init.cc index e101758d3..1757fe387 100644 --- a/benchmark/test_init.cc +++ b/benchmark/test_init.cc @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "labstor/api/labstor_client.h" +#include "hrun/api/hrun_client.h" #include "basic_test.h" #include "test_init.h" diff --git a/benchmark/test_init.h b/benchmark/test_init.h index a6c71f3ec..cb3b36df7 100644 --- a/benchmark/test_init.h +++ b/benchmark/test_init.h @@ -11,9 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ -#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ +#define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ -#include "labstor/labstor_types.h" +#include "hrun/hrun_types.h" -#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/benchmark/test_latency.cc b/benchmark/test_latency.cc index 80abe788e..a3ecba432 100644 --- a/benchmark/test_latency.cc +++ b/benchmark/test_latency.cc @@ -4,28 +4,28 @@ #include #include "basic_test.h" -#include "labstor/api/labstor_client.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" #include "small_message/small_message.h" #include "hermes_shm/util/timer.h" -#include "labstor/work_orchestrator/affinity.h" +#include "hrun/work_orchestrator/affinity.h" #include "hermes/hermes.h" -#include "labstor/work_orchestrator/worker.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun/work_orchestrator/worker.h" +#include "hrun/api/hrun_runtime.h" /** The performance of getting a queue */ TEST_CASE("TestGetQueue") { - /*labstor::QueueId qid(0, 3); - LABSTOR_ADMIN->CreateQueueRoot(labstor::DomainId::GetLocal(), qid, + /*hrun::QueueId qid(0, 3); + HRUN_ADMIN->CreateQueueRoot(hrun::DomainId::GetLocal(), qid, 16, 16, 256, hshm::bitfield32_t(0)); - LABSTOR_CLIENT->GetQueue(qid); + HRUN_CLIENT->GetQueue(qid); hshm::Timer t; t.Resume(); size_t ops = (1 << 20); for (size_t i = 0; i < ops; ++i) { - labstor::MultiQueue *queue = LABSTOR_CLIENT->GetQueue(qid); + hrun::MultiQueue *queue = HRUN_CLIENT->GetQueue(qid); REQUIRE(queue->id_ == qid); } t.Pause(); @@ -41,12 +41,12 @@ TEST_CASE("TestHshmAllocateFree") { size_t count = (1 << 8); size_t reps = ops / count; for (size_t i = 0; i < reps; ++i) { - std::vector tasks(count); + std::vector tasks(count); for (size_t j = 0; j < count; ++j) { - tasks[j] = LABSTOR_CLIENT->NewTaskRoot().ptr_; + tasks[j] = HRUN_CLIENT->NewTaskRoot().ptr_; } for (size_t j = 0; j < count; ++j) { - LABSTOR_CLIENT->DelTask(tasks[j]); + HRUN_CLIENT->DelTask(tasks[j]); } } t.Pause(); @@ -92,39 +92,39 @@ TEST_CASE("TestPointerQueueVecEmplacePop") { /** Single-thread performance of getting, emplacing, and popping a queue */ TEST_CASE("TestHshmQueueEmplacePop") { - labstor::QueueId qid(0, 3); + hrun::QueueId qid(0, 3); u32 ops = (1 << 20); std::vector queue_info = { {16, 16, ops, 0} }; - auto queue = hipc::make_uptr( + auto queue = hipc::make_uptr( qid, queue_info); - labstor::LaneData entry; - auto task = LABSTOR_CLIENT->NewTaskRoot(); + hrun::LaneData entry; + auto task = HRUN_CLIENT->NewTaskRoot(); entry.p_ = task.shm_; hshm::Timer t; t.Resume(); - labstor::Lane &lane = queue->GetLane(0, 0); + hrun::Lane &lane = queue->GetLane(0, 0); for (size_t i = 0; i < ops; ++i) { queue->Emplace(0, 0, entry); lane.pop(); } t.Pause(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } /** Single-thread performance of getting a lane from a queue */ TEST_CASE("TestHshmQueueGetLane") { - labstor::QueueId qid(0, 3); + hrun::QueueId qid(0, 3); std::vector queue_info = { {16, 16, 256, 0} }; - auto queue = hipc::make_uptr( + auto queue = hipc::make_uptr( qid, queue_info); - labstor::LaneGroup group = queue->GetGroup(0); + hrun::LaneGroup group = queue->GetGroup(0); hshm::Timer t; size_t ops = (1 << 20); @@ -139,24 +139,24 @@ TEST_CASE("TestHshmQueueGetLane") { /** Single-thread performance of getting, emplacing, and popping a queue */ TEST_CASE("TestHshmQueueAllocateEmplacePop") { - labstor::QueueId qid(0, 3); + hrun::QueueId qid(0, 3); std::vector queue_info = { {16, 16, 256, 0} }; - auto queue = hipc::make_uptr( + auto queue = hipc::make_uptr( qid, queue_info); - labstor::Lane &lane = queue->GetLane(0, 0); + hrun::Lane &lane = queue->GetLane(0, 0); hshm::Timer t; size_t ops = (1 << 20); t.Resume(); for (size_t i = 0; i < ops; ++i) { - labstor::LaneData entry; - auto task = LABSTOR_CLIENT->NewTaskRoot(); + hrun::LaneData entry; + auto task = HRUN_CLIENT->NewTaskRoot(); entry.p_ = task.shm_; queue->Emplace(0, 0, entry); lane.pop(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } t.Pause(); @@ -202,16 +202,16 @@ TEST_CASE("TestSpawnJoinArgoThread") { } void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) { - LABSTOR_RUNTIME->Create(); + HRUN_RUNTIME->Create(); - labstor::Worker worker(0); - std::vector> queues; + hrun::Worker worker(0); + std::vector> queues; for (u32 i = 0; i < num_queues; ++i) { - labstor::QueueId qid(0, i + 1); + hrun::QueueId qid(0, i + 1); std::vector queue_info = { {num_lanes, num_lanes, 256, 0} }; - auto queue = hipc::make_uptr( + auto queue = hipc::make_uptr( qid, queue_info); queues.emplace_back(std::move(queue)); for (u32 j = 0; j < num_lanes; ++j) { @@ -219,22 +219,22 @@ void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) { } } - labstor::small_message::Client client; - LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetLocal(), "small_message");\ - client.CreateRoot(labstor::DomainId::GetLocal(), "ipc_test"); + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetLocal(), "small_message");\ + client.CreateRoot(hrun::DomainId::GetLocal(), "ipc_test"); hshm::Timer t; t.Resume(); // size_t ops = (1 << 20); size_t ops = 256; for (size_t i = 0; i < ops; ++i) { - hipc::LPointer task; - labstor::TaskNode task_node(labstor::TaskId((u32)0, (u64)i)); + hipc::LPointer task; + hrun::TaskNode task_node(hrun::TaskId((u32)0, (u64)i)); task = client.AsyncMdPushEmplace(queues[num_queues - 1].get(), task_node, - labstor::DomainId::GetLocal()); + hrun::DomainId::GetLocal()); worker.Run(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } t.Pause(); @@ -243,7 +243,7 @@ void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) { /** Time for worker to process a request */ TEST_CASE("TestWorkerLatency") { - TRANSPARENT_LABSTOR(); + TRANSPARENT_HRUN(); TestWorkerIterationLatency(1, 16); TestWorkerIterationLatency(5, 16); TestWorkerIterationLatency(10, 16); @@ -252,16 +252,16 @@ TEST_CASE("TestWorkerLatency") { /** Time to process a request */ TEST_CASE("TestRoundTripLatency") { - TRANSPARENT_LABSTOR(); + TRANSPARENT_HRUN(); HERMES->ClientInit(); - labstor::small_message::Client client; - LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetLocal(), "small_message"); + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetLocal(), "small_message"); // int count = 25; // for (int i = 0; i < count; ++i) { -// labstor::small_message::Client client2; -// client2.CreateRoot(labstor::DomainId::GetLocal(), "ipc_test" + std::to_string(i)); +// hrun::small_message::Client client2; +// client2.CreateRoot(hrun::DomainId::GetLocal(), "ipc_test" + std::to_string(i)); // } - client.CreateRoot(labstor::DomainId::GetLocal(), "ipc_test"); + client.CreateRoot(hrun::DomainId::GetLocal(), "ipc_test"); hshm::Timer t; int pid = getpid(); @@ -271,7 +271,7 @@ TEST_CASE("TestRoundTripLatency") { size_t ops = (1 << 20); // size_t ops = 1024; for (size_t i = 0; i < ops; ++i) { - client.MdPushRoot(labstor::DomainId::GetLocal()); + client.MdPushRoot(hrun::DomainId::GetLocal()); } t.Pause(); diff --git a/benchmark/test_zmq.cc b/benchmark/test_zmq.cc index 2f95ea8e6..1de149bcf 100644 --- a/benchmark/test_zmq.cc +++ b/benchmark/test_zmq.cc @@ -4,11 +4,11 @@ #include #include "basic_test.h" -#include "labstor/api/labstor_client.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" #include "small_message/small_message.h" #include "hermes_shm/util/timer.h" -#include "labstor/work_orchestrator/affinity.h" +#include "hrun/work_orchestrator/affinity.h" /** ZeroMQ allocate + free request */ TEST_CASE("TestZeromqAllocateFree") { @@ -30,7 +30,7 @@ TEST_CASE("TestZeromqAllocateFree") { t.Resume(); size_t ops = (1 << 20); for (size_t i = 0; i < ops; ++i) { - zmq::message_t message(sizeof(labstor::Task)); + zmq::message_t message(sizeof(hrun::Task)); } t.Pause(); @@ -58,13 +58,13 @@ TEST_CASE("TestZeromqAllocateEmplacePop") { size_t ops = (1 << 20); for (size_t i = 0; i < ops; ++i) { // Send a request - zmq::message_t message(sizeof(labstor::Task)); + zmq::message_t message(sizeof(hrun::Task)); pushSocket.send(message, zmq::send_flags::none); // Receive the request zmq::message_t receivedMessage; zmq::recv_result_t result = pullSocket.recv(receivedMessage); - REQUIRE(receivedMessage.size() == sizeof(labstor::Task)); + REQUIRE(receivedMessage.size() == sizeof(hrun::Task)); } t.Pause(); diff --git a/codegen/codegen/labstor_config/generator.py b/codegen/codegen/labstor_config/generator.py index 29b5f1e56..f3de632e8 100644 --- a/codegen/codegen/labstor_config/generator.py +++ b/codegen/codegen/labstor_config/generator.py @@ -22,10 +22,10 @@ def create_config(path, var_name, config_path, macro_name): # Create the configuration config_lines = [] - config_lines.append(f"#ifndef LABSTOR_SRC_CONFIG_{macro_name}_DEFAULT_H_") - config_lines.append(f"#define LABSTOR_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#ifndef HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#define HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") config_lines += string_lines - config_lines.append(f"#endif // LABSTOR_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#endif // HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") # Persist config = "\n".join(config_lines) diff --git a/codegen/codegen/util/paths.py b/codegen/codegen/util/paths.py index ce4ce8fe9..31b641e4a 100644 --- a/codegen/codegen/util/paths.py +++ b/codegen/codegen/util/paths.py @@ -8,4 +8,4 @@ def GetLabstorRoot(): hermes_path = os.path.dirname(code_generators_path) return hermes_path -LABSTOR_ROOT=GetLabstorRoot() \ No newline at end of file +HRUN_ROOT=GetLabstorRoot() \ No newline at end of file diff --git a/codegen/hermes_config b/codegen/hermes_config index 28a523f97..4bbdea946 100755 --- a/codegen/hermes_config +++ b/codegen/hermes_config @@ -10,10 +10,10 @@ OUTPUT: ${HERMES}/src/config_server_default.h (if server) """ -from codegen.labstor_config.generator import create_config -from codegen.util.paths import LABSTOR_ROOT +from codegen.hrun_config.generator import create_config +from codegen.util.paths import HRUN_ROOT -HERMES_ROOT = f'{LABSTOR_ROOT}/tasks/hermes' +HERMES_ROOT = f'{HRUN_ROOT}/tasks/hermes' create_config( path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", var_name="kHermesClientDefaultConfigStr", diff --git a/codegen/labstor_config b/codegen/labstor_config index 7060e4c95..49183dd8f 100755 --- a/codegen/labstor_config +++ b/codegen/labstor_config @@ -3,26 +3,26 @@ """ USAGE: cd codegen/bin - python3 labstor_config.py + python3 hrun_config.py OUTPUT: - ${LABSTOR}/src/config_client_default.h (if client) - ${LABSTOR}/src/config_server_default.h (if server) + ${HRUN}/src/config_client_default.h (if client) + ${HRUN}/src/config_server_default.h (if server) """ -from codegen.labstor_config.generator import create_config -from codegen.util.paths import LABSTOR_ROOT +from codegen.hrun_config.generator import create_config +from codegen.util.paths import HRUN_ROOT create_config( - path=f"{LABSTOR_ROOT}/config/labstor_client_default.yaml", + path=f"{HRUN_ROOT}/config/hrun_client_default.yaml", var_name="kLabstorClientDefaultConfigStr", - config_path=f"{LABSTOR_ROOT}/include/labstor/config/config_client_default.h", + config_path=f"{HRUN_ROOT}/include/hrun/config/config_client_default.h", macro_name="CLIENT" ) create_config( - path=f"{LABSTOR_ROOT}/config/labstor_server_default.yaml", + path=f"{HRUN_ROOT}/config/hrun_server_default.yaml", var_name="kLabstorServerDefaultConfigStr", - config_path=f"{LABSTOR_ROOT}/include/labstor/config/config_server_default.h", + config_path=f"{HRUN_ROOT}/include/hrun/config/config_server_default.h", macro_name="SERVER" ) \ No newline at end of file diff --git a/codegen/make_macro b/codegen/make_macro index d9d0c7a95..24490af05 100755 --- a/codegen/make_macro +++ b/codegen/make_macro @@ -6,7 +6,7 @@ USAGE: ./make_macro [PATH] import os import sys -from codegen.labstor_config.generator import print_macro +from codegen.hrun_config.generator import print_macro PATH = sys.argv[1] MACRO_NAME = os.path.basename(PATH).upper().split('.')[0] diff --git a/codegen/make_task b/codegen/make_task index 0330e2784..6a54f2fb8 100755 --- a/codegen/make_task +++ b/codegen/make_task @@ -6,7 +6,7 @@ USAGE: ./make_task [TASK_ROOT] import os import sys -from codegen.util.paths import LABSTOR_ROOT +from codegen.util.paths import HRUN_ROOT def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: @@ -18,7 +18,7 @@ def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): TASK_ROOT = sys.argv[1] TASK_NAME = os.path.basename(TASK_ROOT) -TASK_TEMPL_ROOT = f'{LABSTOR_ROOT}/tasks_required/TASK_NAME' +TASK_TEMPL_ROOT = f'{HRUN_ROOT}/tasks_required/TASK_NAME' os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) diff --git a/codegen/preamble.py b/codegen/preamble.py index 46036cffa..9813b2826 100644 --- a/codegen/preamble.py +++ b/codegen/preamble.py @@ -2,7 +2,7 @@ Prepends the preabmle to all files in the repo USAGE: - python3 scripts/preamble.py ${LABSTOR_ROOT} + python3 scripts/preamble.py ${HRUN_ROOT} """ import sys,os,re @@ -36,7 +36,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ """.strip() -labstor_preamble = """ +hrun_preamble = """ /* * Copyright (C) 2022 SCS Lab , * Luke Logan , @@ -69,7 +69,7 @@ def PrependPreamble(path): text = fp.read() text = text.replace(hermes_preamble, "") text = text.replace(preamble2, "") - text = text.replace(labstor_preamble, "") + text = text.replace(hrun_preamble, "") text = re.sub("//\n// Created by [^\n]*\n//\n", "", text) text = text.strip() text = hermes_preamble + "\n\n" + text + "\n" diff --git a/codegen/refresh_methods b/codegen/refresh_methods index 1e9484641..cad1bfac4 100755 --- a/codegen/refresh_methods +++ b/codegen/refresh_methods @@ -7,7 +7,7 @@ USAGE: ./referesh_methods [TASK_DIR] import yaml import os import sys -from codegen.util.paths import LABSTOR_ROOT +from codegen.util.paths import HRUN_ROOT def refresh_methods(TASK_ROOT): if not os.path.exists(f'{TASK_ROOT}/include'): @@ -16,8 +16,8 @@ def refresh_methods(TASK_ROOT): METHODS_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.h' METHODS_YAML = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.yaml' LIB_EXEC_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_lib_exec.h' - METHOD_MACRO = f'LABSTOR_{TASK_NAME.upper()}_METHODS_H_' - LIB_EXEC_MACRO = f'LABSTOR_{TASK_NAME.upper()}_LIB_EXEC_H_' + METHOD_MACRO = f'HRUN_{TASK_NAME.upper()}_METHODS_H_' + LIB_EXEC_MACRO = f'HRUN_{TASK_NAME.upper()}_LIB_EXEC_H_' with open(METHODS_YAML) as fp: methods = yaml.load(fp, Loader=yaml.FullLoader) @@ -26,7 +26,7 @@ def refresh_methods(TASK_ROOT): if 'kLast' in methods: del methods['kLast'] methods = sorted(methods.items(), key=lambda x: x[1]) - if TASK_NAME != 'labstor_admin': + if TASK_NAME != 'hrun_admin': methods.insert(0, ('kConstruct', -2)) methods.insert(1, ('kDestruct', -1)) @@ -75,7 +75,7 @@ def refresh_methods(TASK_ROOT): method_name = method_enum_name.replace('k', '', 1) task_name = method_name + "Task" lines += [f' case Method::{method_enum_name}: {{', - f' LABSTOR_CLIENT->DelTask(reinterpret_cast<{task_name} *>(task));', + f' HRUN_CLIENT->DelTask(reinterpret_cast<{task_name} *>(task));', f' break;', f' }}'] lines += [' }'] @@ -89,7 +89,7 @@ def refresh_methods(TASK_ROOT): method_name = method_enum_name.replace('k', '', 1) task_name = method_name + "Task" lines += [f' case Method::{method_enum_name}: {{', - f' labstor::CALL_DUPLICATE(reinterpret_cast<{task_name}*>(orig_task), dups);', + f' hrun::CALL_DUPLICATE(reinterpret_cast<{task_name}*>(orig_task), dups);', f' break;', f' }}'] lines += [' }'] @@ -104,7 +104,7 @@ def refresh_methods(TASK_ROOT): task_name = method_name + "Task" task_cast = f'reinterpret_cast<{task_name}*>' lines += [f' case Method::{method_enum_name}: {{', - f' labstor::CALL_DUPLICATE_END(replica, {task_cast}(orig_task), {task_cast}(dup_task));', + f' hrun::CALL_DUPLICATE_END(replica, {task_cast}(orig_task), {task_cast}(dup_task));', f' break;', f' }}'] lines += [' }'] @@ -118,7 +118,7 @@ def refresh_methods(TASK_ROOT): method_name = method_enum_name.replace('k', '', 1) task_name = method_name + "Task" lines += [f' case Method::{method_enum_name}: {{', - f' labstor::CALL_REPLICA_START(count, reinterpret_cast<{task_name}*>(task));', + f' hrun::CALL_REPLICA_START(count, reinterpret_cast<{task_name}*>(task));', f' break;', f' }}'] lines += [' }'] @@ -132,7 +132,7 @@ def refresh_methods(TASK_ROOT): method_name = method_enum_name.replace('k', '', 1) task_name = method_name + "Task" lines += [f' case Method::{method_enum_name}: {{', - f' labstor::CALL_REPLICA_END(reinterpret_cast<{task_name}*>(task));', + f' hrun::CALL_REPLICA_END(reinterpret_cast<{task_name}*>(task));', f' break;', f' }}'] lines += [' }'] @@ -162,7 +162,7 @@ def refresh_methods(TASK_ROOT): method_name = method_enum_name.replace('k', '', 1) task_name = method_name + "Task" lines += [f' case Method::{method_enum_name}: {{', - f' task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask<{task_name}>(task_ptr.shm_);', + f' task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask<{task_name}>(task_ptr.shm_);', f' ar >> *reinterpret_cast<{task_name}*>(task_ptr.ptr_);', f' break;', f' }}'] diff --git a/codegen/update_task b/codegen/update_task index ffcdd5015..3445a239c 100755 --- a/codegen/update_task +++ b/codegen/update_task @@ -6,7 +6,7 @@ USAGE: ./make_task [TASK_ROOT] import os import sys -from codegen.util.paths import LABSTOR_ROOT +from codegen.util.paths import HRUN_ROOT def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: @@ -18,7 +18,7 @@ def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): TASK_ROOT = sys.argv[1] TASK_NAME = os.path.basename(TASK_ROOT) -TASK_TEMPL_ROOT = f'{LABSTOR_ROOT}/tasks_required/TASK_NAME +TASK_TEMPL_ROOT = f'{HRUN_ROOT}/tasks_required/TASK_NAME os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) diff --git a/config/labstor_server_default.yaml b/config/labstor_server_default.yaml index 977ea6c0c..f31f30ce0 100644 --- a/config/labstor_server_default.yaml +++ b/config/labstor_server_default.yaml @@ -14,7 +14,7 @@ queue_manager: # The shared memory allocator to use shm_allocator: kScalablePageAllocator # The name of the shared memory region to create - shm_name: "labstor_shm" + shm_name: "hrun_shm" # The size of the shared memory region to allocate for general data structures shm_size: 0g # The size of the shared memory to allocate for data buffers diff --git a/include/labstor/api/labstor_client.h b/include/hrun/api/hrun_client.h similarity index 86% rename from include/labstor/api/labstor_client.h rename to include/hrun/api/hrun_client.h index 35523ab5e..72d9aac3a 100644 --- a/include/labstor/api/labstor_client.h +++ b/include/hrun/api/hrun_client.h @@ -2,18 +2,18 @@ // Created by lukemartinlogan on 6/23/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_CLIENT_H_ -#define LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_CLIENT_H_ +#ifndef HRUN_INCLUDE_HRUN_CLIENT_HRUN_CLIENT_H_ +#define HRUN_INCLUDE_HRUN_CLIENT_HRUN_CLIENT_H_ #include #include "manager.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/queue_manager/queue_manager_client.h" // Singleton macros -#define LABSTOR_CLIENT hshm::Singleton::GetInstance() -#define LABSTOR_CLIENT_T labstor::Client* +#define HRUN_CLIENT hshm::Singleton::GetInstance() +#define HRUN_CLIENT_T hrun::Client* -namespace labstor { +namespace hrun { class Client : public ConfigurationManager { public: @@ -246,19 +246,19 @@ class Client : public ConfigurationManager { }; /** A function which creates a new TaskNode value */ -#define LABSTOR_TASK_NODE_ROOT(CUSTOM)\ +#define HRUN_TASK_NODE_ROOT(CUSTOM)\ template\ auto CUSTOM##Root(Args&& ...args) {\ - TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId();\ + TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId();\ return CUSTOM(task_node, std::forward(args)...);\ } /** Fill in common default parameters for task client wrapper function */ -#define LABSTOR_TASK_NODE_ADMIN_ROOT(CUSTOM)\ +#define HRUN_TASK_NODE_ADMIN_ROOT(CUSTOM)\ template\ hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node,\ Args&& ...args) {\ - hipc::LPointer task = LABSTOR_CLIENT->AllocateTask();\ + hipc::LPointer task = HRUN_CLIENT->AllocateTask();\ Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...);\ return task;\ }\ @@ -266,7 +266,7 @@ class Client : public ConfigurationManager { hipc::LPointer Async##CUSTOM(const TaskNode &task_node, \ Args&& ...args) {\ hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...);\ - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_);\ + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_);\ queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_);\ return task;\ }\ @@ -280,17 +280,17 @@ class Client : public ConfigurationManager { }\ template\ hipc::LPointer Async##CUSTOM##Root(Args&& ...args) {\ - TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId();\ + TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId();\ hipc::LPointer task = Async##CUSTOM(task_node + 1, std::forward(args)...);\ return task;\ } /** The default asynchronous method behavior */ -#define LABSTOR_TASK_NODE_PUSH_ROOT(CUSTOM)\ +#define HRUN_TASK_NODE_PUSH_ROOT(CUSTOM)\ template\ hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node,\ Args&& ...args) {\ - hipc::LPointer task = LABSTOR_CLIENT->AllocateTask();\ + hipc::LPointer task = HRUN_CLIENT->AllocateTask();\ Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...);\ return task;\ }\ @@ -298,7 +298,7 @@ class Client : public ConfigurationManager { hipc::LPointer Async##CUSTOM(const TaskNode &task_node, \ Args&& ...args) {\ hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...);\ - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_);\ + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_);\ queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_);\ return task;\ }\ @@ -311,11 +311,11 @@ class Client : public ConfigurationManager { return task;\ }\ template\ - hipc::LPointer> Async##CUSTOM##Root(Args&& ...args) {\ - TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId();\ + hipc::LPointer> Async##CUSTOM##Root(Args&& ...args) {\ + TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId();\ hipc::LPointer task = Async##CUSTOM##Alloc(task_node + 1, std::forward(args)...);\ - hipc::LPointer> push_task =\ - LABSTOR_PROCESS_QUEUE->AsyncPush(task_node,\ + hipc::LPointer> push_task =\ + HRUN_PROCESS_QUEUE->AsyncPush(task_node,\ DomainId::GetLocal(),\ task);\ return push_task;\ @@ -326,8 +326,8 @@ template constexpr inline void CALL_DUPLICATE(TaskT *orig_task, std::vector> &dups) { if constexpr(TaskT::REPLICA) { for (LPointer &dup : dups) { - LPointer task = LABSTOR_CLIENT->NewEmptyTask(); - task.ptr_->Dup(LABSTOR_CLIENT->main_alloc_, *orig_task); + LPointer task = HRUN_CLIENT->NewEmptyTask(); + task.ptr_->Dup(HRUN_CLIENT->main_alloc_, *orig_task); dup.ptr_ = task.ptr_; dup.shm_ = task.shm_; } @@ -355,19 +355,19 @@ constexpr inline void CALL_REPLICA_END(TaskT *task) { } } -} // namespace labstor +} // namespace hrun -static inline bool TRANSPARENT_LABSTOR() { - if (!LABSTOR_CLIENT->IsInitialized() && - !LABSTOR_CLIENT->IsBeingInitialized() && - !LABSTOR_CLIENT->IsTerminated()) { - LABSTOR_CLIENT->Create(); - LABSTOR_CLIENT->is_transparent_ = true; +static inline bool TRANSPARENT_HRUN() { + if (!HRUN_CLIENT->IsInitialized() && + !HRUN_CLIENT->IsBeingInitialized() && + !HRUN_CLIENT->IsTerminated()) { + HRUN_CLIENT->Create(); + HRUN_CLIENT->is_transparent_ = true; return true; } return false; } -#define HASH_TO_NODE_ID(hash) (1 + ((hash) % LABSTOR_CLIENT->GetNumNodes())) +#define HASH_TO_NODE_ID(hash) (1 + ((hash) % HRUN_CLIENT->GetNumNodes())) -#endif // LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_CLIENT_H_ +#endif // HRUN_INCLUDE_HRUN_CLIENT_HRUN_CLIENT_H_ diff --git a/include/labstor/api/labstor_runtime.h b/include/hrun/api/hrun_runtime.h similarity index 74% rename from include/labstor/api/labstor_runtime.h rename to include/hrun/api/hrun_runtime.h index 4feb03bab..bfc806217 100644 --- a/include/labstor/api/labstor_runtime.h +++ b/include/hrun/api/hrun_runtime.h @@ -2,27 +2,27 @@ // Created by lukemartinlogan on 6/27/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_SERVER_H_ -#define LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_SERVER_H_ +#ifndef HRUN_INCLUDE_HRUN_CLIENT_HRUN_SERVER_H_ +#define HRUN_INCLUDE_HRUN_CLIENT_HRUN_SERVER_H_ -#include "labstor/task_registry/task_registry.h" -#include "labstor/work_orchestrator/work_orchestrator.h" -#include "labstor/queue_manager/queue_manager_runtime.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun/task_registry/task_registry.h" +#include "hrun/work_orchestrator/work_orchestrator.h" +#include "hrun/queue_manager/queue_manager_runtime.h" +#include "hrun_admin/hrun_admin.h" #include "remote_queue/remote_queue.h" -#include "labstor_client.h" +#include "hrun_client.h" #include "manager.h" -#include "labstor/network/rpc.h" -#include "labstor/network/rpc_thallium.h" +#include "hrun/network/rpc.h" +#include "hrun/network/rpc_thallium.h" // Singleton macros -#define LABSTOR_RUNTIME hshm::Singleton::GetInstance() -#define LABSTOR_RUNTIME_T labstor::Runtime* -#define LABSTOR_REMOTE_QUEUE (&LABSTOR_RUNTIME->remote_queue_) -#define LABSTOR_THALLIUM (&LABSTOR_RUNTIME->thallium_) -#define LABSTOR_RPC (&LABSTOR_RUNTIME->rpc_) +#define HRUN_RUNTIME hshm::Singleton::GetInstance() +#define HRUN_RUNTIME_T hrun::Runtime* +#define HRUN_REMOTE_QUEUE (&HRUN_RUNTIME->remote_queue_) +#define HRUN_THALLIUM (&HRUN_RUNTIME->thallium_) +#define HRUN_RPC (&HRUN_RUNTIME->rpc_) -namespace labstor { +namespace hrun { class Runtime : public ConfigurationManager { public: @@ -69,33 +69,33 @@ class Runtime : public ConfigurationManager { rpc_.node_id_, &server_config_, header_->queue_manager_); - LABSTOR_CLIENT->Create(server_config_path, "", true); + HRUN_CLIENT->Create(server_config_path, "", true); HERMES_THREAD_MODEL->SetThreadModel(hshm::ThreadType::kPthread); work_orchestrator_.ServerInit(&server_config_, queue_manager_); hipc::mptr admin_task; // Create the admin library - LABSTOR_CLIENT->MakeTaskStateId(); + HRUN_CLIENT->MakeTaskStateId(); admin_task = hipc::make_mptr(); - task_registry_.RegisterTaskLib("labstor_admin"); + task_registry_.RegisterTaskLib("hrun_admin"); task_registry_.CreateTaskState( - "labstor_admin", - "labstor_admin", - LABSTOR_QM_CLIENT->admin_task_state_, + "hrun_admin", + "hrun_admin", + HRUN_QM_CLIENT->admin_task_state_, admin_task.get()); // Create the process queue - LABSTOR_CLIENT->MakeTaskStateId(); + HRUN_CLIENT->MakeTaskStateId(); admin_task = hipc::make_mptr(); task_registry_.RegisterTaskLib("proc_queue"); task_registry_.CreateTaskState( "proc_queue", "proc_queue", - LABSTOR_QM_CLIENT->process_queue_, + HRUN_QM_CLIENT->process_queue_, admin_task.get()); // Create the work orchestrator queue scheduling library - TaskStateId queue_sched_id = LABSTOR_CLIENT->MakeTaskStateId(); + TaskStateId queue_sched_id = HRUN_CLIENT->MakeTaskStateId(); admin_task = hipc::make_mptr(); task_registry_.RegisterTaskLib("worch_queue_round_robin"); task_registry_.CreateTaskState( @@ -105,7 +105,7 @@ class Runtime : public ConfigurationManager { admin_task.get()); // Create the work orchestrator process scheduling library - TaskStateId proc_sched_id = LABSTOR_CLIENT->MakeTaskStateId(); + TaskStateId proc_sched_id = HRUN_CLIENT->MakeTaskStateId(); admin_task = hipc::make_mptr(); task_registry_.RegisterTaskLib("worch_proc_round_robin"); task_registry_.CreateTaskState( @@ -115,13 +115,13 @@ class Runtime : public ConfigurationManager { admin_task.get()); // Set the work orchestrator queue scheduler - LABSTOR_ADMIN->SetWorkOrchQueuePolicyRoot(labstor::DomainId::GetLocal(), queue_sched_id); - LABSTOR_ADMIN->SetWorkOrchProcPolicyRoot(labstor::DomainId::GetLocal(), proc_sched_id); + HRUN_ADMIN->SetWorkOrchQueuePolicyRoot(hrun::DomainId::GetLocal(), queue_sched_id); + HRUN_ADMIN->SetWorkOrchProcPolicyRoot(hrun::DomainId::GetLocal(), proc_sched_id); // Create the remote queue library task_registry_.RegisterTaskLib("remote_queue"); remote_queue_.CreateRoot(DomainId::GetLocal(), "remote_queue", - LABSTOR_CLIENT->MakeTaskStateId()); + HRUN_CLIENT->MakeTaskStateId()); remote_created_ = true; } @@ -152,18 +152,18 @@ class Runtime : public ConfigurationManager { void RunDaemon() { thallium_.RunDaemon(); HILOG(kInfo, "Daemon is running") -// while (LABSTOR_WORK_ORCHESTRATOR->IsRuntimeAlive()) { +// while (HRUN_WORK_ORCHESTRATOR->IsRuntimeAlive()) { // // Scheduler callbacks? // HERMES_THREAD_MODEL->SleepForUs(1000); // } HILOG(kInfo, "Finishing up last requests") - LABSTOR_WORK_ORCHESTRATOR->Join(); + HRUN_WORK_ORCHESTRATOR->Join(); HILOG(kInfo, "Daemon is exiting") } /** Stop the Hermes core Daemon */ void StopDaemon() { - LABSTOR_WORK_ORCHESTRATOR->FinalizeRuntime(); + HRUN_WORK_ORCHESTRATOR->FinalizeRuntime(); } /** Get the set of DomainIds */ @@ -183,6 +183,6 @@ class Runtime : public ConfigurationManager { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_CLIENT_LABSTOR_SERVER_H_ +#endif // HRUN_INCLUDE_HRUN_CLIENT_HRUN_SERVER_H_ diff --git a/include/labstor/api/manager.h b/include/hrun/api/manager.h similarity index 83% rename from include/labstor/api/manager.h rename to include/hrun/api/manager.h index d4a27f119..41948d734 100644 --- a/include/labstor/api/manager.h +++ b/include/hrun/api/manager.h @@ -2,16 +2,16 @@ // Created by lukemartinlogan on 6/27/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_MANAGER_MANAGER_H_ -#define LABSTOR_INCLUDE_LABSTOR_MANAGER_MANAGER_H_ +#ifndef HRUN_INCLUDE_HRUN_MANAGER_MANAGER_H_ +#define HRUN_INCLUDE_HRUN_MANAGER_MANAGER_H_ -#include "labstor/labstor_types.h" -#include "labstor/labstor_constants.h" -#include "labstor/config/config_client.h" -#include "labstor/config/config_server.h" -#include "labstor/queue_manager/queue_manager.h" +#include "hrun/hrun_types.h" +#include "hrun/hrun_constants.h" +#include "hrun/config/config_client.h" +#include "hrun/config/config_server.h" +#include "hrun/queue_manager/queue_manager.h" -namespace labstor { +namespace hrun { /** Shared-memory header for LabStor */ struct LabstorShm { @@ -80,6 +80,6 @@ class ConfigurationManager { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_MANAGER_MANAGER_H_ +#endif // HRUN_INCLUDE_HRUN_MANAGER_MANAGER_H_ diff --git a/include/labstor/api/template/labstor_task_node_admin_root.template b/include/hrun/api/template/hrun_task_node_admin_root.template similarity index 86% rename from include/labstor/api/template/labstor_task_node_admin_root.template rename to include/hrun/api/template/hrun_task_node_admin_root.template index acc5327cc..7df57a967 100644 --- a/include/labstor/api/template/labstor_task_node_admin_root.template +++ b/include/hrun/api/template/hrun_task_node_admin_root.template @@ -1,7 +1,7 @@ template hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node, Args&& ...args) { - hipc::LPointer task = LABSTOR_CLIENT->AllocateTask(); + hipc::LPointer task = HRUN_CLIENT->AllocateTask(); Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...); return task; } @@ -9,7 +9,7 @@ template hipc::LPointer Async##CUSTOM(const TaskNode &task_node, Args&& ...args) { hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); return task; } @@ -23,7 +23,7 @@ hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue, } template hipc::LPointer Async##CUSTOM##Root(Args&& ...args) { - TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId(); + TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId(); hipc::LPointer task = Async##CUSTOM(task_node + 1, std::forward(args)...); return task; } \ No newline at end of file diff --git a/include/labstor/api/template/labstor_task_node_push_root.template b/include/hrun/api/template/hrun_task_node_push_root.template similarity index 76% rename from include/labstor/api/template/labstor_task_node_push_root.template rename to include/hrun/api/template/hrun_task_node_push_root.template index 37b909500..ab274c4e8 100644 --- a/include/labstor/api/template/labstor_task_node_push_root.template +++ b/include/hrun/api/template/hrun_task_node_push_root.template @@ -1,7 +1,7 @@ template hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node, Args&& ...args) { - hipc::LPointer task = LABSTOR_CLIENT->AllocateTask(); + hipc::LPointer task = HRUN_CLIENT->AllocateTask(); Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...); return task; } @@ -9,7 +9,7 @@ template hipc::LPointer Async##CUSTOM(const TaskNode &task_node, Args&& ...args) { hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); return task; } @@ -22,11 +22,11 @@ hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue, return task; } template -hipc::LPointer> Async##CUSTOM##Root(Args&& ...args) { - TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId(); +hipc::LPointer> Async##CUSTOM##Root(Args&& ...args) { + TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId(); hipc::LPointer task = Async##CUSTOM##Alloc(task_node + 1, std::forward(args)...); - hipc::LPointer> push_task = - LABSTOR_PROCESS_QUEUE->AsyncPush(task_node, + hipc::LPointer> push_task = + HRUN_PROCESS_QUEUE->AsyncPush(task_node, DomainId::GetLocal(), task); return push_task; diff --git a/include/labstor/config/config.h b/include/hrun/config/config.h similarity index 91% rename from include/labstor/config/config.h rename to include/hrun/config/config.h index 151c997cc..77ac1d71d 100644 --- a/include/labstor/config/config.h +++ b/include/hrun/config/config.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/17/23. // -#ifndef LABSTOR_SRC_CONFIG_H_ -#define LABSTOR_SRC_CONFIG_H_ +#ifndef HRUN_SRC_CONFIG_H_ +#define HRUN_SRC_CONFIG_H_ #include #include @@ -14,7 +14,7 @@ #include #include "hermes_shm/util/config_parse.h" -namespace labstor::config { +namespace hrun::config { /** * Base class for configuration files @@ -68,6 +68,6 @@ class BaseConfig { virtual void ParseYAML(YAML::Node &yaml_conf) = 0; }; -} // namespace labstor::config +} // namespace hrun::config -#endif // LABSTOR_SRC_CONFIG_H_ +#endif // HRUN_SRC_CONFIG_H_ diff --git a/include/labstor/config/config_client.h b/include/hrun/config/config_client.h similarity index 85% rename from include/labstor/config/config_client.h rename to include/hrun/config/config_client.h index 8636eaab5..49e2e0f2f 100644 --- a/include/labstor/config/config_client.h +++ b/include/hrun/config/config_client.h @@ -10,15 +10,15 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_SRC_CONFIG_CLIENT_H_ -#define LABSTOR_SRC_CONFIG_CLIENT_H_ +#ifndef HRUN_SRC_CONFIG_CLIENT_H_ +#define HRUN_SRC_CONFIG_CLIENT_H_ #include #include "config.h" namespace stdfs = std::filesystem; -namespace labstor::config { +namespace hrun::config { /** * Configuration used to intialize client @@ -33,10 +33,10 @@ class ClientConfig : public BaseConfig { void LoadDefault() override; }; -} // namespace labstor::config +} // namespace hrun::config -namespace labstor { +namespace hrun { using ClientConfig = config::ClientConfig; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_SRC_CONFIG_CLIENT_H_ +#endif // HRUN_SRC_CONFIG_CLIENT_H_ diff --git a/include/hrun/config/config_client_default.h b/include/hrun/config/config_client_default.h new file mode 100644 index 000000000..2056e1ebe --- /dev/null +++ b/include/hrun/config/config_client_default.h @@ -0,0 +1,5 @@ +#ifndef HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ +#define HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ +const char* kLabstorClientDefaultConfigStr = +"thread_model: kStd\n"; +#endif // HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/include/labstor/config/config_server.h b/include/hrun/config/config_server.h similarity index 91% rename from include/labstor/config/config_server.h rename to include/hrun/config/config_server.h index be0af57ed..e72183548 100644 --- a/include/labstor/config/config_server.h +++ b/include/hrun/config/config_server.h @@ -10,13 +10,13 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_SRC_CONFIG_SERVER_H_ -#define LABSTOR_SRC_CONFIG_SERVER_H_ +#ifndef HRUN_SRC_CONFIG_SERVER_H_ +#define HRUN_SRC_CONFIG_SERVER_H_ #include "config.h" -#include "labstor/labstor_types.h" +#include "hrun/hrun_types.h" -namespace labstor::config { +namespace hrun::config { /** * Work orchestrator information defined in server config @@ -87,10 +87,10 @@ class ServerConfig : public BaseConfig { void ParseRpcInfo(YAML::Node yaml_conf); }; -} // namespace labstor::config +} // namespace hrun::config -namespace labstor { +namespace hrun { using ServerConfig = config::ServerConfig; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_SRC_CONFIG_SERVER_H_ +#endif // HRUN_SRC_CONFIG_SERVER_H_ diff --git a/include/labstor/config/config_server_default.h b/include/hrun/config/config_server_default.h similarity index 92% rename from include/labstor/config/config_server_default.h rename to include/hrun/config/config_server_default.h index 009f7e1f2..6a112d3f2 100644 --- a/include/labstor/config/config_server_default.h +++ b/include/hrun/config/config_server_default.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ -#define LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ +#ifndef HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ +#define HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ const char* kLabstorServerDefaultConfigStr = "### Runtime orchestration settings\n" "work_orchestrator:\n" @@ -17,7 +17,7 @@ const char* kLabstorServerDefaultConfigStr = " # The shared memory allocator to use\n" " shm_allocator: kScalablePageAllocator\n" " # The name of the shared memory region to create\n" -" shm_name: \"labstor_shm\"\n" +" shm_name: \"hrun_shm\"\n" " # The size of the shared memory region to allocate for general data structures\n" " shm_size: 0g\n" " # The size of the shared memory to allocate for data buffers\n" @@ -58,4 +58,4 @@ const char* kLabstorServerDefaultConfigStr = " \'posix_bdev\',\n" " \'ram_bdev\'\n" "]\n"; -#endif // LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file +#endif // HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file diff --git a/include/hrun/hrun_constants.h b/include/hrun/hrun_constants.h new file mode 100644 index 000000000..8c710c8c1 --- /dev/null +++ b/include/hrun/hrun_constants.h @@ -0,0 +1,29 @@ +// +// Created by lukemartinlogan on 6/22/23. +// + +#ifndef HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ +#define HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ + +namespace hrun { + +#include + +class Constants { + public: + inline static const std::string kClientConfEnv = "HRUN_CLIENT_CONF"; + inline static const std::string kServerConfEnv = "HRUN_SERVER_CONF"; + + static std::string GetEnvSafe(const std::string &env_name) { + char *data = getenv(env_name.c_str()); + if (data == nullptr) { + return ""; + } else { + return data; + } + } +}; + +} // namespace hrun + +#endif // HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ diff --git a/include/hrun/hrun_namespace.h b/include/hrun/hrun_namespace.h new file mode 100644 index 000000000..20053e3bd --- /dev/null +++ b/include/hrun/hrun_namespace.h @@ -0,0 +1,39 @@ +// +// Created by lukemartinlogan on 8/14/23. +// + +#ifndef HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ +#define HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ + +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" + +using hrun::TaskMethod; +using hrun::BinaryOutputArchive; +using hrun::BinaryInputArchive; +using hrun::Task; +using hrun::TaskPointer; +using hrun::MultiQueue; +using hrun::PriorityInfo; +using hrun::TaskNode; +using hrun::DomainId; +using hrun::TaskStateId; +using hrun::QueueId; +using hrun::TaskFlags; +using hrun::DataTransfer; +using hrun::TaskLib; +using hrun::TaskLibClient; +using hrun::config::QueueManagerInfo; +using hrun::TaskPrio; +using hrun::RunContext; + +using hshm::RwLock; +using hshm::Mutex; +using hshm::bitfield; +using hshm::bitfield32_t; +typedef hshm::bitfield bitfield64_t; +using hshm::ScopedRwReadLock; +using hshm::ScopedRwWriteLock; +using hipc::LPointer; + +#endif // HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ diff --git a/include/labstor/labstor_types.h b/include/hrun/hrun_types.h similarity index 96% rename from include/labstor/labstor_types.h rename to include/hrun/hrun_types.h index 9d01597b3..7d9947097 100644 --- a/include/labstor/labstor_types.h +++ b/include/hrun/hrun_types.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/22/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_LABSTOR_TYPES_H_ -#define LABSTOR_INCLUDE_LABSTOR_LABSTOR_TYPES_H_ +#ifndef HRUN_INCLUDE_HRUN_HRUN_TYPES_H_ +#define HRUN_INCLUDE_HRUN_HRUN_TYPES_H_ #include #include @@ -43,7 +43,7 @@ typedef int64_t i64; /**< 64-bit signed integer */ typedef float f32; /**< 32-bit float */ typedef double f64; /**< 64-bit float */ -namespace labstor { +namespace hrun { using hshm::RwLock; using hshm::Mutex; @@ -361,15 +361,15 @@ enum class IoType { kNone }; -} // namespace labstor +} // namespace hrun namespace std { /** Hash function for UniqueId */ template -struct hash> { +struct hash> { HSHM_ALWAYS_INLINE - std::size_t operator()(const labstor::UniqueId &key) const { + std::size_t operator()(const hrun::UniqueId &key) const { return std::hash{}(key.unique_) + std::hash{}(key.node_id_); @@ -378,9 +378,9 @@ struct hash> { /** Hash function for DomainId */ template<> -struct hash { +struct hash { HSHM_ALWAYS_INLINE - std::size_t operator()(const labstor::DomainId &key) const { + std::size_t operator()(const hrun::DomainId &key) const { return std::hash{}(key.id_) + std::hash{}(key.flags_.bits_); @@ -389,4 +389,4 @@ struct hash { } // namespace std -#endif // LABSTOR_INCLUDE_LABSTOR_LABSTOR_TYPES_H_ +#endif // HRUN_INCLUDE_HRUN_HRUN_TYPES_H_ diff --git a/include/labstor/network/local_serialize.h b/include/hrun/network/local_serialize.h similarity index 89% rename from include/labstor/network/local_serialize.h rename to include/hrun/network/local_serialize.h index 09eb83468..dbf94dc6d 100644 --- a/include/labstor/network/local_serialize.h +++ b/include/hrun/network/local_serialize.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 9/5/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_NETWORK_LOCAL_SERIALIZE_H_ -#define LABSTOR_INCLUDE_LABSTOR_NETWORK_LOCAL_SERIALIZE_H_ +#ifndef HRUN_INCLUDE_HRUN_NETWORK_LOCAL_SERIALIZE_H_ +#define HRUN_INCLUDE_HRUN_NETWORK_LOCAL_SERIALIZE_H_ #include "hermes_shm/data_structures/data_structure.h" -namespace labstor { +namespace hrun { /** A class for serializing simple objects into private memory */ template @@ -71,6 +71,6 @@ class LocalDeserialize { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_NETWORK_LOCAL_SERIALIZE_H_ +#endif // HRUN_INCLUDE_HRUN_NETWORK_LOCAL_SERIALIZE_H_ diff --git a/include/labstor/network/rpc.h b/include/hrun/network/rpc.h similarity index 97% rename from include/labstor/network/rpc.h rename to include/hrun/network/rpc.h index ce4546cef..27f655342 100644 --- a/include/labstor/network/rpc.h +++ b/include/hrun/network/rpc.h @@ -10,8 +10,8 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_RPC_H_ -#define LABSTOR_RPC_H_ +#ifndef HRUN_RPC_H_ +#define HRUN_RPC_H_ #include #include @@ -25,10 +25,10 @@ #include #include -#include "labstor/labstor_types.h" -#include "labstor/config/config_server.h" +#include "hrun/hrun_types.h" +#include "hrun/config/config_server.h" -namespace labstor { +namespace hrun { /** RPC types */ enum class RpcType { @@ -223,4 +223,4 @@ class RpcContext { } // namespace hermes -#endif // LABSTOR_RPC_H_ +#endif // HRUN_RPC_H_ diff --git a/include/labstor/network/rpc_thallium.h b/include/hrun/network/rpc_thallium.h similarity index 99% rename from include/labstor/network/rpc_thallium.h rename to include/hrun/network/rpc_thallium.h index 667ddf278..58873283e 100644 --- a/include/labstor/network/rpc_thallium.h +++ b/include/hrun/network/rpc_thallium.h @@ -21,7 +21,7 @@ namespace tl = thallium; -namespace labstor { +namespace hrun { /** A structure to represent Thallium state diff --git a/include/labstor/network/serialize.h b/include/hrun/network/serialize.h similarity index 96% rename from include/labstor/network/serialize.h rename to include/hrun/network/serialize.h index 1336df3c8..d0fed8a80 100644 --- a/include/labstor/network/serialize.h +++ b/include/hrun/network/serialize.h @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 8/7/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_NETWORK_SERIALIZE_H_ -#define LABSTOR_INCLUDE_LABSTOR_NETWORK_SERIALIZE_H_ +#ifndef HRUN_INCLUDE_HRUN_NETWORK_SERIALIZE_H_ +#define HRUN_INCLUDE_HRUN_NETWORK_SERIALIZE_H_ -#include "labstor/labstor_types.h" -#include "labstor/task_registry/task.h" +#include "hrun/hrun_types.h" +#include "hrun/task_registry/task.h" #include #include #include @@ -16,7 +16,7 @@ #include #include -namespace labstor { +namespace hrun { /** Receiver will read from data_ */ #define DT_RECEIVER_READ BIT_OPT(u32, 0) @@ -276,6 +276,6 @@ class BinaryInputArchive { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_NETWORK_SERIALIZE_H_ +#endif // HRUN_INCLUDE_HRUN_NETWORK_SERIALIZE_H_ diff --git a/include/labstor/queue_manager/queue.h b/include/hrun/queue_manager/queue.h similarity index 90% rename from include/labstor/queue_manager/queue.h rename to include/hrun/queue_manager/queue.h index 647ba69a7..ce4c0e49c 100644 --- a/include/labstor/queue_manager/queue.h +++ b/include/hrun/queue_manager/queue.h @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/27/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_H_ -#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_H_ +#ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_H_ +#define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_H_ -#include "labstor/labstor_types.h" -#include "labstor/task_registry/task.h" +#include "hrun/hrun_types.h" +#include "hrun/task_registry/task.h" #include /** Requests in this queue can be processed in any order */ @@ -22,7 +22,7 @@ /** Requests in this queue are long-running */ #define QUEUE_LONG_RUNNING BIT_OPT(u32, 5) -namespace labstor { +namespace hrun { /** Prioritization info needed to be set by client */ struct PriorityInfo { @@ -102,6 +102,6 @@ struct PriorityInfo { template class MultiQueueT; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_H_ +#endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_H_ diff --git a/include/hrun/queue_manager/queue_factory.h b/include/hrun/queue_manager/queue_factory.h new file mode 100644 index 000000000..7c974c61d --- /dev/null +++ b/include/hrun/queue_manager/queue_factory.h @@ -0,0 +1,19 @@ +// +// Created by llogan on 7/1/23. +// + +#ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ +#define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ + +#include "queues/hshm_queue.h" + +namespace hrun { + +#ifdef QUEUE_TYPE +using MultiQueue = MultiQueueT; +#else +using MultiQueue = MultiQueueT; +#endif +} // namespace hrun + +#endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ diff --git a/include/labstor/queue_manager/queue_manager.h b/include/hrun/queue_manager/queue_manager.h similarity index 79% rename from include/labstor/queue_manager/queue_manager.h rename to include/hrun/queue_manager/queue_manager.h index 52593026a..8e82025ba 100644 --- a/include/labstor/queue_manager/queue_manager.h +++ b/include/hrun/queue_manager/queue_manager.h @@ -2,14 +2,14 @@ // Created by lukemartinlogan on 6/28/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_H_ -#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_H_ +#ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_H_ +#define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_H_ -#include "labstor/config/config_server.h" -#include "labstor/labstor_types.h" +#include "hrun/config/config_server.h" +#include "hrun/hrun_types.h" #include "queue_factory.h" -namespace labstor { +namespace hrun { /** Shared-memory representation of the QueueManager */ struct QueueManagerShm { @@ -44,6 +44,6 @@ class QueueManager { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_H_ +#endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_H_ diff --git a/include/labstor/queue_manager/queue_manager_client.h b/include/hrun/queue_manager/queue_manager_client.h similarity index 62% rename from include/labstor/queue_manager/queue_manager_client.h rename to include/hrun/queue_manager/queue_manager_client.h index 5be5e9001..39d0eaf64 100644 --- a/include/labstor/queue_manager/queue_manager_client.h +++ b/include/hrun/queue_manager/queue_manager_client.h @@ -2,15 +2,15 @@ // Created by lukemartinlogan on 6/28/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ -#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ +#ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ +#define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ #include "queue_manager.h" -namespace labstor { +namespace hrun { -#define LABSTOR_QM_CLIENT \ - (&LABSTOR_CLIENT->queue_manager_) +#define HRUN_QM_CLIENT \ + (&HRUN_CLIENT->queue_manager_) /** Enable client programs to access queues */ class QueueManagerClient : public QueueManager { @@ -32,6 +32,6 @@ class QueueManagerClient : public QueueManager { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ +#endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ diff --git a/include/labstor/queue_manager/queue_manager_runtime.h b/include/hrun/queue_manager/queue_manager_runtime.h similarity index 88% rename from include/labstor/queue_manager/queue_manager_runtime.h rename to include/hrun/queue_manager/queue_manager_runtime.h index aa43b50c3..ee86f0b14 100644 --- a/include/labstor/queue_manager/queue_manager_runtime.h +++ b/include/hrun/queue_manager/queue_manager_runtime.h @@ -2,16 +2,16 @@ // Created by lukemartinlogan on 6/28/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ -#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ +#ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ +#define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ #include "queue_manager.h" #include "queue_manager_client.h" -namespace labstor { +namespace hrun { -#define LABSTOR_QM_RUNTIME \ - (&LABSTOR_RUNTIME->queue_manager_) +#define HRUN_QM_RUNTIME \ + (&HRUN_RUNTIME->queue_manager_) /** Administrative queuing actions */ class QueueManagerRuntime : public QueueManager { @@ -87,6 +87,6 @@ class QueueManagerRuntime : public QueueManager { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ +#endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ diff --git a/include/labstor/queue_manager/queues/hshm_queue.h b/include/hrun/queue_manager/queues/hshm_queue.h similarity index 96% rename from include/labstor/queue_manager/queues/hshm_queue.h rename to include/hrun/queue_manager/queues/hshm_queue.h index d1472b96c..94e921527 100644 --- a/include/labstor/queue_manager/queues/hshm_queue.h +++ b/include/hrun/queue_manager/queues/hshm_queue.h @@ -2,12 +2,12 @@ // Created by llogan on 7/1/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ -#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ +#ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_HSHM_QUEUE_H_ +#define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_HSHM_QUEUE_H_ -#include "labstor/queue_manager/queue.h" +#include "hrun/queue_manager/queue.h" -namespace labstor { +namespace hrun { /** The data stored in a lane */ struct LaneData { @@ -288,7 +288,7 @@ struct MultiQueueT : public hipc::ShmContainer { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_HSHM_QUEUE_H_ +#endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_HSHM_QUEUE_H_ diff --git a/include/labstor/task_registry/task.h b/include/hrun/task_registry/task.h similarity index 98% rename from include/labstor/task_registry/task.h rename to include/hrun/task_registry/task.h index 229698ec9..0348e53d4 100644 --- a/include/labstor/task_registry/task.h +++ b/include/hrun/task_registry/task.h @@ -2,14 +2,14 @@ // Created by lukemartinlogan on 6/23/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_REQUEST_H_ -#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_REQUEST_H_ +#ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_REQUEST_H_ +#define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_REQUEST_H_ -#include "labstor/labstor_types.h" -#include "labstor/network/local_serialize.h" +#include "hrun/hrun_types.h" +#include "hrun/network/local_serialize.h" #include -namespace labstor { +namespace hrun { class TaskLib; @@ -191,7 +191,7 @@ class IsTask {}; #define TASK_FLAG_T constexpr inline static bool /** Determine this is a task */ #define IS_TASK(T) \ - std::is_base_of_v + std::is_base_of_v /** Determine this task supports serialization */ #define IS_SRL(T) \ T::SUPPORTS_SRL @@ -598,6 +598,6 @@ struct Task : public hipc::ShmContainer { #define INOUT #define TEMP -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_REQUEST_H_ +#endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_REQUEST_H_ diff --git a/include/labstor/task_registry/task_lib.h b/include/hrun/task_registry/task_lib.h similarity index 82% rename from include/labstor/task_registry/task_lib.h rename to include/hrun/task_registry/task_lib.h index 053ff5ac9..1c140a3db 100644 --- a/include/labstor/task_registry/task_lib.h +++ b/include/hrun/task_registry/task_lib.h @@ -2,16 +2,16 @@ // Created by lukemartinlogan on 6/23/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_TASK_TASK_H_ -#define LABSTOR_INCLUDE_LABSTOR_TASK_TASK_H_ +#ifndef HRUN_INCLUDE_HRUN_TASK_TASK_H_ +#define HRUN_INCLUDE_HRUN_TASK_TASK_H_ #include -#include "labstor/labstor_types.h" -#include "labstor/queue_manager/queue_factory.h" -#include "labstor/network/serialize.h" +#include "hrun/hrun_types.h" +#include "hrun/queue_manager/queue_factory.h" +#include "hrun/network/serialize.h" #include "task.h" -namespace labstor { +namespace hrun { typedef LPointer TaskPointer; @@ -97,19 +97,19 @@ typedef const char* (*get_task_lib_name_t)(void); } // extern c /** Used internally by task source file */ -#define LABSTOR_TASK_CC(TRAIT_CLASS, TASK_NAME) \ +#define HRUN_TASK_CC(TRAIT_CLASS, TASK_NAME) \ extern "C" { \ - void* create_state(labstor::Admin::CreateTaskStateTask *task, const char *state_name) { \ - labstor::TaskState *exec = reinterpret_cast( \ + void* create_state(hrun::Admin::CreateTaskStateTask *task, const char *state_name) { \ + hrun::TaskState *exec = reinterpret_cast( \ new TYPE_UNWRAP(TRAIT_CLASS)()); \ exec->Init(task->id_, state_name); \ RunContext rctx(0); \ - exec->Run(labstor::TaskMethod::kConstruct, task, rctx); \ + exec->Run(hrun::TaskMethod::kConstruct, task, rctx); \ return exec; \ } \ const char* get_task_lib_name(void) { return TASK_NAME; } \ - bool is_labstor_task_ = true; \ + bool is_hrun_task_ = true; \ } -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_TASK_TASK_H_ +#endif // HRUN_INCLUDE_HRUN_TASK_TASK_H_ diff --git a/include/labstor/task_registry/task_registry.h b/include/hrun/task_registry/task_registry.h similarity index 93% rename from include/labstor/task_registry/task_registry.h rename to include/hrun/task_registry/task_registry.h index 7bf7ef987..fe34dda4f 100644 --- a/include/labstor/task_registry/task_registry.h +++ b/include/hrun/task_registry/task_registry.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/23/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_TASK_TASK_REGISTRY_H_ -#define LABSTOR_INCLUDE_LABSTOR_TASK_TASK_REGISTRY_H_ +#ifndef HRUN_INCLUDE_HRUN_TASK_TASK_REGISTRY_H_ +#define HRUN_INCLUDE_HRUN_TASK_TASK_REGISTRY_H_ #include #include @@ -11,12 +11,12 @@ #include #include #include "task_lib.h" -#include "labstor/config/config_server.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun/config/config_server.h" +#include "hrun_admin/hrun_admin.h" namespace stdfs = std::filesystem; -namespace labstor { +namespace hrun { /** All information needed to create a trait */ struct TaskLibInfo { @@ -91,14 +91,14 @@ class TaskRegistry { ld_lib_path = ld_lib_path_env; } - // Load the LABSTOR_TASK_PATH variable + // Load the HRUN_TASK_PATH variable std::string hermes_lib_path; - auto hermes_lib_path_env = getenv("LABSTOR_TASK_PATH"); + auto hermes_lib_path_env = getenv("HRUN_TASK_PATH"); if (hermes_lib_path_env) { hermes_lib_path = hermes_lib_path_env; } - // Combine LD_LIBRARY_PATH and LABSTOR_TASK_PATH + // Combine LD_LIBRARY_PATH and HRUN_TASK_PATH std::string paths = hermes_lib_path + ":" + ld_lib_path; std::stringstream ss(paths); std::string lib_dir; @@ -202,7 +202,7 @@ class TaskRegistry { return false; } // HILOG(kInfo, "(node {}) Creating an instance of {} with name {}", -// LABSTOR_CLIENT->node_id_, lib_name, state_name) +// HRUN_CLIENT->node_id_, lib_name, state_name) // Find the task library to instantiate auto it = libs_.find(lib_name); @@ -235,7 +235,7 @@ class TaskRegistry { task_state_ids_.emplace(state_name, state_id); task_states_.emplace(state_id, task_state); HILOG(kInfo, "(node {}) Created an instance of {} with name {} and ID {}", - LABSTOR_CLIENT->node_id_, lib_name, state_name, state_id) + HRUN_CLIENT->node_id_, lib_name, state_name, state_id) return true; } @@ -292,8 +292,8 @@ class TaskRegistry { }; /** Singleton macro for task registry */ -#define LABSTOR_TASK_REGISTRY \ - (&LABSTOR_RUNTIME->task_registry_) -} // namespace labstor +#define HRUN_TASK_REGISTRY \ + (&HRUN_RUNTIME->task_registry_) +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_TASK_TASK_REGISTRY_H_ +#endif // HRUN_INCLUDE_HRUN_TASK_TASK_REGISTRY_H_ diff --git a/include/labstor/work_orchestrator/affinity.h b/include/hrun/work_orchestrator/affinity.h similarity index 74% rename from include/labstor/work_orchestrator/affinity.h rename to include/hrun/work_orchestrator/affinity.h index 903eab9ad..3a480e789 100644 --- a/include/labstor/work_orchestrator/affinity.h +++ b/include/hrun/work_orchestrator/affinity.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 7/5/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_AFFINITY_H_ -#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_AFFINITY_H_ +#ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_AFFINITY_H_ +#define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_AFFINITY_H_ #include @@ -24,4 +24,4 @@ class ProcessAffiner { } }; -#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_AFFINITY_H_ +#endif // HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_AFFINITY_H_ diff --git a/include/labstor/work_orchestrator/scheduler.h b/include/hrun/work_orchestrator/scheduler.h similarity index 80% rename from include/labstor/work_orchestrator/scheduler.h rename to include/hrun/work_orchestrator/scheduler.h index 4b9bad446..7e23c1863 100644 --- a/include/labstor/work_orchestrator/scheduler.h +++ b/include/hrun/work_orchestrator/scheduler.h @@ -2,12 +2,12 @@ // Created by llogan on 7/2/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_SCHEDULER_H_ -#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_SCHEDULER_H_ +#ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ +#define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ -#include "labstor/task_registry/task.h" +#include "hrun/task_registry/task.h" -namespace labstor { +namespace hrun { /** The set of methods in the admin task */ struct SchedulerMethod : public TaskMethod { @@ -49,6 +49,6 @@ struct ScheduleTask : public Task, TaskFlags { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_SCHEDULER_H_ +#endif // HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ diff --git a/include/labstor/work_orchestrator/work_orchestrator.h b/include/hrun/work_orchestrator/work_orchestrator.h similarity index 85% rename from include/labstor/work_orchestrator/work_orchestrator.h rename to include/hrun/work_orchestrator/work_orchestrator.h index aad42852c..56394d3f0 100644 --- a/include/labstor/work_orchestrator/work_orchestrator.h +++ b/include/hrun/work_orchestrator/work_orchestrator.h @@ -2,17 +2,17 @@ // Created by lukemartinlogan on 6/23/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ -#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ +#ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ +#define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ -#include "labstor/labstor_types.h" -#include "labstor/api/labstor_runtime.h" -#include "labstor/queue_manager/queue_manager_runtime.h" +#include "hrun/hrun_types.h" +#include "hrun/api/hrun_runtime.h" +#include "hrun/queue_manager/queue_manager_runtime.h" #include "worker.h" -#include "labstor/network/rpc_thallium.h" +#include "hrun/network/rpc_thallium.h" #include -namespace labstor { +namespace hrun { class WorkOrchestrator { public: @@ -118,9 +118,9 @@ class WorkOrchestrator { } }; -} // namespace labstor +} // namespace hrun -#define LABSTOR_WORK_ORCHESTRATOR \ - (&LABSTOR_RUNTIME->work_orchestrator_) +#define HRUN_WORK_ORCHESTRATOR \ + (&HRUN_RUNTIME->work_orchestrator_) -#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ +#endif // HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ diff --git a/include/labstor/work_orchestrator/worker.h b/include/hrun/work_orchestrator/worker.h similarity index 90% rename from include/labstor/work_orchestrator/worker.h rename to include/hrun/work_orchestrator/worker.h index f1aaf7476..59f825e97 100644 --- a/include/labstor/work_orchestrator/worker.h +++ b/include/hrun/work_orchestrator/worker.h @@ -2,21 +2,21 @@ // Created by lukemartinlogan on 6/27/23. // -#ifndef LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORKER_H_ -#define LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORKER_H_ +#ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORKER_H_ +#define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORKER_H_ -#include "labstor/labstor_types.h" -#include "labstor/queue_manager/queue_manager_runtime.h" +#include "hrun/hrun_types.h" +#include "hrun/queue_manager/queue_manager_runtime.h" #include #include #include "affinity.h" -#include "labstor/network/rpc_thallium.h" +#include "hrun/network/rpc_thallium.h" static inline pid_t GetLinuxTid() { return syscall(SYS_gettid); } -namespace labstor { +namespace hrun { #define WORKER_CONTINUOUS_POLLING BIT_OPT(u32, 0) @@ -110,22 +110,22 @@ struct WorkEntry { } }; -} // namespace labstor +} // namespace hrun namespace std { /** Hash function for WorkEntry */ template<> -struct hash { +struct hash { HSHM_ALWAYS_INLINE std::size_t - operator()(const labstor::WorkEntry &key) const { - return std::hash{}(key.queue_) + + operator()(const hrun::WorkEntry &key) const { + return std::hash{}(key.queue_) + std::hash{}(key.lane_id_) + std::hash{}(key.prio_); } }; } // namespace std -namespace labstor { +namespace hrun { class Worker { public: @@ -283,7 +283,7 @@ class Worker { int ret = exec->GetGroup(task->method_, task, group_); if (ret == TASK_UNORDERED || task->IsUnordered()) { // HILOG(kDebug, "(node {}) Task {} is unordered, so count remains 0 worker={}", -// LABSTOR_CLIENT->node_id_, task->task_node_, id_); +// HRUN_CLIENT->node_id_, task->task_node_, id_); return true; } @@ -304,14 +304,14 @@ class Worker { node.node_depth_ = 1; group_map_.emplace(group_, node); // HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", -// LABSTOR_CLIENT->node_id_, node.node_depth_, id_); +// HRUN_CLIENT->node_id_, node.node_depth_, id_); return true; } TaskNode &node_cmp = it->second; if (node_cmp.root_ == node.root_) { node_cmp.node_depth_ += 1; // HILOG(kDebug, "(node {}) Increasing depth of group to {} worker={}", -// LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, id_); +// HRUN_CLIENT->node_id_, node_cmp.node_depth_, id_); return true; } return false; @@ -326,7 +326,7 @@ class Worker { int ret = exec->GetGroup(task->method_, task, group_); if (ret == TASK_UNORDERED || task->IsUnordered()) { // HILOG(kDebug, "(node {}) Decreasing depth of group remains 0 (task_node={} worker={})", -// LABSTOR_CLIENT->node_id_, task->task_node_, id_); +// HRUN_CLIENT->node_id_, task->task_node_, id_); return; } @@ -344,11 +344,11 @@ class Worker { TaskNode &node_cmp = group_map_[group_]; if (node_cmp.node_depth_ == 0) { HELOG(kFatal, "(node {}) Group {} depth is already 0 (task_node={} worker={})", - LABSTOR_CLIENT->node_id_, task->task_node_, id_); + HRUN_CLIENT->node_id_, task->task_node_, id_); } node_cmp.node_depth_ -= 1; // HILOG(kDebug, "(node {}) Decreasing depth of to {} (task_node={} worker={})", -// LABSTOR_CLIENT->node_id_, node_cmp.node_depth_, task->task_node_, id_); +// HRUN_CLIENT->node_id_, node_cmp.node_depth_, task->task_node_, id_); if (node_cmp.node_depth_ == 0) { group_map_.erase(group_); } @@ -358,7 +358,7 @@ class Worker { void EndTask(Lane *lane, TaskState *exec, Task *task, int &off) { PopTask(lane, off); if (exec && task->IsFireAndForget()) { - LABSTOR_CLIENT->DelTask(exec, task); + HRUN_CLIENT->DelTask(exec, task); } else { task->SetComplete(); } @@ -378,6 +378,6 @@ class Worker { static void RunPreemptive(void *data); }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_INCLUDE_LABSTOR_WORK_ORCHESTRATOR_WORKER_H_ +#endif // HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORKER_H_ diff --git a/include/labstor/config/config_client_default.h b/include/labstor/config/config_client_default.h deleted file mode 100644 index 94df443ad..000000000 --- a/include/labstor/config/config_client_default.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifndef LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ -#define LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ -const char* kLabstorClientDefaultConfigStr = -"thread_model: kStd\n"; -#endif // LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/include/labstor/labstor_constants.h b/include/labstor/labstor_constants.h deleted file mode 100644 index 9cfb242e8..000000000 --- a/include/labstor/labstor_constants.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Created by lukemartinlogan on 6/22/23. -// - -#ifndef LABSTOR_INCLUDE_LABSTOR_LABSTOR_CONSTANTS_H_ -#define LABSTOR_INCLUDE_LABSTOR_LABSTOR_CONSTANTS_H_ - -namespace labstor { - -#include - -class Constants { - public: - inline static const std::string kClientConfEnv = "LABSTOR_CLIENT_CONF"; - inline static const std::string kServerConfEnv = "LABSTOR_SERVER_CONF"; - - static std::string GetEnvSafe(const std::string &env_name) { - char *data = getenv(env_name.c_str()); - if (data == nullptr) { - return ""; - } else { - return data; - } - } -}; - -} // namespace labstor - -#endif // LABSTOR_INCLUDE_LABSTOR_LABSTOR_CONSTANTS_H_ diff --git a/include/labstor/labstor_namespace.h b/include/labstor/labstor_namespace.h deleted file mode 100644 index da84cfd71..000000000 --- a/include/labstor/labstor_namespace.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// Created by lukemartinlogan on 8/14/23. -// - -#ifndef LABSTOR_INCLUDE_LABSTOR_LABSTOR_NAMESPACE_H_ -#define LABSTOR_INCLUDE_LABSTOR_LABSTOR_NAMESPACE_H_ - -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" - -using labstor::TaskMethod; -using labstor::BinaryOutputArchive; -using labstor::BinaryInputArchive; -using labstor::Task; -using labstor::TaskPointer; -using labstor::MultiQueue; -using labstor::PriorityInfo; -using labstor::TaskNode; -using labstor::DomainId; -using labstor::TaskStateId; -using labstor::QueueId; -using labstor::TaskFlags; -using labstor::DataTransfer; -using labstor::TaskLib; -using labstor::TaskLibClient; -using labstor::config::QueueManagerInfo; -using labstor::TaskPrio; -using labstor::RunContext; - -using hshm::RwLock; -using hshm::Mutex; -using hshm::bitfield; -using hshm::bitfield32_t; -typedef hshm::bitfield bitfield64_t; -using hshm::ScopedRwReadLock; -using hshm::ScopedRwWriteLock; -using hipc::LPointer; - -#endif // LABSTOR_INCLUDE_LABSTOR_LABSTOR_NAMESPACE_H_ diff --git a/include/labstor/queue_manager/queue_factory.h b/include/labstor/queue_manager/queue_factory.h deleted file mode 100644 index 7a2f3bf46..000000000 --- a/include/labstor/queue_manager/queue_factory.h +++ /dev/null @@ -1,19 +0,0 @@ -// -// Created by llogan on 7/1/23. -// - -#ifndef LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_FACTORY_H_ -#define LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_FACTORY_H_ - -#include "queues/hshm_queue.h" - -namespace labstor { - -#ifdef QUEUE_TYPE -using MultiQueue = MultiQueueT; -#else -using MultiQueue = MultiQueueT; -#endif -} // namespace labstor - -#endif // LABSTOR_INCLUDE_LABSTOR_QUEUE_MANAGER_QUEUE_FACTORY_H_ diff --git a/scripts/lint.sh b/scripts/lint.sh index 1953c341c..1bde6276f 100644 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,6 +1,6 @@ #!/bin/bash -LABSTOR_ROOT=$1 +HRUN_ROOT=$1 cpplint --recursive \ -"${LABSTOR_ROOT}/src" "${LABSTOR_ROOT}/include" "${LABSTOR_ROOT}/test" +"${HRUN_ROOT}/src" "${HRUN_ROOT}/include" "${HRUN_ROOT}/test" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cdfd0b46e..c9248f49e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,11 +6,11 @@ #------------------------------------------------------------------------------ # Build Labstor Client Library #------------------------------------------------------------------------------ -add_library(labstor_client SHARED +add_library(hrun_client SHARED ${CMAKE_CURRENT_SOURCE_DIR}/config_client.cc ${CMAKE_CURRENT_SOURCE_DIR}/config_server.cc - ${CMAKE_CURRENT_SOURCE_DIR}/labstor_client.cc) -target_link_libraries(labstor_client + ${CMAKE_CURRENT_SOURCE_DIR}/hrun_client.cc) +target_link_libraries(hrun_client ${HermesShm_LIBRARIES} yaml-cpp cereal::cereal @@ -20,26 +20,26 @@ target_link_libraries(labstor_client #------------------------------------------------------------------------------ # Build Labstor Runtime Library #------------------------------------------------------------------------------ -add_library(labstor_runtime +add_library(hrun_runtime worker.cc - labstor_runtime.cc) -add_dependencies(labstor_runtime ${Hermes_CLIENT_DEPS}) -target_link_libraries(labstor_runtime thallium ${Hermes_CLIENT_LIBRARIES}) + hrun_runtime.cc) +add_dependencies(hrun_runtime ${Hermes_CLIENT_DEPS}) +target_link_libraries(hrun_runtime thallium ${Hermes_CLIENT_LIBRARIES}) #------------------------------------------------------------------------------ # Build Labstor Runtime Start Function #------------------------------------------------------------------------------ -add_executable(labstor_start_runtime - labstor_start_runtime.cc) -add_dependencies(labstor_start_runtime ${Hermes_RUNTIME_DEPS}) -target_link_libraries(labstor_start_runtime ${Hermes_RUNTIME_LIBRARIES}) +add_executable(hrun_start_runtime + hrun_start_runtime.cc) +add_dependencies(hrun_start_runtime ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hrun_start_runtime ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Build LabStor Runtime Stop Function #------------------------------------------------------------------------------ -add_executable(labstor_stop_runtime labstor_stop_runtime.cc) -add_dependencies(labstor_stop_runtime ${Hermes_CLIENT_DEPS}) -target_link_libraries(labstor_stop_runtime ${Hermes_CLIENT_LIBRARIES}) +add_executable(hrun_stop_runtime hrun_stop_runtime.cc) +add_dependencies(hrun_stop_runtime ${Hermes_CLIENT_DEPS}) +target_link_libraries(hrun_stop_runtime ${Hermes_CLIENT_LIBRARIES}) #----------------------------------------------------------------------------- # Add file(s) to CMake Install @@ -58,10 +58,10 @@ install( #----------------------------------------------------------------------------- install( TARGETS - labstor_client - labstor_runtime - labstor_start_runtime - labstor_stop_runtime + hrun_client + hrun_runtime + hrun_start_runtime + hrun_stop_runtime EXPORT ${HERMES_EXPORTED_TARGETS} LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} @@ -85,10 +85,10 @@ install( # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- set(HERMES_EXPORTED_LIBS - labstor_client - labstor_runtime - labstor_start_runtime - labstor_stop_runtime + hrun_client + hrun_runtime + hrun_start_runtime + hrun_stop_runtime ${HERMES_EXPORTED_LIBS}) if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( @@ -103,8 +103,8 @@ endif() # Coverage #------------------------------------------------------------------------------ if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(labstor_client) - set_coverage_flags(labstor_runtime) - set_coverage_flags(labstor_start_runtime) - set_coverage_flags(labstor_stop_runtime) + set_coverage_flags(hrun_client) + set_coverage_flags(hrun_runtime) + set_coverage_flags(hrun_start_runtime) + set_coverage_flags(hrun_stop_runtime) endif() \ No newline at end of file diff --git a/src/config_client.cc b/src/config_client.cc index 8f7d7dd26..02c53cc65 100644 --- a/src/config_client.cc +++ b/src/config_client.cc @@ -10,14 +10,14 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "labstor/config/config_client.h" -#include "labstor/config/config_client_default.h" +#include "hrun/config/config_client.h" +#include "hrun/config/config_client_default.h" #include "hermes_shm/util/config_parse.h" #include namespace stdfs = std::filesystem; -namespace labstor::config { +namespace hrun::config { /** parse the YAML node */ void ClientConfig::ParseYAML(YAML::Node &yaml_conf) { @@ -31,4 +31,4 @@ void ClientConfig::LoadDefault() { LoadText(kLabstorClientDefaultConfigStr, false); } -} // namespace labstor::config +} // namespace hrun::config diff --git a/src/config_server.cc b/src/config_server.cc index df7d282c5..736f9d283 100644 --- a/src/config_server.cc +++ b/src/config_server.cc @@ -15,11 +15,11 @@ #include #include "hermes_shm/util/logging.h" #include "hermes_shm/util/config_parse.h" -#include "labstor/config/config.h" -#include "labstor/config/config_server.h" -#include "labstor/config/config_server_default.h" +#include "hrun/config/config.h" +#include "hrun/config/config_server.h" +#include "hrun/config/config_server_default.h" -namespace labstor::config { +namespace hrun::config { /** parse work orchestrator info from YAML config */ void ServerConfig::ParseWorkOrchestrator(YAML::Node yaml_conf) { @@ -103,4 +103,4 @@ void ServerConfig::LoadDefault() { LoadText(kLabstorServerDefaultConfigStr, false); } -} // namespace labstor::config +} // namespace hrun::config diff --git a/src/labstor_client.cc b/src/hrun_client.cc similarity index 60% rename from src/labstor_client.cc rename to src/hrun_client.cc index 10e7fcf7a..7c9aa5015 100644 --- a/src/labstor_client.cc +++ b/src/hrun_client.cc @@ -3,7 +3,7 @@ // #include "hermes_shm/util/singleton.h" -#include "labstor/api/labstor_client.h" +#include "hrun/api/hrun_client.h" /** Runtime singleton */ -DEFINE_SINGLETON_CC(labstor::Client) \ No newline at end of file +DEFINE_SINGLETON_CC(hrun::Client) \ No newline at end of file diff --git a/src/labstor_runtime.cc b/src/hrun_runtime.cc similarity index 59% rename from src/labstor_runtime.cc rename to src/hrun_runtime.cc index 7db986499..8cd958036 100644 --- a/src/labstor_runtime.cc +++ b/src/hrun_runtime.cc @@ -3,7 +3,7 @@ // #include "hermes_shm/util/singleton.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun/api/hrun_runtime.h" /** Runtime singleton */ -DEFINE_SINGLETON_CC(labstor::Runtime) \ No newline at end of file +DEFINE_SINGLETON_CC(hrun::Runtime) \ No newline at end of file diff --git a/src/labstor_start_runtime.cc b/src/hrun_start_runtime.cc similarity index 50% rename from src/labstor_start_runtime.cc rename to src/hrun_start_runtime.cc index c7749ccec..938c4b583 100644 --- a/src/labstor_start_runtime.cc +++ b/src/hrun_start_runtime.cc @@ -3,11 +3,11 @@ // #include "hermes_shm/util/singleton.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun/api/hrun_runtime.h" int main(int argc, char **argv) { - LABSTOR_RUNTIME->Create(); - LABSTOR_RUNTIME->RunDaemon(); - LABSTOR_RUNTIME->Finalize(); + HRUN_RUNTIME->Create(); + HRUN_RUNTIME->RunDaemon(); + HRUN_RUNTIME->Finalize(); return 0; } diff --git a/src/hrun_stop_runtime.cc b/src/hrun_stop_runtime.cc new file mode 100644 index 000000000..629f5d58a --- /dev/null +++ b/src/hrun_stop_runtime.cc @@ -0,0 +1,10 @@ +// +// Created by lukemartinlogan on 6/22/23. +// + +#include "hrun_admin/hrun_admin.h" + +int main() { + TRANSPARENT_HRUN(); + HRUN_ADMIN->AsyncStopRuntimeRoot(hrun::DomainId::GetLocal()); +} \ No newline at end of file diff --git a/src/labstor_stop_runtime.cc b/src/labstor_stop_runtime.cc deleted file mode 100644 index 3f9951fa5..000000000 --- a/src/labstor_stop_runtime.cc +++ /dev/null @@ -1,10 +0,0 @@ -// -// Created by lukemartinlogan on 6/22/23. -// - -#include "labstor_admin/labstor_admin.h" - -int main() { - TRANSPARENT_LABSTOR(); - LABSTOR_ADMIN->AsyncStopRuntimeRoot(labstor::DomainId::GetLocal()); -} \ No newline at end of file diff --git a/src/worker.cc b/src/worker.cc index e57aac0ad..b07734bb4 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -2,15 +2,15 @@ // Created by lukemartinlogan on 6/27/23. // -#include "labstor/api/labstor_runtime.h" -#include "labstor/work_orchestrator/worker.h" -#include "labstor/work_orchestrator/work_orchestrator.h" +#include "hrun/api/hrun_runtime.h" +#include "hrun/work_orchestrator/worker.h" +#include "hrun/work_orchestrator/work_orchestrator.h" -namespace labstor { +namespace hrun { void Worker::Loop() { pid_ = GetLinuxTid(); - WorkOrchestrator *orchestrator = LABSTOR_WORK_ORCHESTRATOR; + WorkOrchestrator *orchestrator = HRUN_WORK_ORCHESTRATOR; while (orchestrator->IsAlive()) { try { flush_.pending_ = 0; @@ -19,7 +19,7 @@ void Worker::Loop() { flush_.flushing_ = false; } } catch (hshm::Error &e) { - HELOG(kFatal, "(node {}) Worker {} caught an error: {}", LABSTOR_CLIENT->node_id_, id_, e.what()); + HELOG(kFatal, "(node {}) Worker {} caught an error: {}", HRUN_CLIENT->node_id_, id_, e.what()); } // Yield(); } @@ -61,43 +61,43 @@ void Worker::PollGrouped(WorkEntry &work_entry) { PopTask(lane, off); continue; } - task = LABSTOR_CLIENT->GetPrivatePointer(entry->p_); + task = HRUN_CLIENT->GetPrivatePointer(entry->p_); RunContext &rctx = task->ctx_; rctx.lane_id_ = work_entry.lane_id_; rctx.flush_ = &flush_; // Get the task state TaskState *&exec = rctx.exec_; - exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); + exec = HRUN_TASK_REGISTRY->GetTaskState(task->task_state_); if (!exec) { HELOG(kFatal, "(node {}) Could not find the task state: {}", - LABSTOR_CLIENT->node_id_, task->task_state_); + HRUN_CLIENT->node_id_, task->task_state_); entry->complete_ = true; EndTask(lane, exec, task, off); continue; } // Attempt to run the task if it's ready and runnable - bool is_remote = task->domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); + bool is_remote = task->domain_id_.IsRemote(HRUN_RPC->GetNumHosts(), HRUN_CLIENT->node_id_); if (!task->IsRunDisabled() && CheckTaskGroup(task, exec, work_entry.lane_id_, task->task_node_, is_remote) && task->ShouldRun(work_entry.cur_time_, flush_.flushing_)) { #ifdef REMOTE_DEBUG - if (task->task_state_ != LABSTOR_QM_CLIENT->admin_task_state_ && + if (task->task_state_ != HRUN_QM_CLIENT->admin_task_state_ && !task->task_flags_.Any(TASK_REMOTE_DEBUG_MARK) && task->method_ != TaskMethod::kConstruct && - LABSTOR_RUNTIME->remote_created_) { + HRUN_RUNTIME->remote_created_) { is_remote = true; } task->task_flags_.SetBits(TASK_REMOTE_DEBUG_MARK); #endif // Execute or schedule task if (is_remote) { - auto ids = LABSTOR_RUNTIME->ResolveDomainId(task->domain_id_); - LABSTOR_REMOTE_QUEUE->Disperse(task, exec, ids); + auto ids = HRUN_RUNTIME->ResolveDomainId(task->domain_id_); + HRUN_REMOTE_QUEUE->Disperse(task, exec, ids); task->SetDisableRun(); task->SetUnordered(); task->UnsetCoroutine(); } else if (task->IsLaneAll()) { - LABSTOR_REMOTE_QUEUE->DisperseLocal(task, exec, work_entry.queue_, work_entry.group_); + HRUN_REMOTE_QUEUE->DisperseLocal(task, exec, work_entry.queue_, work_entry.group_); task->SetDisableRun(); task->SetUnordered(); task->UnsetCoroutine(); @@ -123,7 +123,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { } } else if (task->IsPreemptive()) { task->SetDisableRun(); - entry->thread_ = LABSTOR_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunPreemptive, task); + entry->thread_ = HRUN_WORK_ORCHESTRATOR->SpawnAsyncThread(&Worker::RunPreemptive, task); } else { task->SetStarted(); exec->Run(task->method_, task, rctx); @@ -136,7 +136,7 @@ void Worker::PollGrouped(WorkEntry &work_entry) { // Cleanup on task completion if (task->IsModuleComplete()) { // HILOG(kDebug, "(node {}) Ending task: task_node={} task_state={} lane={} queue={} worker={}", -// LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); +// HRUN_CLIENT->node_id_, task->task_node_, task->task_state_, lane_id, queue->id_, id_); entry->complete_ = true; if (task->IsCoroutine()) { free(rctx.stack_ptr_); @@ -166,7 +166,7 @@ void Worker::RunCoroutine(bctx::transfer_t t) { void Worker::RunPreemptive(void *data) { Task *task = reinterpret_cast(data); - TaskState *exec = LABSTOR_TASK_REGISTRY->GetTaskState(task->task_state_); + TaskState *exec = HRUN_TASK_REGISTRY->GetTaskState(task->task_state_); RunContext &real_rctx = task->ctx_; RunContext rctx(0); do { @@ -183,4 +183,4 @@ void Worker::RunPreemptive(void *data) { } while(!task->IsModuleComplete()); } -} // namespace labstor \ No newline at end of file +} // namespace hrun \ No newline at end of file diff --git a/tasks/bdev/include/bdev/bdev.h b/tasks/bdev/include/bdev/bdev.h index a454b46b5..6b20dbdd9 100644 --- a/tasks/bdev/include/bdev/bdev.h +++ b/tasks/bdev/include/bdev/bdev.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_bdev_H_ -#define LABSTOR_bdev_H_ +#ifndef HRUN_bdev_H_ +#define HRUN_bdev_H_ #include "bdev_tasks.h" @@ -40,13 +40,13 @@ class Client : public TaskLibClient { domain_id_ = domain_id; id_ = TaskStateId::GetNull(); CopyDevInfo(dev_info); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {4, 4, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, lib_name, id_, queue_info, dev_info); } @@ -55,10 +55,10 @@ class Client : public TaskLibClient { id_ = task->id_; queue_id_ = QueueId(id_); monitor_task_ = AsyncMonitor(task->task_node_, 100).ptr_; - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } } - LABSTOR_TASK_NODE_ROOT(AsyncCreateTaskState); + HRUN_TASK_NODE_ROOT(AsyncCreateTaskState); template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -71,7 +71,7 @@ class Client : public TaskLibClient { /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const std::string &state_name) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id_, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id_, id_); monitor_task_->SetModuleComplete(); } @@ -80,19 +80,19 @@ class Client : public TaskLibClient { void AsyncMonitorConstruct(MonitorTask *task, const TaskNode &task_node, size_t freq_ms) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id_, id_, freq_ms, max_cap_); } - LABSTOR_TASK_NODE_PUSH_ROOT(Monitor); + HRUN_TASK_NODE_PUSH_ROOT(Monitor); /** Update bdev capacity */ void AsyncUpdateCapacityConstruct(UpdateCapacityTask *task, const TaskNode &task_node, ssize_t size) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id_, id_, size); } - LABSTOR_TASK_NODE_PUSH_ROOT(UpdateCapacity); + HRUN_TASK_NODE_PUSH_ROOT(UpdateCapacity); /** Get bdev remaining capacity */ HSHM_ALWAYS_INLINE @@ -106,10 +106,10 @@ class Client : public TaskLibClient { const TaskNode &task_node, size_t size, std::vector &buffers) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id_, id_, size, &buffers); } - LABSTOR_TASK_NODE_PUSH_ROOT(Allocate); + HRUN_TASK_NODE_PUSH_ROOT(Allocate); /** Free data */ HSHM_ALWAYS_INLINE @@ -117,30 +117,30 @@ class Client : public TaskLibClient { const TaskNode &task_node, const std::vector &buffers, bool fire_and_forget) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id_, id_, buffers, fire_and_forget); } - LABSTOR_TASK_NODE_PUSH_ROOT(Free); + HRUN_TASK_NODE_PUSH_ROOT(Free); /** Allocate buffers from the bdev */ HSHM_ALWAYS_INLINE void AsyncWriteConstruct(WriteTask *task, const TaskNode &task_node, const char *data, size_t off, size_t size) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id_, id_, data, off, size); } - LABSTOR_TASK_NODE_PUSH_ROOT(Write); + HRUN_TASK_NODE_PUSH_ROOT(Write); /** Allocate buffers from the bdev */ HSHM_ALWAYS_INLINE void AsyncReadConstruct(ReadTask *task, const TaskNode &task_node, char *data, size_t off, size_t size) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id_, id_, data, off, size); } - LABSTOR_TASK_NODE_PUSH_ROOT(Read); + HRUN_TASK_NODE_PUSH_ROOT(Read); }; class Server { @@ -181,4 +181,4 @@ struct TargetStats { }; } // namespace hermes -#endif // LABSTOR_bdev_H_ +#endif // HRUN_bdev_H_ diff --git a/tasks/bdev/include/bdev/bdev_lib_exec.h b/tasks/bdev/include/bdev/bdev_lib_exec.h index a1cf7beed..83068c8b8 100644 --- a/tasks/bdev/include/bdev/bdev_lib_exec.h +++ b/tasks/bdev/include/bdev/bdev_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_BDEV_LIB_EXEC_H_ -#define LABSTOR_BDEV_LIB_EXEC_H_ +#ifndef HRUN_BDEV_LIB_EXEC_H_ +#define HRUN_BDEV_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -42,35 +42,35 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kWrite: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRead: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kAllocate: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kFree: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kMonitor: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kUpdateCapacity: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -79,35 +79,35 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kWrite: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRead: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kAllocate: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kFree: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kMonitor: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kUpdateCapacity: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -116,35 +116,35 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kWrite: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRead: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kAllocate: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kFree: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kMonitor: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kUpdateCapacity: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -153,35 +153,35 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kWrite: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRead: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kAllocate: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kFree: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kMonitor: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kUpdateCapacity: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -190,35 +190,35 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kWrite: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRead: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kAllocate: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kFree: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kMonitor: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kUpdateCapacity: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -266,42 +266,42 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kWrite: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRead: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kAllocate: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kFree: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kMonitor: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kUpdateCapacity: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -414,4 +414,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_BDEV_METHODS_H_ \ No newline at end of file +#endif // HRUN_BDEV_METHODS_H_ \ No newline at end of file diff --git a/tasks/bdev/include/bdev/bdev_methods.h b/tasks/bdev/include/bdev/bdev_methods.h index 7ca9b12f4..c6a398de1 100644 --- a/tasks/bdev/include/bdev/bdev_methods.h +++ b/tasks/bdev/include/bdev/bdev_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_BDEV_METHODS_H_ -#define LABSTOR_BDEV_METHODS_H_ +#ifndef HRUN_BDEV_METHODS_H_ +#define HRUN_BDEV_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -11,4 +11,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kUpdateCapacity = kLast + 5; }; -#endif // LABSTOR_BDEV_METHODS_H_ \ No newline at end of file +#endif // HRUN_BDEV_METHODS_H_ \ No newline at end of file diff --git a/tasks/bdev/include/bdev/bdev_namespace.h b/tasks/bdev/include/bdev/bdev_namespace.h index bebc9155a..79e94e114 100644 --- a/tasks/bdev/include/bdev/bdev_namespace.h +++ b/tasks/bdev/include/bdev/bdev_namespace.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ -#define LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ +#ifndef HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ +#define HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ #include "bdev_tasks.h" #include "bdev.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" /** The set of methods in the admin task */ using ::hermes::bdev::Method; @@ -23,4 +23,4 @@ using ::hermes::bdev::UpdateCapacityTask; /** Create admin requests */ using ::hermes::bdev::Client; -#endif // LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ +#endif // HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ diff --git a/tasks/bdev/include/bdev/bdev_tasks.h b/tasks/bdev/include/bdev/bdev_tasks.h index 22548f180..e160efd8d 100644 --- a/tasks/bdev/include/bdev/bdev_tasks.h +++ b/tasks/bdev/include/bdev/bdev_tasks.h @@ -2,13 +2,13 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ -#define LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ +#ifndef HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ +#define HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "hermes/hermes_types.h" #include "hermes/config_server.h" #include "proc_queue/proc_queue.h" @@ -16,12 +16,12 @@ namespace hermes::bdev { #include "bdev_methods.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" /** * A task to create hermes_mdm * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { IN DeviceInfo info_; @@ -47,7 +47,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy hermes_mdm */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -309,4 +309,4 @@ struct UpdateCapacityTask : public Task, TaskFlags { } // namespace hermes::bdev -#endif // LABSTOR_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ +#endif // HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_TASKS_H_ diff --git a/tasks/data_stager/include/data_stager/data_stager.h b/tasks/data_stager/include/data_stager/data_stager.h index 7c0094d7d..dd7fdbc7c 100644 --- a/tasks/data_stager/include/data_stager/data_stager.h +++ b/tasks/data_stager/include/data_stager/data_stager.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_data_stager_H_ -#define LABSTOR_data_stager_H_ +#ifndef HRUN_data_stager_H_ +#define HRUN_data_stager_H_ #include "data_stager_tasks.h" @@ -25,16 +25,16 @@ class Client : public TaskLibClient { const std::string &state_name, const TaskStateId &blob_mdm) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info, blob_mdm); } - LABSTOR_TASK_NODE_ROOT(AsyncCreate) + HRUN_TASK_NODE_ROOT(AsyncCreate) template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -43,13 +43,13 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /** Register task state */ @@ -58,33 +58,33 @@ class Client : public TaskLibClient { const TaskNode &task_node, const BucketId &bkt_id, const hshm::charbuf &url) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_, bkt_id, url); } HSHM_ALWAYS_INLINE void RegisterStagerRoot(const BucketId &bkt_id, const hshm::charbuf &url) { - LPointer> task = + LPointer> task = AsyncRegisterStagerRoot(bkt_id, url); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(RegisterStager); + HRUN_TASK_NODE_PUSH_ROOT(RegisterStager); /** Unregister task state */ HSHM_ALWAYS_INLINE void AsyncUnregisterStagerConstruct(UnregisterStagerTask *task, const TaskNode &task_node, const BucketId &bkt_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_, bkt_id); } HSHM_ALWAYS_INLINE void UnregisterStagerRoot(const BucketId &bkt_id) { - LPointer> task = + LPointer> task = AsyncUnregisterStagerRoot(bkt_id); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(UnregisterStager); + HRUN_TASK_NODE_PUSH_ROOT(UnregisterStager); /** Stage in data from a remote source */ HSHM_ALWAYS_INLINE @@ -94,7 +94,7 @@ class Client : public TaskLibClient { const hshm::charbuf &blob_name, float score, u32 node_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_, bkt_id, blob_name, score, node_id); } @@ -103,11 +103,11 @@ class Client : public TaskLibClient { const hshm::charbuf &blob_name, float score, u32 node_id) { - LPointer> task = + LPointer> task = AsyncStageInRoot(bkt_id, blob_name, score, node_id); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(StageIn); + HRUN_TASK_NODE_PUSH_ROOT(StageIn); /** Stage out data to a remote source */ HSHM_ALWAYS_INLINE @@ -118,7 +118,7 @@ class Client : public TaskLibClient { const hipc::Pointer &data, size_t data_size, u32 task_flags) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_, bkt_id, blob_name, data, data_size, task_flags); } @@ -128,11 +128,11 @@ class Client : public TaskLibClient { const hipc::Pointer &data, size_t data_size, u32 task_flags) { - LPointer> task = + LPointer> task = AsyncStageOutRoot(bkt_id, blob_name, data, data_size, task_flags); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(StageOut); + HRUN_TASK_NODE_PUSH_ROOT(StageOut); /** Parse url */ static inline @@ -172,6 +172,6 @@ class Client : public TaskLibClient { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_data_stager_H_ +#endif // HRUN_data_stager_H_ diff --git a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h index 3a12864da..df65d48be 100644 --- a/tasks/data_stager/include/data_stager/data_stager_lib_exec.h +++ b/tasks/data_stager/include/data_stager/data_stager_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_DATA_STAGER_LIB_EXEC_H_ -#define LABSTOR_DATA_STAGER_LIB_EXEC_H_ +#ifndef HRUN_DATA_STAGER_LIB_EXEC_H_ +#define HRUN_DATA_STAGER_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -34,27 +34,27 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRegisterStager: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kUnregisterStager: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kStageIn: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kStageOut: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -63,27 +63,27 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRegisterStager: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kUnregisterStager: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kStageIn: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kStageOut: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -92,27 +92,27 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRegisterStager: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kUnregisterStager: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kStageIn: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kStageOut: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -121,27 +121,27 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRegisterStager: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kUnregisterStager: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kStageIn: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kStageOut: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -150,27 +150,27 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRegisterStager: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kUnregisterStager: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kStageIn: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kStageOut: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -210,32 +210,32 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRegisterStager: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kUnregisterStager: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kStageIn: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kStageOut: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -326,4 +326,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_DATA_STAGER_METHODS_H_ \ No newline at end of file +#endif // HRUN_DATA_STAGER_METHODS_H_ \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_methods.h b/tasks/data_stager/include/data_stager/data_stager_methods.h index 4b1b9d8f3..2f7d2af52 100644 --- a/tasks/data_stager/include/data_stager/data_stager_methods.h +++ b/tasks/data_stager/include/data_stager/data_stager_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_DATA_STAGER_METHODS_H_ -#define LABSTOR_DATA_STAGER_METHODS_H_ +#ifndef HRUN_DATA_STAGER_METHODS_H_ +#define HRUN_DATA_STAGER_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -9,4 +9,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kStageOut = kLast + 3; }; -#endif // LABSTOR_DATA_STAGER_METHODS_H_ \ No newline at end of file +#endif // HRUN_DATA_STAGER_METHODS_H_ \ No newline at end of file diff --git a/tasks/data_stager/include/data_stager/data_stager_tasks.h b/tasks/data_stager/include/data_stager/data_stager_tasks.h index ba485851b..39dd2f036 100644 --- a/tasks/data_stager/include/data_stager/data_stager_tasks.h +++ b/tasks/data_stager/include/data_stager/data_stager_tasks.h @@ -2,27 +2,27 @@ // Created by lukemartinlogan on 8/11/23. // -#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ -#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ +#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ +#define HRUN_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" #include "hermes/hermes_types.h" namespace hermes::data_stager { #include "data_stager_methods.h" -#include "labstor/labstor_namespace.h" -using labstor::proc_queue::TypedPushTask; -using labstor::proc_queue::PushTask; +#include "hrun/hrun_namespace.h" +using hrun::proc_queue::TypedPushTask; +using hrun::proc_queue::PushTask; /** * A task to create data_stager * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { TaskStateId blob_mdm_; @@ -53,7 +53,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy data_stager */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -299,7 +299,7 @@ struct StageOutTask : public Task, TaskFlags { ~StageOutTask() { HSHM_DESTROY_AR(blob_name_) if (IsDataOwner()) { - LABSTOR_CLIENT->FreeBuffer(data_); + HRUN_CLIENT->FreeBuffer(data_); } } @@ -310,6 +310,6 @@ struct StageOutTask : public Task, TaskFlags { } }; -} // namespace labstor::data_stager +} // namespace hrun::data_stager -#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ +#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_data_stager_data_stager_TASKS_H_ diff --git a/tasks/data_stager/include/data_stager/factory/binary_stager.h b/tasks/data_stager/include/data_stager/factory/binary_stager.h index 981eb5d83..93b958578 100644 --- a/tasks/data_stager/include/data_stager/factory/binary_stager.h +++ b/tasks/data_stager/include/data_stager/factory/binary_stager.h @@ -60,7 +60,7 @@ class BinaryFileStager : public AbstractStager { plcmnt.DecodeBlobName(*task->blob_name_, page_size_); HILOG(kDebug, "Attempting to stage {} bytes from the backend file {} at offset {}", page_size_, url_, plcmnt.bucket_off_); - LPointer blob = LABSTOR_CLIENT->AllocateBuffer(page_size_); + LPointer blob = HRUN_CLIENT->AllocateBuffer(page_size_); ssize_t real_size = HERMES_POSIX_API->pread(fd_, blob.ptr_, page_size_, @@ -82,7 +82,7 @@ class BinaryFileStager : public AbstractStager { 0, real_size, blob.shm_, task->score_, 0, ctx, TASK_DATA_OWNER | TASK_LOW_LATENCY); put_task->Wait(task); - LABSTOR_CLIENT->DelTask(put_task); + HRUN_CLIENT->DelTask(put_task); } /** Stage data out to remote source */ @@ -91,7 +91,7 @@ class BinaryFileStager : public AbstractStager { plcmnt.DecodeBlobName(*task->blob_name_, page_size_); HILOG(kDebug, "Attempting to stage {} bytes to the backend file {} at offset {}", page_size_, url_, plcmnt.bucket_off_); - char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); + char *data = HRUN_CLIENT->GetPrivatePointer(task->data_); ssize_t real_size = HERMES_POSIX_API->pwrite(fd_, data, task->data_size_, diff --git a/tasks/data_stager/src/data_stager.cc b/tasks/data_stager/src/data_stager.cc index 6a816c161..b99459bd2 100644 --- a/tasks/data_stager/src/data_stager.cc +++ b/tasks/data_stager/src/data_stager.cc @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "data_stager/data_stager.h" #include "hermes_adapters/mapper/abstract_mapper.h" #include "hermes_adapters/posix/posix_api.h" @@ -21,7 +21,7 @@ class Server : public TaskLib { Server() = default; void Construct(ConstructTask *task, RunContext &rctx) { - url_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + url_map_.resize(HRUN_QM_RUNTIME->max_lanes_); blob_mdm_.Init(task->blob_mdm_); task->SetModuleComplete(); } @@ -58,6 +58,6 @@ class Server : public TaskLib { #include "data_stager/data_stager_lib_exec.h" }; -} // namespace labstor::data_stager +} // namespace hrun::data_stager -LABSTOR_TASK_CC(hermes::data_stager::Server, "data_stager"); +HRUN_TASK_CC(hermes::data_stager::Server, "data_stager"); diff --git a/tasks/hermes/include/hermes/bucket.h b/tasks/hermes/include/hermes/bucket.h index 9dd50c61d..6418802d5 100644 --- a/tasks/hermes/include/hermes/bucket.h +++ b/tasks/hermes/include/hermes/bucket.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 7/9/23. // -#ifndef LABSTOR_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ -#define LABSTOR_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ +#ifndef HRUN_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ +#define HRUN_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ #include "hermes/hermes_types.h" #include "hermes_mdm/hermes_mdm.h" @@ -11,7 +11,7 @@ namespace hermes { -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" using hermes::blob_mdm::PutBlobTask; using hermes::blob_mdm::GetBlobTask; @@ -217,7 +217,7 @@ class Bucket { BlobId blob_id = orig_blob_id; bitfield32_t flags, task_flags(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY); // Copy data to shared memory - LPointer p = LABSTOR_CLIENT->AllocateBuffer(blob.size()); + LPointer p = HRUN_CLIENT->AllocateBuffer(blob.size()); char *data = p.ptr_; memcpy(data, blob.data(), blob.size()); // Put to shared memory @@ -231,7 +231,7 @@ class Bucket { if constexpr(!PARTIAL) { flags.SetBits(HERMES_BLOB_REPLACE); } - LPointer> push_task; + LPointer> push_task; push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, flags.bits_, ctx, task_flags.bits_); @@ -240,7 +240,7 @@ class Bucket { push_task->Wait(); PutBlobTask *task = push_task->get(); blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } } return blob_id; @@ -364,7 +364,7 @@ class Bucket { * Append \a blob_name Blob into the bucket (fully asynchronous) * */ void Append(const Blob &blob, size_t page_size, Context &ctx) { - LPointer p = LABSTOR_CLIENT->AllocateBuffer(blob.size()); + LPointer p = HRUN_CLIENT->AllocateBuffer(blob.size()); char *data = p.ptr_; memcpy(data, blob.data(), blob.size()); bkt_mdm_->AppendBlobRoot(id_, blob.size(), p.shm_, page_size, ctx.blob_score_, ctx.node_id_, ctx); @@ -414,7 +414,7 @@ class Bucket { /** * Get \a blob_id Blob from the bucket (async) * */ - LPointer> + LPointer> HSHM_ALWAYS_INLINE AsyncBaseGet(const std::string &blob_name, const BlobId &blob_id, @@ -428,8 +428,8 @@ class Bucket { } // Get from shared memory size_t data_size = blob.size(); - LPointer data_p = LABSTOR_CLIENT->AllocateBuffer(data_size); - LPointer> push_task; + LPointer data_p = HRUN_CLIENT->AllocateBuffer(data_size); + LPointer> push_task; push_task = blob_mdm_->AsyncGetBlobRoot(id_, hshm::to_charbuf(blob_name), blob_id, blob_off, data_size, data_p.shm_, @@ -454,15 +454,15 @@ class Bucket { } HILOG(kDebug, "Getting blob of size {}", data_size); BlobId blob_id; - LPointer> push_task; + LPointer> push_task; push_task = AsyncBaseGet(blob_name, orig_blob_id, blob, blob_off, ctx); push_task->Wait(); GetBlobTask *task = push_task->get(); blob_id = task->blob_id_; - char *data = LABSTOR_CLIENT->GetPrivatePointer(task->data_); + char *data = HRUN_CLIENT->GetPrivatePointer(task->data_); memcpy(blob.data(), data, data_size); - LABSTOR_CLIENT->FreeBuffer(task->data_); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->FreeBuffer(task->data_); + HRUN_CLIENT->DelTask(push_task); return blob_id; } @@ -513,7 +513,7 @@ class Bucket { /** * AsyncGet \a blob_name Blob from the bucket * */ - LPointer> + LPointer> AsyncGet(const std::string &blob_name, Blob &blob, Context &ctx) { @@ -523,7 +523,7 @@ class Bucket { /** * AsyncGet \a blob_id Blob from the bucket * */ - LPointer> + LPointer> AsyncGet(const BlobId &blob_id, Blob &blob, Context &ctx) { @@ -553,7 +553,7 @@ class Bucket { /** * AsyncGet \a blob_name Blob from the bucket * */ - LPointer> + LPointer> AsyncPartialGet(const std::string &blob_name, Blob &blob, size_t blob_off, @@ -564,7 +564,7 @@ class Bucket { /** * AsyncGet \a blob_id Blob from the bucket * */ - LPointer> + LPointer> AsyncPartialGet(const BlobId &blob_id, Blob &blob, size_t blob_off, @@ -605,4 +605,4 @@ class Bucket { } // namespace hermes -#endif // LABSTOR_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ +#endif // HRUN_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ diff --git a/tasks/hermes/include/hermes/config_client_default.h b/tasks/hermes/include/hermes/config_client_default.h index e33d413a0..e2d088e58 100644 --- a/tasks/hermes/include/hermes/config_client_default.h +++ b/tasks/hermes/include/hermes/config_client_default.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ -#define LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ +#ifndef HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ +#define HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ static inline const char* kHermesClientDefaultConfigStr = "stop_daemon: false\n" "path_inclusions: [\"/tmp/test_hermes\"]\n" @@ -11,4 +11,4 @@ static inline const char* kHermesClientDefaultConfigStr = " - path: \"/\"\n" " page_size: 1MB\n" " mode: kDefault\n"; -#endif // LABSTOR_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file +#endif // HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/tasks/hermes/include/hermes/config_manager.h b/tasks/hermes/include/hermes/config_manager.h index e29172953..cf69b70fd 100644 --- a/tasks/hermes/include/hermes/config_manager.h +++ b/tasks/hermes/include/hermes/config_manager.h @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 7/9/23. // -#ifndef LABSTOR_TASKS_HERMES_INCLUDE_hermes_H_ -#define LABSTOR_TASKS_HERMES_INCLUDE_hermes_H_ +#ifndef HRUN_TASKS_HERMES_INCLUDE_hermes_H_ +#define HRUN_TASKS_HERMES_INCLUDE_hermes_H_ #include "hermes/hermes_types.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun_admin/hrun_admin.h" #include "hermes_mdm/hermes_mdm.h" #include "hermes_bucket_mdm/hermes_bucket_mdm.h" #include "hermes_blob_mdm/hermes_blob_mdm.h" @@ -76,11 +76,11 @@ class ConfigurationManager { /** Initialize client-side Hermes transparently */ static inline bool TRANSPARENT_HERMES() { - if (TRANSPARENT_LABSTOR()) { + if (TRANSPARENT_HRUN()) { HERMES_CONF->ClientInit(); return true; } return false; } -#endif // LABSTOR_TASKS_HERMES_INCLUDE_hermes_H_ +#endif // HRUN_TASKS_HERMES_INCLUDE_hermes_H_ diff --git a/tasks/hermes/include/hermes/config_server_default.h b/tasks/hermes/include/hermes/config_server_default.h index f6bdcdd3b..0c3dfd0a4 100644 --- a/tasks/hermes/include/hermes/config_server_default.h +++ b/tasks/hermes/include/hermes/config_server_default.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ -#define LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ +#ifndef HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ +#define HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ static inline const char* kHermesServerDefaultConfigStr = "# Example Hermes configuration file\n" "\n" @@ -178,4 +178,4 @@ static inline const char* kHermesServerDefaultConfigStr = " - \"hermes_mpiio_io_client\"\n" " - \"hermes_example_trait\"\n" " - \"hermes_prefetcher_trait\"\n"; -#endif // LABSTOR_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file +#endif // HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file diff --git a/tasks/hermes/include/hermes/hermes.h b/tasks/hermes/include/hermes/hermes.h index 0645a580e..72d110ed1 100644 --- a/tasks/hermes/include/hermes/hermes.h +++ b/tasks/hermes/include/hermes/hermes.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 7/9/23. // -#ifndef LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ -#define LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ +#ifndef HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ +#define HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ #include "hermes/bucket.h" @@ -64,4 +64,4 @@ class Hermes { } // namespace hermes -#endif // LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ +#endif // HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ diff --git a/tasks/hermes/include/hermes/hermes_types.h b/tasks/hermes/include/hermes/hermes_types.h index acc12ea08..c00baa682 100644 --- a/tasks/hermes/include/hermes/hermes_types.h +++ b/tasks/hermes/include/hermes/hermes_types.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 7/8/23. // -#ifndef LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ -#define LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ +#ifndef HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ +#define HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ -#include "labstor/labstor_types.h" -#include "labstor/task_registry/task_registry.h" -#include "labstor/api/labstor_client.h" +#include "hrun/hrun_types.h" +#include "hrun/task_registry/task_registry.h" +#include "hrun/api/hrun_client.h" #include "status.h" #include "statuses.h" @@ -15,21 +15,21 @@ namespace hapi = hermes; namespace hermes { -using labstor::TaskLib; -using labstor::TaskMethod; -using labstor::UniqueId; -using labstor::TaskStateId; -using labstor::DomainId; -using labstor::Task; -using labstor::TaskId; -using labstor::TaskNode; +using hrun::TaskLib; +using hrun::TaskMethod; +using hrun::UniqueId; +using hrun::TaskStateId; +using hrun::DomainId; +using hrun::Task; +using hrun::TaskId; +using hrun::TaskNode; using hshm::bitfield32_t; /** Queue id */ -using labstor::QueueId; +using hrun::QueueId; /** Queue for interprocess-communication */ -using labstor::MultiQueue; +using hrun::MultiQueue; /** Unique blob id */ typedef UniqueId<100> BlobId; @@ -469,4 +469,4 @@ enum LockOwners { } // namespace hermes -#endif // LABSTOR_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ +#endif // HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ diff --git a/tasks/hermes/include/hermes/slab_allocator.h b/tasks/hermes/include/hermes/slab_allocator.h index 8b177a473..c3de53a6d 100644 --- a/tasks/hermes/include/hermes/slab_allocator.h +++ b/tasks/hermes/include/hermes/slab_allocator.h @@ -2,10 +2,10 @@ // Created by lukemartinlogan on 7/13/23. // -#ifndef LABSTOR_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ -#define LABSTOR_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ +#ifndef HRUN_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ +#define HRUN_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ -#include "labstor/labstor_types.h" +#include "hrun/hrun_types.h" #include "hermes/hermes_types.h" namespace hermes { @@ -147,4 +147,4 @@ class SlabAllocator { } // namespace hermes -#endif // LABSTOR_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ +#endif // HRUN_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ diff --git a/tasks/hermes/include/hermes/statuses.h b/tasks/hermes/include/hermes/statuses.h index 83dcd8168..595e63c90 100644 --- a/tasks/hermes/include/hermes/statuses.h +++ b/tasks/hermes/include/hermes/statuses.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 7/13/23. // -#ifndef LABSTOR_TASKS_HERMES_INCLUDE_STATUSES_H_ -#define LABSTOR_TASKS_HERMES_INCLUDE_STATUSES_H_ +#ifndef HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ +#define HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ #include "status.h" @@ -18,4 +18,4 @@ STATUS_T DPE_MIN_IO_TIME_NO_SOLUTION(1, "DPE could not find solution for the min } // namespace hermes -#endif //LABSTOR_TASKS_HERMES_INCLUDE_STATUSES_H_ +#endif //HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h index 463166e26..f7cf8e705 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_hermes_adapters_H_ -#define LABSTOR_hermes_adapters_H_ +#ifndef HRUN_hermes_adapters_H_ +#define HRUN_hermes_adapters_H_ #include "hermes_adapters_tasks.h" -namespace labstor::hermes_adapters { +namespace hrun::hermes_adapters { /** Create hermes_adapters requests */ class Client : public TaskLibClient { @@ -25,16 +25,16 @@ class Client : public TaskLibClient { const DomainId &domain_id, const std::string &state_name) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info); } - LABSTOR_TASK_NODE_ROOT(AsyncCreate) + HRUN_TASK_NODE_ROOT(AsyncCreate) template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -43,13 +43,13 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /** Call a custom method */ @@ -57,17 +57,17 @@ class Client : public TaskLibClient { void AsyncCustomConstruct(CustomTask *task, const TaskNode &task_node, const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id_); } HSHM_ALWAYS_INLINE void CustomRoot(const DomainId &domain_id) { - LPointer> task = AsyncCustomRoot(domain_id); + LPointer> task = AsyncCustomRoot(domain_id); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(Custom); + HRUN_TASK_NODE_PUSH_ROOT(Custom); }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_hermes_adapters_H_ +#endif // HRUN_hermes_adapters_H_ diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h index 7ff739e1c..3d1096611 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_ADAPTERS_LIB_EXEC_H_ -#define LABSTOR_HERMES_ADAPTERS_LIB_EXEC_H_ +#ifndef HRUN_HERMES_ADAPTERS_LIB_EXEC_H_ +#define HRUN_HERMES_ADAPTERS_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -22,15 +22,15 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kCustom: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -39,15 +39,15 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kCustom: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -56,15 +56,15 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kCustom: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -73,15 +73,15 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kCustom: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -90,15 +90,15 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kCustom: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -126,17 +126,17 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kCustom: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -194,4 +194,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h index 19d2b80a6..965777f39 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h @@ -1,9 +1,9 @@ -#ifndef LABSTOR_HERMES_ADAPTERS_METHODS_H_ -#define LABSTOR_HERMES_ADAPTERS_METHODS_H_ +#ifndef HRUN_HERMES_ADAPTERS_METHODS_H_ +#define HRUN_HERMES_ADAPTERS_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kCustom = kLast + 0; }; -#endif // LABSTOR_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h index cfc03fe9a..dc1afdfb1 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h +++ b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h @@ -2,26 +2,26 @@ // Created by lukemartinlogan on 8/11/23. // -#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ -#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ +#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ +#define HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" -namespace labstor::hermes_adapters { +namespace hrun::hermes_adapters { #include "hermes_adapters_methods.h" -#include "labstor/labstor_namespace.h" -using labstor::proc_queue::TypedPushTask; -using labstor::proc_queue::PushTask; +#include "hrun/hrun_namespace.h" +using hrun::proc_queue::TypedPushTask; +using hrun::proc_queue::PushTask; /** * A task to create hermes_adapters * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -48,7 +48,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy hermes_adapters */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -114,6 +114,6 @@ struct CustomTask : public Task, TaskFlags { } }; -} // namespace labstor::hermes_adapters +} // namespace hrun::hermes_adapters -#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ +#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ diff --git a/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h b/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h index 8ca20026f..30c904f6f 100644 --- a/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h +++ b/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h @@ -37,7 +37,7 @@ struct BlobPlacement { /** create a BLOB name from index. */ static hshm::charbuf CreateBlobName(size_t page) { hshm::charbuf buf(sizeof(page)); - labstor::LocalSerialize srl(buf); + hrun::LocalSerialize srl(buf); srl << page; return buf; } @@ -45,7 +45,7 @@ struct BlobPlacement { /** create a BLOB name from index. */ hshm::charbuf CreateBlobName() const { hshm::charbuf buf(sizeof(page_)); - labstor::LocalSerialize srl(buf); + hrun::LocalSerialize srl(buf); srl << page_; return buf; } @@ -53,7 +53,7 @@ struct BlobPlacement { /** decode \a blob_name BLOB name to index. */ template void DecodeBlobName(const StringT &blob_name, size_t page_size) { - labstor::LocalDeserialize srl(blob_name); + hrun::LocalDeserialize srl(blob_name); srl >> page_; bucket_off_ = page_ * page_size; blob_off_ = 0; diff --git a/tasks/hermes_adapters/src/hermes_adapters.cc b/tasks/hermes_adapters/src/hermes_adapters.cc index d87424bd6..41fa46f64 100644 --- a/tasks/hermes_adapters/src/hermes_adapters.cc +++ b/tasks/hermes_adapters/src/hermes_adapters.cc @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "hermes_adapters/hermes_adapters.h" -namespace labstor::hermes_adapters { +namespace hrun::hermes_adapters { class Server : public TaskLib { public: @@ -34,6 +34,6 @@ class Server : public TaskLib { #include "hermes_adapters/hermes_adapters_lib_exec.h" }; -} // namespace labstor::hermes_adapters +} // namespace hrun::hermes_adapters -LABSTOR_TASK_CC(labstor::hermes_adapters::Server, "hermes_adapters"); +HRUN_TASK_CC(hrun::hermes_adapters::Server, "hermes_adapters"); diff --git a/tasks/hermes_adapters/vfd/README.md b/tasks/hermes_adapters/vfd/README.md index 9667f4762..34b1a5e1c 100644 --- a/tasks/hermes_adapters/vfd/README.md +++ b/tasks/hermes_adapters/vfd/README.md @@ -31,7 +31,7 @@ Now we must tell the HDF5 library where to find the Hermes VFD. That is done with the following environment variable: ```sh -HDF5_PLUGIN_PATH=/lib/hermes_vfd +HDF5_PLUGIN_PATH=/lib/hermes_vfd ``` The Hermes VFD has two configuration options. @@ -65,14 +65,14 @@ initialization and finalization: ```sh HERMES_CONF=/hermes.yaml -LD_PRELOAD=/hermes_vfd/libhdf5_hermes_vfd.so +LD_PRELOAD=/hermes_vfd/libhdf5_hermes_vfd.so ``` Heres is a full example of running an HDF5 app with the Hermes VFD: ```sh HDF5_DRIVER=hermes \ - HDF5_PLUGIN_PATH=/hermes.yaml \ ./my_hdf5_app diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h index 047366a62..140282af1 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_hermes_blob_mdm_H_ -#define LABSTOR_hermes_blob_mdm_H_ +#ifndef HRUN_hermes_blob_mdm_H_ +#define HRUN_hermes_blob_mdm_H_ #include "hermes_blob_mdm_tasks.h" @@ -31,23 +31,23 @@ class Client : public TaskLibClient { const DomainId &domain_id, const std::string &state_name) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info); } void AsyncCreateComplete(ConstructTask *task) { if (task->IsComplete()) { id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } } - LABSTOR_TASK_NODE_ROOT(AsyncCreate); + HRUN_TASK_NODE_ROOT(AsyncCreate); template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -59,7 +59,7 @@ class Client : public TaskLibClient { /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /**==================================== @@ -73,19 +73,19 @@ class Client : public TaskLibClient { const TaskStateId &blob_mdm, const TaskStateId &stager_mdm, const TaskStateId &op_mdm) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id_, blob_mdm, stager_mdm, op_mdm); } void SetBucketMdmRoot(const DomainId &domain_id, const TaskStateId &blob_mdm, const TaskStateId &stager_mdm, const TaskStateId &op_mdm) { - LPointer> push_task = + LPointer> push_task = AsyncSetBucketMdmRoot(domain_id, blob_mdm, stager_mdm, op_mdm); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(SetBucketMdm); + HRUN_TASK_NODE_PUSH_ROOT(SetBucketMdm); /** * Get \a blob_name BLOB from \a bkt_id bucket @@ -95,20 +95,20 @@ class Client : public TaskLibClient { TagId tag_id, const hshm::charbuf &blob_name) { u32 hash = std::hash{}(blob_name); - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, blob_name); } BlobId GetOrCreateBlobIdRoot(TagId tag_id, const hshm::charbuf &blob_name) { - LPointer> push_task = + LPointer> push_task = AsyncGetOrCreateBlobIdRoot(tag_id, blob_name); push_task->Wait(); GetOrCreateBlobIdTask *task = push_task->get(); BlobId blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return blob_id; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateBlobId); + HRUN_TASK_NODE_PUSH_ROOT(GetOrCreateBlobId); /** * Create a blob's metadata @@ -132,13 +132,13 @@ class Client : public TaskLibClient { u32 flags, Context ctx = Context(), u32 task_flags = TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id, blob_off, blob_size, blob, score, flags, ctx, task_flags); } - LABSTOR_TASK_NODE_PUSH_ROOT(PutBlob); + HRUN_TASK_NODE_PUSH_ROOT(PutBlob); /** Get a blob's data */ void AsyncGetBlobConstruct(GetBlobTask *task, @@ -152,7 +152,7 @@ class Client : public TaskLibClient { Context ctx = Context(), u32 flags = 0) { // HILOG(kDebug, "Beginning GET (task_node={})", task_node); - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id, off, data_size, data, ctx, flags); } @@ -163,16 +163,16 @@ class Client : public TaskLibClient { hipc::Pointer &data, Context ctx = Context(), u32 flags = 0) { - LPointer> push_task = + LPointer> push_task = AsyncGetBlobRoot(tag_id, hshm::charbuf(""), blob_id, off, data_size, data, ctx, flags); push_task->Wait(); GetBlobTask *task = push_task->get(); data = task->data_; size_t true_size = task->data_size_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return true_size; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetBlob); + HRUN_TASK_NODE_PUSH_ROOT(GetBlob); /** * Reorganize a blob @@ -188,11 +188,11 @@ class Client : public TaskLibClient { float score, u32 node_id) { // HILOG(kDebug, "Beginning REORGANIZE (task_node={})", task_node); - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id, score, node_id); } - LABSTOR_TASK_NODE_PUSH_ROOT(ReorganizeBlob); + HRUN_TASK_NODE_PUSH_ROOT(ReorganizeBlob); /** * Tag a blob @@ -205,19 +205,19 @@ class Client : public TaskLibClient { const TagId &tag_id, const BlobId &blob_id, const TagId &tag) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id, tag); } void TagBlobRoot(const TagId &tag_id, const BlobId &blob_id, const TagId &tag) { - LPointer> push_task = + LPointer> push_task = AsyncTagBlobRoot(tag_id, blob_id, tag); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(TagBlob); + HRUN_TASK_NODE_PUSH_ROOT(TagBlob); /** * Check if blob has a tag @@ -227,22 +227,22 @@ class Client : public TaskLibClient { const TagId &tag_id, const BlobId &blob_id, const TagId &tag) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id, tag); } bool BlobHasTagRoot(const TagId &tag_id, const BlobId &blob_id, const TagId &tag) { - LPointer> push_task = + LPointer> push_task = AsyncBlobHasTagRoot(tag_id, blob_id, tag); push_task->Wait(); BlobHasTagTask *task = push_task->get(); bool has_tag = task->has_tag_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return has_tag; } - LABSTOR_TASK_NODE_PUSH_ROOT(BlobHasTag); + HRUN_TASK_NODE_PUSH_ROOT(BlobHasTag); /** * Get \a blob_name BLOB from \a bkt_id bucket @@ -252,21 +252,21 @@ class Client : public TaskLibClient { const TagId &tag_id, const hshm::charbuf &blob_name) { u32 hash = std::hash{}(blob_name); - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, blob_name); } BlobId GetBlobIdRoot(const TagId &tag_id, const hshm::charbuf &blob_name) { - LPointer> push_task = + LPointer> push_task = AsyncGetBlobIdRoot(tag_id, blob_name); push_task->Wait(); GetBlobIdTask *task = push_task->get(); BlobId blob_id = task->blob_id_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return blob_id; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobId); + HRUN_TASK_NODE_PUSH_ROOT(GetBlobId); /** * Get \a blob_name BLOB name from \a blob_id BLOB id @@ -275,21 +275,21 @@ class Client : public TaskLibClient { const TaskNode &task_node, const TagId &tag_id, const BlobId &blob_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id); } std::string GetBlobNameRoot(const TagId &tag_id, const BlobId &blob_id) { - LPointer> push_task = + LPointer> push_task = AsyncGetBlobNameRoot(tag_id, blob_id); push_task->Wait(); GetBlobNameTask *task = push_task->get(); std::string blob_name = task->blob_name_->str(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return blob_name; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobName); + HRUN_TASK_NODE_PUSH_ROOT(GetBlobName); /** * Get \a size from \a blob_id BLOB id @@ -300,22 +300,22 @@ class Client : public TaskLibClient { const hshm::charbuf &blob_name, const BlobId &blob_id) { // HILOG(kDebug, "Getting blob size {}", task_node); - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_name, blob_id); } size_t GetBlobSizeRoot(const TagId &tag_id, const hshm::charbuf &blob_name, const BlobId &blob_id) { - LPointer> push_task = + LPointer> push_task = AsyncGetBlobSizeRoot(tag_id, blob_name, blob_id); push_task->Wait(); GetBlobSizeTask *task = push_task->get(); size_t size = task->size_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return size; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobSize); + HRUN_TASK_NODE_PUSH_ROOT(GetBlobSize); /** * Get \a score from \a blob_id BLOB id @@ -324,21 +324,21 @@ class Client : public TaskLibClient { const TaskNode &task_node, const TagId &tag_id, const BlobId &blob_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id); } float GetBlobScoreRoot(const TagId &tag_id, const BlobId &blob_id) { - LPointer> push_task = + LPointer> push_task = AsyncGetBlobScoreRoot(tag_id, blob_id); push_task->Wait(); GetBlobScoreTask *task = push_task->get(); float score = task->score_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return score; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobScore); + HRUN_TASK_NODE_PUSH_ROOT(GetBlobScore); /** * Get \a blob_id blob's buffers @@ -347,21 +347,21 @@ class Client : public TaskLibClient { const TaskNode &task_node, const TagId &tag_id, const BlobId &blob_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id); } std::vector GetBlobBuffersRoot(const TagId &tag_id, const BlobId &blob_id) { - LPointer> push_task = AsyncGetBlobBuffersRoot(tag_id, blob_id); + LPointer> push_task = AsyncGetBlobBuffersRoot(tag_id, blob_id); push_task->Wait(); GetBlobBuffersTask *task = push_task->get(); std::vector buffers = hshm::to_stl_vector(*task->buffers_); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return buffers; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetBlobBuffers) + HRUN_TASK_NODE_PUSH_ROOT(GetBlobBuffers) /** * Rename \a blob_id blob to \a new_blob_name new blob name @@ -372,19 +372,19 @@ class Client : public TaskLibClient { const TagId &tag_id, const BlobId &blob_id, const hshm::charbuf &new_blob_name) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id, new_blob_name); } void RenameBlobRoot(const TagId &tag_id, const BlobId &blob_id, const hshm::charbuf &new_blob_name) { - LPointer> push_task = + LPointer> push_task = AsyncRenameBlobRoot(tag_id, blob_id, new_blob_name); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(RenameBlob); + HRUN_TASK_NODE_PUSH_ROOT(RenameBlob); /** * Truncate a blob to a new size @@ -394,19 +394,19 @@ class Client : public TaskLibClient { const TagId &tag_id, const BlobId &blob_id, size_t new_size) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id, new_size); } void TruncateBlobRoot(const TagId &tag_id, const BlobId &blob_id, size_t new_size) { - LPointer> push_task = + LPointer> push_task = AsyncTruncateBlobRoot(tag_id, blob_id, new_size); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(TruncateBlob); + HRUN_TASK_NODE_PUSH_ROOT(TruncateBlob); /** * Destroy \a blob_id blob in \a bkt_id bucket @@ -415,68 +415,68 @@ class Client : public TaskLibClient { const TaskNode &task_node, const TagId &tag_id, const BlobId &blob_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(blob_id.node_id_), id_, tag_id, blob_id); } void DestroyBlobRoot(const TagId &tag_id, const BlobId &blob_id) { - LPointer> push_task = + LPointer> push_task = AsyncDestroyBlobRoot(tag_id, blob_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(DestroyBlob); + HRUN_TASK_NODE_PUSH_ROOT(DestroyBlob); /** Initialize automatic flushing */ void AsyncFlushDataConstruct(FlushDataTask *task, const TaskNode &task_node) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_); } - LABSTOR_TASK_NODE_PUSH_ROOT(FlushData); + HRUN_TASK_NODE_PUSH_ROOT(FlushData); /** * Get all blob metadata * */ void AsyncPollBlobMetadataConstruct(PollBlobMetadataTask *task, const TaskNode &task_node) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_); } std::vector PollBlobMetadataRoot() { - LPointer> push_task = + LPointer> push_task = AsyncPollBlobMetadataRoot(); push_task->Wait(); PollBlobMetadataTask *task = push_task->get(); std::vector blob_mdms = task->DeserializeBlobMetadata(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return blob_mdms; } - LABSTOR_TASK_NODE_PUSH_ROOT(PollBlobMetadata); + HRUN_TASK_NODE_PUSH_ROOT(PollBlobMetadata); /** * Get all target metadata * */ void AsyncPollTargetMetadataConstruct(PollTargetMetadataTask *task, const TaskNode &task_node) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_); } std::vector PollTargetMetadataRoot() { - LPointer> push_task = + LPointer> push_task = AsyncPollTargetMetadataRoot(); push_task->Wait(); PollTargetMetadataTask *task = push_task->get(); std::vector target_mdms = task->DeserializeTargetMetadata(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return target_mdms; } - LABSTOR_TASK_NODE_PUSH_ROOT(PollTargetMetadata); + HRUN_TASK_NODE_PUSH_ROOT(PollTargetMetadata); }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_hermes_blob_mdm_H_ +#endif // HRUN_hermes_blob_mdm_H_ diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h index 3e54512d9..9b702250e 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_BLOB_MDM_LIB_EXEC_H_ -#define LABSTOR_HERMES_BLOB_MDM_LIB_EXEC_H_ +#ifndef HRUN_HERMES_BLOB_MDM_LIB_EXEC_H_ +#define HRUN_HERMES_BLOB_MDM_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -90,83 +90,83 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kPutBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kTruncateBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestroyBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kTagBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kBlobHasTag: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetBlobId: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetOrCreateBlobId: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetBlobName: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetBlobSize: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetBlobScore: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetBlobBuffers: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRenameBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kReorganizeBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kSetBucketMdm: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kFlushData: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kPollBlobMetadata: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kPollTargetMetadata: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -175,83 +175,83 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kPutBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kTruncateBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestroyBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kTagBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kBlobHasTag: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetBlobId: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetOrCreateBlobId: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetBlobName: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetBlobSize: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetBlobScore: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetBlobBuffers: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRenameBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kReorganizeBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kSetBucketMdm: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kFlushData: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kPollBlobMetadata: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kPollTargetMetadata: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -260,83 +260,83 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kPutBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kTruncateBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestroyBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kTagBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kBlobHasTag: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetBlobId: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetOrCreateBlobId: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetBlobName: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetBlobSize: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetBlobScore: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetBlobBuffers: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRenameBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kReorganizeBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kSetBucketMdm: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kFlushData: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kPollBlobMetadata: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kPollTargetMetadata: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -345,83 +345,83 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kPutBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kTruncateBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestroyBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kTagBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kBlobHasTag: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetBlobId: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetOrCreateBlobId: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetBlobName: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetBlobSize: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetBlobScore: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetBlobBuffers: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRenameBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kReorganizeBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kSetBucketMdm: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kFlushData: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kPollBlobMetadata: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kPollTargetMetadata: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -430,83 +430,83 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kPutBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kTruncateBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestroyBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kTagBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kBlobHasTag: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetBlobId: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetOrCreateBlobId: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetBlobName: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetBlobSize: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetBlobScore: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetBlobBuffers: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRenameBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kReorganizeBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kSetBucketMdm: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kFlushData: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kPollBlobMetadata: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kPollTargetMetadata: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -602,102 +602,102 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPutBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTruncateBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kBlobHasTag: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobId: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetOrCreateBlobId: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobName: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobSize: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobScore: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetBlobBuffers: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRenameBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kReorganizeBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetBucketMdm: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kFlushData: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPollBlobMetadata: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPollTargetMetadata: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -942,4 +942,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h index 3cbb59a0b..7bd0cc751 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_BLOB_MDM_METHODS_H_ -#define LABSTOR_HERMES_BLOB_MDM_METHODS_H_ +#ifndef HRUN_HERMES_BLOB_MDM_METHODS_H_ +#define HRUN_HERMES_BLOB_MDM_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -23,4 +23,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kPollTargetMetadata = kLast + 19; }; -#endif // LABSTOR_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_BLOB_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h index 56a2ad7f9..3852fb921 100644 --- a/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h +++ b/tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_tasks.h @@ -2,29 +2,29 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ -#define LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ +#ifndef HRUN_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ +#define HRUN_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "hermes/hermes_types.h" #include "bdev/bdev.h" -#include "labstor/api/labstor_client.h" +#include "hrun/api/hrun_client.h" #include "proc_queue/proc_queue.h" namespace hermes::blob_mdm { #include "hermes_blob_mdm_methods.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" -using labstor::Task; -using labstor::TaskFlags; -using labstor::DataTransfer; +using hrun::Task; +using hrun::TaskFlags; +using hrun::DataTransfer; /** Phases of the construct task */ -using labstor::Admin::CreateTaskStatePhase; +using hrun::Admin::CreateTaskStatePhase; class ConstructTaskPhase : public CreateTaskStatePhase { public: TASK_METHOD_T kCreateTaskStates = kLast + 0; @@ -34,7 +34,7 @@ class ConstructTaskPhase : public CreateTaskStatePhase { /** * A task to create hermes_mdm * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -54,7 +54,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy hermes_mdm */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -288,7 +288,7 @@ struct PutBlobTask : public Task, TaskFlags ~PutBlobTask() { HSHM_DESTROY_AR(blob_name_); if (IsDataOwner()) { - LABSTOR_CLIENT->FreeBuffer(data_); + HRUN_CLIENT->FreeBuffer(data_); } } @@ -324,7 +324,7 @@ struct PutBlobTask : public Task, TaskFlags /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -397,7 +397,7 @@ struct GetBlobTask : public Task, TaskFlags template HSHM_ALWAYS_INLINE void Get(T &obj) { - char *data = LABSTOR_CLIENT->GetPrivatePointer(data_); + char *data = HRUN_CLIENT->GetPrivatePointer(data_); std::stringstream ss(std::string(data, data_size_)); cereal::BinaryInputArchive ar(ss); ar >> obj; @@ -448,7 +448,7 @@ struct GetBlobTask : public Task, TaskFlags /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -504,7 +504,7 @@ struct TagBlobTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -565,7 +565,7 @@ struct BlobHasTagTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -627,7 +627,7 @@ struct GetBlobIdTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -690,7 +690,7 @@ struct GetBlobNameTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -758,7 +758,7 @@ struct GetBlobSizeTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -813,7 +813,7 @@ struct GetBlobScoreTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -874,7 +874,7 @@ struct GetBlobBuffersTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -939,7 +939,7 @@ struct RenameBlobTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -995,7 +995,7 @@ struct TruncateBlobTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -1056,7 +1056,7 @@ struct DestroyBlobTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -1127,7 +1127,7 @@ struct ReorganizeBlobTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -1420,4 +1420,4 @@ struct PollTargetMetadataTask : public Task, TaskFlags } // namespace hermes::blob_mdm -#endif //LABSTOR_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ +#endif //HRUN_TASKS_HERMES_BLOB_MDM_INCLUDE_HERMES_BLOB_MDM_HERMES_BLOB_MDM_TASKS_H_ diff --git a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc index 582fd9d6f..e7f0065c8 100644 --- a/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc +++ b/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc @@ -1,8 +1,8 @@ // // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "hermes/config_server.h" #include "hermes_blob_mdm/hermes_blob_mdm.h" #include "hermes_adapters/mapper/mapper_factory.h" @@ -57,10 +57,10 @@ class Server : public TaskLib { void Construct(ConstructTask *task, RunContext &rctx) { id_alloc_ = 0; - node_id_ = LABSTOR_CLIENT->node_id_; + node_id_ = HRUN_CLIENT->node_id_; // Initialize blob maps - blob_id_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); - blob_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + blob_id_map_.resize(HRUN_QM_RUNTIME->max_lanes_); + blob_map_.resize(HRUN_QM_RUNTIME->max_lanes_); // Initialize targets target_tasks_.reserve(HERMES_SERVER_CONF.devices_.size()); for (DeviceInfo &dev : HERMES_SERVER_CONF.devices_) { @@ -89,7 +89,7 @@ class Server : public TaskLib { target_map_.emplace(client.id_, &client); } blob_mdm_.Init(id_); - HILOG(kInfo, "(node {}) Created Blob MDM", LABSTOR_CLIENT->node_id_); + HILOG(kInfo, "(node {}) Created Blob MDM", HRUN_CLIENT->node_id_); task->SetModuleComplete(); } @@ -101,7 +101,7 @@ class Server : public TaskLib { /** Get the globally unique blob name */ const hshm::charbuf GetBlobNameWithBucket(TagId tag_id, const hshm::charbuf &blob_name) { hshm::charbuf new_name(sizeof(TagId) + blob_name.size()); - labstor::LocalSerialize srl(new_name); + hrun::LocalSerialize srl(new_name); srl << tag_id.node_id_; srl << tag_id.unique_; srl << blob_name; @@ -139,7 +139,7 @@ class Server : public TaskLib { blob_info.mod_count_ = 0; blob_info.access_freq_ = 0; blob_info.UpdateWriteStats(); - LPointer data = LABSTOR_CLIENT->AllocateBuffer(blob_info.blob_size_); + LPointer data = HRUN_CLIENT->AllocateBuffer(blob_info.blob_size_); LPointer get_blob = blob_mdm_.AsyncGetBlob(task->task_node_ + 1, blob_info.tag_id_, @@ -192,7 +192,7 @@ class Server : public TaskLib { blob_info.name_, task->score_, 0); stage_task->Wait(task); - LABSTOR_CLIENT->DelTask(stage_task); + HRUN_CLIENT->DelTask(stage_task); } } else { // Modify existing blob @@ -235,7 +235,7 @@ class Server : public TaskLib { // next_placement.size_ += diff; HELOG(kFatal, "Ran outta space in this tier -- will fix soon") } - LABSTOR_CLIENT->DelTask(alloc_task); + HRUN_CLIENT->DelTask(alloc_task); } } @@ -243,7 +243,7 @@ class Server : public TaskLib { std::vector> write_tasks; write_tasks.reserve(blob_info.buffers_.size()); size_t blob_off = 0, buf_off = 0; - char *blob_buf = LABSTOR_CLIENT->GetPrivatePointer(task->data_); + char *blob_buf = HRUN_CLIENT->GetPrivatePointer(task->data_); HILOG(kDebug, "Number of buffers {}", blob_info.buffers_.size()); for (BufferInfo &buf : blob_info.buffers_) { size_t blob_left = blob_off; @@ -271,7 +271,7 @@ class Server : public TaskLib { // Wait for the placements to complete for (LPointer &write_task : write_tasks) { write_task->Wait(task); - LABSTOR_CLIENT->DelTask(write_task); + HRUN_CLIENT->DelTask(write_task); } // Update information @@ -372,7 +372,7 @@ class Server : public TaskLib { } } for (bdev::ReadTask *&read_task : read_tasks) { - LABSTOR_CLIENT->DelTask(read_task); + HRUN_CLIENT->DelTask(read_task); } HSHM_DESTROY_AR(task->bdev_reads_); HILOG(kDebug, "GetBlobTask complete"); @@ -593,7 +593,7 @@ class Server : public TaskLib { } } for (bdev::FreeTask *&free_task : free_tasks) { - LABSTOR_CLIENT->DelTask(free_task); + HRUN_CLIENT->DelTask(free_task); } BLOB_MAP_T &blob_map = blob_map_[rctx.lane_id_]; BlobInfo &blob_info = blob_map[task->blob_id_]; @@ -621,7 +621,7 @@ class Server : public TaskLib { return; } BlobInfo &blob_info = it->second; - task->data_ = LABSTOR_CLIENT->AllocateBuffer(blob_info.blob_size_).shm_; + task->data_ = HRUN_CLIENT->AllocateBuffer(blob_info.blob_size_).shm_; task->data_size_ = blob_info.blob_size_; task->get_task_ = blob_mdm_.AsyncGetBlob(task->task_node_ + 1, task->tag_id_, @@ -637,7 +637,7 @@ class Server : public TaskLib { if (!task->get_task_->IsComplete()) { return; } - LABSTOR_CLIENT->DelTask(task->get_task_); + HRUN_CLIENT->DelTask(task->get_task_); task->phase_ = ReorganizeBlobPhase::kPut; } case ReorganizeBlobPhase::kPut: { @@ -677,7 +677,7 @@ class Server : public TaskLib { std::vector target_mdms; target_mdms.reserve(targets_.size()); for (const bdev::Client &bdev_client : targets_) { - bool is_remote = bdev_client.domain_id_.IsRemote(LABSTOR_RPC->GetNumHosts(), LABSTOR_CLIENT->node_id_); + bool is_remote = bdev_client.domain_id_.IsRemote(HRUN_RPC->GetNumHosts(), HRUN_CLIENT->node_id_); if (is_remote) { continue; } @@ -698,6 +698,6 @@ class Server : public TaskLib { #include "hermes_blob_mdm/hermes_blob_mdm_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(hermes::blob_mdm::Server, "hermes_blob_mdm"); +HRUN_TASK_CC(hermes::blob_mdm::Server, "hermes_blob_mdm"); diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h index 0da9260d9..ad6d66e52 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_hermes_bucket_mdm_H_ -#define LABSTOR_hermes_bucket_mdm_H_ +#ifndef HRUN_hermes_bucket_mdm_H_ +#define HRUN_hermes_bucket_mdm_H_ #include "hermes_bucket_mdm_tasks.h" @@ -23,13 +23,13 @@ class Client : public TaskLibClient { void CreateRoot(const DomainId &domain_id, const std::string &state_name) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + id_ = HRUN_ADMIN->CreateTaskStateRoot( domain_id, state_name, id_, queue_info); queue_id_ = QueueId(id_); } @@ -37,7 +37,7 @@ class Client : public TaskLibClient { /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /**==================================== @@ -50,18 +50,18 @@ class Client : public TaskLibClient { const DomainId &domain_id, const TaskStateId &blob_mdm, const TaskStateId &stager_mdm) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id_, blob_mdm, stager_mdm); } void SetBlobMdmRoot(const DomainId &domain_id, const TaskStateId &blob_mdm, const TaskStateId &stager_mdm) { - LPointer> push_task = + LPointer> push_task = AsyncSetBlobMdmRoot(domain_id, blob_mdm, stager_mdm); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(SetBlobMdm); + HRUN_TASK_NODE_PUSH_ROOT(SetBlobMdm); /** Update statistics after blob PUT (fire & forget) */ HSHM_ALWAYS_INLINE @@ -70,11 +70,11 @@ class Client : public TaskLibClient { TagId tag_id, size_t update, int mode) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(tag_id.node_id_), id_, tag_id, update, mode); } - LABSTOR_TASK_NODE_PUSH_ROOT(UpdateSize); + HRUN_TASK_NODE_PUSH_ROOT(UpdateSize); /** Append data to the bucket (fire & forget) */ HSHM_ALWAYS_INLINE @@ -83,11 +83,11 @@ class Client : public TaskLibClient { TagId tag_id, size_t data_size, size_t page_size) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(tag_id.node_id_), id_, tag_id, data_size, page_size); } - LABSTOR_TASK_NODE_PUSH_ROOT(AppendBlobSchema); + HRUN_TASK_NODE_PUSH_ROOT(AppendBlobSchema); /** Append data to the bucket (fire & forget) */ HSHM_ALWAYS_INLINE @@ -101,7 +101,7 @@ class Client : public TaskLibClient { float score, u32 node_id, const Context &ctx) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetLocal(), id_, tag_id, data_size, data, page_size, score, node_id, ctx); } @@ -115,7 +115,7 @@ class Client : public TaskLibClient { const Context &ctx) { AsyncAppendBlobRoot(tag_id, data_size, data, page_size, score, node_id, ctx); } - LABSTOR_TASK_NODE_PUSH_ROOT(AppendBlob); + HRUN_TASK_NODE_PUSH_ROOT(AppendBlob); /** Create a tag or get the ID of existing tag */ HSHM_ALWAYS_INLINE @@ -128,7 +128,7 @@ class Client : public TaskLibClient { u32 flags) { HILOG(kDebug, "Creating a tag {}", tag_name.str()); u32 hash = std::hash{}(tag_name); - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_name, blob_owner, traits, backend_size, flags); } @@ -138,55 +138,55 @@ class Client : public TaskLibClient { const std::vector &traits, size_t backend_size, u32 flags) { - LPointer> push_task = + LPointer> push_task = AsyncGetOrCreateTagRoot(tag_name, blob_owner, traits, backend_size, flags); push_task->Wait(); GetOrCreateTagTask *task = push_task->get(); TagId tag_id = task->tag_id_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return tag_id; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetOrCreateTag); + HRUN_TASK_NODE_PUSH_ROOT(GetOrCreateTag); /** Get tag ID */ void AsyncGetTagIdConstruct(GetTagIdTask *task, const TaskNode &task_node, const hshm::charbuf &tag_name) { u32 hash = std::hash{}(tag_name); - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_name); } TagId GetTagIdRoot(const hshm::charbuf &tag_name) { - LPointer> push_task = + LPointer> push_task = AsyncGetTagIdRoot(tag_name); push_task->Wait(); GetTagIdTask *task = push_task->get(); TagId tag_id = task->tag_id_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return tag_id; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetTagId); + HRUN_TASK_NODE_PUSH_ROOT(GetTagId); /** Get tag name */ void AsyncGetTagNameConstruct(GetTagNameTask *task, const TaskNode &task_node, const TagId &tag_id) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); } hshm::string GetTagNameRoot(const TagId &tag_id) { - LPointer> push_task = + LPointer> push_task = AsyncGetTagNameRoot(tag_id); push_task->Wait(); GetTagNameTask *task = push_task->get(); hshm::string tag_name = hshm::to_charbuf(*task->tag_name_.get()); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return tag_name; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetTagName); + HRUN_TASK_NODE_PUSH_ROOT(GetTagName); /** Rename tag */ void AsyncRenameTagConstruct(RenameTagTask *task, @@ -194,34 +194,34 @@ class Client : public TaskLibClient { const TagId &tag_id, const hshm::charbuf &new_tag_name) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, new_tag_name); } void RenameTagRoot(const TagId &tag_id, const hshm::charbuf &new_tag_name) { - LPointer> push_task = + LPointer> push_task = AsyncRenameTagRoot(tag_id, new_tag_name); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(RenameTag); + HRUN_TASK_NODE_PUSH_ROOT(RenameTag); /** Destroy tag */ void AsyncDestroyTagConstruct(DestroyTagTask *task, const TaskNode &task_node, const TagId &tag_id) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); } void DestroyTagRoot(const TagId &tag_id) { - LPointer> push_task = + LPointer> push_task = AsyncDestroyTagRoot(tag_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(DestroyTag); + HRUN_TASK_NODE_PUSH_ROOT(DestroyTag); /** Add a blob to a tag */ void AsyncTagAddBlobConstruct(TagAddBlobTask *task, @@ -229,113 +229,113 @@ class Client : public TaskLibClient { const TagId &tag_id, const BlobId &blob_id) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, blob_id); } void TagAddBlobRoot(const TagId &tag_id, const BlobId &blob_id) { - LPointer> push_task = + LPointer> push_task = AsyncTagAddBlobRoot(tag_id, blob_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(TagAddBlob); + HRUN_TASK_NODE_PUSH_ROOT(TagAddBlob); /** Remove a blob from a tag */ void AsyncTagRemoveBlobConstruct(TagRemoveBlobTask *task, const TaskNode &task_node, const TagId &tag_id, const BlobId &blob_id) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id, blob_id); } void TagRemoveBlobRoot(const TagId &tag_id, const BlobId &blob_id) { - LPointer> push_task = + LPointer> push_task = AsyncTagRemoveBlobRoot(tag_id, blob_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(TagRemoveBlob); + HRUN_TASK_NODE_PUSH_ROOT(TagRemoveBlob); /** Clear blobs from a tag */ void AsyncTagClearBlobsConstruct(TagClearBlobsTask *task, const TaskNode &task_node, const TagId &tag_id) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); } void TagClearBlobsRoot(const TagId &tag_id) { - LPointer> push_task = + LPointer> push_task = AsyncTagClearBlobsRoot(tag_id); push_task->Wait(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); } - LABSTOR_TASK_NODE_PUSH_ROOT(TagClearBlobs); + HRUN_TASK_NODE_PUSH_ROOT(TagClearBlobs); /** Get the size of a bucket */ void AsyncGetSizeConstruct(GetSizeTask *task, const TaskNode &task_node, const TagId &tag_id) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); } size_t GetSizeRoot(const TagId &tag_id) { - LPointer> push_task = + LPointer> push_task = AsyncGetSizeRoot(tag_id); push_task->Wait(); GetSizeTask *task = push_task->get(); size_t size = task->size_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return size; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetSize); + HRUN_TASK_NODE_PUSH_ROOT(GetSize); /** Get contained blob ids */ void AsyncGetContainedBlobIdsConstruct(GetContainedBlobIdsTask *task, const TaskNode &task_node, const TagId &tag_id) { u32 hash = tag_id.hash_; - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetNode(HASH_TO_NODE_ID(hash)), id_, tag_id); } std::vector GetContainedBlobIdsRoot(const TagId &tag_id) { - LPointer> push_task = + LPointer> push_task = AsyncGetContainedBlobIdsRoot(tag_id); push_task->Wait(); GetContainedBlobIdsTask *task = push_task->get(); std::vector blob_ids = task->blob_ids_->vec(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return blob_ids; } - LABSTOR_TASK_NODE_PUSH_ROOT(GetContainedBlobIds); + HRUN_TASK_NODE_PUSH_ROOT(GetContainedBlobIds); /** * Get all tag metadata * */ void AsyncPollTagMetadataConstruct(PollTagMetadataTask *task, const TaskNode &task_node) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_); } std::vector PollTagMetadataRoot() { - LPointer> push_task = + LPointer> push_task = AsyncPollTagMetadataRoot(); push_task->Wait(); PollTagMetadataTask *task = push_task->get(); std::vector target_mdms = task->DeserializeTagMetadata(); - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return target_mdms; } - LABSTOR_TASK_NODE_PUSH_ROOT(PollTagMetadata); + HRUN_TASK_NODE_PUSH_ROOT(PollTagMetadata); }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_hermes_bucket_mdm_H_ +#endif // HRUN_hermes_bucket_mdm_H_ diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h index 560014830..323022042 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_BUCKET_MDM_LIB_EXEC_H_ -#define LABSTOR_HERMES_BUCKET_MDM_LIB_EXEC_H_ +#ifndef HRUN_HERMES_BUCKET_MDM_LIB_EXEC_H_ +#define HRUN_HERMES_BUCKET_MDM_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -78,71 +78,71 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetOrCreateTag: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetTagId: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetTagName: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRenameTag: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestroyTag: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kTagAddBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kTagRemoveBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kTagClearBlobs: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kUpdateSize: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kAppendBlobSchema: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kAppendBlob: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetSize: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kSetBlobMdm: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetContainedBlobIds: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kPollTagMetadata: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -151,71 +151,71 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetOrCreateTag: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetTagId: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetTagName: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRenameTag: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestroyTag: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kTagAddBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kTagRemoveBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kTagClearBlobs: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kUpdateSize: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kAppendBlobSchema: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kAppendBlob: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetSize: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kSetBlobMdm: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetContainedBlobIds: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kPollTagMetadata: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -224,71 +224,71 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetOrCreateTag: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetTagId: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetTagName: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRenameTag: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestroyTag: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kTagAddBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kTagRemoveBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kTagClearBlobs: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kUpdateSize: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kAppendBlobSchema: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kAppendBlob: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetSize: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kSetBlobMdm: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetContainedBlobIds: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kPollTagMetadata: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -297,71 +297,71 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetOrCreateTag: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetTagId: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetTagName: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRenameTag: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestroyTag: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kTagAddBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kTagRemoveBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kTagClearBlobs: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kUpdateSize: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kAppendBlobSchema: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kAppendBlob: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetSize: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kSetBlobMdm: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetContainedBlobIds: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kPollTagMetadata: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -370,71 +370,71 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetOrCreateTag: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetTagId: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetTagName: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRenameTag: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestroyTag: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kTagAddBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kTagRemoveBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kTagClearBlobs: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kUpdateSize: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kAppendBlobSchema: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kAppendBlob: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetSize: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kSetBlobMdm: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetContainedBlobIds: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kPollTagMetadata: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -518,87 +518,87 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetOrCreateTag: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetTagId: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetTagName: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRenameTag: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyTag: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagAddBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagRemoveBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kTagClearBlobs: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kUpdateSize: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kAppendBlobSchema: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kAppendBlob: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetSize: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetBlobMdm: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetContainedBlobIds: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPollTagMetadata: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -810,4 +810,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h index f797d2b18..0146a45ce 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ -#define LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ +#ifndef HRUN_HERMES_BUCKET_MDM_METHODS_H_ +#define HRUN_HERMES_BUCKET_MDM_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -20,4 +20,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kPollTagMetadata = kLast + 17; }; -#endif // LABSTOR_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_BUCKET_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h index 764a168a3..24eaf2dad 100644 --- a/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h +++ b/tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_tasks.h @@ -2,29 +2,29 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ -#define LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ +#ifndef HRUN_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ +#define HRUN_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "hermes/hermes_types.h" #include "bdev/bdev.h" #include "hermes_blob_mdm/hermes_blob_mdm.h" -#include "labstor/api/labstor_client.h" -#include "labstor/labstor_namespace.h" +#include "hrun/api/hrun_client.h" +#include "hrun/hrun_namespace.h" #include "proc_queue/proc_queue.h" namespace hermes::bucket_mdm { #include "hermes_bucket_mdm_methods.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" /** * A task to create hermes_bucket_mdm * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -44,7 +44,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy hermes_bucket_mdm */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -189,7 +189,7 @@ struct UpdateSizeTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -274,7 +274,7 @@ struct AppendBlobSchemaTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -330,14 +330,14 @@ struct AppendBlobTask : public Task, TaskFlags { /** Destructor */ ~AppendBlobTask() { if (IsDataOwner()) { - LABSTOR_CLIENT->FreeBuffer(data_); + HRUN_CLIENT->FreeBuffer(data_); } } /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -519,7 +519,7 @@ struct GetTagNameTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -578,7 +578,7 @@ struct RenameTagTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -635,7 +635,7 @@ struct DestroyTagTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -687,7 +687,7 @@ struct TagAddBlobTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -739,7 +739,7 @@ struct TagRemoveBlobTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -797,7 +797,7 @@ struct TagClearBlobsTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -849,7 +849,7 @@ struct GetSizeTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -907,7 +907,7 @@ struct GetContainedBlobIdsTask : public Task, TaskFlags { /** Create group */ HSHM_ALWAYS_INLINE u32 GetGroup(hshm::charbuf &group) { - labstor::LocalSerialize srl(group); + hrun::LocalSerialize srl(group); srl << tag_id_.unique_; srl << tag_id_.node_id_; return 0; @@ -1028,4 +1028,4 @@ struct PollTagMetadataTask : public Task, TaskFlags { } // namespace hermes::bucket_mdm -#endif // LABSTOR_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ +#endif // HRUN_TASKS_HERMES_BUCKET_MDM_INCLUDE_HERMES_BUCKET_MDM_HERMES_BUCKET_MDM_TASKS_H_ diff --git a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc index 486c5070a..4e167e0ee 100644 --- a/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc +++ b/tasks/hermes_bucket_mdm/src/hermes_bucket_mdm.cc @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "hermes/config_server.h" #include "hermes_bucket_mdm/hermes_bucket_mdm.h" #include "hermes_adapters/mapper/abstract_mapper.h" @@ -31,10 +31,10 @@ class Server : public TaskLib { void Construct(ConstructTask *task, RunContext &rctx) { id_alloc_ = 0; - node_id_ = LABSTOR_CLIENT->node_id_; + node_id_ = HRUN_CLIENT->node_id_; bkt_mdm_.Init(id_); - tag_id_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); - tag_map_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + tag_id_map_.resize(HRUN_QM_RUNTIME->max_lanes_); + tag_map_.resize(HRUN_QM_RUNTIME->max_lanes_); task->SetModuleComplete(); } @@ -74,7 +74,7 @@ class Server : public TaskLib { switch (task->phase_) { case AppendBlobPhase::kGetBlobIds: { HILOG(kDebug, "(node {}) Getting blob IDs for tag {} (task_node={})", - LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) + HRUN_CLIENT->node_id_, task->tag_id_, task->task_node_) TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; TagInfo &tag_info = tag_map[task->tag_id_]; size_t bucket_size = tag_info.internal_size_; @@ -84,7 +84,7 @@ class Server : public TaskLib { size_t max_pages = task->data_size_ / task->page_size_ + 1; size_t cur_size = 0; HILOG(kDebug, "(node {}) Bucket size {}, page_size {}, cur_page {} (task_node={})", - LABSTOR_CLIENT->node_id_, bucket_size, task->page_size_, cur_page, task->task_node_) + HRUN_CLIENT->node_id_, bucket_size, task->page_size_, cur_page, task->task_node_) HSHM_MAKE_AR0(task->append_info_, nullptr); std::vector &append_info = *task->append_info_; append_info.reserve(max_pages); @@ -115,10 +115,10 @@ class Server : public TaskLib { } } HILOG(kDebug, "(node {}) Finished blob IDs for tag {} (task_node={})", - LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) + HRUN_CLIENT->node_id_, task->tag_id_, task->task_node_) for (AppendInfo &append : append_info) { append.blob_id_ = append.blob_id_task_->blob_id_; - LABSTOR_CLIENT->DelTask(append.blob_id_task_); + HRUN_CLIENT->DelTask(append.blob_id_task_); } task->SetModuleComplete(); } @@ -134,7 +134,7 @@ class Server : public TaskLib { switch (task->phase_) { case AppendBlobPhase::kGetBlobIds: { HILOG(kDebug, "(node {}) Appending {} bytes to bucket {} (task_node={})", - LABSTOR_CLIENT->node_id_, task->data_size_, task->tag_id_, task->task_node_); + HRUN_CLIENT->node_id_, task->data_size_, task->tag_id_, task->task_node_); task->schema_ = bkt_mdm_.AsyncAppendBlobSchema(task->task_node_ + 1, task->tag_id_, task->data_size_, @@ -148,10 +148,10 @@ class Server : public TaskLib { std::vector &append_info = *task->schema_->append_info_; size_t buf_off = 0; HILOG(kDebug, "(node {}) Got blob schema of size {} for tag {} (task_node={})", - LABSTOR_CLIENT->node_id_, append_info.size(), task->tag_id_, task->task_node_) + HRUN_CLIENT->node_id_, append_info.size(), task->tag_id_, task->task_node_) for (AppendInfo &append : append_info) { HILOG(kDebug, "(node {}) Spawning blob {} of size {} for tag {} (task_node={} blob_mdm={})", - LABSTOR_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, + HRUN_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, task->tag_id_, task->task_node_, blob_mdm_.id_); append.put_task_ = blob_mdm_.AsyncPutBlob(task->task_node_ + 1, task->tag_id_, @@ -163,7 +163,7 @@ class Server : public TaskLib { task->score_, 0, Context(), 0).ptr_; HILOG(kDebug, "(node {}) Finished spawning blob {} of size {} for tag {} (task_node={} blob_mdm={})", - LABSTOR_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, + HRUN_CLIENT->node_id_, append.blob_name_.str(), append.data_size_, task->tag_id_, task->task_node_, blob_mdm_.id_); buf_off += append.data_size_; } @@ -177,12 +177,12 @@ class Server : public TaskLib { } } HILOG(kDebug, "(node {}) PUT blobs for tag {} (task_node={})", - LABSTOR_CLIENT->node_id_, task->tag_id_, task->task_node_) + HRUN_CLIENT->node_id_, task->tag_id_, task->task_node_) for (AppendInfo &append : append_info) { - LABSTOR_CLIENT->DelTask(append.put_task_); + HRUN_CLIENT->DelTask(append.put_task_); } HSHM_DESTROY_AR(task->schema_->append_info_); - LABSTOR_CLIENT->DelTask(task->schema_); + HRUN_CLIENT->DelTask(task->schema_); task->SetModuleComplete(); } } @@ -207,7 +207,7 @@ class Server : public TaskLib { TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; tag_id.unique_ = id_alloc_.fetch_add(1); tag_id.hash_ = task->lane_hash_; - tag_id.node_id_ = LABSTOR_RUNTIME->rpc_.node_id_; + tag_id.node_id_ = HRUN_RUNTIME->rpc_.node_id_; HILOG(kDebug, "Creating tag for the first time: {} {}", tag_name.str(), tag_id) tag_id_map.emplace(tag_name, tag_id); tag_map.emplace(tag_id, TagInfo()); @@ -303,7 +303,7 @@ class Server : public TaskLib { } } for (blob_mdm::DestroyBlobTask *&blob_task : blob_tasks) { - LABSTOR_CLIENT->DelTask(blob_task); + HRUN_CLIENT->DelTask(blob_task); } HSHM_DESTROY_AR(task->destroy_blob_tasks_); TAG_MAP_T &tag_map = tag_map_[rctx.lane_id_]; @@ -405,6 +405,6 @@ class Server : public TaskLib { #include "hermes_bucket_mdm/hermes_bucket_mdm_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(hermes::bucket_mdm::Server, "hermes_bucket_mdm"); +HRUN_TASK_CC(hermes::bucket_mdm::Server, "hermes_bucket_mdm"); diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h index 61439b0a5..54c583f2d 100644 --- a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_hermes_data_op_H_ -#define LABSTOR_hermes_data_op_H_ +#ifndef HRUN_hermes_data_op_H_ +#define HRUN_hermes_data_op_H_ #include "hermes_data_op_tasks.h" @@ -27,17 +27,17 @@ class Client : public TaskLibClient { TaskStateId &bkt_mdm_id, TaskStateId &blob_mdm_id) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info, bkt_mdm_id, blob_mdm_id); } - LABSTOR_TASK_NODE_ROOT(AsyncCreate) + HRUN_TASK_NODE_ROOT(AsyncCreate) template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -46,13 +46,13 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /** Register the OpGraph to perform on data */ @@ -60,16 +60,16 @@ class Client : public TaskLibClient { void AsyncRegisterOpConstruct(RegisterOpTask *task, const TaskNode &task_node, const OpGraph &op_graph) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, DomainId::GetGlobal(), id_, op_graph); } HSHM_ALWAYS_INLINE void RegisterOpRoot(const OpGraph &op_graph) { - LPointer> task = + LPointer> task = AsyncRegisterOpRoot(op_graph); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(RegisterOp); + HRUN_TASK_NODE_PUSH_ROOT(RegisterOp); /** Register data as ready for operations to be performed */ HSHM_ALWAYS_INLINE @@ -80,22 +80,22 @@ class Client : public TaskLibClient { const BlobId &blob_id, size_t off, size_t size) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_, bkt_id, blob_name, blob_id, off, size); } - LABSTOR_TASK_NODE_PUSH_ROOT(RegisterData); + HRUN_TASK_NODE_PUSH_ROOT(RegisterData); /** Async task to run operators */ HSHM_ALWAYS_INLINE void AsyncRunOpConstruct(RunOpTask *task, const TaskNode &task_node) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, id_); } - LABSTOR_TASK_NODE_PUSH_ROOT(RunOp); + HRUN_TASK_NODE_PUSH_ROOT(RunOp); }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_hermes_data_op_H_ +#endif // HRUN_hermes_data_op_H_ diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h index ce08374ad..51413ffc1 100644 --- a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_DATA_OP_LIB_EXEC_H_ -#define LABSTOR_HERMES_DATA_OP_LIB_EXEC_H_ +#ifndef HRUN_HERMES_DATA_OP_LIB_EXEC_H_ +#define HRUN_HERMES_DATA_OP_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -30,23 +30,23 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRegisterOp: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRegisterData: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRunOp: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -55,23 +55,23 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRegisterOp: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRegisterData: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRunOp: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -80,23 +80,23 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRegisterOp: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRegisterData: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRunOp: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -105,23 +105,23 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRegisterOp: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRegisterData: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRunOp: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -130,23 +130,23 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRegisterOp: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRegisterData: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRunOp: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -182,27 +182,27 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRegisterOp: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRegisterData: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRunOp: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -282,4 +282,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h index 2e04c31ca..06cc34c2b 100644 --- a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_DATA_OP_METHODS_H_ -#define LABSTOR_HERMES_DATA_OP_METHODS_H_ +#ifndef HRUN_HERMES_DATA_OP_METHODS_H_ +#define HRUN_HERMES_DATA_OP_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -8,4 +8,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kRunOp = kLast + 2; }; -#endif // LABSTOR_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_DATA_OP_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h index c159f8a61..65f4f7651 100644 --- a/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h +++ b/tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_tasks.h @@ -2,13 +2,13 @@ // Created by lukemartinlogan on 8/11/23. // -#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ -#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ +#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ +#define HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" #include "hermes/hermes_types.h" #include "hermes_data_op/hermes_data_op.h" @@ -17,9 +17,9 @@ namespace hermes::data_op { #include "hermes_data_op_methods.h" -#include "labstor/labstor_namespace.h" -using labstor::proc_queue::TypedPushTask; -using labstor::proc_queue::PushTask; +#include "hrun/hrun_namespace.h" +using hrun::proc_queue::TypedPushTask; +using hrun::proc_queue::PushTask; /** The bucket to use for op data */ struct OpBucketName { @@ -66,7 +66,7 @@ struct OpGraph { /** * A task to create hermes_data_op * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { IN TaskStateId bkt_mdm_; IN TaskStateId blob_mdm_; @@ -100,7 +100,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy hermes_data_op */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -308,4 +308,4 @@ struct RunOpTask : public Task, TaskFlags { } // namespace hermes::data_op -#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ +#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_data_op_hermes_data_op_TASKS_H_ diff --git a/tasks/hermes_data_op/src/hermes_data_op.cc b/tasks/hermes_data_op/src/hermes_data_op.cc index a411394c6..9aebb1052 100644 --- a/tasks/hermes_data_op/src/hermes_data_op.cc +++ b/tasks/hermes_data_op/src/hermes_data_op.cc @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "hermes_data_op/hermes_data_op.h" #include "hermes_bucket_mdm/hermes_bucket_mdm.h" @@ -38,7 +38,7 @@ class Server : public TaskLib { op_id_map_["min"] = 0; op_id_map_["max"] = 1; run_task_ = client_.AsyncRunOp(task->task_node_ + 1); - op_graphs_.resize(LABSTOR_QM_RUNTIME->max_lanes_); + op_graphs_.resize(HRUN_QM_RUNTIME->max_lanes_); task->SetModuleComplete(); } @@ -79,13 +79,13 @@ class Server : public TaskLib { bkt_name.bkt_id_task_->Wait(task); bkt_name.bkt_id_ = bkt_name.bkt_id_task_->tag_id_; op_data_map_.emplace(bkt_name.bkt_id_, OpPendingData()); - LABSTOR_CLIENT->DelTask(bkt_name.bkt_id_task_); + HRUN_CLIENT->DelTask(bkt_name.bkt_id_task_); } // Spawn bucket ID task for the output op.var_name_.bkt_id_task_->Wait(task); op.var_name_.bkt_id_ = op.var_name_.bkt_id_task_->tag_id_; op_data_map_.emplace(op.var_name_.bkt_id_, OpPendingData()); - LABSTOR_CLIENT->DelTask(op.var_name_.bkt_id_task_); + HRUN_CLIENT->DelTask(op.var_name_.bkt_id_task_); } // Get number of operations that depend on each data object @@ -161,7 +161,7 @@ class Server : public TaskLib { std::vector in_tasks; for (OpData &data : op_data) { // Get the input data - LPointer data_ptr = LABSTOR_CLIENT->AllocateBuffer(data.size_); + LPointer data_ptr = HRUN_CLIENT->AllocateBuffer(data.size_); LPointer in_task = blob_mdm_.AsyncGetBlob(task->task_node_ + 1, data.bkt_id_, @@ -180,7 +180,7 @@ class Server : public TaskLib { in_task->Wait(task); // Calaculate the minimum - LPointer min_ptr = LABSTOR_CLIENT->AllocateBuffer(sizeof(float)); + LPointer min_ptr = HRUN_CLIENT->AllocateBuffer(sizeof(float)); float &min = *((float*)min_ptr.ptr_); min = std::numeric_limits::max(); for (size_t i = 0; i < in_task->data_size_; i += sizeof(float)) { @@ -207,4 +207,4 @@ class Server : public TaskLib { } // namespace hermes::data_op -LABSTOR_TASK_CC(hermes::data_op::Server, "hermes_data_op"); +HRUN_TASK_CC(hermes::data_op::Server, "hermes_data_op"); diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h index 11f6c42e9..0420c3629 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_hermes_mdm_H_ -#define LABSTOR_hermes_mdm_H_ +#ifndef HRUN_hermes_mdm_H_ +#define HRUN_hermes_mdm_H_ #include "hermes_mdm_tasks.h" @@ -26,7 +26,7 @@ class Client : public TaskLibClient { std::vector queue_info = { {1, 1, 1, 0}, }; - id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + id_ = HRUN_ADMIN->CreateTaskStateRoot( domain_id, state_name, id_, queue_info); queue_id_ = QueueId(id_); } @@ -34,10 +34,10 @@ class Client : public TaskLibClient { /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_hermes_mdm_H_ +#endif // HRUN_hermes_mdm_H_ diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h index 411a6065a..17c627803 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_HERMES_MDM_LIB_EXEC_H_ -#define LABSTOR_HERMES_MDM_LIB_EXEC_H_ +#ifndef HRUN_HERMES_MDM_LIB_EXEC_H_ +#define HRUN_HERMES_MDM_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -18,11 +18,11 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -31,11 +31,11 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -44,11 +44,11 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -57,11 +57,11 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -70,11 +70,11 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -98,12 +98,12 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -150,4 +150,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_HERMES_MDM_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h index 8a4383c14..2dc9001ad 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h @@ -1,8 +1,8 @@ -#ifndef LABSTOR_HERMES_MDM_METHODS_H_ -#define LABSTOR_HERMES_MDM_METHODS_H_ +#ifndef HRUN_HERMES_MDM_METHODS_H_ +#define HRUN_HERMES_MDM_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { }; -#endif // LABSTOR_HERMES_MDM_METHODS_H_ \ No newline at end of file +#endif // HRUN_HERMES_MDM_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h index 3a78e6de6..1a590cd07 100644 --- a/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h +++ b/tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_tasks.h @@ -2,13 +2,13 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ -#define LABSTOR_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ +#ifndef HRUN_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ +#define HRUN_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "hermes/hermes_types.h" #include "bdev/bdev.h" #include "proc_queue/proc_queue.h" @@ -16,12 +16,12 @@ namespace hermes::mdm { #include "hermes_mdm_methods.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" /** * A task to create hermes_mdm * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { IN hipc::ShmArchive server_config_path_; @@ -63,7 +63,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy hermes_mdm */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -86,4 +86,4 @@ struct DestructTask : public DestroyTaskStateTask { } // namespace hermes::mdm -#endif // LABSTOR_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ +#endif // HRUN_TASKS_HERMES_MDM_INCLUDE_HERMES_MDM_HERMES_MDM_TASKS_H_ diff --git a/tasks/hermes_mdm/src/hermes_mdm.cc b/tasks/hermes_mdm/src/hermes_mdm.cc index 6a870cf8b..60351b68c 100644 --- a/tasks/hermes_mdm/src/hermes_mdm.cc +++ b/tasks/hermes_mdm/src/hermes_mdm.cc @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "hermes/config_server.h" #include "hermes_mdm/hermes_mdm.h" #include "hermes_blob_mdm/hermes_blob_mdm.h" @@ -29,7 +29,7 @@ class Server : public TaskLib { HILOG(kDebug, "ConstructTaskPhase::kLoadConfig") std::string config_path = task->server_config_path_->str(); HERMES_CONF->LoadServerConfig(config_path); - node_id_ = LABSTOR_CLIENT->node_id_; + node_id_ = HRUN_CLIENT->node_id_; task->SetModuleComplete(); } @@ -41,6 +41,6 @@ class Server : public TaskLib { #include "hermes_mdm/hermes_mdm_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(hermes::mdm::Server, "hermes_mdm"); +HRUN_TASK_CC(hermes::mdm::Server, "hermes_mdm"); diff --git a/tasks/posix_bdev/include/posix_bdev/posix_bdev.h b/tasks/posix_bdev/include/posix_bdev/posix_bdev.h index 849d8f0c3..d7dc2bca2 100644 --- a/tasks/posix_bdev/include/posix_bdev/posix_bdev.h +++ b/tasks/posix_bdev/include/posix_bdev/posix_bdev.h @@ -2,19 +2,19 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_posix_bdev_H_ -#define LABSTOR_posix_bdev_H_ +#ifndef HRUN_posix_bdev_H_ +#define HRUN_posix_bdev_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "hermes/hermes_types.h" #include "bdev/bdev.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" namespace hermes::posix_bdev { #include "bdev/bdev_namespace.h" -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_posix_bdev_H_ +#endif // HRUN_posix_bdev_H_ diff --git a/tasks/posix_bdev/src/posix_bdev.cc b/tasks/posix_bdev/src/posix_bdev.cc index bf0bba084..6e2d7e51e 100644 --- a/tasks/posix_bdev/src/posix_bdev.cc +++ b/tasks/posix_bdev/src/posix_bdev.cc @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "posix_bdev/posix_bdev.h" #include "hermes/slab_allocator.h" @@ -85,6 +85,6 @@ class Server : public TaskLib { #include "bdev/bdev_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(hermes::posix_bdev::Server, "posix_bdev"); +HRUN_TASK_CC(hermes::posix_bdev::Server, "posix_bdev"); diff --git a/tasks/ram_bdev/include/ram_bdev/ram_bdev.h b/tasks/ram_bdev/include/ram_bdev/ram_bdev.h index b6e004101..602b07af3 100644 --- a/tasks/ram_bdev/include/ram_bdev/ram_bdev.h +++ b/tasks/ram_bdev/include/ram_bdev/ram_bdev.h @@ -2,20 +2,20 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_ram_bdev_H_ -#define LABSTOR_ram_bdev_H_ +#ifndef HRUN_ram_bdev_H_ +#define HRUN_ram_bdev_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "hermes/hermes_types.h" #include "bdev/bdev.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" namespace hermes::ram_bdev { #include "bdev/bdev_namespace.h" -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_ram_bdev_H_ +#endif // HRUN_ram_bdev_H_ diff --git a/tasks/ram_bdev/src/ram_bdev.cc b/tasks/ram_bdev/src/ram_bdev.cc index 638b10633..f993b335c 100644 --- a/tasks/ram_bdev/src/ram_bdev.cc +++ b/tasks/ram_bdev/src/ram_bdev.cc @@ -2,8 +2,8 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "ram_bdev/ram_bdev.h" #include "hermes/slab_allocator.h" @@ -64,6 +64,6 @@ class Server : public TaskLib { #include "bdev/bdev_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(hermes::ram_bdev::Server, "ram_bdev"); +HRUN_TASK_CC(hermes::ram_bdev::Server, "ram_bdev"); diff --git a/tasks_required/CMakeLists.txt b/tasks_required/CMakeLists.txt index 0ab7c475e..cb3e75dd7 100644 --- a/tasks_required/CMakeLists.txt +++ b/tasks_required/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(TASK_NAME) add_subdirectory(small_message) -add_subdirectory(labstor_admin) +add_subdirectory(hrun_admin) add_subdirectory(remote_queue) add_subdirectory(worch_proc_round_robin) add_subdirectory(worch_queue_round_robin) diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h index 1c1e71fd7..a59a86814 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_TASK_NAME_H_ -#define LABSTOR_TASK_NAME_H_ +#ifndef HRUN_TASK_NAME_H_ +#define HRUN_TASK_NAME_H_ #include "TASK_NAME_tasks.h" -namespace labstor::TASK_NAME { +namespace hrun::TASK_NAME { /** Create TASK_NAME requests */ class Client : public TaskLibClient { @@ -25,16 +25,16 @@ class Client : public TaskLibClient { const DomainId &domain_id, const std::string &state_name) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info); } - LABSTOR_TASK_NODE_ROOT(AsyncCreate) + HRUN_TASK_NODE_ROOT(AsyncCreate) template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -43,13 +43,13 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /** Call a custom method */ @@ -57,17 +57,17 @@ class Client : public TaskLibClient { void AsyncCustomConstruct(CustomTask *task, const TaskNode &task_node, const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id_); } HSHM_ALWAYS_INLINE void CustomRoot(const DomainId &domain_id) { - LPointer> task = AsyncCustomRoot(domain_id); + LPointer> task = AsyncCustomRoot(domain_id); task.ptr_->Wait(); } - LABSTOR_TASK_NODE_PUSH_ROOT(Custom); + HRUN_TASK_NODE_PUSH_ROOT(Custom); }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_TASK_NAME_H_ +#endif // HRUN_TASK_NAME_H_ diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h index f315a229d..0f75fae53 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_TASK_NAME_LIB_EXEC_H_ -#define LABSTOR_TASK_NAME_LIB_EXEC_H_ +#ifndef HRUN_TASK_NAME_LIB_EXEC_H_ +#define HRUN_TASK_NAME_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -22,15 +22,15 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kCustom: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -39,15 +39,15 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kCustom: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -56,15 +56,15 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kCustom: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -73,15 +73,15 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kCustom: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -90,15 +90,15 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kCustom: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -126,17 +126,17 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kCustom: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -194,4 +194,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_TASK_NAME_METHODS_H_ \ No newline at end of file +#endif // HRUN_TASK_NAME_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h index 79d7137bb..66e99a6cd 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h @@ -1,9 +1,9 @@ -#ifndef LABSTOR_TASK_NAME_METHODS_H_ -#define LABSTOR_TASK_NAME_METHODS_H_ +#ifndef HRUN_TASK_NAME_METHODS_H_ +#define HRUN_TASK_NAME_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kCustom = kLast + 0; }; -#endif // LABSTOR_TASK_NAME_METHODS_H_ \ No newline at end of file +#endif // HRUN_TASK_NAME_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h index 0c714be8a..3f528d97d 100644 --- a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h +++ b/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h @@ -2,26 +2,26 @@ // Created by lukemartinlogan on 8/11/23. // -#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ -#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ +#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ +#define HRUN_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" -namespace labstor::TASK_NAME { +namespace hrun::TASK_NAME { #include "TASK_NAME_methods.h" -#include "labstor/labstor_namespace.h" -using labstor::proc_queue::TypedPushTask; -using labstor::proc_queue::PushTask; +#include "hrun/hrun_namespace.h" +using hrun::proc_queue::TypedPushTask; +using hrun::proc_queue::PushTask; /** * A task to create TASK_NAME * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -48,7 +48,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy TASK_NAME */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -130,6 +130,6 @@ struct CustomTask : public Task, TaskFlags { } }; -} // namespace labstor::TASK_NAME +} // namespace hrun::TASK_NAME -#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ +#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_TASK_NAME_TASK_NAME_TASKS_H_ diff --git a/tasks_required/TASK_NAME/src/TASK_NAME.cc b/tasks_required/TASK_NAME/src/TASK_NAME.cc index 5912dba1f..59fa67256 100644 --- a/tasks_required/TASK_NAME/src/TASK_NAME.cc +++ b/tasks_required/TASK_NAME/src/TASK_NAME.cc @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "TASK_NAME/TASK_NAME.h" -namespace labstor::TASK_NAME { +namespace hrun::TASK_NAME { class Server : public TaskLib { public: @@ -28,6 +28,6 @@ class Server : public TaskLib { #include "TASK_NAME/TASK_NAME_lib_exec.h" }; -} // namespace labstor::TASK_NAME +} // namespace hrun::TASK_NAME -LABSTOR_TASK_CC(labstor::TASK_NAME::Server, "TASK_NAME"); +HRUN_TASK_CC(hrun::TASK_NAME::Server, "TASK_NAME"); diff --git a/tasks_required/labstor_admin/CMakeLists.txt b/tasks_required/hrun_admin/CMakeLists.txt similarity index 100% rename from tasks_required/labstor_admin/CMakeLists.txt rename to tasks_required/hrun_admin/CMakeLists.txt diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h similarity index 79% rename from tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h rename to tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h index 151880126..b6d5ca980 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin.h +++ b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h @@ -2,20 +2,20 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_TASKS_LABSTOR_ADMIN_LABSTOR_ADMIN_H_ -#define LABSTOR_TASKS_LABSTOR_ADMIN_LABSTOR_ADMIN_H_ +#ifndef HRUN_TASKS_HRUN_ADMIN_HRUN_ADMIN_H_ +#define HRUN_TASKS_HRUN_ADMIN_HRUN_ADMIN_H_ -#include "labstor_admin_tasks.h" +#include "hrun_admin_tasks.h" -namespace labstor::Admin { +namespace hrun::Admin { /** Create admin requests */ class Client : public TaskLibClient { public: /** Default constructor */ Client() { - id_ = TaskStateId(LABSTOR_QM_CLIENT->admin_queue_); - queue_id_ = LABSTOR_QM_CLIENT->admin_queue_; + id_ = TaskStateId(HRUN_QM_CLIENT->admin_queue_); + queue_id_ = HRUN_QM_CLIENT->admin_queue_; } /** Destructor */ @@ -27,7 +27,7 @@ class Client : public TaskLibClient { const TaskNode &task_node, const DomainId &domain_id, const std::string &lib_name) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, lib_name); } HSHM_ALWAYS_INLINE @@ -35,9 +35,9 @@ class Client : public TaskLibClient { const std::string &lib_name) { LPointer task = AsyncRegisterTaskLibRoot(domain_id, lib_name); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } - LABSTOR_TASK_NODE_ADMIN_ROOT(RegisterTaskLib) + HRUN_TASK_NODE_ADMIN_ROOT(RegisterTaskLib) /** Unregister a task */ HSHM_ALWAYS_INLINE @@ -45,7 +45,7 @@ class Client : public TaskLibClient { const TaskNode &task_node, const DomainId &domain_id, const std::string &lib_name) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, lib_name); } HSHM_ALWAYS_INLINE @@ -55,9 +55,9 @@ class Client : public TaskLibClient { LPointer task = AsyncDestroyTaskLibRoot(domain_id, lib_name); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } - LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskLib) + HRUN_TASK_NODE_ADMIN_ROOT(DestroyTaskLib) /** Spawn a task state */ template @@ -66,7 +66,7 @@ class Client : public TaskLibClient { const TaskNode &task_node, const DomainId &domain_id, Args&& ...args) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, std::forward(args)...); } template @@ -77,7 +77,7 @@ class Client : public TaskLibClient { domain_id, std::forward(args)...); task->Wait(); TaskStateId new_id = task->id_; - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); if (new_id.IsNull()) { HELOG(kWarning, "Failed to create task state"); } @@ -86,7 +86,7 @@ class Client : public TaskLibClient { template hipc::LPointer AsyncCreateTaskStateAlloc(const TaskNode &task_node, Args&& ...args) { - hipc::LPointer task = LABSTOR_CLIENT->AllocateTask(); + hipc::LPointer task = HRUN_CLIENT->AllocateTask(); AsyncCreateTaskStateConstruct(task.ptr_, task_node, std::forward(args)...); return task; } @@ -95,7 +95,7 @@ class Client : public TaskLibClient { Args&& ...args) { hipc::LPointer task = AsyncCreateTaskStateAlloc(task_node, std::forward(args)...); - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); return task; } @@ -110,7 +110,7 @@ class Client : public TaskLibClient { } template hipc::LPointer AsyncCreateTaskStateRoot(Args&& ...args) { - TaskNode task_node = LABSTOR_CLIENT->MakeTaskNodeId(); + TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId(); hipc::LPointer task = AsyncCreateTaskState(task_node, std::forward(args)...); return task; @@ -121,7 +121,7 @@ class Client : public TaskLibClient { const TaskNode &task_node, const DomainId &domain_id, const std::string &state_name) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, state_name); } TaskStateId GetOrCreateTaskStateIdRoot(const DomainId &domain_id, @@ -130,17 +130,17 @@ class Client : public TaskLibClient { AsyncGetOrCreateTaskStateIdRoot(domain_id, state_name); task->Wait(); TaskStateId new_id = task->id_; - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); return new_id; } - LABSTOR_TASK_NODE_ADMIN_ROOT(GetOrCreateTaskStateId) + HRUN_TASK_NODE_ADMIN_ROOT(GetOrCreateTaskStateId) /** Get the ID of a task state */ void AsyncGetTaskStateIdConstruct(GetTaskStateIdTask *task, const TaskNode &task_node, const DomainId &domain_id, const std::string &state_name) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, state_name); } TaskStateId GetTaskStateIdRoot(const DomainId &domain_id, @@ -149,10 +149,10 @@ class Client : public TaskLibClient { AsyncGetTaskStateIdRoot(domain_id, state_name); task->Wait(); TaskStateId new_id = task->id_; - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); return new_id; } - LABSTOR_TASK_NODE_ADMIN_ROOT(GetTaskStateId) + HRUN_TASK_NODE_ADMIN_ROOT(GetTaskStateId) /** Terminate a task state */ HSHM_ALWAYS_INLINE @@ -160,7 +160,7 @@ class Client : public TaskLibClient { const TaskNode &task_node, const DomainId &domain_id, const TaskStateId &id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id); } HSHM_ALWAYS_INLINE @@ -169,25 +169,25 @@ class Client : public TaskLibClient { LPointer task = AsyncDestroyTaskStateRoot(domain_id, id); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } - LABSTOR_TASK_NODE_ADMIN_ROOT(DestroyTaskState) + HRUN_TASK_NODE_ADMIN_ROOT(DestroyTaskState) /** Terminate the runtime */ void AsyncStopRuntimeConstruct(StopRuntimeTask *task, const TaskNode &task_node, const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id); } - LABSTOR_TASK_NODE_ADMIN_ROOT(StopRuntime); + HRUN_TASK_NODE_ADMIN_ROOT(StopRuntime); /** Set work orchestrator queue policy */ void AsyncSetWorkOrchQueuePolicyConstruct(SetWorkOrchQueuePolicyTask *task, const TaskNode &task_node, const DomainId &domain_id, const TaskStateId &policy) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, policy); } void SetWorkOrchQueuePolicyRoot(const DomainId &domain_id, @@ -195,16 +195,16 @@ class Client : public TaskLibClient { LPointer task = AsyncSetWorkOrchQueuePolicyRoot(domain_id, policy); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } - LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchQueuePolicy); + HRUN_TASK_NODE_ADMIN_ROOT(SetWorkOrchQueuePolicy); /** Set work orchestrator process policy */ void AsyncSetWorkOrchProcPolicyConstruct(SetWorkOrchProcPolicyTask *task, const TaskNode &task_node, const DomainId &domain_id, const TaskStateId &policy) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, policy); } void SetWorkOrchProcPolicyRoot(const DomainId &domain_id, @@ -212,29 +212,29 @@ class Client : public TaskLibClient { LPointer task = AsyncSetWorkOrchProcPolicyRoot(domain_id, policy); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } - LABSTOR_TASK_NODE_ADMIN_ROOT(SetWorkOrchProcPolicy); + HRUN_TASK_NODE_ADMIN_ROOT(SetWorkOrchProcPolicy); /** Flush the runtime */ void AsyncFlushConstruct(FlushTask *task, const TaskNode &task_node, const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id); } void FlushRoot(const DomainId &domain_id) { LPointer task = AsyncFlushRoot(domain_id); task->Wait(); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } - LABSTOR_TASK_NODE_ADMIN_ROOT(Flush); + HRUN_TASK_NODE_ADMIN_ROOT(Flush); }; -} // namespace labstor::Admin +} // namespace hrun::Admin -#define LABSTOR_ADMIN \ - hshm::EasySingleton::GetInstance() +#define HRUN_ADMIN \ + hshm::EasySingleton::GetInstance() -#endif // LABSTOR_TASKS_LABSTOR_ADMIN_LABSTOR_ADMIN_H_ +#endif // HRUN_TASKS_HRUN_ADMIN_HRUN_ADMIN_H_ diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h similarity index 67% rename from tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h rename to tasks_required/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h index 9ac96d764..22f34312c 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_lib_exec.h +++ b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_LABSTOR_ADMIN_LIB_EXEC_H_ -#define LABSTOR_LABSTOR_ADMIN_LIB_EXEC_H_ +#ifndef HRUN_HRUN_ADMIN_LIB_EXEC_H_ +#define HRUN_HRUN_ADMIN_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -50,43 +50,43 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kCreateTaskState: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestroyTaskState: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kRegisterTaskLib: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestroyTaskLib: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetOrCreateTaskStateId: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kGetTaskStateId: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kStopRuntime: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kSetWorkOrchQueuePolicy: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kSetWorkOrchProcPolicy: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kFlush: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -95,43 +95,43 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kCreateTaskState: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestroyTaskState: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kRegisterTaskLib: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestroyTaskLib: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetOrCreateTaskStateId: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kGetTaskStateId: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kStopRuntime: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kSetWorkOrchQueuePolicy: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kSetWorkOrchProcPolicy: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kFlush: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -140,43 +140,43 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kCreateTaskState: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestroyTaskState: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kRegisterTaskLib: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestroyTaskLib: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetOrCreateTaskStateId: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kGetTaskStateId: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kStopRuntime: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kSetWorkOrchQueuePolicy: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kSetWorkOrchProcPolicy: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kFlush: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -185,43 +185,43 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kCreateTaskState: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestroyTaskState: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kRegisterTaskLib: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestroyTaskLib: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetOrCreateTaskStateId: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kGetTaskStateId: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kStopRuntime: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kSetWorkOrchQueuePolicy: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kSetWorkOrchProcPolicy: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kFlush: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -230,43 +230,43 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kCreateTaskState: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestroyTaskState: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kRegisterTaskLib: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestroyTaskLib: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetOrCreateTaskStateId: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kGetTaskStateId: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kStopRuntime: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kSetWorkOrchQueuePolicy: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kSetWorkOrchProcPolicy: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kFlush: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -322,52 +322,52 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kCreateTaskState: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyTaskState: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kRegisterTaskLib: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestroyTaskLib: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetOrCreateTaskStateId: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kGetTaskStateId: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kStopRuntime: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetWorkOrchQueuePolicy: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSetWorkOrchProcPolicy: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kFlush: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -502,4 +502,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_LABSTOR_ADMIN_METHODS_H_ \ No newline at end of file +#endif // HRUN_HRUN_ADMIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h similarity index 81% rename from tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h rename to tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h index 8921a31c3..882809459 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.h +++ b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_LABSTOR_ADMIN_METHODS_H_ -#define LABSTOR_LABSTOR_ADMIN_METHODS_H_ +#ifndef HRUN_HRUN_ADMIN_METHODS_H_ +#define HRUN_HRUN_ADMIN_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -15,4 +15,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kFlush = kLast + 9; }; -#endif // LABSTOR_LABSTOR_ADMIN_METHODS_H_ \ No newline at end of file +#endif // HRUN_HRUN_ADMIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml similarity index 100% rename from tasks_required/labstor_admin/include/labstor_admin/labstor_admin_methods.yaml rename to tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml diff --git a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h similarity index 92% rename from tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h rename to tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h index a64b112a3..f8a962c1c 100644 --- a/tasks_required/labstor_admin/include/labstor_admin/labstor_admin_tasks.h +++ b/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h @@ -2,17 +2,17 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_TASKS_LABSTOR_ADMIN_INCLUDE_LABSTOR_ADMIN_LABSTOR_ADMIN_TASKS_H_ -#define LABSTOR_TASKS_LABSTOR_ADMIN_INCLUDE_LABSTOR_ADMIN_LABSTOR_ADMIN_TASKS_H_ +#ifndef HRUN_TASKS_HRUN_ADMIN_INCLUDE_HRUN_ADMIN_HRUN_ADMIN_TASKS_H_ +#define HRUN_TASKS_HRUN_ADMIN_INCLUDE_HRUN_ADMIN_HRUN_ADMIN_TASKS_H_ -#include "labstor/work_orchestrator/scheduler.h" -#include "labstor/api/labstor_client.h" -#include "labstor/queue_manager/queue_manager_client.h" -#include "labstor/labstor_namespace.h" +#include "hrun/work_orchestrator/scheduler.h" +#include "hrun/api/hrun_client.h" +#include "hrun/queue_manager/queue_manager_client.h" +#include "hrun/hrun_namespace.h" -namespace labstor::Admin { +namespace hrun::Admin { -#include "labstor_admin_methods.h" +#include "hrun_admin_methods.h" /** A template to register or destroy a task library */ template @@ -34,7 +34,7 @@ struct RegisterTaskLibTaskTempl : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; if constexpr(method == 0) { method_ = Method::kRegisterTaskLib; @@ -108,7 +108,7 @@ struct GetOrCreateTaskStateIdTask : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; method_ = Method::kGetOrCreateTaskStateId; task_flags_.SetBits(0); domain_id_ = domain_id; @@ -165,7 +165,7 @@ struct CreateTaskStateTask : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; method_ = Method::kCreateTaskState; task_flags_.SetBits(TASK_COROUTINE); domain_id_ = domain_id; @@ -246,7 +246,7 @@ struct GetTaskStateIdTask : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; method_ = Method::kGetTaskStateId; task_flags_.SetBits(0); domain_id_ = domain_id; @@ -297,7 +297,7 @@ struct DestroyTaskStateTask : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; method_ = Method::kDestroyTaskState; task_flags_.SetBits(0); domain_id_ = domain_id; @@ -339,7 +339,7 @@ struct StopRuntimeTask : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; method_ = Method::kStopRuntime; task_flags_.SetBits(TASK_FIRE_AND_FORGET); domain_id_ = domain_id; @@ -382,7 +382,7 @@ struct SetWorkOrchestratorPolicyTask : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; if constexpr(method == 0) { method_ = Method::kSetWorkOrchQueuePolicy; } else { @@ -430,7 +430,7 @@ struct FlushTask : public Task, TaskFlags { task_node_ = task_node; lane_hash_ = 0; prio_ = TaskPrio::kAdmin; - task_state_ = LABSTOR_QM_CLIENT->admin_task_state_; + task_state_ = HRUN_QM_CLIENT->admin_task_state_; method_ = Method::kFlush; task_flags_.SetBits(TASK_FLUSH | TASK_COROUTINE); domain_id_ = domain_id; @@ -470,6 +470,6 @@ struct FlushTask : public Task, TaskFlags { }; -} // namespace labstor::Admin +} // namespace hrun::Admin -#endif // LABSTOR_TASKS_LABSTOR_ADMIN_INCLUDE_LABSTOR_ADMIN_LABSTOR_ADMIN_TASKS_H_ +#endif // HRUN_TASKS_HRUN_ADMIN_INCLUDE_HRUN_ADMIN_HRUN_ADMIN_TASKS_H_ diff --git a/tasks_required/labstor_admin/src/CMakeLists.txt b/tasks_required/hrun_admin/src/CMakeLists.txt similarity index 85% rename from tasks_required/labstor_admin/src/CMakeLists.txt rename to tasks_required/hrun_admin/src/CMakeLists.txt index ad727d992..39b07560c 100644 --- a/tasks_required/labstor_admin/src/CMakeLists.txt +++ b/tasks_required/hrun_admin/src/CMakeLists.txt @@ -1,17 +1,17 @@ #------------------------------------------------------------------------------ # Build Labstor Admin Task Library #------------------------------------------------------------------------------ -add_library(labstor_admin SHARED - ${CMAKE_CURRENT_SOURCE_DIR}/labstor_admin.cc) -add_dependencies(labstor_admin ${Hermes_RUNTIME_DEPS}) -target_link_libraries(labstor_admin ${Hermes_RUNTIME_LIBRARIES}) +add_library(hrun_admin SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/hrun_admin.cc) +add_dependencies(hrun_admin ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hrun_admin ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ # Install Labstor Admin Task Library #------------------------------------------------------------------------------ install( TARGETS - labstor_admin + hrun_admin EXPORT ${HERMES_EXPORTED_TARGETS} LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} @@ -35,7 +35,7 @@ install( # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- set(HERMES_EXPORTED_LIBS - labstor_admin + hrun_admin ${HERMES_EXPORTED_LIBS}) if(NOT HERMES_EXTERNALLY_CONFIGURED) EXPORT ( @@ -50,5 +50,5 @@ endif() # Coverage #------------------------------------------------------------------------------ if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(labstor_admin) + set_coverage_flags(hrun_admin) endif() diff --git a/tasks_required/labstor_admin/src/labstor_admin.cc b/tasks_required/hrun_admin/src/hrun_admin.cc similarity index 68% rename from tasks_required/labstor_admin/src/labstor_admin.cc rename to tasks_required/hrun_admin/src/hrun_admin.cc index bd91dac4a..bc270d13e 100644 --- a/tasks_required/labstor_admin/src/labstor_admin.cc +++ b/tasks_required/hrun_admin/src/hrun_admin.cc @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" -#include "labstor/work_orchestrator/scheduler.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" +#include "hrun/work_orchestrator/scheduler.h" -namespace labstor::Admin { +namespace hrun::Admin { class Server : public TaskLib { public: @@ -18,19 +18,19 @@ class Server : public TaskLib { void RegisterTaskLib(RegisterTaskLibTask *task, RunContext &rctx) { std::string lib_name = task->lib_name_->str(); - LABSTOR_TASK_REGISTRY->RegisterTaskLib(lib_name); + HRUN_TASK_REGISTRY->RegisterTaskLib(lib_name); task->SetModuleComplete(); } void DestroyTaskLib(DestroyTaskLibTask *task, RunContext &rctx) { std::string lib_name = task->lib_name_->str(); - LABSTOR_TASK_REGISTRY->DestroyTaskLib(lib_name); + HRUN_TASK_REGISTRY->DestroyTaskLib(lib_name); task->SetModuleComplete(); } void GetOrCreateTaskStateId(GetOrCreateTaskStateIdTask *task, RunContext &rctx) { std::string state_name = task->state_name_->str(); - task->id_ = LABSTOR_TASK_REGISTRY->GetOrCreateTaskStateId(state_name); + task->id_ = HRUN_TASK_REGISTRY->GetOrCreateTaskStateId(state_name); task->SetModuleComplete(); } @@ -38,7 +38,7 @@ class Server : public TaskLib { std::string lib_name = task->lib_name_->str(); std::string state_name = task->state_name_->str(); // Check local registry for task state - TaskState *task_state = LABSTOR_TASK_REGISTRY->GetTaskState(state_name, task->id_); + TaskState *task_state = HRUN_TASK_REGISTRY->GetTaskState(state_name, task->id_); if (task_state) { task->id_ = task_state->id_; task->SetModuleComplete(); @@ -48,34 +48,34 @@ class Server : public TaskLib { if (task->id_.IsNull()) { DomainId domain = DomainId::GetNode(1); LPointer get_id = - LABSTOR_ADMIN->AsyncGetOrCreateTaskStateId(task->task_node_ + 1, domain, state_name); + HRUN_ADMIN->AsyncGetOrCreateTaskStateId(task->task_node_ + 1, domain, state_name); get_id->Wait(task); task->id_ = get_id->id_; - LABSTOR_CLIENT->DelTask(get_id); + HRUN_CLIENT->DelTask(get_id); } // Create the task state HILOG(kInfo, "(node {}) Creating task state {} with id {} (task_node={})", - LABSTOR_CLIENT->node_id_, state_name, task->id_, task->task_node_); + HRUN_CLIENT->node_id_, state_name, task->id_, task->task_node_); if (task->id_.IsNull()) { HELOG(kError, "(node {}) The task state {} with id {} is NULL.", - LABSTOR_CLIENT->node_id_, state_name, task->id_); + HRUN_CLIENT->node_id_, state_name, task->id_); task->SetModuleComplete(); return; } // Verify the state doesn't exist - if (LABSTOR_TASK_REGISTRY->TaskStateExists(task->id_)) { + if (HRUN_TASK_REGISTRY->TaskStateExists(task->id_)) { HILOG(kInfo, "(node {}) The task state {} with id {} exists", - LABSTOR_CLIENT->node_id_, state_name, task->id_); + HRUN_CLIENT->node_id_, state_name, task->id_); task->SetModuleComplete(); return; } // Create the task queue for the state QueueId qid(task->id_); - MultiQueue *queue = LABSTOR_QM_RUNTIME->CreateQueue( + MultiQueue *queue = HRUN_QM_RUNTIME->CreateQueue( qid, task->queue_info_->vec()); // Run the task state's submethod task->method_ = Method::kConstruct; - bool ret = LABSTOR_TASK_REGISTRY->CreateTaskState( + bool ret = HRUN_TASK_REGISTRY->CreateTaskState( lib_name.c_str(), state_name.c_str(), task->id_, @@ -83,24 +83,24 @@ class Server : public TaskLib { queue->flags_.SetBits(QUEUE_READY); task->SetModuleComplete(); HILOG(kInfo, "(node {}) Allocated task state {} with id {}", - LABSTOR_CLIENT->node_id_, state_name, task->task_state_); + HRUN_CLIENT->node_id_, state_name, task->task_state_); } void GetTaskStateId(GetTaskStateIdTask *task, RunContext &rctx) { std::string state_name = task->state_name_->str(); - task->id_ = LABSTOR_TASK_REGISTRY->GetTaskStateId(state_name); + task->id_ = HRUN_TASK_REGISTRY->GetTaskStateId(state_name); task->SetModuleComplete(); } void DestroyTaskState(DestroyTaskStateTask *task, RunContext &rctx) { - LABSTOR_TASK_REGISTRY->DestroyTaskState(task->id_); + HRUN_TASK_REGISTRY->DestroyTaskState(task->id_); task->SetModuleComplete(); } void StopRuntime(StopRuntimeTask *task, RunContext &rctx) { HILOG(kInfo, "Stopping (server mode)"); - LABSTOR_WORK_ORCHESTRATOR->FinalizeRuntime(); - LABSTOR_THALLIUM->StopThisDaemon(); + HRUN_WORK_ORCHESTRATOR->FinalizeRuntime(); + HRUN_THALLIUM->StopThisDaemon(); task->SetModuleComplete(); } @@ -111,10 +111,10 @@ class Server : public TaskLib { if (queue_sched_ && !queue_sched_->IsComplete()) { return; } - auto queue_sched = LABSTOR_CLIENT->NewTask( + auto queue_sched = HRUN_CLIENT->NewTask( task->task_node_, DomainId::GetLocal(), task->policy_id_); queue_sched_ = queue_sched.ptr_; - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(0, 0, queue_sched.shm_); task->SetModuleComplete(); } @@ -126,22 +126,22 @@ class Server : public TaskLib { if (proc_sched_ && !proc_sched_->IsComplete()) { return; } - auto proc_sched = LABSTOR_CLIENT->NewTask( + auto proc_sched = HRUN_CLIENT->NewTask( task->task_node_, DomainId::GetLocal(), task->policy_id_); proc_sched_ = proc_sched.ptr_; - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(0, 0, proc_sched.shm_); task->SetModuleComplete(); } void Flush(FlushTask *task, RunContext &rctx) { HILOG(kDebug, "Beginning to flush runtime"); - for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { + for (Worker &worker : HRUN_WORK_ORCHESTRATOR->workers_) { worker.flush_.flushing_ = true; } while (true) { int count = 0; - for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { + for (Worker &worker : HRUN_WORK_ORCHESTRATOR->workers_) { if (worker.flush_.flushing_) { count += 1; break; @@ -158,9 +158,9 @@ class Server : public TaskLib { } public: -#include "labstor_admin/labstor_admin_lib_exec.h" +#include "hrun_admin/hrun_admin_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(labstor::Admin::Server, "labstor_admin"); +HRUN_TASK_CC(hrun::Admin::Server, "hrun_admin"); diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue.h b/tasks_required/proc_queue/include/proc_queue/proc_queue.h index 170307a68..1c8635360 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue.h @@ -2,20 +2,20 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_proc_queue_H_ -#define LABSTOR_proc_queue_H_ +#ifndef HRUN_proc_queue_H_ +#define HRUN_proc_queue_H_ #include "proc_queue_tasks.h" -namespace labstor::proc_queue { +namespace hrun::proc_queue { /** Create proc_queue requests */ class Client : public TaskLibClient { public: /** Default constructor */ Client() { - id_ = TaskStateId(LABSTOR_QM_CLIENT->process_queue_); - queue_id_ = LABSTOR_QM_CLIENT->process_queue_; + id_ = TaskStateId(HRUN_QM_CLIENT->process_queue_); + queue_id_ = HRUN_QM_CLIENT->process_queue_; } /** Destructor */ @@ -27,16 +27,16 @@ class Client : public TaskLibClient { const DomainId &domain_id, const std::string &state_name) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info); } - LABSTOR_TASK_NODE_ROOT(AsyncCreate); + HRUN_TASK_NODE_ROOT(AsyncCreate); template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -45,44 +45,44 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /** Call a custom method */ template HSHM_ALWAYS_INLINE - void AsyncPushConstruct(labpq::TypedPushTask *task, + void AsyncPushConstruct(hrunpq::TypedPushTask *task, const TaskNode &task_node, const DomainId &domain_id, const hipc::LPointer &subtask) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id_, subtask); } template HSHM_ALWAYS_INLINE - LPointer> + LPointer> AsyncPush(const TaskNode &task_node, const DomainId &domain_id, const hipc::LPointer &subtask) { - LPointer> push_task = - LABSTOR_CLIENT->AllocateTask>(); + LPointer> push_task = + HRUN_CLIENT->AllocateTask>(); AsyncPushConstruct(push_task.ptr_, task_node, domain_id, subtask); - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(push_task->prio_, push_task->lane_hash_, push_task.shm_); return push_task; } - LABSTOR_TASK_NODE_ROOT(AsyncPush); + HRUN_TASK_NODE_ROOT(AsyncPush); }; -} // namespace labstor +} // namespace hrun -#define LABSTOR_PROCESS_QUEUE \ - hshm::EasySingleton::GetInstance() +#define HRUN_PROCESS_QUEUE \ + hshm::EasySingleton::GetInstance() -#endif // LABSTOR_proc_queue_H_ +#endif // HRUN_proc_queue_H_ diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h index a3357d770..7033c12e5 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_PROC_QUEUE_LIB_EXEC_H_ -#define LABSTOR_PROC_QUEUE_LIB_EXEC_H_ +#ifndef HRUN_PROC_QUEUE_LIB_EXEC_H_ +#define HRUN_PROC_QUEUE_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -22,15 +22,15 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kPush: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -39,15 +39,15 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kPush: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -56,15 +56,15 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kPush: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -73,15 +73,15 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kPush: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -90,15 +90,15 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kPush: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -126,17 +126,17 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPush: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -194,4 +194,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_PROC_QUEUE_METHODS_H_ \ No newline at end of file +#endif // HRUN_PROC_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h index 095b84bf1..f18c5072e 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h @@ -1,9 +1,9 @@ -#ifndef LABSTOR_PROC_QUEUE_METHODS_H_ -#define LABSTOR_PROC_QUEUE_METHODS_H_ +#ifndef HRUN_PROC_QUEUE_METHODS_H_ +#define HRUN_PROC_QUEUE_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kPush = kLast + 0; }; -#endif // LABSTOR_PROC_QUEUE_METHODS_H_ \ No newline at end of file +#endif // HRUN_PROC_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index 08adb8457..339baf22a 100644 --- a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -2,23 +2,23 @@ // Created by lukemartinlogan on 8/11/23. // -#ifndef LABSTOR_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ -#define LABSTOR_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ +#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ +#define HRUN_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" -namespace labstor::proc_queue { +namespace hrun::proc_queue { -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" #include "proc_queue_methods.h" /** * A task to create proc_queue * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -45,7 +45,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy proc_queue */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -110,9 +110,9 @@ struct TypedPushTask : public Task, TaskFlags { /** Destructor */ ~TypedPushTask() { if (!IsFireAndForget()) { - LABSTOR_CLIENT->DelTask(sub_cli_); + HRUN_CLIENT->DelTask(sub_cli_); } else { - LABSTOR_CLIENT->DelTask(sub_run_); + HRUN_CLIENT->DelTask(sub_run_); } } @@ -131,10 +131,10 @@ struct TypedPushTask : public Task, TaskFlags { } }; -using PushTask = labstor::proc_queue::TypedPushTask; +using PushTask = hrun::proc_queue::TypedPushTask; -} // namespace labstor::proc_queue +} // namespace hrun::proc_queue -namespace labpq = labstor::proc_queue; +namespace hrunpq = hrun::proc_queue; -#endif // LABSTOR_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ +#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/tasks_required/proc_queue/src/proc_queue.cc index c911bb65a..56f7d215a 100644 --- a/tasks_required/proc_queue/src/proc_queue.cc +++ b/tasks_required/proc_queue/src/proc_queue.cc @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "proc_queue/proc_queue.h" -namespace labstor::proc_queue { +namespace hrun::proc_queue { class Server : public TaskLib { public: @@ -24,14 +24,14 @@ class Server : public TaskLib { switch (task->phase_) { case PushTaskPhase::kSchedule: { task->sub_run_.shm_ = task->sub_cli_.shm_; - task->sub_run_.ptr_ = LABSTOR_CLIENT->GetPrivatePointer(task->sub_cli_.shm_); + task->sub_run_.ptr_ = HRUN_CLIENT->GetPrivatePointer(task->sub_cli_.shm_); Task *&ptr = task->sub_run_.ptr_; // HILOG(kDebug, "Scheduling task {} on state {} tid {}", // ptr->task_node_, ptr->task_state_, GetLinuxTid()); if (ptr->IsFireAndForget()) { ptr->UnsetFireAndForget(); } - MultiQueue *real_queue = LABSTOR_CLIENT->GetQueue(QueueId(ptr->task_state_)); + MultiQueue *real_queue = HRUN_CLIENT->GetQueue(QueueId(ptr->task_state_)); real_queue->Emplace(ptr->prio_, ptr->lane_hash_, task->sub_run_.shm_); task->phase_ = PushTaskPhase::kWaitSchedule; } @@ -50,6 +50,6 @@ class Server : public TaskLib { #include "proc_queue/proc_queue_lib_exec.h" }; -} // namespace labstor::proc_queue +} // namespace hrun::proc_queue -LABSTOR_TASK_CC(labstor::proc_queue::Server, "proc_queue"); +HRUN_TASK_CC(hrun::proc_queue::Server, "proc_queue"); diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/tasks_required/remote_queue/include/remote_queue/remote_queue.h index 8c304b1da..a12c4586d 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_remote_queue_H_ -#define LABSTOR_remote_queue_H_ +#ifndef HRUN_remote_queue_H_ +#define HRUN_remote_queue_H_ #include "remote_queue_tasks.h" -namespace labstor::remote_queue { +namespace hrun::remote_queue { /** * Create remote_queue requests @@ -31,17 +31,17 @@ class Client : public TaskLibClient { const std::string &state_name, const TaskStateId &state_id) { id_ = state_id; - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, // {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} {1, 1, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - return LABSTOR_ADMIN->AsyncCreateTaskState( + return HRUN_ADMIN->AsyncCreateTaskState( task_node, domain_id, state_name, id_, queue_info); } - LABSTOR_TASK_NODE_ROOT(AsyncCreate); + HRUN_TASK_NODE_ROOT(AsyncCreate); template HSHM_ALWAYS_INLINE void CreateRoot(Args&& ...args) { @@ -50,13 +50,13 @@ class Client : public TaskLibClient { task->Wait(); id_ = task->id_; queue_id_ = QueueId(id_); - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); } /** Destroy task state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /** Disperse a task among a domain of nodes */ @@ -67,16 +67,16 @@ class Client : public TaskLibClient { // Serialize task + create the wait task HILOG(kDebug, "Beginning dispersion for (task_node={}, task_state={}, method={})", orig_task->task_node_ + 1, orig_task->task_state_, orig_task->method_) - BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); + BinaryOutputArchive ar(DomainId::GetNode(HRUN_CLIENT->node_id_)); std::vector xfer = exec->SaveStart(orig_task->method_, ar, orig_task); // Create subtasks exec->ReplicateStart(orig_task->method_, domain_ids.size(), orig_task); - LPointer push_task = LABSTOR_CLIENT->NewTask( + LPointer push_task = HRUN_CLIENT->NewTask( orig_task->task_node_ + 1, DomainId::GetLocal(), id_, domain_ids, orig_task, exec, orig_task->method_, xfer); - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(TaskPrio::kLowLatency, orig_task->lane_hash_, push_task.shm_); } @@ -99,10 +99,10 @@ class Client : public TaskLibClient { // Create duplicate task exec->ReplicateStart(orig_task->method_, lane_group->num_lanes_, orig_task); - LPointer dup_task = LABSTOR_CLIENT->NewTask( + LPointer dup_task = HRUN_CLIENT->NewTask( orig_task->task_node_ + 1, id_, orig_task, exec, orig_task->method_, dups); - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); queue->Emplace(orig_task->prio_, orig_task->lane_hash_, dup_task.shm_); } @@ -110,14 +110,14 @@ class Client : public TaskLibClient { // HSHM_ALWAYS_INLINE // AcceptTask* AsyncAcceptThread() { // hipc::Pointer p; -// MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); -// auto *task = LABSTOR_CLIENT->NewTask( +// MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); +// auto *task = HRUN_CLIENT->NewTask( // p, TaskNode::GetNull(), DomainId::GetLocal(), id_); // queue->Emplace(0, 0, p); // return task; // } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_remote_queue_H_ +#endif // HRUN_remote_queue_H_ diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h index 8fb284193..aa55b5fdb 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_REMOTE_QUEUE_LIB_EXEC_H_ -#define LABSTOR_REMOTE_QUEUE_LIB_EXEC_H_ +#ifndef HRUN_REMOTE_QUEUE_LIB_EXEC_H_ +#define HRUN_REMOTE_QUEUE_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -26,19 +26,19 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kPush: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDup: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -47,19 +47,19 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kPush: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDup: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -68,19 +68,19 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kPush: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDup: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -89,19 +89,19 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kPush: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDup: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -110,19 +110,19 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kPush: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDup: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -154,22 +154,22 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kPush: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDup: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -238,4 +238,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_REMOTE_QUEUE_METHODS_H_ \ No newline at end of file +#endif // HRUN_REMOTE_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h index 001659a9a..d401adc2b 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_REMOTE_QUEUE_METHODS_H_ -#define LABSTOR_REMOTE_QUEUE_METHODS_H_ +#ifndef HRUN_REMOTE_QUEUE_METHODS_H_ +#define HRUN_REMOTE_QUEUE_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -7,4 +7,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kDup = kLast + 1; }; -#endif // LABSTOR_REMOTE_QUEUE_METHODS_H_ \ No newline at end of file +#endif // HRUN_REMOTE_QUEUE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h index e3c8f2d87..673ce7c88 100644 --- a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h +++ b/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h @@ -2,13 +2,13 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ -#define LABSTOR_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ +#ifndef HRUN_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ +#define HRUN_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" #include @@ -19,15 +19,15 @@ namespace tl = thallium; -namespace labstor::remote_queue { +namespace hrun::remote_queue { -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" #include "remote_queue_methods.h" /** * A task to create remote_queue * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -48,7 +48,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy remote_queue */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -194,6 +194,6 @@ struct DupTask : public Task, TaskFlags { // } //}; -} // namespace labstor::remote_queue +} // namespace hrun::remote_queue -#endif //LABSTOR_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ +#endif //HRUN_TASKS_REMOTE_QUEUE_INCLUDE_REMOTE_QUEUE_REMOTE_QUEUE_TASKS_H_ diff --git a/tasks_required/remote_queue/src/CMakeLists.txt b/tasks_required/remote_queue/src/CMakeLists.txt index dce89518d..f130ef443 100644 --- a/tasks_required/remote_queue/src/CMakeLists.txt +++ b/tasks_required/remote_queue/src/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(remote_queue SHARED remote_queue.cc) add_dependencies(remote_queue ${Hermes_RUNTIME_DEPS}) target_link_libraries(remote_queue - labstor_runtime ${libfabric_LIBRARIES}) + hrun_runtime ${libfabric_LIBRARIES}) #------------------------------------------------------------------------------ # Install Small Message Task Library diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/tasks_required/remote_queue/src/remote_queue.cc index 50190e29d..f3cda24c7 100644 --- a/tasks_required/remote_queue/src/remote_queue.cc +++ b/tasks_required/remote_queue/src/remote_queue.cc @@ -2,22 +2,22 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "remote_queue/remote_queue.h" namespace thallium { /** Serialize I/O type enum */ -SERIALIZE_ENUM(labstor::IoType); +SERIALIZE_ENUM(hrun::IoType); } // namespace thallium -namespace labstor::remote_queue { +namespace hrun::remote_queue { class Server : public TaskLib { public: - labstor::remote_queue::Client client_; + hrun::remote_queue::Client client_; public: Server() = default; @@ -25,14 +25,14 @@ class Server : public TaskLib { /** Construct remote queue */ void Construct(ConstructTask *task, RunContext &rctx) { HILOG(kInfo, "(node {}) Constructing remote queue (task_node={}, task_state={}, method={})", - LABSTOR_CLIENT->node_id_, task->task_node_, task->task_state_, task->method_); - LABSTOR_THALLIUM->RegisterRpc("RpcPushSmall", [this](const tl::request &req, + HRUN_CLIENT->node_id_, task->task_node_, task->task_state_, task->method_); + HRUN_THALLIUM->RegisterRpc("RpcPushSmall", [this](const tl::request &req, TaskStateId state_id, u32 method, std::string ¶ms) { this->RpcPushSmall(req, state_id, method, params); }); - LABSTOR_THALLIUM->RegisterRpc("RpcPushBulk", [this](const tl::request &req, + HRUN_THALLIUM->RegisterRpc("RpcPushBulk", [this](const tl::request &req, const tl::bulk &bulk, TaskStateId state_id, u32 method, @@ -89,9 +89,9 @@ class Server : public TaskLib { task->orig_task_->task_node_, task->orig_task_->task_state_, task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, + HRUN_CLIENT->node_id_, domain_id.id_); - std::string ret = LABSTOR_THALLIUM->SyncCall(domain_id.id_, + std::string ret = HRUN_THALLIUM->SyncCall(domain_id.id_, "RpcPushSmall", task->exec_->id_, task->exec_method_, @@ -101,7 +101,7 @@ class Server : public TaskLib { task->orig_task_->task_node_, task->orig_task_->task_state_, task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, + HRUN_CLIENT->node_id_, domain_id.id_); ClientHandlePushReplicaOutput(replica, ret, task); } @@ -123,10 +123,10 @@ class Server : public TaskLib { task->orig_task_->task_node_, task->orig_task_->task_state_, task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, + HRUN_CLIENT->node_id_, domain_id.id_, static_cast(io_type)); - std::string ret = LABSTOR_THALLIUM->SyncIoCall(domain_id.id_, + std::string ret = HRUN_THALLIUM->SyncIoCall(domain_id.id_, "RpcPushBulk", io_type, data, @@ -141,7 +141,7 @@ class Server : public TaskLib { task->orig_task_->task_node_, task->orig_task_->task_state_, task->orig_task_->method_, - LABSTOR_CLIENT->node_id_, + HRUN_CLIENT->node_id_, domain_id.id_, static_cast(io_type)); ClientHandlePushReplicaOutput(replica, ret, task); @@ -173,7 +173,7 @@ class Server : public TaskLib { LPointer &dup = task->dups_[i]; dup->Wait(task); task->exec_->DupEnd(dup->method_, i, task->orig_task_, dup.ptr_); - LABSTOR_CLIENT->DelTask(dup); + HRUN_CLIENT->DelTask(dup); } task->exec_->ReplicateEnd(task->exec_method_, task->orig_task_); task->orig_task_->SetModuleComplete(); @@ -222,15 +222,15 @@ class Server : public TaskLib { // Process the message if (io_type == IoType::kWrite) { - LABSTOR_THALLIUM->IoCallServer(req, bulk, io_type, data.data(), data_size); - HILOG(kDebug, "(node {}) Write blob integer: {}", LABSTOR_CLIENT->node_id_, (int)data[0]) + HRUN_THALLIUM->IoCallServer(req, bulk, io_type, data.data(), data_size); + HILOG(kDebug, "(node {}) Write blob integer: {}", HRUN_CLIENT->node_id_, (int)data[0]) } TaskState *exec; Task *orig_task; RpcExec(req, state_id, method, xfer, orig_task, exec); if (io_type == IoType::kRead) { - HILOG(kDebug, "(node {}) Read blob integer: {}", LABSTOR_CLIENT->node_id_, (int)data[0]) - LABSTOR_THALLIUM->IoCallServer(req, bulk, io_type, data.data(), data_size); + HILOG(kDebug, "(node {}) Read blob integer: {}", HRUN_CLIENT->node_id_, (int)data[0]) + HRUN_THALLIUM->IoCallServer(req, bulk, io_type, data.data(), data_size); } // Return @@ -247,24 +247,24 @@ class Server : public TaskLib { BinaryInputArchive ar(xfer); // Deserialize task - exec = LABSTOR_TASK_REGISTRY->GetTaskState(state_id); + exec = HRUN_TASK_REGISTRY->GetTaskState(state_id); if (exec == nullptr) { HELOG(kFatal, "(node {}) Could not find the task state {}", - LABSTOR_CLIENT->node_id_, state_id); + HRUN_CLIENT->node_id_, state_id); req.respond(std::string()); return; } else { HILOG(kDebug, "(node {}) Found task state {}", - LABSTOR_CLIENT->node_id_, + HRUN_CLIENT->node_id_, state_id); } TaskPointer task_ptr = exec->LoadStart(method, ar); orig_task = task_ptr.ptr_; hipc::Pointer &p = task_ptr.shm_; - orig_task->domain_id_ = DomainId::GetNode(LABSTOR_CLIENT->node_id_); + orig_task->domain_id_ = DomainId::GetNode(HRUN_CLIENT->node_id_); // Execute task - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(QueueId(state_id)); + MultiQueue *queue = HRUN_CLIENT->GetQueue(QueueId(state_id)); orig_task->UnsetFireAndForget(); orig_task->UnsetStarted(); orig_task->UnsetDataOwner(); @@ -272,7 +272,7 @@ class Server : public TaskLib { queue->Emplace(orig_task->prio_, orig_task->lane_hash_, p); HILOG(kDebug, "(node {}) Executing task (task_node={}, task_state={}/{}, state_name={}, method={}, size={}, lane_hash={})", - LABSTOR_CLIENT->node_id_, + HRUN_CLIENT->node_id_, orig_task->task_node_, orig_task->task_state_, state_id, @@ -286,16 +286,16 @@ class Server : public TaskLib { void RpcComplete(const tl::request &req, u32 method, Task *orig_task, TaskState *exec, TaskStateId state_id) { - BinaryOutputArchive ar(DomainId::GetNode(LABSTOR_CLIENT->node_id_)); + BinaryOutputArchive ar(DomainId::GetNode(HRUN_CLIENT->node_id_)); std::vector out_xfer = exec->SaveEnd(method, ar, orig_task); HILOG(kDebug, "(node {}) Returning {} bytes of data (task_node={}, task_state={}/{}, method={})", - LABSTOR_CLIENT->node_id_, + HRUN_CLIENT->node_id_, out_xfer[0].data_size_, orig_task->task_node_, orig_task->task_state_, state_id, method); - LABSTOR_CLIENT->DelTask(exec, orig_task); + HRUN_CLIENT->DelTask(exec, orig_task); if (out_xfer.size() > 0 && out_xfer[0].data_size_ > 0) { req.respond(std::string((char *) out_xfer[0].data_, out_xfer[0].data_size_)); } else { @@ -306,6 +306,6 @@ class Server : public TaskLib { public: #include "remote_queue/remote_queue_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(labstor::remote_queue::Server, "remote_queue"); +HRUN_TASK_CC(hrun::remote_queue::Server, "remote_queue"); diff --git a/tasks_required/small_message/include/small_message/small_message.h b/tasks_required/small_message/include/small_message/small_message.h index bb5f8c1a2..f66d120f2 100644 --- a/tasks_required/small_message/include/small_message/small_message.h +++ b/tasks_required/small_message/include/small_message/small_message.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_small_message_H_ -#define LABSTOR_small_message_H_ +#ifndef HRUN_small_message_H_ +#define HRUN_small_message_H_ #include "small_message_tasks.h" -namespace labstor::small_message { +namespace hrun::small_message { /** Create admin requests */ class Client : public TaskLibClient { @@ -24,13 +24,13 @@ class Client : public TaskLibClient { void CreateRoot(const DomainId &domain_id, const std::string &state_name) { id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = LABSTOR_CLIENT->server_config_.queue_manager_; + QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; std::vector queue_info = { {1, 1, qm.queue_depth_, 0}, {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} }; - id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + id_ = HRUN_ADMIN->CreateTaskStateRoot( domain_id, state_name, id_, queue_info); queue_id_ = QueueId(id_); } @@ -38,24 +38,24 @@ class Client : public TaskLibClient { /** Destroy state + queue */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } /** Metadata task */ LPointer AsyncMd(const TaskNode &task_node, const DomainId &domain_id) { - MultiQueue *queue = LABSTOR_CLIENT->GetQueue(queue_id_); - auto task = LABSTOR_CLIENT->NewTask( + MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); + auto task = HRUN_CLIENT->NewTask( task_node, domain_id, id_); queue->Emplace(TaskPrio::kLowLatency, 3, task.shm_); return task; } - LABSTOR_TASK_NODE_ROOT(AsyncMd); + HRUN_TASK_NODE_ROOT(AsyncMd); int MdRoot(const DomainId &domain_id) { LPointer task = AsyncMdRoot(domain_id); task->Wait(); int ret = task->ret_[0]; - LABSTOR_CLIENT->DelTask(task); + HRUN_CLIENT->DelTask(task); return ret; } @@ -63,37 +63,37 @@ class Client : public TaskLibClient { void AsyncMdPushConstruct(MdPushTask *task, const TaskNode &task_node, const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id_); } int MdPushRoot(const DomainId &domain_id) { - LPointer> push_task = + LPointer> push_task = AsyncMdPushRoot(domain_id); push_task->Wait(); MdPushTask *task = push_task->get(); int ret = task->ret_[0]; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return ret; } - LABSTOR_TASK_NODE_PUSH_ROOT(MdPush); + HRUN_TASK_NODE_PUSH_ROOT(MdPush); /** Io task */ void AsyncIoConstruct(IoTask *task, const TaskNode &task_node, const DomainId &domain_id) { - LABSTOR_CLIENT->ConstructTask( + HRUN_CLIENT->ConstructTask( task, task_node, domain_id, id_); } int IoRoot(const DomainId &domain_id) { - LPointer> push_task = AsyncIoRoot(domain_id); + LPointer> push_task = AsyncIoRoot(domain_id); push_task->Wait(); IoTask *task = push_task->get(); int ret = task->ret_; - LABSTOR_CLIENT->DelTask(push_task); + HRUN_CLIENT->DelTask(push_task); return ret; } - LABSTOR_TASK_NODE_PUSH_ROOT(Io) + HRUN_TASK_NODE_PUSH_ROOT(Io) }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_small_message_H_ +#endif // HRUN_small_message_H_ diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/tasks_required/small_message/include/small_message/small_message_lib_exec.h index 3dbb98838..692bcd8f5 100644 --- a/tasks_required/small_message/include/small_message/small_message_lib_exec.h +++ b/tasks_required/small_message/include/small_message/small_message_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_SMALL_MESSAGE_LIB_EXEC_H_ -#define LABSTOR_SMALL_MESSAGE_LIB_EXEC_H_ +#ifndef HRUN_SMALL_MESSAGE_LIB_EXEC_H_ +#define HRUN_SMALL_MESSAGE_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -30,23 +30,23 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kMd: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kIo: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kMdPush: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -55,23 +55,23 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kMd: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kIo: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kMdPush: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -80,23 +80,23 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kMd: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kIo: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kMdPush: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -105,23 +105,23 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kMd: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kIo: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kMdPush: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -130,23 +130,23 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kMd: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kIo: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kMdPush: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -182,27 +182,27 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kMd: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kIo: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kMdPush: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -282,4 +282,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_SMALL_MESSAGE_METHODS_H_ \ No newline at end of file +#endif // HRUN_SMALL_MESSAGE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/small_message/include/small_message/small_message_methods.h b/tasks_required/small_message/include/small_message/small_message_methods.h index fc0e6e35d..0d02897c9 100644 --- a/tasks_required/small_message/include/small_message/small_message_methods.h +++ b/tasks_required/small_message/include/small_message/small_message_methods.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_SMALL_MESSAGE_METHODS_H_ -#define LABSTOR_SMALL_MESSAGE_METHODS_H_ +#ifndef HRUN_SMALL_MESSAGE_METHODS_H_ +#define HRUN_SMALL_MESSAGE_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { @@ -8,4 +8,4 @@ struct Method : public TaskMethod { TASK_METHOD_T kMdPush = kLast + 2; }; -#endif // LABSTOR_SMALL_MESSAGE_METHODS_H_ \ No newline at end of file +#endif // HRUN_SMALL_MESSAGE_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/small_message/include/small_message/small_message_tasks.h b/tasks_required/small_message/include/small_message/small_message_tasks.h index 1503e9f96..f1a074a38 100644 --- a/tasks_required/small_message/include/small_message/small_message_tasks.h +++ b/tasks_required/small_message/include/small_message/small_message_tasks.h @@ -2,24 +2,24 @@ // Created by lukemartinlogan on 8/11/23. // -#ifndef LABSTOR_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ -#define LABSTOR_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ +#ifndef HRUN_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ +#define HRUN_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" -namespace labstor::small_message { +namespace hrun::small_message { #include "small_message_methods.h" -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" /** * A task to create small_message * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -39,7 +39,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy small_message */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -245,6 +245,6 @@ struct IoTask : public Task, TaskFlags { } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ +#endif // HRUN_TASKS_SMALL_MESSAGE_INCLUDE_SMALL_MESSAGE_SMALL_MESSAGE_TASKS_H_ diff --git a/tasks_required/small_message/src/small_message.cc b/tasks_required/small_message/src/small_message.cc index 05ed670e2..0c5a815d2 100644 --- a/tasks_required/small_message/src/small_message.cc +++ b/tasks_required/small_message/src/small_message.cc @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "small_message/small_message.h" -namespace labstor::small_message { +namespace hrun::small_message { class Server : public TaskLib { public: @@ -46,6 +46,6 @@ class Server : public TaskLib { #include "small_message/small_message_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(labstor::small_message::Server, "small_message"); +HRUN_TASK_CC(hrun::small_message::Server, "small_message"); diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h index 0ffcffd08..f57feabaa 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_worch_proc_round_robin_H_ -#define LABSTOR_worch_proc_round_robin_H_ +#ifndef HRUN_worch_proc_round_robin_H_ +#define HRUN_worch_proc_round_robin_H_ #include "worch_proc_round_robin_tasks.h" -namespace labstor::worch_proc_round_robin { +namespace hrun::worch_proc_round_robin { /** Create admin requests */ class Client : public TaskLibClient { @@ -27,17 +27,17 @@ class Client : public TaskLibClient { std::vector queue_info = { {1, 1, 4, 0}, }; - id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + id_ = HRUN_ADMIN->CreateTaskStateRoot( domain_id, state_name, id_, queue_info); } /** Destroy state */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_worch_proc_round_robin_H_ +#endif // HRUN_worch_proc_round_robin_H_ diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h index ed3c310cd..b859d2bf8 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ -#define LABSTOR_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ +#ifndef HRUN_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ +#define HRUN_WORCH_PROC_ROUND_ROBIN_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -22,15 +22,15 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kSchedule: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -39,15 +39,15 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kSchedule: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -56,15 +56,15 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kSchedule: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -73,15 +73,15 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kSchedule: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -90,15 +90,15 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kSchedule: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -126,17 +126,17 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSchedule: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -194,4 +194,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ \ No newline at end of file +#endif // HRUN_WORCH_PROC_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h index 382707b85..3256067b1 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h @@ -1,9 +1,9 @@ -#ifndef LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ -#define LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ +#ifndef HRUN_WORCH_PROC_ROUND_ROBIN_METHODS_H_ +#define HRUN_WORCH_PROC_ROUND_ROBIN_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kSchedule = kLast + 0; }; -#endif // LABSTOR_WORCH_PROC_ROUND_ROBIN_METHODS_H_ \ No newline at end of file +#endif // HRUN_WORCH_PROC_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h index e9e6a27af..7f8324945 100644 --- a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h +++ b/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h @@ -2,19 +2,19 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_WORCH_PROC_ROUND_ROBIN_TASKS_H__ -#define LABSTOR_WORCH_PROC_ROUND_ROBIN_TASKS_H__ +#ifndef HRUN_WORCH_PROC_ROUND_ROBIN_TASKS_H__ +#define HRUN_WORCH_PROC_ROUND_ROBIN_TASKS_H__ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor/work_orchestrator/scheduler.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun/work_orchestrator/scheduler.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" -namespace labstor::worch_proc_round_robin { +namespace hrun::worch_proc_round_robin { -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" /** The set of methods in the worch task */ typedef SchedulerMethod Method; @@ -22,7 +22,7 @@ typedef SchedulerMethod Method; /** * A task to create worch_proc_round_robin * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -42,7 +42,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy worch_proc_round_robin */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -63,6 +63,6 @@ struct DestructTask : public DestroyTaskStateTask { } }; -} // namespace labstor::worch_proc_round_robin +} // namespace hrun::worch_proc_round_robin -#endif // LABSTOR_WORCH_PROC_ROUND_ROBIN_TASKS_H__ +#endif // HRUN_WORCH_PROC_ROUND_ROBIN_TASKS_H__ diff --git a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc index 81411bf8f..41b02cccd 100644 --- a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc +++ b/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "worch_proc_round_robin/worch_proc_round_robin.h" -namespace labstor::worch_proc_round_robin { +namespace hrun::worch_proc_round_robin { class Server : public TaskLib { public: @@ -20,7 +20,7 @@ class Server : public TaskLib { void Schedule(ScheduleTask *task, RunContext &rctx) { int rr = 0; - for (Worker &worker : LABSTOR_WORK_ORCHESTRATOR->workers_) { + for (Worker &worker : HRUN_WORK_ORCHESTRATOR->workers_) { worker.SetCpuAffinity(rr % HERMES_SYSTEM_INFO->ncpu_); ++rr; } @@ -29,6 +29,6 @@ class Server : public TaskLib { #include "worch_proc_round_robin/worch_proc_round_robin_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(labstor::worch_proc_round_robin::Server, "worch_proc_round_robin"); +HRUN_TASK_CC(hrun::worch_proc_round_robin::Server, "worch_proc_round_robin"); diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h index 1b4796079..539f96fb3 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h @@ -2,12 +2,12 @@ // Created by lukemartinlogan on 6/29/23. // -#ifndef LABSTOR_worch_queue_round_robin_H_ -#define LABSTOR_worch_queue_round_robin_H_ +#ifndef HRUN_worch_queue_round_robin_H_ +#define HRUN_worch_queue_round_robin_H_ #include "worch_queue_round_robin_tasks.h" -namespace labstor::worch_queue_round_robin { +namespace hrun::worch_queue_round_robin { /** Create admin requests */ class Client : public TaskLibClient { @@ -27,17 +27,17 @@ class Client : public TaskLibClient { std::vector queue_info = { {1, 1, 4, 0}, }; - id_ = LABSTOR_ADMIN->CreateTaskStateRoot( + id_ = HRUN_ADMIN->CreateTaskStateRoot( domain_id, state_name, id_, queue_info); } /** Destroy task state */ HSHM_ALWAYS_INLINE void DestroyRoot(const DomainId &domain_id) { - LABSTOR_ADMIN->DestroyTaskStateRoot(domain_id, id_); + HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); } }; -} // namespace labstor +} // namespace hrun -#endif // LABSTOR_worch_queue_round_robin_H_ +#endif // HRUN_worch_queue_round_robin_H_ diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h index 51b17b71d..4f4c3237d 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h @@ -1,5 +1,5 @@ -#ifndef LABSTOR_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ -#define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ +#ifndef HRUN_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ +#define HRUN_WORCH_QUEUE_ROUND_ROBIN_LIB_EXEC_H_ /** Execute a task */ void Run(u32 method, Task *task, RunContext &rctx) override { @@ -22,15 +22,15 @@ void Run(u32 method, Task *task, RunContext &rctx) override { void Del(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kDestruct: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } case Method::kSchedule: { - LABSTOR_CLIENT->DelTask(reinterpret_cast(task)); + HRUN_CLIENT->DelTask(reinterpret_cast(task)); break; } } @@ -39,15 +39,15 @@ void Del(u32 method, Task *task) override { void Dup(u32 method, Task *orig_task, std::vector> &dups) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } case Method::kSchedule: { - labstor::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); + hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); break; } } @@ -56,15 +56,15 @@ void Dup(u32 method, Task *orig_task, std::vector> &dups) overrid void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kDestruct: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } case Method::kSchedule: { - labstor::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); + hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); break; } } @@ -73,15 +73,15 @@ void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { void ReplicateStart(u32 method, u32 count, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } case Method::kSchedule: { - labstor::CALL_REPLICA_START(count, reinterpret_cast(task)); + hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); break; } } @@ -90,15 +90,15 @@ void ReplicateStart(u32 method, u32 count, Task *task) override { void ReplicateEnd(u32 method, Task *task) override { switch (method) { case Method::kConstruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kDestruct: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } case Method::kSchedule: { - labstor::CALL_REPLICA_END(reinterpret_cast(task)); + hrun::CALL_REPLICA_END(reinterpret_cast(task)); break; } } @@ -126,17 +126,17 @@ TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { TaskPointer task_ptr; switch (method) { case Method::kConstruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kDestruct: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } case Method::kSchedule: { - task_ptr.ptr_ = LABSTOR_CLIENT->NewEmptyTask(task_ptr.shm_); + task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); ar >> *reinterpret_cast(task_ptr.ptr_); break; } @@ -194,4 +194,4 @@ u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { return -1; } -#endif // LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ \ No newline at end of file +#endif // HRUN_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h index 87ae85fc8..b099e4b47 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h @@ -1,9 +1,9 @@ -#ifndef LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ -#define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ +#ifndef HRUN_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ +#define HRUN_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ /** The set of methods in the admin task */ struct Method : public TaskMethod { TASK_METHOD_T kSchedule = kLast + 0; }; -#endif // LABSTOR_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ \ No newline at end of file +#endif // HRUN_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ \ No newline at end of file diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h index 68ebda34b..666231d06 100644 --- a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h +++ b/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h @@ -2,19 +2,19 @@ // Created by lukemartinlogan on 8/14/23. // -#ifndef LABSTOR_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ -#define LABSTOR_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ +#ifndef HRUN_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ +#define HRUN_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ -#include "labstor/api/labstor_client.h" -#include "labstor/task_registry/task_lib.h" -#include "labstor_admin/labstor_admin.h" -#include "labstor/work_orchestrator/scheduler.h" -#include "labstor/queue_manager/queue_manager_client.h" +#include "hrun/api/hrun_client.h" +#include "hrun/task_registry/task_lib.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/work_orchestrator/scheduler.h" +#include "hrun/queue_manager/queue_manager_client.h" #include "proc_queue/proc_queue.h" -namespace labstor::worch_queue_round_robin { +namespace hrun::worch_queue_round_robin { -#include "labstor/labstor_namespace.h" +#include "hrun/hrun_namespace.h" /** The set of methods in the worch task */ typedef SchedulerMethod Method; @@ -22,7 +22,7 @@ typedef SchedulerMethod Method; /** * A task to create worch_queue_round_robin * */ -using labstor::Admin::CreateTaskStateTask; +using hrun::Admin::CreateTaskStateTask; struct ConstructTask : public CreateTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -46,7 +46,7 @@ struct ConstructTask : public CreateTaskStateTask { }; /** A task to destroy worch_queue_round_robin */ -using labstor::Admin::DestroyTaskStateTask; +using hrun::Admin::DestroyTaskStateTask; struct DestructTask : public DestroyTaskStateTask { /** SHM default constructor */ HSHM_ALWAYS_INLINE explicit @@ -67,6 +67,6 @@ struct DestructTask : public DestroyTaskStateTask { } }; -} // namespace labstor::worch_queue_round_robin +} // namespace hrun::worch_queue_round_robin -#endif // LABSTOR_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ +#endif // HRUN_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ diff --git a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc index e1b484b11..1089e30eb 100644 --- a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc +++ b/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc @@ -2,11 +2,11 @@ // Created by lukemartinlogan on 6/29/23. // -#include "labstor_admin/labstor_admin.h" -#include "labstor/api/labstor_runtime.h" +#include "hrun_admin/hrun_admin.h" +#include "hrun/api/hrun_runtime.h" #include "worch_queue_round_robin/worch_queue_round_robin.h" -namespace labstor::worch_queue_round_robin { +namespace hrun::worch_queue_round_robin { class Server : public TaskLib { public: @@ -24,7 +24,7 @@ class Server : public TaskLib { void Schedule(ScheduleTask *task, RunContext &rctx) { // Check if any new queues need to be scheduled - for (MultiQueue &queue : *LABSTOR_QM_RUNTIME->queue_map_) { + for (MultiQueue &queue : *HRUN_QM_RUNTIME->queue_map_) { if (queue.id_.IsNull() || !queue.flags_.Any(QUEUE_READY)) { continue; } @@ -33,15 +33,15 @@ class Server : public TaskLib { if (lane_group.IsLowPriority()) { for (u32 lane_id = lane_group.num_scheduled_; lane_id < lane_group.num_lanes_; ++lane_id) { // HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); - Worker &worker = LABSTOR_WORK_ORCHESTRATOR->workers_[0]; + Worker &worker = HRUN_WORK_ORCHESTRATOR->workers_[0]; worker.PollQueues({WorkEntry(lane_group.prio_, lane_id, &queue)}); } lane_group.num_scheduled_ = lane_group.num_lanes_; } else { for (u32 lane_id = lane_group.num_scheduled_; lane_id < lane_group.num_lanes_; ++lane_id) { // HILOG(kDebug, "Scheduling the queue {} (lane {})", queue.id_, lane_id); - u32 worker_id = (count_ % (LABSTOR_WORK_ORCHESTRATOR->workers_.size() - 1)) + 1; - Worker &worker = LABSTOR_WORK_ORCHESTRATOR->workers_[worker_id]; + u32 worker_id = (count_ % (HRUN_WORK_ORCHESTRATOR->workers_.size() - 1)) + 1; + Worker &worker = HRUN_WORK_ORCHESTRATOR->workers_[worker_id]; worker.PollQueues({WorkEntry(lane_group.prio_, lane_id, &queue)}); count_ += 1; } @@ -54,6 +54,6 @@ class Server : public TaskLib { #include "worch_queue_round_robin/worch_queue_round_robin_lib_exec.h" }; -} // namespace labstor +} // namespace hrun -LABSTOR_TASK_CC(labstor::worch_queue_round_robin::Server, "worch_queue_round_robin"); +HRUN_TASK_CC(hrun::worch_queue_round_robin::Server, "worch_queue_round_robin"); diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index da3d58de3..559ffe832 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -4,7 +4,7 @@ project(hermes) set(CMAKE_CXX_STANDARD 17) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_SOURCE_DIR}/tasks/labstor_admin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hrun_admin/include) add_subdirectory(ipc) add_subdirectory(hermes) add_subdirectory(hermes_adapters) diff --git a/test/unit/basic_test.h b/test/unit/basic_test.h index 278bf6d85..a5428131d 100644 --- a/test/unit/basic_test.h +++ b/test/unit/basic_test.h @@ -10,8 +10,8 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_TEST_UNIT_BASIC_TEST_H_ -#define LABSTOR_TEST_UNIT_BASIC_TEST_H_ +#ifndef HRUN_TEST_UNIT_BASIC_TEST_H_ +#define HRUN_TEST_UNIT_BASIC_TEST_H_ #define CATCH_CONFIG_RUNNER #include @@ -51,4 +51,4 @@ void MainPosttest(); #define PAGE_DIVIDE(TEXT) -#endif // LABSTOR_TEST_UNIT_BASIC_TEST_H_ +#endif // HRUN_TEST_UNIT_BASIC_TEST_H_ diff --git a/test/unit/boost/test_init.cc b/test/unit/boost/test_init.cc index e101758d3..1757fe387 100644 --- a/test/unit/boost/test_init.cc +++ b/test/unit/boost/test_init.cc @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "labstor/api/labstor_client.h" +#include "hrun/api/hrun_client.h" #include "basic_test.h" #include "test_init.h" diff --git a/test/unit/boost/test_init.h b/test/unit/boost/test_init.h index a6c71f3ec..cb3b36df7 100644 --- a/test/unit/boost/test_init.h +++ b/test/unit/boost/test_init.h @@ -11,9 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ -#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ +#define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ -#include "labstor/labstor_types.h" +#include "hrun/hrun_types.h" -#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/test/unit/hermes/config/labstor_server.yaml b/test/unit/hermes/config/labstor_server.yaml index cb4051d8d..42e4ecedb 100644 --- a/test/unit/hermes/config/labstor_server.yaml +++ b/test/unit/hermes/config/labstor_server.yaml @@ -14,7 +14,7 @@ queue_manager: # The shared memory allocator to use shm_allocator: kScalablePageAllocator # The name of the shared memory region to create - shm_name: "labstor_shm" + shm_name: "hrun_shm" # The size of the shared memory region to allocate shm_size: 0g diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index b4dd27838..2c908ee97 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -3,8 +3,8 @@ // #include "basic_test.h" -#include "labstor/api/labstor_client.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" #include "hermes/hermes.h" #include "hermes/bucket.h" #include "data_stager/factory/binary_stager.h" @@ -523,7 +523,7 @@ TEST_CASE("TestHermesDataOp") { } MPI_Barrier(MPI_COMM_WORLD); - // LABSTOR_ADMIN->FlushRoot(DomainId::GetGlobal()); + // HRUN_ADMIN->FlushRoot(DomainId::GetGlobal()); // Verify derived operator happens hermes::Bucket bkt_min("data_bkt_min", 0, 0); size_t size = bkt_min.GetSize(); diff --git a/test/unit/hermes/test_init.cc b/test/unit/hermes/test_init.cc index 63132b16a..91734b043 100644 --- a/test/unit/hermes/test_init.cc +++ b/test/unit/hermes/test_init.cc @@ -11,12 +11,12 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "labstor/api/labstor_client.h" +#include "hrun/api/hrun_client.h" #include "basic_test.h" #include "test_init.h" void MainPretest() { - TRANSPARENT_LABSTOR(); + TRANSPARENT_HRUN(); } void MainPosttest() { diff --git a/test/unit/hermes/test_init.h b/test/unit/hermes/test_init.h index a6c71f3ec..cb3b36df7 100644 --- a/test/unit/hermes/test_init.h +++ b/test/unit/hermes/test_init.h @@ -11,9 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ -#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ +#define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ -#include "labstor/labstor_types.h" +#include "hrun/hrun_types.h" -#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/test/unit/ipc/test_finalize.cc b/test/unit/ipc/test_finalize.cc index 8834bd3d2..94dc67576 100644 --- a/test/unit/ipc/test_finalize.cc +++ b/test/unit/ipc/test_finalize.cc @@ -3,9 +3,9 @@ // #include "basic_test.h" -#include "labstor/api/labstor_client.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" TEST_CASE("TestFinalize") { - LABSTOR_ADMIN->AsyncStopRuntimeRoot(labstor::DomainId::GetGlobal()); + HRUN_ADMIN->AsyncStopRuntimeRoot(hrun::DomainId::GetGlobal()); } \ No newline at end of file diff --git a/test/unit/ipc/test_init.cc b/test/unit/ipc/test_init.cc index 63132b16a..91734b043 100644 --- a/test/unit/ipc/test_init.cc +++ b/test/unit/ipc/test_init.cc @@ -11,12 +11,12 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "labstor/api/labstor_client.h" +#include "hrun/api/hrun_client.h" #include "basic_test.h" #include "test_init.h" void MainPretest() { - TRANSPARENT_LABSTOR(); + TRANSPARENT_HRUN(); } void MainPosttest() { diff --git a/test/unit/ipc/test_init.h b/test/unit/ipc/test_init.h index a6c71f3ec..cb3b36df7 100644 --- a/test/unit/ipc/test_init.h +++ b/test/unit/ipc/test_init.h @@ -11,9 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ -#define LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ +#define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ -#include "labstor/labstor_types.h" +#include "hrun/hrun_types.h" -#endif // LABSTOR_TEST_UNIT_IPC_TEST_INIT_H_ +#endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/test/unit/ipc/test_ipc.cc b/test/unit/ipc/test_ipc.cc index 643af0c59..a2f40633c 100644 --- a/test/unit/ipc/test_ipc.cc +++ b/test/unit/ipc/test_ipc.cc @@ -4,12 +4,12 @@ #include "basic_test.h" #include -#include "labstor/api/labstor_client.h" -#include "labstor_admin/labstor_admin.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" #include "small_message/small_message.h" #include "hermes_shm/util/timer.h" -#include "labstor/work_orchestrator/affinity.h" +#include "hrun/work_orchestrator/affinity.h" #include "omp.h" TEST_CASE("TestIpc") { @@ -17,9 +17,9 @@ TEST_CASE("TestIpc") { MPI_Barrier(MPI_COMM_WORLD); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - labstor::small_message::Client client; - LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); - client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(hrun::DomainId::GetGlobal(), "ipc_test"); MPI_Barrier(MPI_COMM_WORLD); hshm::Timer t; @@ -32,7 +32,7 @@ TEST_CASE("TestIpc") { int ret; HILOG(kInfo, "Sending message {}", i); int node_id = 1 + ((rank + 1) % nprocs); - ret = client.MdRoot(labstor::DomainId::GetNode(node_id)); + ret = client.MdRoot(hrun::DomainId::GetNode(node_id)); REQUIRE(ret == 1); } t.Pause(); @@ -45,9 +45,9 @@ TEST_CASE("TestFlush") { MPI_Barrier(MPI_COMM_WORLD); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - labstor::small_message::Client client; - LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); - client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(hrun::DomainId::GetGlobal(), "ipc_test"); MPI_Barrier(MPI_COMM_WORLD); hshm::Timer t; @@ -60,19 +60,19 @@ TEST_CASE("TestFlush") { int ret; HILOG(kInfo, "Sending message {}", i); int node_id = 1 + ((rank + 1) % nprocs); - LPointer task = - client.AsyncMdRoot(labstor::DomainId::GetNode(node_id)); + LPointer task = + client.AsyncMdRoot(hrun::DomainId::GetNode(node_id)); } - LABSTOR_ADMIN->FlushRoot(DomainId::GetGlobal()); + HRUN_ADMIN->FlushRoot(DomainId::GetGlobal()); t.Pause(); HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } void TestIpcMultithread(int nprocs) { - labstor::small_message::Client client; - LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); - client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(hrun::DomainId::GetGlobal(), "ipc_test"); #pragma omp parallel shared(client, nprocs) num_threads(nprocs) { @@ -82,7 +82,7 @@ void TestIpcMultithread(int nprocs) { int ret; HILOG(kInfo, "Sending message {}", i); int node_id = 1 + ((rank + 1) % nprocs); - ret = client.MdRoot(labstor::DomainId::GetNode(node_id)); + ret = client.MdRoot(hrun::DomainId::GetNode(node_id)); REQUIRE(ret == 1); } } @@ -109,9 +109,9 @@ TEST_CASE("TestIO") { MPI_Barrier(MPI_COMM_WORLD); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - labstor::small_message::Client client; - LABSTOR_ADMIN->RegisterTaskLibRoot(labstor::DomainId::GetGlobal(), "small_message"); - client.CreateRoot(labstor::DomainId::GetGlobal(), "ipc_test"); + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetGlobal(), "small_message"); + client.CreateRoot(hrun::DomainId::GetGlobal(), "ipc_test"); hshm::Timer t; int pid = getpid(); @@ -125,7 +125,7 @@ TEST_CASE("TestIO") { int ret; HILOG(kInfo, "Sending message {}", i); int node_id = 1 + ((rank + 1) % nprocs); - ret = client.IoRoot(labstor::DomainId::GetNode(node_id)); + ret = client.IoRoot(hrun::DomainId::GetNode(node_id)); REQUIRE(ret == 1); } t.Pause(); @@ -134,7 +134,7 @@ TEST_CASE("TestIO") { } //TEST_CASE("TestHostfile") { -// for (u32 node_id = 1; node_id < LABSTOR_THALLIUM->rpc_->hosts_.size() + 1; ++node_id) { -// HILOG(kInfo, "Node {}: {}", node_id, LABSTOR_THALLIUM->GetServerName(node_id)); +// for (u32 node_id = 1; node_id < HRUN_THALLIUM->rpc_->hosts_.size() + 1; ++node_id) { +// HILOG(kInfo, "Node {}: {}", node_id, HRUN_THALLIUM->GetServerName(node_id)); // } //} \ No newline at end of file diff --git a/test/unit/ipc/test_serialize.cc b/test/unit/ipc/test_serialize.cc index e3f1c7ed9..05e83b1c7 100644 --- a/test/unit/ipc/test_serialize.cc +++ b/test/unit/ipc/test_serialize.cc @@ -3,15 +3,15 @@ // #include "basic_test.h" -#include "labstor/network/serialize.h" -#include "labstor/task_registry/task.h" - -using labstor::DomainId; -using labstor::BinaryOutputArchive; -using labstor::BinaryInputArchive; -using labstor::DataTransfer; -using labstor::Task; -using labstor::TaskFlags; +#include "hrun/network/serialize.h" +#include "hrun/task_registry/task.h" + +using hrun::DomainId; +using hrun::BinaryOutputArchive; +using hrun::BinaryInputArchive; +using hrun::DataTransfer; +using hrun::Task; +using hrun::TaskFlags; struct TestObj : public Task, TaskFlags { std::vector data_; From a5bacc5d464a29f415a141c03f56ddc4132546fb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 21:32:12 -0500 Subject: [PATCH 178/191] change configuration names to hrun --- config/{labstor_client_default.yaml => hrun_client_default.yaml} | 0 config/{labstor_server_default.yaml => hrun_server_default.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename config/{labstor_client_default.yaml => hrun_client_default.yaml} (100%) rename config/{labstor_server_default.yaml => hrun_server_default.yaml} (100%) diff --git a/config/labstor_client_default.yaml b/config/hrun_client_default.yaml similarity index 100% rename from config/labstor_client_default.yaml rename to config/hrun_client_default.yaml diff --git a/config/labstor_server_default.yaml b/config/hrun_server_default.yaml similarity index 100% rename from config/labstor_server_default.yaml rename to config/hrun_server_default.yaml From 6740957095d28998acbaf0376b5ba88e92243df0 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 21:34:42 -0500 Subject: [PATCH 179/191] Change codegens to use hrun --- codegen/codegen/{labstor_config => hrun_config}/generator.py | 0 codegen/{labstor_config => hrun_config} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename codegen/codegen/{labstor_config => hrun_config}/generator.py (100%) rename codegen/{labstor_config => hrun_config} (100%) diff --git a/codegen/codegen/labstor_config/generator.py b/codegen/codegen/hrun_config/generator.py similarity index 100% rename from codegen/codegen/labstor_config/generator.py rename to codegen/codegen/hrun_config/generator.py diff --git a/codegen/labstor_config b/codegen/hrun_config similarity index 100% rename from codegen/labstor_config rename to codegen/hrun_config From 9ef996824a0a86b92a2811fb814ccf363f27ebe2 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 22:00:06 -0500 Subject: [PATCH 180/191] Repo reorganization --- .gitignore | 4 +- CMakeLists.txt | 25 +-- codegen/codegen/util/paths.py | 5 +- codegen/hermes_config | 3 +- codegen/hrun_config | 4 +- .../hermes_client_default.yaml | 0 .../hermes_server_default.yaml | 0 .../CMakeLists.txt | 6 +- .../adapter_constants.h | 0 .../adapter_types.h | 0 .../filesystem/CMakeLists.txt | 2 +- .../filesystem/filesystem.cc | 0 .../filesystem/filesystem.h | 0 .../filesystem/filesystem_io_client.h | 0 .../filesystem/filesystem_mdm.cc | 0 .../filesystem/filesystem_mdm.h | 0 .../filesystem/filesystem_mdm_singleton.cc | 0 .../interceptor.h | 0 .../mapper/abstract_mapper.h | 0 .../mapper/balanced_mapper.h | 0 .../mapper/mapper_factory.h | 0 .../mpiio/CMakeLists.txt | 2 +- .../mpiio/mpiio_api.cc | 0 .../mpiio/mpiio_api.h | 0 .../mpiio/mpiio_fs_api.h | 0 .../mpiio/mpiio_io_client.cc | 0 .../mpiio/mpiio_io_client.h | 0 .../posix/CMakeLists.txt | 2 +- .../posix/posix_api.cc | 0 .../posix/posix_api.h | 0 .../posix/posix_fs_api.h | 0 .../posix/posix_io_client.cc | 0 .../posix/posix_io_client.h | 0 .../real_api.h | 0 .../stdio/CMakeLists.txt | 2 +- .../stdio/stdio_api.cc | 0 .../stdio/stdio_api.h | 0 .../stdio/stdio_fs_api.h | 0 .../stdio/stdio_io_client.cc | 0 .../stdio/stdio_io_client.h | 0 .../vfd/CMakeLists.txt | 4 +- .../vfd/H5FDhermes.cc | 0 .../vfd/H5FDhermes.h | 0 .../vfd/README.md | 0 hrun/CMakeLists.txt | 160 ++++++++++++++ .../config}/hrun_client_default.yaml | 0 .../config}/hrun_server_default.yaml | 0 .../include}/hrun/api/hrun_client.h | 4 +- .../include}/hrun/api/hrun_runtime.h | 6 +- {include => hrun/include}/hrun/api/manager.h | 14 +- .../hrun_task_node_admin_root.template | 0 .../hrun_task_node_push_root.template | 0 .../include}/hrun/config/config.h | 0 .../include}/hrun/config/config_client.h | 0 .../hrun/config/config_client_default.h | 2 +- .../include}/hrun/config/config_server.h | 0 .../hrun/config/config_server_default.h | 2 +- .../include}/hrun/hrun_constants.h | 4 +- .../include}/hrun/hrun_namespace.h | 0 {include => hrun/include}/hrun/hrun_types.h | 4 +- .../include}/hrun/network/local_serialize.h | 0 {include => hrun/include}/hrun/network/rpc.h | 0 .../include}/hrun/network/rpc_thallium.h | 0 .../include}/hrun/network/serialize.h | 0 .../include}/hrun/queue_manager/queue.h | 0 .../hrun/queue_manager/queue_factory.h | 0 .../hrun/queue_manager/queue_manager.h | 0 .../hrun/queue_manager/queue_manager_client.h | 0 .../queue_manager/queue_manager_runtime.h | 0 .../hrun/queue_manager/queues/hshm_queue.h | 0 .../include}/hrun/task_registry/task.h | 0 .../include}/hrun/task_registry/task_lib.h | 0 .../hrun/task_registry/task_registry.h | 0 .../hrun/work_orchestrator/affinity.h | 0 .../hrun/work_orchestrator/scheduler.h | 0 .../work_orchestrator/work_orchestrator.h | 0 .../include}/hrun/work_orchestrator/worker.h | 0 hrun/src/CMakeLists.txt | 110 ++++++++++ {src => hrun/src}/config_client.cc | 2 +- {src => hrun/src}/config_server.cc | 2 +- {src => hrun/src}/hrun_client.cc | 0 {src => hrun/src}/hrun_runtime.cc | 0 {src => hrun/src}/hrun_start_runtime.cc | 0 {src => hrun/src}/hrun_stop_runtime.cc | 0 {src => hrun/src}/worker.cc | 0 .../tasks_required}/CMakeLists.txt | 0 .../tasks_required/TASK_NAME}/CMakeLists.txt | 4 +- .../TASK_NAME/include/TASK_NAME/TASK_NAME.h | 0 .../include/TASK_NAME/TASK_NAME_lib_exec.h | 0 .../include/TASK_NAME/TASK_NAME_methods.h | 0 .../include/TASK_NAME/TASK_NAME_methods.yaml | 0 .../include/TASK_NAME/TASK_NAME_tasks.h | 0 .../TASK_NAME/src/CMakeLists.txt | 0 .../TASK_NAME/src/TASK_NAME.cc | 0 .../tasks_required}/hrun_admin/CMakeLists.txt | 4 +- .../include/hrun_admin/hrun_admin.h | 0 .../include/hrun_admin/hrun_admin_lib_exec.h | 0 .../include/hrun_admin/hrun_admin_methods.h | 0 .../hrun_admin/hrun_admin_methods.yaml | 0 .../include/hrun_admin/hrun_admin_tasks.h | 0 .../hrun_admin/src/CMakeLists.txt | 4 +- .../hrun_admin/src/hrun_admin.cc | 0 .../tasks_required}/proc_queue/CMakeLists.txt | 4 +- .../include/proc_queue/proc_queue.h | 0 .../include/proc_queue/proc_queue_lib_exec.h | 0 .../include/proc_queue/proc_queue_methods.h | 0 .../proc_queue/proc_queue_methods.yaml | 0 .../include/proc_queue/proc_queue_tasks.h | 0 .../proc_queue/src/CMakeLists.txt | 0 .../proc_queue/src/proc_queue.cc | 0 .../remote_queue/CMakeLists.txt | 4 +- .../include/remote_queue/remote_queue.h | 0 .../remote_queue/remote_queue_lib_exec.h | 0 .../remote_queue/remote_queue_methods.h | 0 .../remote_queue/remote_queue_methods.yaml | 0 .../include/remote_queue/remote_queue_tasks.h | 0 .../remote_queue/src/CMakeLists.txt | 0 .../remote_queue/src/remote_queue.cc | 0 .../small_message}/CMakeLists.txt | 4 +- .../include/small_message/small_message.h | 0 .../small_message/small_message_lib_exec.h | 0 .../small_message/small_message_methods.h | 0 .../small_message/small_message_methods.yaml | 0 .../small_message/small_message_tasks.h | 0 .../small_message/src/CMakeLists.txt | 0 .../small_message/src/small_message.cc | 0 .../worch_proc_round_robin/CMakeLists.txt | 10 + .../worch_proc_round_robin.h | 0 .../worch_proc_round_robin_lib_exec.h | 0 .../worch_proc_round_robin_methods.h | 0 .../worch_proc_round_robin_methods.yaml | 0 .../worch_proc_round_robin_tasks.h | 0 .../worch_proc_round_robin/src/CMakeLists.txt | 0 .../src/worch_proc_round_robin.cc | 0 .../worch_queue_round_robin/CMakeLists.txt | 10 + .../worch_queue_round_robin.h | 0 .../worch_queue_round_robin_lib_exec.h | 0 .../worch_queue_round_robin_methods.h | 0 .../worch_queue_round_robin_methods.yaml | 0 .../worch_queue_round_robin_tasks.h | 0 .../src/CMakeLists.txt | 0 .../src/worch_queue_round_robin.cc | 0 .../hermes/adapter_types.h | 0 .../include => include}/hermes/bucket.h | 0 .../include => include}/hermes/config.h | 0 .../hermes/config_client.h | 0 .../hermes/config_client_default.h | 0 .../hermes/config_manager.h | 0 .../hermes/config_server.h | 0 .../hermes/config_server_default.h | 0 .../include => include}/hermes/dpe/dpe.h | 0 .../hermes/dpe/dpe_factory.h | 0 .../hermes/dpe/minimize_io_time.h | 0 .../include => include}/hermes/dpe/random.h | 0 .../hermes/dpe/round_robin.h | 0 .../include => include}/hermes/hermes.h | 0 .../include => include}/hermes/hermes_types.h | 0 .../hermes/slab_allocator.h | 0 .../include => include}/hermes/status.h | 0 .../include => include}/hermes/statuses.h | 0 src/CMakeLists.txt | 100 ++------- .../hermes_config_manager.cc | 0 tasks/CMakeLists.txt | 2 - tasks/bdev/CMakeLists.txt | 4 +- tasks/data_stager/CMakeLists.txt | 4 +- tasks/hermes/src/CMakeLists.txt | 54 ----- .../include/hermes_adapters/hermes_adapters.h | 73 ------- .../hermes_adapters_lib_exec.h | 197 ------------------ .../hermes_adapters/hermes_adapters_methods.h | 9 - .../hermes_adapters_methods.yaml | 1 - .../hermes_adapters/hermes_adapters_tasks.h | 119 ----------- tasks/hermes_adapters/src/CMakeLists.txt | 54 ----- tasks/hermes_adapters/src/hermes_adapters.cc | 39 ---- tasks/hermes_blob_mdm/CMakeLists.txt | 2 +- tasks/hermes_bucket_mdm/CMakeLists.txt | 2 +- tasks/hermes_data_op/CMakeLists.txt | 4 +- tasks/hermes_mdm/CMakeLists.txt | 4 +- tasks/posix_bdev/CMakeLists.txt | 4 +- tasks/ram_bdev/CMakeLists.txt | 4 +- tasks_required/small_message/CMakeLists.txt | 10 - .../worch_proc_round_robin/CMakeLists.txt | 10 - .../worch_queue_round_robin/CMakeLists.txt | 10 - 182 files changed, 387 insertions(+), 732 deletions(-) rename {tasks/hermes/config => config}/hermes_client_default.yaml (100%) rename {tasks/hermes/config => config}/hermes_server_default.yaml (100%) rename {tasks/hermes_adapters => hermes_adapters}/CMakeLists.txt (82%) rename {tasks/hermes_adapters/include/hermes_adapters => hermes_adapters}/adapter_constants.h (100%) rename {tasks/hermes_adapters/include/hermes_adapters => hermes_adapters}/adapter_types.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/filesystem/CMakeLists.txt (98%) rename {tasks/hermes_adapters => hermes_adapters}/filesystem/filesystem.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/filesystem/filesystem.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/filesystem/filesystem_io_client.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/filesystem/filesystem_mdm.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/filesystem/filesystem_mdm.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/filesystem/filesystem_mdm_singleton.cc (100%) rename {tasks/hermes_adapters/include/hermes_adapters => hermes_adapters}/interceptor.h (100%) rename {tasks/hermes_adapters/include/hermes_adapters => hermes_adapters}/mapper/abstract_mapper.h (100%) rename {tasks/hermes_adapters/include/hermes_adapters => hermes_adapters}/mapper/balanced_mapper.h (100%) rename {tasks/hermes_adapters/include/hermes_adapters => hermes_adapters}/mapper/mapper_factory.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/mpiio/CMakeLists.txt (98%) rename {tasks/hermes_adapters => hermes_adapters}/mpiio/mpiio_api.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/mpiio/mpiio_api.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/mpiio/mpiio_fs_api.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/mpiio/mpiio_io_client.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/mpiio/mpiio_io_client.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/posix/CMakeLists.txt (98%) rename {tasks/hermes_adapters => hermes_adapters}/posix/posix_api.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/posix/posix_api.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/posix/posix_fs_api.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/posix/posix_io_client.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/posix/posix_io_client.h (100%) rename {tasks/hermes_adapters/include/hermes_adapters => hermes_adapters}/real_api.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/stdio/CMakeLists.txt (98%) rename {tasks/hermes_adapters => hermes_adapters}/stdio/stdio_api.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/stdio/stdio_api.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/stdio/stdio_fs_api.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/stdio/stdio_io_client.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/stdio/stdio_io_client.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/vfd/CMakeLists.txt (99%) rename {tasks/hermes_adapters => hermes_adapters}/vfd/H5FDhermes.cc (100%) rename {tasks/hermes_adapters => hermes_adapters}/vfd/H5FDhermes.h (100%) rename {tasks/hermes_adapters => hermes_adapters}/vfd/README.md (100%) create mode 100644 hrun/CMakeLists.txt rename {config => hrun/config}/hrun_client_default.yaml (100%) rename {config => hrun/config}/hrun_server_default.yaml (100%) rename {include => hrun/include}/hrun/api/hrun_client.h (99%) rename {include => hrun/include}/hrun/api/hrun_runtime.h (98%) rename {include => hrun/include}/hrun/api/manager.h (89%) rename {include => hrun/include}/hrun/api/template/hrun_task_node_admin_root.template (100%) rename {include => hrun/include}/hrun/api/template/hrun_task_node_push_root.template (100%) rename {include => hrun/include}/hrun/config/config.h (100%) rename {include => hrun/include}/hrun/config/config_client.h (100%) rename {include => hrun/include}/hrun/config/config_client_default.h (76%) rename {include => hrun/include}/hrun/config/config_server.h (100%) rename {include => hrun/include}/hrun/config/config_server_default.h (97%) rename {include => hrun/include}/hrun/hrun_constants.h (76%) rename {include => hrun/include}/hrun/hrun_namespace.h (100%) rename {include => hrun/include}/hrun/hrun_types.h (99%) rename {include => hrun/include}/hrun/network/local_serialize.h (100%) rename {include => hrun/include}/hrun/network/rpc.h (100%) rename {include => hrun/include}/hrun/network/rpc_thallium.h (100%) rename {include => hrun/include}/hrun/network/serialize.h (100%) rename {include => hrun/include}/hrun/queue_manager/queue.h (100%) rename {include => hrun/include}/hrun/queue_manager/queue_factory.h (100%) rename {include => hrun/include}/hrun/queue_manager/queue_manager.h (100%) rename {include => hrun/include}/hrun/queue_manager/queue_manager_client.h (100%) rename {include => hrun/include}/hrun/queue_manager/queue_manager_runtime.h (100%) rename {include => hrun/include}/hrun/queue_manager/queues/hshm_queue.h (100%) rename {include => hrun/include}/hrun/task_registry/task.h (100%) rename {include => hrun/include}/hrun/task_registry/task_lib.h (100%) rename {include => hrun/include}/hrun/task_registry/task_registry.h (100%) rename {include => hrun/include}/hrun/work_orchestrator/affinity.h (100%) rename {include => hrun/include}/hrun/work_orchestrator/scheduler.h (100%) rename {include => hrun/include}/hrun/work_orchestrator/work_orchestrator.h (100%) rename {include => hrun/include}/hrun/work_orchestrator/worker.h (100%) create mode 100644 hrun/src/CMakeLists.txt rename {src => hrun/src}/config_client.cc (96%) rename {src => hrun/src}/config_server.cc (98%) rename {src => hrun/src}/hrun_client.cc (100%) rename {src => hrun/src}/hrun_runtime.cc (100%) rename {src => hrun/src}/hrun_start_runtime.cc (100%) rename {src => hrun/src}/hrun_stop_runtime.cc (100%) rename {src => hrun/src}/worker.cc (100%) rename {tasks_required => hrun/tasks_required}/CMakeLists.txt (100%) rename {tasks/hermes => hrun/tasks_required/TASK_NAME}/CMakeLists.txt (84%) rename {tasks_required => hrun/tasks_required}/TASK_NAME/include/TASK_NAME/TASK_NAME.h (100%) rename {tasks_required => hrun/tasks_required}/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h (100%) rename {tasks_required => hrun/tasks_required}/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h (100%) rename {tasks_required => hrun/tasks_required}/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml (100%) rename {tasks_required => hrun/tasks_required}/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h (100%) rename {tasks_required => hrun/tasks_required}/TASK_NAME/src/CMakeLists.txt (100%) rename {tasks_required => hrun/tasks_required}/TASK_NAME/src/TASK_NAME.cc (100%) rename {tasks_required => hrun/tasks_required}/hrun_admin/CMakeLists.txt (84%) rename {tasks_required => hrun/tasks_required}/hrun_admin/include/hrun_admin/hrun_admin.h (100%) rename {tasks_required => hrun/tasks_required}/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h (100%) rename {tasks_required => hrun/tasks_required}/hrun_admin/include/hrun_admin/hrun_admin_methods.h (100%) rename {tasks_required => hrun/tasks_required}/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml (100%) rename {tasks_required => hrun/tasks_required}/hrun_admin/include/hrun_admin/hrun_admin_tasks.h (100%) rename {tasks_required => hrun/tasks_required}/hrun_admin/src/CMakeLists.txt (96%) rename {tasks_required => hrun/tasks_required}/hrun_admin/src/hrun_admin.cc (100%) rename {tasks_required => hrun/tasks_required}/proc_queue/CMakeLists.txt (84%) rename {tasks_required => hrun/tasks_required}/proc_queue/include/proc_queue/proc_queue.h (100%) rename {tasks_required => hrun/tasks_required}/proc_queue/include/proc_queue/proc_queue_lib_exec.h (100%) rename {tasks_required => hrun/tasks_required}/proc_queue/include/proc_queue/proc_queue_methods.h (100%) rename {tasks_required => hrun/tasks_required}/proc_queue/include/proc_queue/proc_queue_methods.yaml (100%) rename {tasks_required => hrun/tasks_required}/proc_queue/include/proc_queue/proc_queue_tasks.h (100%) rename {tasks_required => hrun/tasks_required}/proc_queue/src/CMakeLists.txt (100%) rename {tasks_required => hrun/tasks_required}/proc_queue/src/proc_queue.cc (100%) rename {tasks_required => hrun/tasks_required}/remote_queue/CMakeLists.txt (88%) rename {tasks_required => hrun/tasks_required}/remote_queue/include/remote_queue/remote_queue.h (100%) rename {tasks_required => hrun/tasks_required}/remote_queue/include/remote_queue/remote_queue_lib_exec.h (100%) rename {tasks_required => hrun/tasks_required}/remote_queue/include/remote_queue/remote_queue_methods.h (100%) rename {tasks_required => hrun/tasks_required}/remote_queue/include/remote_queue/remote_queue_methods.yaml (100%) rename {tasks_required => hrun/tasks_required}/remote_queue/include/remote_queue/remote_queue_tasks.h (100%) rename {tasks_required => hrun/tasks_required}/remote_queue/src/CMakeLists.txt (100%) rename {tasks_required => hrun/tasks_required}/remote_queue/src/remote_queue.cc (100%) rename {tasks_required/TASK_NAME => hrun/tasks_required/small_message}/CMakeLists.txt (84%) rename {tasks_required => hrun/tasks_required}/small_message/include/small_message/small_message.h (100%) rename {tasks_required => hrun/tasks_required}/small_message/include/small_message/small_message_lib_exec.h (100%) rename {tasks_required => hrun/tasks_required}/small_message/include/small_message/small_message_methods.h (100%) rename {tasks_required => hrun/tasks_required}/small_message/include/small_message/small_message_methods.yaml (100%) rename {tasks_required => hrun/tasks_required}/small_message/include/small_message/small_message_tasks.h (100%) rename {tasks_required => hrun/tasks_required}/small_message/src/CMakeLists.txt (100%) rename {tasks_required => hrun/tasks_required}/small_message/src/small_message.cc (100%) create mode 100644 hrun/tasks_required/worch_proc_round_robin/CMakeLists.txt rename {tasks_required => hrun/tasks_required}/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h (100%) rename {tasks_required => hrun/tasks_required}/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h (100%) rename {tasks_required => hrun/tasks_required}/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h (100%) rename {tasks_required => hrun/tasks_required}/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml (100%) rename {tasks_required => hrun/tasks_required}/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h (100%) rename {tasks_required => hrun/tasks_required}/worch_proc_round_robin/src/CMakeLists.txt (100%) rename {tasks_required => hrun/tasks_required}/worch_proc_round_robin/src/worch_proc_round_robin.cc (100%) create mode 100644 hrun/tasks_required/worch_queue_round_robin/CMakeLists.txt rename {tasks_required => hrun/tasks_required}/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h (100%) rename {tasks_required => hrun/tasks_required}/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h (100%) rename {tasks_required => hrun/tasks_required}/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h (100%) rename {tasks_required => hrun/tasks_required}/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml (100%) rename {tasks_required => hrun/tasks_required}/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h (100%) rename {tasks_required => hrun/tasks_required}/worch_queue_round_robin/src/CMakeLists.txt (100%) rename {tasks_required => hrun/tasks_required}/worch_queue_round_robin/src/worch_queue_round_robin.cc (100%) rename {tasks/hermes/include => include}/hermes/adapter_types.h (100%) rename {tasks/hermes/include => include}/hermes/bucket.h (100%) rename {tasks/hermes/include => include}/hermes/config.h (100%) rename {tasks/hermes/include => include}/hermes/config_client.h (100%) rename {tasks/hermes/include => include}/hermes/config_client_default.h (100%) rename {tasks/hermes/include => include}/hermes/config_manager.h (100%) rename {tasks/hermes/include => include}/hermes/config_server.h (100%) rename {tasks/hermes/include => include}/hermes/config_server_default.h (100%) rename {tasks/hermes/include => include}/hermes/dpe/dpe.h (100%) rename {tasks/hermes/include => include}/hermes/dpe/dpe_factory.h (100%) rename {tasks/hermes/include => include}/hermes/dpe/minimize_io_time.h (100%) rename {tasks/hermes/include => include}/hermes/dpe/random.h (100%) rename {tasks/hermes/include => include}/hermes/dpe/round_robin.h (100%) rename {tasks/hermes/include => include}/hermes/hermes.h (100%) rename {tasks/hermes/include => include}/hermes/hermes_types.h (100%) rename {tasks/hermes/include => include}/hermes/slab_allocator.h (100%) rename {tasks/hermes/include => include}/hermes/status.h (100%) rename {tasks/hermes/include => include}/hermes/statuses.h (100%) rename tasks/hermes/src/config_manager.cc => src/hermes_config_manager.cc (100%) delete mode 100644 tasks/hermes/src/CMakeLists.txt delete mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h delete mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h delete mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h delete mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml delete mode 100644 tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h delete mode 100644 tasks/hermes_adapters/src/CMakeLists.txt delete mode 100644 tasks/hermes_adapters/src/hermes_adapters.cc delete mode 100644 tasks_required/small_message/CMakeLists.txt delete mode 100644 tasks_required/worch_proc_round_robin/CMakeLists.txt delete mode 100644 tasks_required/worch_queue_round_robin/CMakeLists.txt diff --git a/.gitignore b/.gitignore index c1b647efc..eb001e77a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,7 @@ cmake-build-* .idea .clang-format __pycache__/ -/src/hermes_adapters/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt +/hrun/src/hermes_adapters/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt /hermes_adapters/test/posix/Testing/Temporary/LastTest.log .cache @@ -39,7 +39,7 @@ Testing .gradle **/build/ -!src/**/build/ +!hrun/src/**/build/ # Ignore Gradle GUI config gradle-app.setting diff --git a/CMakeLists.txt b/CMakeLists.txt index f316ce7ae..71cf9b545 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,15 +135,16 @@ endif() include_directories(${HermesShm_INCLUDE_DIRS}) include_directories(${CMAKE_SOURCE_DIR}) include_directories(${CMAKE_SOURCE_DIR}/include) +# Required Task includes +include_directories(${CMAKE_SOURCE_DIR}/hrun/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/hrun_admin/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/small_message/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/remote_queue/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/worch_proc_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/worch_queue_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/proc_queue/include) # Task includes -include_directories(${CMAKE_SOURCE_DIR}/tasks_required) -include_directories(${CMAKE_SOURCE_DIR}/tasks_required/hrun_admin/include) -include_directories(${CMAKE_SOURCE_DIR}/tasks_required/small_message/include) -include_directories(${CMAKE_SOURCE_DIR}/tasks_required/remote_queue/include) -include_directories(${CMAKE_SOURCE_DIR}/tasks_required/worch_proc_round_robin/include) -include_directories(${CMAKE_SOURCE_DIR}/tasks_required/worch_queue_round_robin/include) -include_directories(${CMAKE_SOURCE_DIR}/tasks_required/proc_queue/include) - include_directories(${CMAKE_SOURCE_DIR}/tasks) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/bdev/include) @@ -155,7 +156,6 @@ include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_bucket_mdm/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_adapters/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/data_stager/include) include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_data_op/include) - # Test includes include_directories(${CMAKE_SOURCE_DIR}/test/unit) @@ -174,8 +174,9 @@ set(Hermes_RUNTIME_DEPS hrun_client hrun_runtime) set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) +add_subdirectory(hrun) add_subdirectory(src) -add_subdirectory(tasks_required) +add_subdirectory(hermes_adapters) add_subdirectory(tasks) add_subdirectory(benchmark) add_custom_target(lint COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/lint.sh ${CMAKE_SOURCE_DIR}) @@ -194,9 +195,9 @@ enable_testing() add_subdirectory(test) #----------------------------------------------------------------------------- -# Install LabStor Headers +# Install HRUN Headers #----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) +install(DIRECTORY hrun/include DESTINATION ${CMAKE_INSTALL_PREFIX}) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/CMake/HermesConfig.cmake diff --git a/codegen/codegen/util/paths.py b/codegen/codegen/util/paths.py index 31b641e4a..90ce03e41 100644 --- a/codegen/codegen/util/paths.py +++ b/codegen/codegen/util/paths.py @@ -1,11 +1,12 @@ import os,sys import pathlib -def GetLabstorRoot(): +def GetHermesRoot(): util_path = pathlib.Path(__file__).parent.resolve() code_generators_path = os.path.dirname(util_path) code_generators_path = os.path.dirname(code_generators_path) hermes_path = os.path.dirname(code_generators_path) return hermes_path -HRUN_ROOT=GetLabstorRoot() \ No newline at end of file +HERMES_ROOT = GetHermesRoot() +HRUN_ROOT = f'{HERMES_ROOT}/hrun' diff --git a/codegen/hermes_config b/codegen/hermes_config index 4bbdea946..0a35757bf 100755 --- a/codegen/hermes_config +++ b/codegen/hermes_config @@ -11,9 +11,8 @@ OUTPUT: """ from codegen.hrun_config.generator import create_config -from codegen.util.paths import HRUN_ROOT +from codegen.util.paths import HERMES_ROOT -HERMES_ROOT = f'{HRUN_ROOT}/tasks/hermes' create_config( path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", var_name="kHermesClientDefaultConfigStr", diff --git a/codegen/hrun_config b/codegen/hrun_config index 49183dd8f..e62fde410 100755 --- a/codegen/hrun_config +++ b/codegen/hrun_config @@ -15,14 +15,14 @@ from codegen.util.paths import HRUN_ROOT create_config( path=f"{HRUN_ROOT}/config/hrun_client_default.yaml", - var_name="kLabstorClientDefaultConfigStr", + var_name="kHrunClientDefaultConfigStr", config_path=f"{HRUN_ROOT}/include/hrun/config/config_client_default.h", macro_name="CLIENT" ) create_config( path=f"{HRUN_ROOT}/config/hrun_server_default.yaml", - var_name="kLabstorServerDefaultConfigStr", + var_name="kHrunServerDefaultConfigStr", config_path=f"{HRUN_ROOT}/include/hrun/config/config_server_default.h", macro_name="SERVER" ) \ No newline at end of file diff --git a/tasks/hermes/config/hermes_client_default.yaml b/config/hermes_client_default.yaml similarity index 100% rename from tasks/hermes/config/hermes_client_default.yaml rename to config/hermes_client_default.yaml diff --git a/tasks/hermes/config/hermes_server_default.yaml b/config/hermes_server_default.yaml similarity index 100% rename from tasks/hermes/config/hermes_server_default.yaml rename to config/hermes_server_default.yaml diff --git a/tasks/hermes_adapters/CMakeLists.txt b/hermes_adapters/CMakeLists.txt similarity index 82% rename from tasks/hermes_adapters/CMakeLists.txt rename to hermes_adapters/CMakeLists.txt index ad7a2441d..1e8a02eff 100644 --- a/tasks/hermes_adapters/CMakeLists.txt +++ b/hermes_adapters/CMakeLists.txt @@ -1,7 +1,6 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ -include_directories(include) add_subdirectory(filesystem) add_subdirectory(posix) if (HERMES_ENABLE_STDIO_ADAPTER) @@ -13,9 +12,8 @@ endif() if (HERMES_ENABLE_HDF5_ADAPTER) add_subdirectory(vfd) endif() -add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_adapters/include/hermes_adapters/adapter_constants.h b/hermes_adapters/adapter_constants.h similarity index 100% rename from tasks/hermes_adapters/include/hermes_adapters/adapter_constants.h rename to hermes_adapters/adapter_constants.h diff --git a/tasks/hermes_adapters/include/hermes_adapters/adapter_types.h b/hermes_adapters/adapter_types.h similarity index 100% rename from tasks/hermes_adapters/include/hermes_adapters/adapter_types.h rename to hermes_adapters/adapter_types.h diff --git a/tasks/hermes_adapters/filesystem/CMakeLists.txt b/hermes_adapters/filesystem/CMakeLists.txt similarity index 98% rename from tasks/hermes_adapters/filesystem/CMakeLists.txt rename to hermes_adapters/filesystem/CMakeLists.txt index bdcd61ac0..d9ea0f618 100644 --- a/tasks/hermes_adapters/filesystem/CMakeLists.txt +++ b/hermes_adapters/filesystem/CMakeLists.txt @@ -3,7 +3,7 @@ include_directories( ${HERMES_SRC_DIR} ${HERMES_ADAPTER_DIR} ${HERMES_IO_CLIENT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}) + .) # Create the metadata manager singleton + FS base class add_library(hermes_fs_base SHARED diff --git a/tasks/hermes_adapters/filesystem/filesystem.cc b/hermes_adapters/filesystem/filesystem.cc similarity index 100% rename from tasks/hermes_adapters/filesystem/filesystem.cc rename to hermes_adapters/filesystem/filesystem.cc diff --git a/tasks/hermes_adapters/filesystem/filesystem.h b/hermes_adapters/filesystem/filesystem.h similarity index 100% rename from tasks/hermes_adapters/filesystem/filesystem.h rename to hermes_adapters/filesystem/filesystem.h diff --git a/tasks/hermes_adapters/filesystem/filesystem_io_client.h b/hermes_adapters/filesystem/filesystem_io_client.h similarity index 100% rename from tasks/hermes_adapters/filesystem/filesystem_io_client.h rename to hermes_adapters/filesystem/filesystem_io_client.h diff --git a/tasks/hermes_adapters/filesystem/filesystem_mdm.cc b/hermes_adapters/filesystem/filesystem_mdm.cc similarity index 100% rename from tasks/hermes_adapters/filesystem/filesystem_mdm.cc rename to hermes_adapters/filesystem/filesystem_mdm.cc diff --git a/tasks/hermes_adapters/filesystem/filesystem_mdm.h b/hermes_adapters/filesystem/filesystem_mdm.h similarity index 100% rename from tasks/hermes_adapters/filesystem/filesystem_mdm.h rename to hermes_adapters/filesystem/filesystem_mdm.h diff --git a/tasks/hermes_adapters/filesystem/filesystem_mdm_singleton.cc b/hermes_adapters/filesystem/filesystem_mdm_singleton.cc similarity index 100% rename from tasks/hermes_adapters/filesystem/filesystem_mdm_singleton.cc rename to hermes_adapters/filesystem/filesystem_mdm_singleton.cc diff --git a/tasks/hermes_adapters/include/hermes_adapters/interceptor.h b/hermes_adapters/interceptor.h similarity index 100% rename from tasks/hermes_adapters/include/hermes_adapters/interceptor.h rename to hermes_adapters/interceptor.h diff --git a/tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h b/hermes_adapters/mapper/abstract_mapper.h similarity index 100% rename from tasks/hermes_adapters/include/hermes_adapters/mapper/abstract_mapper.h rename to hermes_adapters/mapper/abstract_mapper.h diff --git a/tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h b/hermes_adapters/mapper/balanced_mapper.h similarity index 100% rename from tasks/hermes_adapters/include/hermes_adapters/mapper/balanced_mapper.h rename to hermes_adapters/mapper/balanced_mapper.h diff --git a/tasks/hermes_adapters/include/hermes_adapters/mapper/mapper_factory.h b/hermes_adapters/mapper/mapper_factory.h similarity index 100% rename from tasks/hermes_adapters/include/hermes_adapters/mapper/mapper_factory.h rename to hermes_adapters/mapper/mapper_factory.h diff --git a/tasks/hermes_adapters/mpiio/CMakeLists.txt b/hermes_adapters/mpiio/CMakeLists.txt similarity index 98% rename from tasks/hermes_adapters/mpiio/CMakeLists.txt rename to hermes_adapters/mpiio/CMakeLists.txt index 448ba7593..b12f78c59 100644 --- a/tasks/hermes_adapters/mpiio/CMakeLists.txt +++ b/hermes_adapters/mpiio/CMakeLists.txt @@ -3,7 +3,7 @@ include_directories( ${HERMES_SRC_DIR} ${HERMES_ADAPTER_DIR} ${HERMES_IO_CLIENT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}) + .) # Create the MPIIO I/O client add_library(hermes_mpiio_io_client SHARED mpiio_io_client.cc) diff --git a/tasks/hermes_adapters/mpiio/mpiio_api.cc b/hermes_adapters/mpiio/mpiio_api.cc similarity index 100% rename from tasks/hermes_adapters/mpiio/mpiio_api.cc rename to hermes_adapters/mpiio/mpiio_api.cc diff --git a/tasks/hermes_adapters/mpiio/mpiio_api.h b/hermes_adapters/mpiio/mpiio_api.h similarity index 100% rename from tasks/hermes_adapters/mpiio/mpiio_api.h rename to hermes_adapters/mpiio/mpiio_api.h diff --git a/tasks/hermes_adapters/mpiio/mpiio_fs_api.h b/hermes_adapters/mpiio/mpiio_fs_api.h similarity index 100% rename from tasks/hermes_adapters/mpiio/mpiio_fs_api.h rename to hermes_adapters/mpiio/mpiio_fs_api.h diff --git a/tasks/hermes_adapters/mpiio/mpiio_io_client.cc b/hermes_adapters/mpiio/mpiio_io_client.cc similarity index 100% rename from tasks/hermes_adapters/mpiio/mpiio_io_client.cc rename to hermes_adapters/mpiio/mpiio_io_client.cc diff --git a/tasks/hermes_adapters/mpiio/mpiio_io_client.h b/hermes_adapters/mpiio/mpiio_io_client.h similarity index 100% rename from tasks/hermes_adapters/mpiio/mpiio_io_client.h rename to hermes_adapters/mpiio/mpiio_io_client.h diff --git a/tasks/hermes_adapters/posix/CMakeLists.txt b/hermes_adapters/posix/CMakeLists.txt similarity index 98% rename from tasks/hermes_adapters/posix/CMakeLists.txt rename to hermes_adapters/posix/CMakeLists.txt index 7d3aea406..bcc973f3c 100644 --- a/tasks/hermes_adapters/posix/CMakeLists.txt +++ b/hermes_adapters/posix/CMakeLists.txt @@ -3,7 +3,7 @@ include_directories( ${HERMES_SRC_DIR} ${HERMES_ADAPTER_DIR} ${HERMES_IO_CLIENT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}) + .) # Creates the POSIX I/O client add_library(hermes_posix_io_client SHARED posix_io_client.cc) diff --git a/tasks/hermes_adapters/posix/posix_api.cc b/hermes_adapters/posix/posix_api.cc similarity index 100% rename from tasks/hermes_adapters/posix/posix_api.cc rename to hermes_adapters/posix/posix_api.cc diff --git a/tasks/hermes_adapters/posix/posix_api.h b/hermes_adapters/posix/posix_api.h similarity index 100% rename from tasks/hermes_adapters/posix/posix_api.h rename to hermes_adapters/posix/posix_api.h diff --git a/tasks/hermes_adapters/posix/posix_fs_api.h b/hermes_adapters/posix/posix_fs_api.h similarity index 100% rename from tasks/hermes_adapters/posix/posix_fs_api.h rename to hermes_adapters/posix/posix_fs_api.h diff --git a/tasks/hermes_adapters/posix/posix_io_client.cc b/hermes_adapters/posix/posix_io_client.cc similarity index 100% rename from tasks/hermes_adapters/posix/posix_io_client.cc rename to hermes_adapters/posix/posix_io_client.cc diff --git a/tasks/hermes_adapters/posix/posix_io_client.h b/hermes_adapters/posix/posix_io_client.h similarity index 100% rename from tasks/hermes_adapters/posix/posix_io_client.h rename to hermes_adapters/posix/posix_io_client.h diff --git a/tasks/hermes_adapters/include/hermes_adapters/real_api.h b/hermes_adapters/real_api.h similarity index 100% rename from tasks/hermes_adapters/include/hermes_adapters/real_api.h rename to hermes_adapters/real_api.h diff --git a/tasks/hermes_adapters/stdio/CMakeLists.txt b/hermes_adapters/stdio/CMakeLists.txt similarity index 98% rename from tasks/hermes_adapters/stdio/CMakeLists.txt rename to hermes_adapters/stdio/CMakeLists.txt index 3cadb6c7b..a2da0baf1 100644 --- a/tasks/hermes_adapters/stdio/CMakeLists.txt +++ b/hermes_adapters/stdio/CMakeLists.txt @@ -3,7 +3,7 @@ include_directories( ${HERMES_SRC_DIR} ${HERMES_ADAPTER_DIR} ${HERMES_IO_CLIENT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}) + .) # Creates the STDIO I/O client add_library(hermes_stdio_io_client SHARED stdio_io_client.cc) diff --git a/tasks/hermes_adapters/stdio/stdio_api.cc b/hermes_adapters/stdio/stdio_api.cc similarity index 100% rename from tasks/hermes_adapters/stdio/stdio_api.cc rename to hermes_adapters/stdio/stdio_api.cc diff --git a/tasks/hermes_adapters/stdio/stdio_api.h b/hermes_adapters/stdio/stdio_api.h similarity index 100% rename from tasks/hermes_adapters/stdio/stdio_api.h rename to hermes_adapters/stdio/stdio_api.h diff --git a/tasks/hermes_adapters/stdio/stdio_fs_api.h b/hermes_adapters/stdio/stdio_fs_api.h similarity index 100% rename from tasks/hermes_adapters/stdio/stdio_fs_api.h rename to hermes_adapters/stdio/stdio_fs_api.h diff --git a/tasks/hermes_adapters/stdio/stdio_io_client.cc b/hermes_adapters/stdio/stdio_io_client.cc similarity index 100% rename from tasks/hermes_adapters/stdio/stdio_io_client.cc rename to hermes_adapters/stdio/stdio_io_client.cc diff --git a/tasks/hermes_adapters/stdio/stdio_io_client.h b/hermes_adapters/stdio/stdio_io_client.h similarity index 100% rename from tasks/hermes_adapters/stdio/stdio_io_client.h rename to hermes_adapters/stdio/stdio_io_client.h diff --git a/tasks/hermes_adapters/vfd/CMakeLists.txt b/hermes_adapters/vfd/CMakeLists.txt similarity index 99% rename from tasks/hermes_adapters/vfd/CMakeLists.txt rename to hermes_adapters/vfd/CMakeLists.txt index 24cc3effa..6ac9e99cd 100644 --- a/tasks/hermes_adapters/vfd/CMakeLists.txt +++ b/hermes_adapters/vfd/CMakeLists.txt @@ -162,14 +162,14 @@ endforeach() set(HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES ${HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES} PARENT_SCOPE) set(HDF5_HERMES_VFD_INCLUDES_BUILD_TIME - ${CMAKE_CURRENT_SOURCE_DIR} + . ${CMAKE_CURRENT_BINARY_DIR} ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} PARENT_SCOPE ) set(HDF5_HERMES_VFD_INCLUDES_INSTALL_TIME - ${CMAKE_CURRENT_SOURCE_DIR} + . ${CMAKE_CURRENT_BINARY_DIR} ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} PARENT_SCOPE) diff --git a/tasks/hermes_adapters/vfd/H5FDhermes.cc b/hermes_adapters/vfd/H5FDhermes.cc similarity index 100% rename from tasks/hermes_adapters/vfd/H5FDhermes.cc rename to hermes_adapters/vfd/H5FDhermes.cc diff --git a/tasks/hermes_adapters/vfd/H5FDhermes.h b/hermes_adapters/vfd/H5FDhermes.h similarity index 100% rename from tasks/hermes_adapters/vfd/H5FDhermes.h rename to hermes_adapters/vfd/H5FDhermes.h diff --git a/tasks/hermes_adapters/vfd/README.md b/hermes_adapters/vfd/README.md similarity index 100% rename from tasks/hermes_adapters/vfd/README.md rename to hermes_adapters/vfd/README.md diff --git a/hrun/CMakeLists.txt b/hrun/CMakeLists.txt new file mode 100644 index 000000000..76c70b8a1 --- /dev/null +++ b/hrun/CMakeLists.txt @@ -0,0 +1,160 @@ +cmake_minimum_required(VERSION 3.10) +project(hermes) + +#----------------------------------------------------------------------------- +# Define Options +#----------------------------------------------------------------------------- +option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" ON) +option(BUILD_MPI_TESTS "Build tests which depend on MPI" ON) +option(BUILD_OpenMP_TESTS "Build tests which depend on OpenMP" ON) +option(HERMES_ENABLE_COVERAGE "Check how well tests cover code" OFF) +option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" OFF) + +#----------------------------------------------------------------------------- +# Compiler Optimization +#----------------------------------------------------------------------------- +set(CMAKE_CXX_STANDARD 17) +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + message("IN DEBUG MODE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0") + add_compile_definitions(HERMES_LOG_VERBOSITY=10) +else() + message("IN RELEASE MODE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O3") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3") + add_compile_definitions(HERMES_LOG_VERBOSITY=1) +endif() +add_compile_options(-march=native -fomit-frame-pointer) + +#----------------------------------------------------------------------------- +# Targets built within this project are exported at Install time for use +# by other projects. +#----------------------------------------------------------------------------- +if(NOT HERMES_EXPORTED_TARGETS) + set(HERMES_EXPORTED_TARGETS "hrun-targets") +endif() + +#----------------------------------------------------------------------------- +# Find Packages +#----------------------------------------------------------------------------- + +# HermesShm +find_package(HermesShm CONFIG REQUIRED) +message(STATUS "found hermes_shm.h at ${HermesShm_INCLUDE_DIRS}") + +# YAML-CPP +find_package(yaml-cpp REQUIRED) +message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") + +# Catch2 +find_package(Catch2 3.0.1 REQUIRED) +message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") + +# MPICH +if(BUILD_MPI_TESTS) + find_package(MPI REQUIRED COMPONENTS C CXX) + message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") +endif() + +# OpenMP +if(BUILD_OpenMP_TESTS) + find_package(OpenMP REQUIRED COMPONENTS C CXX) + message(STATUS "found omp.h at ${OpenMP_CXX_INCLUDE_DIRS}") +endif() + +# Cereal +find_package(cereal REQUIRED) +if(cereal) + message(STATUS "found cereal") +endif() + +# Pkg-Config +find_package(PkgConfig REQUIRED) +if(PkgConfig) + message(STATUS "found pkg config") +endif() + +# Zeromq +#pkg_check_modules(ZMQ REQUIRED libzmq) +#include_directories(${ZMQ_INCLUDE_DIRS}) +#message("Found libzmq at: ${ZMQ_INCLUDE_DIRS}") + +# Thallium +find_package(thallium CONFIG REQUIRED) +if(thallium_FOUND) + message(STATUS "found thallium at ${thallium_DIR}") +endif() + +# Boost +find_package(Boost REQUIRED COMPONENTS regex system filesystem fiber REQUIRED) +if (Boost_FOUND) + message(STATUS "found boost at ${Boost_INCLUDE_DIRS}") +endif() +include_directories(${Boost_INCLUDE_DIRS}) +message("Boost: ${Boost_LIBRARIES}") + +#------------------------------------------------------------------------------ +# Setup CMake Environment +#------------------------------------------------------------------------------ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables.") +set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries.") + +#------------------------------------------------------------------------------ +# Setup install and output Directories +#------------------------------------------------------------------------------ +if(NOT HERMES_INSTALL_BIN_DIR) + set(HERMES_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) +endif() +if(NOT HERMES_INSTALL_LIB_DIR) + set(HERMES_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) +endif() +if(NOT HERMES_INSTALL_INCLUDE_DIR) + set(HERMES_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) +endif() +if(NOT HERMES_INSTALL_DATA_DIR) + set(HERMES_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) +endif() + +#----------------------------------------------------------------------------- +# Build Hermes Main Packages +#----------------------------------------------------------------------------- +# Main includes +include_directories(${HermesShm_INCLUDE_DIRS}) +include_directories(${CMAKE_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +# Task includes +include_directories(${CMAKE_SOURCE_DIR}/tasks_required) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/hrun_admin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/small_message/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/remote_queue/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/worch_proc_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/worch_queue_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks_required/proc_queue/include) + + +set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) +add_subdirectory(tasks_required) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Build + Enable Testing +#----------------------------------------------------------------------------- +# General function used to hook ctest to python test tool lib +function(pytest test_type test_name) + set(script ${CMAKE_SOURCE_DIR}/scripts/ci/py_hermes_ci/bin/run_test) + add_test(NAME ${test_name} + COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER}) +endfunction() + +enable_testing() + +#----------------------------------------------------------------------------- +# Install HRUN Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/config/hrun_client_default.yaml b/hrun/config/hrun_client_default.yaml similarity index 100% rename from config/hrun_client_default.yaml rename to hrun/config/hrun_client_default.yaml diff --git a/config/hrun_server_default.yaml b/hrun/config/hrun_server_default.yaml similarity index 100% rename from config/hrun_server_default.yaml rename to hrun/config/hrun_server_default.yaml diff --git a/include/hrun/api/hrun_client.h b/hrun/include/hrun/api/hrun_client.h similarity index 99% rename from include/hrun/api/hrun_client.h rename to hrun/include/hrun/api/hrun_client.h index 72d9aac3a..87d41fd06 100644 --- a/include/hrun/api/hrun_client.h +++ b/hrun/include/hrun/api/hrun_client.h @@ -34,7 +34,7 @@ class Client : public ConfigurationManager { if (is_initialized_) { return this; } - mode_ = LabstorMode::kClient; + mode_ = HrunMode::kClient; is_being_initialized_ = true; ClientInit(std::move(server_config_path), std::move(client_config_path), @@ -70,7 +70,7 @@ class Client : public ConfigurationManager { server_config_.queue_manager_.shm_name_); } main_alloc_ = mem_mngr->GetAllocator(main_alloc_id_); - header_ = main_alloc_->GetCustomHeader(); + header_ = main_alloc_->GetCustomHeader(); unique_ = &header_->unique_; node_id_ = header_->node_id_; } diff --git a/include/hrun/api/hrun_runtime.h b/hrun/include/hrun/api/hrun_runtime.h similarity index 98% rename from include/hrun/api/hrun_runtime.h rename to hrun/include/hrun/api/hrun_runtime.h index bfc806217..e057d6e08 100644 --- a/include/hrun/api/hrun_runtime.h +++ b/hrun/include/hrun/api/hrun_runtime.h @@ -45,7 +45,7 @@ class Runtime : public ConfigurationManager { if (is_initialized_) { return this; } - mode_ = LabstorMode::kServer; + mode_ = HrunMode::kServer; is_being_initialized_ = true; ServerInit(std::move(server_config_path)); is_initialized_ = true; @@ -141,8 +141,8 @@ class Runtime : public ConfigurationManager { mem_mngr->CreateAllocator( server_config_.queue_manager_.shm_name_, main_alloc_id_, - sizeof(LabstorShm)); - header_ = main_alloc_->GetCustomHeader(); + sizeof(HrunShm)); + header_ = main_alloc_->GetCustomHeader(); } /** Finalize Hermes explicitly */ diff --git a/include/hrun/api/manager.h b/hrun/include/hrun/api/manager.h similarity index 89% rename from include/hrun/api/manager.h rename to hrun/include/hrun/api/manager.h index 41948d734..b24c45add 100644 --- a/include/hrun/api/manager.h +++ b/hrun/include/hrun/api/manager.h @@ -13,8 +13,8 @@ namespace hrun { -/** Shared-memory header for LabStor */ -struct LabstorShm { +/** Shared-memory header for HRUN */ +struct HrunShm { u32 node_id_; QueueManagerShm queue_manager_; std::atomic unique_; @@ -24,8 +24,8 @@ struct LabstorShm { /** The configuration used inherited by runtime + client */ class ConfigurationManager { public: - LabstorMode mode_; - LabstorShm *header_; + HrunMode mode_; + HrunShm *header_; ClientConfig client_config_; ServerConfig server_config_; static inline const hipc::allocator_id_t main_alloc_id_ = @@ -47,13 +47,13 @@ class ConfigurationManager { /** Destructor */ ~ConfigurationManager() {} - /** Whether or not Labstor is currently being initialized */ + /** Whether or not Hrun is currently being initialized */ bool IsBeingInitialized() { return is_being_initialized_; } - /** Whether or not Labstor is initialized */ + /** Whether or not Hrun is initialized */ bool IsInitialized() { return is_initialized_; } - /** Whether or not Labstor is finalized */ + /** Whether or not Hrun is finalized */ bool IsTerminated() { return is_terminated_; } /** Load the server-side configuration */ diff --git a/include/hrun/api/template/hrun_task_node_admin_root.template b/hrun/include/hrun/api/template/hrun_task_node_admin_root.template similarity index 100% rename from include/hrun/api/template/hrun_task_node_admin_root.template rename to hrun/include/hrun/api/template/hrun_task_node_admin_root.template diff --git a/include/hrun/api/template/hrun_task_node_push_root.template b/hrun/include/hrun/api/template/hrun_task_node_push_root.template similarity index 100% rename from include/hrun/api/template/hrun_task_node_push_root.template rename to hrun/include/hrun/api/template/hrun_task_node_push_root.template diff --git a/include/hrun/config/config.h b/hrun/include/hrun/config/config.h similarity index 100% rename from include/hrun/config/config.h rename to hrun/include/hrun/config/config.h diff --git a/include/hrun/config/config_client.h b/hrun/include/hrun/config/config_client.h similarity index 100% rename from include/hrun/config/config_client.h rename to hrun/include/hrun/config/config_client.h diff --git a/include/hrun/config/config_client_default.h b/hrun/include/hrun/config/config_client_default.h similarity index 76% rename from include/hrun/config/config_client_default.h rename to hrun/include/hrun/config/config_client_default.h index 2056e1ebe..2d4ad8bf1 100644 --- a/include/hrun/config/config_client_default.h +++ b/hrun/include/hrun/config/config_client_default.h @@ -1,5 +1,5 @@ #ifndef HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ #define HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ -const char* kLabstorClientDefaultConfigStr = +const char* kHrunClientDefaultConfigStr = "thread_model: kStd\n"; #endif // HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/include/hrun/config/config_server.h b/hrun/include/hrun/config/config_server.h similarity index 100% rename from include/hrun/config/config_server.h rename to hrun/include/hrun/config/config_server.h diff --git a/include/hrun/config/config_server_default.h b/hrun/include/hrun/config/config_server_default.h similarity index 97% rename from include/hrun/config/config_server_default.h rename to hrun/include/hrun/config/config_server_default.h index 6a112d3f2..2cc61dccf 100644 --- a/include/hrun/config/config_server_default.h +++ b/hrun/include/hrun/config/config_server_default.h @@ -1,6 +1,6 @@ #ifndef HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ #define HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ -const char* kLabstorServerDefaultConfigStr = +const char* kHrunServerDefaultConfigStr = "### Runtime orchestration settings\n" "work_orchestrator:\n" " # The number of worker threads to spawn\n" diff --git a/include/hrun/hrun_constants.h b/hrun/include/hrun/hrun_constants.h similarity index 76% rename from include/hrun/hrun_constants.h rename to hrun/include/hrun/hrun_constants.h index 8c710c8c1..8183cb2a7 100644 --- a/include/hrun/hrun_constants.h +++ b/hrun/include/hrun/hrun_constants.h @@ -11,8 +11,8 @@ namespace hrun { class Constants { public: - inline static const std::string kClientConfEnv = "HRUN_CLIENT_CONF"; - inline static const std::string kServerConfEnv = "HRUN_SERVER_CONF"; + inline static const std::string kClientConfEnv = "HERMES_CLIENT_CONF"; + inline static const std::string kServerConfEnv = "HERMES_CONF"; static std::string GetEnvSafe(const std::string &env_name) { char *data = getenv(env_name.c_str()); diff --git a/include/hrun/hrun_namespace.h b/hrun/include/hrun/hrun_namespace.h similarity index 100% rename from include/hrun/hrun_namespace.h rename to hrun/include/hrun/hrun_namespace.h diff --git a/include/hrun/hrun_types.h b/hrun/include/hrun/hrun_types.h similarity index 99% rename from include/hrun/hrun_types.h rename to hrun/include/hrun/hrun_types.h index 7d9947097..f0c72cab5 100644 --- a/include/hrun/hrun_types.h +++ b/hrun/include/hrun/hrun_types.h @@ -54,8 +54,8 @@ using hshm::ScopedRwReadLock; using hshm::ScopedRwWriteLock; using hipc::LPointer; -/** Determine the mode that LabStor is initialized for */ -enum class LabstorMode { +/** Determine the mode that HRUN is initialized for */ +enum class HrunMode { kNone, kClient, kServer diff --git a/include/hrun/network/local_serialize.h b/hrun/include/hrun/network/local_serialize.h similarity index 100% rename from include/hrun/network/local_serialize.h rename to hrun/include/hrun/network/local_serialize.h diff --git a/include/hrun/network/rpc.h b/hrun/include/hrun/network/rpc.h similarity index 100% rename from include/hrun/network/rpc.h rename to hrun/include/hrun/network/rpc.h diff --git a/include/hrun/network/rpc_thallium.h b/hrun/include/hrun/network/rpc_thallium.h similarity index 100% rename from include/hrun/network/rpc_thallium.h rename to hrun/include/hrun/network/rpc_thallium.h diff --git a/include/hrun/network/serialize.h b/hrun/include/hrun/network/serialize.h similarity index 100% rename from include/hrun/network/serialize.h rename to hrun/include/hrun/network/serialize.h diff --git a/include/hrun/queue_manager/queue.h b/hrun/include/hrun/queue_manager/queue.h similarity index 100% rename from include/hrun/queue_manager/queue.h rename to hrun/include/hrun/queue_manager/queue.h diff --git a/include/hrun/queue_manager/queue_factory.h b/hrun/include/hrun/queue_manager/queue_factory.h similarity index 100% rename from include/hrun/queue_manager/queue_factory.h rename to hrun/include/hrun/queue_manager/queue_factory.h diff --git a/include/hrun/queue_manager/queue_manager.h b/hrun/include/hrun/queue_manager/queue_manager.h similarity index 100% rename from include/hrun/queue_manager/queue_manager.h rename to hrun/include/hrun/queue_manager/queue_manager.h diff --git a/include/hrun/queue_manager/queue_manager_client.h b/hrun/include/hrun/queue_manager/queue_manager_client.h similarity index 100% rename from include/hrun/queue_manager/queue_manager_client.h rename to hrun/include/hrun/queue_manager/queue_manager_client.h diff --git a/include/hrun/queue_manager/queue_manager_runtime.h b/hrun/include/hrun/queue_manager/queue_manager_runtime.h similarity index 100% rename from include/hrun/queue_manager/queue_manager_runtime.h rename to hrun/include/hrun/queue_manager/queue_manager_runtime.h diff --git a/include/hrun/queue_manager/queues/hshm_queue.h b/hrun/include/hrun/queue_manager/queues/hshm_queue.h similarity index 100% rename from include/hrun/queue_manager/queues/hshm_queue.h rename to hrun/include/hrun/queue_manager/queues/hshm_queue.h diff --git a/include/hrun/task_registry/task.h b/hrun/include/hrun/task_registry/task.h similarity index 100% rename from include/hrun/task_registry/task.h rename to hrun/include/hrun/task_registry/task.h diff --git a/include/hrun/task_registry/task_lib.h b/hrun/include/hrun/task_registry/task_lib.h similarity index 100% rename from include/hrun/task_registry/task_lib.h rename to hrun/include/hrun/task_registry/task_lib.h diff --git a/include/hrun/task_registry/task_registry.h b/hrun/include/hrun/task_registry/task_registry.h similarity index 100% rename from include/hrun/task_registry/task_registry.h rename to hrun/include/hrun/task_registry/task_registry.h diff --git a/include/hrun/work_orchestrator/affinity.h b/hrun/include/hrun/work_orchestrator/affinity.h similarity index 100% rename from include/hrun/work_orchestrator/affinity.h rename to hrun/include/hrun/work_orchestrator/affinity.h diff --git a/include/hrun/work_orchestrator/scheduler.h b/hrun/include/hrun/work_orchestrator/scheduler.h similarity index 100% rename from include/hrun/work_orchestrator/scheduler.h rename to hrun/include/hrun/work_orchestrator/scheduler.h diff --git a/include/hrun/work_orchestrator/work_orchestrator.h b/hrun/include/hrun/work_orchestrator/work_orchestrator.h similarity index 100% rename from include/hrun/work_orchestrator/work_orchestrator.h rename to hrun/include/hrun/work_orchestrator/work_orchestrator.h diff --git a/include/hrun/work_orchestrator/worker.h b/hrun/include/hrun/work_orchestrator/worker.h similarity index 100% rename from include/hrun/work_orchestrator/worker.h rename to hrun/include/hrun/work_orchestrator/worker.h diff --git a/hrun/src/CMakeLists.txt b/hrun/src/CMakeLists.txt new file mode 100644 index 000000000..c7e819a91 --- /dev/null +++ b/hrun/src/CMakeLists.txt @@ -0,0 +1,110 @@ +#------------------------------------------------------------------------------ +# External dependencies +#------------------------------------------------------------------------------ +# None for now + +#------------------------------------------------------------------------------ +# Build Hrun Client Library +#------------------------------------------------------------------------------ +add_library(hrun_client SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/config_client.cc + ${CMAKE_CURRENT_SOURCE_DIR}/config_server.cc + ${CMAKE_CURRENT_SOURCE_DIR}/hrun_client.cc) +target_link_libraries(hrun_client + ${HermesShm_LIBRARIES} + yaml-cpp + cereal::cereal + thallium + -ldl -lrt -lc -pthread) + +#------------------------------------------------------------------------------ +# Build Hrun Runtime Library +#------------------------------------------------------------------------------ +add_library(hrun_runtime + worker.cc + hrun_runtime.cc) +add_dependencies(hrun_runtime ${Hermes_CLIENT_DEPS}) +target_link_libraries(hrun_runtime thallium ${Hermes_CLIENT_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Build Hrun Runtime Start Function +#------------------------------------------------------------------------------ +add_executable(hrun_start_runtime + hrun_start_runtime.cc) +add_dependencies(hrun_start_runtime ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hrun_start_runtime ${Hermes_RUNTIME_LIBRARIES}) + +#------------------------------------------------------------------------------ +# Build HRUN Runtime Stop Function +#------------------------------------------------------------------------------ +add_executable(hrun_stop_runtime hrun_stop_runtime.cc) +add_dependencies(hrun_stop_runtime ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hrun_stop_runtime ${Hermes_RUNTIME_LIBRARIES}) + +#----------------------------------------------------------------------------- +# Add file(s) to CMake Install +#----------------------------------------------------------------------------- +install( + FILES + ${HERMES_HEADERS} + DESTINATION + ${HERMES_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install +#----------------------------------------------------------------------------- +install( + TARGETS + hrun_client + hrun_runtime + hrun_start_runtime + hrun_stop_runtime + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${HERMES_EXPORTED_TARGETS} + DESTINATION + ${HERMES_INSTALL_DATA_DIR}/cmake/hermes + FILE + ${HERMES_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project +#----------------------------------------------------------------------------- +set(HERMES_EXPORTED_LIBS + hrun_client + hrun_runtime + hrun_start_runtime + hrun_stop_runtime + ${HERMES_EXPORTED_LIBS}) +if(NOT HERMES_EXTERNALLY_CONFIGURED) + EXPORT ( + TARGETS + ${HERMES_EXPORTED_LIBS} + FILE + ${HERMES_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Coverage +#------------------------------------------------------------------------------ +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(hrun_client) + set_coverage_flags(hrun_runtime) + set_coverage_flags(hrun_start_runtime) + set_coverage_flags(hrun_stop_runtime) +endif() \ No newline at end of file diff --git a/src/config_client.cc b/hrun/src/config_client.cc similarity index 96% rename from src/config_client.cc rename to hrun/src/config_client.cc index 02c53cc65..abc17d266 100644 --- a/src/config_client.cc +++ b/hrun/src/config_client.cc @@ -28,7 +28,7 @@ void ClientConfig::ParseYAML(YAML::Node &yaml_conf) { /** Load the default configuration */ void ClientConfig::LoadDefault() { - LoadText(kLabstorClientDefaultConfigStr, false); + LoadText(kHrunClientDefaultConfigStr, false); } } // namespace hrun::config diff --git a/src/config_server.cc b/hrun/src/config_server.cc similarity index 98% rename from src/config_server.cc rename to hrun/src/config_server.cc index 736f9d283..5cb7a478c 100644 --- a/src/config_server.cc +++ b/hrun/src/config_server.cc @@ -100,7 +100,7 @@ void ServerConfig::ParseYAML(YAML::Node &yaml_conf) { /** Load the default configuration */ void ServerConfig::LoadDefault() { - LoadText(kLabstorServerDefaultConfigStr, false); + LoadText(kHrunServerDefaultConfigStr, false); } } // namespace hrun::config diff --git a/src/hrun_client.cc b/hrun/src/hrun_client.cc similarity index 100% rename from src/hrun_client.cc rename to hrun/src/hrun_client.cc diff --git a/src/hrun_runtime.cc b/hrun/src/hrun_runtime.cc similarity index 100% rename from src/hrun_runtime.cc rename to hrun/src/hrun_runtime.cc diff --git a/src/hrun_start_runtime.cc b/hrun/src/hrun_start_runtime.cc similarity index 100% rename from src/hrun_start_runtime.cc rename to hrun/src/hrun_start_runtime.cc diff --git a/src/hrun_stop_runtime.cc b/hrun/src/hrun_stop_runtime.cc similarity index 100% rename from src/hrun_stop_runtime.cc rename to hrun/src/hrun_stop_runtime.cc diff --git a/src/worker.cc b/hrun/src/worker.cc similarity index 100% rename from src/worker.cc rename to hrun/src/worker.cc diff --git a/tasks_required/CMakeLists.txt b/hrun/tasks_required/CMakeLists.txt similarity index 100% rename from tasks_required/CMakeLists.txt rename to hrun/tasks_required/CMakeLists.txt diff --git a/tasks/hermes/CMakeLists.txt b/hrun/tasks_required/TASK_NAME/CMakeLists.txt similarity index 84% rename from tasks/hermes/CMakeLists.txt rename to hrun/tasks_required/TASK_NAME/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks/hermes/CMakeLists.txt +++ b/hrun/tasks_required/TASK_NAME/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h b/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h similarity index 100% rename from tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h rename to hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h b/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h similarity index 100% rename from tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h rename to hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_lib_exec.h diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h b/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h similarity index 100% rename from tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h rename to hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml b/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml similarity index 100% rename from tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml rename to hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml diff --git a/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h b/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h similarity index 100% rename from tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h rename to hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_tasks.h diff --git a/tasks_required/TASK_NAME/src/CMakeLists.txt b/hrun/tasks_required/TASK_NAME/src/CMakeLists.txt similarity index 100% rename from tasks_required/TASK_NAME/src/CMakeLists.txt rename to hrun/tasks_required/TASK_NAME/src/CMakeLists.txt diff --git a/tasks_required/TASK_NAME/src/TASK_NAME.cc b/hrun/tasks_required/TASK_NAME/src/TASK_NAME.cc similarity index 100% rename from tasks_required/TASK_NAME/src/TASK_NAME.cc rename to hrun/tasks_required/TASK_NAME/src/TASK_NAME.cc diff --git a/tasks_required/hrun_admin/CMakeLists.txt b/hrun/tasks_required/hrun_admin/CMakeLists.txt similarity index 84% rename from tasks_required/hrun_admin/CMakeLists.txt rename to hrun/tasks_required/hrun_admin/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks_required/hrun_admin/CMakeLists.txt +++ b/hrun/tasks_required/hrun_admin/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h similarity index 100% rename from tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h rename to hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h diff --git a/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h similarity index 100% rename from tasks_required/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h rename to hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_lib_exec.h diff --git a/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h similarity index 100% rename from tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h rename to hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h diff --git a/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml similarity index 100% rename from tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml rename to hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml diff --git a/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h similarity index 100% rename from tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h rename to hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h diff --git a/tasks_required/hrun_admin/src/CMakeLists.txt b/hrun/tasks_required/hrun_admin/src/CMakeLists.txt similarity index 96% rename from tasks_required/hrun_admin/src/CMakeLists.txt rename to hrun/tasks_required/hrun_admin/src/CMakeLists.txt index 39b07560c..bd40c8706 100644 --- a/tasks_required/hrun_admin/src/CMakeLists.txt +++ b/hrun/tasks_required/hrun_admin/src/CMakeLists.txt @@ -1,5 +1,5 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ add_library(hrun_admin SHARED ${CMAKE_CURRENT_SOURCE_DIR}/hrun_admin.cc) @@ -7,7 +7,7 @@ add_dependencies(hrun_admin ${Hermes_RUNTIME_DEPS}) target_link_libraries(hrun_admin ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ -# Install Labstor Admin Task Library +# Install Hrun Admin Task Library #------------------------------------------------------------------------------ install( TARGETS diff --git a/tasks_required/hrun_admin/src/hrun_admin.cc b/hrun/tasks_required/hrun_admin/src/hrun_admin.cc similarity index 100% rename from tasks_required/hrun_admin/src/hrun_admin.cc rename to hrun/tasks_required/hrun_admin/src/hrun_admin.cc diff --git a/tasks_required/proc_queue/CMakeLists.txt b/hrun/tasks_required/proc_queue/CMakeLists.txt similarity index 84% rename from tasks_required/proc_queue/CMakeLists.txt rename to hrun/tasks_required/proc_queue/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks_required/proc_queue/CMakeLists.txt +++ b/hrun/tasks_required/proc_queue/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue.h b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue.h similarity index 100% rename from tasks_required/proc_queue/include/proc_queue/proc_queue.h rename to hrun/tasks_required/proc_queue/include/proc_queue/proc_queue.h diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h similarity index 100% rename from tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h rename to hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_lib_exec.h diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h similarity index 100% rename from tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h rename to hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml similarity index 100% rename from tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml rename to hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml diff --git a/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h similarity index 100% rename from tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h rename to hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h diff --git a/tasks_required/proc_queue/src/CMakeLists.txt b/hrun/tasks_required/proc_queue/src/CMakeLists.txt similarity index 100% rename from tasks_required/proc_queue/src/CMakeLists.txt rename to hrun/tasks_required/proc_queue/src/CMakeLists.txt diff --git a/tasks_required/proc_queue/src/proc_queue.cc b/hrun/tasks_required/proc_queue/src/proc_queue.cc similarity index 100% rename from tasks_required/proc_queue/src/proc_queue.cc rename to hrun/tasks_required/proc_queue/src/proc_queue.cc diff --git a/tasks_required/remote_queue/CMakeLists.txt b/hrun/tasks_required/remote_queue/CMakeLists.txt similarity index 88% rename from tasks_required/remote_queue/CMakeLists.txt rename to hrun/tasks_required/remote_queue/CMakeLists.txt index da1eafa4d..00c53a80d 100644 --- a/tasks_required/remote_queue/CMakeLists.txt +++ b/hrun/tasks_required/remote_queue/CMakeLists.txt @@ -3,12 +3,12 @@ # message("LibFabric found at: ${libfabric_INCLUDE_DIRS} with libs ${libfabric_LIBRARIES} in dir ${libfabric_LIBRARY_DIRS}") #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue.h b/hrun/tasks_required/remote_queue/include/remote_queue/remote_queue.h similarity index 100% rename from tasks_required/remote_queue/include/remote_queue/remote_queue.h rename to hrun/tasks_required/remote_queue/include/remote_queue/remote_queue.h diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h b/hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h similarity index 100% rename from tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h rename to hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_lib_exec.h diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h b/hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h similarity index 100% rename from tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h rename to hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml b/hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml similarity index 100% rename from tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml rename to hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml diff --git a/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h b/hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h similarity index 100% rename from tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h rename to hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_tasks.h diff --git a/tasks_required/remote_queue/src/CMakeLists.txt b/hrun/tasks_required/remote_queue/src/CMakeLists.txt similarity index 100% rename from tasks_required/remote_queue/src/CMakeLists.txt rename to hrun/tasks_required/remote_queue/src/CMakeLists.txt diff --git a/tasks_required/remote_queue/src/remote_queue.cc b/hrun/tasks_required/remote_queue/src/remote_queue.cc similarity index 100% rename from tasks_required/remote_queue/src/remote_queue.cc rename to hrun/tasks_required/remote_queue/src/remote_queue.cc diff --git a/tasks_required/TASK_NAME/CMakeLists.txt b/hrun/tasks_required/small_message/CMakeLists.txt similarity index 84% rename from tasks_required/TASK_NAME/CMakeLists.txt rename to hrun/tasks_required/small_message/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks_required/TASK_NAME/CMakeLists.txt +++ b/hrun/tasks_required/small_message/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/small_message/include/small_message/small_message.h b/hrun/tasks_required/small_message/include/small_message/small_message.h similarity index 100% rename from tasks_required/small_message/include/small_message/small_message.h rename to hrun/tasks_required/small_message/include/small_message/small_message.h diff --git a/tasks_required/small_message/include/small_message/small_message_lib_exec.h b/hrun/tasks_required/small_message/include/small_message/small_message_lib_exec.h similarity index 100% rename from tasks_required/small_message/include/small_message/small_message_lib_exec.h rename to hrun/tasks_required/small_message/include/small_message/small_message_lib_exec.h diff --git a/tasks_required/small_message/include/small_message/small_message_methods.h b/hrun/tasks_required/small_message/include/small_message/small_message_methods.h similarity index 100% rename from tasks_required/small_message/include/small_message/small_message_methods.h rename to hrun/tasks_required/small_message/include/small_message/small_message_methods.h diff --git a/tasks_required/small_message/include/small_message/small_message_methods.yaml b/hrun/tasks_required/small_message/include/small_message/small_message_methods.yaml similarity index 100% rename from tasks_required/small_message/include/small_message/small_message_methods.yaml rename to hrun/tasks_required/small_message/include/small_message/small_message_methods.yaml diff --git a/tasks_required/small_message/include/small_message/small_message_tasks.h b/hrun/tasks_required/small_message/include/small_message/small_message_tasks.h similarity index 100% rename from tasks_required/small_message/include/small_message/small_message_tasks.h rename to hrun/tasks_required/small_message/include/small_message/small_message_tasks.h diff --git a/tasks_required/small_message/src/CMakeLists.txt b/hrun/tasks_required/small_message/src/CMakeLists.txt similarity index 100% rename from tasks_required/small_message/src/CMakeLists.txt rename to hrun/tasks_required/small_message/src/CMakeLists.txt diff --git a/tasks_required/small_message/src/small_message.cc b/hrun/tasks_required/small_message/src/small_message.cc similarity index 100% rename from tasks_required/small_message/src/small_message.cc rename to hrun/tasks_required/small_message/src/small_message.cc diff --git a/hrun/tasks_required/worch_proc_round_robin/CMakeLists.txt b/hrun/tasks_required/worch_proc_round_robin/CMakeLists.txt new file mode 100644 index 000000000..d48611782 --- /dev/null +++ b/hrun/tasks_required/worch_proc_round_robin/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Hrun Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install HRUN Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h b/hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h similarity index 100% rename from tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h rename to hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h b/hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h similarity index 100% rename from tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h rename to hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_lib_exec.h diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h b/hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h similarity index 100% rename from tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h rename to hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml b/hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml similarity index 100% rename from tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml rename to hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml diff --git a/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h b/hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h similarity index 100% rename from tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h rename to hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h diff --git a/tasks_required/worch_proc_round_robin/src/CMakeLists.txt b/hrun/tasks_required/worch_proc_round_robin/src/CMakeLists.txt similarity index 100% rename from tasks_required/worch_proc_round_robin/src/CMakeLists.txt rename to hrun/tasks_required/worch_proc_round_robin/src/CMakeLists.txt diff --git a/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc b/hrun/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc similarity index 100% rename from tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc rename to hrun/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc diff --git a/hrun/tasks_required/worch_queue_round_robin/CMakeLists.txt b/hrun/tasks_required/worch_queue_round_robin/CMakeLists.txt new file mode 100644 index 000000000..d48611782 --- /dev/null +++ b/hrun/tasks_required/worch_queue_round_robin/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------ +# Build Hrun Admin Task Library +#------------------------------------------------------------------------------ +include_directories(include) +add_subdirectory(src) + +#----------------------------------------------------------------------------- +# Install HRUN Admin Task Library Headers +#----------------------------------------------------------------------------- +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h b/hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h similarity index 100% rename from tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h rename to hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h b/hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h similarity index 100% rename from tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h rename to hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_lib_exec.h diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h b/hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h similarity index 100% rename from tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h rename to hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml b/hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml similarity index 100% rename from tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml rename to hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml diff --git a/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h b/hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h similarity index 100% rename from tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h rename to hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h diff --git a/tasks_required/worch_queue_round_robin/src/CMakeLists.txt b/hrun/tasks_required/worch_queue_round_robin/src/CMakeLists.txt similarity index 100% rename from tasks_required/worch_queue_round_robin/src/CMakeLists.txt rename to hrun/tasks_required/worch_queue_round_robin/src/CMakeLists.txt diff --git a/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc similarity index 100% rename from tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc rename to hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc diff --git a/tasks/hermes/include/hermes/adapter_types.h b/include/hermes/adapter_types.h similarity index 100% rename from tasks/hermes/include/hermes/adapter_types.h rename to include/hermes/adapter_types.h diff --git a/tasks/hermes/include/hermes/bucket.h b/include/hermes/bucket.h similarity index 100% rename from tasks/hermes/include/hermes/bucket.h rename to include/hermes/bucket.h diff --git a/tasks/hermes/include/hermes/config.h b/include/hermes/config.h similarity index 100% rename from tasks/hermes/include/hermes/config.h rename to include/hermes/config.h diff --git a/tasks/hermes/include/hermes/config_client.h b/include/hermes/config_client.h similarity index 100% rename from tasks/hermes/include/hermes/config_client.h rename to include/hermes/config_client.h diff --git a/tasks/hermes/include/hermes/config_client_default.h b/include/hermes/config_client_default.h similarity index 100% rename from tasks/hermes/include/hermes/config_client_default.h rename to include/hermes/config_client_default.h diff --git a/tasks/hermes/include/hermes/config_manager.h b/include/hermes/config_manager.h similarity index 100% rename from tasks/hermes/include/hermes/config_manager.h rename to include/hermes/config_manager.h diff --git a/tasks/hermes/include/hermes/config_server.h b/include/hermes/config_server.h similarity index 100% rename from tasks/hermes/include/hermes/config_server.h rename to include/hermes/config_server.h diff --git a/tasks/hermes/include/hermes/config_server_default.h b/include/hermes/config_server_default.h similarity index 100% rename from tasks/hermes/include/hermes/config_server_default.h rename to include/hermes/config_server_default.h diff --git a/tasks/hermes/include/hermes/dpe/dpe.h b/include/hermes/dpe/dpe.h similarity index 100% rename from tasks/hermes/include/hermes/dpe/dpe.h rename to include/hermes/dpe/dpe.h diff --git a/tasks/hermes/include/hermes/dpe/dpe_factory.h b/include/hermes/dpe/dpe_factory.h similarity index 100% rename from tasks/hermes/include/hermes/dpe/dpe_factory.h rename to include/hermes/dpe/dpe_factory.h diff --git a/tasks/hermes/include/hermes/dpe/minimize_io_time.h b/include/hermes/dpe/minimize_io_time.h similarity index 100% rename from tasks/hermes/include/hermes/dpe/minimize_io_time.h rename to include/hermes/dpe/minimize_io_time.h diff --git a/tasks/hermes/include/hermes/dpe/random.h b/include/hermes/dpe/random.h similarity index 100% rename from tasks/hermes/include/hermes/dpe/random.h rename to include/hermes/dpe/random.h diff --git a/tasks/hermes/include/hermes/dpe/round_robin.h b/include/hermes/dpe/round_robin.h similarity index 100% rename from tasks/hermes/include/hermes/dpe/round_robin.h rename to include/hermes/dpe/round_robin.h diff --git a/tasks/hermes/include/hermes/hermes.h b/include/hermes/hermes.h similarity index 100% rename from tasks/hermes/include/hermes/hermes.h rename to include/hermes/hermes.h diff --git a/tasks/hermes/include/hermes/hermes_types.h b/include/hermes/hermes_types.h similarity index 100% rename from tasks/hermes/include/hermes/hermes_types.h rename to include/hermes/hermes_types.h diff --git a/tasks/hermes/include/hermes/slab_allocator.h b/include/hermes/slab_allocator.h similarity index 100% rename from tasks/hermes/include/hermes/slab_allocator.h rename to include/hermes/slab_allocator.h diff --git a/tasks/hermes/include/hermes/status.h b/include/hermes/status.h similarity index 100% rename from tasks/hermes/include/hermes/status.h rename to include/hermes/status.h diff --git a/tasks/hermes/include/hermes/statuses.h b/include/hermes/statuses.h similarity index 100% rename from tasks/hermes/include/hermes/statuses.h rename to include/hermes/statuses.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c9248f49e..4bcda8599 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,72 +1,22 @@ #------------------------------------------------------------------------------ -# External dependencies +# Build Small Message Task Library #------------------------------------------------------------------------------ -# None for now +add_library(hermes SHARED + hermes_config_manager.cc) +add_dependencies(hermes ${Hermes_RUNTIME_DEPS}) +target_link_libraries(hermes ${Hermes_RUNTIME_LIBRARIES}) #------------------------------------------------------------------------------ -# Build Labstor Client Library +# Install Small Message Task Library #------------------------------------------------------------------------------ -add_library(hrun_client SHARED - ${CMAKE_CURRENT_SOURCE_DIR}/config_client.cc - ${CMAKE_CURRENT_SOURCE_DIR}/config_server.cc - ${CMAKE_CURRENT_SOURCE_DIR}/hrun_client.cc) -target_link_libraries(hrun_client - ${HermesShm_LIBRARIES} - yaml-cpp - cereal::cereal - thallium - -ldl -lrt -lc -pthread) - -#------------------------------------------------------------------------------ -# Build Labstor Runtime Library -#------------------------------------------------------------------------------ -add_library(hrun_runtime - worker.cc - hrun_runtime.cc) -add_dependencies(hrun_runtime ${Hermes_CLIENT_DEPS}) -target_link_libraries(hrun_runtime thallium ${Hermes_CLIENT_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Build Labstor Runtime Start Function -#------------------------------------------------------------------------------ -add_executable(hrun_start_runtime - hrun_start_runtime.cc) -add_dependencies(hrun_start_runtime ${Hermes_RUNTIME_DEPS}) -target_link_libraries(hrun_start_runtime ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Build LabStor Runtime Stop Function -#------------------------------------------------------------------------------ -add_executable(hrun_stop_runtime hrun_stop_runtime.cc) -add_dependencies(hrun_stop_runtime ${Hermes_CLIENT_DEPS}) -target_link_libraries(hrun_stop_runtime ${Hermes_CLIENT_LIBRARIES}) - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install( - FILES - ${HERMES_HEADERS} - DESTINATION - ${HERMES_INSTALL_INCLUDE_DIR} - COMPONENT - headers -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- install( - TARGETS - hrun_client - hrun_runtime - hrun_start_runtime - hrun_stop_runtime - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} + TARGETS + hermes + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) #----------------------------------------------------------------------------- @@ -85,26 +35,20 @@ install( # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- set(HERMES_EXPORTED_LIBS - hrun_client - hrun_runtime - hrun_start_runtime - hrun_stop_runtime + hermes ${HERMES_EXPORTED_LIBS}) if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) + EXPORT ( + TARGETS + ${HERMES_EXPORTED_LIBS} + FILE + ${HERMES_EXPORTED_TARGETS}.cmake + ) endif() #------------------------------------------------------------------------------ # Coverage #------------------------------------------------------------------------------ if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hrun_client) - set_coverage_flags(hrun_runtime) - set_coverage_flags(hrun_start_runtime) - set_coverage_flags(hrun_stop_runtime) -endif() \ No newline at end of file + set_coverage_flags(hermes) +endif() diff --git a/tasks/hermes/src/config_manager.cc b/src/hermes_config_manager.cc similarity index 100% rename from tasks/hermes/src/config_manager.cc rename to src/hermes_config_manager.cc diff --git a/tasks/CMakeLists.txt b/tasks/CMakeLists.txt index 52b374e69..4251d8161 100644 --- a/tasks/CMakeLists.txt +++ b/tasks/CMakeLists.txt @@ -1,10 +1,8 @@ -add_subdirectory(hermes) add_subdirectory(bdev) add_subdirectory(ram_bdev) add_subdirectory(posix_bdev) add_subdirectory(hermes_mdm) add_subdirectory(hermes_blob_mdm) add_subdirectory(hermes_bucket_mdm) -add_subdirectory(hermes_adapters) add_subdirectory(hermes_data_op) add_subdirectory(data_stager) \ No newline at end of file diff --git a/tasks/bdev/CMakeLists.txt b/tasks/bdev/CMakeLists.txt index c6f04b8f0..1fea7b11e 100644 --- a/tasks/bdev/CMakeLists.txt +++ b/tasks/bdev/CMakeLists.txt @@ -1,9 +1,9 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/data_stager/CMakeLists.txt b/tasks/data_stager/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks/data_stager/CMakeLists.txt +++ b/tasks/data_stager/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes/src/CMakeLists.txt b/tasks/hermes/src/CMakeLists.txt deleted file mode 100644 index f9ebf9a1c..000000000 --- a/tasks/hermes/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(hermes SHARED - config_manager.cc) -add_dependencies(hermes ${Hermes_RUNTIME_DEPS}) -target_link_libraries(hermes ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - hermes - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes) -endif() diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h deleted file mode 100644 index f7cf8e705..000000000 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters.h +++ /dev/null @@ -1,73 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#ifndef HRUN_hermes_adapters_H_ -#define HRUN_hermes_adapters_H_ - -#include "hermes_adapters_tasks.h" - -namespace hrun::hermes_adapters { - -/** Create hermes_adapters requests */ -class Client : public TaskLibClient { - - public: - /** Default constructor */ - Client() = default; - - /** Destructor */ - ~Client() = default; - - /** Async create a task state */ - HSHM_ALWAYS_INLINE - LPointer AsyncCreate(const TaskNode &task_node, - const DomainId &domain_id, - const std::string &state_name) { - id_ = TaskStateId::GetNull(); - QueueManagerInfo &qm = HRUN_CLIENT->server_config_.queue_manager_; - std::vector queue_info = { - {1, 1, qm.queue_depth_, 0}, - {1, 1, qm.queue_depth_, QUEUE_LONG_RUNNING}, - {qm.max_lanes_, qm.max_lanes_, qm.queue_depth_, QUEUE_LOW_LATENCY} - }; - return HRUN_ADMIN->AsyncCreateTaskState( - task_node, domain_id, state_name, id_, queue_info); - } - HRUN_TASK_NODE_ROOT(AsyncCreate) - template - HSHM_ALWAYS_INLINE - void CreateRoot(Args&& ...args) { - LPointer task = - AsyncCreateRoot(std::forward(args)...); - task->Wait(); - id_ = task->id_; - queue_id_ = QueueId(id_); - HRUN_CLIENT->DelTask(task); - } - - /** Destroy task state + queue */ - HSHM_ALWAYS_INLINE - void DestroyRoot(const DomainId &domain_id) { - HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); - } - - /** Call a custom method */ - HSHM_ALWAYS_INLINE - void AsyncCustomConstruct(CustomTask *task, - const TaskNode &task_node, - const DomainId &domain_id) { - HRUN_CLIENT->ConstructTask( - task, task_node, domain_id, id_); - } - HSHM_ALWAYS_INLINE - void CustomRoot(const DomainId &domain_id) { - LPointer> task = AsyncCustomRoot(domain_id); - task.ptr_->Wait(); - } - HRUN_TASK_NODE_PUSH_ROOT(Custom); -}; - -} // namespace hrun - -#endif // HRUN_hermes_adapters_H_ diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h deleted file mode 100644 index 3d1096611..000000000 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_lib_exec.h +++ /dev/null @@ -1,197 +0,0 @@ -#ifndef HRUN_HERMES_ADAPTERS_LIB_EXEC_H_ -#define HRUN_HERMES_ADAPTERS_LIB_EXEC_H_ - -/** Execute a task */ -void Run(u32 method, Task *task, RunContext &rctx) override { - switch (method) { - case Method::kConstruct: { - Construct(reinterpret_cast(task), rctx); - break; - } - case Method::kDestruct: { - Destruct(reinterpret_cast(task), rctx); - break; - } - case Method::kCustom: { - Custom(reinterpret_cast(task), rctx); - break; - } - } -} -/** Delete a task */ -void Del(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - HRUN_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - HRUN_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - case Method::kCustom: { - HRUN_CLIENT->DelTask(reinterpret_cast(task)); - break; - } - } -} -/** Duplicate a task */ -void Dup(u32 method, Task *orig_task, std::vector> &dups) override { - switch (method) { - case Method::kConstruct: { - hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kDestruct: { - hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - case Method::kCustom: { - hrun::CALL_DUPLICATE(reinterpret_cast(orig_task), dups); - break; - } - } -} -/** Register the duplicate output with the origin task */ -void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override { - switch (method) { - case Method::kConstruct: { - hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kDestruct: { - hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - case Method::kCustom: { - hrun::CALL_DUPLICATE_END(replica, reinterpret_cast(orig_task), reinterpret_cast(dup_task)); - break; - } - } -} -/** Ensure there is space to store replicated outputs */ -void ReplicateStart(u32 method, u32 count, Task *task) override { - switch (method) { - case Method::kConstruct: { - hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - case Method::kCustom: { - hrun::CALL_REPLICA_START(count, reinterpret_cast(task)); - break; - } - } -} -/** Determine success and handle failures */ -void ReplicateEnd(u32 method, Task *task) override { - switch (method) { - case Method::kConstruct: { - hrun::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - hrun::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - case Method::kCustom: { - hrun::CALL_REPLICA_END(reinterpret_cast(task)); - break; - } - } -} -/** Serialize a task when initially pushing into remote */ -std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kCustom: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when popping from remote queue */ -TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override { - TaskPointer task_ptr; - switch (method) { - case Method::kConstruct: { - task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kDestruct: { - task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - case Method::kCustom: { - task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask(task_ptr.shm_); - ar >> *reinterpret_cast(task_ptr.ptr_); - break; - } - } - return task_ptr; -} -/** Serialize a task when returning from remote queue */ -std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kDestruct: { - ar << *reinterpret_cast(task); - break; - } - case Method::kCustom: { - ar << *reinterpret_cast(task); - break; - } - } - return ar.Get(); -} -/** Deserialize a task when returning from remote queue */ -void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override { - switch (method) { - case Method::kConstruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kDestruct: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - case Method::kCustom: { - ar.Deserialize(replica, *reinterpret_cast(task)); - break; - } - } -} -/** Get the grouping of the task */ -u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override { - switch (method) { - case Method::kConstruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kDestruct: { - return reinterpret_cast(task)->GetGroup(group); - } - case Method::kCustom: { - return reinterpret_cast(task)->GetGroup(group); - } - } - return -1; -} - -#endif // HRUN_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h deleted file mode 100644 index 965777f39..000000000 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef HRUN_HERMES_ADAPTERS_METHODS_H_ -#define HRUN_HERMES_ADAPTERS_METHODS_H_ - -/** The set of methods in the admin task */ -struct Method : public TaskMethod { - TASK_METHOD_T kCustom = kLast + 0; -}; - -#endif // HRUN_HERMES_ADAPTERS_METHODS_H_ \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml deleted file mode 100644 index b1b54e2a4..000000000 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_methods.yaml +++ /dev/null @@ -1 +0,0 @@ -kCustom: 0 \ No newline at end of file diff --git a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h b/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h deleted file mode 100644 index dc1afdfb1..000000000 --- a/tasks/hermes_adapters/include/hermes_adapters/hermes_adapters_tasks.h +++ /dev/null @@ -1,119 +0,0 @@ -// -// Created by lukemartinlogan on 8/11/23. -// - -#ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ -#define HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ - -#include "hrun/api/hrun_client.h" -#include "hrun/task_registry/task_lib.h" -#include "hrun_admin/hrun_admin.h" -#include "hrun/queue_manager/queue_manager_client.h" -#include "proc_queue/proc_queue.h" - -namespace hrun::hermes_adapters { - -#include "hermes_adapters_methods.h" -#include "hrun/hrun_namespace.h" -using hrun::proc_queue::TypedPushTask; -using hrun::proc_queue::PushTask; - -/** - * A task to create hermes_adapters - * */ -using hrun::Admin::CreateTaskStateTask; -struct ConstructTask : public CreateTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(hipc::Allocator *alloc) - : CreateTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - ConstructTask(hipc::Allocator *alloc, - const TaskNode &task_node, - const DomainId &domain_id, - const std::string &state_name, - const TaskStateId &id, - const std::vector &queue_info) - : CreateTaskStateTask(alloc, task_node, domain_id, state_name, - "hermes_adapters", id, queue_info) { - // Custom params - } - - HSHM_ALWAYS_INLINE - ~ConstructTask() { - // Custom params - } -}; - -/** A task to destroy hermes_adapters */ -using hrun::Admin::DestroyTaskStateTask; -struct DestructTask : public DestroyTaskStateTask { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(hipc::Allocator *alloc) - : DestroyTaskStateTask(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - DestructTask(hipc::Allocator *alloc, - const TaskNode &task_node, - const DomainId &domain_id, - TaskStateId &state_id) - : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } -}; - -/** - * A custom task in hermes_adapters - * */ -struct CustomTask : public Task, TaskFlags { - /** SHM default constructor */ - HSHM_ALWAYS_INLINE explicit - CustomTask(hipc::Allocator *alloc) : Task(alloc) {} - - /** Emplace constructor */ - HSHM_ALWAYS_INLINE explicit - CustomTask(hipc::Allocator *alloc, - const TaskNode &task_node, - const DomainId &domain_id, - const TaskStateId &state_id) : Task(alloc) { - // Initialize task - task_node_ = task_node; - lane_hash_ = 0; - prio_ = TaskPrio::kLowLatency; - task_state_ = state_id; - method_ = Method::kCustom; - task_flags_.SetBits(0); - domain_id_ = domain_id; - - // Custom params - } - - /** (De)serialize message call */ - template - void SerializeStart(Ar &ar) { - task_serialize(ar); - } - - /** (De)serialize message return */ - template - void SerializeEnd(u32 replica, Ar &ar) { - } - - /** Create group */ - HSHM_ALWAYS_INLINE - u32 GetGroup(hshm::charbuf &group) { - return TASK_UNORDERED; - } -}; - -} // namespace hrun::hermes_adapters - -#endif // HRUN_TASKS_TASK_TEMPL_INCLUDE_hermes_adapters_hermes_adapters_TASKS_H_ diff --git a/tasks/hermes_adapters/src/CMakeLists.txt b/tasks/hermes_adapters/src/CMakeLists.txt deleted file mode 100644 index 8c5d06d8e..000000000 --- a/tasks/hermes_adapters/src/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Small Message Task Library -#------------------------------------------------------------------------------ -add_library(hermes_adapters SHARED - hermes_adapters.cc) -add_dependencies(hermes_adapters ${Hermes_RUNTIME_DEPS}) -target_link_libraries(hermes_adapters ${Hermes_RUNTIME_LIBRARIES}) - -#------------------------------------------------------------------------------ -# Install Small Message Task Library -#------------------------------------------------------------------------------ -install( - TARGETS - hermes_adapters - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install for import into other projects -#----------------------------------------------------------------------------- -install( - EXPORT - ${HERMES_EXPORTED_TARGETS} - DESTINATION - ${HERMES_INSTALL_DATA_DIR}/cmake/hermes - FILE - ${HERMES_EXPORTED_TARGETS}.cmake -) - -#----------------------------------------------------------------------------- -# Export all exported targets to the build tree for use by parent project -#----------------------------------------------------------------------------- -set(HERMES_EXPORTED_LIBS - hermes_adapters - ${HERMES_EXPORTED_LIBS}) -if(NOT HERMES_EXTERNALLY_CONFIGURED) - EXPORT ( - TARGETS - ${HERMES_EXPORTED_LIBS} - FILE - ${HERMES_EXPORTED_TARGETS}.cmake - ) -endif() - -#------------------------------------------------------------------------------ -# Coverage -#------------------------------------------------------------------------------ -if(HERMES_ENABLE_COVERAGE) - set_coverage_flags(hermes_adapters) -endif() diff --git a/tasks/hermes_adapters/src/hermes_adapters.cc b/tasks/hermes_adapters/src/hermes_adapters.cc deleted file mode 100644 index 41fa46f64..000000000 --- a/tasks/hermes_adapters/src/hermes_adapters.cc +++ /dev/null @@ -1,39 +0,0 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - -#include "hrun_admin/hrun_admin.h" -#include "hrun/api/hrun_runtime.h" -#include "hermes_adapters/hermes_adapters.h" - -namespace hrun::hermes_adapters { - -class Server : public TaskLib { - public: - Server() = default; - - void Construct(ConstructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - - void Destruct(DestructTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - - void Custom(CustomTask *task, RunContext &rctx) { - task->SetModuleComplete(); - } - - void FileStageIn() { - } - - void FileStageOut() { - } - - public: -#include "hermes_adapters/hermes_adapters_lib_exec.h" -}; - -} // namespace hrun::hermes_adapters - -HRUN_TASK_CC(hrun::hermes_adapters::Server, "hermes_adapters"); diff --git a/tasks/hermes_blob_mdm/CMakeLists.txt b/tasks/hermes_blob_mdm/CMakeLists.txt index 61dbbc349..7cd7a5538 100644 --- a/tasks/hermes_blob_mdm/CMakeLists.txt +++ b/tasks/hermes_blob_mdm/CMakeLists.txt @@ -5,6 +5,6 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_bucket_mdm/CMakeLists.txt b/tasks/hermes_bucket_mdm/CMakeLists.txt index 67b8141e5..be52b3c1e 100644 --- a/tasks/hermes_bucket_mdm/CMakeLists.txt +++ b/tasks/hermes_bucket_mdm/CMakeLists.txt @@ -5,6 +5,6 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_data_op/CMakeLists.txt b/tasks/hermes_data_op/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks/hermes_data_op/CMakeLists.txt +++ b/tasks/hermes_data_op/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/hermes_mdm/CMakeLists.txt b/tasks/hermes_mdm/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks/hermes_mdm/CMakeLists.txt +++ b/tasks/hermes_mdm/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/posix_bdev/CMakeLists.txt b/tasks/posix_bdev/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks/posix_bdev/CMakeLists.txt +++ b/tasks/posix_bdev/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks/ram_bdev/CMakeLists.txt b/tasks/ram_bdev/CMakeLists.txt index 5b409c224..d48611782 100644 --- a/tasks/ram_bdev/CMakeLists.txt +++ b/tasks/ram_bdev/CMakeLists.txt @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# Build Labstor Admin Task Library +# Build Hrun Admin Task Library #------------------------------------------------------------------------------ include_directories(include) add_subdirectory(src) #----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers +# Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/small_message/CMakeLists.txt b/tasks_required/small_message/CMakeLists.txt deleted file mode 100644 index 5b409c224..000000000 --- a/tasks_required/small_message/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Labstor Admin Task Library -#------------------------------------------------------------------------------ -include_directories(include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/worch_proc_round_robin/CMakeLists.txt b/tasks_required/worch_proc_round_robin/CMakeLists.txt deleted file mode 100644 index 5b409c224..000000000 --- a/tasks_required/worch_proc_round_robin/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Labstor Admin Task Library -#------------------------------------------------------------------------------ -include_directories(include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/tasks_required/worch_queue_round_robin/CMakeLists.txt b/tasks_required/worch_queue_round_robin/CMakeLists.txt deleted file mode 100644 index 5b409c224..000000000 --- a/tasks_required/worch_queue_round_robin/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -#------------------------------------------------------------------------------ -# Build Labstor Admin Task Library -#------------------------------------------------------------------------------ -include_directories(include) -add_subdirectory(src) - -#----------------------------------------------------------------------------- -# Install LabStor Admin Task Library Headers -#----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) From 2d87f6da05a1238091f4a88ea4506c7ad651b590 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 22:36:18 -0500 Subject: [PATCH 181/191] Tested restructure on compiling external program --- CMake/HermesConfig.cmake | 9 ++++++--- CMakeLists.txt | 8 ++++---- codegen/codegen/hrun_config/generator.py | 2 +- codegen/hermes_config | 4 ++-- codegen/hrun_config | 4 ++-- hermes_adapters/CMakeLists.txt | 11 ++++++++++- hrun/include/hrun/config/config_client_default.h | 8 ++++---- hrun/include/hrun/config/config_server_default.h | 8 ++++---- include/hermes/config_client_default.h | 8 ++++---- include/hermes/config_server_default.h | 8 ++++---- 10 files changed, 41 insertions(+), 29 deletions(-) diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index 961ad449a..a9c12a958 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -11,6 +11,7 @@ find_path( Hermes_INCLUDE_DIR hermes/hermes_types.h ) +message("Hermes_INCLUDE_DIR: ${Hermes_INCLUDE_DIR}") if( Hermes_INCLUDE_DIR ) get_filename_component(Hermes_DIR ${Hermes_INCLUDE_DIR} PATH) @@ -20,9 +21,11 @@ if( Hermes_INCLUDE_DIR ) #----------------------------------------------------------------------------- find_library( Hermes_LIBRARY - NAMES hrun_client hrun_runtime + NAMES hrun_client hrun_server ) + message("Hermes_LIBRARY: ${Hermes_LIBRARY}") + # HermesShm find_package(HermesShm CONFIG REQUIRED) message(STATUS "found hermes_shm.h at ${HermesShm_INCLUDE_DIRS}") @@ -91,10 +94,10 @@ endif(Hermes_INCLUDE_DIR) if(Hermes_FOUND) if(NOT Hermes_FIND_QUIETLY) - message(STATUS "FindHermes: Found both Hermes.h and libHermes_client.so") + message(STATUS "FindHermes: Found both Hermes.h and libhrun_client.so") endif(NOT Hermes_FIND_QUIETLY) else(Hermes_FOUND) if(Hermes_FIND_REQUIRED) - message(STATUS "FindHermes: Could not find Hermes.h and/or libHermes_client.so") + message(STATUS "FindHermes: Could not find Hermes.h and/or libhrun_client.so") endif(Hermes_FIND_REQUIRED) endif(Hermes_FOUND) diff --git a/CMakeLists.txt b/CMakeLists.txt index 71cf9b545..e823e943c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,10 +195,8 @@ enable_testing() add_subdirectory(test) #----------------------------------------------------------------------------- -# Install HRUN Headers +# Install Hermes Headers #----------------------------------------------------------------------------- -install(DIRECTORY hrun/include DESTINATION ${CMAKE_INSTALL_PREFIX}) - configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/CMake/HermesConfig.cmake ${PROJECT_BINARY_DIR}/CMakeFiles/HermesConfig.cmake @ONLY @@ -209,4 +207,6 @@ install( ${PROJECT_BINARY_DIR}/CMakeFiles/HermesConfig.cmake DESTINATION ${CMAKE_INSTALL_PREFIX}/cmake -) \ No newline at end of file +) + +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) \ No newline at end of file diff --git a/codegen/codegen/hrun_config/generator.py b/codegen/codegen/hrun_config/generator.py index f3de632e8..76ea0562e 100644 --- a/codegen/codegen/hrun_config/generator.py +++ b/codegen/codegen/hrun_config/generator.py @@ -13,7 +13,7 @@ def create_config(path, var_name, config_path, macro_name): # Create the hermes config string string_lines = [] - string_lines.append(f"const char* {var_name} = ") + string_lines.append(f"const inline char* {var_name} = ") for line in yaml_config_lines: line = line.replace('\"', '\\\"') line = line.replace('\'', '\\\'') diff --git a/codegen/hermes_config b/codegen/hermes_config index 0a35757bf..54b36cced 100755 --- a/codegen/hermes_config +++ b/codegen/hermes_config @@ -17,12 +17,12 @@ create_config( path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", var_name="kHermesClientDefaultConfigStr", config_path=f"{HERMES_ROOT}/include/hermes/config_client_default.h", - macro_name="CLIENT" + macro_name="HERMES_CLIENT" ) create_config( path=f"{HERMES_ROOT}/config/hermes_server_default.yaml", var_name="kHermesServerDefaultConfigStr", config_path=f"{HERMES_ROOT}/include/hermes/config_server_default.h", - macro_name="SERVER" + macro_name="HERMES_SERVER" ) diff --git a/codegen/hrun_config b/codegen/hrun_config index e62fde410..9882e8321 100755 --- a/codegen/hrun_config +++ b/codegen/hrun_config @@ -17,12 +17,12 @@ create_config( path=f"{HRUN_ROOT}/config/hrun_client_default.yaml", var_name="kHrunClientDefaultConfigStr", config_path=f"{HRUN_ROOT}/include/hrun/config/config_client_default.h", - macro_name="CLIENT" + macro_name="HRUN_CLIENT" ) create_config( path=f"{HRUN_ROOT}/config/hrun_server_default.yaml", var_name="kHrunServerDefaultConfigStr", config_path=f"{HRUN_ROOT}/include/hrun/config/config_server_default.h", - macro_name="SERVER" + macro_name="HRUN_SERVER" ) \ No newline at end of file diff --git a/hermes_adapters/CMakeLists.txt b/hermes_adapters/CMakeLists.txt index 1e8a02eff..e335f4d0b 100644 --- a/hermes_adapters/CMakeLists.txt +++ b/hermes_adapters/CMakeLists.txt @@ -16,4 +16,13 @@ endif() #----------------------------------------------------------------------------- # Install HRUN Admin Task Library Headers #----------------------------------------------------------------------------- -install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) +file(GLOB_RECURSE HERMES_HEADERS "*.h") +install( + FILES + ${HERMES_HEADERS} + DESTINATION + ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters + COMPONENT + headers +) +install(DIRECTORY mapper DESTINATION ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters) \ No newline at end of file diff --git a/hrun/include/hrun/config/config_client_default.h b/hrun/include/hrun/config/config_client_default.h index 2d4ad8bf1..7b188d56a 100644 --- a/hrun/include/hrun/config/config_client_default.h +++ b/hrun/include/hrun/config/config_client_default.h @@ -1,5 +1,5 @@ -#ifndef HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ -#define HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ -const char* kHrunClientDefaultConfigStr = +#ifndef HRUN_SRC_CONFIG_HRUN_CLIENT_DEFAULT_H_ +#define HRUN_SRC_CONFIG_HRUN_CLIENT_DEFAULT_H_ +const inline char* kHrunClientDefaultConfigStr = "thread_model: kStd\n"; -#endif // HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file +#endif // HRUN_SRC_CONFIG_HRUN_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/hrun/include/hrun/config/config_server_default.h b/hrun/include/hrun/config/config_server_default.h index 2cc61dccf..f8e5c5334 100644 --- a/hrun/include/hrun/config/config_server_default.h +++ b/hrun/include/hrun/config/config_server_default.h @@ -1,6 +1,6 @@ -#ifndef HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ -#define HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ -const char* kHrunServerDefaultConfigStr = +#ifndef HRUN_SRC_CONFIG_HRUN_SERVER_DEFAULT_H_ +#define HRUN_SRC_CONFIG_HRUN_SERVER_DEFAULT_H_ +const inline char* kHrunServerDefaultConfigStr = "### Runtime orchestration settings\n" "work_orchestrator:\n" " # The number of worker threads to spawn\n" @@ -58,4 +58,4 @@ const char* kHrunServerDefaultConfigStr = " \'posix_bdev\',\n" " \'ram_bdev\'\n" "]\n"; -#endif // HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file +#endif // HRUN_SRC_CONFIG_HRUN_SERVER_DEFAULT_H_ \ No newline at end of file diff --git a/include/hermes/config_client_default.h b/include/hermes/config_client_default.h index e2d088e58..aff693b94 100644 --- a/include/hermes/config_client_default.h +++ b/include/hermes/config_client_default.h @@ -1,6 +1,6 @@ -#ifndef HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ -#define HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ -static inline const char* kHermesClientDefaultConfigStr = +#ifndef HRUN_SRC_CONFIG_HERMES_CLIENT_DEFAULT_H_ +#define HRUN_SRC_CONFIG_HERMES_CLIENT_DEFAULT_H_ +const inline char* kHermesClientDefaultConfigStr = "stop_daemon: false\n" "path_inclusions: [\"/tmp/test_hermes\"]\n" "path_exclusions: [\"/\"]\n" @@ -11,4 +11,4 @@ static inline const char* kHermesClientDefaultConfigStr = " - path: \"/\"\n" " page_size: 1MB\n" " mode: kDefault\n"; -#endif // HRUN_SRC_CONFIG_CLIENT_DEFAULT_H_ \ No newline at end of file +#endif // HRUN_SRC_CONFIG_HERMES_CLIENT_DEFAULT_H_ \ No newline at end of file diff --git a/include/hermes/config_server_default.h b/include/hermes/config_server_default.h index 0c3dfd0a4..5e543e058 100644 --- a/include/hermes/config_server_default.h +++ b/include/hermes/config_server_default.h @@ -1,6 +1,6 @@ -#ifndef HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ -#define HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ -static inline const char* kHermesServerDefaultConfigStr = +#ifndef HRUN_SRC_CONFIG_HERMES_SERVER_DEFAULT_H_ +#define HRUN_SRC_CONFIG_HERMES_SERVER_DEFAULT_H_ +const inline char* kHermesServerDefaultConfigStr = "# Example Hermes configuration file\n" "\n" "### Define properties of the storage devices\n" @@ -178,4 +178,4 @@ static inline const char* kHermesServerDefaultConfigStr = " - \"hermes_mpiio_io_client\"\n" " - \"hermes_example_trait\"\n" " - \"hermes_prefetcher_trait\"\n"; -#endif // HRUN_SRC_CONFIG_SERVER_DEFAULT_H_ \ No newline at end of file +#endif // HRUN_SRC_CONFIG_HERMES_SERVER_DEFAULT_H_ \ No newline at end of file From 5a8991c2e57467689335a73c1bdf30a14d6e93f5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sat, 14 Oct 2023 23:00:10 -0500 Subject: [PATCH 182/191] Make it so the hrun and hermes can use the same exact config --- config/hermes_server_default.yaml | 94 ++++++++++------- include/hermes/config_server.h | 15 --- include/hermes/config_server_default.h | 94 ++++++++++------- test/unit/hermes/config/hermes_server.yaml | 111 +++++--------------- test/unit/hermes/config/labstor_server.yaml | 56 ---------- 5 files changed, 138 insertions(+), 232 deletions(-) delete mode 100644 test/unit/hermes/config/labstor_server.yaml diff --git a/config/hermes_server_default.yaml b/config/hermes_server_default.yaml index 5d88d855b..9741de8dd 100644 --- a/config/hermes_server_default.yaml +++ b/config/hermes_server_default.yaml @@ -82,32 +82,6 @@ devices: # any amount of memory max_memory: 0g -### Define properties of RPCs -rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "" - - # Host names can be defined using the following syntax: - # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... - # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... - host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. - protocol: "ofi+sockets" - - # RPC domain name for verbs transport. Blank for tcp. - domain: "" - - # Desired RPC port number. - port: 8080 - - # The number of handler threads for each RPC server. - num_threads: 4 - ### Define properties of the BORG buffer_organizer: # The number of threads used in the background organization of internal Hermes buffers. @@ -161,17 +135,63 @@ mdm: est_bucket_count: 100000 est_num_traits: 256 -# The shared memory prefix for the hermes shared memory segment. A username -# will be automatically appended. -shmem_name: "/hermes_shm_" - # The interval in milliseconds at which to update the global system view. system_view_state_update_interval_ms: 1000 -### Define the names of the traits to search LD_LIBRARY_PATH for -traits: - - "hermes_posix_io_client" - - "hermes_stdio_io_client" - - "hermes_mpiio_io_client" - - "hermes_example_trait" - - "hermes_prefetcher_trait" +### Runtime orchestration settings +work_orchestrator: + # The number of worker threads to spawn + max_workers: 4 + +### Queue Manager settings +queue_manager: + # The default depth of allocated queues + queue_depth: 256 + # The maximum number of lanes per queue + max_lanes: 16 + # The maximum number of queues + max_queues: 1024 + # The shared memory allocator to use + shm_allocator: kScalablePageAllocator + # The name of the shared memory region to create + shm_name: "hrun_shm" + # The size of the shared memory region to allocate for general data structures + shm_size: 0g + # The size of the shared memory to allocate for data buffers + +### Define properties of RPCs +rpc: + # A path to a file containing a list of server names, 1 per line. If your + # servers are named according to a pattern (e.g., server-1, server-2, etc.), + # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this + # option is not empty, it will override anything in `rpc_server_base_name`. + host_file: "" + + # Host names can be defined using the following syntax: + # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... + # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... + host_names: ["localhost"] + + # The RPC protocol. This must come from the documentation of the specific RPC + # library in use. + protocol: "ofi+sockets" + + # RPC domain name for verbs transport. Blank for tcp. + domain: "" + + # Desired RPC port number. + port: 8080 + + # The number of handler threads for each RPC server. + num_threads: 4 + +### Task Registry +task_registry: [ + 'hermes_mdm', + 'hermes_blob_mdm', + 'hermes_bucket_mdm', + 'hermes_data_op', + 'data_stager', + 'posix_bdev', + 'ram_bdev' +] \ No newline at end of file diff --git a/include/hermes/config_server.h b/include/hermes/config_server.h index 24a8ae938..e9ea2e66e 100644 --- a/include/hermes/config_server.h +++ b/include/hermes/config_server.h @@ -168,14 +168,6 @@ class ServerConfig : public BaseConfig { /** The length of a view state epoch */ u32 system_view_state_update_interval_ms; - /** The max amount of memory hermes uses for non-buffering tasks */ - size_t max_memory_; - - /** A base name for the BufferPool shared memory segement. Hermes appends the - * value of the USER environment variable to this string. - */ - std::string shmem_name_; - public: /** Default constructor */ ServerConfig() = default; @@ -213,13 +205,6 @@ class ServerConfig : public BaseConfig { if (yaml_conf["traits"]) { ParseTraitInfo(yaml_conf["traits"]); } - if (yaml_conf["shmem_name"]) { - shmem_name_ = yaml_conf["shmem_name"].as(); - } - if (yaml_conf["max_memory"]) { - max_memory_ = hshm::ConfigParse::ParseSize( - yaml_conf["max_memory"].as()); - } } diff --git a/include/hermes/config_server_default.h b/include/hermes/config_server_default.h index 5e543e058..75ce716a1 100644 --- a/include/hermes/config_server_default.h +++ b/include/hermes/config_server_default.h @@ -85,32 +85,6 @@ const inline char* kHermesServerDefaultConfigStr = "# any amount of memory\n" "max_memory: 0g\n" "\n" -"### Define properties of RPCs\n" -"rpc:\n" -" # A path to a file containing a list of server names, 1 per line. If your\n" -" # servers are named according to a pattern (e.g., server-1, server-2, etc.),\n" -" # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this\n" -" # option is not empty, it will override anything in `rpc_server_base_name`.\n" -" host_file: \"\"\n" -"\n" -" # Host names can be defined using the following syntax:\n" -" # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ...\n" -" # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ...\n" -" host_names: [\"localhost\"]\n" -"\n" -" # The RPC protocol. This must come from the documentation of the specific RPC\n" -" # library in use.\n" -" protocol: \"ofi+sockets\"\n" -"\n" -" # RPC domain name for verbs transport. Blank for tcp.\n" -" domain: \"\"\n" -"\n" -" # Desired RPC port number.\n" -" port: 8080\n" -"\n" -" # The number of handler threads for each RPC server.\n" -" num_threads: 4\n" -"\n" "### Define properties of the BORG\n" "buffer_organizer:\n" " # The number of threads used in the background organization of internal Hermes buffers.\n" @@ -164,18 +138,64 @@ const inline char* kHermesServerDefaultConfigStr = " est_bucket_count: 100000\n" " est_num_traits: 256\n" "\n" -"# The shared memory prefix for the hermes shared memory segment. A username\n" -"# will be automatically appended.\n" -"shmem_name: \"/hermes_shm_\"\n" -"\n" "# The interval in milliseconds at which to update the global system view.\n" "system_view_state_update_interval_ms: 1000\n" "\n" -"### Define the names of the traits to search LD_LIBRARY_PATH for\n" -"traits:\n" -" - \"hermes_posix_io_client\"\n" -" - \"hermes_stdio_io_client\"\n" -" - \"hermes_mpiio_io_client\"\n" -" - \"hermes_example_trait\"\n" -" - \"hermes_prefetcher_trait\"\n"; +"### Runtime orchestration settings\n" +"work_orchestrator:\n" +" # The number of worker threads to spawn\n" +" max_workers: 4\n" +"\n" +"### Queue Manager settings\n" +"queue_manager:\n" +" # The default depth of allocated queues\n" +" queue_depth: 256\n" +" # The maximum number of lanes per queue\n" +" max_lanes: 16\n" +" # The maximum number of queues\n" +" max_queues: 1024\n" +" # The shared memory allocator to use\n" +" shm_allocator: kScalablePageAllocator\n" +" # The name of the shared memory region to create\n" +" shm_name: \"hrun_shm\"\n" +" # The size of the shared memory region to allocate for general data structures\n" +" shm_size: 0g\n" +" # The size of the shared memory to allocate for data buffers\n" +"\n" +"### Define properties of RPCs\n" +"rpc:\n" +" # A path to a file containing a list of server names, 1 per line. If your\n" +" # servers are named according to a pattern (e.g., server-1, server-2, etc.),\n" +" # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this\n" +" # option is not empty, it will override anything in `rpc_server_base_name`.\n" +" host_file: \"\"\n" +"\n" +" # Host names can be defined using the following syntax:\n" +" # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ...\n" +" # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ...\n" +" host_names: [\"localhost\"]\n" +"\n" +" # The RPC protocol. This must come from the documentation of the specific RPC\n" +" # library in use.\n" +" protocol: \"ofi+sockets\"\n" +"\n" +" # RPC domain name for verbs transport. Blank for tcp.\n" +" domain: \"\"\n" +"\n" +" # Desired RPC port number.\n" +" port: 8080\n" +"\n" +" # The number of handler threads for each RPC server.\n" +" num_threads: 4\n" +"\n" +"### Task Registry\n" +"task_registry: [\n" +" \'hermes_mdm\',\n" +" \'hermes_blob_mdm\',\n" +" \'hermes_bucket_mdm\',\n" +" \'hermes_data_op\',\n" +" \'data_stager\',\n" +" \'posix_bdev\',\n" +" \'ram_bdev\'\n" +"]\n"; #endif // HRUN_SRC_CONFIG_HERMES_SERVER_DEFAULT_H_ \ No newline at end of file diff --git a/test/unit/hermes/config/hermes_server.yaml b/test/unit/hermes/config/hermes_server.yaml index 0dd5b841c..d7b3dcf3f 100644 --- a/test/unit/hermes/config/hermes_server.yaml +++ b/test/unit/hermes/config/hermes_server.yaml @@ -2,48 +2,14 @@ ### Define properties of the storage devices devices: - # The name of the device. - # It can be whatever the user wants, there are no special names ram: - # The mount point of each device. RAM should be the empty string. For block - # devices, this is the directory where Hermes will create buffering files. For - # object storage or cloud targets, this will be a url. mount_point: "" - - # The maximum buffering capacity in MiB of each device. capacity: 4GB - - # The size of the smallest available buffer in KiB. In general this should be - # the page size of your system for byte addressable storage, and the block size - # of the storage device for block addressable storage. block_size: 4KB - - # The number of blocks (the size of which is chosen in block_sizes_kb) that each - # device should contain for each slab (controlled by num_slabs). This allows for - # precise control of the distibution of buffer sizes. slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - - # The maximum theoretical bandwidth (as advertised by the manufacturer) in - # Possible units: KBps, MBps, GBps bandwidth: 6000MBps - - # The latency of each device (as advertised by the manufacturer). - # Possible units: ns, us, ms, s latency: 15us - - # For each device, indicate '1' if it is shared among nodes (e.g., burst - # buffers), or '0' if it is per node (e.g., local NVMe). is_shared_device: false - - # For each device, the minimum and maximum percent capacity threshold at which - # the BufferOrganizer will trigger. Decreasing the maximum thresholds will cause - # the BufferOrganizer to move data to lower devices, making more room in faster - # devices (ideal for write-heavy workloads). Conversely, increasing the minimum - # threshold will cause data to be moved from slower devices into faster devices - # (ideal for read-heavy workloads). For example, a maximum capacity threshold of - # 0.8 would have the effect of always keeping 20% of the device's space free for - # incoming writes. Conversely, a minimum capacity threshold of 0.3 would ensure - # that the device is always at least 30% occupied. borg_capacity_thresh: [0.0, 1.0] nvme: @@ -76,68 +42,28 @@ devices: is_shared_device: true borg_capacity_thresh: [ 0.0, 1.0 ] -# Define the maximum amount of memory Hermes can use for non-buffering tasks. -# This includes metadata management and memory allocations. -# This memory will not be preallocated, so if you don't know, 0 indicates -# any amount of memory -max_memory: 0g - ### Define properties of RPCs rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. host_file: "" - - # Host names can be defined using the following syntax: - # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... - # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. protocol: "ofi+sockets" - - # RPC domain name for verbs transport. Blank for tcp. domain: "" - - # Desired RPC port number. port: 8080 - - # The number of handler threads for each RPC server. num_threads: 4 ### Define properties of the BORG buffer_organizer: - # The number of threads used in the background organization of internal Hermes buffers. num_threads: 1 - - # Interval (ms) where blobs are checked for flushing flush_period: 1024 - - # Interval (ms) where blobs are checked for re-organization blob_reorg_period: 1024 - - ## What does "recently accessed" mean? - # Time when score is equal to 1 (seconds) recency_min: 0 - # Time when score is equal to 0 (seconds) recency_max: 60 - - ## What does "frequently accessed" mean? - # Number of accesses for score to be equal to 1 (count) freq_max: 15 - # Number of accesses for score to be equal to 0 (count) freq_min: 0 ### Define the default data placement policy dpe: - # Choose Random, RoundRobin, or MinimizeIoTime default_placement_policy: "MinimizeIoTime" - - # If true (1) the RoundRobin placement policy algorithm will split each Blob - # into a random number of smaller Blobs. default_rr_split: 0 ### Define I/O tracing properties @@ -155,23 +81,34 @@ prefetch: ### Define mdm properties mdm: - # This represents the number of blobs and buckets before collisions start - # to happen in the unordered_map tables. est_blob_count: 100000 est_bucket_count: 100000 est_num_traits: 256 -# The shared memory prefix for the hermes shared memory segment. A username -# will be automatically appended. -shmem_name: "/hermes_shm_" - # The interval in milliseconds at which to update the global system view. system_view_state_update_interval_ms: 1000 -### Define the names of the traits to search LD_LIBRARY_PATH for -traits: - - "hermes_posix_io_client" - - "hermes_stdio_io_client" - - "hermes_mpiio_io_client" - - "hermes_example_trait" - - "hermes_prefetcher_trait" +### Runtime orchestration settings +work_orchestrator: + # The number of worker threads to spawn + max_workers: 4 + +### Queue Manager settings +queue_manager: + queue_depth: 256 + max_lanes: 16 + max_queues: 1024 + shm_allocator: kScalablePageAllocator + shm_name: "hrun_shm" + shm_size: 0g + +### Task Registry +task_registry: [ + 'hermes_mdm', + 'hermes_blob_mdm', + 'hermes_bucket_mdm', + 'hermes_data_op', + 'data_stager', + 'posix_bdev', + 'ram_bdev' +] \ No newline at end of file diff --git a/test/unit/hermes/config/labstor_server.yaml b/test/unit/hermes/config/labstor_server.yaml deleted file mode 100644 index 42e4ecedb..000000000 --- a/test/unit/hermes/config/labstor_server.yaml +++ /dev/null @@ -1,56 +0,0 @@ -### Runtime orchestration settings -work_orchestrator: - # The number of worker threads to spawn - max_workers: 2 - -### Queue Manager settings -queue_manager: - # The default depth of allocated queues - queue_depth: 8192 - # The maximum number of lanes per queue - max_lanes: 4 - # The maximum number of queues - max_queues: 1024 - # The shared memory allocator to use - shm_allocator: kScalablePageAllocator - # The name of the shared memory region to create - shm_name: "hrun_shm" - # The size of the shared memory region to allocate - shm_size: 0g - -### Define properties of RPCs -rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "" - - # Host names can be defined using the following syntax: - # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... - # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... - host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. - protocol: "sockets" - - # RPC domain name for verbs transport. Blank for tcp. - domain: "" - - # Desired RPC port number. - port: 9192 - - # The number of handler threads for each RPC server. - num_threads: 4 - -### Task Registry -task_registry: [ - 'hermes_mdm', - 'hermes_blob_mdm', - 'hermes_bucket_mdm', - 'hermes_data_op', - 'data_stager', - 'posix_bdev', - 'ram_bdev' -] \ No newline at end of file From 3ce70d0c4d9e2533d111e687def28a086708a948 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 15 Oct 2023 06:12:38 -0500 Subject: [PATCH 183/191] Add python binding infrastructure --- .gitmodules | 3 +++ CMakeLists.txt | 2 ++ ci/hermes/packages/hermes/package.py | 15 ++++++++++----- external/pybind11 | 1 + wrapper/CMakeLists.txt | 4 ++++ wrapper/python/CMakeLists.txt | 2 ++ wrapper/python/cpp/CMakeLists.txt | 21 +++++++++++++++++++++ wrapper/python/cpp/py_hermes.cpp | 11 +++++++++++ wrapper/python/requirements.txt | 2 ++ wrapper/python/setup.py | 23 +++++++++++++++++++++++ wrapper/python/test/unit/test_hermes.py | 8 ++++++++ 11 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 .gitmodules create mode 160000 external/pybind11 create mode 100644 wrapper/CMakeLists.txt create mode 100644 wrapper/python/CMakeLists.txt create mode 100644 wrapper/python/cpp/CMakeLists.txt create mode 100644 wrapper/python/cpp/py_hermes.cpp create mode 100644 wrapper/python/requirements.txt create mode 100644 wrapper/python/setup.py create mode 100644 wrapper/python/test/unit/test_hermes.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..7676f3941 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/pybind11"] + path = external/pybind11 + url = https://github.com/pybind/pybind11.git diff --git a/CMakeLists.txt b/CMakeLists.txt index e823e943c..cf9dc2750 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -174,11 +174,13 @@ set(Hermes_RUNTIME_DEPS hrun_client hrun_runtime) set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) +add_subdirectory(external/pybind11) add_subdirectory(hrun) add_subdirectory(src) add_subdirectory(hermes_adapters) add_subdirectory(tasks) add_subdirectory(benchmark) +add_subdirectory(wrapper) add_custom_target(lint COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/lint.sh ${CMAKE_SOURCE_DIR}) #----------------------------------------------------------------------------- diff --git a/ci/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py index 607150dd9..138b1818e 100644 --- a/ci/hermes/packages/hermes/package.py +++ b/ci/hermes/packages/hermes/package.py @@ -5,11 +5,16 @@ class Hermes(CMakePackage): url = "https://github.com/HDFGroup/hermes/tarball/master" git = "https://github.com/HDFGroup/hermes.git" - version('master', branch='master') - version('pnnl', branch='pnnl') - version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', branch='dev') - version('dev-1.1', git='https://github.com/lukemartinlogan/hermes.git', branch='hermes-1.1') - version('hdf-1.1', git='https://github.com/HDFGroup/hermes.git', branch='hermes-1.1') + version('master', + branch='master', submodules=True) + version('pnnl', + branch='pnnl', submodules=True) + version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', + branch='dev', submodules=True) + version('dev-1.1', git='https://github.com/lukemartinlogan/hermes.git', + branch='hermes-1.1', submodules=True) + version('hdf-1.1', git='https://github.com/HDFGroup/hermes.git', + branch='hermes-1.1', submodules=True) variant('vfd', default=False, description='Enable HDF5 VFD') variant('ares', default=False, description='Enable full libfabric install') diff --git a/external/pybind11 b/external/pybind11 new file mode 160000 index 000000000..0e2c3e5db --- /dev/null +++ b/external/pybind11 @@ -0,0 +1 @@ +Subproject commit 0e2c3e5db41b6b2af4038734c84ab855ccaaa5f0 diff --git a/wrapper/CMakeLists.txt b/wrapper/CMakeLists.txt new file mode 100644 index 000000000..99821e045 --- /dev/null +++ b/wrapper/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.10) +project(hermes) + +add_subdirectory(python) \ No newline at end of file diff --git a/wrapper/python/CMakeLists.txt b/wrapper/python/CMakeLists.txt new file mode 100644 index 000000000..44fdf033a --- /dev/null +++ b/wrapper/python/CMakeLists.txt @@ -0,0 +1,2 @@ + +add_subdirectory(cpp) \ No newline at end of file diff --git a/wrapper/python/cpp/CMakeLists.txt b/wrapper/python/cpp/CMakeLists.txt new file mode 100644 index 000000000..d45903283 --- /dev/null +++ b/wrapper/python/cpp/CMakeLists.txt @@ -0,0 +1,21 @@ +#------------------------------------------------------------------------------ +# Find Packages +#------------------------------------------------------------------------------ +# find_package(pybind11 CONFIG REQUIRED) +# message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") + +#------------------------------------------------------------------------------ +# Compile Python GL +#------------------------------------------------------------------------------ +pybind11_add_module(py_hermes MODULE py_hermes.cpp) +add_dependencies(py_hermes ${Hermes_CLIENT_DEPS}) +target_link_libraries(py_hermes PUBLIC + ${Hermes_CLIENT_LIBRARIES} pybind11::module) + +#------------------------------------------------------------------------------ +# Install Targets +#------------------------------------------------------------------------------ +install(TARGETS + hermes + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) \ No newline at end of file diff --git a/wrapper/python/cpp/py_hermes.cpp b/wrapper/python/cpp/py_hermes.cpp new file mode 100644 index 000000000..8889fb859 --- /dev/null +++ b/wrapper/python/cpp/py_hermes.cpp @@ -0,0 +1,11 @@ +// +// Created by lukemartinlogan on 6/24/2023. +// + +#include +#include + +namespace py = pybind11; + +PYBIND11_MODULE(py_hermes, m) { +} diff --git a/wrapper/python/requirements.txt b/wrapper/python/requirements.txt new file mode 100644 index 000000000..852bb4206 --- /dev/null +++ b/wrapper/python/requirements.txt @@ -0,0 +1,2 @@ +pybind11 +pytest \ No newline at end of file diff --git a/wrapper/python/setup.py b/wrapper/python/setup.py new file mode 100644 index 000000000..9f2c15225 --- /dev/null +++ b/wrapper/python/setup.py @@ -0,0 +1,23 @@ +import setuptools + +setuptools.setup( + name="degenerate_studio", + packages=setuptools.find_packages(), + version="0.0.1", + author="Luke Logan", + author_email="lukemartinlogan@gmail.com", + description="A graphics library", + url="https://github.com/scs-lab/jarvis-util", + classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Development Status :: 0 - Pre-Alpha", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: None", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Graphics", + ], + long_description="" +) \ No newline at end of file diff --git a/wrapper/python/test/unit/test_hermes.py b/wrapper/python/test/unit/test_hermes.py new file mode 100644 index 000000000..fb803fca4 --- /dev/null +++ b/wrapper/python/test/unit/test_hermes.py @@ -0,0 +1,8 @@ +from unittest import TestCase +from py_hermes import Hermes, MetadataTable +import pathlib +import os + +class TestHermes(TestCase): + def test_metadata_query(self): + pass From f48679cbef0c8f7af720717dd4ed1d3bfb08c1ee Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 15 Oct 2023 06:15:00 -0500 Subject: [PATCH 184/191] Change wrapper api --- wrapper/python/setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wrapper/python/setup.py b/wrapper/python/setup.py index 9f2c15225..9ed6bbd05 100644 --- a/wrapper/python/setup.py +++ b/wrapper/python/setup.py @@ -1,23 +1,23 @@ import setuptools setuptools.setup( - name="degenerate_studio", + name="hermes", packages=setuptools.find_packages(), version="0.0.1", author="Luke Logan", author_email="lukemartinlogan@gmail.com", - description="A graphics library", - url="https://github.com/scs-lab/jarvis-util", + description="An I/O buffering library", + url="https://github.com/HDFGroup/hermes.git", classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", - "Development Status :: 0 - Pre-Alpha", + "Development Status :: 1.1", "Environment :: Other Environment", "Intended Audience :: Developers", "License :: None", - "Operating System :: OS Independent", + "Operating System :: Linux", "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Graphics", + "Topic :: I/O", ], long_description="" ) \ No newline at end of file From 6c4625adce73ffd2892cb1dd482ff9f4848d5dbb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 15 Oct 2023 06:15:42 -0500 Subject: [PATCH 185/191] Comment out python bindings --- CMakeLists.txt | 1 + wrapper/CMakeLists.txt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf9dc2750..438aa9fae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,7 @@ option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON) option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF) option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF) option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) +option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" OFF) #----------------------------------------------------------------------------- # Compiler Optimization diff --git a/wrapper/CMakeLists.txt b/wrapper/CMakeLists.txt index 99821e045..711dfa18b 100644 --- a/wrapper/CMakeLists.txt +++ b/wrapper/CMakeLists.txt @@ -1,4 +1,6 @@ cmake_minimum_required(VERSION 3.10) project(hermes) -add_subdirectory(python) \ No newline at end of file +if (HERMES_ENABLE_PYTHON) + add_subdirectory(python) +endif() From 0d17fe507da456e73ab58ac59db4eb2822a76fe3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 15 Oct 2023 06:17:14 -0500 Subject: [PATCH 186/191] Only add pybind if python --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 438aa9fae..f3df747ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -175,7 +175,9 @@ set(Hermes_RUNTIME_DEPS hrun_client hrun_runtime) set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) +if (HERMES_ENABLE_PYTHON) add_subdirectory(external/pybind11) +endif() add_subdirectory(hrun) add_subdirectory(src) add_subdirectory(hermes_adapters) From b14b6d27c078129b7a7acf4a048a09bd869babf5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Sun, 15 Oct 2023 08:54:25 -0500 Subject: [PATCH 187/191] Python bindings pass import --- CMakeLists.txt | 4 +- wrapper/CMakeLists.txt | 1 + wrapper/python/cpp/CMakeLists.txt | 8 +- wrapper/python/cpp/py_hermes.cpp | 108 ++++++++++++++++++++++++ wrapper/python/test/unit/test_hermes.py | 5 +- 5 files changed, 119 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f3df747ea..4e189e1d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,9 @@ option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON) option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF) option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF) option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) -option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" OFF) +option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" ON) + +message("HERMES_ENABLE_PYTHON: ${HERMES_ENABLE_PYTHON}") #----------------------------------------------------------------------------- # Compiler Optimization diff --git a/wrapper/CMakeLists.txt b/wrapper/CMakeLists.txt index 711dfa18b..082f8c4d6 100644 --- a/wrapper/CMakeLists.txt +++ b/wrapper/CMakeLists.txt @@ -2,5 +2,6 @@ cmake_minimum_required(VERSION 3.10) project(hermes) if (HERMES_ENABLE_PYTHON) + message("Python bindings enabled") add_subdirectory(python) endif() diff --git a/wrapper/python/cpp/CMakeLists.txt b/wrapper/python/cpp/CMakeLists.txt index d45903283..103c27142 100644 --- a/wrapper/python/cpp/CMakeLists.txt +++ b/wrapper/python/cpp/CMakeLists.txt @@ -5,17 +5,17 @@ # message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") #------------------------------------------------------------------------------ -# Compile Python GL +# Compile Python Hermes #------------------------------------------------------------------------------ pybind11_add_module(py_hermes MODULE py_hermes.cpp) -add_dependencies(py_hermes ${Hermes_CLIENT_DEPS}) +add_dependencies(py_hermes ${Hermes_CLIENT_DEPS} hermes) target_link_libraries(py_hermes PUBLIC - ${Hermes_CLIENT_LIBRARIES} pybind11::module) + ${Hermes_CLIENT_LIBRARIES} hermes pybind11::module) #------------------------------------------------------------------------------ # Install Targets #------------------------------------------------------------------------------ install(TARGETS - hermes + py_hermes LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) \ No newline at end of file diff --git a/wrapper/python/cpp/py_hermes.cpp b/wrapper/python/cpp/py_hermes.cpp index 8889fb859..17e6a89fa 100644 --- a/wrapper/python/cpp/py_hermes.cpp +++ b/wrapper/python/cpp/py_hermes.cpp @@ -4,8 +4,116 @@ #include #include +#include +#include "hermes/hermes.h" +#include "hermes/bucket.h" namespace py = pybind11; +using hermes::BlobId; +using hermes::BucketId; +using hermes::TagId; +using hermes::TargetId; +using hermes::BlobInfo; +using hermes::TargetStats; +using hermes::TagInfo; +using hermes::MetadataTable; +using hermes::Hermes; +using hrun::UniqueId; + +bool TRANSPARENT_HERMES_FUN() { + if (TRANSPARENT_HRUN()) { + HERMES_CONF->ClientInit(); + return true; + } + return false; +} + +template +void BindUniqueId(py::module &m, const std::string &name) { + py::class_(m, name.c_str()) + .def(py::init<>()) + .def(py::init(), py::arg("node_id"), py::arg("unique")) + .def(py::init(), py::arg("node_id"), py::arg("hash"), py::arg("unique")) + .def("IsNull", &UniqueT::IsNull) + .def("GetNull", &UniqueT::GetNull) + .def("SetNull", &UniqueT::SetNull) + .def("GetNodeId", &UniqueT::GetNodeId) + .def("__eq__", &UniqueT::operator==) + .def("__ne__", &UniqueT::operator!=) + .def_readonly("node_id", &UniqueT::node_id_) + .def_readonly("hash", &UniqueT::hash_) + .def_readonly("unique", &UniqueT::unique_); +} + +void BindBlobInfo(py::module &m) { + py::class_(m, "BlobInfo") + .def(py::init<>()) + .def("UpdateWriteStats", &BlobInfo::UpdateWriteStats) + .def("UpdateReadStats", &BlobInfo::UpdateReadStats) + .def_readonly("tag_id", &BlobInfo::tag_id_) + .def_readonly("blob_id", &BlobInfo::blob_id_) + .def_readonly("name", &BlobInfo::name_) + .def_readonly("buffers", &BlobInfo::buffers_) + .def_readonly("tags", &BlobInfo::tags_) + .def_readonly("blob_size", &BlobInfo::blob_size_) + .def_readonly("max_blob_size", &BlobInfo::max_blob_size_) + .def_readonly("score", &BlobInfo::score_) + .def_readonly("access_freq", &BlobInfo::access_freq_) + .def_readonly("last_access", &BlobInfo::last_access_) + .def_readonly("mod_count", &BlobInfo::mod_count_) + .def_readonly("last_flush", &BlobInfo::last_flush_) + .def_static("GetTimeFromStartNs", &BlobInfo::GetTimeFromStartNs); +} + +void BindTargetStats(py::module &m) { + py::class_(m, "TargetStats") + .def(py::init<>()) + .def_readonly("tgt_id", &TargetStats::tgt_id_) + .def_readonly("rem_cap", &TargetStats::rem_cap_) + .def_readonly("max_cap", &TargetStats::max_cap_) + .def_readonly("bandwidth", &TargetStats::bandwidth_) + .def_readonly("latency", &TargetStats::latency_) + .def_readonly("score", &TargetStats::score_); +} + +void BindTagInfo(py::module &m) { + py::class_(m, "TagInfo") + .def(py::init<>()) + .def_readonly("tag_id", &TagInfo::tag_id_) + .def_readonly("name", &TagInfo::name_) + .def_readonly("blobs", &TagInfo::blobs_) + .def_readonly("traits", &TagInfo::traits_) + .def_readonly("internal_size", &TagInfo::internal_size_) + .def_readonly("page_size", &TagInfo::page_size_) + .def_readonly("owner", &TagInfo::owner_); +} + +void BindMetadataTable(py::module &m) { + py::class_(m, "MetadataTable") + .def(py::init<>()) + .def_readonly("blob_info", &MetadataTable::blob_info_) + .def_readonly("target_info", &MetadataTable::target_info_) + .def_readonly("bkt_info", &MetadataTable::bkt_info_); +} + +void BindHermes(py::module &m) { + py::class_(m, "Hermes") + .def(py::init<>()) + .def("ClientInit", &Hermes::ClientInit) + .def("IsInitialized", &Hermes::IsInitialized) + .def("GetTagId", &Hermes::GetTagId) + .def("CollectMetadataSnapshot", &Hermes::CollectMetadataSnapshot); + m.def("TRANSPARENT_HERMES", &TRANSPARENT_HERMES_FUN); +} + PYBIND11_MODULE(py_hermes, m) { + BindUniqueId(m, "BlobId"); + BindUniqueId(m, "BucketId"); + BindUniqueId(m, "TagId"); + BindUniqueId(m, "TargetId"); + BindBlobInfo(m); + BindTargetStats(m); + BindMetadataTable(m); + BindHermes(m); } diff --git a/wrapper/python/test/unit/test_hermes.py b/wrapper/python/test/unit/test_hermes.py index fb803fca4..1a66d462f 100644 --- a/wrapper/python/test/unit/test_hermes.py +++ b/wrapper/python/test/unit/test_hermes.py @@ -1,8 +1,9 @@ from unittest import TestCase -from py_hermes import Hermes, MetadataTable +import py_hermes +# from py_hermes import Hermes, MetadataTable import pathlib import os class TestHermes(TestCase): def test_metadata_query(self): - pass + print("HERE?") From b6e932661164a781710d7c0483d4d0b68119f6bc Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 16 Oct 2023 07:53:55 -0500 Subject: [PATCH 188/191] Fix linter issues --- CMakeLists.txt | 4 +- benchmark/CMakeLists.txt | 4 +- hrun/include/hrun/api/hrun_client.h | 14 +++- hrun/include/hrun/api/hrun_runtime.h | 14 +++- hrun/include/hrun/api/manager.h | 14 +++- hrun/include/hrun/config/config.h | 14 +++- hrun/include/hrun/hrun_constants.h | 14 +++- hrun/include/hrun/hrun_namespace.h | 14 +++- hrun/include/hrun/hrun_types.h | 14 +++- hrun/include/hrun/network/local_serialize.h | 14 +++- hrun/include/hrun/network/serialize.h | 14 +++- hrun/include/hrun/queue_manager/queue.h | 14 +++- .../hrun/queue_manager/queue_factory.h | 14 +++- .../hrun/queue_manager/queue_manager.h | 14 +++- .../hrun/queue_manager/queue_manager_client.h | 14 +++- .../queue_manager/queue_manager_runtime.h | 14 +++- hrun/include/hrun/task_registry/task.h | 14 +++- hrun/include/hrun/task_registry/task_lib.h | 14 +++- .../hrun/task_registry/task_registry.h | 14 +++- .../include/hrun/work_orchestrator/affinity.h | 14 +++- .../hrun/work_orchestrator/scheduler.h | 14 +++- .../work_orchestrator/work_orchestrator.h | 14 +++- hrun/include/hrun/work_orchestrator/worker.h | 14 +++- hrun/src/hrun_client.cc | 14 +++- hrun/src/hrun_runtime.cc | 14 +++- hrun/src/hrun_start_runtime.cc | 14 +++- hrun/src/hrun_stop_runtime.cc | 14 +++- hrun/src/worker.cc | 14 +++- .../TASK_NAME/include/TASK_NAME/TASK_NAME.h | 14 +++- .../tasks_required/TASK_NAME/src/TASK_NAME.cc | 14 +++- .../include/hrun_admin/hrun_admin.h | 4 -- .../include/hrun_admin/hrun_admin_tasks.h | 4 -- .../hrun_admin/src/hrun_admin.cc | 14 +++- .../include/proc_queue/proc_queue.h | 14 +++- .../include/proc_queue/proc_queue_tasks.h | 14 +++- .../proc_queue/src/proc_queue.cc | 14 +++- .../remote_queue/src/remote_queue.cc | 14 +++- .../small_message/src/small_message.cc | 14 +++- .../src/worch_proc_round_robin.cc | 14 +++- .../src/worch_queue_round_robin.cc | 14 +++- include/hermes/CPPLINT.cfg | 2 + include/hermes/bucket.h | 57 ++++++++++----- include/hermes/config_client.h | 6 +- include/hermes/config_manager.h | 36 +++++++--- include/hermes/config_server.h | 15 ++-- include/hermes/hermes.h | 14 +++- include/hermes/hermes_types.h | 16 +++-- include/hermes/slab_allocator.h | 24 +++++-- include/hermes/status.h | 2 +- include/hermes/statuses.h | 19 +++-- scripts/lint.sh | 4 +- src/hermes_config_manager.cc | 14 +++- test/unit/boost/CMakeLists.txt | 4 +- test/unit/boost/test_boost.cc | 72 ++++++++++--------- test/unit/external/CPPLINT.cfg | 1 + test/unit/hermes/CMakeLists.txt | 4 +- test/unit/hermes/test_bucket.cc | 28 +++++--- test/unit/ipc/CMakeLists.txt | 4 +- test/unit/ipc/test_finalize.cc | 16 +++-- test/unit/ipc/test_ipc.cc | 24 +++++-- test/unit/ipc/test_serialize.cc | 17 +++-- wrapper/python/cpp/py_hermes.cpp | 14 +++- wrapper/python/test/unit/test_hermes.py | 9 ++- 63 files changed, 675 insertions(+), 247 deletions(-) create mode 100644 include/hermes/CPPLINT.cfg create mode 100644 test/unit/external/CPPLINT.cfg diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e189e1d5..9bc09c3fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -194,8 +194,8 @@ add_custom_target(lint COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/lint.sh ${CMAKE_ # General function used to hook ctest to python test tool lib function(pytest test_type test_name) set(script ${CMAKE_SOURCE_DIR}/scripts/ci/py_hermes_ci/bin/run_test) - add_test(NAME ${test_name} - COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER}) +# add_test(NAME ${test_name} +# COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER}) endfunction() enable_testing() diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 6055ce288..ee640a1ae 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -42,8 +42,8 @@ target_link_libraries(hermes_api_bench #------------------------------------------------------------------------------ # STRING TESTS -add_test(NAME test_performance COMMAND - ${CMAKE_BINARY_DIR}/bin/test_performance_exec "TestIpc") +#add_test(NAME test_performance COMMAND +# ${CMAKE_BINARY_DIR}/bin/test_performance_exec "TestIpc") #------------------------------------------------------------------------------ # Install Targets diff --git a/hrun/include/hrun/api/hrun_client.h b/hrun/include/hrun/api/hrun_client.h index 87d41fd06..202259052 100644 --- a/hrun/include/hrun/api/hrun_client.h +++ b/hrun/include/hrun/api/hrun_client.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/23/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_CLIENT_HRUN_CLIENT_H_ #define HRUN_INCLUDE_HRUN_CLIENT_HRUN_CLIENT_H_ diff --git a/hrun/include/hrun/api/hrun_runtime.h b/hrun/include/hrun/api/hrun_runtime.h index e057d6e08..d6d949b83 100644 --- a/hrun/include/hrun/api/hrun_runtime.h +++ b/hrun/include/hrun/api/hrun_runtime.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/27/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_CLIENT_HRUN_SERVER_H_ #define HRUN_INCLUDE_HRUN_CLIENT_HRUN_SERVER_H_ diff --git a/hrun/include/hrun/api/manager.h b/hrun/include/hrun/api/manager.h index b24c45add..0f2ee22c9 100644 --- a/hrun/include/hrun/api/manager.h +++ b/hrun/include/hrun/api/manager.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/27/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_MANAGER_MANAGER_H_ #define HRUN_INCLUDE_HRUN_MANAGER_MANAGER_H_ diff --git a/hrun/include/hrun/config/config.h b/hrun/include/hrun/config/config.h index 77ac1d71d..08c3d9942 100644 --- a/hrun/include/hrun/config/config.h +++ b/hrun/include/hrun/config/config.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/17/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_SRC_CONFIG_H_ #define HRUN_SRC_CONFIG_H_ diff --git a/hrun/include/hrun/hrun_constants.h b/hrun/include/hrun/hrun_constants.h index 8183cb2a7..621cf625d 100644 --- a/hrun/include/hrun/hrun_constants.h +++ b/hrun/include/hrun/hrun_constants.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/22/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ #define HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ diff --git a/hrun/include/hrun/hrun_namespace.h b/hrun/include/hrun/hrun_namespace.h index 20053e3bd..88ec1e8a1 100644 --- a/hrun/include/hrun/hrun_namespace.h +++ b/hrun/include/hrun/hrun_namespace.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 8/14/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ #define HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ diff --git a/hrun/include/hrun/hrun_types.h b/hrun/include/hrun/hrun_types.h index f0c72cab5..2e0952d9d 100644 --- a/hrun/include/hrun/hrun_types.h +++ b/hrun/include/hrun/hrun_types.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/22/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_HRUN_TYPES_H_ #define HRUN_INCLUDE_HRUN_HRUN_TYPES_H_ diff --git a/hrun/include/hrun/network/local_serialize.h b/hrun/include/hrun/network/local_serialize.h index dbf94dc6d..f0a870be9 100644 --- a/hrun/include/hrun/network/local_serialize.h +++ b/hrun/include/hrun/network/local_serialize.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 9/5/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_NETWORK_LOCAL_SERIALIZE_H_ #define HRUN_INCLUDE_HRUN_NETWORK_LOCAL_SERIALIZE_H_ diff --git a/hrun/include/hrun/network/serialize.h b/hrun/include/hrun/network/serialize.h index d0fed8a80..873d03a57 100644 --- a/hrun/include/hrun/network/serialize.h +++ b/hrun/include/hrun/network/serialize.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 8/7/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_NETWORK_SERIALIZE_H_ #define HRUN_INCLUDE_HRUN_NETWORK_SERIALIZE_H_ diff --git a/hrun/include/hrun/queue_manager/queue.h b/hrun/include/hrun/queue_manager/queue.h index ce4c0e49c..09fd3213a 100644 --- a/hrun/include/hrun/queue_manager/queue.h +++ b/hrun/include/hrun/queue_manager/queue.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/27/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_H_ #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_H_ diff --git a/hrun/include/hrun/queue_manager/queue_factory.h b/hrun/include/hrun/queue_manager/queue_factory.h index 7c974c61d..9f712c893 100644 --- a/hrun/include/hrun/queue_manager/queue_factory.h +++ b/hrun/include/hrun/queue_manager/queue_factory.h @@ -1,6 +1,14 @@ -// -// Created by llogan on 7/1/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ diff --git a/hrun/include/hrun/queue_manager/queue_manager.h b/hrun/include/hrun/queue_manager/queue_manager.h index 8e82025ba..1a714e0b3 100644 --- a/hrun/include/hrun/queue_manager/queue_manager.h +++ b/hrun/include/hrun/queue_manager/queue_manager.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/28/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_H_ #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_H_ diff --git a/hrun/include/hrun/queue_manager/queue_manager_client.h b/hrun/include/hrun/queue_manager/queue_manager_client.h index 39d0eaf64..0ff0111e9 100644 --- a/hrun/include/hrun/queue_manager/queue_manager_client.h +++ b/hrun/include/hrun/queue_manager/queue_manager_client.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/28/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ diff --git a/hrun/include/hrun/queue_manager/queue_manager_runtime.h b/hrun/include/hrun/queue_manager/queue_manager_runtime.h index ee86f0b14..f60f55b6c 100644 --- a/hrun/include/hrun/queue_manager/queue_manager_runtime.h +++ b/hrun/include/hrun/queue_manager/queue_manager_runtime.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/28/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_SERVER_H_ diff --git a/hrun/include/hrun/task_registry/task.h b/hrun/include/hrun/task_registry/task.h index 0348e53d4..6a6a3d451 100644 --- a/hrun/include/hrun/task_registry/task.h +++ b/hrun/include/hrun/task_registry/task.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/23/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_REQUEST_H_ #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_REQUEST_H_ diff --git a/hrun/include/hrun/task_registry/task_lib.h b/hrun/include/hrun/task_registry/task_lib.h index 1c140a3db..a6cce4743 100644 --- a/hrun/include/hrun/task_registry/task_lib.h +++ b/hrun/include/hrun/task_registry/task_lib.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/23/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_TASK_TASK_H_ #define HRUN_INCLUDE_HRUN_TASK_TASK_H_ diff --git a/hrun/include/hrun/task_registry/task_registry.h b/hrun/include/hrun/task_registry/task_registry.h index fe34dda4f..af7a9ddaa 100644 --- a/hrun/include/hrun/task_registry/task_registry.h +++ b/hrun/include/hrun/task_registry/task_registry.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/23/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_TASK_TASK_REGISTRY_H_ #define HRUN_INCLUDE_HRUN_TASK_TASK_REGISTRY_H_ diff --git a/hrun/include/hrun/work_orchestrator/affinity.h b/hrun/include/hrun/work_orchestrator/affinity.h index 3a480e789..55f54fc1f 100644 --- a/hrun/include/hrun/work_orchestrator/affinity.h +++ b/hrun/include/hrun/work_orchestrator/affinity.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/5/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_AFFINITY_H_ #define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_AFFINITY_H_ diff --git a/hrun/include/hrun/work_orchestrator/scheduler.h b/hrun/include/hrun/work_orchestrator/scheduler.h index 7e23c1863..25bdca4c8 100644 --- a/hrun/include/hrun/work_orchestrator/scheduler.h +++ b/hrun/include/hrun/work_orchestrator/scheduler.h @@ -1,6 +1,14 @@ -// -// Created by llogan on 7/2/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ #define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ diff --git a/hrun/include/hrun/work_orchestrator/work_orchestrator.h b/hrun/include/hrun/work_orchestrator/work_orchestrator.h index 56394d3f0..33d2182e1 100644 --- a/hrun/include/hrun/work_orchestrator/work_orchestrator.h +++ b/hrun/include/hrun/work_orchestrator/work_orchestrator.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/23/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ #define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORK_ORCHESTRATOR_H_ diff --git a/hrun/include/hrun/work_orchestrator/worker.h b/hrun/include/hrun/work_orchestrator/worker.h index 59f825e97..92e29aeb6 100644 --- a/hrun/include/hrun/work_orchestrator/worker.h +++ b/hrun/include/hrun/work_orchestrator/worker.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/27/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORKER_H_ #define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_WORKER_H_ diff --git a/hrun/src/hrun_client.cc b/hrun/src/hrun_client.cc index 7c9aa5015..55aaeaf47 100644 --- a/hrun/src/hrun_client.cc +++ b/hrun/src/hrun_client.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/27/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hermes_shm/util/singleton.h" #include "hrun/api/hrun_client.h" diff --git a/hrun/src/hrun_runtime.cc b/hrun/src/hrun_runtime.cc index 8cd958036..6484ff3df 100644 --- a/hrun/src/hrun_runtime.cc +++ b/hrun/src/hrun_runtime.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/17/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hermes_shm/util/singleton.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/src/hrun_start_runtime.cc b/hrun/src/hrun_start_runtime.cc index 938c4b583..8562f2be9 100644 --- a/hrun/src/hrun_start_runtime.cc +++ b/hrun/src/hrun_start_runtime.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/17/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hermes_shm/util/singleton.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/src/hrun_stop_runtime.cc b/hrun/src/hrun_stop_runtime.cc index 629f5d58a..079533cab 100644 --- a/hrun/src/hrun_stop_runtime.cc +++ b/hrun/src/hrun_stop_runtime.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/22/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" diff --git a/hrun/src/worker.cc b/hrun/src/worker.cc index b07734bb4..0b191b680 100644 --- a/hrun/src/worker.cc +++ b/hrun/src/worker.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/27/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun/api/hrun_runtime.h" #include "hrun/work_orchestrator/worker.h" diff --git a/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h b/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h index a59a86814..e2f541956 100644 --- a/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h +++ b/hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASK_NAME_H_ #define HRUN_TASK_NAME_H_ diff --git a/hrun/tasks_required/TASK_NAME/src/TASK_NAME.cc b/hrun/tasks_required/TASK_NAME/src/TASK_NAME.cc index 59fa67256..43c5aec9f 100644 --- a/hrun/tasks_required/TASK_NAME/src/TASK_NAME.cc +++ b/hrun/tasks_required/TASK_NAME/src/TASK_NAME.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h index b6d5ca980..0762ba70f 100644 --- a/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h +++ b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin.h @@ -1,7 +1,3 @@ -// -// Created by lukemartinlogan on 6/29/23. -// - #ifndef HRUN_TASKS_HRUN_ADMIN_HRUN_ADMIN_H_ #define HRUN_TASKS_HRUN_ADMIN_HRUN_ADMIN_H_ diff --git a/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h index f8a962c1c..602185310 100644 --- a/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h +++ b/hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_tasks.h @@ -1,7 +1,3 @@ -// -// Created by lukemartinlogan on 8/14/23. -// - #ifndef HRUN_TASKS_HRUN_ADMIN_INCLUDE_HRUN_ADMIN_HRUN_ADMIN_TASKS_H_ #define HRUN_TASKS_HRUN_ADMIN_INCLUDE_HRUN_ADMIN_HRUN_ADMIN_TASKS_H_ diff --git a/hrun/tasks_required/hrun_admin/src/hrun_admin.cc b/hrun/tasks_required/hrun_admin/src/hrun_admin.cc index bc270d13e..193125aab 100644 --- a/hrun/tasks_required/hrun_admin/src/hrun_admin.cc +++ b/hrun/tasks_required/hrun_admin/src/hrun_admin.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue.h b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue.h index 1c8635360..1db95bc66 100644 --- a/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue.h +++ b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_proc_queue_H_ #define HRUN_proc_queue_H_ diff --git a/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h index 339baf22a..94a8406a8 100644 --- a/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h +++ b/hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_tasks.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 8/11/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ #define HRUN_TASKS_TASK_TEMPL_INCLUDE_proc_queue_proc_queue_TASKS_H_ diff --git a/hrun/tasks_required/proc_queue/src/proc_queue.cc b/hrun/tasks_required/proc_queue/src/proc_queue.cc index 56f7d215a..95a6c3260 100644 --- a/hrun/tasks_required/proc_queue/src/proc_queue.cc +++ b/hrun/tasks_required/proc_queue/src/proc_queue.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/tasks_required/remote_queue/src/remote_queue.cc b/hrun/tasks_required/remote_queue/src/remote_queue.cc index f3cda24c7..1de787101 100644 --- a/hrun/tasks_required/remote_queue/src/remote_queue.cc +++ b/hrun/tasks_required/remote_queue/src/remote_queue.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/tasks_required/small_message/src/small_message.cc b/hrun/tasks_required/small_message/src/small_message.cc index 0c5a815d2..30d737b28 100644 --- a/hrun/tasks_required/small_message/src/small_message.cc +++ b/hrun/tasks_required/small_message/src/small_message.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc b/hrun/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc index 41b02cccd..0c8be3c24 100644 --- a/hrun/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc +++ b/hrun/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" #include "hrun/api/hrun_runtime.h" diff --git a/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc b/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc index 1089e30eb..15596232f 100644 --- a/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc +++ b/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/29/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hrun_admin/hrun_admin.h" #include "hrun/api/hrun_runtime.h" diff --git a/include/hermes/CPPLINT.cfg b/include/hermes/CPPLINT.cfg new file mode 100644 index 000000000..4bca945b7 --- /dev/null +++ b/include/hermes/CPPLINT.cfg @@ -0,0 +1,2 @@ +exclude_files=.*config_client_default +exclude_files=.*config_server_default \ No newline at end of file diff --git a/include/hermes/bucket.h b/include/hermes/bucket.h index 6418802d5..f607d6ef6 100644 --- a/include/hermes/bucket.h +++ b/include/hermes/bucket.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/9/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ #define HRUN_TASKS_HERMES_CONF_INCLUDE_HERMES_CONF_BUCKET_H_ @@ -44,8 +52,9 @@ class Bucket { mdm_ = &HERMES_CONF->mdm_; blob_mdm_ = &HERMES_CONF->blob_mdm_; bkt_mdm_ = &HERMES_CONF->bkt_mdm_; - id_ = bkt_mdm_->GetOrCreateTagRoot(hshm::charbuf(bkt_name), true, - std::vector(), backend_size, flags); + id_ = bkt_mdm_->GetOrCreateTagRoot( + hshm::charbuf(bkt_name), true, + std::vector(), backend_size, flags); name_ = bkt_name; } @@ -62,8 +71,9 @@ class Bucket { mdm_ = &HERMES_CONF->mdm_; blob_mdm_ = &HERMES_CONF->blob_mdm_; bkt_mdm_ = &HERMES_CONF->bkt_mdm_; - id_ = bkt_mdm_->GetOrCreateTagRoot(hshm::charbuf(bkt_name), true, - std::vector(), backend_size, flags); + id_ = bkt_mdm_->GetOrCreateTagRoot( + hshm::charbuf(bkt_name), true, + std::vector(), backend_size, flags); name_ = bkt_name; } @@ -215,7 +225,8 @@ class Bucket { size_t blob_off, Context &ctx) { BlobId blob_id = orig_blob_id; - bitfield32_t flags, task_flags(TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY); + bitfield32_t flags, task_flags( + TASK_FIRE_AND_FORGET | TASK_DATA_OWNER | TASK_LOW_LATENCY); // Copy data to shared memory LPointer p = HRUN_CLIENT->AllocateBuffer(blob.size()); char *data = p.ptr_; @@ -233,7 +244,8 @@ class Bucket { } LPointer> push_task; push_task = blob_mdm_->AsyncPutBlobRoot(id_, blob_name_buf, - blob_id, blob_off, blob.size(), p.shm_, ctx.blob_score_, + blob_id, blob_off, blob.size(), + p.shm_, ctx.blob_score_, flags.bits_, ctx, task_flags.bits_); if constexpr (!ASYNC) { if (flags.Any(HERMES_GET_BLOB_ID)) { @@ -270,9 +282,11 @@ class Bucket { const T &blob, Context &ctx) { if constexpr(std::is_same_v) { - return BasePut(blob_name, BlobId::GetNull(), blob, 0, ctx); + return BasePut( + blob_name, BlobId::GetNull(), blob, 0, ctx); } else { - return SrlBasePut(blob_name, BlobId::GetNull(), blob, ctx); + return SrlBasePut( + blob_name, BlobId::GetNull(), blob, ctx); } } @@ -327,7 +341,9 @@ class Bucket { const Blob &blob, size_t blob_off, Context &ctx) { - return BasePut(blob_name, BlobId::GetNull(), blob, blob_off, ctx); + return BasePut(blob_name, + BlobId::GetNull(), + blob, blob_off, ctx); } /** @@ -367,7 +383,9 @@ class Bucket { LPointer p = HRUN_CLIENT->AllocateBuffer(blob.size()); char *data = p.ptr_; memcpy(data, blob.data(), blob.size()); - bkt_mdm_->AppendBlobRoot(id_, blob.size(), p.shm_, page_size, ctx.blob_score_, ctx.node_id_, ctx); + bkt_mdm_->AppendBlobRoot( + id_, blob.size(), p.shm_, page_size, + ctx.blob_score_, ctx.node_id_, ctx); } /** @@ -408,7 +426,8 @@ class Bucket { * Get the current size of the blob in the bucket * */ size_t GetBlobSize(const std::string &name) { - return blob_mdm_->GetBlobSizeRoot(id_, hshm::charbuf(name), BlobId::GetNull()); + return blob_mdm_->GetBlobSizeRoot( + id_, hshm::charbuf(name), BlobId::GetNull()); } /** @@ -449,7 +468,8 @@ class Bucket { // TODO(llogan): make GetBlobSize work with blob_name size_t data_size = blob.size(); if (blob.size() == 0) { - data_size = blob_mdm_->GetBlobSizeRoot(id_, hshm::charbuf(blob_name), orig_blob_id); + data_size = blob_mdm_->GetBlobSizeRoot( + id_, hshm::charbuf(blob_name), orig_blob_id); blob.resize(data_size); } HILOG(kDebug, "Getting blob of size {}", data_size); @@ -576,14 +596,17 @@ class Bucket { * Determine if the bucket contains \a blob_id BLOB * */ bool ContainsBlob(const std::string &blob_name) { - BlobId new_blob_id = blob_mdm_->GetBlobIdRoot(id_, hshm::to_charbuf(blob_name)); + BlobId new_blob_id = blob_mdm_->GetBlobIdRoot( + id_, hshm::to_charbuf(blob_name)); return !new_blob_id.IsNull(); } /** * Rename \a blob_id blob to \a new_blob_name new name * */ - void RenameBlob(const BlobId &blob_id, std::string new_blob_name, Context &ctx) { + void RenameBlob(const BlobId &blob_id, + std::string new_blob_name, + Context &ctx) { blob_mdm_->RenameBlobRoot(id_, blob_id, hshm::to_charbuf(new_blob_name)); } diff --git a/include/hermes/config_client.h b/include/hermes/config_client.h index aab5db713..9dcf043d4 100644 --- a/include/hermes/config_client.h +++ b/include/hermes/config_client.h @@ -168,7 +168,8 @@ class ClientConfig : public BaseConfig { } if (yaml_conf["flushing_mode"]) { flushing_mode_ = - FlushingModeConv::GetEnum(yaml_conf["flushing_mode"].as()); + FlushingModeConv::GetEnum( + yaml_conf["flushing_mode"].as()); auto flush_mode_env = getenv("HERMES_FLUSH_MODE"); if (flush_mode_env) { flushing_mode_ = FlushingModeConv::GetEnum(flush_mode_env); @@ -188,7 +189,8 @@ class ClientConfig : public BaseConfig { path = hshm::ConfigParse::ExpandPath(path); path = stdfs::absolute(path).string(); if (yaml_conf["mode"]) { - conf.mode_ = AdapterModeConv::to_enum(yaml_conf["mode"].as()); + conf.mode_ = AdapterModeConv::to_enum( + yaml_conf["mode"].as()); } if (yaml_conf["page_size"]) { conf.page_size_ = hshm::ConfigParse::ParseSize( diff --git a/include/hermes/config_manager.h b/include/hermes/config_manager.h index cf69b70fd..1cf44a040 100644 --- a/include/hermes/config_manager.h +++ b/include/hermes/config_manager.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/9/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASKS_HERMES_INCLUDE_hermes_H_ #define HRUN_TASKS_HERMES_INCLUDE_hermes_H_ @@ -42,10 +50,15 @@ class ConfigurationManager { mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_mdm"); blob_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_blob_mdm"); bkt_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_bkt_mdm"); - op_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_op_mdm", bkt_mdm_.id_, blob_mdm_.id_); - stager_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_stager_mdm", blob_mdm_.id_); - blob_mdm_.SetBucketMdmRoot(DomainId::GetGlobal(), bkt_mdm_.id_, stager_mdm_.id_, op_mdm_.id_); - bkt_mdm_.SetBlobMdmRoot(DomainId::GetGlobal(), blob_mdm_.id_, stager_mdm_.id_); + op_mdm_.CreateRoot(DomainId::GetGlobal(), "hermes_op_mdm", + bkt_mdm_.id_, blob_mdm_.id_); + stager_mdm_.CreateRoot(DomainId::GetGlobal(), + "hermes_stager_mdm", blob_mdm_.id_); + blob_mdm_.SetBucketMdmRoot(DomainId::GetGlobal(), + bkt_mdm_.id_, + stager_mdm_.id_, op_mdm_.id_); + bkt_mdm_.SetBlobMdmRoot(DomainId::GetGlobal(), + blob_mdm_.id_, stager_mdm_.id_); is_initialized_ = true; } @@ -70,9 +83,12 @@ class ConfigurationManager { } // namespace hermes -#define HERMES_CONF hshm::Singleton<::hermes::ConfigurationManager>::GetInstance() -#define HERMES_CLIENT_CONF HERMES_CONF->client_config_ -#define HERMES_SERVER_CONF HERMES_CONF->server_config_ +#define HERMES_CONF \ +hshm::Singleton<::hermes::ConfigurationManager>::GetInstance() +#define HERMES_CLIENT_CONF \ +HERMES_CONF->client_config_ +#define HERMES_SERVER_CONF \ +HERMES_CONF->server_config_ /** Initialize client-side Hermes transparently */ static inline bool TRANSPARENT_HERMES() { diff --git a/include/hermes/config_server.h b/include/hermes/config_server.h index e9ea2e66e..9c6dbd68d 100644 --- a/include/hermes/config_server.h +++ b/include/hermes/config_server.h @@ -225,19 +225,24 @@ class ServerConfig : public BaseConfig { dev.is_shared_ = dev_info["is_shared_device"].as(); dev.block_size_ = - hshm::ConfigParse::ParseSize(dev_info["block_size"].as()); + hshm::ConfigParse::ParseSize( + dev_info["block_size"].as()); dev.capacity_ = - hshm::ConfigParse::ParseSize(dev_info["capacity"].as()); + hshm::ConfigParse::ParseSize( + dev_info["capacity"].as()); dev.bandwidth_ = - hshm::ConfigParse::ParseSize(dev_info["bandwidth"].as()); + hshm::ConfigParse::ParseSize( + dev_info["bandwidth"].as()); dev.latency_ = - hshm::ConfigParse::ParseLatency(dev_info["latency"].as()); + hshm::ConfigParse::ParseLatency( + dev_info["latency"].as()); std::vector size_vec; ParseVector>( dev_info["slab_sizes"], size_vec); dev.slab_sizes_.reserve(size_vec.size()); for (const std::string &size_str : size_vec) { - dev.slab_sizes_.emplace_back(hshm::ConfigParse::ParseSize(size_str)); + dev.slab_sizes_.emplace_back( + hshm::ConfigParse::ParseSize(size_str)); } } } diff --git a/include/hermes/hermes.h b/include/hermes/hermes.h index 72d110ed1..7cf45ffea 100644 --- a/include/hermes/hermes.h +++ b/include/hermes/hermes.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/9/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ #define HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_H_ diff --git a/include/hermes/hermes_types.h b/include/hermes/hermes_types.h index c00baa682..d27638f27 100644 --- a/include/hermes/hermes_types.h +++ b/include/hermes/hermes_types.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/8/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ #define HRUN_TASKS_HERMES_INCLUDE_HERMES_HERMES_TYPES_H_ @@ -186,7 +194,7 @@ class FlushingModeConv { }; /** A class with static constants */ -#define CONST_T inline const static +#define CONST_T static inline const class Constant { public: /** Hermes server environment variable */ diff --git a/include/hermes/slab_allocator.h b/include/hermes/slab_allocator.h index c3de53a6d..33ff777da 100644 --- a/include/hermes/slab_allocator.h +++ b/include/hermes/slab_allocator.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/13/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ #define HRUN_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ @@ -37,12 +45,14 @@ class SlabAllocator { ~SlabAllocator() = default; /** Initialize slab allocator */ - void Init(TargetId target_id, size_t dev_size, std::vector &slab_sizes_) { + void Init(TargetId target_id, + size_t dev_size, + std::vector &slab_sizes) { heap_ = 0; dev_size_ = dev_size; target_id_ = target_id; - slab_lists_.reserve(slab_sizes_.size()); - for (auto &slab_size : slab_sizes_) { + slab_lists_.reserve(slab_sizes.size()); + for (auto &slab_size : slab_sizes) { slab_lists_.emplace_back(); auto &slab = slab_lists_.back(); slab.slab_size_ = slab_size; @@ -147,4 +157,4 @@ class SlabAllocator { } // namespace hermes -#endif // HRUN_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ +#endif // HRUN_TASKS_HERMES_INCLUDE_HERMES_SLAB_ALLOCATOR_H_ diff --git a/include/hermes/status.h b/include/hermes/status.h index 06863c7ab..6e1ffb60c 100644 --- a/include/hermes/status.h +++ b/include/hermes/status.h @@ -47,5 +47,5 @@ class Status { } }; -} // namespace hermes::api +} // namespace hermes #endif // HERMES_STATUS_H_ diff --git a/include/hermes/statuses.h b/include/hermes/statuses.h index 595e63c90..30e130bce 100644 --- a/include/hermes/statuses.h +++ b/include/hermes/statuses.h @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/13/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #ifndef HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ #define HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ @@ -14,8 +22,9 @@ namespace hermes { STATUS_T NOT_IMPLEMENTED(0, "Function was not implemented"); STATUS_T DPE_PLACEMENT_SCHEMA_EMPTY(1, "DPE placement schema is empty"); STATUS_T DPE_NO_SPACE(1, "Placement failed. Non-fatal."); -STATUS_T DPE_MIN_IO_TIME_NO_SOLUTION(1, "DPE could not find solution for the minimize I/O time DPE"); +STATUS_T DPE_MIN_IO_TIME_NO_SOLUTION( + 1, "DPE could not find solution for the minimize I/O time DPE"); } // namespace hermes -#endif //HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ +#endif // HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ diff --git a/scripts/lint.sh b/scripts/lint.sh index 1bde6276f..9e40817ad 100644 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -2,5 +2,7 @@ HRUN_ROOT=$1 +echo "RUNNING CPPLINT" cpplint --recursive \ -"${HRUN_ROOT}/src" "${HRUN_ROOT}/include" "${HRUN_ROOT}/test" +"${HRUN_ROOT}/src" "${HRUN_ROOT}/include" "${HRUN_ROOT}/test" \ +--exclude="${HRUN_ROOT}/test/unit/external" \ No newline at end of file diff --git a/src/hermes_config_manager.cc b/src/hermes_config_manager.cc index 134861c35..163876111 100644 --- a/src/hermes_config_manager.cc +++ b/src/hermes_config_manager.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 7/28/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hermes/config_manager.h" diff --git a/test/unit/boost/CMakeLists.txt b/test/unit/boost/CMakeLists.txt index b3b17fcde..019a213c8 100644 --- a/test/unit/boost/CMakeLists.txt +++ b/test/unit/boost/CMakeLists.txt @@ -21,8 +21,8 @@ target_link_libraries(test_boost_exec # Test Cases #------------------------------------------------------------------------------ -add_test(NAME test_boost COMMAND - ${CMAKE_BINARY_DIR}/bin/test_messages "TestBoost") +#add_test(NAME test_boost COMMAND +# ${CMAKE_BINARY_DIR}/bin/test_messages "TestBoost") #------------------------------------------------------------------------------ # Install Targets diff --git a/test/unit/boost/test_boost.cc b/test/unit/boost/test_boost.cc index 84c517341..4b5a525ac 100644 --- a/test/unit/boost/test_boost.cc +++ b/test/unit/boost/test_boost.cc @@ -1,6 +1,14 @@ -// -// Created by llogan on 7/1/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "basic_test.h" #include @@ -30,37 +38,35 @@ TEST_CASE("TestBoostFiber") { } template< std::size_t Max, std::size_t Default, std::size_t Min > -class simple_stack_allocator -{ +class simple_stack_allocator { public: - static std::size_t maximum_stacksize() - { return Max; } - - static std::size_t default_stacksize() - { return Default; } - - static std::size_t minimum_stacksize() - { return Min; } + static std::size_t maximum_stacksize() { + return Max; + } - void * allocate( std::size_t size) const - { - BOOST_ASSERT( minimum_stacksize() <= size); - BOOST_ASSERT( maximum_stacksize() >= size); + static std::size_t default_stacksize() { + return Default; + } - void * limit = malloc( size); - if ( ! limit) throw std::bad_alloc(); + static std::size_t minimum_stacksize() { + return Min; + } - return static_cast< char * >( limit) + size; + void *allocate(std::size_t size) const { + BOOST_ASSERT(minimum_stacksize() <= size); + BOOST_ASSERT(maximum_stacksize() >= size); + void *limit = malloc(size); + if (!limit) throw std::bad_alloc(); + return static_cast(limit) + size; } - void deallocate( void * vp, std::size_t size) const - { - BOOST_ASSERT( vp); - BOOST_ASSERT( minimum_stacksize() <= size); - BOOST_ASSERT( maximum_stacksize() >= size); + void deallocate(void * vp, std::size_t size) const { + BOOST_ASSERT(vp); + BOOST_ASSERT(minimum_stacksize() <= size); + BOOST_ASSERT(maximum_stacksize() >= size); - void * limit = static_cast< char * >( vp) - size; - free( limit); + void *limit = static_cast(vp) - size; + free(limit); } }; @@ -74,12 +80,12 @@ namespace bctx = boost::context::detail; bctx::transfer_t shared_xfer; -void f3( bctx::transfer_t t) { +void f3(bctx::transfer_t t) { ++value1; shared_xfer = t; shared_xfer = bctx::jump_fcontext(shared_xfer.fctx, 0); ++value1; - shared_xfer = bctx::jump_fcontext( shared_xfer.fctx, shared_xfer.data); + shared_xfer = bctx::jump_fcontext(shared_xfer.fctx, shared_xfer.data); } @@ -104,9 +110,9 @@ TEST_CASE("TestBoostFcontext") { HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } -using namespace boost::coroutines2; +namespace co = boost::coroutines2; -void myCoroutine(coroutine::push_type& yield) { +void myCoroutine(co::coroutine::push_type& yield) { for (int i = 1; i <= 5; ++i) { yield(); } @@ -118,10 +124,10 @@ TEST_CASE("TestBoostCoroutine") { size_t ops = (1 << 20); for (size_t i = 0; i < ops; ++i) { - coroutine::pull_type myCoroutineInstance(myCoroutine); + co::coroutine::pull_type myCoroutineInstance(myCoroutine); myCoroutineInstance(); } t.Pause(); HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); -} \ No newline at end of file +} diff --git a/test/unit/external/CPPLINT.cfg b/test/unit/external/CPPLINT.cfg new file mode 100644 index 000000000..ff51e24b9 --- /dev/null +++ b/test/unit/external/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=.*build \ No newline at end of file diff --git a/test/unit/hermes/CMakeLists.txt b/test/unit/hermes/CMakeLists.txt index 06e1418b4..a0ab24237 100644 --- a/test/unit/hermes/CMakeLists.txt +++ b/test/unit/hermes/CMakeLists.txt @@ -22,8 +22,8 @@ target_link_libraries(test_hermes_exec #------------------------------------------------------------------------------ # STRING TESTS -add_test(NAME test_ipc COMMAND - ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") +#add_test(NAME test_ipc COMMAND +# ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") #------------------------------------------------------------------------------ # Install Targets diff --git a/test/unit/hermes/test_bucket.cc b/test/unit/hermes/test_bucket.cc index 2c908ee97..941ea4bac 100644 --- a/test/unit/hermes/test_bucket.cc +++ b/test/unit/hermes/test_bucket.cc @@ -1,6 +1,14 @@ -// -// Created by llogan on 7/1/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "basic_test.h" #include "hrun/api/hrun_client.h" @@ -42,7 +50,8 @@ TEST_CASE("TestHermesPut1n") { // Put a blob hermes::Blob blob(KILOBYTES(4)); memset(blob.data(), i % 256, blob.size()); - hermes::BlobId blob_id = bkt.Put(std::to_string(i % max_blobs), blob, ctx); + hermes::BlobId blob_id = + bkt.Put(std::to_string(i % max_blobs), blob, ctx); // Get a blob HILOG(kInfo, "Put {} returned successfully", i); @@ -147,8 +156,10 @@ TEST_CASE("TestHermesPartialPutGet") { memset(rblob.data(), (i + 1) % 256, rblob.size()); // PartialPut a blob - hermes::BlobId lblob_id = bkt.PartialPut(std::to_string(i), lblob, 0, ctx); - hermes::BlobId rblob_id = bkt.PartialPut(std::to_string(i), rblob, half_blob, ctx); + hermes::BlobId lblob_id = + bkt.PartialPut(std::to_string(i), lblob, 0, ctx); + hermes::BlobId rblob_id = + bkt.PartialPut(std::to_string(i), rblob, half_blob, ctx); REQUIRE(lblob_id == rblob_id); // PartialGet a blob @@ -453,7 +464,8 @@ TEST_CASE("TestHermesDataStager") { using hermes::data_stager::BinaryFileStager; hermes::Context ctx; ctx.flags_.SetBits(HERMES_IS_FILE); - hshm::charbuf url = BinaryFileStager::BuildFileUrl("/tmp/test.txt", page_size); + hshm::charbuf url = + BinaryFileStager::BuildFileUrl("/tmp/test.txt", page_size); hermes::Bucket bkt(url.str(), file_size, HERMES_IS_FILE); // Put a few blobs in the bucket @@ -559,4 +571,4 @@ TEST_CASE("TestHermesCollectMetadata") { REQUIRE(table.bkt_info_.size() == 1); REQUIRE(table.target_info_.size() >= 4); MPI_Barrier(MPI_COMM_WORLD); -} \ No newline at end of file +} diff --git a/test/unit/ipc/CMakeLists.txt b/test/unit/ipc/CMakeLists.txt index f3cafba1e..d6eff102a 100644 --- a/test/unit/ipc/CMakeLists.txt +++ b/test/unit/ipc/CMakeLists.txt @@ -24,8 +24,8 @@ target_link_libraries(test_ipc_exec #------------------------------------------------------------------------------ # STRING TESTS -add_test(NAME test_ipc COMMAND - ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") +#add_test(NAME test_ipc COMMAND +# ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") #------------------------------------------------------------------------------ # Install Targets diff --git a/test/unit/ipc/test_finalize.cc b/test/unit/ipc/test_finalize.cc index 94dc67576..4e3ece644 100644 --- a/test/unit/ipc/test_finalize.cc +++ b/test/unit/ipc/test_finalize.cc @@ -1,6 +1,14 @@ -// -// Created by llogan on 7/1/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "basic_test.h" #include "hrun/api/hrun_client.h" @@ -8,4 +16,4 @@ TEST_CASE("TestFinalize") { HRUN_ADMIN->AsyncStopRuntimeRoot(hrun::DomainId::GetGlobal()); -} \ No newline at end of file +} diff --git a/test/unit/ipc/test_ipc.cc b/test/unit/ipc/test_ipc.cc index a2f40633c..f4518757f 100644 --- a/test/unit/ipc/test_ipc.cc +++ b/test/unit/ipc/test_ipc.cc @@ -1,6 +1,14 @@ -// -// Created by llogan on 7/1/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "basic_test.h" #include @@ -133,8 +141,10 @@ TEST_CASE("TestIO") { HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); } -//TEST_CASE("TestHostfile") { -// for (u32 node_id = 1; node_id < HRUN_THALLIUM->rpc_->hosts_.size() + 1; ++node_id) { -// HILOG(kInfo, "Node {}: {}", node_id, HRUN_THALLIUM->GetServerName(node_id)); +// TEST_CASE("TestHostfile") { +// for (u32 node_id = 1; node_id < +// HRUN_THALLIUM->rpc_->hosts_.size() + 1; ++node_id) { +// HILOG(kInfo, "Node {}: {}", node_id, +// HRUN_THALLIUM->GetServerName(node_id)); // } -//} \ No newline at end of file +// } diff --git a/test/unit/ipc/test_serialize.cc b/test/unit/ipc/test_serialize.cc index 05e83b1c7..607df85bd 100644 --- a/test/unit/ipc/test_serialize.cc +++ b/test/unit/ipc/test_serialize.cc @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 8/7/23. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "basic_test.h" #include "hrun/network/serialize.h" @@ -27,7 +35,6 @@ struct TestObj : public Task, TaskFlags { template void serialize(Ar &ar) { - } /** (De)serialize message call */ @@ -60,4 +67,4 @@ TEST_CASE("TestSerialize") { in >> obj2; REQUIRE(obj == obj2); -} \ No newline at end of file +} diff --git a/wrapper/python/cpp/py_hermes.cpp b/wrapper/python/cpp/py_hermes.cpp index 17e6a89fa..86059d8d9 100644 --- a/wrapper/python/cpp/py_hermes.cpp +++ b/wrapper/python/cpp/py_hermes.cpp @@ -1,6 +1,14 @@ -// -// Created by lukemartinlogan on 6/24/2023. -// +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include #include diff --git a/wrapper/python/test/unit/test_hermes.py b/wrapper/python/test/unit/test_hermes.py index 1a66d462f..7f3a3e7a8 100644 --- a/wrapper/python/test/unit/test_hermes.py +++ b/wrapper/python/test/unit/test_hermes.py @@ -1,9 +1,12 @@ from unittest import TestCase -import py_hermes -# from py_hermes import Hermes, MetadataTable +from py_hermes import Hermes, TRANSPARENT_HERMES import pathlib import os class TestHermes(TestCase): def test_metadata_query(self): - print("HERE?") + TRANSPARENT_HERMES() + hermes = Hermes() + mdm = hermes.CollectMetadataSnapshot() + print(mdm.blob_info) + print("Done") From 0f8d146ebb3f9026a96b2381d3dd866814368b84 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 16 Oct 2023 07:55:46 -0500 Subject: [PATCH 189/191] Move lint and docs to ci --- CMakeLists.txt | 2 +- {scripts => ci}/docs.sh | 0 {scripts => ci}/lint.sh | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {scripts => ci}/docs.sh (100%) rename {scripts => ci}/lint.sh (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bc09c3fc..37b728f66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -186,7 +186,7 @@ add_subdirectory(hermes_adapters) add_subdirectory(tasks) add_subdirectory(benchmark) add_subdirectory(wrapper) -add_custom_target(lint COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/lint.sh ${CMAKE_SOURCE_DIR}) +add_custom_target(lint COMMAND bash ${CMAKE_SOURCE_DIR}/ci/lint.sh ${CMAKE_SOURCE_DIR}) #----------------------------------------------------------------------------- # Build + Enable Testing diff --git a/scripts/docs.sh b/ci/docs.sh similarity index 100% rename from scripts/docs.sh rename to ci/docs.sh diff --git a/scripts/lint.sh b/ci/lint.sh similarity index 100% rename from scripts/lint.sh rename to ci/lint.sh From 821905e6e0acda1db91f0d1ccb108913d83bea6a Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 16 Oct 2023 08:15:37 -0500 Subject: [PATCH 190/191] Improve actions --- .github/workflows/main.yml | 8 ++++---- ci/{install_hshm.sh => build_hermes.sh} | 11 ++++------- ci/install_deps.sh | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) rename ci/{install_hshm.sh => build_hermes.sh} (80%) mode change 100644 => 100755 ci/install_deps.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1b37ea5aa..28705e6da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,7 +52,7 @@ jobs: with: path: | ~/build - key: ${{ runner.os }}-${{ hashFiles('wrapper', 'traits', 'test', 'src', 'hermes_shm', 'benchmarks', 'adapter') }} + key: ${{ runner.os }}-${{ hashFiles('wrapper', 'test', 'tasks', 'src', 'include', 'hrun', 'hermes_adapters', 'benchmark') }} - name: Install APT Dependencies run: | @@ -71,9 +71,9 @@ jobs: if: steps.spack-cache.outputs.cache-hit != 'true' run: ci/install_deps.sh - - name: Build - if: steps.hermes-cache.outputs.cache-hit != 'true' - run: ci/build_hermes.sh +# - name: Build +# if: steps.hermes-cache.outputs.cache-hit != 'true' +# run: ci/build_hermes.sh # # - name: Test # run: bash ci/test_hermes.sh diff --git a/ci/install_hshm.sh b/ci/build_hermes.sh similarity index 80% rename from ci/install_hshm.sh rename to ci/build_hermes.sh index 3a0b2be1d..a13d3c167 100755 --- a/ci/install_hshm.sh +++ b/ci/build_hermes.sh @@ -15,15 +15,12 @@ SPACK_DIR=${INSTALL_DIR}/spack mkdir -p "${HOME}/install" mkdir build cd build -spack load --only dependencies hermes_shm +spack load hermes_shm cmake ../ \ -DCMAKE_BUILD_TYPE=Debug \ --DHERMES_ENABLE_COVERAGE=ON \ --DHERMES_ENABLE_DOXYGEN=ON \ --DBUILD_HSHM_BENCHMARKS=ON \ --DBUILD_HSHM_TESTS=ON \ --DCMAKE_INSTALL_PREFIX=${HOME}/install +-DCMAKE_INSTALL_PREFIX="${HOME}/install" make -j8 +make install export CXXFLAGS=-Wall ctest -VV @@ -39,7 +36,7 @@ export CMAKE_PREFIX_PATH="${INSTALL_PREFIX}:${CMAKE_PREFIX_PATH}" export CXXFLAGS="-I${INSTALL_PREFIX}/include:${CXXFLAGS}" # Run make install unit test -cd ci/external +cd test/unit/external mkdir build cd build cmake ../ diff --git a/ci/install_deps.sh b/ci/install_deps.sh old mode 100644 new mode 100755 index 1b8b819a2..26cc2462d --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -37,5 +37,5 @@ cp scripts/ci/packages.yaml ${SPACK_DIR}/etc/spack/packages.yaml # Install hermes_shm (needed for dependencies) # -spack repo add scripts/hermes_shm +spack repo add ci/hermes spack install hermes_shm From 78414ca8211397b0eb984fe6418176bbe0ec6374 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 16 Oct 2023 08:29:50 -0500 Subject: [PATCH 191/191] Install hermes_shm --- ci/install_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install_deps.sh b/ci/install_deps.sh index 26cc2462d..03958f316 100755 --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -33,7 +33,7 @@ set -x # available from the system. For example, autoconf, cmake, m4, etc. # Modify ci/pckages.yaml to skip building compilers or build tools via Spack. cd ${GITHUB_WORKSPACE} -cp scripts/ci/packages.yaml ${SPACK_DIR}/etc/spack/packages.yaml +cp ci/packages.yaml ${SPACK_DIR}/etc/spack/packages.yaml # Install hermes_shm (needed for dependencies) #