diff --git a/CMakeLists.txt b/CMakeLists.txt index d6418ba74..cd6d20dbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,13 +12,14 @@ option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" OFF) option(HERMES_ENABLE_POSIX_ADAPTER "Build the Hermes POSIX adapter." ON) option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." ON) -option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON) +option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." OFF) +option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF) option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF) -option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" ON) -message("HERMES_ENABLE_PYTHON: ${HERMES_ENABLE_PYTHON}") +option(HERMES_MPICH "Specify that this a MPICH build" OFF) +option(HERMES_OPENMPI "Specify that this a OpenMPI build" OFF) #----------------------------------------------------------------------------- # Compiler Optimization @@ -61,7 +62,7 @@ message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") find_package(Catch2 3.0.1 REQUIRED) message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") -# MPICH +# MPI if(BUILD_MPI_TESTS) find_package(MPI REQUIRED COMPONENTS C CXX) message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") @@ -105,6 +106,38 @@ endif() include_directories(${Boost_INCLUDE_DIRS}) message("Boost: ${Boost_LIBRARIES}") +# HDF5 +if(HERMES_ENABLE_VFD) + set(HERMES_REQUIRED_HDF5_VERSION 1.14.0) + set(HERMES_REQUIRED_HDF5_COMPONENTS C) + find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} CONFIG NAMES hdf5 + COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS} shared) + if(HDF5_FOUND) + message(STATUS "found HDF5 ${HDF5_VERSION} at ${HDF5_INCLUDE_DIR}") + set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ${HDF5_INCLUDE_DIR}) + set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_C_SHARED_LIBRARY}) + else() + # Allow for HDF5 autotools builds + find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} MODULE REQUIRED + COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS}) + if(HDF5_FOUND) + set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ${HDF5_INCLUDE_DIRS}) + set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_LIBRARIES}) + else() + message(FATAL_ERROR "Could not find HDF5, please set HDF5_DIR (1.13.0) or HDF5_ROOT (1.13.1).") + endif() + endif() +endif() + + #------------------------------------------------------------------------------ # Setup CMake Environment #------------------------------------------------------------------------------ diff --git a/ci/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py index 38a697f3a..8095446b2 100644 --- a/ci/hermes/packages/hermes/package.py +++ b/ci/hermes/packages/hermes/package.py @@ -8,8 +8,6 @@ class Hermes(CMakePackage): version('master', branch='master', submodules=True) version('dev', branch='dev', submodules=True) - version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', - branch='dev', submodules=True) version("1.0.5-beta", sha256="1f3ba51a8beda4bc1314d6541b800de1525f5e233a6f498fcde6dc43562ddcb7") version("1.0.0-beta", sha256="301084cced32aa00532ab4bebd638c31b0512c881ffab20bf5da4b7739defac2") version("0.9.9-beta", sha256="d2e0025a9bd7a3f05d3ab608c727ed15d86ed30cf582549fe996875daf6cb649") diff --git a/ci/hermes/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py index 9d31d00c6..5af8c607f 100644 --- a/ci/hermes/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -8,20 +8,22 @@ class HermesShm(CMakePackage): version('master', branch='master') version("1.0.0", sha256="a79f01d531ce89985ad59a2f62b41d74c2385e48d929e2f4ad895ae34137573b") + variant('mpiio', default=True, description='Enable MPI I/O adapter') + variant('stdio', default=True, description='Enable STDIO adapter') + variant('vfd', default=False, description='Enable HDF5 VFD') variant('ares', default=False, description='Enable full libfabric install') variant('only_verbs', default=False, description='Only verbs') - variant('vfd', default=False, description='Enable HDF5 VFD') variant('debug', default=False, description='Build shared libraries') variant('zmq', default=False, description='Build ZeroMQ tests') depends_on('mochi-thallium~cereal@0.10.1') depends_on('catch2@3.0.1') - depends_on('mpich@3.3.2') + depends_on('mpi') depends_on('cereal') depends_on('yaml-cpp') depends_on('doxygen@1.9.3') depends_on('boost@1.7: +context +fiber +filesystem +system +atomic +chrono +serialization +signals +pic +regex') - depends_on('libfabric fabrics=sockets,tcp,udp,rxm,rxd,verbs', + depends_on('libfabric fabrics=sockets,tcp,udp,verbs', when='+ares') depends_on('libfabric fabrics=verbs', when='+only_verbs') @@ -34,6 +36,16 @@ def cmake_args(self): args.append('-DCMAKE_BUILD_TYPE=Debug') else: args.append('-DCMAKE_BUILD_TYPE=Release') + if '+mpiio' in self.spec: + args.append('-DHERMES_ENABLE_MPIIO_ADAPTER=ON') + if 'openmpi' in self.spec: + args.append('-DHERMES_OPENMPI=ON') + elif 'mpich' in self.spec: + args.append('-DHERMES_MPICH=ON') + if '+stdio' in self.spec: + args.append('-HERMES_ENABLE_STDIO_ADAPTER=ON') + if '+vfd' in self.spec: + args.append('-HERMES_ENABLE_VFD=ON') return args def set_include(self, env, path): diff --git a/hermes_adapters/CMakeLists.txt b/hermes_adapters/CMakeLists.txt index e335f4d0b..1c37de7a2 100644 --- a/hermes_adapters/CMakeLists.txt +++ b/hermes_adapters/CMakeLists.txt @@ -9,7 +9,7 @@ endif() if (HERMES_ENABLE_MPIIO_ADAPTER) add_subdirectory(mpiio) endif() -if (HERMES_ENABLE_HDF5_ADAPTER) +if (HERMES_ENABLE_VFD) add_subdirectory(vfd) endif() diff --git a/hermes_adapters/mpiio/CMakeLists.txt b/hermes_adapters/mpiio/CMakeLists.txt index b12f78c59..60cbee153 100644 --- a/hermes_adapters/mpiio/CMakeLists.txt +++ b/hermes_adapters/mpiio/CMakeLists.txt @@ -11,6 +11,13 @@ add_dependencies(hermes_mpiio_io_client hermes hermes_fs_base) target_link_libraries(hermes_mpiio_io_client hermes hermes_fs_base MPI::MPI_CXX stdc++fs dl) +if (HERMES_MPICH) + message(STATUS "Using HERMES_MPICH") + add_definitions(-DHERMES_MPICH) +elseif(HERMES_OPENMPI) + message(STATUS "Using HERMES_OPENMPI") + add_definitions(-DHERMES_OPENMPI) +endif() # Create the MPIIO interceptor set(INTERCEPTOR_DEPS diff --git a/hermes_adapters/mpiio/mpiio_api.h b/hermes_adapters/mpiio/mpiio_api.h index 1328447a4..6d9424971 100644 --- a/hermes_adapters/mpiio/mpiio_api.h +++ b/hermes_adapters/mpiio/mpiio_api.h @@ -17,7 +17,9 @@ #include #include "hermes_shm/util/logging.h" #include +#ifdef HERMES_MPICH #include +#endif #include "hermes_adapters/real_api.h" #ifndef MPI_MODE_TRUNCATE diff --git a/hermes_adapters/mpiio/mpiio_io_client.cc b/hermes_adapters/mpiio/mpiio_io_client.cc index 09d23237f..b42f4e18b 100644 --- a/hermes_adapters/mpiio/mpiio_io_client.cc +++ b/hermes_adapters/mpiio/mpiio_io_client.cc @@ -214,8 +214,15 @@ void MpiioIoClient::ReadBlob(const std::string &bkt_name, /** Update the I/O status after a ReadBlob or WriteBlob */ void MpiioIoClient::UpdateIoStatus(size_t count, IoStatus &status) { +#ifdef HERMES_OPENMPI + status.mpi_status_ptr_->_cancelled = 0; + status.mpi_status_ptr_->_ucount = count; +#elif defined(HERMES_MPICH) status.mpi_status_ptr_->count_hi_and_cancelled = 0; status.mpi_status_ptr_->count_lo = count; +#else +#error "No MPI implementation specified for MPIIO adapter" +#endif } } // namespace hermes::adapter::fs diff --git a/hermes_adapters/vfd/H5FDhermes.cc b/hermes_adapters/vfd/H5FDhermes.cc index 60fff9455..d8ad22a68 100644 --- a/hermes_adapters/vfd/H5FDhermes.cc +++ b/hermes_adapters/vfd/H5FDhermes.cc @@ -40,7 +40,7 @@ #include "H5FDhermes.h" /* Hermes file driver */ #include "hermes_adapters/posix/posix_io_client.h" -#include "posix/posix_fs_api.h" +#include "hermes_adapters/posix/posix_fs_api.h" /** * Make this adapter use Hermes. @@ -211,7 +211,8 @@ H5FD__hermes_term(void) { /* Reset VFL ID */ H5FD_HERMES_g = H5I_INVALID_HID; - HERMES->Finalize(); + // TODO(llogan): Probably should add back at some point. + // HERMES->Finalize(); return ret_value; } /* end H5FD__hermes_term() */ @@ -515,13 +516,13 @@ static herr_t H5FD__hermes_write(H5FD_t *_file, H5FD_mem_t type, */ H5PL_type_t H5PLget_plugin_type(void) { - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); return H5PL_TYPE_VFD; } const void* H5PLget_plugin_info(void) { - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); return &H5FD_hermes_g; } diff --git a/tasks/data_stager/src/data_stager.cc b/tasks/data_stager/src/data_stager.cc index 858392858..e27b51434 100644 --- a/tasks/data_stager/src/data_stager.cc +++ b/tasks/data_stager/src/data_stager.cc @@ -65,6 +65,7 @@ class Server : public TaskLib { std::unordered_map>::iterator it = url_map_[rctx.lane_id_].find(task->bkt_id_); if (it == url_map_[rctx.lane_id_].end()) { + // // HELOG(kError, "Could not find stager for bucket: {}", task->bkt_id_); // TODO(llogan): Probably should add back... // task->SetModuleComplete(); diff --git a/test/unit/hermes_adapters/vfd/hermes_vfd_test.cc b/test/unit/hermes_adapters/vfd/hermes_vfd_test.cc index ce2711315..4356c526d 100644 --- a/test/unit/hermes_adapters/vfd/hermes_vfd_test.cc +++ b/test/unit/hermes_adapters/vfd/hermes_vfd_test.cc @@ -23,12 +23,10 @@ #include #include "hermes/hermes.h" -#include "hermes_types.h" +#include "hermes/hermes_types.h" #include "adapter_test_utils.h" #include "catch_config.h" -using hermes::f32; -using hermes::u32; namespace stdfs = std::filesystem; namespace hermes::adapter::vfd::test { @@ -399,7 +397,7 @@ void RemoveFiles() { int init(int* argc, char*** argv) { #if HERMES_INTERCEPT == 1 setenv("HERMES_FLUSH_MODE", "kSync", 1); - TRANSPARENT_HERMES + TRANSPARENT_HERMES(); HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync; #endif MPI_Init(argc, argv); diff --git a/test/unit/pipelines/test_ior.yaml b/test/unit/pipelines/test_ior.yaml new file mode 100644 index 000000000..d84366214 --- /dev/null +++ b/test/unit/pipelines/test_ior.yaml @@ -0,0 +1,14 @@ +name: hermes_unit_ior +env: hermes +pkgs: + - pkg_type: asan + pkg_name: asan + - pkg_type: hermes_run + pkg_name: hermes_run + sleep: 5 + - pkg_type: hermes_api + pkg_name: hermes_api + posix: true + - pkg_type: ior + pkg_name: ior + api: posix \ No newline at end of file