Merge pull request #684 from lukemartinlogan/adios2

Adios2

.github/workflows/build_and_push_docker_images.yml

@@ -25,7 +25,7 @@ jobs:
         id: docker-deps-cache
         with:
           path: ci/deps/deps.Dockerfile
-          key: docker-${{ hashFiles('ci/deps/deps.Dockerfile') }}
+          key: docker-${{ hashFiles('docker/deps.Dockerfile') }}
       - name: Checkout
         uses: actions/checkout@v3
       - name: Login to Docker Hub

CMake/HermesConfig.cmake

@@ -91,6 +91,15 @@ if(thallium_FOUND)
   message(STATUS "found thallium at ${thallium_DIR}")
 endif()
 
+# ADIOS
+if(HERMES_ENABLE_ADIOS)
+    find_package(ADIOS2 REQUIRED)
+    message(STATUS "found adios2")
+    include_directories(${ADIOS2_INCLUDE_DIRS})
+    link_directories(${ADIOS2_LIBRARY_DIRS})
+    add_compile_definitions(HERMES_ENABLE_ADIOS)
+endif()
+
 #-----------------------------------------------------------------------------
 # Mark hermes as found and set all needed packages
 #-----------------------------------------------------------------------------
@@ -107,6 +116,7 @@ set(Hermes_LIBRARIES
         -ldl -lrt -lc -pthread
         thallium
         hermes
+        ${ADIOS2_LIBRARIES}
         ${Boost_LIBRARIES} ${Hermes_LIBRARY})
 set(Hermes_LIBRARY_DIRS ${HermeShm_LIBRARY_DIRS})
 # Set Hermes client dirs (equal to Hermes dirs)

CMakeLists.txt

@@ -24,8 +24,8 @@ option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." OFF)
 option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF)
 option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF)
 option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF)
-option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" ON)
-option(HERMES_ENABLE_ADIOS "Build the Hermes Python wrapper" ON)
+option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" OFF)
+option(HERMES_ENABLE_ADIOS "Build the Hermes Python wrapper" OFF)
 
 option(HERMES_MPICH "Specify that this a MPICH build" OFF)
 option(HERMES_OPENMPI "Specify that this a OpenMPI build" OFF)
@@ -169,6 +169,12 @@ endif()
 include_directories(${Boost_INCLUDE_DIRS})
 message("Boost: ${Boost_LIBRARIES}")
 
+# Libelf
+pkg_check_modules(libelf REQUIRED libelf)
+message(STATUS "found libelf at ${libelf_INCLUDE_DIRS}")
+include_directories(${libelf_INCLUDE_DIRS})
+link_directories(${libelf_LIBRARY_DIRS})
+
 # HDF5
 if(HERMES_ENABLE_VFD)
     set(HERMES_REQUIRED_HDF5_VERSION 1.14.0)
@@ -200,6 +206,14 @@ if(HERMES_ENABLE_VFD)
     endif()
 endif()
 
+# ADIOS
+if(HERMES_ENABLE_ADIOS)
+    find_package(ADIOS2 REQUIRED)
+    message(STATUS "found adios2 at ${ADIOS2_INCLUDE_DIRS}")
+    include_directories(${ADIOS2_INCLUDE_DIRS})
+    link_directories(${ADIOS2_LIBRARY_DIRS})
+    add_compile_definitions(HERMES_ENABLE_ADIOS)
+endif()
 
 #------------------------------------------------------------------------------
 # Setup CMake Environment
@@ -233,6 +247,7 @@ endif()
 #-----------------------------------------------------------------------------
 # Main includes
 include_directories(${HermesShm_INCLUDE_DIRS})
+link_directories(${HermesShm_LIBRARY_DIRS})
 include_directories(${CMAKE_SOURCE_DIR})
 include_directories(${CMAKE_SOURCE_DIR}/include)
 # Required Task includes
@@ -263,13 +278,15 @@ set(Hermes_CLIENT_LIBRARIES
         ${HermesShm_LIBRARIES}
         yaml-cpp
         cereal::cereal
-        -ldl -lrt -lc -pthread hrun_client)
+        -ldl -lrt -lc -pthread hrun_client
+        ${ADIOS2_LIBRARIES})
 set(Hermes_CLIENT_DEPS
         hrun_client)
 set(Hermes_RUNTIME_LIBRARIES
         ${Hermes_CLIENT_LIBRARIES}
         hrun_runtime
-        ${Boost_LIBRARIES})
+        ${Boost_LIBRARIES}
+        ${ADIOS2_LIBRARIES})
 set(Hermes_RUNTIME_DEPS
         hrun_client hrun_runtime)
 

ci/hermes/packages/hermes/package.py

@@ -38,6 +38,7 @@ class Hermes(CMakePackage):
     variant('ares', default=False, description='Enable full libfabric install')
     variant('zmq', default=False, description='Build ZeroMQ tests')
     variant('adios', default=False, description='Build Adios tests')
+    variant('python', default=False, description='Build Python Wrapper')
     variant('encrypt', default=False, description='Build Adios tests')
     variant('compress', default=False, description='Build Adios tests')
 
@@ -73,6 +74,10 @@ def cmake_args(self):
             args.append(self.define('HERMES_ENABLE_COMPRESSION', 'ON'))
         if '+encrypt' in self.spec:
             args.append(self.define('HERMES_ENABLE_ENCRYPTION', 'ON'))
+        if '+adios' in self.spec:
+            args.append(self.define('HERMES_ENABLE_ADIOS', 'ON'))
+        if '+python' in self.spec:
+            args.append(self.define('HERMES_ENABLE_PYTHON', 'ON'))
         return args
 
     def set_include(self, env, path):

ci/hermes/packages/hermes_shm/package.py

@@ -4,8 +4,6 @@ class HermesShm(CMakePackage):
     homepage = "https://github.com/lukemartinlogan/hermes_shm/wiki"
     git = "https://github.com/lukemartinlogan/hermes_shm.git"
     version('master', branch='master')
-    version("1.1.0", sha256="080d5361cff22794b670e4544c532926ca8b6d6ec695af25596efe035bfffea5")
-    version("1.0.0", sha256="a79f01d531ce89985ad59a2f62b41d74c2385e48d929e2f4ad895ae34137573b")
 
     # Main variants
     variant('debug', default=False, description='Build shared libraries')
@@ -21,6 +19,7 @@ class HermesShm(CMakePackage):
     depends_on('[email protected]')
     depends_on('yaml-cpp')
     depends_on('[email protected]')
+    depends_on('libelf')
 
     # Machine variants
     variant('ares', default=False, description='Build in ares')
@@ -30,7 +29,7 @@ class HermesShm(CMakePackage):
     # Main dependencies
     depends_on('[email protected]', when='+mochi')
     depends_on('cereal', when='+cereal')
-    depends_on('[email protected]: +context +fiber', when='+boost')
+    depends_on('[email protected]: +context +fiber +coroutine +regex +system +filesystem +serialization +pic +math', when='+boost')
     depends_on('mpi', when='+mpiio')
     depends_on('[email protected]', when='+vfd')
     depends_on('libzmq', '+zmq')

ci/install_deps.sh

@@ -10,8 +10,8 @@ docker run -d \
 --mount src=${PWD},target=/hermes,type=bind \
 --name hermes_deps_c \
 --network host \
---memory=4G \
---shm-size=4G \
+--memory=8G \
+--shm-size=8G \
 -p 4000:4000 \
 -p 4001:4001 \
 lukemartinlogan/hermes_deps \

config/hermes_client_default.yaml

@@ -1,10 +1,10 @@
 stop_daemon: false
-path_inclusions: ["/tmp/test_hermes"]
-path_exclusions: ["/"]
+path_inclusions: ["/tmp/test_hermes/*"]
+path_exclusions: ["/*"]
 file_page_size: 1024KB
 base_adapter_mode: kDefault
 flushing_mode: kAsync
 file_adapter_configs:
-  - path: "/"
+  - path: "/*"
     page_size: 1MB
     mode: kDefault

docker/deps.Dockerfile

@@ -1,5 +1,5 @@
 # NOTE(llogan): This dockerfile assumes that
-# hermes github is mounted on /hermes
+# hermes github is the current working directory
 
 # Install ubuntu 22.04
 FROM ubuntu:22.04
@@ -38,9 +38,10 @@ ENV SPACK_DIR="${HOME}/spack"
 ENV SPACK_VERSION="v0.20.2"
 ENV HERMES_DEPS_DIR="${HOME}/hermes_deps"
 ENV HERMES_DIR="${HOME}/hermes"
+COPY ci/module_load.sh module_load.sh
 
 # Install Spack
-RUN . /hermes/ci/module_load.sh && \
+RUN . /module_load.sh && \
     git clone -b ${SPACK_VERSION} https://github.com/spack/spack ${SPACK_DIR} && \
     . "${SPACK_DIR}/share/spack/setup-env.sh" && \
     git clone -b dev https://github.com/lukemartinlogan/hermes.git ${HERMES_DEPS_DIR} && \
@@ -50,7 +51,7 @@ RUN . /hermes/ci/module_load.sh && \
     spack external find
 
 # Install hermes_shm
-RUN . /hermes/ci/module_load.sh && \
+RUN . /module_load.sh && \
     . "${SPACK_DIR}/share/spack/setup-env.sh" && \
     spack external find && \
     spack install hermes_shm@master+vfd+mpiio^[email protected]

hermes_adapters/mpiio/CMakeLists.txt

@@ -18,7 +18,8 @@ set(INTERCEPTOR_DEPS
 add_library(hermes_mpiio SHARED
         ${CMAKE_CURRENT_SOURCE_DIR}/mpiio_api.cc)
 add_dependencies(hermes_mpiio ${INTERCEPTOR_DEPS})
-target_link_libraries(hermes_mpiio MPI::MPI_CXX stdc++fs dl ${INTERCEPTOR_DEPS})
+target_link_libraries(hermes_mpiio
+        MPI::MPI_CXX stdc++fs ${libelf_LIBRARIES} dl ${INTERCEPTOR_DEPS})
 
 #-----------------------------------------------------------------------------
 # Add Target(s) to CMake Install

hermes_adapters/posix/CMakeLists.txt

@@ -11,7 +11,7 @@ set(INTERCEPTOR_DEPS
 add_library(hermes_posix SHARED
         ${CMAKE_CURRENT_SOURCE_DIR}/posix_api.cc)
 add_dependencies(hermes_posix ${INTERCEPTOR_DEPS})
-target_link_libraries(hermes_posix MPI::MPI_CXX stdc++fs dl ${INTERCEPTOR_DEPS})
+target_link_libraries(hermes_posix MPI::MPI_CXX stdc++fs ${libelf_LIBRARIES} dl ${INTERCEPTOR_DEPS})
 
 #-----------------------------------------------------------------------------
 # Add Target(s) to CMake Install

hermes_adapters/posix/posix_api.cc

@@ -37,17 +37,16 @@ namespace stdfs = std::filesystem;
 
 extern "C" {
 
-static __attribute__((constructor(101))) void init_posix(void) {
-  HERMES_POSIX_API;
-  HERMES_POSIX_FS;
-  TRANSPARENT_HERMES();;
-}/**/
+//static __attribute__((constructor(101))) void init_posix(void) {
+//  HERMES_POSIX_API;
+//  HERMES_POSIX_FS;
+//  TRANSPARENT_HERMES();;
+//}
 
 /**
  * POSIX
  */
 int HERMES_DECL(open)(const char *path, int flags, ...) {
-//  TRANSPARENT_HERMES();
   int mode = 0;
   auto real_api = HERMES_POSIX_API;
   auto fs_api = HERMES_POSIX_FS;
@@ -57,7 +56,10 @@ int HERMES_DECL(open)(const char *path, int flags, ...) {
     mode = va_arg(arg, int);
     va_end(arg);
   }
-  if (fs_api->IsPathTracked(path)) {
+  if (real_api->IsInterceptorLoaded()) {
+    TRANSPARENT_HERMES();
+  }
+  if (real_api->IsInterceptorLoaded() && fs_api->IsPathTracked(path)) {
     HILOG(kDebug, "Intercept open for filename: {}"
           " and mode: {}"
           " is tracked.", path, flags)
@@ -68,24 +70,29 @@ int HERMES_DECL(open)(const char *path, int flags, ...) {
     return f.hermes_fd_;
   }
 
+  int fd = -1;
   if (flags & O_CREAT || flags & O_TMPFILE) {
-    return real_api->open(path, flags, mode);
+    fd = real_api->open(path, flags, mode);
+  } else {
+    fd = real_api->open(path, flags);
   }
-  return real_api->open(path, flags);
+  return fd;
 }
 
 int HERMES_DECL(open64)(const char *path, int flags, ...) {
-//  TRANSPARENT_HERMES();
   int mode = 0;
   auto real_api = HERMES_POSIX_API;
   auto fs_api = HERMES_POSIX_FS;
-  if (flags & O_CREAT) {
+  if (flags & O_CREAT || flags & O_TMPFILE) {
     va_list arg;
     va_start(arg, flags);
     mode = va_arg(arg, int);
     va_end(arg);
   }
-  if (fs_api->IsPathTracked(path)) {
+  if (real_api->IsInterceptorLoaded()) {
+    TRANSPARENT_HERMES();
+  }
+  if (real_api->IsInterceptorLoaded() && fs_api->IsPathTracked(path)) {
     HILOG(kDebug, "Intercept open64 for filename: {}"
           " and mode: {}"
           " is tracked.", path, flags)
@@ -94,17 +101,22 @@ int HERMES_DECL(open64)(const char *path, int flags, ...) {
     stat.st_mode_ = mode;
     return fs_api->Open(stat, path).hermes_fd_;
   }
-  if (flags & O_CREAT) {
-    return real_api->open64(path, flags, mode);
+  int fd = -1;
+  if (flags & O_CREAT || flags & O_TMPFILE) {
+    fd = real_api->open64(path, flags, mode);
+  } else {
+    fd = real_api->open64(path, flags);
   }
-  return real_api->open64(path, flags);
+  return fd;
 }
 
 int HERMES_DECL(__open_2)(const char *path, int oflag) {
-//  TRANSPARENT_HERMES();
   auto real_api = HERMES_POSIX_API;
   auto fs_api = HERMES_POSIX_FS;
-  if (fs_api->IsPathTracked(path)) {
+  if (real_api->IsInterceptorLoaded()) {
+    TRANSPARENT_HERMES();
+  }
+  if (real_api->IsInterceptorLoaded() && fs_api->IsPathTracked(path)) {
     HILOG(kDebug, "Intercept __open_2 for filename: {}"
           " and mode: {}"
           " is tracked.", path, oflag)
@@ -117,11 +129,13 @@ int HERMES_DECL(__open_2)(const char *path, int oflag) {
 }
 
 int HERMES_DECL(creat)(const char *path, mode_t mode) {
-//  TRANSPARENT_HERMES();
   std::string path_str(path);
   auto real_api = HERMES_POSIX_API;
   auto fs_api = HERMES_POSIX_FS;
-  if (fs_api->IsPathTracked(path)) {
+  if (real_api->IsInterceptorLoaded()) {
+    TRANSPARENT_HERMES();
+  }
+  if (real_api->IsInterceptorLoaded() && fs_api->IsPathTracked(path)) {
     HILOG(kDebug, "Intercept creat for filename: {}"
           " and mode: {}"
           " is tracked.", path, mode)
@@ -134,11 +148,13 @@ int HERMES_DECL(creat)(const char *path, mode_t mode) {
 }
 
 int HERMES_DECL(creat64)(const char *path, mode_t mode) {
-//  TRANSPARENT_HERMES();
   std::string path_str(path);
   auto real_api = HERMES_POSIX_API;
   auto fs_api = HERMES_POSIX_FS;
-  if (fs_api->IsPathTracked(path)) {
+  if (real_api->IsInterceptorLoaded()) {
+    TRANSPARENT_HERMES();
+  }
+  if (real_api->IsInterceptorLoaded() && fs_api->IsPathTracked(path)) {
     HILOG(kDebug, "Intercept creat64 for filename: {}"
           " and mode: {}"
           " is tracked.", path, mode)
@@ -440,6 +456,30 @@ int HERMES_DECL(fsync)(int fd) {
   return real_api->fsync(fd);
 }
 
+int HERMES_DECL(ftruncate)(int fd, off_t length) {
+  bool stat_exists;
+  auto real_api = HERMES_POSIX_API;
+  auto fs_api = HERMES_POSIX_FS;
+  if (fs_api->IsFdTracked(fd)) {
+    File f; f.hermes_fd_ = fd;
+    HILOG(kDebug, "Intercepted ftruncate.")
+    return fs_api->Truncate(f, stat_exists, length);
+  }
+  return real_api->ftruncate(fd, length);
+}
+
+int HERMES_DECL(ftruncate64)(int fd, off64_t length) {
+  bool stat_exists;
+  auto real_api = HERMES_POSIX_API;
+  auto fs_api = HERMES_POSIX_FS;
+  if (fs_api->IsFdTracked(fd)) {
+    File f; f.hermes_fd_ = fd;
+    HILOG(kDebug, "Intercepted ftruncate.")
+    return fs_api->Truncate(f, stat_exists, length);
+  }
+  return real_api->ftruncate64(fd, length);
+}
+
 int HERMES_DECL(close)(int fd) {
   bool stat_exists;
   auto real_api = HERMES_POSIX_API;
@@ -462,7 +502,7 @@ int HERMES_DECL(flock)(int fd, int operation) {
     // TODO(llogan): implement?
     return 0;
   }
-  return real_api->close(fd);
+  return real_api->flock(fd, operation);
 }
 
 int HERMES_DECL(remove)(const char *pathname) {