diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1b37ea5aa..28705e6da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,7 +52,7 @@ jobs: with: path: | ~/build - key: ${{ runner.os }}-${{ hashFiles('wrapper', 'traits', 'test', 'src', 'hermes_shm', 'benchmarks', 'adapter') }} + key: ${{ runner.os }}-${{ hashFiles('wrapper', 'test', 'tasks', 'src', 'include', 'hrun', 'hermes_adapters', 'benchmark') }} - name: Install APT Dependencies run: | @@ -71,9 +71,9 @@ jobs: if: steps.spack-cache.outputs.cache-hit != 'true' run: ci/install_deps.sh - - name: Build - if: steps.hermes-cache.outputs.cache-hit != 'true' - run: ci/build_hermes.sh +# - name: Build +# if: steps.hermes-cache.outputs.cache-hit != 'true' +# run: ci/build_hermes.sh # # - name: Test # run: bash ci/test_hermes.sh diff --git a/.gitignore b/.gitignore index e3ed55357..eb001e77a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,17 @@ +cmake* +.idea +*.cmd +*.symvers +*.cmd +*.mod +*.mod.c +*.ko +*.order +*.o +*.o.cmd +*.o.d +build + build/ benchmarks/HermesVFD @@ -15,8 +29,8 @@ cmake-build-* .idea .clang-format __pycache__/ -/src/adapter/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt -/adapter/test/posix/Testing/Temporary/LastTest.log +/hrun/src/hermes_adapters/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt +/hermes_adapters/test/posix/Testing/Temporary/LastTest.log .cache wrapper/java/out @@ -25,7 +39,7 @@ Testing .gradle **/build/ -!src/**/build/ +!hrun/src/**/build/ # Ignore Gradle GUI config gradle-app.setting diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..7676f3941 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/pybind11"] + path = external/pybind11 + url = https://github.com/pybind/pybind11.git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3fe28b8d2..000000000 --- a/.travis.yml +++ /dev/null @@ -1,67 +0,0 @@ -language: cpp -os: linux -dist: bionic - -addons: - apt: - update: true - packages: - - build-essential - - cmake - - libboost-dev - - libmpich-dev - - autoconf - - autoconf-archive - - automake - - gfortran - - g++-7 - - libstdc++-7-dev - -before_install: - - test -n $CC && unset CC - - test -n $CXX && unset CXX - - git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git - - (cd $HOME/spack; git describe) || git clone https://github.com/spack/spack $HOME/spack - - | - test -f $HOME/spack/etc/spack/packages.yaml || cat > $HOME/spack/etc/spack/packages.yaml << ' EOF' - packages: - all: - target: [x86_64] - boost: - paths: - boost@1.72.0: /usr - buildable: False - mpich: - paths: - mpich@3.3: /opt/mpich-3.3-intel - buildable: False - cmake: - paths: - cmake@3.10.0: /usr/local/cmake-3.10.0 - buildable: False - autoconf: - paths: - autoconf@2.69: /usr - buildable: False - automake: - paths: - automake@1.15: /usr - buildable: False - libtool: - paths: - libtool@2.4.6: /usr - buildable: False - m4: - paths: - m4@4.17: /usr - buildable: False - EOF -install: - - . $HOME/spack/share/spack/setup-env.sh - - spack install gotcha@develop && spack load gotcha@develop - - spack install glog@0.3.5 && spack load glog@0.3.5 - - spack repo add sds-repo - - spack install mochi-thallium~cereal && spack load -r mochi-thallium~cereal - -script: ./ci/install_hshm.sh - diff --git a/CMake/HermesConfig.cmake b/CMake/HermesConfig.cmake index f727e2f2c..a9c12a958 100644 --- a/CMake/HermesConfig.cmake +++ b/CMake/HermesConfig.cmake @@ -1,96 +1,103 @@ -# Find hermes header and library. +# Find Hermes header and library. # # This module defines the following uncached variables: -# Hermes_FOUND, if false, do not try to use hermes. -# Hermes_INCLUDE_DIRS, where to find hermes.h. -# Hermes_LIBRARIES, the libraries to link against to use the hermes library -# Hermes_LIBRARY_DIRS, the directory where the hermes library is found. +# Hermes_FOUND, if false, do not try to use Hermes. +# Hermes_INCLUDE_DIRS, where to find Hermes.h. +# Hermes_LIBRARIES, the libraries to link against to use the Hermes library +# Hermes_LIBRARY_DIRS, the directory where the Hermes library is found. -#----------------------------------------------------------------------------- -# Version Strings -#----------------------------------------------------------------------------- -set(HERMES_VERSION_STRING @HERMES_PACKAGE_VERSION@) -set(HERMES_VERSION_MAJOR @HERMES_VERSION_MAJOR@) -set(HERMES_VERSION_MINOR @HERMES_VERSION_MINOR@) -set(HERMES_VERSION_PATCH @HERMES_VERSION_PATCH@) +find_path( + Hermes_INCLUDE_DIR + hermes/hermes_types.h +) +message("Hermes_INCLUDE_DIR: ${Hermes_INCLUDE_DIR}") -#----------------------------------------------------------------------------- -# C++ Version -#----------------------------------------------------------------------------- -set(CMAKE_CXX_STANDARD 17) +if( Hermes_INCLUDE_DIR ) + get_filename_component(Hermes_DIR ${Hermes_INCLUDE_DIR} PATH) -#----------------------------------------------------------------------------- -# Find hermes.h -#----------------------------------------------------------------------------- -find_path(Hermes_INCLUDE_DIR hermes.h) -if (NOT Hermes_INCLUDE_DIR) - set(Hermes_FOUND OFF) - message(SEND_ERROR "FindHermes: Could not find hermes.h") -else() - #----------------------------------------------------------------------------- - # Find hermes install dir and hermes.so - #----------------------------------------------------------------------------- - get_filename_component(Hermes_DIR ${Hermes_INCLUDE_DIR} PATH) - find_library( - Hermes_LIBRARY - NAMES hermes - ) + #----------------------------------------------------------------------------- + # Find all packages needed by Hermes + #----------------------------------------------------------------------------- + find_library( + Hermes_LIBRARY + NAMES hrun_client hrun_server + ) - #----------------------------------------------------------------------------- - # Mark hermes as found - #----------------------------------------------------------------------------- - if( Hermes_LIBRARY ) - set(Hermes_LIBRARY_DIR "") - get_filename_component(Hermes_LIBRARY_DIRS ${Hermes_LIBRARY} PATH) - # Set uncached variables as per standard. - set(Hermes_FOUND ON) - set(Hermes_INCLUDE_DIRS ${Hermes_INCLUDE_DIR}) - set(Hermes_LIBRARIES ${Hermes_LIBRARY}) - endif(Hermes_LIBRARY) + message("Hermes_LIBRARY: ${Hermes_LIBRARY}") - #----------------------------------------------------------------------------- - # Find all packages needed by hermes - #----------------------------------------------------------------------------- - # thallium - if(@HERMES_RPC_THALLIUM@) - find_package(thallium CONFIG REQUIRED) - if(thallium_FOUND) - message(STATUS "found thallium at ${thallium_DIR}") - endif() - endif() + # HermesShm + find_package(HermesShm CONFIG REQUIRED) + message(STATUS "found hermes_shm.h at ${HermesShm_INCLUDE_DIRS}") - #YAML-CPP - find_package(yaml-cpp REQUIRED) - if(yaml-cpp_FOUND) - message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") - endif() + # YAML-CPP + find_package(yaml-cpp REQUIRED) + message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") - #Cereal - find_package(cereal REQUIRED) - if(cereal) - message(STATUS "found cereal") - endif() + # Catch2 + find_package(Catch2 3.0.1 REQUIRED) + message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") - #if(HERMES_COMMUNICATION_MPI) + # MPICH + if(BUILD_MPI_TESTS) find_package(MPI REQUIRED COMPONENTS C CXX) message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") - #endif() + endif() - # librt - if(NOT APPLE) - find_library(LIBRT rt) - if(NOT LIBRT) - message(FATAL_ERROR "librt is required for POSIX shared memory") - endif() - endif() + # OpenMP + if(BUILD_OpenMP_TESTS) + find_package(OpenMP REQUIRED COMPONENTS C CXX) + message(STATUS "found omp.h at ${OpenMP_CXX_INCLUDE_DIRS}") + endif() - #----------------------------------------------------------------------------- - # Set all packages needed by hermes - #----------------------------------------------------------------------------- + # Cereal + find_package(cereal REQUIRED) + if(cereal) + message(STATUS "found cereal") + endif() - # Set the Hermes library dependencies - set(Hermes_LIBRARIES yaml-cpp thallium - MPI::MPI_CXX stdc++fs dl cereal::cereal - hermes_shm_data_structures hermes) -endif() + # Boost + find_package(Boost REQUIRED COMPONENTS regex system filesystem fiber REQUIRED) + if (Boost_FOUND) + message(STATUS "found boost at ${Boost_INCLUDE_DIRS}") + endif() + + # Thallium + find_package(thallium CONFIG REQUIRED) + if(thallium_FOUND) + message(STATUS "found thallium at ${thallium_DIR}") + endif() + + #----------------------------------------------------------------------------- + # Mark hermes as found and set all needed packages + #----------------------------------------------------------------------------- + if( Hermes_LIBRARY ) + set(Hermes_LIBRARY_DIR "") + get_filename_component(Hermes_LIBRARY_DIRS ${Hermes_LIBRARY} PATH) + # Set uncached variables as per standard. + set(Hermes_FOUND ON) + set(Hermes_INCLUDE_DIRS ${Boost_INCLUDE_DIRS} ${Hermes_INCLUDE_DIR}) + set(Hermes_LIBRARIES + ${HermesShm_LIBRARIES} + yaml-cpp + cereal::cereal + -ldl -lrt -lc -pthread + thallium + hermes + ${Boost_LIBRARIES} ${Hermes_LIBRARY}) + set(Hermes_CLIENT_LIBRARIES ${Hermes_LIBRARIES}) + endif(Hermes_LIBRARY) + +else(Hermes_INCLUDE_DIR) + message(STATUS "FindHermes: Could not find Hermes.h") +endif(Hermes_INCLUDE_DIR) + +if(Hermes_FOUND) + if(NOT Hermes_FIND_QUIETLY) + message(STATUS "FindHermes: Found both Hermes.h and libhrun_client.so") + endif(NOT Hermes_FIND_QUIETLY) +else(Hermes_FOUND) + if(Hermes_FIND_REQUIRED) + message(STATUS "FindHermes: Could not find Hermes.h and/or libhrun_client.so") + endif(Hermes_FIND_REQUIRED) +endif(Hermes_FOUND) diff --git a/CMake/UseDoxygenDoc.cmake b/CMake/UseDoxygenDoc.cmake deleted file mode 100644 index b9601910b..000000000 --- a/CMake/UseDoxygenDoc.cmake +++ /dev/null @@ -1,33 +0,0 @@ -find_package(Perl REQUIRED) -find_package(Doxygen REQUIRED) - -function(add_doxygen_doc) - set(options) - set(oneValueArgs BUILD_DIR DOXY_FILE TARGET_NAME COMMENT) - set(multiValueArgs) - - cmake_parse_arguments(DOXY_DOC - "${options}" - "${oneValueArgs}" - "${multiValueArgs}" - ${ARGN} - ) - - configure_file( - ${DOXY_DOC_DOXY_FILE} - ${DOXY_DOC_BUILD_DIR}/Doxyfile - @ONLY - ) - - add_custom_target(${DOXY_DOC_TARGET_NAME} - COMMAND - ${DOXYGEN_EXECUTABLE} Doxyfile - WORKING_DIRECTORY - ${DOXY_DOC_BUILD_DIR} - COMMENT - "Building ${DOXY_DOC_COMMENT} with Doxygen" - VERBATIM - ) - - message(STATUS "Added ${DOXY_DOC_TARGET_NAME} [Doxygen] target to build documentation") -endfunction() \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index a8c4391ce..37b728f66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,504 +1,219 @@ -# CMakeLists files in this project can -# refer to the root source directory of the project as ${HERMES_SOURCE_DIR} and -# to the root binary directory of the project as ${HERMES_BINARY_DIR}. -cmake_minimum_required (VERSION 3.10) - -if(CMAKE_SIZEOF_VOID_P LESS 8) - message(FATAL_ERROR "Hermes currently requires a 64-bit processor") -endif() - -# Set a consistent MACOSX_RPATH default across all CMake versions. -if(NOT DEFINED CMAKE_MACOSX_RPATH) - set(CMAKE_MACOSX_RPATH 0) -endif() - -project(HERMES) -set(IS_HERMES_MAIN ON) - -#------------------------------------------------------------------------------ -# Compiler optimization -#------------------------------------------------------------------------------ -set(CMAKE_CXX_STANDARD 17) -add_compile_options("-fPIC") -if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -fPIC") - # Ensure that debug logging is enabled - # This will keep logs which have beneath priority 10 - add_compile_definitions(HERMES_LOG_VERBOSITY=10) - message("This is NOT a release build: ${CMAKE_BUILD_TYPE} ${CMAKE_CXX_FLAGS}") -elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fPIC") - # Ensure that debug logging is enabled - # This will keep logs which have beneath priority 10 - add_compile_definitions(HERMES_LOG_VERBOSITY=10) - message("This IS a release + debug build: ${CMAKE_BUILD_TYPE} ${CMAKE_CXX_FLAGS}") -else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC") - # Ensure that debug logging is disabled - # This will keep only VLOGs which have priroity 0 - add_compile_definitions(HERMES_LOG_VERBOSITY=1) - message("This IS a release build: ${CMAKE_BUILD_TYPE} ${CMAKE_CXX_FLAGS}") -endif() - -#------------------------------------------------------------------------------ -# Version information -#------------------------------------------------------------------------------ -set(HERMES_VERSION_MAJOR "1") -set(HERMES_VERSION_MINOR "0") -set(HERMES_VERSION_PATCH "0") -set(HERMES_PACKAGE "hermes") -set(HERMES_PACKAGE_NAME "HERMES") -set(HERMES_PACKAGE_VERSION "${HERMES_VERSION_MAJOR}.${HERMES_VERSION_MINOR}.${HERMES_VERSION_PATCH}") -set(HERMES_PACKAGE_VERSION_MAJOR "${HERMES_VERSION_MAJOR}.${HERMES_VERSION_MINOR}") -set(HERMES_PACKAGE_VERSION_MINOR "${HERMES_VERSION_PATCH}") -set(HERMES_PACKAGE_STRING "${HERMES_PACKAGE_NAME} ${HERMES_PACKAGE_VERSION}") -set(HERMES_PACKAGE_TARNAME "${HERMES_PACKAGE}") - -#set( CMAKE_VERBOSE_MAKEFILE on ) - -#------------------------------------------------------------------------------ -# Setup install and output Directories -#------------------------------------------------------------------------------ -if(NOT HERMES_INSTALL_BIN_DIR) - set(HERMES_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) -endif() -if(NOT HERMES_INSTALL_LIB_DIR) - set(HERMES_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) -endif() -if(NOT HERMES_INSTALL_INCLUDE_DIR) - set(HERMES_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) -endif() -if(NOT HERMES_INSTALL_DATA_DIR) - set(HERMES_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) -endif() - -#------------------------------------------------------------------------------ -# Setup CMake Environment -#------------------------------------------------------------------------------ -# Export compile commands for autocode generation with CLANG -# set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -if(APPLE) - # We are doing a unix-style install i.e. everything will be installed in - # CMAKE_INSTALL_PREFIX/bin and CMAKE_INSTALL_PREFIX/lib etc. as on other unix - # platforms. We still need to setup CMAKE_INSTALL_NAME_DIR correctly so that - # the binaries point to appropriate location for the libraries. - - # 1. Make CMAKE_INSTALL_PREFIX publicly accessible, if it was hidden in - # previous pass - get_property(is_internal CACHE CMAKE_INSTALL_PREFIX PROPERTY TYPE) - if(is_internal STREQUAL "INTERNAL") - set(CMAKE_INSTALL_PREFIX ${CACHED_CMAKE_INSTALL_PREFIX} CACHE PATH "Install prefix" FORCE) - else() - set(CMAKE_INSTALL_PREFIX ${CACHED_CMAKE_INSTALL_PREFIX} CACHE PATH "Install prefix") - endif() - unset(MACOSX_APP_INSTALL_PREFIX CACHE) - - set(CMAKE_INSTALL_NAME_DIR "@rpath") - mark_as_advanced( - CMAKE_OSX_ARCHITECTURES - CMAKE_OSX_DEPLOYMENT_TARGET - CMAKE_OSX_SYSROOT - ) -endif() - -if(NOT CMAKE_INSTALL_RPATH) - set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") - set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -endif() - -#------------------------------------------------------------------------------ -if(NOT HERMES_EXTERNALLY_CONFIGURED) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." - ) - set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" - ) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." - ) -endif() - -set(HERMES_CMAKE_DIR "${HERMES_SOURCE_DIR}/CMake") -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${HERMES_CMAKE_DIR}) -set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${ORTOOLS_DIR} ${ORTOOLS_ROOT}) - -#------------------------------------------------------------------------------ -# Disallow in-source build -#------------------------------------------------------------------------------ -if("${HERMES_SOURCE_DIR}" STREQUAL "${HERMES_BINARY_DIR}") - message(FATAL_ERROR - "HERMES requires an out of source Build. " - "Please create a separate binary directory and run CMake there.") -endif() - -#------------------------------------------------------------------------------ -# Set a default build type if none was specified -#------------------------------------------------------------------------------ -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") - set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" - "MinSizeRel" "RelWithDebInfo") -endif() - -if(NOT "${CMAKE_CXX_STANDARD}") - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_EXTENSIONS OFF) -endif() +cmake_minimum_required(VERSION 3.10) +project(hermes) #----------------------------------------------------------------------------- -# Targets built within this project are exported at Install time for use -# by other projects. +# Define Options #----------------------------------------------------------------------------- -if(NOT HERMES_EXPORTED_TARGETS) - set(HERMES_EXPORTED_TARGETS "hermes-targets") -endif() +option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" ON) +option(BUILD_MPI_TESTS "Build tests which depend on MPI" ON) +option(BUILD_OpenMP_TESTS "Build tests which depend on OpenMP" ON) +option(HERMES_ENABLE_COVERAGE "Check how well tests cover code" OFF) +option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" OFF) -#------------------------------------------------------------------------------ -# Build options -#------------------------------------------------------------------------------ -option(BUILD_SHARED_LIBS "Build with shared libraries." ON) -option(HERMES_COMMUNICATION_MPI "Use MPI as the communication layer." ON) -option(HERMES_ENABLE_DOXYGEN "Enable hermes documentation." OFF) -option(HERMES_BUILD_BUFFER_POOL_VISUALIZER "Build the BufferPool visualizer" OFF) -option(HERMES_USE_ADDRESS_SANITIZER "Enable -fsanitize=address in Debug builds" OFF) -option(HERMES_USE_THREAD_SANITIZER "Enable -fsanitize=thread in Debug builds" OFF) -option(HERMES_RPC_THALLIUM "Use Thallium as the RPC library." ON) -option(HERMES_BUILD_BENCHMARKS "Build the Hermes benchmark suite." ON) -option(HERMES_ENABLE_COVERAGE "Enable code coverage." OFF) option(HERMES_ENABLE_POSIX_ADAPTER "Build the Hermes POSIX adapter." ON) option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." ON) option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON) -option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." ON) +option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF) option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF) option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) -option(HERMES_BUILD_ADAPTER_TESTS "Enable installation of tests." ON) -option(HERMES_INSTALL_TESTS "Enable installation of tests." OFF) -option(HERMES_ONLY_RPC "Avoids the shared-memory case in RPC for testing" OFF) -option(HERMES_ENABLE_PROFILING "Enables profiling of certain functions" OFF) -option(HERMES_ENABLE_C_BINDINGS "Used for compiling java API" OFF) -option(HERMES_ENABLE_JAVA_BINDINGS "Used for compiling java API" OFF) -option(HERMES_ENABLE_PYTHON_BINDINGS "Used for compiling python API" OFF) -option(BUILD_HSHM_TESTS "Build data structure tests" OFF) - -option(HERMES_PTHREADS_ENABLED "Support spawning pthreads" ON) -option(HERMES_DEBUG_LOCK "Used for debugging locks" OFF) +option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" ON) -# Compile definitions -if (HERMES_PTHREADS_ENABLED) - add_compile_definitions(HERMES_PTHREADS_ENABLED) -endif() -if (HERMES_RPC_THALLIUM) - add_compile_definitions(HERMES_RPC_THALLIUM) -endif() -if (HERMES_DEBUG_LOCK) - message("Lock debugging enabled") - add_compile_definitions(HERMES_DEBUG_LOCK) -endif() - -# Calculate code coverage with debug mode -if(HERMES_ENABLE_COVERAGE) - if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - message(STATUS "Use code coverage with debug mode") - set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "" FORCE) - endif() -endif() +message("HERMES_ENABLE_PYTHON: ${HERMES_ENABLE_PYTHON}") -if(BUILD_SHARED_LIBS) - set(HERMES_BUILD_SHARED_LIBS 1) - set(HERMES_LIBTYPE SHARED) +#----------------------------------------------------------------------------- +# Compiler Optimization +#----------------------------------------------------------------------------- +set(CMAKE_CXX_STANDARD 17) +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + message("IN DEBUG MODE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0") + add_compile_definitions(HERMES_LOG_VERBOSITY=10) else() - message(FATAL_ERROR "Must build hermes with shared libs") + message("IN RELEASE MODE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O3") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3") + add_compile_definitions(HERMES_LOG_VERBOSITY=1) endif() +add_compile_options(-march=native -fomit-frame-pointer) -# TODO(chogan): Expose this once we have more than one communication layer -mark_as_advanced(HERMES_COMMUNICATION_MPI) -# TODO(chogan): Expose this once we support more than one RPC layer -mark_as_advanced(HERMES_RPC_THALLIUM) - -if (CMAKE_BUILD_TYPE STREQUAL "Debug") - if (NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-g3.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g3" - CACHE STRING "" FORCE) - endif() - - if (HERMES_USE_ADDRESS_SANITIZER) - if(NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=address.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address" - CACHE STRING "" FORCE) - endif() - else() - string(REPLACE - "-fsanitize=address" "" FLAGS_NO_SANITIZE "${CMAKE_CXX_FLAGS_DEBUG}") - set(CMAKE_CXX_FLAGS_DEBUG "${FLAGS_NO_SANITIZE}" CACHE STRING "" FORCE) - endif() - - if (HERMES_USE_THREAD_SANITIZER AND HERMES_USE_ADDRESS_SANITIZER) - message(FATAL_ERROR "Cannont use -fsanitize=address and -fsanitize=thread " - "at the same time") - else() - if (HERMES_USE_THREAD_SANITIZER) - if(NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=thread.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=thread" - CACHE STRING "" FORCE) - endif() - else() - string(REPLACE - "-fsanitize=thread" "" FLAGS_NO_SANITIZE "${CMAKE_CXX_FLAGS_DEBUG}") - set(CMAKE_CXX_FLAGS_DEBUG "${FLAGS_NO_SANITIZE}" CACHE STRING "" FORCE) - endif() - endif() +#----------------------------------------------------------------------------- +# Targets built within this project are exported at Install time for use +# by other projects. +#----------------------------------------------------------------------------- +if(NOT HERMES_EXPORTED_TARGETS) + set(HERMES_EXPORTED_TARGETS "hrun-targets") endif() -#------------------------------------------------------------------------------- -function(hermes_set_lib_options libtarget libname libtype) - if(${libtype} MATCHES "SHARED") - if(WIN32 AND NOT MINGW) - set(LIB_RELEASE_NAME "${libname}") - set(LIB_DEBUG_NAME "${libname}_D") - set(LIB_VERSION ${HERMES_PACKAGE_VERSION_MAJOR}) - else() - set(LIB_RELEASE_NAME "${libname}") - set(LIB_DEBUG_NAME "${libname}_debug") - set(LIB_VERSION ${HERMES_PACKAGE_VERSION}) - endif() - else() - if(WIN32 AND NOT MINGW) - set(LIB_RELEASE_NAME "lib${libname}") - set(LIB_DEBUG_NAME "lib${libname}_D") - else() - # if the generator supports configuration types or if the CMAKE_BUILD_TYPE has a value - if(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - set(LIB_RELEASE_NAME "${libname}") - set(LIB_DEBUG_NAME "${libname}_debug") - else() - set(LIB_RELEASE_NAME "lib${libname}") - set(LIB_DEBUG_NAME "lib${libname}_debug") - endif() - endif() - endif() +#----------------------------------------------------------------------------- +# Find Packages +#----------------------------------------------------------------------------- - set_target_properties(${libtarget} - PROPERTIES - DEBUG_OUTPUT_NAME ${LIB_DEBUG_NAME} - RELEASE_OUTPUT_NAME ${LIB_RELEASE_NAME} - MINSIZEREL_OUTPUT_NAME ${LIB_RELEASE_NAME} - RELWITHDEBINFO_OUTPUT_NAME ${LIB_RELEASE_NAME} - VERSION ${LIB_VERSION} - SOVERSION ${LIB_VERSION} - ) +# HermesShm +find_package(HermesShm CONFIG REQUIRED) +message(STATUS "found hermes_shm.h at ${HermesShm_INCLUDE_DIRS}") - #----- Use MSVC Naming conventions for Shared Libraries - if(MINGW AND ${libtype} MATCHES "SHARED") - set_target_properties(${libtarget} - PROPERTIES - IMPORT_SUFFIX ".lib" - IMPORT_PREFIX "" - PREFIX "" - ) - endif() -endfunction() +# YAML-CPP +find_package(yaml-cpp REQUIRED) +message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") -#----------------------------------------------------------------------------- -# Dependencies common to all subdirectories -#----------------------------------------------------------------------------- -# thallium -if(HERMES_RPC_THALLIUM) - find_package(thallium CONFIG REQUIRED) - if(thallium_FOUND) - message(STATUS "found thallium at ${thallium_DIR}") - endif() +# Catch2 +find_package(Catch2 3.0.1 REQUIRED) +message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}") + +# MPICH +if(BUILD_MPI_TESTS) + find_package(MPI REQUIRED COMPONENTS C CXX) + message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") endif() -#YAML-CPP -find_package(yaml-cpp REQUIRED) -if(yaml-cpp_FOUND) - message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}") +# OpenMP +if(BUILD_OpenMP_TESTS) + find_package(OpenMP REQUIRED COMPONENTS C CXX) + message(STATUS "found omp.h at ${OpenMP_CXX_INCLUDE_DIRS}") endif() -#Cereal +# Cereal find_package(cereal REQUIRED) if(cereal) - message(STATUS "found cereal") -endif() - -#if(HERMES_COMMUNICATION_MPI) -find_package(MPI REQUIRED COMPONENTS C CXX) -message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") -#endif() - -# librt -if(NOT APPLE) - find_library(LIBRT rt) - if(NOT LIBRT) - message(FATAL_ERROR "librt is required for POSIX shared memory") - endif() + message(STATUS "found cereal") endif() -# HDF5 -if(HERMES_ENABLE_VFD) - set(HERMES_REQUIRED_HDF5_VERSION 1.14.0) - set(HERMES_REQUIRED_HDF5_COMPONENTS C) - find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} CONFIG NAMES hdf5 - COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS} shared) - if(HDF5_FOUND) - message(STATUS "found HDF5 ${HDF5_VERSION} at ${HDF5_INCLUDE_DIR}") - set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - ${HDF5_INCLUDE_DIR}) - set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} - ${HDF5_C_SHARED_LIBRARY}) - else() - # Allow for HDF5 autotools builds - find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} MODULE REQUIRED - COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS}) - if(HDF5_FOUND) - set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - ${HDF5_INCLUDE_DIRS}) - set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES - ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} - ${HDF5_LIBRARIES}) - else() - message(FATAL_ERROR "Could not find HDF5, please set HDF5_DIR (1.13.0) or HDF5_ROOT (1.13.1).") - endif() - endif() +# Pkg-Config +find_package(PkgConfig REQUIRED) +if(PkgConfig) + message(STATUS "found pkg config") endif() -#----------------------------------------------------------------------------- -# Coverage -#----------------------------------------------------------------------------- -if(HERMES_ENABLE_COVERAGE) - set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage" CACHE STRING - "Flags to the coverage program to perform coverage inspection" - ) - mark_as_advanced(COVERAGE_FLAGS) +# Zeromq +#pkg_check_modules(ZMQ REQUIRED libzmq) +#include_directories(${ZMQ_INCLUDE_DIRS}) +#message("Found libzmq at: ${ZMQ_INCLUDE_DIRS}") - macro(set_coverage_flags target) - set_target_properties(${target} - PROPERTIES - COMPILE_FLAGS ${COVERAGE_FLAGS} - LINK_FLAGS ${COVERAGE_FLAGS} - ) - endmacro() +# Thallium +find_package(thallium CONFIG REQUIRED) +if(thallium_FOUND) + message(STATUS "found thallium at ${thallium_DIR}") endif() -#----------------------------------------------------------------------------- -# Enable Testing -#----------------------------------------------------------------------------- -# General function used to hook ctest to python test tool lib -function(pytest test_type test_name) - set(script ${CMAKE_SOURCE_DIR}/ci/py_hermes_ci/bin/run_test) - add_test(NAME ${test_name} - COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER}) -endfunction() - -include(CTest) - -# Add testing directory -if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) - find_package(Catch2 3.0.1 REQUIRED) - find_program(IOR_EXE ior) - if(IOR_EXE) - message(STATUS "Found ior at ${IOR_EXE}") - set(HERMES_HAVE_IOR "YES") - else() - message(WARNING "Couldn't find the 'ior' executable. Some tests will be skipped.") - set(HERMES_HAVE_IOR "NO") - endif() - enable_testing() +# Boost +find_package(Boost REQUIRED COMPONENTS regex system filesystem fiber REQUIRED) +if (Boost_FOUND) + message(STATUS "found boost at ${Boost_INCLUDE_DIRS}") endif() +include_directories(${Boost_INCLUDE_DIRS}) +message("Boost: ${Boost_LIBRARIES}") -#----------------------------------------------------------------------------- -# Source -#----------------------------------------------------------------------------- - -set(HERMES_EXPORTED_LIBS "") - -include_directories(${CMAKE_SOURCE_DIR}/hermes_shm/include) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src) -add_subdirectory(${CMAKE_SOURCE_DIR}/hermes_shm) -add_subdirectory(${CMAKE_SOURCE_DIR}/wrapper) - -add_custom_target(rpc COMMAND bash - ${CMAKE_SOURCE_DIR}/code_generators/code_generators/rpc/rpcgen.sh - "${CMAKE_SOURCE_DIR}" - "${CMAKE_BINARY_DIR}") -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/data_stager) +#------------------------------------------------------------------------------ +# Setup CMake Environment +#------------------------------------------------------------------------------ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables.") +set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries.") -##### ADAPTERS -if(HERMES_ENABLE_STDIO_ADAPTER OR HERMES_ENABLE_POSIX_ADAPTER OR - HERMES_ENABLE_MPIIO_ADAPTER OR HERMES_ENABLE_PUBSUB_ADAPTER OR - HERMES_ENABLE_VFD) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/adapter) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/traits) +#------------------------------------------------------------------------------ +# Setup install and output Directories +#------------------------------------------------------------------------------ +if(NOT HERMES_INSTALL_BIN_DIR) + set(HERMES_INSTALL_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin) endif() - -#----------------------------------------------------------------------------- -# Testing Sources -#----------------------------------------------------------------------------- -if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) - add_subdirectory(test) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/data_stager/test) +if(NOT HERMES_INSTALL_LIB_DIR) + set(HERMES_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib) endif() - -#----------------------------------------------------------------------------- -# Documentation -#----------------------------------------------------------------------------- - -if(HERMES_ENABLE_DOXYGEN) - include(UseDoxygenDoc) - - add_doxygen_doc( - BUILD_DIR - ${CMAKE_CURRENT_BINARY_DIR}/_build - DOXY_FILE - ${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in - TARGET_NAME - dox - COMMENT - "HTML documentation" - ) +if(NOT HERMES_INSTALL_INCLUDE_DIR) + set(HERMES_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_PREFIX}/include) endif() - -#----------------------------------------------------------------------------- -# Benchmarks +if(NOT HERMES_INSTALL_DATA_DIR) + set(HERMES_INSTALL_DATA_DIR ${CMAKE_INSTALL_PREFIX}/share) +endif() + +#----------------------------------------------------------------------------- +# Build Hermes Main Packages +#----------------------------------------------------------------------------- +# Main includes +include_directories(${HermesShm_INCLUDE_DIRS}) +include_directories(${CMAKE_SOURCE_DIR}) +include_directories(${CMAKE_SOURCE_DIR}/include) +# Required Task includes +include_directories(${CMAKE_SOURCE_DIR}/hrun/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/hrun_admin/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/small_message/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/remote_queue/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/worch_proc_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/worch_queue_round_robin/include) +include_directories(${CMAKE_SOURCE_DIR}/hrun/tasks_required/proc_queue/include) +# Task includes +include_directories(${CMAKE_SOURCE_DIR}/tasks) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/bdev/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/ram_bdev/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/posix_bdev/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_mdm/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_blob_mdm/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_bucket_mdm/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_adapters/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/data_stager/include) +include_directories(${CMAKE_SOURCE_DIR}/tasks/hermes_data_op/include) +# Test includes +include_directories(${CMAKE_SOURCE_DIR}/test/unit) + +set(Hermes_CLIENT_LIBRARIES + ${HermesShm_LIBRARIES} + yaml-cpp + cereal::cereal + -ldl -lrt -lc -pthread hrun_client) +set(Hermes_CLIENT_DEPS + hrun_client) +set(Hermes_RUNTIME_LIBRARIES + ${Hermes_CLIENT_LIBRARIES} + hrun_runtime + ${Boost_LIBRARIES}) +set(Hermes_RUNTIME_DEPS + hrun_client hrun_runtime) + +set(TEST_MAIN ${CMAKE_SOURCE_DIR}/test/unit) +if (HERMES_ENABLE_PYTHON) +add_subdirectory(external/pybind11) +endif() +add_subdirectory(hrun) +add_subdirectory(src) +add_subdirectory(hermes_adapters) +add_subdirectory(tasks) +add_subdirectory(benchmark) +add_subdirectory(wrapper) +add_custom_target(lint COMMAND bash ${CMAKE_SOURCE_DIR}/ci/lint.sh ${CMAKE_SOURCE_DIR}) + +#----------------------------------------------------------------------------- +# Build + Enable Testing #----------------------------------------------------------------------------- -if(HERMES_BUILD_BENCHMARKS) - message("Building benchmarks") - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/benchmarks) -endif() - -install( - FILES - test/data/hermes_server.yaml - TYPE - DATA - RENAME - hermes_sample.yaml -) +# General function used to hook ctest to python test tool lib +function(pytest test_type test_name) + set(script ${CMAKE_SOURCE_DIR}/scripts/ci/py_hermes_ci/bin/run_test) +# add_test(NAME ${test_name} +# COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER}) +endfunction() -add_custom_target(lint - COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/ci/lint.sh ${CMAKE_CURRENT_SOURCE_DIR} -) +enable_testing() +add_subdirectory(test) #----------------------------------------------------------------------------- -# Install HermesConfig.cmake +# Install Hermes Headers #----------------------------------------------------------------------------- configure_file( - ${HERMES_SOURCE_DIR}/CMake/HermesConfig.cmake - ${HERMES_BINARY_DIR}/CMakeFiles/HermesConfig.cmake @ONLY + ${CMAKE_CURRENT_SOURCE_DIR}/CMake/HermesConfig.cmake + ${PROJECT_BINARY_DIR}/CMakeFiles/HermesConfig.cmake @ONLY ) install( - FILES - ${HERMES_BINARY_DIR}/CMakeFiles/HermesConfig.cmake - DESTINATION - ${CMAKE_INSTALL_PREFIX}/cmake + FILES + ${PROJECT_BINARY_DIR}/CMakeFiles/HermesConfig.cmake + DESTINATION + ${CMAKE_INSTALL_PREFIX}/cmake ) + +install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) \ No newline at end of file diff --git a/COPYING b/COPYING deleted file mode 100644 index bb9689a91..000000000 --- a/COPYING +++ /dev/null @@ -1,52 +0,0 @@ -Copyright Notice and License Terms for -Hermes I/O Buffering Software Library and Utilities - ------------------------------------------------------------------------------- - -Hermes I/O Buffering Software Library and Utilities -Copyright 2018-2021, The HDF Group and Illinois Institute of Technology - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of The HDF Group, Illinois Institute of Technology, nor - the names of Contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -DISCLAIMER: - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -You are under no obligation whatsoever to provide any bug fixes, patches, or -upgrades to the features, functionality or performance of the source code -("Enhancements") to anyone; however, if you choose to make your Enhancements -available either publicly, or directly to the copyright holders, without -imposing a separate written license agreement for such Enhancements, then you -hereby grant the following license: a non-exclusive, royalty-free perpetual -license to install, use, modify, prepare derivative works, incorporate into -other computer software, distribute, and sublicense such enhancements or -derivative works thereof, in binary and source code form. - ------------------------------------------------------------------------------- - -Hermes was developed with support from the National Science Foundation (NSF) -under award NSF OCI-1835764. diff --git a/README.md b/README.md index 53d224120..dec668244 100644 --- a/README.md +++ b/README.md @@ -22,50 +22,28 @@ Hermes is a heterogeneous-aware, multi-tiered, dynamic, and distributed I/O buff [Spack](https://spack.io/) is the easiest way to get Hermes and all its dependencies installed. -```bash -# Install spack if you don't already have it -SPACK_DIR=~/spack -git clone https://github.com/spack/spack ${SPACK_DIR} -. ${SPACK_DIR}/share/spack/setup-env.sh -spack install hermes -``` - -If this doesn't work, the Hermes and Mochi spack recipes might be out of sync -with the version of spack you're using. In that case, you can try the most -up-to-date repos. - ```bash # set location of hermes_file_staging -STAGE_DIR=~/hermes_stage -MOCHI_REPO=${STAGE_DIR}/mochi -HERMES_REPO=${STAGE_DIR}/hermes -git clone https://github.com/mochi-hpc/mochi-spack-packages.git ${MOCHI_REPO} -git clone https://github.com/HDFGroup/hermes ${HERMES_REPO} -spack repo add ${MOCHI_REPO} +git clone https://github.com/HDFGroup/hermes spack repo add ${HERMES_REPO}/ci/hermes -spack install hermes +# The stable 1.1. development branch +spack install hermes@hdf-1.1 +# The unstable 1.1. development branch +spack install hermes@dev-1.1 ``` ### CMake Hermes makes use of the CMake build system and requires an out of source build. - cd /path/to/hermes - mkdir build - cd build - ccmake .. - -Type 'c' to configure until there are no errors, then generate the makefile with 'g'. The default options should suffice for most use cases. In addition, we recommend the following options. - - -DCMAKE_INSTALL_PREFIX=/installation/prefix - -DCMAKE_PREFIX_PATH=/path/to/dependencies - -DCMAKE_BUILD_RPATH=/path/to/dependencies/lib - -DCMAKE_INSTALL_RPATH=/path/to/dependencies/lib - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_COMPILER=`which mpicxx` - -DBUILD_SHARED_LIBS=ON - -After the makefile has been generated, you can type `make -j 4` or `cmake --build . -- -j 4`. Add `VERBOSE=1` to see detailed compiler output. +``` +cd /path/to/hermes +mkdir build +cd build +cmake ../ -DCMAKE_BUILD_TYPE=Relase -DCMAKE_INSTALL_PREFIX=... +make -j8 +make install +``` ### Testing and Installation diff --git a/adapter/CMakeLists.txt b/adapter/CMakeLists.txt deleted file mode 100644 index 1f4b884f0..000000000 --- a/adapter/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ -# Set hermes to preload if adapter is linked. -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_PRELOAD -DHERMES_RPC_THALLIUM") - -set(HERMES_SRC_DIR ${CMAKE_SOURCE_DIR}/src) -set(HERMES_ADAPTER_DIR ${CMAKE_SOURCE_DIR}/adapter) -set(HERMES_IO_CLIENT_DIR ${CMAKE_SOURCE_DIR}/io_client) - -include_directories( - ${CMAKE_SOURCE_DIR} - ${HERMES_SRC_DIR} - ${HERMES_ADAPTER_DIR} - ${HERMES_IO_CLIENT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}) - -add_subdirectory(filesystem) -if(HERMES_ENABLE_POSIX_ADAPTER) - add_subdirectory(posix) -endif() -if(HERMES_ENABLE_STDIO_ADAPTER) - add_subdirectory(stdio) -endif() -if(HERMES_ENABLE_MPIIO_ADAPTER) - add_subdirectory(mpiio) -endif() -if(HERMES_ENABLE_VFD) - add_subdirectory(vfd) -endif() -if(HERMES_ENABLE_KVSTORE) - add_subdirectory(kvstore) -endif() -if(HERMES_BUILD_ADAPTER_TESTS) - add_subdirectory(test) -endif() - -#----------------------------------------------------------------------------- -# Specify project header files to be installed -#----------------------------------------------------------------------------- -file(GLOB_RECURSE HERMES_HEADERS "*.h") - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install( - FILES - ${HERMES_HEADERS} - DESTINATION - ${HERMES_INSTALL_INCLUDE_DIR}/adapter - COMPONENT - headers -) \ No newline at end of file diff --git a/adapter/kvstore/README.md b/adapter/kvstore/README.md deleted file mode 100644 index 96b62be0e..000000000 --- a/adapter/kvstore/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# KVStore - -The Hermes KVStore adapter provides a simplistic in-memory key-value store -interface similar to that of Redis. Currently, we only provide the minimum -functionality to benchmark KVStore over the YCSB benchmark. - -## Current Operation Support - -Below we describe the operations our KVStore currently supports: -1. Create a table: a table contains a set of records. -2. Create a record: a record is a )>. The ARRAY -is stored as a single blob in Hermes. Updating a field in a record will -require reading the entire blob. -3. Update a record -4. Get a set of fields from a record -5. Scan a subset of records -6. Delete a record -7. Delete a table \ No newline at end of file diff --git a/adapter/kvstore/java/build.gradle b/adapter/kvstore/java/build.gradle deleted file mode 100644 index 1c0d21482..000000000 --- a/adapter/kvstore/java/build.gradle +++ /dev/null @@ -1,49 +0,0 @@ -/* - * This build file was generated by the Gradle 'init' task. - * - * This generated file contains a commented-out sample Java project to get you started. - * For more details take a look at the Java Quickstart chapter in the Gradle - * user guide available at https://docs.gradle.org/4.4.1/userguide/tutorial_java_projects.html - */ - -// Apply the java plugin to add support for Java -apply plugin: 'java' - -// In this section you declare where to find the dependencies of your project -repositories { - // Use 'jcenter' for resolving your dependencies. - // You can declare any Maven/Ivy/file repository here. - jcenter() -} - -// In this section you declare the dependencies for your production and test code -dependencies { - // The production code uses the SLF4J logging API at compile time - compile 'org.slf4j:slf4j-api:1.7.25' - - // Declare the dependency for your favourite test framework you want to use in your tests. - // TestNG is also supported by the Gradle Test task. Just change the - // testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add - // 'test.useTestNG()' to your build script. - testCompile 'junit:junit:4.12' - - compile files('../../../wrapper/java/build/libs/hermes-1.0.0.jar') -} - -sourceSets { - main { - java { - srcDir 'src/kvstore/java' - } - } - - test { - java { - srcDir 'src/test/java' - } - } -} - -test { - useJUnit() -} \ No newline at end of file diff --git a/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.jar b/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 99340b4ad..000000000 Binary files a/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.properties b/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 2c2bbe5f9..000000000 --- a/adapter/kvstore/java/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-bin.zip diff --git a/adapter/kvstore/java/gradlew b/adapter/kvstore/java/gradlew deleted file mode 100755 index cccdd3d51..000000000 --- a/adapter/kvstore/java/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/adapter/kvstore/java/gradlew.bat b/adapter/kvstore/java/gradlew.bat deleted file mode 100644 index e95643d6a..000000000 --- a/adapter/kvstore/java/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/adapter/kvstore/java/settings.gradle b/adapter/kvstore/java/settings.gradle deleted file mode 100644 index 371bf447f..000000000 --- a/adapter/kvstore/java/settings.gradle +++ /dev/null @@ -1,18 +0,0 @@ -/* - * This settings file was generated by the Gradle 'init' task. - * - * The settings file is used to specify which projects to include in your build. - * In a single project build this file can be empty or even removed. - * - * Detailed information about configuring a multi-project build in Gradle can be found - * in the user guide at https://docs.gradle.org/4.4.1/userguide/multi_project_builds.html - */ - -/* -// To declare projects as part of a multi-project build use the 'include' method -include 'shared' -include 'api' -include 'services:webservice' -*/ - -rootProject.name = 'hermes_kvstore-1.0.0' diff --git a/adapter/kvstore/java/src/kvstore/java/KVStore.java b/adapter/kvstore/java/src/kvstore/java/KVStore.java deleted file mode 100644 index 8bd924830..000000000 --- a/adapter/kvstore/java/src/kvstore/java/KVStore.java +++ /dev/null @@ -1,14 +0,0 @@ -package hermes_kvstore.java; -import hermes_kvstore.java.KVTable; -import hermes.java.Hermes; - -public class KVStore { - public static void connect() { - Hermes hermes = Hermes.getInstance(); - hermes.create(); - } - - public static KVTable getTable(String table_name) { - return new KVTable(table_name); - } -} diff --git a/adapter/kvstore/java/src/kvstore/java/KVTable.java b/adapter/kvstore/java/src/kvstore/java/KVTable.java deleted file mode 100644 index 596226ada..000000000 --- a/adapter/kvstore/java/src/kvstore/java/KVTable.java +++ /dev/null @@ -1,160 +0,0 @@ -package hermes_kvstore.java; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.HashMap; -import java.util.Set; -import java.util.HashSet; -import java.io.IOException; -import java.io.ByteArrayOutputStream; -import java.io.ByteArrayInputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - -import hermes.java.Hermes; -import hermes.java.Bucket; -import hermes.java.Blob; -import hermes.java.MdLockType; -import hermes.java.UniqueId; - -public class KVTable { - Bucket bkt_; - - /** Emplace Constructor */ - KVTable(String table_name) { - Hermes hermes = Hermes.getInstance(); - bkt_ = hermes.getBucket(table_name); - } - - /** Serialize a Map into a blob */ - private Blob blobSerialize(T obj) throws IOException { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(baos); - oos.writeObject(obj); - oos.close(); - byte[] bytes = baos.toByteArray(); - return new Blob(ByteBuffer.wrap(bytes)); - } catch (IOException e) { - e.printStackTrace(); - throw e; - } - } - - /** Deserialize a Map from a blob */ - private T blobDeserialize(Blob blob) throws IOException, ClassNotFoundException { - try { - byte[] bytes = blob.array(); - ByteArrayInputStream istream = new ByteArrayInputStream(bytes); - ObjectInputStream ois = new ObjectInputStream(istream); - T obj = (T) ois.readObject(); - ois.close(); - blob.close(); - return obj; - } catch (IOException e) { - e.printStackTrace(); - throw e; - } catch (ClassNotFoundException e) { - e.printStackTrace(); - throw e; - } - } - - public String getFieldKey(String key, String field_name) { - return String.format("%s-%s", key, field_name); - } - - public String getKvMetadataKey() { - return "hermes-kvmetadata"; - } - - /** - * Insert a new record into the table - * */ - public void insert(String key, Map val) throws IOException { - /*UniqueId md_id = bkt_.getBlobId(getKvMetadataKey()); - if (md_id.isNull()) { - Set keys = new HashSet(val.keySet()); - Blob md_blob = blobSerialize(keys); - bkt_.put(getKvMetadataKey(), md_blob); - md_blob.close(); - } - for (Map.Entry entry : val.entrySet()) { - String field_key = getFieldKey(key, entry.getKey()); - Blob blob = blobSerialize(entry.getValue()); - bkt_.put(field_key, blob); - blob.close(); - }*/ - Blob blob = blobSerialize(val); - bkt_.put(key, blob); - blob.close(); - } - - /** - * Create or insert a record into the table - * - * @param key the record key - * @param val the values to update in the record - * @return None - * */ - public void update(String key, Map val) throws IOException, ClassNotFoundException { - /*insert(key, val);*/ - UniqueId blob_id = bkt_.getBlobId(key); - if (blob_id.isNull()) { - insert(key, val); - } else { - bkt_.lockBlob(blob_id, MdLockType.kExternalWrite); - Blob orig_blob = bkt_.get(blob_id); - Map old_val = blobDeserialize(orig_blob); - old_val.putAll(val); - Blob new_blob = blobSerialize(old_val); - bkt_.put(key, new_blob); - bkt_.unlockBlob(blob_id, MdLockType.kExternalWrite); - new_blob.close(); - } - } - - /** - * Get a subset of fields from a record - * - * @param key the record key - * @param field_set the field in the record to update - * @return The blob containing only the field's data - * */ - public Map read(String key, Set field_set) throws IOException, ClassNotFoundException { - /*HashMap map = new HashMap(); - if (field_set.isEmpty()) { - UniqueId md_id = bkt_.getBlobId(getKvMetadataKey()); - Blob md_blob = bkt_.get(md_id); - field_set = (HashSet)blobDeserialize(md_blob); - } - for (String field_name : field_set) { - UniqueId blob_id = bkt_.getBlobId(getFieldKey(key, field_name)); - Blob blob = bkt_.get(blob_id); - map.put(field_name, blobDeserialize(blob)); - } - return map;*/ - - UniqueId blob_id = bkt_.getBlobId(key); - bkt_.lockBlob(blob_id, MdLockType.kExternalRead); - Blob orig_blob = bkt_.get(blob_id); - Map old_val = blobDeserialize(orig_blob); - bkt_.unlockBlob(blob_id, MdLockType.kExternalRead); - return old_val; - } - - public Map read(String key) throws IOException, ClassNotFoundException { - return read(key, new HashSet()); - } - - /** Delete a record */ - public void erase(String key) { - UniqueId blob_id = bkt_.getBlobId(key); - bkt_.destroyBlob(blob_id); - } - - /** Destroy this table */ - public void destroy() { - bkt_.destroy(); - } -}; diff --git a/adapter/kvstore/java/src/test/java/KvstoreJniTest.java b/adapter/kvstore/java/src/test/java/KvstoreJniTest.java deleted file mode 100644 index 12168bf67..000000000 --- a/adapter/kvstore/java/src/test/java/KvstoreJniTest.java +++ /dev/null @@ -1,34 +0,0 @@ -import org.junit.Test; -import static org.junit.Assert.*; -import java.nio.*; -import java.util.*; -import hermes_kvstore.java.KVStore; -import hermes_kvstore.java.KVTable; - -import java.lang.management.ManagementFactory; - -public class KvstoreJniTest { - @Test - public void testKvPutGet() throws Exception { - String pid = ManagementFactory.getRuntimeMXBean().getName(); - System.out.println(pid); - KVStore.connect(); - KVTable table = KVStore.getTable("hello");/**/ - - HashMap record = new HashMap(); - try { - record.put("f0", "abcde"); - record.put("f1", "12345"); - record.put("f2", "ABCDE"); - table.update("0", record); - table.update("0", record); - Map read_record = table.read("0"); - for (Map.Entry entry : read_record.entrySet()) { - assertTrue(entry.getValue().equals(record.get(entry.getKey()))); - } - } catch (Exception e) { - e.printStackTrace(); - throw e; - } - } -} diff --git a/adapter/kvstore/kvstore.cc b/adapter/kvstore/kvstore.cc deleted file mode 100644 index a40648c15..000000000 --- a/adapter/kvstore/kvstore.cc +++ /dev/null @@ -1,140 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "kvstore.h" - -namespace hermes::adapter { - -/**==================================== - * KVRecord - * ===================================*/ - -/** Convert the record from a single array of bytes */ -KVRecord::KVRecord(hapi::Blob &blob) { - BlobSerializer serial(blob); - while (serial.off_ < blob.size()) { - std::string field_name; - hapi::Blob field_val; - serial >> field_name; - serial >> field_val; - record_.emplace(std::move(field_name), std::move(field_val)); - } -} - -/** Convert the record into a single array of bytes */ -hapi::Blob KVRecord::value() { - BlobSerializeCounter counter; - - // Get the total size of the serialized blob - for (auto &[field_name, field_val] : record_) { - counter << field_name; - counter << field_val; - } - - // Produce the serialized blob - hapi::Blob blob(counter.size_); - BlobSerializer serial(blob); - for (auto &[field_name, field_val] : record_) { - serial << field_name; - serial << field_val; - } - - return blob; -} - -/**==================================== - * KVTable - * ===================================*/ - -/** - * Create or insert a record into the table - * - * @param key the record key - * @param val the values to update in the record - * @return None - * */ -void KVTable::Update(const std::string &key, KVRecord &val) { - hermes::BlobId blob_id; - bkt_.GetBlobId(key, blob_id); - if (!blob_id.IsNull()) { - hapi::Context ctx; - hapi::Blob record_serial; - bkt_.LockBlob(blob_id, MdLockType::kExternalWrite); - bkt_.Get(blob_id, record_serial, ctx); - KVRecord orig(record_serial); - for (auto &[field, field_val] : val.record_) { - orig.record_[field] = field_val; - } - bkt_.Put(key, orig.value(), blob_id, ctx); - bkt_.UnlockBlob(blob_id, MdLockType::kExternalWrite); - } else { - hapi::Context ctx; - hapi::Blob record_serial; - bkt_.LockBlob(blob_id, MdLockType::kExternalWrite); - bkt_.Put(key, val.value(), blob_id, ctx); - bkt_.UnlockBlob(blob_id, MdLockType::kExternalWrite); - } -} - -/** - * Get a subset of fields from a record - * - * @param key the record key - * @param field the field in the record to update - * @return The blob containing only the field's data - * */ -KVRecord KVTable::Read(const std::string &key, const KVFieldSet &field) { - hapi::Blob blob; - hermes::BlobId blob_id; - hapi::Context ctx; - bkt_.GetBlobId(key, blob_id); - bkt_.Get(blob_id, blob, ctx); - KVRecord orig_record(blob); - if (field.size() == 0) { - return orig_record; - } - - KVRecord new_record; - for (const std::string &field_name : field) { - new_record.record_[field_name] = orig_record.record_[field_name]; - } - return new_record; -} - -/** Delete a record */ -void KVTable::Erase(const std::string &key) { - hapi::Context ctx; - hermes::BlobId blob_id; - bkt_.GetBlobId(key, blob_id); - bkt_.DestroyBlob(blob_id, ctx); -} - -/** Destroy this table */ -void KVTable::Destroy() { - bkt_.Destroy(); -} - -/**==================================== - * KVStore - * ===================================*/ - -/** Connect to Hermes */ -void KVStore::Connect() { - HERMES->Create(hermes::HermesType::kClient); -} - -/** Get or create a table */ -KVTable KVStore::GetTable(const std::string table_name) { - return KVTable(HERMES->GetBucket(table_name)); -} - -} // namespace hermes::adapter diff --git a/adapter/kvstore/kvstore.h b/adapter/kvstore/kvstore.h deleted file mode 100644 index b914e9bb0..000000000 --- a/adapter/kvstore/kvstore.h +++ /dev/null @@ -1,177 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ADAPTER_KVSTORE_KVSTORE_H_ -#define HERMES_ADAPTER_KVSTORE_KVSTORE_H_ - -#include -#include -#include "hermes.h" - -namespace hermes::adapter { - -/** Denotes the set of fields for a record */ -typedef std::unordered_set KVFieldSet; - -/** Measure the serialized size */ -struct BlobSerializeCounter { - size_t size_; - - /** Constructor */ - BlobSerializeCounter() : size_(0) {} - - /** Serialize std::string into blob */ - BlobSerializeCounter& operator<<(const std::string &data) { - (*this) << data.size(); - size_ += data.size(); - return *this; - } - - /** Serialize blob into blob */ - BlobSerializeCounter& operator<<(const hapi::Blob &data) { - (*this) << data.size(); - size_ += data.size(); - return *this; - } - - /** Serialize size_t into blob */ - BlobSerializeCounter& operator<<(const size_t &data) { - size_ += sizeof(size_t); - return *this; - } -}; - -/** Serialize data into and out of blob */ -struct BlobSerializer { - hapi::Blob &blob_; - size_t off_; - - /** Constructor */ - explicit BlobSerializer(hapi::Blob &blob) : blob_(blob), off_(0) {} - - /** Serialize a string type */ - template - void SerializeString(const StringT &data) { - (*this) << data.size(); - memcpy(blob_.data() + off_, data.data(), data.size()); - off_ += data.size(); - } - - /** Deserialize a string type */ - template - void DeserializeString(StringT &data) { - size_t size; - (*this) >> size; - data.resize(size); - memcpy(data.data(), blob_.data() + off_, data.size()); - off_ += data.size(); - } - - /** Serialize std::string into blob */ - BlobSerializer& operator<<(const std::string &data) { - SerializeString(data); - return *this; - } - - /** Deserialize std::string from blob */ - BlobSerializer& operator>>(std::string &data) { - DeserializeString(data); - return *this; - } - - /** Serialize blob into blob */ - BlobSerializer& operator<<(const hapi::Blob &data) { - SerializeString(data); - return *this; - } - - /** Deserialize blob from blob */ - BlobSerializer& operator>>(hapi::Blob &data) { - DeserializeString(data); - return *this; - } - - /** Serialize size_t into blob */ - BlobSerializer& operator<<(const size_t &data) { - memcpy(blob_.data() + off_, &data, sizeof(size_t)); - off_ += sizeof(size_t); - return *this; - } - - /** Deserialize size_t from blob */ - BlobSerializer& operator>>(size_t &data) { - memcpy(&data, blob_.data() + off_, sizeof(size_t)); - off_ += sizeof(size_t); - return *this; - } -}; - -/** Denotes a record to place in a table */ -struct KVRecord { - std::unordered_map record_; - - /** Default constructor */ - KVRecord() = default; - - /** Convert the record from a single array of bytes */ - explicit KVRecord(hapi::Blob &blob); - - /** Convert the record into a single array of bytes */ - hapi::Blob value(); -}; - -/** Denotes a set of records */ -class KVTable { - public: - hapi::Bucket bkt_; - - /** Emplace Constructor */ - explicit KVTable(const hapi::Bucket &bkt) : bkt_(bkt) {} - - /** - * Create or insert a record into the table - * - * @param key the record key - * @param val the values to update in the record - * @return None - * */ - void Update(const std::string &key, KVRecord &val); - - /** - * Get a subset of fields from a record - * - * @param key the record key - * @param field the field in the record to update - * @return The blob containing only the field's data - * */ - KVRecord Read(const std::string &key, const KVFieldSet &field); - - /** Delete a record */ - void Erase(const std::string &key); - - /** Destroy this table */ - void Destroy(); -}; - -/** The key-value store instance */ -class KVStore { - public: - /** Connect to Hermes */ - void Connect(); - - /** Get or create a table */ - KVTable GetTable(const std::string table_name); -}; - -} // namespace hermes::adapter - -#endif // HERMES_ADAPTER_KVSTORE_KVSTORE_H_ diff --git a/adapter/kvstore/tests.py b/adapter/kvstore/tests.py deleted file mode 100644 index e5718b51b..000000000 --- a/adapter/kvstore/tests.py +++ /dev/null @@ -1,22 +0,0 @@ -from py_hermes_ci.test_manager import TestManager -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - - -class KvstoreTestManager(TestManager): - def spawn_all_nodes(self): - return self.spawn_info() - - def set_paths(self): - self.KVSTORE_CMD = f"./gradlew test" - self.disable_testing = False - - def test_hermes_kvstore_java(self): - return - spawn_info = self.spawn_info(nprocs=1, - hermes_conf='hermes_server', - cwd=f"{self.MY_DIR}/java") - self.start_daemon(spawn_info) - node = Exec(self.KVSTORE_CMD, spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code diff --git a/adapter/mapper/balanced_mapper.cc b/adapter/mapper/balanced_mapper.cc deleted file mode 100644 index 3afece24e..000000000 --- a/adapter/mapper/balanced_mapper.cc +++ /dev/null @@ -1,43 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "balanced_mapper.h" -#include "api/hermes.h" - -namespace hermes::adapter { - -/** - * Convert a range defined by \a off and \a size into specific - * blob placements. - */ -void BalancedMapper::map(size_t off, size_t size, - size_t page_size, - BlobPlacements &ps) { - HILOG(kDebug, "Mapping File with offset {} and size {}", off, size); - size_t kPageSize = page_size; - size_t size_mapped = 0; - while (size > size_mapped) { - BlobPlacement p; - p.bucket_off_ = off + size_mapped; - p.page_ = p.bucket_off_ / kPageSize; - p.page_size_ = page_size; - p.blob_off_ = p.bucket_off_ % kPageSize; - auto left_size_page = kPageSize - p.blob_off_; - p.blob_size_ = left_size_page < size - size_mapped ? left_size_page - : size - size_mapped; - ps.emplace_back(p); - size_mapped += p.blob_size_; - } -} - - -} // namespace hermes::adapter diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 000000000..ee640a1ae --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,66 @@ +cmake_minimum_required(VERSION 3.10) +project(hermes) + +set(CMAKE_CXX_STANDARD 17) + +#------------------------------------------------------------------------------ +# Build Tests +#------------------------------------------------------------------------------ + +add_executable(test_performance_exec + ${TEST_MAIN}/main.cc + test_init.cc + test_latency.cc) +add_dependencies(test_performance_exec + ${Hermes_RUNTIME_DEPS} hermes) +target_link_libraries(test_performance_exec + ${Hermes_RUNTIME_LIBRARIES} hermes Catch2::Catch2 + MPI::MPI_CXX) + +#add_executable(test_performance_exec +# ${TEST_MAIN}/main.cc +# test_init.cc +# test_latency.cc +# test_zmq.cc +#) +#add_dependencies(test_performance_exec +# ${Hermes_CLIENT_DEPS} hermes) +#target_link_libraries(test_performance_exec +# ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 +# MPI::MPI_CXX ${ZMQ_LIBRARIES}) + +add_executable(hermes_api_bench + hermes_api_bench.cc) +add_dependencies(hermes_api_bench + ${Hermes_CLIENT_DEPS} hermes) +target_link_libraries(hermes_api_bench + ${Hermes_CLIENT_LIBRARIES} hermes + Catch2::Catch2 MPI::MPI_CXX) + +#------------------------------------------------------------------------------ +# Test Cases +#------------------------------------------------------------------------------ + +# STRING TESTS +#add_test(NAME test_performance COMMAND +# ${CMAKE_BINARY_DIR}/bin/test_performance_exec "TestIpc") + +#------------------------------------------------------------------------------ +# Install Targets +#------------------------------------------------------------------------------ +install(TARGETS + test_performance_exec + hermes_api_bench + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_performance_exec) + set_coverage_flags(test_hermes_api) +endif() diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc new file mode 100644 index 000000000..19ba38889 --- /dev/null +++ b/benchmark/hermes_api_bench.cc @@ -0,0 +1,296 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include +#include "hermes/hermes.h" +#include "hermes/bucket.h" +#include "hrun/work_orchestrator/affinity.h" + +namespace hapi = hermes; +using hshm::MpiTimer; + +/** Gather times per-process */ +void GatherTimes(std::string test_name, size_t io_size, MpiTimer &t) { + t.Collect(); + if (t.rank_ == 0) { + double max = t.GetSec(); + double mbps = io_size / t.GetUsec(); + HIPRINT("{}: Time: {} sec, MBps (or MOps): {}, Count: {}, Nprocs: {}\n", + test_name, max, mbps, io_size, t.nprocs_); + } +} + +/** Each process PUTS into the same bucket, but with different blob names */ +void PutTest(int nprocs, int rank, + int repeat, size_t blobs_per_rank, size_t blob_size) { + MpiTimer t(MPI_COMM_WORLD); + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); + hermes::Blob blob(blob_size); + t.Resume(); + for (int j = 0; j < repeat; ++j) { + for (size_t i = 0; i < blobs_per_rank; ++i) { + size_t blob_name_int = rank * blobs_per_rank + i; + std::string name = std::to_string(blob_name_int); + bkt.AsyncPut(name, blob, ctx); + } + } + t.Pause(); + HILOG(kInfo, "Finished PUT") + GatherTimes("Put", nprocs * blobs_per_rank * blob_size * repeat, t); +} + +/** + * Each process GETS from the same bucket, but with different blob names + * MUST run PutTest first. + * */ +void GetTest(int nprocs, int rank, + int repeat, size_t blobs_per_rank, size_t blob_size) { + MpiTimer t(MPI_COMM_WORLD); + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); + t.Resume(); + for (int j = 0; j < repeat; ++j) { + for (size_t i = 0; i < blobs_per_rank; ++i) { + hermes::Blob ret(blob_size); + size_t blob_name_int = rank * blobs_per_rank + i; + std::string name = std::to_string(blob_name_int); + bkt.Get(name, ret, ctx); + } + } + t.Pause(); + GatherTimes("Get", nprocs * blobs_per_rank * blob_size * repeat, t); +} + +/** Each process PUTs then GETs */ +void PutGetTest(int nprocs, int rank, int repeat, + size_t blobs_per_rank, size_t blob_size) { + PutTest(nprocs, rank, repeat, blobs_per_rank, blob_size); + MPI_Barrier(MPI_COMM_WORLD); + GetTest(nprocs, rank, repeat, blobs_per_rank, blob_size); +} + +/** Each process PUTS into the same bucket, but with different blob names */ +void PartialPutTest(int nprocs, int rank, + int repeat, size_t blobs_per_rank, + size_t blob_size, size_t part_size) { + MpiTimer t(MPI_COMM_WORLD); + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); + hermes::Blob blob(blob_size); + t.Resume(); + for (int j = 0; j < repeat; ++j) { + for (size_t i = 0; i < blobs_per_rank; ++i) { + size_t blob_name_int = rank * blobs_per_rank + i; + std::string name = std::to_string(blob_name_int); + for (size_t cur_size = 0; cur_size < blob_size; cur_size += part_size) { + bkt.PartialPut(name, blob, cur_size, ctx); + } + } + } + t.Pause(); + GatherTimes("PartialPut", nprocs * blobs_per_rank * blob_size * repeat, t); +} + +/** + * Each process GETS from the same bucket, but with different blob names + * MUST run PutTest first. + * */ +void PartialGetTest(int nprocs, int rank, + int repeat, size_t blobs_per_rank, + size_t blob_size, size_t part_size) { + MpiTimer t(MPI_COMM_WORLD); + hermes::Context ctx; + hermes::Bucket bkt("hello", ctx); + t.Resume(); + for (int j = 0; j < repeat; ++j) { + for (size_t i = 0; i < blobs_per_rank; ++i) { + size_t blob_name_int = rank * blobs_per_rank + i; + std::string name = std::to_string(blob_name_int); + hermes::Blob ret(blob_size); + for (size_t cur_size = 0; cur_size < blob_size; cur_size += part_size) { + bkt.PartialGet(name, ret, cur_size, ctx); + } + } + } + t.Pause(); + GatherTimes("PartialGet", nprocs * blobs_per_rank * blob_size * repeat, t); +} + +/** Each process PUTs then GETs */ +void PartialPutGetTest(int nprocs, int rank, int repeat, + size_t blobs_per_rank, size_t blob_size, + size_t part_size) { + PartialPutTest(nprocs, rank, repeat, blobs_per_rank, blob_size, part_size); + MPI_Barrier(MPI_COMM_WORLD); + PartialGetTest(nprocs, rank, repeat, blobs_per_rank, blob_size, part_size); +} + +/** Each process creates a set of buckets */ +void CreateBucketTest(int nprocs, int rank, + size_t bkts_per_rank) { + MpiTimer t(MPI_COMM_WORLD); + t.Resume(); + hapi::Context ctx; + std::unordered_map mdm_; + for (size_t i = 0; i < bkts_per_rank; ++i) { + int bkt_name_int = rank * bkts_per_rank + i; + std::string bkt_name = std::to_string(bkt_name_int); + hermes::Bucket bkt(bkt_name, ctx); + } + t.Pause(); + GatherTimes("CreateBucket", bkts_per_rank * nprocs, t); +} + +/** Each process gets existing buckets */ +void GetBucketTest(int nprocs, int rank, + size_t bkts_per_rank) { + // Initially create the buckets + hapi::Context ctx; + for (size_t i = 0; i < bkts_per_rank; ++i) { + int bkt_name = rank * bkts_per_rank + i; + hermes::Bucket bkt(std::to_string(bkt_name), ctx); + } + + // Get existing buckets + MpiTimer t(MPI_COMM_WORLD); + t.Resume(); + for (size_t i = 0; i < bkts_per_rank; ++i) { + int bkt_name = rank * bkts_per_rank + i; + hapi::Bucket bkt(std::to_string(bkt_name), ctx); + } + t.Pause(); + GatherTimes("GetBucket", bkts_per_rank * nprocs, t); +} + +/** Each process deletes a number of buckets */ +void DeleteBucketTest(int nprocs, int rank, + size_t bkt_per_rank, + size_t blobs_per_bucket) { + MpiTimer t(MPI_COMM_WORLD); + hapi::Context ctx; + + // Create the buckets + for (size_t i = 0; i < bkt_per_rank; ++i) { + hapi::Bucket bkt(hshm::Formatter::format("DeleteBucket{}", rank), ctx); + hapi::Blob blob; + for (size_t j = 0; j < blobs_per_bucket; ++j) { + std::string name = std::to_string(j); + bkt.Put(name, blob, ctx); + } + } + + // Delete the buckets + t.Resume(); + for (size_t i = 0; i < bkt_per_rank; ++i) { + hapi::Bucket bkt(hshm::Formatter::format("DeleteBucket{}", rank), ctx); + bkt.Destroy(); + } + t.Pause(); + GatherTimes("DeleteBucket", nprocs * bkt_per_rank * blobs_per_bucket, t); +} + +/** Each process deletes blobs from a single bucket */ +void DeleteBlobOneBucket(int nprocs, int rank, + size_t blobs_per_rank) { +} + + +#define REQUIRE_ARGC_GE(N) \ + if (argc < (N)) { \ + HIPRINT("Requires fewer than {} params\n", N); \ + help(); \ + } + +#define REQUIRE_ARGC(N) \ + if (argc != (N)) { \ + HIPRINT("Requires exactly {} params\n", N); \ + help(); \ + } + +void help() { + printf("USAGE: ./api_bench [mode] ...\n"); + printf("USAGE: ./api_bench put [blob_size (K/M/G)] [blobs_per_rank]\n"); + printf("USAGE: ./api_bench putget [blob_size (K/M/G)] [blobs_per_rank]\n"); + printf("USAGE: ./api_bench pputget [blob_size (K/M/G)] [part_size (K/M/G)] [blobs_per_rank]\n"); + printf("USAGE: ./api_bench create_bkt [bkts_per_rank]\n"); + printf("USAGE: ./api_bench get_bkt [bkts_per_rank]\n"); + printf("USAGE: ./api_bench create_blob_1bkt [blobs_per_rank]\n"); + printf("USAGE: ./api_bench create_blob_Nbkt [blobs_per_rank]\n"); + printf("USAGE: ./api_bench del_bkt [bkt_per_rank] [blobs_per_bkt]\n"); + printf("USAGE: ./api_bench del_blobs [blobs_per_rank]\n"); + exit(1); +} + +int main(int argc, char **argv) { + int rank, nprocs; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + TRANSPARENT_HRUN(); + HERMES->ClientInit(); + + // Get mode + REQUIRE_ARGC_GE(2) + std::string mode = argv[1]; + + // Set CPU affinity + // TODO(logan): remove + ProcessAffiner::SetCpuAffinity(getpid(), 8); + + MPI_Barrier(MPI_COMM_WORLD); + + HIPRINT("Beginning {}\n", mode) + + // Run tests + try { + if (mode == "put") { + REQUIRE_ARGC(4) + size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); + size_t blobs_per_rank = atoi(argv[3]); + PutTest(nprocs, rank, 1, blobs_per_rank, blob_size); + } else if (mode == "putget") { + REQUIRE_ARGC(4) + size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); + size_t blobs_per_rank = atoi(argv[3]); + PutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size); + } else if (mode == "pputget") { + REQUIRE_ARGC(5) + size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); + size_t part_size = hshm::ConfigParse::ParseSize(argv[3]); + size_t blobs_per_rank = atoi(argv[4]); + PartialPutGetTest(nprocs, rank, 1, blobs_per_rank, blob_size, part_size); + } else if (mode == "create_bkt") { + REQUIRE_ARGC(3) + size_t bkts_per_rank = atoi(argv[2]); + CreateBucketTest(nprocs, rank, bkts_per_rank); + } else if (mode == "get_bkt") { + REQUIRE_ARGC(3) + size_t bkts_per_rank = atoi(argv[2]); + GetBucketTest(nprocs, rank, bkts_per_rank); + } else if (mode == "del_bkt") { + REQUIRE_ARGC(4) + size_t bkt_per_rank = atoi(argv[2]); + size_t blobs_per_bkt = atoi(argv[3]); + DeleteBucketTest(nprocs, rank, bkt_per_rank, blobs_per_bkt); + } else if (mode == "del_blobs") { + REQUIRE_ARGC(4) + size_t blobs_per_rank = atoi(argv[2]); + DeleteBlobOneBucket(nprocs, rank, blobs_per_rank); + } + } catch (hshm::Error &err) { + HELOG(kFatal, "Error: {}", err.what()); + } + MPI_Finalize(); +} diff --git a/hermes_shm/benchmark/lock/test_init.cc b/benchmark/test_init.cc similarity index 94% rename from hermes_shm/benchmark/lock/test_init.cc rename to benchmark/test_init.cc index 0fe134776..1757fe387 100644 --- a/hermes_shm/benchmark/lock/test_init.cc +++ b/benchmark/test_init.cc @@ -10,6 +10,9 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hrun/api/hrun_client.h" +#include "basic_test.h" #include "test_init.h" void MainPretest() { diff --git a/hermes_shm/test/unit/thread/test_init.cc b/benchmark/test_init.h similarity index 85% rename from hermes_shm/test/unit/thread/test_init.cc rename to benchmark/test_init.h index 82bac123b..cb3b36df7 100644 --- a/hermes_shm/test/unit/thread/test_init.cc +++ b/benchmark/test_init.h @@ -11,8 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "basic_test.h" +#ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ +#define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ -void MainPretest() {} +#include "hrun/hrun_types.h" -void MainPosttest() {} +#endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/benchmark/test_latency.cc b/benchmark/test_latency.cc new file mode 100644 index 000000000..a3ecba432 --- /dev/null +++ b/benchmark/test_latency.cc @@ -0,0 +1,391 @@ +// +// Created by llogan on 7/1/23. +// + +#include +#include "basic_test.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" +#include "small_message/small_message.h" +#include "hermes_shm/util/timer.h" +#include "hrun/work_orchestrator/affinity.h" +#include "hermes/hermes.h" +#include "hrun/work_orchestrator/worker.h" +#include "hrun/api/hrun_runtime.h" + +/** The performance of getting a queue */ +TEST_CASE("TestGetQueue") { + /*hrun::QueueId qid(0, 3); + HRUN_ADMIN->CreateQueueRoot(hrun::DomainId::GetLocal(), qid, + 16, 16, 256, + hshm::bitfield32_t(0)); + HRUN_CLIENT->GetQueue(qid); + + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + hrun::MultiQueue *queue = HRUN_CLIENT->GetQueue(qid); + REQUIRE(queue->id_ == qid); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec());*/ +} + +/** Single-thread performance of allocating + freeing tasks */ +TEST_CASE("TestHshmAllocateFree") { + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + size_t count = (1 << 8); + size_t reps = ops / count; + for (size_t i = 0; i < reps; ++i) { + std::vector tasks(count); + for (size_t j = 0; j < count; ++j) { + tasks[j] = HRUN_CLIENT->NewTaskRoot().ptr_; + } + for (size_t j = 0; j < count; ++j) { + HRUN_CLIENT->DelTask(tasks[j]); + } + } + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of emplacing, and popping a mpsc_ptr_queue */ +TEST_CASE("TestPointerQueueEmplacePop") { + size_t ops = (1 << 20); + auto queue_ptr = hipc::make_uptr>(ops); + auto queue = queue_ptr.get(); + hipc::Pointer p; + + hshm::Timer t; + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + queue->emplace(p); + queue->pop(p); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of empacling + popping vec */ +TEST_CASE("TestPointerQueueVecEmplacePop") { + auto queues_ptr = hipc::make_uptr>>(16); + auto queues = queues_ptr.get(); + hipc::Pointer p; + + hshm::Timer t; + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + t.Resume(); + auto &queue = (*queues)[0]; + queue.emplace(p); + queue.pop(p); + t.Pause(); + } + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of getting, emplacing, and popping a queue */ +TEST_CASE("TestHshmQueueEmplacePop") { + hrun::QueueId qid(0, 3); + u32 ops = (1 << 20); + std::vector queue_info = { + {16, 16, ops, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + hrun::LaneData entry; + auto task = HRUN_CLIENT->NewTaskRoot(); + entry.p_ = task.shm_; + + hshm::Timer t; + t.Resume(); + hrun::Lane &lane = queue->GetLane(0, 0); + for (size_t i = 0; i < ops; ++i) { + queue->Emplace(0, 0, entry); + lane.pop(); + } + t.Pause(); + + HRUN_CLIENT->DelTask(task); + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of getting a lane from a queue */ +TEST_CASE("TestHshmQueueGetLane") { + hrun::QueueId qid(0, 3); + std::vector queue_info = { + {16, 16, 256, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + hrun::LaneGroup group = queue->GetGroup(0); + + hshm::Timer t; + size_t ops = (1 << 20); + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + queue->GetLane(0, i % group.num_lanes_); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of getting, emplacing, and popping a queue */ +TEST_CASE("TestHshmQueueAllocateEmplacePop") { + hrun::QueueId qid(0, 3); + std::vector queue_info = { + {16, 16, 256, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + hrun::Lane &lane = queue->GetLane(0, 0); + + hshm::Timer t; + size_t ops = (1 << 20); + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + hrun::LaneData entry; + auto task = HRUN_CLIENT->NewTaskRoot(); + entry.p_ = task.shm_; + queue->Emplace(0, 0, entry); + lane.pop(); + HRUN_CLIENT->DelTask(task); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Time to spawn and join a thread */ +TEST_CASE("TestSpawnJoinThread") { + hshm::Timer t; + t.Resume(); + size_t count = 16 * (1 << 10); + for (int i = 0; i < count; ++i) { + std::thread thread([]() { }); + thread.join(); + } + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", count / t.GetUsec()); +} + +/** Time to spawn and join a thread */ +TEST_CASE("TestSpawnJoinArgoThread") { + hshm::Timer t; + t.Resume(); + ABT_xstream xstream; + ABT_thread tl_thread_; + size_t count = 16 * (1 << 10); + ABT_init(0, nullptr); + int ret = ABT_xstream_create(ABT_SCHED_NULL, &xstream); + if (ret != ABT_SUCCESS) { + HELOG(kFatal, "Could not create argobots xstream"); + } + for (int i = 0; i < count; ++i) { + int ret = ABT_thread_create_on_xstream(xstream, + [](void *args) { }, nullptr, + ABT_THREAD_ATTR_NULL, &tl_thread_); + if (ret != ABT_SUCCESS) { + HELOG(kFatal, "Couldn't spawn worker"); + } + ABT_thread_join(tl_thread_); + } + t.Pause(); + HILOG(kInfo, "Latency: {} MOps", count / t.GetUsec()); +} + +void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) { + HRUN_RUNTIME->Create(); + + hrun::Worker worker(0); + std::vector> queues; + for (u32 i = 0; i < num_queues; ++i) { + hrun::QueueId qid(0, i + 1); + std::vector queue_info = { + {num_lanes, num_lanes, 256, 0} + }; + auto queue = hipc::make_uptr( + qid, queue_info); + queues.emplace_back(std::move(queue)); + for (u32 j = 0; j < num_lanes; ++j) { + worker.PollQueues({{0, j, queue.get()}}); + } + } + + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetLocal(), "small_message");\ + client.CreateRoot(hrun::DomainId::GetLocal(), "ipc_test"); + + hshm::Timer t; + t.Resume(); + // size_t ops = (1 << 20); + size_t ops = 256; + for (size_t i = 0; i < ops; ++i) { + hipc::LPointer task; + hrun::TaskNode task_node(hrun::TaskId((u32)0, (u64)i)); + task = client.AsyncMdPushEmplace(queues[num_queues - 1].get(), + task_node, + hrun::DomainId::GetLocal()); + worker.Run(); + HRUN_CLIENT->DelTask(task); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Time for worker to process a request */ +TEST_CASE("TestWorkerLatency") { + TRANSPARENT_HRUN(); + TestWorkerIterationLatency(1, 16); + TestWorkerIterationLatency(5, 16); + TestWorkerIterationLatency(10, 16); + TestWorkerIterationLatency(15, 16); +} + +/** Time to process a request */ +TEST_CASE("TestRoundTripLatency") { + TRANSPARENT_HRUN(); + HERMES->ClientInit(); + hrun::small_message::Client client; + HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetLocal(), "small_message"); +// int count = 25; +// for (int i = 0; i < count; ++i) { +// hrun::small_message::Client client2; +// client2.CreateRoot(hrun::DomainId::GetLocal(), "ipc_test" + std::to_string(i)); +// } + client.CreateRoot(hrun::DomainId::GetLocal(), "ipc_test"); + hshm::Timer t; + + int pid = getpid(); + ProcessAffiner::SetCpuAffinity(pid, 8); + + t.Resume(); + size_t ops = (1 << 20); + // size_t ops = 1024; + for (size_t i = 0; i < ops; ++i) { + client.MdPushRoot(hrun::DomainId::GetLocal()); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +TEST_CASE("TestTimespecLatency") { + size_t ops = (1 << 20); + hshm::Timer t; + + t.Resume(); + for (size_t i = 0; i < ops; ++i) { + struct timespec ts; + timespec_get(&ts, TIME_UTC); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +TEST_CASE("TestTimerLatency") { + size_t ops = (1 << 20); + hshm::Timer t, tmp; + + t.Resume(); + double usec; + for (size_t i = 0; i < ops; ++i) { + usec = tmp.GetUsecFromStart(); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps (usec={})", ops / t.GetUsec(), usec); +} + +TEST_CASE("TestTimepointLatency") { + size_t ops = (1 << 20); + hshm::Timer t; + hshm::Timepoint tmp; + + t.Resume(); + double usec; + for (size_t i = 0; i < ops; ++i) { + usec = tmp.GetUsecFromStart(); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps (usec={})", ops / t.GetUsec(), usec); +} + +/** Time to process a request */ +//TEST_CASE("TestHermesGetBlobIdLatency") { +// HERMES->ClientInit(); +// hshm::Timer t; +// +// int pid = getpid(); +// ProcessAffiner::SetCpuAffinity(pid, 8); +// hermes::Bucket bkt = HERMES_FILESYSTEM_API->Open("/home/lukemartinlogan/hi.txt"); +// +// t.Resume(); +// size_t ops = (1 << 20); +// hermes::Context ctx; +// std::string data(ctx.page_size_, 0); +// for (size_t i = 0; i < ops; ++i) { +// bkt.GetBlobId(std::to_string(i)); +// } +// t.Pause(); +// +// HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +//} +// +///** Time to process a request */ +//TEST_CASE("TestHermesFsWriteLatency") { +// HERMES->ClientInit(); +// hshm::Timer t; +// +// int pid = getpid(); +// ProcessAffiner::SetCpuAffinity(pid, 8); +// hermes::Bucket bkt = HERMES_FILESYSTEM_API->Open("/home/lukemartinlogan/hi.txt"); +// +// t.Resume(); +// size_t ops = 150; +// hermes::Context ctx; +// ctx.page_size_ = 4096; +// std::string data(ctx.page_size_, 0); +// for (size_t i = 0; i < ops; ++i) { +// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, rctx); +// } +// t.Pause(); +// +// HILOG(kInfo, "Latency: {} MBps", ops * 4096 / t.GetUsec()); +//} +// +///** Time to process a request */ +//TEST_CASE("TestHermesFsReadLatency") { +// HERMES->ClientInit(); +// hshm::Timer t; +// +// int pid = getpid(); +// ProcessAffiner::SetCpuAffinity(pid, 8); +// hermes::Bucket bkt = HERMES_FILESYSTEM_API->Open("/home/lukemartinlogan/hi.txt"); +// +// size_t ops = 1024; +// hermes::Context ctx; +// ctx.page_size_ = 4096; +// std::string data(ctx.page_size_, 0); +// for (size_t i = 0; i < ops; ++i) { +// HERMES_FILESYSTEM_API->Write(bkt, data.data(), i * ctx.page_size_, data.size(), false, rctx); +// } +// +// t.Resume(); +// for (size_t i = 0; i < ops; ++i) { +// HERMES_FILESYSTEM_API->Read(bkt, data.data(), i * ctx.page_size_, data.size(), false, rctx); +// } +// t.Pause(); +// +// HILOG(kInfo, "Latency: {} MBps", ops * 4096 / t.GetUsec()); +//} \ No newline at end of file diff --git a/benchmark/test_zmq.cc b/benchmark/test_zmq.cc new file mode 100644 index 000000000..1de149bcf --- /dev/null +++ b/benchmark/test_zmq.cc @@ -0,0 +1,72 @@ +// +// Created by lukemartinlogan on 8/25/23. +// + +#include +#include "basic_test.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" +#include "small_message/small_message.h" +#include "hermes_shm/util/timer.h" +#include "hrun/work_orchestrator/affinity.h" + +/** ZeroMQ allocate + free request */ +TEST_CASE("TestZeromqAllocateFree") { + zmq::context_t context(1); + + // Create a PUSH socket to push requests + zmq::socket_t pushSocket(context, ZMQ_PUSH); + + // Bind the PUSH socket to a specific address + pushSocket.bind("ipc:///tmp/shared_memory"); + + // Create a PULL socket to receive requests + zmq::socket_t pullSocket(context, ZMQ_PULL); + + // Connect the PULL socket to the same address + pullSocket.connect("ipc:///tmp/shared_memory"); + + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + zmq::message_t message(sizeof(hrun::Task)); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} + +/** Single-thread performance of zeromq */ +TEST_CASE("TestZeromqAllocateEmplacePop") { + zmq::context_t context(1); + + // Create a PUSH socket to push requests + zmq::socket_t pushSocket(context, ZMQ_PUSH); + + // Bind the PUSH socket to a specific address + pushSocket.bind("ipc:///tmp/shared_memory"); + + // Create a PULL socket to receive requests + zmq::socket_t pullSocket(context, ZMQ_PULL); + + // Connect the PULL socket to the same address + pullSocket.connect("ipc:///tmp/shared_memory"); + + hshm::Timer t; + t.Resume(); + size_t ops = (1 << 20); + for (size_t i = 0; i < ops; ++i) { + // Send a request + zmq::message_t message(sizeof(hrun::Task)); + pushSocket.send(message, zmq::send_flags::none); + + // Receive the request + zmq::message_t receivedMessage; + zmq::recv_result_t result = pullSocket.recv(receivedMessage); + REQUIRE(receivedMessage.size() == sizeof(hrun::Task)); + } + t.Pause(); + + HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); +} \ No newline at end of file diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt deleted file mode 100644 index faae961b1..000000000 --- a/benchmarks/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -include_directories( - ${CMAKE_SOURCE_DIR} - ${PROJECT_SOURCE_DIR}/src/api - ${PROJECT_SOURCE_DIR}/test - ${PROJECT_SOURCE_DIR}/io_client -) - -set(BENCHMARKS - api_bench - memcpy_bench - reorganize) - -foreach(benchmark ${BENCHMARKS}) - message("Building ${benchmark}") - add_executable(${benchmark} ${benchmark}.cc) - add_dependencies(${benchmark} hermes hermes_prefetcher_trait) - target_link_libraries(${benchmark} hermes hermes_prefetcher_trait - MPI::MPI_CXX - $<$:thallium>) -endforeach() diff --git a/benchmarks/api_bench.cc b/benchmarks/api_bench.cc deleted file mode 100644 index 3cdd40c91..000000000 --- a/benchmarks/api_bench.cc +++ /dev/null @@ -1,301 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include -#include -#include "hermes.h" -#include "bucket.h" - -namespace hapi = hermes::api; -using Timer = hshm::HighResMonotonicTimer; - -/** Gather times per-process */ -void GatherTimes(std::string test_name, size_t io_size, Timer &t) { - MPI_Barrier(MPI_COMM_WORLD); - int rank, nprocs; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - double time = t.GetSec(), max; - MPI_Reduce(&time, &max, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - if (rank == 0) { - double mbps = io_size / (max * 1000000); - HIPRINT("{}: Time: {} sec, MBps (or MOps): {}, Count: {}, Nprocs: {}\n", - test_name, max, mbps, io_size, nprocs); - } -} - -/** Each process PUTS into the same bucket, but with different blob names */ -void PutTest(hapi::Hermes *hermes, - int nprocs, int rank, - int repeat, size_t blobs_per_rank, size_t blob_size) { - Timer t; - auto bkt = hermes->GetBucket("hello"); - hermes::api::Context ctx; - hermes::BlobId blob_id; - hermes::Blob blob(blob_size); - t.Resume(); - for (int j = 0; j < repeat; ++j) { - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.Put(name, blob, blob_id, ctx); - } - } - t.Pause(); - GatherTimes("Put", nprocs * blobs_per_rank * blob_size * repeat, t); -} - -/** - * Each process GETS from the same bucket, but with different blob names - * MUST run PutTest first. - * */ -void GetTest(hapi::Hermes *hermes, - int nprocs, int rank, - int repeat, size_t blobs_per_rank, size_t blob_size) { - Timer t; - auto bkt = hermes->GetBucket("hello"); - hermes::api::Context ctx; - hermes::BlobId blob_id; - t.Resume(); - for (int j = 0; j < repeat; ++j) { - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - hermes::Blob ret; - bkt.GetBlobId(name, blob_id); - bkt.Get(blob_id, ret, ctx); - } - } - t.Pause(); - GatherTimes("Get", nprocs * blobs_per_rank * blob_size * repeat, t); -} - -/** Each process PUTs then GETs */ -void PutGetTest(hapi::Hermes *hermes, - int nprocs, int rank, int repeat, - size_t blobs_per_rank, size_t blob_size) { - PutTest(hermes, nprocs, rank, repeat, blobs_per_rank, blob_size); - MPI_Barrier(MPI_COMM_WORLD); - GetTest(hermes, nprocs, rank, repeat, blobs_per_rank, blob_size); -} - -/** Each process creates a set of buckets */ -void CreateBucketTest(hapi::Hermes *hermes, int nprocs, int rank, - size_t bkts_per_rank) { - Timer t; - t.Resume(); - for (size_t i = 0; i < bkts_per_rank; ++i) { - int bkt_name = rank * bkts_per_rank + i; - hapi::Bucket bkt = hermes->GetBucket(std::to_string(bkt_name)); - } - t.Pause(); - GatherTimes("CreateBucket", bkts_per_rank * nprocs, t); -} - -/** Each process gets existing buckets */ -void GetBucketTest(hapi::Hermes *hermes, int nprocs, int rank, - size_t bkts_per_rank) { - // Initially create the buckets - for (size_t i = 0; i < bkts_per_rank; ++i) { - int bkt_name = rank * bkts_per_rank + i; - hapi::Bucket bkt = hermes->GetBucket(std::to_string(bkt_name)); - } - - // Get existing buckets - Timer t; - t.Resume(); - for (size_t i = 0; i < bkts_per_rank; ++i) { - int bkt_name = rank * bkts_per_rank + i; - hapi::Bucket bkt = hermes->GetBucket(std::to_string(bkt_name)); - } - t.Pause(); - GatherTimes("CreateBucket", bkts_per_rank * nprocs, t); -} - -/** Each process creates (no data) a blob in a single bucket */ -void CreateBlobOneBucketTest(hapi::Hermes *hermes, - int nprocs, int rank, - size_t blobs_per_rank) { - Timer t; - hapi::Bucket bkt = hermes->GetBucket("CreateBlobOneBucket"); - hapi::Context ctx; - hermes::BlobId blob_id; - - t.Resume(); - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.TryCreateBlob(name, blob_id, ctx); - } - t.Pause(); - GatherTimes("CreateBlobOneBucket", blobs_per_rank * nprocs, t); -} - -/** Each process creates (no data) a blob in a process-specific bucket */ -void CreateBlobPerBucketTest(hapi::Hermes *hermes, - int nprocs, int rank, - size_t blobs_per_rank) { - Timer t; - hapi::Bucket bkt = hermes->GetBucket( - hshm::Formatter::format("CreateBlobPerBucket{}", rank)); - hapi::Context ctx; - hermes::BlobId blob_id; - - t.Resume(); - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.TryCreateBlob(name, blob_id, ctx); - } - t.Pause(); - GatherTimes("CreateBlobPerBucket", nprocs * blobs_per_rank, t); -} - -/** Each process deletes a number of buckets */ -void DeleteBucketTest(hapi::Hermes *hermes, - int nprocs, int rank, - size_t bkt_per_rank, - size_t blobs_per_bucket) { - Timer t; - hapi::Context ctx; - hermes::BlobId blob_id; - - // Create the buckets - for (size_t i = 0; i < bkt_per_rank; ++i) { - hapi::Bucket bkt = hermes->GetBucket( - hshm::Formatter::format("DeleteBucket{}", rank)); - for (size_t j = 0; j < blobs_per_bucket; ++j) { - std::string name = std::to_string(j); - bkt.TryCreateBlob(name, blob_id, ctx); - } - } - - // Delete the buckets - t.Resume(); - for (size_t i = 0; i < bkt_per_rank; ++i) { - hapi::Bucket bkt = hermes->GetBucket( - hshm::Formatter::format("DeleteBucket{}", rank)); - size_t blob_name_int = rank * bkt_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.TryCreateBlob(name, blob_id, ctx); - } - t.Pause(); - GatherTimes("DeleteBucket", nprocs * bkt_per_rank * blobs_per_bucket, t); -} - -/** Each process deletes blobs from a single bucket */ -void DeleteBlobOneBucket(hapi::Hermes *hermes, - int nprocs, int rank, - size_t blobs_per_rank) { - Timer t; - CreateBlobOneBucketTest(hermes, nprocs, rank, blobs_per_rank); - MPI_Barrier(MPI_COMM_WORLD); - hapi::Bucket bkt = hermes->GetBucket("CreateBlobOneBucket"); - hermes::BlobId blob_id; - hapi::Context ctx; - - // Delete blobs from a bucket - t.Resume(); - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_name_int = rank * blobs_per_rank + i; - std::string name = std::to_string(blob_name_int); - bkt.GetBlobId(name, blob_id); - bkt.DestroyBlob(blob_id, ctx); - } - t.Pause(); - GatherTimes("CreateBlobOneBucket", nprocs * blobs_per_rank, t); -} - - -#define REQUIRE_ARGC_GE(N) \ - if (argc < (N)) { \ - HIPRINT("Requires fewer than {} params\n", N); \ - help(); \ - } - -#define REQUIRE_ARGC(N) \ - if (argc != (N)) { \ - HIPRINT("Requires exactly {} params\n", N); \ - help(); \ - } - -void help() { - printf("USAGE: ./api_bench [mode] ...\n"); - printf("USAGE: ./api_bench put [blob_size (K/M/G)] [blobs_per_rank]\n"); - printf("USAGE: ./api_bench putget [blob_size (K/M/G)] [blobs_per_rank]\n"); - printf("USAGE: ./api_bench create_bkt [bkts_per_rank]\n"); - printf("USAGE: ./api_bench get_bkt [bkts_per_rank]\n"); - printf("USAGE: ./api_bench create_blob_1bkt [blobs_per_rank]\n"); - printf("USAGE: ./api_bench create_blob_Nbkt [blobs_per_rank]\n"); - printf("USAGE: ./api_bench del_bkt [bkt_per_rank] [blobs_per_bkt]\n"); - printf("USAGE: ./api_bench del_blobs [blobs_per_rank]\n"); - exit(1); -} - -int main(int argc, char **argv) { - int rank, nprocs; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - auto hermes = hapi::Hermes::Create(hermes::HermesType::kClient); - - // Get mode - REQUIRE_ARGC_GE(2) - std::string mode = argv[1]; - - MPI_Barrier(MPI_COMM_WORLD); - - HIPRINT("Beginning {}\n", mode) - - // Run tests - if (mode == "put") { - REQUIRE_ARGC(4) - size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); - size_t blobs_per_rank = atoi(argv[3]); - PutTest(hermes, nprocs, rank, 1, blobs_per_rank, blob_size); - } else if (mode == "putget") { - REQUIRE_ARGC(4) - size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); - size_t blobs_per_rank = atoi(argv[3]); - PutGetTest(hermes, nprocs, rank, 1, blobs_per_rank, blob_size); - } else if (mode == "create_bkt") { - REQUIRE_ARGC(3) - size_t bkts_per_rank = atoi(argv[2]); - CreateBucketTest(hermes, nprocs, rank, bkts_per_rank); - } else if (mode == "get_bkt") { - REQUIRE_ARGC(3) - size_t bkts_per_rank = atoi(argv[2]); - GetBucketTest(hermes, nprocs, rank, bkts_per_rank); - } else if (mode == "create_blob_1bkt") { - REQUIRE_ARGC(3) - size_t blobs_per_rank = atoi(argv[2]); - CreateBlobOneBucketTest(hermes, nprocs, rank, blobs_per_rank); - } else if (mode == "create_blob_Nbkt") { - REQUIRE_ARGC(3) - size_t blobs_per_rank = atoi(argv[2]); - CreateBlobPerBucketTest(hermes, nprocs, rank, blobs_per_rank); - } else if (mode == "del_bkt") { - REQUIRE_ARGC(4) - size_t bkt_per_rank = atoi(argv[2]); - size_t blobs_per_bkt = atoi(argv[3]); - DeleteBucketTest(hermes, nprocs, rank, bkt_per_rank, blobs_per_bkt); - } else if (mode == "del_blobs") { - REQUIRE_ARGC(4) - size_t blobs_per_rank = atoi(argv[2]); - DeleteBlobOneBucket(hermes, nprocs, rank, blobs_per_rank); - } - - hermes->Finalize(); - MPI_Finalize(); -} diff --git a/benchmarks/memcpy_bench.cc b/benchmarks/memcpy_bench.cc deleted file mode 100644 index 267439cb6..000000000 --- a/benchmarks/memcpy_bench.cc +++ /dev/null @@ -1,154 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// #define HERMES_ENABLE_PROFILING 1 - -#include "mpi.h" -#include -#include "hermes.h" -#include "hermes_shm/memory/backend/posix_shm_mmap.h" -#include - -using hshm::ipc::PosixShmMmap; -using Timer = hshm::HighResMonotonicTimer; -hipc::MemoryBackend *backend; - -void GatherTimes(const std::string &backend_type, - const std::string &test_name, - size_t size_per_rank, Timer &t) { - MPI_Barrier(MPI_COMM_WORLD); - int nprocs, rank; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - double time = t.GetSec(), max; - MPI_Reduce(&time, &max, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - if (rank == 0) { - HIPRINT("{} {} {}: MBps: {}\n", - nprocs, backend_type, test_name, - size_per_rank * nprocs / t.GetUsec()) - } -} - -template -size_t GetBlobOff(int rank, size_t i, - size_t blobs_per_rank, size_t blob_size) { - if constexpr(SHM) { - return (rank * blobs_per_rank + i) * blob_size; - } else { - return i * blob_size; - } -} - -template -void PutTest(const std::string &backend_type, int rank, int repeat, - size_t blobs_per_rank, size_t blob_size) { - Timer t; - std::vector data(blob_size); - t.Resume(); - for (int j = 0; j < repeat; ++j) { - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_off = GetBlobOff(rank, i, blobs_per_rank, blob_size); - memcpy(backend->data_ + blob_off, data.data(), data.size()); - } - } - t.Pause(); - GatherTimes(backend_type, "Put", blobs_per_rank * blob_size * repeat, t); -} - -template -void GetTest(const std::string &backend_type, int rank, int repeat, - size_t blobs_per_rank, size_t blob_size) { - Timer t; - std::vector data(blob_size); - t.Resume(); - for (int j = 0; j < repeat; ++j) { - for (size_t i = 0; i < blobs_per_rank; ++i) { - size_t blob_off = GetBlobOff(rank, i, blobs_per_rank, blob_size); - memcpy(data.data(), backend->data_ + blob_off, data.size()); - } - } - t.Pause(); - GatherTimes(backend_type, "Get", blobs_per_rank * blob_size * repeat, t); -} - -template -void MemcpyBench(int nprocs, int rank, - size_t blob_size, - size_t blobs_per_rank) { - std::string shm_url = "test_mem_backend"; - std::string backend_type; - size_t backend_size; - hipc::MemoryBackendType type = hipc::MemoryBackendType::kPosixShmMmap; - - if constexpr(std::is_same_v) { - backend_type = "kPosixShmMmap"; - type = hipc::MemoryBackendType::kPosixShmMmap; - backend_size = nprocs * blob_size * blobs_per_rank; - } else if constexpr(std::is_same_v) { - backend_type = "kPosixMmap"; - type = hipc::MemoryBackendType::kPosixMmap; - backend_size = blob_size * blobs_per_rank; - } else { - HELOG(kFatal, "Invalid backend type"); - } - - // Create the backend - try { - if constexpr(std::is_same_v) { - MPI_Barrier(MPI_COMM_WORLD); - if (rank == 0) { - backend = HERMES_MEMORY_MANAGER-> - CreateBackend(backend_size, shm_url); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank != 0) { - backend = HERMES_MEMORY_MANAGER->AttachBackend(type, shm_url); - } - MPI_Barrier(MPI_COMM_WORLD); - } else if constexpr(std::is_same_v) { - MPI_Barrier(MPI_COMM_WORLD); - backend = HERMES_MEMORY_MANAGER-> - CreateBackend(backend_size, shm_url); - MPI_Barrier(MPI_COMM_WORLD); - } - } catch (std::exception &e) { - HELOG(kFatal, "{}\n", e.what()); - } - - if (backend == nullptr) { - HELOG(kFatal, "Backend was null: {}", backend_type) - } - - PutTest(backend_type, rank, 1, blobs_per_rank, blob_size); - MPI_Barrier(MPI_COMM_WORLD); - GetTest(backend_type, rank, 1, blobs_per_rank, blob_size); -} - -int main(int argc, char **argv) { - int rank, nprocs; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc != 3) { - printf("USAGE: ./memcpy_bench [blob_size (K/M/G)] [blobs_per_rank]\n"); - exit(1); - } - size_t blob_size = hshm::ConfigParse::ParseSize(argv[1]); - size_t blobs_per_rank = atoi(argv[2]); - - MemcpyBench(nprocs, rank, blob_size, blobs_per_rank); - MemcpyBench(nprocs, rank, - blob_size, blobs_per_rank); - - MPI_Finalize(); -} diff --git a/benchmarks/reorganize.cc b/benchmarks/reorganize.cc deleted file mode 100644 index 785f0fbe3..000000000 --- a/benchmarks/reorganize.cc +++ /dev/null @@ -1,150 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// #define HERMES_ENABLE_PROFILING 1 - -#include "mpi.h" -#include -#include "hermes.h" -#include "hermes_shm/memory/backend/posix_shm_mmap.h" -#include -#include "traits/prefetcher/prefetcher_trait.h" - -using hshm::ipc::PosixShmMmap; -using Timer = hshm::HighResMonotonicTimer; -hipc::MemoryBackend *backend; -const char *kBucketName = "/tmp/test_hermes/hi.txt"; - -void GatherTimes(const std::string &test_name, - size_t total_size, - Timer &t, Timer &io_t) { - MPI_Barrier(MPI_COMM_WORLD); - int nprocs, rank; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - double time = t.GetUsec(), max_runtime; - MPI_Reduce(&time, &max_runtime, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - double io_time = t.GetUsec(), max_io_time; - MPI_Reduce(&io_time, &max_io_time, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - if (rank == 0) { - HIPRINT("{} {}: MBps: {}, Time: {}\n", - nprocs, test_name, - total_size / max_io_time, - max_runtime) - } -} - -/** Each process PUTS into the same bucket, but with different blob names */ -void PutTest(int nprocs, int rank, - size_t blobs_per_checkpt, - size_t num_checkpts, - size_t blob_size, - int compute_sec) { - Timer t, io_t; - auto bkt = HERMES->GetBucket(kBucketName); - hermes::api::Context ctx; - hermes::BlobId blob_id; - hermes::Blob blob(blob_size); - t.Resume(); - - size_t blobs_per_rank = blobs_per_checkpt * num_checkpts; - size_t cur_blob = 0; - for (size_t i = 0; i < num_checkpts; ++i) { - io_t.Resume(); - for (size_t j = 0; j < blobs_per_checkpt; ++j) { - size_t blob_name_int = rank * blobs_per_rank + cur_blob; - std::string name = std::to_string(blob_name_int); - bkt.Put(name, blob, blob_id, ctx); - ++cur_blob; - } - io_t.Pause(); - sleep(compute_sec); - } - t.Pause(); - GatherTimes("Put", nprocs * blobs_per_rank * blob_size, t, io_t); -} - -/** - * Each process GETS from the same bucket, but with different blob names - * MUST run PutTest first. - * */ -void GetTest(int nprocs, int rank, - size_t blobs_per_checkpt, - size_t num_checkpts, - size_t blob_size, - int compute_sec) { - Timer t, io_t; - auto bkt = HERMES->GetBucket(kBucketName); - hermes::api::Context ctx; - hermes::BlobId blob_id; - t.Resume(); - size_t blobs_per_rank = blobs_per_checkpt * num_checkpts; - size_t cur_blob = 0; - for (size_t i = 0; i < num_checkpts; ++i) { - io_t.Resume(); - for (size_t j = 0; j < blobs_per_checkpt; ++j) { - size_t blob_name_int = rank * blobs_per_rank + cur_blob; - std::string name = std::to_string(blob_name_int); - hermes::Blob ret; - bkt.GetBlobId(name, blob_id); - bkt.Get(blob_id, ret, ctx); - ++cur_blob; - } - io_t.Pause(); - sleep(compute_sec); - } - t.Pause(); - GatherTimes("Get", nprocs * blobs_per_rank * blob_size, t, io_t); -} - -int main(int argc, char **argv) { - int rank, nprocs; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc != 7) { - printf("USAGE: ./reorganize [prefetch] [blob_size (K/M/G)] " - "[blobs_per_checkpt] [num_checkpts] " - "[compute_put (sec)] [compute_get (sec)]\n"); - exit(1); - } - int with_prefetch = atoi(argv[1]); - size_t blob_size = hshm::ConfigParse::ParseSize(argv[2]); - size_t blobs_per_checkpt = atoi(argv[3]); - size_t num_checkpts = atoi(argv[4]); - int compute_put = atoi(argv[5]); - int compute_get = atoi(argv[6]); - - // Start Hermes - HERMES->Create(hermes::HermesType::kClient); - - // Register the Apriori trait - hermes::TraitId apriori_trait = - HERMES->RegisterTrait( - "apriori", hermes::PrefetcherType::kApriori); - if (with_prefetch) { - auto bkt = HERMES->GetBucket(kBucketName); - bkt.AttachTrait(apriori_trait); - } - - PutTest(nprocs, rank, blobs_per_checkpt, num_checkpts, - blob_size, compute_put); - MPI_Barrier(MPI_COMM_WORLD); - GetTest(nprocs, rank, blobs_per_checkpt, num_checkpts, - blob_size, compute_get); - - // Finalize - HERMES->Finalize(); - MPI_Finalize(); -} diff --git a/ci/jarvis-util/jarvis_util/introspect/__init__.py b/ci/__init__.py similarity index 100% rename from ci/jarvis-util/jarvis_util/introspect/__init__.py rename to ci/__init__.py diff --git a/ci/build_hermes.sh b/ci/build_hermes.sh index ae1a9b563..a13d3c167 100755 --- a/ci/build_hermes.sh +++ b/ci/build_hermes.sh @@ -1,47 +1,43 @@ #!/bin/bash +# CD into git workspace +cd ${GITHUB_WORKSPACE} + set -x set -e set -o pipefail +# Set spack env INSTALL_DIR="${HOME}" -INSTALL_PREFIX="${HOME}/install" -mkdir -p "${INSTALL_PREFIX}" - -# This will build our small python library for running unit tests -pushd ci/jarvis-util -python3 -m pip install -r requirements.txt -python3 -m pip install -e . -popd - -mkdir build -pushd build - -# Load hermes dependencies via spack -# Copy from install_deps.sh SPACK_DIR=${INSTALL_DIR}/spack -set +x . ${SPACK_DIR}/share/spack/setup-env.sh -set -x -spack load --only dependencies hermes -# Build hermes -# export CXXFLAGS="${CXXFLAGS} -std=c++17 -Werror -Wall -Wextra" -cmake \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DBUILD_SHARED_LIBS=ON \ - -DHERMES_ENABLE_COVERAGE=ON \ - -DHERMES_BUILD_BENCHMARKS=OFF \ - -DHERMES_BUILD_BUFFER_POOL_VISUALIZER=OFF \ - -DHERMES_USE_ADDRESS_SANITIZER=OFF \ - -DHERMES_USE_THREAD_SANITIZER=OFF \ - -DHERMES_RPC_THALLIUM=ON \ - -DBUILD_TESTING=ON \ - -DHERMES_ENABLE_VFD=ON \ - -DHERMES_DEBUG_LOCK=OFF \ - .. +mkdir -p "${HOME}/install" +mkdir build +cd build +spack load hermes_shm +cmake ../ \ +-DCMAKE_BUILD_TYPE=Debug \ +-DCMAKE_INSTALL_PREFIX="${HOME}/install" make -j8 make install -popd +export CXXFLAGS=-Wall +ctest -VV + +# Set proper flags for cmake to find Hermes +INSTALL_PREFIX="${HOME}/install" +export LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LIBRARY_PATH}" +export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}" +export LDFLAGS="-L${INSTALL_PREFIX}/lib:${LDFLAGS}" +export CFLAGS="-I${INSTALL_PREFIX}/include:${CFLAGS}" +export CPATH="${INSTALL_PREFIX}/include:${CPATH}" +export CMAKE_PREFIX_PATH="${INSTALL_PREFIX}:${CMAKE_PREFIX_PATH}" +export CXXFLAGS="-I${INSTALL_PREFIX}/include:${CXXFLAGS}" + +# Run make install unit test +cd test/unit/external +mkdir build +cd build +cmake ../ +make -j8 diff --git a/ci/cluster/Dockerfile b/ci/cluster/Dockerfile deleted file mode 100644 index d3600723a..000000000 --- a/ci/cluster/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM ubuntu:20.04 - -ENV USER mpirun - -ENV DEBIAN_FRONTEND=noninteractive \ - HOME=/home/${USER} - -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends \ - sudo \ - apt-utils \ - && apt-get install -y --no-install-recommends \ - openssh-server \ - gcc \ - g++ \ - libfabric-dev \ - mpich \ - binutils \ - libjson-c-dev \ - libunwind-dev \ - && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -RUN mkdir /var/run/sshd -RUN echo 'root:${USER}' | chpasswd -RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config - -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -ENV NOTVISIBLE "in users profile" -RUN echo "export VISIBLE=now" >> /etc/profile - -# ------------------------------------------------------------ -# Add an 'mpirun' user -# ------------------------------------------------------------ - -ARG USER_ID -ARG GROUP_ID - -RUN addgroup --gid ${GROUP_ID} ${USER} -RUN adduser --disabled-password --gecos "" --uid ${USER_ID} --gid ${GROUP_ID} ${USER} && \ - echo "${USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - -# ------------------------------------------------------------ -# Set-Up SSH with our Github deploy key -# ------------------------------------------------------------ - -ENV SSHDIR ${HOME}/.ssh/ - -RUN mkdir -p ${SSHDIR} -RUN echo "StrictHostKeyChecking no" > ${SSHDIR}/config - -RUN chmod -R 600 ${SSHDIR}* && \ - chown -R ${USER}:${USER} ${SSHDIR} - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/ci/cluster/cluster_down.sh b/ci/cluster/cluster_down.sh deleted file mode 100755 index 71377a9bf..000000000 --- a/ci/cluster/cluster_down.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -. cluster_utils.sh -hermes_cluster_down diff --git a/ci/cluster/cluster_test.sh b/ci/cluster/cluster_test.sh deleted file mode 100755 index 6ec34caaf..000000000 --- a/ci/cluster/cluster_test.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -. cluster_utils.sh -hermes_cluster_test ${1:-} diff --git a/ci/cluster/cluster_up.sh b/ci/cluster/cluster_up.sh deleted file mode 100755 index ce7ef43bf..000000000 --- a/ci/cluster/cluster_up.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -. cluster_utils.sh -hermes_cluster_up diff --git a/ci/cluster/cluster_utils.sh b/ci/cluster/cluster_utils.sh deleted file mode 100644 index b1a388034..000000000 --- a/ci/cluster/cluster_utils.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -x -e - -node_names=($(awk '/hostname:/ {print $2}' docker-compose.yml)) -docker_user=mpirun -docker_home=/home/${docker_user} -script_dir="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)" -hermes_dir=${script_dir}/../.. -cluster_conf=${hermes_dir}/test/data/hermes_server_ci.yaml -hermes_build_dir=${hermes_dir}/build -project_name="$(basename ${script_dir})" -host1=${project_name}_${node_names[0]}_1 -host2=${project_name}_${node_names[1]}_1 - - -# Build images and start a cluster -function hermes_cluster_up() { - local num_workers=${1:-1} - - # Build the images, passing our user id and group id so the container user can - # modify the .gcda coverage files - for n in "${node_names[@]}"; do - docker-compose build --build-arg GROUP_ID=$(id -g) --build-arg USER_ID=$(id -u) ${n} - done - - docker-compose up -d --scale ${node_names[0]}=1 --scale ${node_names[1]}=${num_workers} --no-recreate - - for h in ${host1} ${host2}; do - # Create the hosts file - docker exec --user ${docker_user} -w ${hermes_build_dir} ${h} \ - bash -c "echo -e \"${host1}\n${host2}\n\" > hermes_hosts" - # Copy ssh keys to ${docker_home}/.ssh - docker exec ${h} bash -c "cp ${HOME}/.ssh/id_rsa ${docker_home}/.ssh/id_rsa" - docker exec ${h} bash -c "cp ${HOME}/.ssh/id_rsa.pub ${docker_home}/.ssh/id_rsa.pub" - docker exec ${h} bash -c "cp ${HOME}/.ssh/id_rsa.pub ${docker_home}/.ssh/authorized_keys" - docker exec ${h} chown -R ${docker_user}:${docker_user} ${docker_home}/.ssh - # Create the data directory - docker exec ${h} bash -c "mkdir /tmp/test_hermes" - done -} - -function hermes_cluster_test() { - local allocate_tty=${1:-} - local hosts=${host1},${host2} - - docker-compose exec ${allocate_tty} \ - -e GLOG_vmodule=rpc_thallium=1 \ - -e LSAN_OPTIONS=suppressions=../test/data/asan.supp \ - --user ${docker_user} \ - -w ${hermes_build_dir} \ - ${node_names[0]} \ - echo `mpirun -n 2 -ppn 1 -hosts ${hosts} bin/hermes_daemon ${cluster_conf} &` \ - && sleep 3 \ - && mpirun -n 4 -ppn 2 -hosts ${hosts} -genv ${cluster_conf} bin/test_multinode_put_get -} - -# Stop the cluster -function hermes_cluster_down() { - docker-compose down -} diff --git a/ci/cluster/docker-compose.yml b/ci/cluster/docker-compose.yml deleted file mode 100644 index 389c73c45..000000000 --- a/ci/cluster/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3" - -services: - node1: - build: . - links: - - node2 - networks: - - net - volumes: - - $HOME:$HOME - hostname: node1 - - node2: - build: . - networks: - - net - volumes: - - $HOME:$HOME - hostname: node2 - -networks: - net: - driver: bridge diff --git a/ci/cluster/multi_node_ci_test.sh b/ci/cluster/multi_node_ci_test.sh deleted file mode 100755 index 5fcfe0238..000000000 --- a/ci/cluster/multi_node_ci_test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -x -e - -if [[ "${CI}" != "true" ]]; then - echo "This script is only meant to be run within Github actions" - exit 1 -fi - -echo "TODO(llogan): Add back. disabling for now" -exit 0 - -. cluster_utils.sh - -# Create ssh keys for the cluster to use -echo -e 'y\n' | ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa - -# Start a two node cluster -hermes_cluster_up - -# Run the Hermes tests on the cluster (without allocating a tty) -hermes_cluster_test "-T" - -# Shutdown the cluster -hermes_cluster_down diff --git a/ci/coverage.sh b/ci/coverage.sh index c6b27ad2c..59e8dbd01 100644 --- a/ci/coverage.sh +++ b/ci/coverage.sh @@ -8,3 +8,15 @@ lcov --remove "${COVERAGE_DIR}/tmp.info" \ "*/spack/*" \ -o "${COVERAGE_DIR}/lcov.info" genhtml "${COVERAGE_DIR}/lcov.info" --output-directory coverage_report + +echo "Placed coverage info in ${COVERAGE_DIR}/lcov.info" +cd ${COVERAGE_DIR} +`pwd` +ls + +#lcov -c -d . -o "tmp.info" +#lcov --remove "tmp.info" \ +# "/usr/*" \ +# "*/spack/*" \ +# -o "lcov.info" +#genhtml "lcov.info" --output-directory coverage_report diff --git a/hermes_shm/scripts/docs.sh b/ci/docs.sh similarity index 100% rename from hermes_shm/scripts/docs.sh rename to ci/docs.sh diff --git a/ci/external/CMakeLists.txt b/ci/external/CMakeLists.txt deleted file mode 100644 index 826f316b6..000000000 --- a/ci/external/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -project(UNIT_TESTS) -cmake_minimum_required (VERSION 3.10) - -set(CMAKE_MESSAGE_LOG_LEVEL "VERBOSE") -find_package(Hermes CONFIG REQUIRED) -message("Found Hermes at ${Hermes_INCLUDE_DIR} ${Hermes_LIBRARIES}") -add_executable(test_hermes_install test_install.cc) -target_link_libraries(test_hermes_install ${Hermes_LIBRARIES}) \ No newline at end of file diff --git a/ci/jarvis-util/jarvis_util/serialize/__init__.py b/ci/hermes/__init__.py similarity index 100% rename from ci/jarvis-util/jarvis_util/serialize/__init__.py rename to ci/hermes/__init__.py diff --git a/ci/jarvis-util/jarvis_util/shell/__init__.py b/ci/hermes/packages/hermes/__init__.py similarity index 100% rename from ci/jarvis-util/jarvis_util/shell/__init__.py rename to ci/hermes/packages/hermes/__init__.py diff --git a/ci/hermes/packages/hermes/package.py b/ci/hermes/packages/hermes/package.py index 0a3462f95..138b1818e 100644 --- a/ci/hermes/packages/hermes/package.py +++ b/ci/hermes/packages/hermes/package.py @@ -4,44 +4,50 @@ class Hermes(CMakePackage): homepage = "http://www.cs.iit.edu/~scs/assets/projects/Hermes/Hermes.html" url = "https://github.com/HDFGroup/hermes/tarball/master" git = "https://github.com/HDFGroup/hermes.git" - version('master', branch='master') - version('pnnl', branch='pnnl') - version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', branch='dev') + + version('master', + branch='master', submodules=True) + version('pnnl', + branch='pnnl', submodules=True) + version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', + branch='dev', submodules=True) + version('dev-1.1', git='https://github.com/lukemartinlogan/hermes.git', + branch='hermes-1.1', submodules=True) + version('hdf-1.1', git='https://github.com/HDFGroup/hermes.git', + branch='hermes-1.1', submodules=True) + variant('vfd', default=False, description='Enable HDF5 VFD') variant('ares', default=False, description='Enable full libfabric install') variant('debug', default=False, description='Enable debug mode') - depends_on('mochi-thallium~cereal@0.8.3') + variant('zmq', default=False, description='Build ZeroMQ tests') + + depends_on('mochi-thallium~cereal@0.10.1') depends_on('cereal') - depends_on('catch2@3.0.1') - depends_on('mpich@3.3.2:') + # depends_on('catch2@3.0.1') + depends_on('catch2') + depends_on('mpich@3.3.2') depends_on('yaml-cpp') depends_on('boost@1.7:') - # TODO(): we need to figure out how to add a python library as a dependency - # depends_on('py-jarvis-util') - depends_on('libfabric@1.14.1 fabrics=mlx,rxd,rxm,shm,sockets,tcp,udp,verbs,xpmem', - when='+ares') - depends_on('hdf5@1.14.0:', when='+vfd') + depends_on('hermes_shm') + depends_on('libzmq', '+zmq') def cmake_args(self): - args = ['-DCMAKE_INSTALL_PREFIX={}'.format(self.prefix), - '-DHERMES_RPC_THALLIUM=ON', - '-DHERMES_INSTALL_TESTS=ON', - '-DBUILD_TESTING=ON'] + args = ['-DCMAKE_INSTALL_PREFIX={}'.format(self.prefix)] if '+debug' in self.spec: args.append('-DCMAKE_BUILD_TYPE=Debug') - if '+vfd' in self.spec: - args.append(self.define('HERMES_ENABLE_VFD', 'ON')) + else: + args.append('-DCMAKE_BUILD_TYPE=Release') return args def set_include(self, env, path): env.append_flags('CFLAGS', '-I{}'.format(path)) env.append_flags('CXXFLAGS', '-I{}'.format(path)) + env.prepend_path('INCLUDE', '{}'.format(path)) env.prepend_path('CPATH', '{}'.format(path)) - env.prepend_path('CMAKE_PREFIX_PATH', '{}'.format(path)) def set_lib(self, env, path): + env.prepend_path('LIBRARY_PATH', path) env.prepend_path('LD_LIBRARY_PATH', path) - env.prepend_path('LIBRARY_PATH', '{}'.format(path)) env.append_flags('LDFLAGS', '-L{}'.format(path)) def set_flags(self, env): @@ -49,6 +55,7 @@ def set_flags(self, env): self.set_include(env, '{}/include'.format(self.prefix)) self.set_lib(env, '{}/lib'.format(self.prefix)) self.set_lib(env, '{}/lib64'.format(self.prefix)) + env.prepend_path('CMAKE_PREFIX_PATH', '{}/cmake'.format(self.prefix)) def setup_dependent_environment(self, spack_env, run_env, dependent_spec): self.set_flags(spack_env) diff --git a/ci/jarvis-util/jarvis_util/util/__init__.py b/ci/hermes/packages/hermes_shm/__init__.py similarity index 100% rename from ci/jarvis-util/jarvis_util/util/__init__.py rename to ci/hermes/packages/hermes_shm/__init__.py diff --git a/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py b/ci/hermes/packages/hermes_shm/package.py similarity index 52% rename from hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py rename to ci/hermes/packages/hermes_shm/package.py index 36fd17ec6..0ba3d2b1c 100644 --- a/hermes_shm/scripts/hermes_shm/packages/hermes_shm/package.py +++ b/ci/hermes/packages/hermes_shm/package.py @@ -4,21 +4,40 @@ class HermesShm(CMakePackage): homepage = "https://github.com/lukemartinlogan/hermes_shm/wiki" git = "https://github.com/lukemartinlogan/hermes_shm.git" version('master', branch='master') - depends_on('mochi-thallium~cereal@0.8.3') + variant('ares', default=False, description='Enable full libfabric install') + variant('only_verbs', default=False, description='Only verbs') + + depends_on('mochi-thallium~cereal@0.10.1') depends_on('catch2@3.0.1') - depends_on('mpich@3.3.2:') + # depends_on('mpi') + depends_on('mpich@3.3.2') + depends_on('boost@1.7: +context +fiber') + depends_on('cereal') depends_on('yaml-cpp') - depends_on('boost@1.7:') depends_on('doxygen@1.9.3') + depends_on('libfabric fabrics=sockets,tcp,udp,rxm,rxd,verbs', + when='+ares') + depends_on('libfabric fabrics=verbs', + when='+only_verbs') + + variant('debug', default=False, description='Build shared libraries') def cmake_args(self): - return [] + args = [] + if '+debug' in self.spec: + args.append('-DCMAKE_BUILD_TYPE=Debug') + else: + args.append('-DCMAKE_BUILD_TYPE=Release') + return args def set_include(self, env, path): env.append_flags('CFLAGS', '-I{}'.format(path)) env.append_flags('CXXFLAGS', '-I{}'.format(path)) + env.prepend_path('INCLUDE', '{}'.format(path)) + env.prepend_path('CPATH', '{}'.format(path)) def set_lib(self, env, path): + env.prepend_path('LIBRARY_PATH', path) env.prepend_path('LD_LIBRARY_PATH', path) env.append_flags('LDFLAGS', '-L{}'.format(path)) @@ -27,6 +46,7 @@ def set_flags(self, env): self.set_include(env, '{}/include'.format(self.prefix)) self.set_lib(env, '{}/lib'.format(self.prefix)) self.set_lib(env, '{}/lib64'.format(self.prefix)) + env.prepend_path('CMAKE_PREFIX_PATH', '{}/cmake'.format(self.prefix)) def setup_dependent_environment(self, spack_env, run_env, dependent_spec): self.set_flags(spack_env) diff --git a/ci/hermes/repo.yaml b/ci/hermes/repo.yaml index fdd6f5fb9..039fd3384 100644 --- a/ci/hermes/repo.yaml +++ b/ci/hermes/repo.yaml @@ -1,2 +1,2 @@ repo: - namespace: hermes + namespace: 'hermes' diff --git a/ci/install_deps.sh b/ci/install_deps.sh index 45e6be4c4..03958f316 100755 --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -1,25 +1,30 @@ #!/bin/bash +# CD into git workspace +cd ${GITHUB_WORKSPACE} + +# This script will build and install them via Spack from source +# because Hermes requires a very specific version and configuration options +# for each package. + set -x set -e set -o pipefail -sudo apt-get install -y pkg-config - -# Where to install data +# Change this especially when your $HOME doesn't have enough disk space. INSTALL_DIR="${HOME}" -INSTALL_PREFIX="${HOME}/install" SPACK_DIR=${INSTALL_DIR}/spack SPACK_VERSION=0.18.1 -# Install spack echo "Installing dependencies at ${INSTALL_DIR}" mkdir -p ${INSTALL_DIR} + +# Load Spack git clone https://github.com/spack/spack ${SPACK_DIR} -pushd ${SPACK_DIR} +cd ${SPACK_DIR} git checkout v${SPACK_VERSION} -popd +# Set spack env set +x . ${SPACK_DIR}/share/spack/setup-env.sh set -x @@ -27,17 +32,10 @@ set -x # This will allow Spack to skip building some packages that are directly # available from the system. For example, autoconf, cmake, m4, etc. # Modify ci/pckages.yaml to skip building compilers or build tools via Spack. +cd ${GITHUB_WORKSPACE} cp ci/packages.yaml ${SPACK_DIR}/etc/spack/packages.yaml -# This will override Spack's default package repository to allow building -# a custom package when the same package is available from Spack. -spack repo add ./ci/hermes - -# This will build our small python library for running unit tests -cd ci/jarvis-util -python3 -m pip install -r requirements.txt -python3 -m pip install -e . - -# NOTE(llogan): Modify version string per release. -HERMES_VERSION=1.0.0 -spack install hermes +vfd +# Install hermes_shm (needed for dependencies) +# +spack repo add ci/hermes +spack install hermes_shm diff --git a/ci/install_docs.sh b/ci/install_docs.sh index 38d213025..63f912bd8 100755 --- a/ci/install_docs.sh +++ b/ci/install_docs.sh @@ -1,14 +1,13 @@ #!/bin/bash -# set -x -# set -e -# set -o pipefail +# CD into git workspace +cd ${GITHUB_WORKSPACE} +# Make build directory mkdir build -pushd build +cd build -#LOCAL=local -INSTALL_PREFIX="${HOME}/${LOCAL}" +INSTALL_PREFIX="${HOME}" export CXXFLAGS="${CXXFLAGS} -std=c++17 -Werror -Wall -Wextra" cmake \ @@ -17,25 +16,12 @@ cmake \ -DCMAKE_BUILD_RPATH=${INSTALL_PREFIX}/lib \ -DCMAKE_INSTALL_RPATH=${INSTALL_PREFIX}/lib \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DCMAKE_CXX_COMPILER=`which mpicxx` \ - -DCMAKE_C_COMPILER=`which mpicc` \ -DBUILD_SHARED_LIBS=ON \ -DHERMES_ENABLE_DOXYGEN=ON \ -DHERMES_ENABLE_COVERAGE=ON \ - -DHERMES_BUILD_BENCHMARKS=ON \ - -DHERMES_COMMUNICATION_MPI=ON \ - -DHERMES_BUILD_BUFFER_POOL_VISUALIZER=ON \ - -DORTOOLS_DIR=${INSTALL_PREFIX} \ - -DHERMES_USE_ADDRESS_SANITIZER=ON \ - -DHERMES_USE_THREAD_SANITIZER=OFF \ - -DHERMES_RPC_THALLIUM=ON \ - -DHERMES_ENABLE_VFD=ON \ - -DBUILD_TESTING=ON \ .. make dox >& out.txt # cat out.txt | grep warning | grep -v "ignoring unsupported tag" -popd rec="$( grep warning build/out.txt | grep -v "ignoring unsupported tag" | wc -l )" echo "$rec" exit "$rec" - diff --git a/ci/install_hermes.sh b/ci/install_hermes.sh deleted file mode 100755 index b34a3aa40..000000000 --- a/ci/install_hermes.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. ci/build_hermes.sh -pushd build -ctest -VV -popd diff --git a/ci/jarvis-util/.coveragerc b/ci/jarvis-util/.coveragerc deleted file mode 100644 index f1db9952b..000000000 --- a/ci/jarvis-util/.coveragerc +++ /dev/null @@ -1,4 +0,0 @@ -# .coveragerc -[run] -source = . -omit = *test* \ No newline at end of file diff --git a/ci/jarvis-util/.gitignore b/ci/jarvis-util/.gitignore deleted file mode 100644 index 80ced04e4..000000000 --- a/ci/jarvis-util/.gitignore +++ /dev/null @@ -1,139 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -.idea -hostfile.txt -lcov.info - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ -/.idea/jarvis-cd.iml -/.idea/misc.xml -/.idea/modules.xml -/.idea/inspectionProfiles/profiles_settings.xml -/.idea/inspectionProfiles/Project_Default.xml -/.idea/vcs.xml -/.idea/deployment.xml diff --git a/ci/jarvis-util/LICENSE b/ci/jarvis-util/LICENSE deleted file mode 100644 index 5db7c2cc8..000000000 --- a/ci/jarvis-util/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022-present Luke Logan and other contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/ci/jarvis-util/README.md b/ci/jarvis-util/README.md deleted file mode 100644 index 711a1466e..000000000 --- a/ci/jarvis-util/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Jarvis UTIL - -Jarvis-util is a library which contains various utilities to aid with -creating shell scripts within Python. This library contains wrappers -for executing shell commands locally, SSH, SCP, MPI, argument parsing, -and various other random utilities. - -![Build](https://github.com/lukemartinlogan/jarvis-util/workflows/GitHub%20Actions/badge.svg) - -[![Coverage Status](https://coveralls.io/repos/github/lukemartinlogan/jarvis-util/badge.svg?branch=master)](https://coveralls.io/github/lukemartinlogan/jarvis-util?branch=master) - -## Installation - -For now, we only consider manual installation -```bash -cd jarvis-util -python3 -m pip install -r requirements.txt -python3 -m pip install -e . -``` - -## Executing a program - -The following code will execute a command on the local machine. -The output will NOT be collected into an in-memory buffer. -The output will be printed to the terminal as it occurs. - -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - -node = Exec('echo hello', LocalExecInfo(collect_output=False)) -``` - -Programs can also be executed asynchronously: -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - -node = Exec('echo hello', LocalExecInfo(collect_output=False, - exec_async=True)) -node.wait() -``` - -Various examples of outputs: -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo - -# Will ONLY print to the terminal -node = Exec('echo hello', LocalExecInfo(collect_output=False)) -# Will collect AND print to the terminal -node = Exec('echo hello', LocalExecInfo(collect_output=True)) -# Will collect BUT NOT print to the terminal -node = Exec('echo hello', LocalExecInfo(collect_output=True, - hide_output=True)) -# Will collect, pipe to file, and print to terminal -node = Exec('echo hello', LocalExecInfo(collect_output=True, - pipe_stdout='/tmp/stdout.txt', - pipe_stderr='/tmp/stderr.txt')) -``` - -This is useful if you have a program which runs using a daemon mode. - -## Executing an MPI program - -The following code will execute the "hostname" command on the local -machine 24 times using MPI. - -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.mpi_exec import MpiExecInfo - -node = Exec('hostname', MpiExecInfo(hostfile=None, - nprocs=24, - ppn=None, - collect_output=False)) -``` - -## Executing an SSH program - -The following code will execute the "hostname" command on all machines -in the hostfile - -```python -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.pssh_exec import PsshExecInfo - -node = Exec('hostname', PsshExecInfo(hostfile="/tmp/hostfile.txt", - collect_output=False)) -``` - -## The contents of a hostfile - -A hostfile can have the following syntax: -``` -ares-comp-01 -ares-comp-[02-04] -ares-comp-[05-09,11,12-14]-40g -``` - -These will be expanded internally by PSSH and MPI. - -# Unit tests - -We run our unit tests in a docker container, which is located underneath -the CI directory. This is because we need to test multi-node functionality, -without having multiple nodes. To setup unit testing, perform the following: - -1. Install Docker -2. Setup our "ci" container -3. Run the unit tests - -``` -``` - -# Contributing - -We use the Google Python Style guide (pylintrc). \ No newline at end of file diff --git a/ci/jarvis-util/bin/jarvis-imports b/ci/jarvis-util/bin/jarvis-imports deleted file mode 100755 index 25fc19a9a..000000000 --- a/ci/jarvis-util/bin/jarvis-imports +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 - -from jarvis_util.util.import_all import * -import pathlib -import os - - -build_global_import_from_bin('jarvis_util') diff --git a/ci/jarvis-util/ci/cluster/Dockerfile b/ci/jarvis-util/ci/cluster/Dockerfile deleted file mode 100644 index 548937e02..000000000 --- a/ci/jarvis-util/ci/cluster/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -# Install ubuntu 20.04 -FROM ubuntu:20.04 -LABEL maintainer="llogan@hawk.iit.edu" -LABEL version="0.0" -LABEL description="An example docker image" - -# Disable Prompt During Packages Installation -ARG DEBIAN_FRONTEND=noninteractive - -# Update ubuntu -RUN apt update && apt install - -# Install some basic packages -RUN apt install -y \ - openssh-server \ - sudo git nano vim \ - docker \ - mpich \ - gcc \ - g++ \ - gfortran \ - libtool \ - libtool-bin \ - automake \ - autoconf - -# Create a new user -RUN useradd -m sshuser -RUN usermod -aG sudo sshuser -RUN passwd -d sshuser - -# Copy the host's SSH keys -# Docker requires COPY be relative to the current working -# directory. We cannot pass ~/.ssh/id_rsa unfortunately... -ENV SSHDIR=/home/sshuser/.ssh -RUN sudo -u sshuser mkdir ${SSHDIR} -COPY id_rsa ${SSHDIR}/id_rsa -COPY id_rsa.pub ${SSHDIR}/id_rsa.pub - -# Authorize host SSH keys -RUN sudo -u sshuser touch ${SSHDIR}/authorized_keys -RUN cat ${SSHDIR}/id_rsa.pub >> ${SSHDIR}/authorized_keys - -# Set SSH permissions -RUN chmod 700 ${SSHDIR} -RUN chmod 644 ${SSHDIR}/id_rsa.pub -RUN chmod 600 ${SSHDIR}/id_rsa -RUN chmod 600 ${SSHDIR}/authorized_keys - -# Enable passwordless SSH -RUN sed -i 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/' /etc/ssh/sshd_config - -# Start SSHD and wait forever -RUN mkdir /run/sshd -CMD ["/usr/sbin/sshd", "-D"] \ No newline at end of file diff --git a/ci/jarvis-util/ci/cluster/docker-compose.yml b/ci/jarvis-util/ci/cluster/docker-compose.yml deleted file mode 100644 index aa7477089..000000000 --- a/ci/jarvis-util/ci/cluster/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3" - -services: - node1: - build: . - links: - - node2 - networks: - - net - hostname: node1 - stdin_open: true - tty: true - - node2: - build: . - networks: - - net - hostname: node2 - stdin_open: true - tty: true - -networks: - net: - driver: bridge \ No newline at end of file diff --git a/ci/jarvis-util/ci/install_deps.sh b/ci/jarvis-util/ci/install_deps.sh deleted file mode 100644 index 0d1f0447d..000000000 --- a/ci/jarvis-util/ci/install_deps.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -sudo apt update -sudo apt install -y \ -docker \ -mpich \ -gcc \ -g++ \ -gfortran \ -libtool \ -libtool-bin \ -automake \ -autoconf \ No newline at end of file diff --git a/ci/jarvis-util/ci/install_jarvis.sh b/ci/jarvis-util/ci/install_jarvis.sh deleted file mode 100644 index 58aa283c0..000000000 --- a/ci/jarvis-util/ci/install_jarvis.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -python3 -m pip install -r requirements.txt -python3 -m pip install -e . diff --git a/ci/jarvis-util/ci/install_spack.sh b/ci/jarvis-util/ci/install_spack.sh deleted file mode 100644 index 0d1f0447d..000000000 --- a/ci/jarvis-util/ci/install_spack.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -sudo apt update -sudo apt install -y \ -docker \ -mpich \ -gcc \ -g++ \ -gfortran \ -libtool \ -libtool-bin \ -automake \ -autoconf \ No newline at end of file diff --git a/ci/jarvis-util/ci/lint.sh b/ci/jarvis-util/ci/lint.sh deleted file mode 100644 index a1af3ff48..000000000 --- a/ci/jarvis-util/ci/lint.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -pylint "${PWD}"/jarvis_util \ No newline at end of file diff --git a/ci/jarvis-util/ci/run_tests.sh b/ci/jarvis-util/ci/run_tests.sh deleted file mode 100644 index abd4e290a..000000000 --- a/ci/jarvis-util/ci/run_tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -coverage run -m pytest -rm -rf "*.pyc" -coverage report -coverage-lcov \ No newline at end of file diff --git a/ci/jarvis-util/jarvis_util/__init__.py b/ci/jarvis-util/jarvis_util/__init__.py deleted file mode 100644 index 1019dd213..000000000 --- a/ci/jarvis-util/jarvis_util/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Import all modules""" -from jarvis_util.util.expand_env import * -from jarvis_util.util.naming import * -from jarvis_util.util.hostfile import * -from jarvis_util.util.size_conv import * -from jarvis_util.util.import_all import * -from jarvis_util.util.import_mod import * -from jarvis_util.util.argparse import * -from jarvis_util.serialize.ini_file import * -from jarvis_util.serialize.yaml_file import * -from jarvis_util.serialize.text_file import * -from jarvis_util.serialize.serializer import * -from jarvis_util.serialize.pickle import * -from jarvis_util.shell.filesystem import * -from jarvis_util.shell.exec import * -from jarvis_util.shell.exec_info import * -from jarvis_util.shell.ssh_exec import * -from jarvis_util.shell.pssh_exec import * -from jarvis_util.shell.process import * -from jarvis_util.shell.pscp import * -from jarvis_util.shell.scp import * -from jarvis_util.shell.mpi_exec import * -from jarvis_util.shell.local_exec import * -from jarvis_util.introspect.system_info import * -from jarvis_util.jutil_manager import * diff --git a/ci/jarvis-util/jarvis_util/introspect/system_info.py b/ci/jarvis-util/jarvis_util/introspect/system_info.py deleted file mode 100644 index 5a9d94482..000000000 --- a/ci/jarvis-util/jarvis_util/introspect/system_info.py +++ /dev/null @@ -1,635 +0,0 @@ -""" -This module provides methods for querying the information of the host -system. This can be used to make scripts more portable. -""" - -import re -import platform -from jarvis_util.shell.exec import Exec -from jarvis_util.util.size_conv import SizeConv -from jarvis_util.serialize.yaml_file import YamlFile -import json -import pandas as pd -import numpy as np -from enum import Enum -import shlex - -# pylint: disable=C0121 - -class SystemInfo: - """ - This class queries information about the host machine, such as OS, - CPU, and kernel - """ - - instance_ = None - - @staticmethod - def get_instance(): - if SystemInfo.instance_ is None: - SystemInfo.instance_ = SystemInfo() - return SystemInfo.instance_ - - def __init__(self): - with open('/etc/os-release', 'r', encoding='utf-8') as fp: - lines = fp.read().splitlines() - self.os = self._detect_os_type(lines) - self.os_like = self._detect_os_like_type(lines) - self.os_version = self._detect_os_version(lines) - self.ksemantic = platform.platform() - self.krelease = platform.release() - self.ktype = platform.system() - self.cpu = platform.processor() - self.cpu_family = platform.machine() - - def _detect_os_type(self, lines): - for line in lines: - if 'ID=' in line: - if 'ubuntu' in line: - return 'ubuntu' - elif 'centos' in line: - return 'centos' - elif 'debian' in line: - return 'debian' - - def _detect_os_like_type(self, lines): - for line in lines: - if 'ID_LIKE=' in line: - if 'ubuntu' in line: - return 'ubuntu' - elif 'centos' in line: - return 'centos' - elif 'debian' in line: - return 'debian' - - def _detect_os_version(self, lines): - for line in lines: - grp = re.match('VERSION_ID=\"(.*)\"', line) - if grp: - return grp.group(1) - - def __hash__(self): - return hash(str([self.os, self.os_like, - self.os_version, self.ksemantic, - self.krelease, self.ktype, - self.cpu, self.cpu_family])) - - def __eq__(self, other): - return ( - (self.os == other.os) and - (self.os_like == other.os_like) and - (self.os_version == other.os_version) and - (self.ksemantic == other.ksemantic) and - (self.krelease == other.krelease) and - (self.cpu == other.cpu) and - (self.cpu_family == other.cpu_family) - ) - - -class Lsblk(Exec): - """ - List all block devices in the system per-node. Lsblk will return - a JSON output - - A table is stored per-host: - parent: the parent device of the partition (e.g., /dev/sda or NaN) - device: the name of the partition (e.g., /dev/sda1) - size: total size of the partition (bytes) - mount: where the partition is mounted (if anywhere) - model: the exact model of the device - tran: the transport of the device (e.g., /dev/nvme) - rota: whether or not the device is rotational - host: the host this record corresponds to - """ - - def __init__(self, exec_info): - cmd = 'lsblk -o NAME,SIZE,MODEL,TRAN,MOUNTPOINT,ROTA -J -s' - super().__init__(cmd, exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - lsblk_data = json.loads(stdout)['blockdevices'] - partitions = [] - devs = {} - for partition in lsblk_data: - dev = partition['children'][0] - partitions.append({ - 'parent': f'/dev/{dev["name"]}', - 'device': f'/dev/{partition["name"]}', - 'size': SizeConv.to_int(partition['size']), - 'mount': partition['mountpoint'], - 'host': host - }) - devs[dev['name']] = { - 'parent': f'/dev/{dev["name"]}', - 'size': SizeConv.to_int(dev['size']), - 'model': dev['model'], - 'tran': dev['tran'].lower(), - 'mount': dev['mountpoint'], - 'rota': dev['rota'], - 'host': host - } - devs = list(devs.values()) - part_df = pd.DataFrame(partitions) - dev_df = pd.DataFrame(devs) - total_df = pd.merge(part_df, - dev_df[['parent', 'model', 'tran', 'host']], - on=['parent', 'host']) - dev_df = dev_df.rename(columns={'parent': 'device'}) - total_df = pd.concat([total_df, dev_df]) - self.df = total_df - - -class Blkid(Exec): - """ - List all filesystems (even those unmounted) and their properties - - Stores a per-host table with the following: - device: the device (or partition) which stores the data (e.g., /dev/sda) - fs_type: the type of filesystem (e.g., ext4) - uuid: filesystem-levle uuid from the FS metadata - partuuid: the partition-lable UUID for the partition - host: the host this entry corresponds to - """ - def __init__(self, exec_info): - cmd = 'blkid' - super().__init__(cmd, exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - devices = stdout.splitlines() - dev_list = [] - for dev in devices: - dev_dict = {} - toks = shlex.split(dev) - dev_name = toks[0].split(':')[0] - dev_dict['device'] = dev_name - dev_dict['host'] = host - for tok in toks[1:]: - keyval = tok.split('=') - key = keyval[0].lower() - val = ' '.join(keyval[1:]) - dev_dict[key] = val - dev_list.append(dev_dict) - df = pd.DataFrame(dev_list) - df = df.rename(columns={'type': 'fs_type'}) - self.df = df - - -class ListFses(Exec): - """ - List all mounted filesystems - - Will store a per-host dictionary containing: - device: the device which contains the filesystem - fs_size: total size of the filesystem - used: total nubmer of bytes used - avail: total number of bytes remaining - use%: the percent of capacity used - fs_mount: where the filesystem is mounted - host: the host this entry corresponds to - """ - - def __init__(self, exec_info): - cmd = 'df -h' - super().__init__(cmd, exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - lines = stdout.strip().splitlines() - columns = ['device', 'fs_size', 'used', - 'avail', 'use%', 'fs_mount', 'host'] - rows = [line.split() + [host] for line in lines[1:]] - df = pd.DataFrame(rows, columns=columns) - # pylint: disable=W0108 - df.loc[:, 'fs_size'] = df['fs_size'].apply( - lambda x : SizeConv.to_int(x)) - df.loc[:, 'used'] = df['used'].apply( - lambda x: SizeConv.to_int(x)) - df.loc[:, 'avail'] = df['avail'].apply( - lambda x : SizeConv.to_int(x)) - # pylint: enable=W0108 - self.df = df - - -class FiInfo(Exec): - """ - List all networks and their information - provider: network protocol (e.g., sockets, tcp, ib) - fabric: IP address - domain: network domain - version: network version - type: packet type (e.g., DGRAM) - protocol: protocol constant - host: the host this network corresponds to - """ - def __init__(self, exec_info): - super().__init__('fi_info', exec_info.mod(collect_output=True)) - self.exec_async = exec_info.exec_async - self.graph = {} - if not self.exec_async: - self.wait() - - def wait(self): - super().wait() - for host, stdout in self.stdout.items(): - lines = stdout.strip().splitlines() - providers = [] - for line in lines: - if 'provider' in line: - providers.append({ - 'provider': line.split(':')[1], - 'host': host - }) - else: - splits = line.split(':') - key = splits[0].strip() - val = splits[1].strip() - if 'fabric' in key: - val = val.split('/')[0] - providers[-1][key] = val - self.df = pd.DataFrame(providers) - - -class StorageDeviceType(Enum): - PMEM='pmem' - NVME='nvme' - SSD='ssd' - HDD='hdd' - - -class ResourceGraph: - """ - Stores helpful information about storage and networking info for machines. - - Two tables are stored to make decisions on application deployment. - fs: - parent: the parent device of the partition (e.g., /dev/sda or NaN) - device: the name of the device (e.g., /dev/sda1 or /dev/sda) - size: total size of the device (bytes) - mount: where the device is mounted (if anywhere) - model: the exact model of the device - rota: whether the device is rotational or not - tran: the transport of the device (e.g., /dev/nvme) - fs_type: the type of filesystem (e.g., ext4) - uuid: filesystem-levle uuid from the FS metadata - fs_size: total size of the filesystem - avail: total number of bytes remaining - shared: whether the this is a shared service or not - host: the host this record corresponds to - net: - provider: network protocol (e.g., sockets, tcp, ib) - fabric: IP address - domain: network domain - host: the host this network corresponds to - - TODO: Need to verify on more than ubuntu20.04 - TODO: Can we make this work for windows? - TODO: Can we make this work even when hosts have different OSes? - """ - - def __init__(self): - self.lsblk = None - self.blkid = None - self.list_fs = None - self.fi_info = None - self.fs_columns = [ - 'parent', 'device', 'size', 'mount', 'model', 'rota', - 'tran', 'fs_type', 'uuid', 'fs_size', - 'avail', 'shared', 'host' - ] - self.net_columns = [ - 'provider', 'fabric', 'domain', 'host' - ] - self.all_fs = pd.DataFrame(columns=self.fs_columns) - self.all_net = pd.DataFrame(columns=self.net_columns) - self.fs_settings = { - 'register': [], - 'filter_mounts': {} - } - self.net_settings = { - 'register': [], - 'track_ips': {} - } - self.hosts = None - - def build(self, exec_info, introspect=True): - """ - Build a resource graph. - - :param exec_info: Where to collect resource information - :param introspect: Whether to introspect system info, or rely solely - on admin-defined settings - :return: self - """ - if introspect: - self._introspect(exec_info) - self.apply() - return self - - def _introspect(self, exec_info): - """ - Introspect the cluster for resources. - - :param exec_info: Where to collect resource information - :return: None - """ - self.lsblk = Lsblk(exec_info) - self.blkid = Blkid(exec_info) - self.list_fs = ListFses(exec_info) - self.fi_info = FiInfo(exec_info) - self.hosts = exec_info.hostfile.hosts - self.all_fs = pd.merge(self.lsblk.df, - self.blkid.df, - on=['device', 'host'], - how='outer') - self.all_fs['shared'] = False - self.all_fs = pd.merge(self.all_fs, - self.list_fs.df, - on=['device', 'host'], - how='outer') - self.all_fs['shared'].fillna(True, inplace=True) - self.all_fs.drop(['used', 'use%', 'fs_mount', 'partuuid'], - axis=1, inplace=True) - self.all_fs['mount'].fillna(value='', inplace=True) - net_df = self.fi_info.df - net_df['speed'] = np.nan - net_df.drop(['version', 'type', 'protocol'], - axis=1, inplace=True) - self.all_net = net_df - - def save(self, path): - """ - Save the resource graph YAML file - - :param path: the path to save the file - :return: None - """ - graph = { - 'hosts': self.hosts, - 'fs': self.all_fs.to_dict('records'), - 'net': self.all_net.to_dict('records'), - 'fs_settings': self.fs_settings, - 'net_settings': self.net_settings - } - YamlFile(path).save(graph) - - def load(self, path): - """ - Load resource graph from storage. - - :param path: The path to the resource graph YAML file - :return: self - """ - graph = YamlFile(path).load() - self.hosts = graph['hosts'] - self.all_fs = pd.DataFrame(graph['fs']) - self.all_net = pd.DataFrame(graph['net']) - self.fs = None - self.net = None - self.fs_settings = graph['fs_settings'] - self.net_settings = graph['net_settings'] - self.apply() - return self - - def set_hosts(self, hosts): - """ - Set the set of hosts this resource graph covers - - :param hosts: Hostfile() - :return: None - """ - self.hosts = hosts.hosts_ip - - def add_storage(self, hosts, **kwargs): - """ - Register a storage device record - - :param hosts: Hostfile() indicating set of hosts to make record for - :param kwargs: storage record - :return: None - """ - for host in hosts.hosts: - record = kwargs.copy() - record['host'] = host - self.fs_settings['register'].append(record) - - def add_net(self, hosts, **kwargs): - """ - Register a network record - - :param hosts: Hostfile() indicating set of hosts to make record for - :param kwargs: net record - :return: None - """ - for host, ip in zip(hosts.hosts, hosts.hosts_ip): - record = kwargs.copy() - record['fabric'] = ip - record['host'] = host - self.net_settings['register'].append(record) - - def filter_fs(self, mount_re, - mount_suffix=None, tran=None): - """ - Track all filesystems + devices matching the mount regex. - - :param mount_re: The regex to match a set of mountpoints - :param mount_suffix: After the mount_re is matched, append this path - to the mountpoint to indicate where users can access data. A typical - value for this is /${USER}, indicating the mountpoint has a subdirectory - per-user where they can access data. - :param shared: Whether this mount point is shared - :param tran: The transport of this device - :return: self - """ - self.fs_settings['filter_mounts']['mount_re'] = { - 'mount_re': mount_re, - 'mount_suffix': mount_suffix, - 'tran': tran - } - return self - - def filter_ip(self, ip_re, speed=None): - """ - Track all IPs matching the regex. The IPs with this regex all have - a certain speed. - - :param ip_re: The regex to match - :param speed: The speed of the fabric - :return: self - """ - self.net_settings['track_ips'][ip_re] = { - 'ip_re': ip_re, - 'speed': SizeConv.to_int(speed) if speed is not None else speed - } - return self - - def filter_hosts(self, hosts, speed=None): - """ - Track all ips matching the hostnames. - - :param hosts: Hostfile() of the hosts to filter for - :param speed: Speed of the interconnect (e.g., 1gbps) - :return: self - """ - for host in hosts.hosts_ip: - self.filter_ip(host, speed) - return self - - def apply(self): - """ - Apply fs and net settings to the resource graph - - :return: self - """ - self._apply_fs_settings() - self._apply_net_settings() - # self.fs.size = self.fs.size.fillna(0) - # self.fs.avail = self.fs.avail.fillna(0) - # self.fs.fs_size = self.fs.fs_size.fillna(0) - return self - - def _apply_fs_settings(self): - if len(self.fs_settings) == 0: - self.fs = self.all_fs - return - df = self.all_fs - self.fs = pd.DataFrame(columns=self.all_net.columns) - for fs_set in self.fs_settings['filter_mounts'].values(): - mount_re = fs_set['mount_re'] - mount_suffix = fs_set['mount_suffix'] - tran = fs_set['tran'] - with_mount = df[df.mount.str.contains(mount_re)] - if mount_suffix is not None: - with_mount['mount'] += mount_suffix - if tran is not None: - with_mount['tran'] = tran - self.fs = pd.concat([self.fs, with_mount]) - admin_df = pd.DataFrame(self.fs_settings['register'], - columns=self.fs_columns) - self.fs = pd.concat([self.fs, admin_df]) - - def _apply_net_settings(self): - if len(self.net_settings) == 0: - self.net = self.all_net - return - self.net = pd.DataFrame(columns=self.all_net.columns) - df = self.all_net - for net_set in self.net_settings['track_ips'].values(): - ip_re = net_set['ip_re'] - speed = net_set['speed'] - with_ip = df[df['fabric'].str.contains(ip_re)] - with_ip['speed'] = speed - self.net = pd.concat([self.net, with_ip]) - admin_df = pd.DataFrame(self.net_settings['register'], - columns=self.net_columns) - self.net = pd.concat([self.net, admin_df]) - - def find_shared_storage(self): - """ - Find the set of shared storage services - - :return: Dataframe - """ - df = self.fs - return df[df.shared == True] - - def find_storage(self, - dev_types=None, - is_mounted=True, - common=False, - count_per_node=None, - count_per_dev=None, - min_cap=None, - min_avail=None): - """ - Find a set of storage devices. - - :param dev_types: Search for devices of type in order. Either a list - or a string. - :param is_mounted: Search only for mounted devices - :param common: Remove mount points that are not common across all hosts - :param count_per_node: Choose only a subset of devices matching query - :param count_per_dev: Choose only a subset of devices matching query - :param min_cap: Remove devices with too little overall capacity - :param min_avail: Remove devices with too little available space - :return: Dataframe - """ - df = self.fs - # Remove pfs - df = df[df.shared == False] - # Filter devices by whether or not a mount is needed - if is_mounted: - df = df[df.mount.notna()] - # Find devices of a particular type - if dev_types is not None: - matching_devs = pd.DataFrame(columns=df.columns) - if isinstance(dev_types, str): - dev_types = [dev_types] - for dev_type in dev_types: - if dev_type == StorageDeviceType.HDD: - devs = df[(df.tran == 'sata') & (df.rota == True)] - elif dev_type == StorageDeviceType.SSD: - devs = df[(df.tran == 'sata') & (df.rota == False)] - elif dev_type == StorageDeviceType.NVME: - devs = df[(df.tran == 'nvme')] - matching_devs = pd.concat([matching_devs, devs]) - df = matching_devs - # Get the set of mounts common between all hosts - if common: - df = df.groupby(['mount']).filter( - lambda x: len(x) == len(self.hosts)).reset_index(drop=True) - # Remove storage with too little capacity - if min_cap is not None: - df = df[df.size >= min_cap] - # Remove storage with too little available space - if min_avail is not None: - df = df[df.avail >= min_avail] - # Take a certain number of each device per-host - if count_per_dev is not None: - df = df.groupby(['tran', 'rota', 'host']).\ - head(count_per_dev).reset_index(drop=True) - # Take a certain number of matched devices per-host - if count_per_node is not None: - df = df.groupby('host').head(count_per_node).reset_index(drop=True) - return df - - def find_net_info(self, hosts, - providers=None): - """ - Find the set of networks common between each host. - - :param hosts: A Hostfile() data structure containing the set of - all hosts to find network information for - :param providers: The network protocols to search for. - :return: Dataframe - """ - df = self.net - # Get the set of fabrics corresponding to these hosts - df = df[df.fabric.isin(hosts.hosts_ip)] - # Filter out protocols which are not common between these hosts - df = df.groupby('provider').filter( - lambda x: len(x) == len(hosts)).reset_index(drop=True) - # Choose only a subset of providers - if providers is not None: - if isinstance(providers, str): - providers = [providers] - df = df[df.provider.isin(providers)] - return df - -# pylint: enable=C0121 diff --git a/ci/jarvis-util/jarvis_util/jutil_manager.py b/ci/jarvis-util/jarvis_util/jutil_manager.py deleted file mode 100644 index c229b2445..000000000 --- a/ci/jarvis-util/jarvis_util/jutil_manager.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -This file contains properties which are globally accessible to all -jarvis-util modules. This can be used to configure various aspects -of jarvis, such as output. -""" - - -class JutilManager: - """ - A singleton which stores various properties that can be queried by - internally by jutil modules. This includes properties such output - management. - """ - - instance_ = None - - @staticmethod - def get_instance(): - if JutilManager.instance_ is None: - JutilManager.instance_ = JutilManager() - return JutilManager.instance_ - - def __init__(self): - self.collect_output = False - self.hide_output = False - self.debug_mpi_exec = False - self.debug_local_exec = False - diff --git a/ci/jarvis-util/jarvis_util/serialize/ini_file.py b/ci/jarvis-util/jarvis_util/serialize/ini_file.py deleted file mode 100644 index 8da174ff1..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/ini_file.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -This module contains methods to serialize and deserialize data from -a human-readable ini file. -""" -import configparser -from jarvis_util.serialize.serializer import Serializer - - -class IniFile(Serializer): - """ - This class contains methods to serialize and deserialize data from - a human-readable ini file. - """ - def __init__(self, path): - self.path = path - - def load(self): - config = configparser.ConfigParser() - config.read(self.path) - return config - - def save(self, data): - with open(self.path, 'w', encoding='utf-8') as fp: - data.write(fp) diff --git a/ci/jarvis-util/jarvis_util/serialize/pickle.py b/ci/jarvis-util/jarvis_util/serialize/pickle.py deleted file mode 100644 index a90ae12de..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/pickle.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -This module contains methods to serialize and deserialize data from -a pickle file. -""" - -import pickle as pkl -from jarvis_util.serialize.serializer import Serializer - - -class PickleFile(Serializer): - """ - This class serializes and deserializes data from a pickle file - """ - - def __init__(self, path): - self.path = path - - def load(self): - with open(self.path, 'rb') as fp: - return pkl.load(fp) - - def save(self, data): - with open(self.path, 'wb') as fp: - pkl.dump(data, fp) diff --git a/ci/jarvis-util/jarvis_util/serialize/serializer.py b/ci/jarvis-util/jarvis_util/serialize/serializer.py deleted file mode 100644 index 8647945a7..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/serializer.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -This module contains an abstract class used to define classes which -serialize data to a file. -""" - -from abc import ABC, abstractmethod - - -class Serializer(ABC): - """ - An abstract class which loads serialized data from a file and - saves serialized data to a file. - """ - - @abstractmethod - def load(self): - pass - - @abstractmethod - def save(self, data): - pass diff --git a/ci/jarvis-util/jarvis_util/serialize/text_file.py b/ci/jarvis-util/jarvis_util/serialize/text_file.py deleted file mode 100644 index 214c6bb9f..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/text_file.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -This module stores data into a file in a human-readable way -""" -from jarvis_util.serialize.serializer import Serializer - - -class TextFile(Serializer): - """ - This class stores data directly into a file using str() as the - serialization method. The data is intended to be human-readable. - """ - def __init__(self, path): - self.path = path - - def load(self): - with open(self.path, 'r', encoding='utf-8') as fp: - data = fp.read() - return data - - def save(self, data): - with open(self.path, 'w', encoding='utf-8') as fp: - fp.write(data) diff --git a/ci/jarvis-util/jarvis_util/serialize/yaml_file.py b/ci/jarvis-util/jarvis_util/serialize/yaml_file.py deleted file mode 100644 index 5892f6bca..000000000 --- a/ci/jarvis-util/jarvis_util/serialize/yaml_file.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -This module contains methods to serialize and deserialize data from -a human-readable YAML file. -""" -from jarvis_util.serialize.serializer import Serializer -import yaml - - -class YamlFile(Serializer): - """ - This class contains methods to serialize and deserialize data from - a human-readable YAML file. - """ - def __init__(self, path): - self.path = path - - def load(self): - with open(self.path, 'r', encoding='utf-8') as fp: - return yaml.load(fp, Loader=yaml.FullLoader) - return None - - def save(self, data): - with open(self.path, 'w', encoding='utf-8') as fp: - yaml.dump(data, fp) diff --git a/ci/jarvis-util/jarvis_util/shell/exec.py b/ci/jarvis-util/jarvis_util/shell/exec.py deleted file mode 100644 index f9fef066d..000000000 --- a/ci/jarvis-util/jarvis_util/shell/exec.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -This module provides mechanisms to execute binaries either locally or -remotely. -""" - -from .local_exec import LocalExec -from .pssh_exec import PsshExec -from .pssh_exec import SshExec -from .mpi_exec import MpiExec -from .exec_info import ExecInfo, ExecType, Executable - - -class Exec(Executable): - """ - This class is a factory which wraps around various shell command - execution stragies, such as MPI and SSH. - """ - - def __init__(self, cmd, exec_info=None): - """ - Execute a command or list of commands - - :param cmd: list of commands or a single command string - :param exec_info: Info needed to execute processes locally - """ - super().__init__() - if exec_info is None: - exec_info = ExecInfo() - if exec_info.exec_type == ExecType.LOCAL: - self.exec_ = LocalExec(cmd, exec_info) - elif exec_info.exec_type == ExecType.SSH: - self.exec_ = SshExec(cmd, exec_info) - elif exec_info.exec_type == ExecType.PSSH: - self.exec_ = PsshExec(cmd, exec_info) - elif exec_info.exec_type == ExecType.MPI: - self.exec_ = MpiExec(cmd, exec_info) - self.set_exit_code() - self.set_output() - - def wait(self): - self.exec_.wait() - self.set_output() - self.set_exit_code() - return self.exit_code - - def set_output(self): - self.stdout = self.exec_.stdout - self.stderr = self.exec_.stderr - if isinstance(self.stdout, str): - if hasattr(self.exec_, 'addr'): - host = self.exec_.addr - else: - host = 'localhost' - self.stdout = {host: self.stdout} - self.stderr = {host: self.stderr} - - def set_exit_code(self): - self.exec_.set_exit_code() - self.exit_code = self.exec_.exit_code diff --git a/ci/jarvis-util/jarvis_util/shell/exec_info.py b/ci/jarvis-util/jarvis_util/shell/exec_info.py deleted file mode 100644 index 2b28246ae..000000000 --- a/ci/jarvis-util/jarvis_util/shell/exec_info.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -This module contains data structures for determining how to execute -a subcommand. This includes information such as storing SSH keys, -passwords, working directory, etc. -""" - -from enum import Enum -from jarvis_util.util.hostfile import Hostfile -import os -from abc import ABC, abstractmethod - - -class ExecType(Enum): - """ - Different program execution methods. - """ - - LOCAL = 'LOCAL' - SSH = 'SSH' - PSSH = 'PSSH' - MPI = 'MPI' - - -class ExecInfo: - """ - Contains all information needed to execute a program. This includes - parameters such as the path to key-pairs, the hosts to run the program - on, number of processes, etc. - """ - def __init__(self, exec_type=ExecType.LOCAL, nprocs=None, ppn=None, - user=None, pkey=None, port=None, - hostfile=None, hosts=None, env=None, - sleep_ms=0, sudo=False, cwd=None, - collect_output=None, pipe_stdout=None, pipe_stderr=None, - hide_output=None, exec_async=False, stdin=None): - """ - - :param exec_type: How to execute a program. SSH, MPI, Local, etc. - :param nprocs: Number of processes to spawn. E.g., MPI uses this - :param ppn: Number of processes per node. E.g., MPI uses this - :param user: The user to execute command under. E.g., SSH, PSSH - :param pkey: The path to the private key. E.g., SSH, PSSH - :param port: The port to use for connection. E.g., SSH, PSSH - :param hostfile: The hosts to launch command on. E.g., PSSH, MPI - :param hosts: A list (or single string) of host names to run command on. - :param env: The environment variables to use for command. - :param sleep_ms: Sleep for a period of time AFTER executing - :param sudo: Execute command with root privilege. E.g., SSH, PSSH - :param cwd: Set current working directory. E.g., SSH, PSSH - :param collect_output: Collect program output in python buffer - :param pipe_stdout: Pipe STDOUT into a file. (path string) - :param pipe_stderr: Pipe STDERR into a file. (path string) - :param hide_output: Whether to print output to console - :param exec_async: Whether to execute program asynchronously - :param stdin: Any input needed by the program. Only local - """ - - self.exec_type = exec_type - self.nprocs = nprocs - self.user = user - self.pkey = pkey - self.port = port - self.ppn = ppn - self.hostfile = hostfile - self._set_hostfile(hostfile=hostfile, hosts=hosts) - self.env = env - self.basic_env = {} - self._set_env(env) - self.cwd = cwd - self.sudo = sudo - self.sleep_ms = sleep_ms - self.collect_output = collect_output - self.pipe_stdout = pipe_stdout - self.pipe_stderr = pipe_stderr - self.hide_output = hide_output - self.exec_async = exec_async - self.stdin = stdin - self.keys = ['exec_type', 'nprocs', 'ppn', 'user', 'pkey', 'port', - 'hostfile', 'env', 'sleep_ms', 'sudo', - 'cwd', 'hosts', 'collect_output', - 'pipe_stdout', 'pipe_stderr', 'hide_output', - 'exec_async', 'stdin'] - - def _set_env(self, env): - if env is None: - self.env = {} - else: - self.env = env - basic_env = [ - 'PATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'CMAKE_PREFIX_PATH', - 'PYTHON_PATH', 'CPATH', 'INCLUDE', 'JAVA_HOME' - ] - self.basic_env = {} - for key in basic_env: - if key not in os.environ: - continue - self.basic_env[key] = os.getenv(key) - for key, val in self.basic_env.items(): - if key not in self.env: - self.env[key] = val - self.basic_env.update(self.env) - if 'LD_PRELOAD' in self.basic_env: - del self.basic_env['LD_PRELOAD'] - - def _set_hostfile(self, hostfile=None, hosts=None): - if hostfile is not None: - if isinstance(hostfile, str): - self.hostfile = Hostfile(hostfile=hostfile) - elif isinstance(hostfile, Hostfile): - self.hostfile = hostfile - else: - raise Exception('Hostfile is neither string nor Hostfile') - if hosts is not None: - if isinstance(hosts, list): - self.hostfile = Hostfile(all_hosts=hosts) - elif isinstance(hosts, str): - self.hostfile = Hostfile(all_hosts=[hosts]) - elif isinstance(hosts, Hostfile): - self.hostfile = hosts - else: - raise Exception('Host set is neither str, list or Hostfile') - - if hosts is not None and hostfile is not None: - raise Exception('Must choose either hosts or hostfile, not both') - - if self.hostfile is None: - self.hostfile = Hostfile() - - def mod(self, **kwargs): - self._mod_kwargs(kwargs) - return ExecInfo(**kwargs) - - def _mod_kwargs(self, kwargs): - for key in self.keys: - if key not in kwargs and hasattr(self, key): - kwargs[key] = getattr(self, key) - - def copy(self): - return self.mod() - - -class Executable(ABC): - """ - An abstract class representing a class which is intended to run - shell commands. This includes SSH, MPI, etc. - """ - - def __init__(self): - self.exit_code = None - self.stdout = '' - self.stderr = '' - - def failed(self): - return self.exit_code != 0 - - @abstractmethod - def set_exit_code(self): - pass - - @abstractmethod - def wait(self): - pass - - def smash_cmd(self, cmds): - """ - Convert a list of commands into a single command for the shell - to execute. - - :param cmds: A list of commands or a single command string - :return: - """ - if isinstance(cmds, list): - return ' && '.join(cmds) - elif isinstance(cmds, str): - return cmds - else: - raise Exception('Command must be either list or string') - - def wait_list(self, nodes): - for node in nodes: - node.wait() - - def smash_list_outputs(self, nodes): - """ - Combine the outputs of a set of nodes into a single output. - For example, used if executing multiple commands in sequence. - - :param nodes: - :return: - """ - self.stdout = '\n'.join([node.stdout for node in nodes]) - self.stderr = '\n'.join([node.stderr for node in nodes]) - - def per_host_outputs(self, nodes): - """ - Convert the outputs of a set of nodes to a per-host dictionary. - Used if sending commands to multiple hosts - - :param nodes: - :return: - """ - self.stdout = {} - self.stderr = {} - self.stdout = {node.addr: node.stdout for node in nodes} - self.stderr = {node.addr: node.stderr for node in nodes} - - def set_exit_code_list(self, nodes): - """ - Set the exit code from a set of nodes. - - :param nodes: The set of execution nodes that have been executed - :return: - """ - for node in nodes: - if node.exit_code: - self.exit_code = node.exit_code diff --git a/ci/jarvis-util/jarvis_util/shell/filesystem.py b/ci/jarvis-util/jarvis_util/shell/filesystem.py deleted file mode 100644 index a5cb2b55e..000000000 --- a/ci/jarvis-util/jarvis_util/shell/filesystem.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -This module contains various wrappers over typical filesystem commands seen -in shell scripts. This includes operations such as creating directories, -changing file permissions, etc. -""" -from .exec import Exec - - -class Mkdir(Exec): - """ - Create directories + subdirectories. - """ - - def __init__(self, paths, exec_info=None): - """ - Create directories + subdirectories. Does not fail if the dirs - already exist. - - :param paths: A list of paths or a single path string. - :param exec_info: Info needed to execute the mkdir command - """ - - if isinstance(paths, str): - paths = [paths] - path = ' '.join(paths) - super().__init__(f'mkdir -p {path}', exec_info) - - -class Rm(Exec): - """ - Remove a file and its subdirectories - """ - - def __init__(self, paths, exec_info=None): - """ - Execute file or directory remove. - - :param paths: Either a list of paths or a single path string - :param exec_info: Information needed to execute rm - """ - - if isinstance(paths, str): - paths = [paths] - path = ' '.join(paths) - super().__init__(f'rm -rf {path}', exec_info) - - -class Chmod(Exec): - """ - Change the mode of a file - """ - - def __init__(self, path=None, mode=None, modes=None, exec_info=None): - """ - Change the mode of a file - - :param path: path to file to mode change - :param mode: the mode to change to - :param modes: A list of tuples [(Path, Mode)] - :param exec_info: How to execute commands - """ - cmds = [] - if path is not None and mode is not None: - cmds.append(f'chmod {mode} {path}') - if modes is not None: - cmds += [f'chmod {mode[1]} {mode[0]}' for mode in modes] - if len(cmds) == 0: - raise Exception('Must set either path+mode or modes') - super().__init__(cmds, exec_info) diff --git a/ci/jarvis-util/jarvis_util/shell/local_exec.py b/ci/jarvis-util/jarvis_util/shell/local_exec.py deleted file mode 100644 index c5487fe1e..000000000 --- a/ci/jarvis-util/jarvis_util/shell/local_exec.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Provides methods for executing a program or workflow locally. This class -is intended to be called from Exec, not by general users. -""" - -import time -import subprocess -import os -import sys -import io -import threading -from jarvis_util.jutil_manager import JutilManager -from .exec_info import ExecInfo, ExecType, Executable - - -class LocalExec(Executable): - """ - Provides methods for executing a program or workflow locally. - """ - - def __init__(self, cmd, exec_info): - """ - Execute a program or workflow - - :param cmd: list of commands or a single command string - :param exec_info: Info needed to execute processes locally - """ - - super().__init__() - jutil = JutilManager.get_instance() - cmd = self.smash_cmd(cmd) - - # Managing console output and collection - self.collect_output = exec_info.collect_output - self.pipe_stdout = exec_info.pipe_stdout - self.pipe_stderr = exec_info.pipe_stderr - self.pipe_stdout_fp = None - self.pipe_stderr_fp = None - self.hide_output = exec_info.hide_output - # pylint: disable=R1732 - if self.collect_output is None: - self.collect_output = jutil.collect_output - if self.hide_output is None: - self.hide_output = jutil.hide_output - # pylint: enable=R1732 - self.stdout = io.StringIO() - self.stderr = io.StringIO() - self.last_stdout_size = 0 - self.last_stderr_size = 0 - self.executing_ = True - self.exit_code = 0 - - # Copy ENV - self.env = exec_info.env.copy() - for key, val in os.environ.items(): - if key not in self.env: - self.env[key] = val - - # Managing command execution - self.cmd = cmd - self.sudo = exec_info.sudo - self.stdin = exec_info.stdin - self.exec_async = exec_info.exec_async - self.sleep_ms = exec_info.sleep_ms - if exec_info.cwd is None: - self.cwd = os.getcwd() - else: - self.cwd = exec_info.cwd - if jutil.debug_local_exec: - print(cmd) - self._start_bash_processes() - - def _start_bash_processes(self): - if self.sudo: - self.cmd = f'sudo {self.cmd}' - time.sleep(self.sleep_ms) - # pylint: disable=R1732 - self.proc = subprocess.Popen(self.cmd, - cwd=self.cwd, - env=self.env, - shell=True) - # pylint: enable=R1732 - if not self.exec_async: - self.wait() - - def wait(self): - self.proc.wait() - self.set_exit_code() - return self.exit_code - - def set_exit_code(self): - self.exit_code = self.proc.returncode - - def get_pid(self): - if self.proc is not None: - return self.proc.pid - else: - return None - - -class LocalExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.LOCAL, **kwargs) diff --git a/ci/jarvis-util/jarvis_util/shell/mpi_exec.py b/ci/jarvis-util/jarvis_util/shell/mpi_exec.py deleted file mode 100644 index e104bf50f..000000000 --- a/ci/jarvis-util/jarvis_util/shell/mpi_exec.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -This module provides methods to execute a process in parallel using the -Message Passing Interface (MPI). This module assumes MPI is installed -on the system. This class is intended to be called from Exec, -not by general users. -""" - -from jarvis_util.jutil_manager import JutilManager -from jarvis_util.shell.local_exec import LocalExec -from .exec_info import ExecInfo, ExecType - - -class MpiExec(LocalExec): - """ - This class contains methods for executing a command in parallel - using MPI. - """ - - def __init__(self, cmd, exec_info): - """ - Execute a command using MPI - - :param cmd: A command (string) to execute - :param exec_info: Information needed by MPI - """ - - self.cmd = cmd - self.nprocs = exec_info.nprocs - self.ppn = exec_info.ppn - self.hostfile = exec_info.hostfile - self.mpi_env = exec_info.env - super().__init__(self.mpicmd(), - exec_info.mod(env=exec_info.basic_env)) - - def mpicmd(self): - params = [f"mpirun -n {self.nprocs}"] - if self.ppn is not None: - params.append(f"-ppn {self.ppn}") - if len(self.hostfile): - if self.hostfile.is_subset() or self.hostfile.path is None: - params.append(f"--host {','.join(self.hostfile.hosts)}") - else: - params.append(f"--hostfile {self.hostfile.path}") - params += [f"-genv {key}={val}" for key, val in self.mpi_env.items()] - params.append(self.cmd) - cmd = " ".join(params) - jutil = JutilManager.get_instance() - if jutil.debug_mpi_exec: - print(cmd) - return cmd - - -class MpiExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.MPI, **kwargs) diff --git a/ci/jarvis-util/jarvis_util/shell/process.py b/ci/jarvis-util/jarvis_util/shell/process.py deleted file mode 100644 index 7aba2f0fc..000000000 --- a/ci/jarvis-util/jarvis_util/shell/process.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -This module provides various wrappers for methods which manage processes -in the cluster. Examples include killing processes, determining whether -or not a process exists, etc. -""" - -from .exec import Exec - - -class Kill(Exec): - """ - Kill all processes which match the name regex. - """ - - def __init__(self, cmd, exec_info): - """ - Kill all processes which match the name regex. - - :param cmd: A regex for the command to kill - :param exec_info: Info needed to execute the command - """ - super().__init__(f"pkill {cmd}", exec_info) diff --git a/ci/jarvis-util/jarvis_util/shell/pscp.py b/ci/jarvis-util/jarvis_util/shell/pscp.py deleted file mode 100644 index f74da34d0..000000000 --- a/ci/jarvis-util/jarvis_util/shell/pscp.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -This module provides methods to distribute a command among multiple -nodes using SSH. This class is intended to be called from Exec, -not by general users. -""" - -from .scp import Scp -from .exec_info import Executable - - -class Pscp(Executable): - """ - Execute commands on multiple hosts using SSH. - """ - - def __init__(self, paths, exec_info): - """ - Copy files to a set of remote hosts via rsync. - - Case 1: Paths is a single file: - paths = '/tmp/hi.txt' - '/tmp/hi.txt' will be copied to user@host:/tmp/hi.txt - - Case 2: Paths is a list of files: - paths = ['/tmp/hi1.txt', '/tmp/hi2.txt'] - Repeat Case 1 twice. - - Case 3: Paths is a list of tuples of files: - paths = [('/tmp/hi.txt', '/tmp/remote_hi.txt')] - '/tmp/hi.txt' will be copied to user@host:'/tmp/remote_hi.txt' - - :param paths: Either a path to a file, a list of files, or a list of - tuples of files. - :param exec_info: Connection information for SSH - """ - super().__init__() - self.exec_async = exec_info.exec_async - self.hosts = exec_info.hostfile.hosts - self.scp_nodes = [] - self.stdout = {} - self.stderr = {} - self.hosts = exec_info.hostfile.hosts - for host in self.hosts: - ssh_exec_info = exec_info.mod(hostfile=None, - hosts=host, - exec_async=True) - self.scp_nodes.append(Scp(paths, ssh_exec_info)) - if self.exec_async: - self.wait() - - def wait(self): - self.wait_list(self.scp_nodes) - self.per_host_outputs(self.scp_nodes) - self.set_exit_code() - - def set_exit_code(self): - self.set_exit_code_list(self.scp_nodes) - diff --git a/ci/jarvis-util/jarvis_util/shell/pssh_exec.py b/ci/jarvis-util/jarvis_util/shell/pssh_exec.py deleted file mode 100644 index 720869391..000000000 --- a/ci/jarvis-util/jarvis_util/shell/pssh_exec.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -This module provides methods to distribute a command among multiple -nodes using SSH. This class is intended to be called from Exec, -not by general users. -""" - -from .ssh_exec import SshExec -from .local_exec import LocalExec -from .exec_info import ExecInfo, ExecType, Executable - - -class PsshExec(Executable): - """ - Execute commands on multiple hosts using SSH. - """ - - def __init__(self, cmd, exec_info): - """ - Execute commands on multiple hosts. - - :param cmd: A list of commands or a single command string - :param exec_info: Info needed to execute command with SSH - """ - super().__init__() - self.cmd = self.smash_cmd(cmd) - self.exec_async = exec_info.exec_async - self.hosts = exec_info.hostfile.hosts - self.execs_ = [] - self.stdout = {} - self.stderr = {} - if len(self.hosts): - for host in self.hosts: - ssh_exec_info = exec_info.mod(hostfile=None, - hosts=host, - exec_async=True) - self.execs_.append(SshExec(cmd, ssh_exec_info)) - else: - self.execs_.append( - LocalExec(cmd, exec_info)) - return - if not self.exec_async: - self.wait() - - def wait(self): - self.wait_list(self.execs_) - self.per_host_outputs(self.execs_) - self.set_exit_code() - - def set_exit_code(self): - self.set_exit_code_list(self.execs_) - - -class PsshExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.PSSH, **kwargs) - diff --git a/ci/jarvis-util/jarvis_util/shell/scp.py b/ci/jarvis-util/jarvis_util/shell/scp.py deleted file mode 100644 index 8b39301e6..000000000 --- a/ci/jarvis-util/jarvis_util/shell/scp.py +++ /dev/null @@ -1,115 +0,0 @@ -""" -This module provides methods to execute a single command remotely using SSH. -This class is intended to be called from Exec, not by general users. -""" -from .local_exec import LocalExec -from .exec_info import Executable - - -class _Scp(LocalExec): - """ - This class provides methods to copy data over SSH using the "rsync" - command utility in Linux - """ - - def __init__(self, src_path, dst_path, exec_info): - """ - Copy a file or directory from source to destination via rsync - - :param src_path: The path to the file on the host - :param dst_path: The desired file path on the remote host - :param exec_info: Info needed to execute command with SSH - """ - - self.addr = exec_info.hostfile.hosts[0] - self.src_path = src_path - self.dst_path = dst_path - self.user = exec_info.user - self.pkey = exec_info.pkey - self.port = exec_info.port - self.sudo = exec_info.sudo - super().__init__(self.rsync_cmd(src_path, dst_path), - exec_info.mod(env=exec_info.basic_env)) - - def rsync_cmd(self, src_path, dst_path): - lines = ['rsync -ha'] - if self.pkey is not None or self.port is not None: - ssh_lines = ['ssh'] - if self.pkey is not None: - ssh_lines.append(f'-i {self.pkey}') - if self.port is not None: - ssh_lines.append(f'-p {self.port}') - ssh_cmd = ' '.join(ssh_lines) - lines.append(f'-e \'{ssh_cmd}\'') - lines.append(src_path) - if self.user is not None: - lines.append(f'{self.user}@{self.addr}:{dst_path}') - else: - lines.append(f'{self.addr}:{dst_path}') - rsync_cmd = ' '.join(lines) - return rsync_cmd - - -class Scp(Executable): - """ - Secure copy data between two hosts. - """ - - def __init__(self, paths, exec_info): - """ - Copy files via rsync. - - Case 1: Paths is a single file: - paths = '/tmp/hi.txt' - '/tmp/hi.txt' will be copied to user@host:/tmp/hi.txt - - Case 2: Paths is a list of files: - paths = ['/tmp/hi1.txt', '/tmp/hi2.txt'] - Repeat Case 1 twice. - - Case 3: Paths is a list of tuples of files: - paths = [('/tmp/hi.txt', '/tmp/remote_hi.txt')] - '/tmp/hi.txt' will be copied to user@host:'/tmp/remote_hi.txt' - - :param paths: Either a path to a file, a list of files, or a list of - tuples of files. - :param exec_info: Connection information for SSH - """ - - super().__init__() - self.paths = paths - self.exec_info = exec_info - self.scp_nodes = [] - if isinstance(paths, str): - self._exec_single_path(paths) - if isinstance(paths, list): - if len(paths) == 0: - raise Exception('Must have at least one path to scp') - elif isinstance(paths[0], str): - self._exec_many_paths(paths) - elif isinstance(paths[0], tuple): - self._exec_many_paths_tuple(paths) - elif isinstance(paths[0], list): - self._exec_many_paths_tuple(paths) - if not self.exec_info.exec_async: - self.wait() - - def _exec_single_path(self, path): - self.scp_nodes.append(_Scp(path, path, self.exec_info)) - - def _exec_many_paths(self, paths): - for path in paths: - self.scp_nodes.append(_Scp(path, path, self.exec_info)) - - def _exec_many_paths_tuple(self, path_tlist): - for src, dst in path_tlist: - self.scp_nodes.append(_Scp(src, dst, self.exec_info)) - - def wait(self): - self.wait_list(self.scp_nodes) - self.smash_list_outputs(self.scp_nodes) - self.set_exit_code() - return self.exit_code - - def set_exit_code(self): - self.set_exit_code_list(self.scp_nodes) diff --git a/ci/jarvis-util/jarvis_util/shell/ssh_exec.py b/ci/jarvis-util/jarvis_util/shell/ssh_exec.py deleted file mode 100644 index bd9b85135..000000000 --- a/ci/jarvis-util/jarvis_util/shell/ssh_exec.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -This module provides methods to execute a single command remotely using SSH. -This class is intended to be called from Exec, not by general users. -""" -from .local_exec import LocalExec -from .exec_info import ExecInfo, ExecType - - -class SshExec(LocalExec): - """ - This class provides methods to execute a command via SSH. - """ - - def __init__(self, cmd, exec_info): - """ - Execute a command remotely via SSH - - :param cmd: A list of commands or a single command string - :param exec_info: Info needed to execute command with SSH - """ - - cmd = self.smash_cmd(cmd) - self.addr = exec_info.hostfile.hosts[0] - self.user = exec_info.user - self.pkey = exec_info.pkey - self.port = exec_info.port - self.sudo = exec_info.sudo - self.ssh_env = exec_info.env - super().__init__(self.ssh_cmd(cmd), - exec_info.mod(env=exec_info.basic_env)) - - def ssh_cmd(self, cmd): - lines = ['ssh'] - if self.pkey is not None: - lines.append(f'-i {self.pkey}') - if self.port is not None: - lines.append(f'-p {self.port}') - if self.user is not None: - lines.append(f'{self.user}@{self.addr}') - else: - lines.append(f'{self.addr}') - ssh_cmd = ' '.join(lines) - - cmd_lines = [] - if self.ssh_env is not None: - for key, val in self.ssh_env.items(): - cmd_lines.append(f'{key}={val}') - cmd_lines.append(cmd) - env_cmd = ' '.join(cmd_lines) - real_cmd = f'{ssh_cmd} \"{env_cmd}\"' - return real_cmd - - -class SshExecInfo(ExecInfo): - def __init__(self, **kwargs): - super().__init__(exec_type=ExecType.SSH, **kwargs) diff --git a/ci/jarvis-util/jarvis_util/util/argparse.py b/ci/jarvis-util/jarvis_util/util/argparse.py deleted file mode 100644 index e58486be5..000000000 --- a/ci/jarvis-util/jarvis_util/util/argparse.py +++ /dev/null @@ -1,454 +0,0 @@ -""" -This module contains an argument parser which defines -""" - -import sys -import os -from abc import ABC, abstractmethod -import shlex -from tabulate import tabulate - - -class ArgParse(ABC): - """ - A class for parsing command line arguments. - Parsed menu name stored in self.menu_name - Parsed menu arguments stored in self.kwargs - Parsed remaining arguments stored in self.remainder - """ - - def __init__(self, args=None, exit_on_fail=True): - if args is None: - args = sys.argv[1:] - elif isinstance(args, str): - args = shlex.split(args) - args = ' '.join(args) - self.binary_name = os.path.basename(sys.argv[0]) - self.orig_args = shlex.split(args) - self.args = self.orig_args - self.error = None - self.exit_on_fail = exit_on_fail - self.menus = [] - self.vars = {} - self.remainder = None - self.pos_required = False - self.use_remainder = False - - self.menu = None - self.menu_name = None - self.kwargs = {} - self.define_options() - self._parse() - - @abstractmethod - def define_options(self): - """ - User-defined options menu - - :return: - """ - pass - - def process_args(self): - """ - After args have been parsed, can call this function to process - the arguments. Assumes that derived ArgParse class has a function - for each menu option. - - :return: - """ - - func_name = self.menu_name.replace(' ', '_') - func = getattr(self, func_name) - func(self) - - def add_menu(self, name=None, msg=None, - use_remainder=False): - """ - A menu is a container of arguments. - - :param name: The name that appears in the CLI to trigger the menu. - Spaces indicate menu nesting. E.g., 'repo add' will trigger the - menu argparser only if 'repo' and 'add' appear next to each other - in the argument list. - :param msg: The message to print if the user selects an improper menu - in the CLI. - :param use_remainder: Whether or not the menu should store all remaining - arguments for further use later. - :return: - """ - - toks = [] - if name is not None: - toks = name.split() - self.menus.append({ - 'name_str': ' '.join(toks), - 'name': toks, - 'msg': msg, - 'num_required': 0, - 'pos_opts': [], - 'kw_opts': {}, - 'use_remainder': use_remainder - }) - self.pos_required = False - self.menu = self.menus[-1] - - def start_required(self): - """ - Define a set of required positional arguments. - - :return: None - """ - self.pos_required = True - - def end_required(self): - """ - Finish the set of required positional arguments. - - :return: None - """ - self.pos_required = False - - def add_arg(self, - name, - argtype=str, - choices=None, - default=None, - msg=None, - action=None, - aliases=None): - """ - Append an argument to a menu. - Arguments can either be positional or key-value. - Positional arguments do NOT start with a dash - Key-value arguments are separated by dashes - - :param name: The name of the argument. If name starts with a dash, - it will be interpreted as a positional arg - :param argtype: The type of the argument being stored. - :param choices: The set of acceptable inputs as a list - :param default: The default value to store - :param msg: The help message to print if there is a problem - :param action: An action to execute if the argument exists - :param aliases: Other names for the same thing (list) - :return: - """ - - # Add all aliases - if aliases is not None: - for alias in aliases: - if '-' in alias: - self.add_arg(alias, argtype, choices, default, msg, action) - else: - raise f"Can't have a non-keyword alias: {alias}" - # Handle the specific boolean argument case - is_kwarg = '-' in name - if is_kwarg and argtype == bool: - self._add_bool_kw_arg(name, default, msg) - return - # Add general argument - menu = self.menu - arg = { - 'name': name, - 'dict_name': self._get_opt_name(name), - 'type': argtype, - 'choices': choices, - 'default': default, - 'action': action, - 'msg': msg, - 'required': self.pos_required, - 'has_input': True - } - if is_kwarg: - self.pos_required = False - menu['kw_opts'][name] = arg - else: - if self.pos_required: - menu['num_required'] += 1 - menu['pos_opts'].append(arg) - self.kwargs[arg['dict_name']] = default - - def _add_bool_kw_arg(self, - name, - default, - msg=None, - is_other=False, - dict_name=None): - """ - Boolean arguments can be indicated using a +-. - + indicates true, - indicates false. - - :param name: The name of the boolean arg - :param default: Default value of the boolean arg - :param msg: Help message - :param is_other: Indicates this is an alias of the +- syntax. - :param dict_name: Name to make the argument in final kwargs - :return: None - """ - menu = self.menu - if dict_name is None: - dict_name = self._get_opt_name(name, True) - arg = { - 'name': name, - 'dict_name': dict_name, - 'type': bool, - 'choices': None, - 'default': default, - 'action': None, - 'msg': msg, - 'required': False, - 'has_input': not is_other - } - if not is_other: - self._add_bool_kw_arg('--with-' + name.strip('-'), - True, msg, True, dict_name) - self._add_bool_kw_arg('--no-' + name.strip('-'), - False, msg, True, dict_name) - self.pos_required = False - menu['kw_opts'][name] = arg - - def _parse(self): - """ - Parse the CLI arguments. - Will modify self.menu to indicate which menu is used - Will modify self.args to create a key-value store of arguments - - :return: None. - """ - self.menus.sort(key=lambda x: len(x['name']), reverse=True) - self._parse_menu() - - def _parse_menu(self): - """ - Determine which menu is used in the CLI. - - :return: Modify self.menu. No return value. - """ - - self.menu = None - for menu in self.menus: - menu_name = menu['name'] - if len(menu_name) > len(self.args): - continue - if menu_name == self.args[0:len(menu_name)]: - self.menu = menu - break - if self.menu is None: - self._invalid_menu() - self.menu_name = self.menu['name_str'] - self.add_arg('-h', - default=None, - msg='print help message', - action=self._print_help, - aliases=['--help']) - menu_name = self.menu['name'] - self.use_remainder = self.menu['use_remainder'] - self.args = self.args[len(menu_name):] - self._parse_args() - - def _parse_args(self): - self._set_defaults() - i = self._parse_pos_args() - self._parse_kw_args(i) - - def _set_defaults(self): - all_opts = self.menu['pos_opts'] + list(self.menu['kw_opts'].values()) - for opt_info in all_opts: - if opt_info['default'] is None: - continue - self.__dict__[opt_info['dict_name']] = opt_info['default'] - - def _parse_pos_args(self): - """ - Parse positional arguments - Modify the self.kwargs dictionary - - :return: - """ - - i = 0 - args = self.args - menu = self.menu - while i < len(menu['pos_opts']): - # Get the positional arg info - opt_name = menu['pos_opts'][i]['name'] - opt_dict_name = menu['pos_opts'][i]['dict_name'] - opt_type = menu['pos_opts'][i]['type'] - opt_choices = menu['pos_opts'][i]['choices'] - if i >= len(args): - if i >= menu['num_required']: - break - else: - self._missing_positional(opt_name) - - # Get the arg value - arg = args[i] - if arg in menu['kw_opts']: - break - arg = self._convert_opt(opt_name, opt_type, opt_choices, arg) - - # Set the argument - self.kwargs[opt_dict_name] = arg - i += 1 - return i - - def _parse_kw_args(self, i): - """ - Parse key-word arguments. - Modify the self.kwargs dictionary - - :param i: The starting index in the self.args list where kv pairs start - :return: - """ - - menu = self.menu - args = self.args - while i < len(args): - # Get argument name - opt_name = args[i] - if opt_name not in menu['kw_opts']: - if self.use_remainder: - self.remainder = ' '.join(args[i:]) - return - else: - self._invalid_kwarg(opt_name) - - # Get argument type - opt = menu['kw_opts'][opt_name] - opt_has_input = opt['has_input'] - opt_dict_name = opt['dict_name'] - opt_type = opt['type'] - opt_default = opt['default'] - opt_action = opt['action'] - opt_choices = opt['choices'] - if not opt_has_input: - arg = opt_default - i += 1 - elif opt_action is not None: - opt_action() - arg = None - i += 1 - elif self._next_is_kw_value(i): - arg = args[i + 1] - i += 2 - elif opt_default is not None: - arg = opt_default - i += 1 - else: - arg = None - self._invalid_kwarg_default(opt_name) - - # Convert argument to type - arg = self._convert_opt(opt_name, opt_type, opt_choices, arg) - - # Set the argument - self.kwargs[opt_dict_name] = arg - - def _convert_opt(self, opt_name, opt_type, opt_choices, arg): - if opt_type is not None: - # pylint: disable=W0702 - try: - arg = opt_type(arg) - if opt_choices is not None: - if arg not in opt_choices: - self._invalid_choice(opt_name, arg) - except: - self._invalid_type(opt_name, opt_type) - # pylint: enable=W0702 - return arg - - def _next_is_kw_value(self, i): - if i + 1 >= len(self.args): - return False - return self.args[i + 1] not in self.menu['kw_opts'] - - def _get_opt_name(self, opt_name, is_bool_arg=False): - """ - Normalize option names - '-' are converted into '_' - '--with-' and '--no-' are removed - '+' and '-' for boolean args are removed - - :param opt_name: The menu option name - :param is_bool_arg: Whether the arg is a boolean arg - :return: - """ - - if not is_bool_arg: - return opt_name.strip('-').replace('-', '_') - else: - return opt_name.replace('--with-', '', 1)\ - .replace('--no-', '', 1).\ - strip('-').replace('-', '_') - - def _invalid_menu(self): - self._print_error('Could not find a menu') - - def _invalid_choice(self, opt_name, arg): - self._print_menu_error(f'{opt_name}={arg} is not a valid choice') - - def _missing_positional(self, opt_name): - self._print_menu_error(f'{opt_name} was required, but not defined') - - def _invalid_kwarg(self, opt_name): - self._print_menu_error(f'{opt_name} is not a valid key-word argument') - - def _invalid_kwarg_default(self, opt_name): - self._print_menu_error( - f'{opt_name} was not given a value, but requires one') - - def _invalid_type(self, opt_name, opt_type): - self._print_menu_error(f'{opt_name} was not of type {opt_type}') - - def _print_menu_error(self, msg): - self._print_error(f'{self.menu["name_str"]} {msg}') - - def _print_error(self, msg): - print(f'{msg}') - self._print_help() - if self.exit_on_fail: - sys.exit(1) - else: - raise Exception(msg) - - def _print_help(self): - if self.menu is not None: - self._print_menu_help() - else: - self._print_menus() - - def _print_menus(self): - for menu in self.menus: - self.menu = menu - self._print_menu_help(True) - - def _print_menu_help(self, only_usage=False): - pos_args = [] - for arg in self.menu['pos_opts']: - if arg['required']: - pos_args.append(f'[{arg["name"]}]') - else: - pos_args.append(f'[{arg["name"]} (opt)]') - pos_args = ' '.join(pos_args) - menu_str = self.menu['name_str'] - print(f'USAGE: {self.binary_name} {menu_str} {pos_args} ...') - if self.menu['msg'] is not None: - print(self.menu['msg']) - print() - if only_usage: - return - - headers = ['Name', 'Default', 'Type', 'Description'] - table = [] - all_opts = self.menu['pos_opts'] + list(self.menu['kw_opts'].values()) - for arg in all_opts: - default = arg['default'] - if self._is_bool_arg(arg): - default = None - table.append( - [arg['name'], default, arg['type'], arg['msg']]) - print(tabulate(table, headers=headers)) - - def _is_bool_arg(self, arg): - return arg['type'] == bool and (arg['name'].startswith('--with-') or - arg['name'].startswith('--no-')) diff --git a/ci/jarvis-util/jarvis_util/util/expand_env.py b/ci/jarvis-util/jarvis_util/util/expand_env.py deleted file mode 100644 index a9d8aefa7..000000000 --- a/ci/jarvis-util/jarvis_util/util/expand_env.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -This module contains functions for expanding environment variables for -dictionaries -""" - -import os - - -def expand_env(data): - """ - Expand environment variables for dictionaries - - :param data: A dict where strings may contain environment variables to - expand - :return: - """ - if isinstance(data, str): - return os.path.expandvars(data) - if isinstance(data, dict): - for key, val in data.items(): - data[key] = expand_env(val) - if isinstance(data, (list, tuple)): - for i, val in enumerate(data): - data[i] = expand_env(val) - return data diff --git a/ci/jarvis-util/jarvis_util/util/hostfile.py b/ci/jarvis-util/jarvis_util/util/hostfile.py deleted file mode 100644 index 4d22ae233..000000000 --- a/ci/jarvis-util/jarvis_util/util/hostfile.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -This module contains methods for parsing hostfiles and storing hosts -""" - -import os -import socket -import re -import itertools - - -class Hostfile: - """ - Parse a hostfile or store a set of hosts passed in manually. - """ - - def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, - text=None, find_ips=True): - """ - Constructor. Parse hostfile or store existing host list. - - :param hostfile: The path to the hostfile - :param all_hosts: a list of strings representing all hostnames - :param all_hosts_ip: a list of strings representing all host IPs - :param text: Text of a hostfile - :param find_ips: Whether to construct host_ip and all_host_ip fields - """ - self.hosts_ip = [] - self.hosts = [] - self.all_hosts = [] - self.all_hosts_ip = [] - self.path = hostfile - self.find_ips = find_ips - - # Set the host ips directly - if all_hosts_ip is not None: - self.all_hosts_ip = all_hosts_ip - self.hosts_ip = all_hosts_ip - self.find_ips = False - - # Direct constructor - if all_hosts is not None: - self._set_hosts(all_hosts) - - # From hostfile path - elif hostfile is not None: - self._load_hostfile(self.path) - - # From hostfile text - elif text is not None: - self.parse(text) - - # Both hostfile and hosts are None - else: - self._set_hosts(['localhost']) - - def _load_hostfile(self, path): - """ - Expand a hostfile - - :param path: the path to the hostfile - :return: - """ - if not os.path.exists(path): - raise Exception('hostfile not found') - self.path = path - with open(path, 'r', encoding='utf-8') as fp: - text = fp.read() - self.parse(text) - return self - - def parse(self, text): - """ - Parse a hostfile text. - - :param text: Hostfile text - :param set_hosts: Whether or not to set hosts - :return: None - """ - - lines = text.strip().splitlines() - hosts = [] - for line in lines: - self._expand_line(hosts, line) - self._set_hosts(hosts) - - def _expand_line(self, hosts, line): - """ - Will expand brackets in a host declaration. - E.g., host-[0-5,...]-name - - :param hosts: the current set of hosts - :param line: the line to parse - :return: None - """ - toks = re.split(r'[\[\]]', line) - brkts = [tok for i, tok in enumerate(toks) if i % 2 == 1] - num_set = [] - - # Get the expanded set of numbers for each bracket - for i, brkt in enumerate(brkts): - num_set.append([]) - self._expand_set(num_set[-1], brkt) - - # Expand the host string - host_nums = self._product(num_set) - for host_num in host_nums: - host = [] - for i, tok in enumerate(toks): - if i % 2 == 1: - host.append(host_num[int(i/2)]) - else: - host.append(tok) - hosts.append(''.join(host)) - - def _expand_set(self, num_set, brkt): - """ - Expand a bracket set. - The bracket initially has notation: [0-5,0-9,...] - """ - - rngs = brkt.split(',') - for rng in rngs: - self._expand_range(num_set, rng) - - def _expand_range(self, num_set, brkt): - """ - Expand a range. - The range has notation: A-B or A - - :param num_set: the numbers in the range - :param brkt: - :return: - """ - if len(brkt) == 0: - return - if '-' in brkt: - min_max = brkt.split('-') - if len(min_max[0]) == len(min_max[1]): - nums = range(int(min_max[0]), int(min_max[1]) + 1) - num_set += [str(num).zfill(len(min_max[0])) for num in nums] - else: - nums = range(int(min_max[0]), int(min_max[1]) + 1) - num_set += [str(num) for num in nums] - else: - num_set.append(brkt) - - def _product(self, num_set): - """ - Return the cartesian product of the number set - - :param num_set: The numbers to product - :return: - """ - return list(itertools.product(*num_set)) - - def _set_hosts(self, all_hosts): - self.all_hosts = all_hosts - if self.find_ips: - self.all_hosts_ip = [socket.gethostbyname(host) - for host in all_hosts] - self.hosts = self.all_hosts - if self.find_ips: - self.hosts_ip = self.all_hosts_ip - return self - - def subset(self, count): - sub = Hostfile() - sub.path = self.path - sub.all_hosts = self.all_hosts - sub.all_hosts_ip = self.all_hosts_ip - sub.hosts = self.hosts[:count] - sub.hosts_ip = self.hosts_ip[:count] - return sub - - def is_subset(self): - return len(self.hosts) != len(self.all_hosts) - - def save(self, path): - self.all_hosts = self.hosts - self.all_hosts_ip = self.hosts_ip - self.path = path - with open(path, 'w', encoding='utf-8') as fp: - fp.write('\n'.join(self.all_hosts)) - return self - - def ip_list(self): - return self.hosts_ip - - def hostname_list(self): - return self.hosts - - def enumerate(self): - return enumerate(self.hosts) - - def host_str(self, sep=','): - return sep.join(self.hosts) - - def ip_str(self, sep=','): - return sep.join(self.hosts_ip) - - def __len__(self): - return len(self.hosts) - - def __getitem__(self, idx): - return self.hosts[idx] - - def __str__(self): - return str(self.hosts) - - def __repr__(self): - return str(self) - - def __eq__(self, other): - return (self.hosts == other.hosts and - self.all_hosts == other.all_hosts) - diff --git a/ci/jarvis-util/jarvis_util/util/import_all.py b/ci/jarvis-util/jarvis_util/util/import_all.py deleted file mode 100644 index 8a10e6125..000000000 --- a/ci/jarvis-util/jarvis_util/util/import_all.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -This file contains methods to automate large import __init__.py files -""" - -import pathlib -import os - - -def _import_recurse(root_path, root, stmts): - """ - Identify the set of files in the current "root" directory - - :param root_path: The path to the root of the python package - :param root: The current subdirectory of the python package - :param stmts: The current set of import statements - :return: - """ - for file in os.listdir(root): - file = os.path.join(root, file) - if os.path.isfile(file): - file = os.path.relpath(file, root_path) - ext = file.split('.') - if ext[-1] == 'py': - toks = ext[0].split('/') - if toks[-1] == '__init__': - continue - import_stmt = '.'.join(toks) - stmts.append(f'from {import_stmt} import *') - elif os.path.isdir(file): - _import_recurse(root_path, file, stmts) - return stmts - - -def import_all(root_path, root): - """ - Create all import statement to do: from root import *. - - :param root_path: The root of the python repo - :param root: The current directory we are in within the repo - :return: - """ - stmts = [] - _import_recurse(root_path, root, stmts) - return '\"\"\"Import all modules\"\"\"\n' + '\n'.join(stmts) + '\n' - - -def build_global_import_file(root_path, pkg_name): - """ - Build a file to be able to do: from pkg_name import * - - :param root_path: The path to the python package's root directory - :param pkg_name: The name of the python package - :return: - """ - path = os.path.join(root_path, pkg_name) - imports = import_all(root_path, path) - with open(os.path.join(path, '__init__.py'), 'w', - encoding='utf-8') as fp: - fp.write(imports) - - -def build_global_import_from_bin(pkg_name): - """ - Build a file to be able to do: from pkg_name import * - This function is assumed to be called in the "bin" directory - of the main python repo - - :param pkg_name: The name of the python package being built - :return: - """ - root_path = str(pathlib.Path(__file__).parent.parent.parent.resolve()) - build_global_import_file(root_path, pkg_name) diff --git a/ci/jarvis-util/jarvis_util/util/import_mod.py b/ci/jarvis-util/jarvis_util/util/import_mod.py deleted file mode 100644 index 9c55c508d..000000000 --- a/ci/jarvis-util/jarvis_util/util/import_mod.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -This file contains helper methods to load a class dynamically from a file -""" - -import sys - -# NOTE(llogan): To get the path of the directory this file is in, use -# str(pathlib.Path(__file__).parent.resolve()) - - -def load_class(import_str, path, class_name): - """ - Loads a class from a python file. - - :param import_str: A python import string. E.g., for "myrepo.dir1.pkg" - :param path: The absolute path to the directory which contains the - beginning of the import statement. Let's say you have a git repo located - at "/home/hello/myrepo". The git repo has a subdirectory called "myrepo", - so "/home/hello/myrepo/myrepo". In this case, path would be - "/home/hello/myrepo". The import string "myrepo.dir1.pkg" will find - the "myrepo" part of the import string at "/home/hello/myrepo/myrepo". - :param class_name: The name of the class in the file - :return: - """ - sys.path.insert(0, path) - module = __import__(import_str, fromlist=[class_name]) - sys.path.pop(0) - return getattr(module, class_name) diff --git a/ci/jarvis-util/jarvis_util/util/naming.py b/ci/jarvis-util/jarvis_util/util/naming.py deleted file mode 100644 index af6505633..000000000 --- a/ci/jarvis-util/jarvis_util/util/naming.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -This module contains methods to create strings which follow a particular -naming convention. -""" - -import re - - -def to_camel_case(string): - """ - Convert a string in snake case to camel case - - :param string: - :return: - """ - if string is None: - return - words = re.sub(r'(_|-)+', ' ', string).split() - words = [word.capitalize() for word in words] - return ''.join(words) - - -def to_snake_case(string): - """ - Convert a string in CamelCase to snake case - :param string: - :return: - """ - if string is None: - return - words = re.split('([A-Z][a-z0-9_]*)', string) - words = [word for word in words if len(word)] - string = '_'.join(words) - return string.lower() diff --git a/ci/jarvis-util/jarvis_util/util/size_conv.py b/ci/jarvis-util/jarvis_util/util/size_conv.py deleted file mode 100644 index 266959bf0..000000000 --- a/ci/jarvis-util/jarvis_util/util/size_conv.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -This module provides methods to convert a semantic size string to an integer. -""" - - -class SizeConv: - """ - A class which provides methods to convert a semantic size string to an int. - """ - - @staticmethod - def to_int(text): - text = text.lower() - if 'k' in text: - return SizeConv.kb(text) - if 'm' in text: - return SizeConv.mb(text) - if 'g' in text: - return SizeConv.gb(text) - if 't' in text: - return SizeConv.tb(text) - if 'p' in text: - return SizeConv.pb(text) - return int(text) - - @staticmethod - def kb(num): - return int(float(num.split('k')[0]) * (1 << 10)) - - @staticmethod - def mb(num): - return int(float(num.split('m')[0]) * (1 << 20)) - - @staticmethod - def gb(num): - return int(float(num.split('g')[0]) * (1 << 30)) - - @staticmethod - def tb(num): - return int(float(num.split('t')[0]) * (1 << 40)) - - @staticmethod - def pb(num): - return int(float(num.split('p')[0]) * (1 << 50)) diff --git a/ci/jarvis-util/pylintrc b/ci/jarvis-util/pylintrc deleted file mode 100644 index 5371019ba..000000000 --- a/ci/jarvis-util/pylintrc +++ /dev/null @@ -1,429 +0,0 @@ -# This Pylint rcfile contains a best-effort configuration to uphold the -# best-practices and style described in the Google Python style guide: -# https://google.github.io/styleguide/pyguide.html -# -# Its canonical open-source location is: -# https://google.github.io/styleguide/pylintrc - -[MASTER] - -# Files or directories to be skipped. They should be base names, not paths. -ignore=third_party - -# Files or directories matching the regex patterns are skipped. The regex -# matches against base names, not paths. -ignore-patterns= - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=4 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=abstract-method, - apply-builtin, - arguments-differ, - attribute-defined-outside-init, - backtick, - bad-option-value, - basestring-builtin, - buffer-builtin, - c-extension-no-member, - consider-using-enumerate, - cmp-builtin, - cmp-method, - coerce-builtin, - coerce-method, - delslice-method, - div-method, - duplicate-code, - eq-without-hash, - execfile-builtin, - file-builtin, - filter-builtin-not-iterating, - fixme, - getslice-method, - global-statement, - hex-method, - idiv-method, - implicit-str-concat, - import-error, - import-self, - import-star-module-level, - inconsistent-return-statements, - input-builtin, - intern-builtin, - invalid-str-codec, - locally-disabled, - long-builtin, - long-suffix, - map-builtin-not-iterating, - misplaced-comparison-constant, - missing-function-docstring, - metaclass-assignment, - next-method-called, - next-method-defined, - no-absolute-import, - no-else-break, - no-else-continue, - no-else-raise, - no-else-return, - no-init, # added - no-member, - no-name-in-module, - no-self-use, - nonzero-method, - oct-method, - old-division, - old-ne-operator, - old-octal-literal, - old-raise-syntax, - parameter-unpacking, - print-statement, - raising-string, - range-builtin-not-iterating, - raw_input-builtin, - rdiv-method, - reduce-builtin, - relative-import, - reload-builtin, - round-builtin, - setslice-method, - signature-differs, - standarderror-builtin, - suppressed-message, - sys-max-int, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-boolean-expressions, - too-many-branches, - too-many-instance-attributes, - too-many-locals, - too-many-nested-blocks, - too-many-public-methods, - too-many-return-statements, - too-many-statements, - trailing-newlines, - unichr-builtin, - unicode-builtin, - unnecessary-pass, - unpacking-in-except, - useless-else-on-loop, - useless-object-inheritance, - useless-suppression, - using-cmp-argument, - wrong-import-order, - xrange-builtin, - zip-builtin-not-iterating, - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=main,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl - -# Regular expression matching correct function names -function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct constant names -const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct attribute names -attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ - -# Regular expression matching correct argument names -argument-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=^_?[A-Z][a-zA-Z0-9]*$ - -# Regular expression matching correct module names -module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ - -# Regular expression matching correct method names -method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=10 - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt -# lines made too long by directives to pytype. - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=(?x)( - ^\s*(\#\ )??$| - ^\s*(from\s+\S+\s+)?import\s+.+$) - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=yes - -# Maximum number of lines in a module -max-module-lines=99999 - -# String used as indentation unit. The internal Google style guide mandates 2 -# spaces. Google's externaly-published style guide says 4, consistent with -# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google -# projects (like TensorFlow). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=TODO - - -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=yes - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging,absl.logging,tensorflow.io.logging - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec, - sets - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant, absl - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls, - class_ - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=StandardError, - Exception, - BaseException diff --git a/ci/jarvis-util/requirements.txt b/ci/jarvis-util/requirements.txt deleted file mode 100644 index 70849fea6..000000000 --- a/ci/jarvis-util/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -pyyaml -pylint==2.15.0 -coverage==5.5 -# coverage -coverage-lcov==0.2.4 -# coverage-lcov -pytest==6.2.5 -pandas -tabulate \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/argparse_main.py b/ci/jarvis-util/test/unit/argparse_main.py deleted file mode 100644 index 7b636ca60..000000000 --- a/ci/jarvis-util/test/unit/argparse_main.py +++ /dev/null @@ -1,4 +0,0 @@ -from jarvis_util.util.argparse import ArgParse - -if __name__ == 'main': - args = ArgParse() \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/print5s.py b/ci/jarvis-util/test/unit/print5s.py deleted file mode 100644 index a6581b31c..000000000 --- a/ci/jarvis-util/test/unit/print5s.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -NOTE: this is a helper utility for test_local_exec -""" - -import time -import sys - - -for i in range(5): - sys.stdout.write(f"COUT: {i}\n") - sys.stderr.write(f"CERR: {i}\n") - time.sleep(1) \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/printNone.py b/ci/jarvis-util/test/unit/printNone.py deleted file mode 100644 index cf728ec39..000000000 --- a/ci/jarvis-util/test/unit/printNone.py +++ /dev/null @@ -1,4 +0,0 @@ -from jarvis_util.shell.local_exec import LocalExec, LocalExecInfo - -spawn_info = LocalExecInfo(hide_output=True) -LocalExec("echo hello", spawn_info) diff --git a/ci/jarvis-util/test/unit/test_argparse.py b/ci/jarvis-util/test/unit/test_argparse.py deleted file mode 100644 index 3c3f3d32f..000000000 --- a/ci/jarvis-util/test/unit/test_argparse.py +++ /dev/null @@ -1,10 +0,0 @@ -from jarvis_util.util.argparse import ArgParse -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo -import pathlib -from unittest import TestCase - - -class TestArgparse(TestCase): - def test_argparse_main(self): - pass \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/test_hostfile.py b/ci/jarvis-util/test/unit/test_hostfile.py deleted file mode 100644 index 47bd7c853..000000000 --- a/ci/jarvis-util/test/unit/test_hostfile.py +++ /dev/null @@ -1,57 +0,0 @@ -from jarvis_util.util.hostfile import Hostfile -import pathlib -from unittest import TestCase - - -class TestHostfile(TestCase): - def test_no_expand_int(self): - host = Hostfile(text='0', find_ips=False) - self.assertTrue(len(host.hosts) == 1) - self.assertTrue(host.hosts[0] == '0') - - def test_no_expand(self): - host = Hostfile(text='ares-comp-01', find_ips=False) - self.assertTrue(len(host.hosts) == 1) - self.assertTrue(host.hosts[0] == 'ares-comp-01') - - def test_expand_set(self): - host = Hostfile(text='ares-comp-[01-04]-40g', find_ips=False) - self.assertTrue(len(host.hosts) == 4) - self.assertTrue(host.hosts[0] == 'ares-comp-01-40g') - self.assertTrue(host.hosts[1] == 'ares-comp-02-40g') - self.assertTrue(host.hosts[2] == 'ares-comp-03-40g') - self.assertTrue(host.hosts[3] == 'ares-comp-04-40g') - - def test_expand_two_sets(self): - host = Hostfile(text='ares-comp-[01-02]-40g-[01-02]', find_ips=False) - self.assertTrue(len(host.hosts) == 4) - self.assertTrue(host.hosts[0] == 'ares-comp-01-40g-01') - self.assertTrue(host.hosts[1] == 'ares-comp-01-40g-02') - self.assertTrue(host.hosts[2] == 'ares-comp-02-40g-01') - self.assertTrue(host.hosts[3] == 'ares-comp-02-40g-02') - - def test_subset(self): - host = Hostfile(text='ares-comp-[01-02]-40g-[01-02]', find_ips=False) - host = host.subset(3) - self.assertTrue(len(host.hosts) == 3) - self.assertTrue(host.is_subset()) - self.assertTrue(host.hosts[0] == 'ares-comp-01-40g-01') - self.assertTrue(host.hosts[1] == 'ares-comp-01-40g-02') - self.assertTrue(host.hosts[2] == 'ares-comp-02-40g-01') - - def test_read_hostfile(self): - HERE = str(pathlib.Path(__file__).parent.resolve()) - hf = Hostfile(hostfile=f'{HERE}/test_hostfile.txt', find_ips=False) - print(hf.hosts) - self.assertEqual(len(hf), 15) - - def test_save_hostfile(self): - HERE = str(pathlib.Path(__file__).parent.resolve()) - hf = Hostfile(hostfile=f'{HERE}/test_hostfile.txt', find_ips=False) - hf_sub = hf.subset(4) - self.assertEqual(len(hf_sub), 4) - hf_sub.save('/tmp/test_hostfile.txt') - hf_sub_reload = Hostfile(hostfile=f'/tmp/test_hostfile.txt', - find_ips=False) - self.assertEqual(len(hf_sub_reload), 4) - self.assertEqual(hf_sub, hf_sub_reload) diff --git a/ci/jarvis-util/test/unit/test_hostfile.txt b/ci/jarvis-util/test/unit/test_hostfile.txt deleted file mode 100644 index 7ecc164e9..000000000 --- a/ci/jarvis-util/test/unit/test_hostfile.txt +++ /dev/null @@ -1 +0,0 @@ -ares-comp-[01-10,11,12-15]-40g \ No newline at end of file diff --git a/ci/jarvis-util/test/unit/test_local_exec.py b/ci/jarvis-util/test/unit/test_local_exec.py deleted file mode 100644 index c0ced8c3b..000000000 --- a/ci/jarvis-util/test/unit/test_local_exec.py +++ /dev/null @@ -1,65 +0,0 @@ -import pathlib -import os -from jarvis_util.shell.local_exec import LocalExec, LocalExecInfo -from jarvis_util.shell.exec import Exec -from unittest import TestCase - - -class TestLocalExec(TestCase): - def _setup_files(self): - self.stdout = '/tmp/test_out.log' - self.stderr = '/tmp/test_err.log' - try: - os.remove(self.stdout) - except OSError: - pass - try: - os.remove(self.stderr) - except: - pass - - def test_default(self): - ret = Exec("echo hello") - self.assertEqual(ret.exit_code, 0) - self.assertEqual(len(ret.stdout['localhost']), 0) - - def test_pipe_stdout(self): - self._setup_files() - spawn_info = LocalExecInfo(pipe_stdout=self.stdout, - pipe_stderr=self.stderr, - collect_output=True) - ret = Exec("echo hello", spawn_info) - self.assertEqual(ret.stdout['localhost'].strip(), "hello") - self.assertEqual(ret.stderr['localhost'].strip(), "") - self.assertFile(self.stdout, "hello") - self.assertFile(self.stderr, "") - - def test_hide_stdout(self): - HERE = str(pathlib.Path(__file__).parent.resolve()) - PRINTNONE = os.path.join(HERE, 'printNone.py') - spawn_info = LocalExecInfo(collect_output=True) - ret = Exec(f"python3 {PRINTNONE}", spawn_info) - self.assertEqual(ret.stdout['localhost'].strip(), "") - self.assertEqual(ret.stderr['localhost'].strip(), "") - - def test_periodic_print(self): - self._setup_files() - HERE = str(pathlib.Path(__file__).parent.resolve()) - PRINT5s = os.path.join(HERE, 'print5s.py') - ret = Exec(f"python3 {PRINT5s}", - LocalExecInfo(pipe_stdout=self.stdout, - pipe_stderr=self.stderr)) - stdout_data = "\n".join([f"COUT: {i}" for i in range(5)]) - stderr_data = "\n".join([f"CERR: {i}" for i in range(5)]) - self.assertFile(self.stdout, stdout_data) - self.assertFile(self.stderr, stderr_data) - - def assertFile(self, path, data, strip=True): - self.assertTrue(os.path.exists(path)) - with open(path, 'r') as fp: - if strip: - data = data.strip() - file_data = fp.read().strip() - else: - file_data = fp.read() - self.assertEqual(data, file_data) diff --git a/ci/jarvis-util/test/unit/test_system_info.py b/ci/jarvis-util/test/unit/test_system_info.py deleted file mode 100644 index a9b6c9d91..000000000 --- a/ci/jarvis-util/test/unit/test_system_info.py +++ /dev/null @@ -1,137 +0,0 @@ -from jarvis_util.util.argparse import ArgParse -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo -from jarvis_util.util.hostfile import Hostfile -from jarvis_util.introspect.system_info import Lsblk, \ - ListFses, FiInfo, Blkid, ResourceGraph, StorageDeviceType -from jarvis_util.util.size_conv import SizeConv -import pathlib -import itertools -from unittest import TestCase - - -class TestSystemInfo(TestCase): - def test_lsblk(self): - Lsblk(LocalExecInfo(hide_output=True)) - - def test_list_fses(self): - ListFses(LocalExecInfo(hide_output=True)) - - def test_fi_info(self): - FiInfo(LocalExecInfo(hide_output=True)) - - def test_blkid(self): - Blkid(LocalExecInfo(hide_output=True)) - - def test_resource_graph(self): - rg = ResourceGraph() - rg.build(LocalExecInfo(hide_output=True)) - rg.save('/tmp/resource_graph.yaml') - rg.load('/tmp/resource_graph.yaml') - rg.filter_fs(r'/$', '/${USER}', 'NVME') - rg.filter_hosts(Hostfile(), '1gbps') - rg.save('/tmp/resource_graph.yaml') - - def test_custom_resource_graph(self): - rg = ResourceGraph() - all_hosts = ['host1', 'host2', 'host3'] - all_hosts_ip = ['192.168.1.0', '192.168.1.1', '192.168.1.2'] - providers = ['tcp', 'ib', 'roce'] - hosts = Hostfile(all_hosts=all_hosts, all_hosts_ip=all_hosts_ip) - - # Add networks for each node - rg.set_hosts(hosts) - for provider in providers: - rg.add_net(hosts, - provider=provider) - rg.add_net(hosts.subset(1), - provider='uncommon') - - # Add common storage for each node - rg.add_storage(hosts, - device='/dev/sda1', - mount='/', - tran='sata', - rota=True, - size=SizeConv.to_int('10g'), - shared=False) - rg.add_storage(hosts, - device='/dev/sda2', - mount='/mnt/hdd/$USER', - tran='sata', - rota=True, - size=SizeConv.to_int('200g'), - shared=False) - rg.add_storage(hosts, - device='/dev/sdb1', - mount='/mnt/ssd/$USER', - tran='sata', - rota=False, - size=SizeConv.to_int('50g'), - shared=False) - rg.add_storage(hosts, - device='/dev/nvme0n1', - mount='/mnt/nvme/$USER', - tran='nvme', - rota=False, - size=SizeConv.to_int('100g'), - shared=False) - rg.add_storage(hosts.subset(1), - device='/dev/nvme0n2', - mount='/mnt/nvme2/$USER', - tran='nvme', - rota=False, - size=SizeConv.to_int('10g'), - shared=False) - rg.add_storage(hosts, - device='/dev/nvme0n3', - tran='nvme', - rota=False, - size=SizeConv.to_int('100g'), - shared=False) - - # Filter only mounts in '/mnt' - rg.filter_fs('/mnt/*') - - # Apply changes - rg.apply() - - # Find all mounted NVMes - df = rg.find_storage([StorageDeviceType.NVME]) - self.assertTrue(len(df[df.tran == 'nvme']) == 4) - self.assertTrue(len(df[df.tran == 'sata']) == 0) - self.assertTrue(len(df) == 4) - - # Find all mounted & common NVMes and SSDs - df = rg.find_storage([StorageDeviceType.NVME, - StorageDeviceType.SSD], - common=True) - self.assertTrue(len(df[df.tran == 'nvme']) == 3) - self.assertTrue(len(df[df.tran == 'sata']) == 3) - self.assertTrue(len(df) == 6) - - # Select a single nvme per-node - df = rg.find_storage([StorageDeviceType.NVME, - StorageDeviceType.SSD], - common=True, - count_per_node=1) - self.assertTrue(len(df[df.tran == 'nvme']) == 3) - self.assertTrue(len(df[df.tran == 'sata']) == 0) - self.assertTrue(len(df) == 3) - - # Select a single nvme and ssd per-node - df = rg.find_storage([StorageDeviceType.NVME, - StorageDeviceType.SSD], - common=True, - count_per_dev=1) - self.assertTrue(len(df[df.tran == 'nvme']) == 3) - self.assertTrue(len(df[df.tran == 'sata']) == 3) - self.assertTrue(len(df) == 6) - - # Find common networks between hosts - df = rg.find_net_info(hosts) - self.assertTrue(len(df) == 9) - - # Find common tcp networks - df = rg.find_net_info(hosts, providers='tcp') - self.assertTrue(len(df) == 3) diff --git a/ci/lint.sh b/ci/lint.sh index 63ecafd32..9e40817ad 100644 --- a/ci/lint.sh +++ b/ci/lint.sh @@ -1,12 +1,8 @@ #!/bin/bash -HERMES_ROOT=$1 -ADAPTER=${HERMES_ROOT}/adapter +HRUN_ROOT=$1 + +echo "RUNNING CPPLINT" cpplint --recursive \ ---exclude="${HERMES_ROOT}/src/config_server_default.h" \ ---exclude="${HERMES_ROOT}/src/config_client_default.h" \ ---exclude="${ADAPTER}/posix/posix_api.h" \ ---exclude="${ADAPTER}/stdio/stdio_api.h" \ ---exclude="${ADAPTER}/mpiio/mpiio_api.h" \ -"${HERMES_ROOT}/adapter" "${HERMES_ROOT}/benchmarks" "${HERMES_ROOT}/data_stager" \ -"${HERMES_ROOT}/src" "${HERMES_ROOT}/test" +"${HRUN_ROOT}/src" "${HRUN_ROOT}/include" "${HRUN_ROOT}/test" \ +--exclude="${HRUN_ROOT}/test/unit/external" \ No newline at end of file diff --git a/ci/py_hermes_ci/bin/run_test b/ci/py_hermes_ci/bin/run_test index 0427ac0fd..51383167f 100755 --- a/ci/py_hermes_ci/bin/run_test +++ b/ci/py_hermes_ci/bin/run_test @@ -17,10 +17,8 @@ if __name__ == '__main__': address_sanitizer = False # The root of Hermes HERMES_ROOT = str(pathlib.Path(__file__).parent. - parent.parent.parent.resolve()) - ADAPTER_TEST_ROOT = f"{HERMES_ROOT}/adapter/test" - # Ensure that all calls beneath know how to resolve jarvis_util - sys.path.insert(0, f"{HERMES_ROOT}/ci/jarvis-util") + parent.parent.parent.parent.resolve()) + ADAPTER_TEST_ROOT = f"{HERMES_ROOT}/test/unit/hermes_adapters" # Ensure subsequent classes know how to resolve py_hermes_ci package sys.path.insert(0, f"{HERMES_ROOT}/ci/py_hermes_ci") # Choose which unit test file to load @@ -35,9 +33,9 @@ if __name__ == '__main__': elif test_type == 'native': pkg_dir = f"{HERMES_ROOT}/test" elif test_type == 'data_stager': - pkg_dir = f"{HERMES_ROOT}/data_stager/test" + pkg_dir = f"{HERMES_ROOT}data_stager/test" elif test_type == 'kvstore': - pkg_dir = f"{HERMES_ROOT}/adapter/kvstore" + pkg_dir = f"{HERMES_ROOT}/hermes_adapters/kvstore" elif test_type == 'java_wrapper': pkg_dir = f"{HERMES_ROOT}/wrapper/java" else: diff --git a/ci/test_hermes.sh b/ci/test_hermes.sh deleted file mode 100644 index e31c7714e..000000000 --- a/ci/test_hermes.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Run main unit tests -pushd build -ctest -VV -R test_hermes_posix_basic_small -popd build - -# Set proper flags for cmake to find Hermes -INSTALL_PREFIX="${HOME}/install" -export LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LIBRARY_PATH}" -export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}" -export LDFLAGS="-L${INSTALL_PREFIX}/lib:${LDFLAGS}" -export CFLAGS="-I${INSTALL_PREFIX}/include:${CFLAGS}" -export CPATH="${INSTALL_PREFIX}/include:${CPATH}" -export CMAKE_PREFIX_PATH="${INSTALL_PREFIX}:${CMAKE_PREFIX_PATH}" -export CXXFLAGS="-I${INSTALL_PREFIX}/include:${CXXFLAGS}" - -# Run make install unit test -cd ci/external -mkdir build -cd build -cmake ../ -make -j8 - diff --git a/code_generators/hermes_config.py b/code_generators/hermes_config.py deleted file mode 100644 index f6e770ef7..000000000 --- a/code_generators/hermes_config.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -USAGE: - cd code_generators/bin - python3 hermes_config.py - -OUTPUT: - ${HERMES}/src/config_client_default.h (if client) - ${HERMES}/src/config_server_default.h (if server) -""" - -from code_generators.hermes_config.generator import create_config -from code_generators.util.paths import HERMES_ROOT - -create_config( - path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", - var_name="kClientDefaultConfigStr", - config_path=f"{HERMES_ROOT}/src/config_client_default.h", - macro_name="CLIENT" -) - -create_config( - path=f"{HERMES_ROOT}/config/hermes_server_default.yaml", - var_name="kServerDefaultConfigStr", - config_path=f"{HERMES_ROOT}/src/config_server_default.h", - macro_name="SERVER" -) \ No newline at end of file diff --git a/code_generators/code_generators/hermes_config/generator.py b/codegen/codegen/hrun_config/generator.py similarity index 55% rename from code_generators/code_generators/hermes_config/generator.py rename to codegen/codegen/hrun_config/generator.py index a59951050..76ea0562e 100644 --- a/code_generators/code_generators/hermes_config/generator.py +++ b/codegen/codegen/hrun_config/generator.py @@ -1,12 +1,19 @@ import sys, os +def print_macro(path, macro_name): + with open(path) as fp: + lines = fp.read().splitlines() + macro_def = f'#define {macro_name}\\\n' + macro_body = '\\\n'.join(lines) + print(f'{macro_def}{macro_body}') + def create_config(path, var_name, config_path, macro_name): with open(path) as fp: yaml_config_lines = fp.read().splitlines() # Create the hermes config string string_lines = [] - string_lines.append(f"const char* {var_name} = ") + string_lines.append(f"const inline char* {var_name} = ") for line in yaml_config_lines: line = line.replace('\"', '\\\"') line = line.replace('\'', '\\\'') @@ -15,10 +22,10 @@ def create_config(path, var_name, config_path, macro_name): # Create the configuration config_lines = [] - config_lines.append(f"#ifndef HERMES_SRC_CONFIG_{macro_name}_DEFAULT_H_") - config_lines.append(f"#define HERMES_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#ifndef HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#define HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") config_lines += string_lines - config_lines.append(f"#endif // HERMES_SRC_CONFIG_{macro_name}_DEFAULT_H_") + config_lines.append(f"#endif // HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") # Persist config = "\n".join(config_lines) diff --git a/code_generators/code_generators/util/conv.py b/codegen/codegen/util/conv.py similarity index 100% rename from code_generators/code_generators/util/conv.py rename to codegen/codegen/util/conv.py diff --git a/code_generators/code_generators/util/naming.py b/codegen/codegen/util/naming.py similarity index 100% rename from code_generators/code_generators/util/naming.py rename to codegen/codegen/util/naming.py diff --git a/code_generators/code_generators/util/paths.py b/codegen/codegen/util/paths.py similarity index 82% rename from code_generators/code_generators/util/paths.py rename to codegen/codegen/util/paths.py index 2052c3aab..90ce03e41 100644 --- a/code_generators/code_generators/util/paths.py +++ b/codegen/codegen/util/paths.py @@ -8,4 +8,5 @@ def GetHermesRoot(): hermes_path = os.path.dirname(code_generators_path) return hermes_path -HERMES_ROOT=GetHermesRoot() \ No newline at end of file +HERMES_ROOT = GetHermesRoot() +HRUN_ROOT = f'{HERMES_ROOT}/hrun' diff --git a/codegen/hermes_config b/codegen/hermes_config new file mode 100755 index 000000000..54b36cced --- /dev/null +++ b/codegen/hermes_config @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +""" +USAGE: + cd code_generators/bin + python3 hermes_config.py + +OUTPUT: + ${HERMES}/src/config_client_default.h (if client) + ${HERMES}/src/config_server_default.h (if server) +""" + +from codegen.hrun_config.generator import create_config +from codegen.util.paths import HERMES_ROOT + +create_config( + path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", + var_name="kHermesClientDefaultConfigStr", + config_path=f"{HERMES_ROOT}/include/hermes/config_client_default.h", + macro_name="HERMES_CLIENT" +) + +create_config( + path=f"{HERMES_ROOT}/config/hermes_server_default.yaml", + var_name="kHermesServerDefaultConfigStr", + config_path=f"{HERMES_ROOT}/include/hermes/config_server_default.h", + macro_name="HERMES_SERVER" +) diff --git a/codegen/hrun_config b/codegen/hrun_config new file mode 100755 index 000000000..9882e8321 --- /dev/null +++ b/codegen/hrun_config @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +""" +USAGE: + cd codegen/bin + python3 hrun_config.py + +OUTPUT: + ${HRUN}/src/config_client_default.h (if client) + ${HRUN}/src/config_server_default.h (if server) +""" + +from codegen.hrun_config.generator import create_config +from codegen.util.paths import HRUN_ROOT + +create_config( + path=f"{HRUN_ROOT}/config/hrun_client_default.yaml", + var_name="kHrunClientDefaultConfigStr", + config_path=f"{HRUN_ROOT}/include/hrun/config/config_client_default.h", + macro_name="HRUN_CLIENT" +) + +create_config( + path=f"{HRUN_ROOT}/config/hrun_server_default.yaml", + var_name="kHrunServerDefaultConfigStr", + config_path=f"{HRUN_ROOT}/include/hrun/config/config_server_default.h", + macro_name="HRUN_SERVER" +) \ No newline at end of file diff --git a/codegen/make_macro b/codegen/make_macro new file mode 100755 index 000000000..24490af05 --- /dev/null +++ b/codegen/make_macro @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./make_macro [PATH] +""" + +import os +import sys +from codegen.hrun_config.generator import print_macro + +PATH = sys.argv[1] +MACRO_NAME = os.path.basename(PATH).upper().split('.')[0] +print_macro(PATH, MACRO_NAME) diff --git a/codegen/make_task b/codegen/make_task new file mode 100755 index 000000000..6a54f2fb8 --- /dev/null +++ b/codegen/make_task @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./make_task [TASK_ROOT] +""" + +import os +import sys +from codegen.util.paths import HRUN_ROOT + +def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): + with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: + text = fp.read() + text = text.replace('TASK_NAME', TASK_NAME) + rel_path = rel_path.replace('TASK_NAME', TASK_NAME) + with open(f'{TASK_ROOT}/{rel_path}', 'w') as fp: + fp.write(text) + +TASK_ROOT = sys.argv[1] +TASK_NAME = os.path.basename(TASK_ROOT) +TASK_TEMPL_ROOT = f'{HRUN_ROOT}/tasks_required/TASK_NAME' + +os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) +os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/TASK_NAME.cc', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_lib_exec.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_tasks.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_methods.h', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_methods.yaml', TASK_NAME) diff --git a/code_generators/preamble.py b/codegen/preamble.py similarity index 96% rename from code_generators/preamble.py rename to codegen/preamble.py index b55ea20d1..9813b2826 100644 --- a/code_generators/preamble.py +++ b/codegen/preamble.py @@ -2,7 +2,7 @@ Prepends the preabmle to all files in the repo USAGE: - python3 scripts/preamble.py ${HERMES_ROOT} + python3 scripts/preamble.py ${HRUN_ROOT} """ import sys,os,re @@ -36,7 +36,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ """.strip() -labstor_preamble = """ +hrun_preamble = """ /* * Copyright (C) 2022 SCS Lab , * Luke Logan , @@ -69,7 +69,7 @@ def PrependPreamble(path): text = fp.read() text = text.replace(hermes_preamble, "") text = text.replace(preamble2, "") - text = text.replace(labstor_preamble, "") + text = text.replace(hrun_preamble, "") text = re.sub("//\n// Created by [^\n]*\n//\n", "", text) text = text.strip() text = hermes_preamble + "\n\n" + text + "\n" diff --git a/codegen/refresh_methods b/codegen/refresh_methods new file mode 100755 index 000000000..cad1bfac4 --- /dev/null +++ b/codegen/refresh_methods @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./referesh_methods [TASK_DIR] +""" + +import yaml +import os +import sys +from codegen.util.paths import HRUN_ROOT + +def refresh_methods(TASK_ROOT): + if not os.path.exists(f'{TASK_ROOT}/include'): + return + TASK_NAME = os.path.basename(TASK_ROOT) + METHODS_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.h' + METHODS_YAML = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_methods.yaml' + LIB_EXEC_H = f'{TASK_ROOT}/include/{TASK_NAME}/{TASK_NAME}_lib_exec.h' + METHOD_MACRO = f'HRUN_{TASK_NAME.upper()}_METHODS_H_' + LIB_EXEC_MACRO = f'HRUN_{TASK_NAME.upper()}_LIB_EXEC_H_' + + with open(METHODS_YAML) as fp: + methods = yaml.load(fp, Loader=yaml.FullLoader) + if methods is None: + methods = {} + if 'kLast' in methods: + del methods['kLast'] + methods = sorted(methods.items(), key=lambda x: x[1]) + if TASK_NAME != 'hrun_admin': + methods.insert(0, ('kConstruct', -2)) + methods.insert(1, ('kDestruct', -1)) + + # Produce the TASK_NAME_methods.h file + lines = [] + lines += [f'#ifndef {METHOD_MACRO}', + f'#define {METHOD_MACRO}', + '', + '/** The set of methods in the admin task */', + 'struct Method : public TaskMethod {'] + for method_enum_name, method_off in methods: + if method_enum_name == 'kConstruct': + continue + if method_enum_name == 'kDestruct': + continue + lines += f' TASK_METHOD_T {method_enum_name} = kLast + {method_off};', + lines += ['};', '', f'#endif // {METHOD_MACRO}'] + with open(METHODS_H, 'w') as fp: + fp.write('\n'.join(lines)) + + + # Produce the TASK_NAME_lib_exec.h file + lines = [] + lines += [f'#ifndef {LIB_EXEC_MACRO}', + f'#define {LIB_EXEC_MACRO}', + ''] + ## Create the Run method + lines += ['/** Execute a task */', + 'void Run(u32 method, Task *task, RunContext &rctx) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' {method_name}(reinterpret_cast<{task_name} *>(task), rctx);', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the Del method + lines += ['/** Delete a task */', + 'void Del(u32 method, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' HRUN_CLIENT->DelTask(reinterpret_cast<{task_name} *>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the Dup method + lines += ['/** Duplicate a task */', + 'void Dup(u32 method, Task *orig_task, std::vector> &dups) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' hrun::CALL_DUPLICATE(reinterpret_cast<{task_name}*>(orig_task), dups);', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Register a duplicate's output + lines += ['/** Register the duplicate output with the origin task */', + 'void DupEnd(u32 method, u32 replica, Task *orig_task, Task *dup_task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + task_cast = f'reinterpret_cast<{task_name}*>' + lines += [f' case Method::{method_enum_name}: {{', + f' hrun::CALL_DUPLICATE_END(replica, {task_cast}(orig_task), {task_cast}(dup_task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the ReplicateStart method + lines += ['/** Ensure there is space to store replicated outputs */', + 'void ReplicateStart(u32 method, u32 count, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' hrun::CALL_REPLICA_START(count, reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the ReplicateEnd method + lines += ['/** Determine success and handle failures */', + 'void ReplicateEnd(u32 method, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' hrun::CALL_REPLICA_END(reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the SaveStart Method + lines += ['/** Serialize a task when initially pushing into remote */', + 'std::vector SaveStart(u32 method, BinaryOutputArchive &ar, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar << *reinterpret_cast<{task_name}*>(task);', + f' break;', + f' }}'] + lines += [' }'] + lines += [' return ar.Get();'] + lines += ['}'] + + ## Create the LoadStart Method + lines += ['/** Deserialize a task when popping from remote queue */', + 'TaskPointer LoadStart(u32 method, BinaryInputArchive &ar) override {', + ' TaskPointer task_ptr;', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' task_ptr.ptr_ = HRUN_CLIENT->NewEmptyTask<{task_name}>(task_ptr.shm_);', + f' ar >> *reinterpret_cast<{task_name}*>(task_ptr.ptr_);', + f' break;', + f' }}'] + lines += [' }'] + lines += [' return task_ptr;'] + lines += ['}'] + + ## Create the SaveEnd Method + lines += ['/** Serialize a task when returning from remote queue */', + 'std::vector SaveEnd(u32 method, BinaryOutputArchive &ar, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar << *reinterpret_cast<{task_name}*>(task);', + f' break;', + f' }}'] + lines += [' }'] + lines += [' return ar.Get();'] + lines += ['}'] + + ## Create the LoadEnd Method + lines += ['/** Deserialize a task when returning from remote queue */', + 'void LoadEnd(u32 replica, u32 method, BinaryInputArchive &ar, Task *task) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' ar.Deserialize(replica, *reinterpret_cast<{task_name}*>(task));', + f' break;', + f' }}'] + lines += [' }'] + lines += ['}'] + + ## Create the CheckIfConcurrent Method + lines += ['/** Get the grouping of the task */', + 'u32 GetGroup(u32 method, Task *task, hshm::charbuf &group) override {', + ' switch (method) {'] + for method_enum_name, method_off in methods: + method_name = method_enum_name.replace('k', '', 1) + task_name = method_name + "Task" + lines += [f' case Method::{method_enum_name}: {{', + f' return reinterpret_cast<{task_name}*>(task)->GetGroup(group);', + f' }}'] + lines += [' }'] + lines += [' return -1;'] + lines += ['}'] + + ## Finish the file + lines += ['', f'#endif // {METHOD_MACRO}'] + + ## Write TASK_NAME_lib_exec.h + with open(LIB_EXEC_H, 'w') as fp: + fp.write('\n'.join(lines)) + +TASK_DIR = sys.argv[1] +TASK_ROOTS = [os.path.join(TASK_DIR, item) for item in os.listdir(TASK_DIR)] +for TASK_ROOT in TASK_ROOTS: + try: + refresh_methods(TASK_ROOT) + except: + pass diff --git a/codegen/update_task b/codegen/update_task new file mode 100755 index 000000000..3445a239c --- /dev/null +++ b/codegen/update_task @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +""" +USAGE: ./make_task [TASK_ROOT] +""" + +import os +import sys +from codegen.util.paths import HRUN_ROOT + +def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): + with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: + text = fp.read() + text = text.replace('TASK_NAME', TASK_NAME) + rel_path = rel_path.replace('TASK_NAME', TASK_NAME) + with open(f'{TASK_ROOT}/{rel_path}', 'w') as fp: + fp.write(text) + +TASK_ROOT = sys.argv[1] +TASK_NAME = os.path.basename(TASK_ROOT) +TASK_TEMPL_ROOT = f'{HRUN_ROOT}/tasks_required/TASK_NAME + +os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) +os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/CMakeLists.txt', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/TASK_NAME.cc', TASK_NAME) +copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME.h', TASK_NAME) diff --git a/config/hermes_server_default.yaml b/config/hermes_server_default.yaml index 5d88d855b..9741de8dd 100644 --- a/config/hermes_server_default.yaml +++ b/config/hermes_server_default.yaml @@ -82,32 +82,6 @@ devices: # any amount of memory max_memory: 0g -### Define properties of RPCs -rpc: - # A path to a file containing a list of server names, 1 per line. If your - # servers are named according to a pattern (e.g., server-1, server-2, etc.), - # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this - # option is not empty, it will override anything in `rpc_server_base_name`. - host_file: "" - - # Host names can be defined using the following syntax: - # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... - # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... - host_names: ["localhost"] - - # The RPC protocol. This must come from the documentation of the specific RPC - # library in use. - protocol: "ofi+sockets" - - # RPC domain name for verbs transport. Blank for tcp. - domain: "" - - # Desired RPC port number. - port: 8080 - - # The number of handler threads for each RPC server. - num_threads: 4 - ### Define properties of the BORG buffer_organizer: # The number of threads used in the background organization of internal Hermes buffers. @@ -161,17 +135,63 @@ mdm: est_bucket_count: 100000 est_num_traits: 256 -# The shared memory prefix for the hermes shared memory segment. A username -# will be automatically appended. -shmem_name: "/hermes_shm_" - # The interval in milliseconds at which to update the global system view. system_view_state_update_interval_ms: 1000 -### Define the names of the traits to search LD_LIBRARY_PATH for -traits: - - "hermes_posix_io_client" - - "hermes_stdio_io_client" - - "hermes_mpiio_io_client" - - "hermes_example_trait" - - "hermes_prefetcher_trait" +### Runtime orchestration settings +work_orchestrator: + # The number of worker threads to spawn + max_workers: 4 + +### Queue Manager settings +queue_manager: + # The default depth of allocated queues + queue_depth: 256 + # The maximum number of lanes per queue + max_lanes: 16 + # The maximum number of queues + max_queues: 1024 + # The shared memory allocator to use + shm_allocator: kScalablePageAllocator + # The name of the shared memory region to create + shm_name: "hrun_shm" + # The size of the shared memory region to allocate for general data structures + shm_size: 0g + # The size of the shared memory to allocate for data buffers + +### Define properties of RPCs +rpc: + # A path to a file containing a list of server names, 1 per line. If your + # servers are named according to a pattern (e.g., server-1, server-2, etc.), + # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this + # option is not empty, it will override anything in `rpc_server_base_name`. + host_file: "" + + # Host names can be defined using the following syntax: + # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... + # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... + host_names: ["localhost"] + + # The RPC protocol. This must come from the documentation of the specific RPC + # library in use. + protocol: "ofi+sockets" + + # RPC domain name for verbs transport. Blank for tcp. + domain: "" + + # Desired RPC port number. + port: 8080 + + # The number of handler threads for each RPC server. + num_threads: 4 + +### Task Registry +task_registry: [ + 'hermes_mdm', + 'hermes_blob_mdm', + 'hermes_bucket_mdm', + 'hermes_data_op', + 'data_stager', + 'posix_bdev', + 'ram_bdev' +] \ No newline at end of file diff --git a/data_stager/CMakeLists.txt b/data_stager/CMakeLists.txt deleted file mode 100644 index 3fc1b9008..000000000 --- a/data_stager/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -#----------------------------------------------------------------------------- -# Build Stage In -#----------------------------------------------------------------------------- -include_directories( - ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/adapter - ${CMAKE_CURRENT_SOURCE_DIR}) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_RPC_THALLIUM") - -add_library(hermes_data_stager SHARED - ${CMAKE_CURRENT_SOURCE_DIR}/stagers/posix_stager.cc) -add_dependencies(hermes_data_stager hermes_posix_io_client) -target_link_libraries(hermes_data_stager hermes_posix_io_client) - -add_executable(stage_in ${CMAKE_CURRENT_SOURCE_DIR}/stage_in.cc) -add_dependencies(stage_in hermes_data_stager) -target_link_libraries(stage_in hermes_data_stager) - -add_executable(stage_out ${CMAKE_CURRENT_SOURCE_DIR}/stage_out.cc) -add_dependencies(stage_out hermes_data_stager) -target_link_libraries(stage_out hermes_data_stager) - -#----------------------------------------------------------------------------- -# Add file(s) to CMake Install -#----------------------------------------------------------------------------- -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DESTINATION ${HERMES_INSTALL_INCLUDE_DIR} - FILES_MATCHING PATTERN "*.h") - -#----------------------------------------------------------------------------- -# Add Target(s) to CMake Install -#----------------------------------------------------------------------------- -install( - TARGETS - hermes_data_stager - stage_in - stage_out - EXPORT - ${HERMES_EXPORTED_TARGETS} - LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} -) \ No newline at end of file diff --git a/data_stager/data_stager.h b/data_stager/data_stager.h deleted file mode 100644 index 3c9f09622..000000000 --- a/data_stager/data_stager.h +++ /dev/null @@ -1,74 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_DATA_STAGER_STAGE_IN_H_ -#define HERMES_DATA_STAGER_STAGE_IN_H_ - -#include -#include - -#include "hermes_types.h" -#include "posix/posix_io_client.h" - -namespace hermes { - -enum class DataStagerType { - kPosix, - kHdf5 -}; - -class DataStagerTypeConv { - public: - static DataStagerType from_url(const std::string &url) { - if (url.rfind("h5::", 0) != std::string::npos) { - return DataStagerType::kHdf5; - } else { - return DataStagerType::kPosix; - } - } -}; - -class DataStager { - public: - virtual void StageIn(std::string url, hapi::PlacementPolicy dpe) = 0; - virtual void StageIn(std::string url, off_t off, size_t size, - hapi::PlacementPolicy dpe) = 0; - virtual void StageOut(std::string url) = 0; - - protected: - void DivideRange(off_t off, size_t size, off_t &new_off, size_t &new_size) { - int nprocs = 1; - int rank = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - int ranks_for_io = nprocs; - - // Ensure that all ranks perform at least 32MB of I/O - size_t min_io_per_rank = MEGABYTES(32); - if (size < nprocs * min_io_per_rank) { - ranks_for_io = size / min_io_per_rank; - if (size % min_io_per_rank) { - ranks_for_io += 1; - } - } - - new_size = size / ranks_for_io; - new_off = off + new_size * rank; - if (rank == ranks_for_io - 1) { - new_size += size % ranks_for_io; - } - } -}; - -} // namespace hermes - -#endif // HERMES_DATA_STAGER_STAGE_IN_H_ diff --git a/data_stager/stage_in.cc b/data_stager/stage_in.cc deleted file mode 100644 index 75a321d3d..000000000 --- a/data_stager/stage_in.cc +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include "hermes_types.h" -#include -#include "data_stager_factory.h" -#include - -using hermes::api::PlacementPolicyConv; -using hermes::api::PlacementPolicy; -using hermes::DataStager; -using hermes::DataStagerFactory; - -int main(int argc, char **argv) { - if (argc != 5) { - std::cout << "Usage: mpirun -n [nprocs]" << - " ./stage_in [url] [offset] [size] [dpe]" << std::endl; - exit(1); - } - MPI_Init(&argc, &argv); - HERMES->Create(hermes::HermesType::kClient); - std::string url = argv[1]; - off_t off = (off_t) hshm::ConfigParse::ParseSize(argv[2]); - size_t size = hshm::ConfigParse::ParseSize(argv[3]); - PlacementPolicy dpe = PlacementPolicyConv::to_enum(argv[4]); - auto stager = DataStagerFactory::Get(url); - if (size == 0) { - HILOG(kInfo, "Full stage-in") - stager->StageIn(url, dpe); - } else { - HILOG(kInfo, "Partial stage-in") - stager->StageIn(url, off, size, dpe); - } - HERMES->Finalize(); - MPI_Finalize(); -} diff --git a/data_stager/stage_out.cc b/data_stager/stage_out.cc deleted file mode 100644 index b99db66e3..000000000 --- a/data_stager/stage_out.cc +++ /dev/null @@ -1,33 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include -#include "posix/posix_io_client.h" -#include "hermes_types.h" -#include -#include "data_stager_factory.h" - -using hermes::api::PlacementPolicyConv; -using hermes::api::PlacementPolicy; -using hermes::DataStager; -using hermes::DataStagerFactory; - -int main(int argc, char **argv) { - if (argc != 1) { - std::cout << "Usage: ./stage_out" << std::endl; - exit(1); - } - HERMES->Create(hermes::HermesType::kClient); - HERMES->Flush(); - HERMES->Finalize(); - MPI_Finalize(); -} diff --git a/data_stager/stagers/posix_stager.cc b/data_stager/stagers/posix_stager.cc deleted file mode 100644 index d1d1254f5..000000000 --- a/data_stager/stagers/posix_stager.cc +++ /dev/null @@ -1,106 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Distributed under BSD 3-Clause license. * -* Copyright by The HDF Group. * -* Copyright by the Illinois Institute of Technology. * -* All rights reserved. * -* * -* This file is part of Hermes. The full Hermes copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * -* the COPYING file, which can be found at the top directory. If you do not * -* have access to the file, you may request a copy from help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "posix_stager.h" - - -using hermes::api::PlacementPolicyConv; -using hermes::api::PlacementPolicy; -using hermes::adapter::fs::AdapterStat; -using hermes::adapter::fs::IoStatus; -using hermes::adapter::fs::File; -using hermes::adapter::fs::FsIoOptions; -namespace stdfs = std::filesystem; - -namespace hermes { - -void PosixStager::StageIn(std::string path, PlacementPolicy dpe) { - if (stdfs::is_directory(path)) { - DirectoryStageIn(path, dpe); - } else if (stdfs::is_regular_file(path)) { - FileStageIn(path, dpe); - } else { - HELOG(kError, "{} is neither directory or file", path); - } -} - -void PosixStager::FileStageIn(std::string path, PlacementPolicy dpe) { - off_t per_proc_off; - size_t per_proc_size; - DivideRange(0, stdfs::file_size(path), per_proc_off, per_proc_size); - FileStageIn(path, per_proc_off, per_proc_size, dpe); -} - -void PosixStager::DirectoryStageIn(std::string path, PlacementPolicy dpe) { - HILOG(kInfo, "Staging in the directory {}", path) - for (auto &file_path : stdfs::directory_iterator(path)) { - FileStageIn(file_path.path(), dpe); - } -} - -void PosixStager::StageIn(std::string path, off_t off, - size_t size, PlacementPolicy dpe) { - if (stdfs::is_directory(path)) { - HELOG(kError, "Posix stage-in with offset is " - "not supported for directories") - } else if (stdfs::is_regular_file(path)) { - FileStageIn(path, off, size, dpe); - } else { - HELOG(kError, "{} is neither directory or file", path); - } -} - -void PosixStager::FileStageIn(std::string path, - off_t off, size_t size, PlacementPolicy dpe) { - auto fs_api = HERMES_POSIX_FS; - std::vector buf(size); - AdapterStat stat; - bool stat_exists; - IoStatus io_status; - HILOG(kInfo, "Staging in {}", path) - File f = fs_api->Open(stat, path); - fs_api->Read(f, stat, buf.data(), off, size, - io_status, FsIoOptions::WithDpe(dpe)); - fs_api->Close(f, stat_exists); -} - -void PosixStager::StageOut(std::string path) { - if (stdfs::is_regular_file(path)) { - FileStageOut(path); - } else if (stdfs::is_directory(path)) { - DirectoryStageOut(path); - } else { - HILOG(kDebug, "Posix stage-out is neither a file or directory") - } -} - -void PosixStager::FileStageOut(std::string path) { - HILOG(kInfo, "Staging out the file {}", path) - auto fs_api = HERMES_POSIX_FS; - AdapterStat stat; - bool stat_exists; - File f = fs_api->Open(stat, path); - if (!f.status_) { - HELOG(kError, "Couldn't open file: {}", path) - return; - } - fs_api->Close(f, stat_exists); -} - -void PosixStager::DirectoryStageOut(std::string path) { - HILOG(kInfo, "Staging out the directory {}", path) - for (auto &file_path : stdfs::directory_iterator(path)) { - FileStageOut(file_path.path()); - } -} - -} // namespace hermes diff --git a/data_stager/stagers/posix_stager.h b/data_stager/stagers/posix_stager.h deleted file mode 100644 index 234ac8be1..000000000 --- a/data_stager/stagers/posix_stager.h +++ /dev/null @@ -1,40 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_DATA_STAGER_STAGERS_POSIX_STAGE_H -#define HERMES_DATA_STAGER_STAGERS_POSIX_STAGE_H - -#include "../data_stager.h" -#include "posix/posix_fs_api.h" -#include "posix/posix_api.h" - -namespace hermes { - -class PosixStager : public DataStager { - public: - void StageIn(std::string url, hapi::PlacementPolicy dpe) override; - void FileStageIn(std::string path, hapi::PlacementPolicy dpe); - void DirectoryStageIn(std::string path, hapi::PlacementPolicy dpe); - - void StageIn(std::string url, off_t off, size_t size, - hapi::PlacementPolicy dpe) override; - void FileStageIn(std::string path, off_t off, size_t size, - hapi::PlacementPolicy dpe); - - void StageOut(std::string url) override; - void FileStageOut(std::string path); - void DirectoryStageOut(std::string path); -}; - -} // namespace hermes - -#endif // HERMES_DATA_STAGER_STAGERS_POSIX_STAGE_H diff --git a/data_stager/test/CMakeLists.txt b/data_stager/test/CMakeLists.txt deleted file mode 100644 index 20b66297c..000000000 --- a/data_stager/test/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -pytest(data_stager test_data_stager_posix_large_aligned) -pytest(data_stager test_data_stager_posix_large_unaligned) -pytest(data_stager test_data_stager_posix_small_misaligned) -pytest(data_stager test_data_stager_directory) \ No newline at end of file diff --git a/data_stager/test/tests.py b/data_stager/test/tests.py deleted file mode 100644 index 834484f26..000000000 --- a/data_stager/test/tests.py +++ /dev/null @@ -1,78 +0,0 @@ -from py_hermes_ci.test_manager import TestManager -from jarvis_util.shell.exec import Exec -from jarvis_util.shell.local_exec import LocalExecInfo -from jarvis_util.util.size_conv import SizeConv -import os - - -class DataStagerTestManager(TestManager): - def spawn_all_nodes(self): - return self.spawn_info() - - def set_paths(self): - self.TEST_STAGE_IN_CMD = f"{self.CMAKE_BINARY_DIR}/bin/stage_in" - - def test_data_stager_posix_large_aligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_large_aligned.bin' - self._make_file(path, '128m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} 0 128m kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_posix_large_unaligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_large_unaligned.bin' - self._make_file(path, '128m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} .5m 127.5m " - f"kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_posix_small_aligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_small_aligned.bin' - self._make_file(path, '64m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} 0 64m kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_posix_small_misaligned(self): - path = '/tmp/test_hermes/test_data_stager_posix_small_misaligned.bin' - self._make_file(path, '64m') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} .5m 63.5m kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def test_data_stager_directory(self): - os.makedirs('/tmp/test_hermes/a', exist_ok=True) - path = '/tmp/test_hermes/a/test_data_stager_posix_small_misaligned.bin' - self._make_file(path + "0", '64m') - self._make_file(path + "1", '128m') - self._make_file(path + "2", '4k') - self._make_file(path + "3", '16k') - spawn_info = self.spawn_info(nprocs=4, - hermes_conf='hermes_server') - self.start_daemon(spawn_info) - node = Exec(f"{self.TEST_STAGE_IN_CMD} {path} 0 0 kMinimizeIoTime", - spawn_info) - self.stop_daemon(spawn_info) - return node.exit_code - - def _make_file(self, path, size): - size = SizeConv.to_int(size) - with open(path, 'w') as fp: - fp.write("\0" * size) \ No newline at end of file diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in deleted file mode 100644 index 0f405154e..000000000 --- a/doc/Doxyfile.in +++ /dev/null @@ -1,2685 +0,0 @@ -# Doxyfile 1.9.5 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). -# -# Note: -# -# Use doxygen to compare the used configuration file with the template -# configuration file: -# doxygen -x [configFile] -# Use doxygen to compare the used configuration file with the template -# configuration file without replacing the environment variables or CMake type -# replacement variables: -# doxygen -x_noenv [configFile] - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the configuration -# file that follow. The default is UTF-8 which is also the encoding used for all -# text before the first occurrence of this tag. Doxygen uses libiconv (or the -# iconv built into libc) for the transcoding. See -# https://www.gnu.org/software/libiconv/ for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = Hermes - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = 1.0.0-beta - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = "Hierarchical Distributed I/O Buffering System" - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 -# sub-directories (in 2 levels) under the output directory of each output format -# and will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to -# control the number of sub-directories. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# Controls the number of sub-directories that will be created when -# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every -# level increment doubles the number of directories, resulting in 4096 -# directories at level 8 which is the default and also the maximum value. The -# sub-directories are organized in 2 levels, the first level always has a fixed -# numer of 16 directories. -# Minimum value: 0, maximum value: 8, default value: 8. -# This tag requires that the tag CREATE_SUBDIRS is set to YES. - -CREATE_SUBDIRS_LEVEL = 8 - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, -# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English -# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, -# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with -# English messages), Korean, Korean-en (Korean with English messages), Latvian, -# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, -# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, -# Swedish, Turkish, Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = NO - -# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line -# such as -# /*************** -# as being the beginning of a Javadoc-style comment "banner". If set to NO, the -# Javadoc-style will behave just like regular comments and it will not be -# interpreted by doxygen. -# The default value is: NO. - -JAVADOC_BANNER = NO - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = NO - -# By default Python docstrings are displayed as preformatted text and doxygen's -# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the -# doxygen's special commands can be used and the contents of the docstring -# documentation blocks is shown as doxygen documentation. -# The default value is: YES. - -PYTHON_DOCSTRING = YES - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:^^" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". Note that you cannot put \n's in the value part of an alias -# to insert newlines (in the resulting output). You can put ^^ in the value part -# of an alias to insert a newline as if a physical newline was in the original -# file. When you need a literal { or } or , in the value part of an alias you -# have to escape them by means of a backslash (\), this can lead to conflicts -# with the commands \{ and \} for these it is advised to use the version @{ and -# @} or use a double escape (\\{ and \\}) - -ALIASES = "status=A Status object." \ - "ctx{1}=ctx The Context for this \1." \ - "bool{1}=true if \1, otherwise false." - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice -# sources only. Doxygen will then generate output that is more tailored for that -# language. For instance, namespaces will be presented as modules, types will be -# separated into more groups, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_SLICE = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, -# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: -# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser -# tries to guess whether the code is fixed or free formatted code, this is the -# default for Fortran type files). For instance to make doxygen treat .inc files -# as Fortran files (default is PHP), and .f files as C (default is Fortran), -# use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. When specifying no_extension you should add -# * to the FILE_PATTERNS. -# -# Note see also the list of default file extension mappings. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See https://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 5. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 5 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = NO - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use -# during processing. When set to 0 doxygen will based this on the number of -# cores available in the system. You can set it explicitly to a value larger -# than 0 to get more control over the balance between CPU load and processing -# speed. At this moment only the input processing can be done using multiple -# threads. Since this is still an experimental feature the default is set to 1, -# which effectively disables parallel processing. Please report any issues you -# encounter. Generating dot graphs in parallel is controlled by the -# DOT_NUM_THREADS setting. -# Minimum value: 0, maximum value: 32, default value: 1. - -NUM_PROC_THREADS = 1 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual -# methods of a class will be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIV_VIRTUAL = NO - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = NO - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = NO - -# If this flag is set to YES, the name of an unnamed parameter in a declaration -# will be determined by the corresponding definition. By default unnamed -# parameters remain unnamed in the output. -# The default value is: YES. - -RESOLVE_UNNAMED_PARAMS = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# declarations. If set to NO, these declarations will be included in the -# documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = NO - -# With the correct setting of option CASE_SENSE_NAMES doxygen will better be -# able to match the capabilities of the underlying filesystem. In case the -# filesystem is case sensitive (i.e. it supports files in the same directory -# whose names only differ in casing), the option must be set to YES to properly -# deal with such files in case they appear in the input. For filesystems that -# are not case sensitive the option should be set to NO to properly deal with -# output files written for symbols that only differ in casing, such as for two -# classes, one named CLASS and the other named Class, and to also support -# references to files without having to specify the exact matching casing. On -# Windows (including Cygwin) and MacOS, users should typically set this option -# to NO, whereas on Linux or other Unix flavors it should typically be set to -# YES. -# Possible values are: SYSTEM, NO and YES. -# The default value is: SYSTEM. - -CASE_SENSE_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class -# will show which file needs to be included to use the class. -# The default value is: YES. - -SHOW_HEADERFILE = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = NO - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = NO - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. See also section "Changing the -# layout of pages" for information. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = YES - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as documenting some parameters in -# a documented function twice, or documenting parameters that don't exist or -# using markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete -# function parameter documentation. If set to NO, doxygen will accept that some -# parameters have no documentation without warning. -# The default value is: YES. - -WARN_IF_INCOMPLETE_DOC = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong parameter -# documentation, but not about the absence of documentation. If EXTRACT_ALL is -# set to YES then this flag will automatically be disabled. See also -# WARN_IF_INCOMPLETE_DOC -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS -# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but -# at the end of the doxygen process doxygen will return with a non-zero status. -# Possible values are: NO, YES and FAIL_ON_WARNINGS. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# See also: WARN_LINE_FORMAT -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# In the $text part of the WARN_FORMAT command it is possible that a reference -# to a more specific place is given. To make it easier to jump to this place -# (outside of doxygen) the user can define a custom "cut" / "paste" string. -# Example: -# WARN_LINE_FORMAT = "'vi $file +$line'" -# See also: WARN_FORMAT -# The default value is: at line $line of file $file. - -WARN_LINE_FORMAT = "at line $line of file $file" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). In case the file specified cannot be opened for writing the -# warning and error messages are written to standard error. When as file - is -# specified the warning and error messages are written to standard output -# (stdout). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = @PROJECT_SOURCE_DIR@/adapter \ - @PROJECT_SOURCE_DIR@/gotcha_intercept \ - @PROJECT_SOURCE_DIR@/src - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: -# https://www.gnu.org/software/libiconv/) for the list of possible encodings. -# See also: INPUT_FILE_ENCODING -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify -# character encoding on a per file pattern basis. Doxygen will compare the file -# name with each pattern and apply the encoding instead of the default -# INPUT_ENCODING) if there is a match. The character encodings are a list of the -# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding -# "INPUT_ENCODING" for further information on supported encodings. - -INPUT_FILE_ENCODING = - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# Note the list of default checked file patterns might differ from the list of -# default file extension mappings. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, -# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C -# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, -# *.vhdl, *.ucf, *.qsf and *.ice. - -FILE_PATTERNS = *.h \ - *.cc - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = @PROJECT_SOURCE_DIR@/src/stb_ds.h - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# ANamespace::AClass, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that doxygen will use the data processed and written to standard output -# for further processing, therefore nothing else, like debug statements or used -# commands (so in case of a Windows batch file always use @echo OFF), should be -# written to standard output. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = front_page.md - -# The Fortran standard specifies that for fixed formatted Fortran code all -# characters from position 72 are to be considered as comment. A common -# extension is to allow longer lines before the automatic comment starts. The -# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can -# be processed before the automatic comment starts. -# Minimum value: 7, maximum value: 10000, default value: 72. - -FORTRAN_COMMENT_AFTER = 72 - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# entity all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see https://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot of -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output -# should be rendered with a dark or light theme. Default setting AUTO_LIGHT -# enables light output unless the user preference is dark output. Other options -# are DARK to always use dark mode, LIGHT to always use light mode, AUTO_DARK to -# default to dark mode unless the user prefers light mode, and TOGGLE to let the -# user toggle between dark and light mode via a button. -# Possible values are: LIGHT Always generate light output., DARK Always generate -# dark output., AUTO_LIGHT Automatically set the mode according to the user -# preference, use light mode if no preference is set (the default)., AUTO_DARK -# Automatically set the mode according to the user preference, use dark mode if -# no preference is set. and TOGGLE Allow to user to switch between light and -# dark mode via a button.. -# The default value is: AUTO_LIGHT. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE = AUTO_LIGHT - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a color-wheel, see -# https://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use gray-scales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - -# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML -# documentation will contain a main index with vertical navigation menus that -# are dynamically created via JavaScript. If disabled, the navigation index will -# consists of multiple levels of tabs that are statically embedded in every HTML -# page. Disable this option to support browsers that do not have JavaScript, -# like the Qt help browser. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_MENUS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = NO - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 100 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: -# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To -# create a documentation set, doxygen will generate a Makefile in the HTML -# output directory. Running make will produce the docset in that directory and -# running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy -# genXcode/_index.html for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag determines the URL of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDURL = - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# on Windows. In the beginning of 2021 Microsoft took the original page, with -# a.o. the download links, offline the HTML help workshop was already many years -# in maintenance mode). You can download the HTML help workshop from the web -# archives at Installation executable (see: -# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo -# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the main .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location (absolute path -# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to -# run qhelpgenerator on the generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine tune the look of the index (see "Fine-tuning the output"). As an -# example, the default style sheet generated by doxygen has an example that -# shows how to put an image at the root of the tree instead of the PROJECT_NAME. -# Since the tree basically has the same information as the tab index, you could -# consider setting DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = YES - -# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the -# FULL_SIDEBAR option determines if the side bar is limited to only the treeview -# area (value NO) or if it should extend to the full height of the window (value -# YES). Setting this to YES gives a layout similar to -# https://docs.readthedocs.io with more room for contents, but less room for the -# project logo, title, and description. If either GENERATE_TREEVIEW or -# DISABLE_INDEX is set to NO, this option has no effect. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FULL_SIDEBAR = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email -# addresses. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -OBFUSCATE_EMAILS = YES - -# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg -# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see -# https://inkscape.org) to generate formulas as SVG images instead of PNGs for -# the HTML output. These images will generally look nicer at scaled resolutions. -# Possible values are: png (the default) and svg (looks nicer but requires the -# pdf2svg or inkscape tool). -# The default value is: png. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FORMULA_FORMAT = png - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands -# to create new LaTeX commands to be used in formulas as building blocks. See -# the section "Including formulas" for details. - -FORMULA_MACROFILE = - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# https://www.mathjax.org) which uses client side JavaScript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. -# Note that the different versions of MathJax have different requirements with -# regards to the different settings, so it is possible that also other MathJax -# settings have to be changed when switching between the different MathJax -# versions. -# Possible values are: MathJax_2 and MathJax_3. -# The default value is: MathJax_2. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_VERSION = MathJax_2 - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. For more details about the output format see MathJax -# version 2 (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 -# (see: -# http://docs.mathjax.org/en/latest/web/components/output.html). -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility. This is the name for Mathjax version 2, for MathJax version 3 -# this will be translated into chtml), NativeMML (i.e. MathML. Only supported -# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This -# is the name for Mathjax version 3, for MathJax version 2 this will be -# translated into HTML-CSS) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. The default value is: -# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 -# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/ - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# for MathJax version 2 (see -# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# For example for MathJax version 3 (see -# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): -# MATHJAX_EXTENSIONS = ams -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /