Merge branch 'main' into nodegrpPrio

CMakeLists.txt

@@ -155,6 +155,9 @@ option(OMP "Enable OpenMP support in Charm++" OFF)
 option(TCP "use TCP sockets for communication (only for netlrts)" OFF)
 option(PTHREADS "compile with pthreads Converse threads" OFF)
 
+# Slingshot-11 specific, off by default, auto enabled if support is found
+option(CXI "Enable CXI (Cassini/Slingshot-11) extensions to OFI" OFF)
+
 # Advanced options:
 option(OOC "Compile with out of core support" OFF)
 option(SYNCFT "Compile with Charm++ fault tolerance support" OFF)
@@ -211,7 +214,7 @@ option(CUDA                "Build with CUDA support" OFF)
 option(PXSHM               "Build with PXSHM" OFF)
 
 # LRTS PMI options
-set(LRTS_PMI "" CACHE STRING "PMI type for UCX and OFI layers, can be one of simplepmi, slurmpmi, slurmpmi2, ompipmix, or openpmix")
+set(LRTS_PMI "" CACHE STRING "PMI type for UCX and OFI layers, can be one of simplepmi, slurmpmi, slurmpmi2, slurmpmi2cray, ompipmix, or openpmix")
 
 set(EXTRA_OPTS "" CACHE STRING "Extra flags to pass to compilers.")
 
@@ -439,11 +442,10 @@ if(${NETWORK} STREQUAL "ucx")
 endif()
 
 set(CMK_BUILD_OFI 0)
-if(${NETWORK} STREQUAL "ofi")
+if(${NETWORK} MATCHES "ofi")
   set(CMK_BUILD_OFI 1)
 endif()
 
-
 set(CMK_ERROR_CHECKING ${ERROR_CHECKING})
 set(CMK_LOCKLESS_QUEUE 0)
 
@@ -533,6 +535,7 @@ endif()
 include(cmake/detect-features.cmake)
 include(cmake/ci-files.cmake)
 
+
 if(${TARGET} STREQUAL "all-test")
   add_custom_target(all-test ALL COMMAND $(MAKE) -C ${CMAKE_BINARY_DIR}/tmp all-test
         COMMENT "Building all-test. This will take several minutes.")
@@ -708,6 +711,16 @@ if(CUDA)
   endif()
 endif()
 
+if(EXISTS ${CMAKE_SOURCE_DIR}/src/arch/${VDIR}/conv-mach-cxi.sh)
+  configure_file(src/arch/${VDIR}/conv-mach-cxi.sh         include/ COPYONLY)
+  configure_file(src/arch/${VDIR}/conv-mach-cxi.h          include/ COPYONLY)
+endif()
+
+if(EXISTS ${CMAKE_SOURCE_DIR}/src/arch/${VDIR}/conv-mach-slurmpmi2cray.sh)
+  configure_file(src/arch/${VDIR}/conv-mach-slurmpmi2cray.sh         include/ COPYONLY)
+  configure_file(src/arch/${VDIR}/conv-mach-slurmpmi2cray.h          include/ COPYONLY)
+endif()
+
 if(EXISTS ${CMAKE_SOURCE_DIR}/src/arch/${VDIR}/conv-mach-pxshm.sh)
   configure_file(src/arch/${VDIR}/conv-mach-pxshm.sh         include/ COPYONLY)
   configure_file(src/arch/${VDIR}/conv-mach-pxshm.h          include/ COPYONLY)
@@ -832,6 +845,8 @@ if(${NETWORK} MATCHES "ucx" OR ${NETWORK} MATCHES "ofi")
   # file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/include/proc_management/simple_pmi)
 
   set(proc_management-sources
+  src/arch/util/proc_management/runtime-craypmi2.C
+  src/arch/util/proc_management/runtime-craypmi.C
   src/arch/util/proc_management/runtime-pmi.C
   src/arch/util/proc_management/runtime-pmix.C
   src/arch/util/proc_management/runtime-codec.h
@@ -1013,6 +1028,12 @@ else()
   set(CMK_USER_DISABLED_TLS "0")
 endif()
 
+
+if(CMK_BUILD_ON_CXI)
+	set(CMK_CXI 1)
+	set(CXI ON)
+endif()
+
 foreach(l BUILDOPTS CMK_AMPI_WITH_ROMIO CMK_BUILD_PYTHON CMK_CAN_LINK_FORTRAN
         CMK_CHARMDEBUG CMK_COMPILER_KNOWS_TLSDIRECTSEGREFS CMK_HAS_INT16 CMK_HAS_MMAP
         CMK_LIBJPEG CMK_MOD_EXT CMK_SUPPORTS_FSGLOBALS CMK_SUPPORTS_PIPGLOBALS
@@ -1022,7 +1043,7 @@ foreach(l BUILDOPTS CMK_AMPI_WITH_ROMIO CMK_BUILD_PYTHON CMK_CAN_LINK_FORTRAN
         CXX_NO_AS_NEEDED LDXX_WHOLE_ARCHIVE_PRE LDXX_WHOLE_ARCHIVE_POST
         CMK_MACOSX CMK_POST_EXE CMK_SHARED_SUF CMK_USER_SUFFIX OPTS_LD
         CMK_COMPILER_KNOWS_FVISIBILITY CMK_LINKER_KNOWS_UNDEFINED
-        CMK_SUPPORTS_MEMORY_ISOMALLOC CUDA_DIR CMK_USER_DISABLED_TLS)
+        CMK_SUPPORTS_MEMORY_ISOMALLOC CUDA_DIR CMK_USER_DISABLED_TLS CMK_CXI)
     file(APPEND ${optfile_sh} "${l}=\"${${l}}\"\n" )
 endforeach(l)
 
@@ -1043,7 +1064,7 @@ set(optfile_h ${CMAKE_BINARY_DIR}/include/conv-mach-opt.h)
 
 file(WRITE  ${optfile_h} "/* Build-time options header, automatically generated by cmake. */\n")
 foreach(l CMK_AMPI_WITH_ROMIO CMK_OPTIMIZE CMK_AMPI_ONLY CMK_POST_EXE CMK_SHARED_SUF
-        CMK_USER_SUFFIX)
+        CMK_USER_SUFFIX CMK_CXI)
     file(APPEND ${optfile_h} "#define ${l} ${${l}}\n")
 endforeach(l)
 
@@ -1065,20 +1086,20 @@ foreach(l CUDA_DIR BUILD_CUDA CMK_AMPI_WITH_ROMIO CMK_MACOSX CMK_BUILD_PYTHON
         CMK_SMP CMK_SUPPORTS_FSGLOBALS CMK_SUPPORTS_PIPGLOBALS CMK_SUPPORTS_PIEGLOBALS
         CMK_SUPPORTS_SWAPGLOBALS CMK_SUPPORTS_TLSGLOBALS CMK_HAS_OPENMP
         CMK_TRACE_ENABLED CMK_USE_LRTS CMK_VDIR OPTSATBUILDTIME CMK_AMPI_ONLY CMK_WINDOWS
-        CMK_USE_CMA CMK_USER_SUFFIX CMK_CAN_LINK_FORTRAN CMK_SUPPORTS_MEMORY_ISOMALLOC)
+        CMK_USE_CMA CMK_USER_SUFFIX CMK_CAN_LINK_FORTRAN CMK_SUPPORTS_MEMORY_ISOMALLOC
+	CMK_CXI)
     file(APPEND ${optfile_mak} "${l}:=${${l}}\n" )
 endforeach(l)
 
 # Add options
-foreach(opt SMP OMP TCP PTHREADS SYNCFT PXSHM PERSISTENT OOC CUDA PAPI)
+foreach(opt SMP OMP TCP PTHREADS SYNCFT PXSHM PERSISTENT OOC CUDA PAPI CXI)
     if(${opt})
         string(TOLOWER ${opt} optl)
         file(APPEND ${optfile_sh} ". ${CMAKE_BINARY_DIR}/include/conv-mach-${optl}.sh\n")
         file(APPEND ${optfile_h} "#include \"conv-mach-${optl}.h\"\n")
         set(opts_enabled "${opts_enabled}${opt} ")
     endif()
 endforeach()
-
 # Options that need no .h/.sh additions
 foreach(opt TRACING TRACING_COMMTHREAD ERROR_CHECKING LBUSERDATA QLOGIC
   BUILD_SHARED TASK_QUEUE DRONE_MODE LOCKLESS_QUEUE CHARMDEBUG CCS CONTROLPOINT
@@ -1160,6 +1181,7 @@ message("  Charmc flags:    ${MAIN_CFLAGS}")
 message("  Enabled options: ${opts_enabled}")
 message("==============================")
 
+
 # Make symlinks
 if(CMK_WINDOWS)
   # add_executable(createlink src/arch/win/createlink.cpp)

README.md

@@ -165,7 +165,7 @@ for more information:
 * `regularpages` - On Cray systems, Charm++'s default is to use `hugepages`. This
     option disables `hugepages`, and uses regular `malloc` for messages.
 * `persistent` - On Cray systems, this option enables use of persistent mode for
-    communication.
+* `cxi` - On HPE Slingshot-11 systems, this option enables use of Cassini extensions for communication.  Usually autodetected and enabled where available.
 * `pxshm` - Use POSIX Shared Memory for communication between Charm++ processes
     within a shared-memory host.
 * `syncft` - Enable in-memory fault tolerance support in Charm++.

buildcmake

@@ -92,7 +92,8 @@ function parse_triplet() {
 parse_triplet
 
 extra_triplet_opts=${extra_triplet_opts//-/ }
-
+opt_build_ofi=0
+opt_cxi=0
 opt_ampi_error_checking=0
 opt_ampi_mpich_tests=0
 opt_ampi_only=0
@@ -175,6 +176,9 @@ function parse_platform_compilers() {
     cuda)
       opt_cuda=1
       ;;
+    cxi)
+      opt_cxi=1
+      ;;
     smp)
       opt_smp=1
       ;;
@@ -193,7 +197,7 @@ function parse_platform_compilers() {
     pxshm)
       opt_pxshm=1
       ;;
-    simplepmi|slurmpmi|slurmpmi2|ompipmix|openpmix)
+    simplepmi|slurmpmi|slurmpmi2|ompipmix|openpmix|slurmpmi2cray)
       opt_lrts_pmi="$arg"
       ;;
     persistent)
@@ -492,18 +496,22 @@ fi
 
 # Append certain features and compilers to the end of $builddir
 builddir_extra=""
-for flag in opt_omp opt_smp opt_tcp opt_pthreads opt_pxshm opt_syncft opt_ooc opt_persistent opt_cuda; do
+for flag in opt_omp opt_smp opt_tcp opt_pthreads opt_pxshm opt_syncft opt_ooc opt_persistent opt_cuda opt_cxi; do
   [[ $flag -eq 1 ]] && builddir_extra+="-${flag/opt_/}"
 done
 
 for c in "$opt_lrts_pmi" "${opt_compiler[@]}"; do
   [[ -n "$c" ]] && builddir_extra+="-$c"
 done
 
-# Use slurmpmi2 by default for ofi builds on Cray platforms (e.g. Cray
-# Shasta/EX) since it matches the interface of cray-pmi
+if [[ "$actual_triplet" = ofi* ]]; then
+    opt_build_ofi=1
+fi
+
+# Use slurmpmi2cray by default for ofi builds on Cray platforms (e.g. Cray
+# Shasta/EX) to access cray extentions to PMI from cray-pmi
 if [[ "$actual_triplet" = ofi-cray* && -z "$opt_lrts_pmi" ]]; then
-    opt_lrts_pmi="slurmpmi2"
+    opt_lrts_pmi="slurmpmi2cray"
 fi
 
 # Default to using simplepmi on non-Cray OFI platforms
@@ -673,6 +681,8 @@ CC=$opt_CC CXX=$opt_CXX FC=$opt_FC cmake "$my_srcdir" \
   -DTCP="$opt_tcp" \
   -DTRACING="$opt_tracing" \
   -DTRACING_COMMTHREAD="$opt_tracing_commthread" \
+  -DCXI="$opt_cxi" \
+  -DCMK_BUILD_OFI="$opt_build_ofi" \
   -DZLIB="$opt_zlib"
 
 

cmake/detect-features-c.cmake

@@ -367,9 +367,43 @@ int main() {
 }
 " CMK_BALANCED_INJECTION_API)
 
-if(${CMK_BUILD_OFI} EQUAL 1)
+if(${NETWORK} STREQUAL "ofi" OR ${NETWORK} STREQUAL "ofi-crayshasta" OR ${NETWORK} STREQUAL "ofi-linux")
+# assume HPC installation
+include(CMakePrintHelpers)
+  find_package(EnvModules REQUIRED)
+  find_package(PkgConfig REQUIRED)
+  if(EnvModules_FOUND)
+  #at least get libfabric loaded if it isn't already
+	env_module(load libfabric)
+  endif()
   set(tmp ${CMAKE_REQUIRED_LIBRARIES})
-  set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES} -lfabric")
+  if(${PkgConfig_FOUND})
+# this is tortured because pkg-config and cmake are infuriating
+	set(myconfigCommand "pkg-config")
+	set(myargs1 "libfabric")
+	set(myargs2 "--libs")
+	execute_process(COMMAND ${myconfigCommand} ${myargs1} ${myargs2}
+	                OUTPUT_VARIABLE PKG_CONFIG_OFI_LIBS_OUTPUT
+			RESULT_VARIABLE PKG_CONFIG_OFI_LIBS_RESULT
+			WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+			ERROR_VARIABLE thiserror
+			)
+	string(STRIP ${PKG_CONFIG_OFI_LIBS_OUTPUT} CMAKE_PKG_CONFIG_OFI_LIBS)
+	set(myargs2 "--cflags")
+	execute_process(COMMAND ${myconfigCommand} ${myargs1} ${myargs2}
+			OUTPUT_VARIABLE PKG_CONFIG_OFI_CFLAGS_OUTPUT
+			RESULT_VARIABLE PKG_CONFIG_OFI_CFLAGS_RESULT
+			WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+			ERROR_VARIABLE $thaterror
+			)
+	string(STRIP ${PKG_CONFIG_OFI_CFLAGS_OUTPUT} CMAKE_PKG_CONFIG_OFI_CFLAGS)
+	set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${CMAKE_PKG_CONFIG_OFI_CFLAGS}")
+	set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES} ${CMAKE_PKG_CONFIG_OFI_LIBS}")
+  else()
+	message(WARNING "cmake can't find pkg-config")
+	set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
+  endif()
+
   check_c_source_compiles("
     #include <rdma/fabric.h>
     int main(int argc, char **argv)
@@ -379,9 +413,13 @@ if(${CMK_BUILD_OFI} EQUAL 1)
       return 0;
     }
   " CMK_BUILD_ON_OFI)
-  set(CMAKE_REQUIRED_LIBRARIES ${tmp})
+
   if("${CMK_BUILD_ON_OFI}" STREQUAL "")
-    message(FATAL_ERROR "Unable to build ofi.")
+    message(FATAL_ERROR "Unable to build ofi with FLAGS ${CMAKE_REQUIRED_FLAGS} LIBS ${CMAKE_REQUIRED_LIBRARIES}  for network ${NETWORK}.")
+    set(CMAKE_REQUIRED_LIBRARIES ${tmp})
+  else()
+#    set(CMAKE_EXTRA_INCLUDE_FILES "{CMAKE_EXTRA_INCLUDE_FILES} CMAKE_PKG_CONFIG_OFI_CFLAGS")
+#    set(CMK_LIBDIR "{CMK_LIBS} CMAKE_PKG_CONFIG_OFI_LIBS")
   endif()
 endif()
 

cmake/detect-features-cxx.cmake

@@ -165,3 +165,55 @@ int main() {
 
 # Unset workaround from above
 set(CMAKE_REQUIRED_FLAGS "")
+if(${NETWORK} STREQUAL "ofi" OR ${NETWORK} STREQUAL "ofi-crayshasta" OR ${NETWORK} STREQUAL "ofi-linux")
+# assume HPC installation with LMOD
+  include(CMakePrintHelpers)
+  find_package(EnvModules REQUIRED)
+  find_package(PkgConfig REQUIRED)
+  if(EnvModules_FOUND)
+  # we need libfabric cray-libpals and cray-pmi
+	env_module(load libfabric)
+	env_module(load cray-libpals)
+	env_module(load cray-pmi)
+  endif()
+  set(tmp ${CMAKE_REQUIRED_LIBRARIES})
+  if(${PkgConfig_FOUND})
+# this is tortured because pkg-config and cmake are infuriating
+	set(myconfigCommand "pkg-config")
+	set(myargs1 "libfabric")
+	set(myargs2 "--libs")
+	execute_process(COMMAND ${myconfigCommand} ${myargs1} ${myargs2}
+	                OUTPUT_VARIABLE PKG_CONFIG_OFI_LIBS_OUTPUT
+			RESULT_VARIABLE PKG_CONFIG_OFI_LIBS_RESULT
+			WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+			ERROR_VARIABLE thiserror
+			)
+	string(STRIP ${PKG_CONFIG_OFI_LIBS_OUTPUT} CMAKE_PKG_CONFIG_OFI_LIBS)
+	set(myargs2 "--cflags")
+	execute_process(COMMAND ${myconfigCommand} ${myargs1} ${myargs2}
+			OUTPUT_VARIABLE PKG_CONFIG_OFI_CFLAGS_OUTPUT
+			RESULT_VARIABLE PKG_CONFIG_OFI_CFLAGS_RESULT
+			WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+			ERROR_VARIABLE $thaterror
+			)
+	string(STRIP ${PKG_CONFIG_OFI_CFLAGS_OUTPUT} CMAKE_PKG_CONFIG_OFI_CFLAGS)
+	set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${CMAKE_PKG_CONFIG_OFI_CFLAGS}")
+	set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES} ${CMAKE_PKG_CONFIG_OFI_LIBS}")
+  else()
+	message(WARNING "cmake can't find pkg-config")
+	set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
+  endif()
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${CMAKE_PKG_CONFIG_OFI_CFLAGS}")
+    set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES} ${CMAKE_PKG_CONFIG_OFI_LIBS}")
+
+check_cxx_source_compiles("
+    #include <rdma/fabric.h>
+    #include <rdma/fi_cxi_ext.h>
+    int main(int argc, char **argv)
+    {
+      struct fi_info *providers;
+      int ret = fi_getinfo(FI_VERSION(1,0), NULL, NULL, 0ULL, NULL, &providers);
+      return 0;
+    }
+  " CMK_BUILD_ON_CXI)
+endif()