Skip to content

Building and Running on Stampede Phi's

Jump to bottom Edit New page
Cameron Smith edited this page Jul 11, 2016 · 19 revisions

Table of Contents

The following instructions are for execution on the Stampede Phi/MIC only; the host processors are not used.

See the Stampede wiki page for system details https://portal.xsede.org/tacc-stampede

Download 

git clone [email protected]:cwsmith/phastaChef.git
cd phastaChef
git clone [email protected]:PHASTA/phasta.git
git clone [email protected]:SCOREC/core.git

wget www.scorec.rpi.edu/~cwsmith/phastaChefTests.tar.gz .
tar xzf phastaChefTests.tar.gz # use for CASES path below
export CASES=$PWD/phastaChefTests

wget https://www.scorec.rpi.edu/pumi/pumi_test_meshes.tar.gz
tar xzf pumi_test_meshes.tar.gz # use for MESHES path below
export MESHES=$PWD/meshes

Build

Setup the environment 

module swap mvapich2 impi
module load cmake
module use /work/02422/cwsmith/software/modules
compiler=intel15.0.2_impi5.0.2
module load parmetis/4.0.3/$compiler zoltan/3.81/$compiler bzip2/1.0.6/$compiler

Create a toolchain file

Create 'StampedePhi.cmake' with the following contents 

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_C_COMPILER mpicc)
set(CMAKE_CXX_COMPILER mpicxx)
set(CMAKE_Fortran_COMPILER ifort)
set(CMAKE_AR /usr/bin/ar CACHE STRING "" FORCE)

set(intel "/opt/apps/intel/15/composer_xe_2015.2.164")
set(impi "/opt/apps/intel15/impi/5.0.2.044")
set(compilermic "${intel}/compiler/lib/mic")
set(cxxflags "-mmic -Wl,-rpath-link=${compilermic} -Wl,--as-needed ")
set(opt "-opt-assume-safe-padding -opt-streaming-stores always -opt-streaming-cache-evict=0")

set(CMAKE_C_FLAGS -mmic CACHE STRING "" FORCE)
set(CMAKE_CXX_FLAGS "${cxxflags}" CACHE STRING "" FORCE)
set(CMAKE_Fortran_FLAGS -mmic CACHE STRING "" FORCE)

set(CMAKE_C_FLAGS_RELEASE "-mmic ${opt} " CACHE STRING "" FORCE)
set(CMAKE_CXX_FLAGS_RELEASE "-mmic ${opt} " CACHE STRING "" FORCE)
set(CMAKE_Fortran_FLAGS_RELEASE "-mmic ${opt} -align array64byte " CACHE STRING "" FORCE)

set(CMAKE_FIND_ROOT_PATH 
    /usr/linux-k1om-4.7/linux-k1om/usr/lib64/
    /usr/linux-k1om-4.7/linux-k1om/usr/
    ${impi}/mic/lib/release_mt
    ${impi}/mic/lib/
    ${impi}/mic/)

Create a run script

Place the following in ~/bin/micrun.sh: 

#!/bin/bash -x
processes=$2
exe=$3

#OPTIONAL - the next two lines are experimental
$HOME/bin/build_romio_host.sh
export ROMIO_HINTS="./romio_hints"

#REQUIRED
export MIC_MY_NSLOTS=$processes
export MIC_PPN=$processes
export myparam=${@:4}
ibrun.symm -m "$exe $myparam"

Create a ROMIO hints script

Place the following in ~/bin/build_romio_host.sh 

#!/bin/bash
#Set the number of aggregators per node
agg_per_node=1
#Get the hostlist from Slurm
declare -a hostlist=(`scontrol show hostname $SLURM_NODELIST `)
#Get the domainname
domainname=`hostname -d`
#Put all the hosts in a bash array
hosts=""
#Add the domain name ang the number of aggregators per node 
# (The full domain name must be added to match the hostname ROMIO
#  receives from MPI_Get_processor_name )
for host in "${hostlist[@]}"; do
   hosts="$hosts ${host}-mic0.${domainname}:${agg_per_node}"
done
echo $hosts
romio_hints="cb_config_list $hosts"
#Create the hint file 
echo $romio_hints > ./romio_hints
echo "Create romio_hints file with the following hint:"
echo "$romio_hints"
echo " "

Create a build directory 

mkdir buildPhi
cd buildPhi

Run CMake 

cmake \
-DCMAKE_TOOLCHAIN_FILE=../StampedePhi.cmake \
-DCMAKE_BUILD_TYPE=Release \
-DPHASTA_INCOMPRESSIBLE=OFF \
-DPHASTA_COMPRESSIBLE=ON \
-DPHASTA_USE_SVLS=OFF \
-DPHASTA_USE_PETSC=OFF \
-DPHASTA_TESTING=ON \
-DCASES=$CASES \
\
-DPCU_COMPRESS=ON \
-DENABLE_ZOLTAN=ON \
-DIS_TESTING=True \
-DMESHES=$MESHES \
\
-DMPIRUN=~/bin/micrun.sh \
..

Build 

make VERBOSE=1 -j4

Run Tests

Start an interactive session: 

idev -m 30
and once it starts run the tests 
ctest

TAU

With a few changes the above process supports building and running with TAU.

CMake Toolchain 

set(CMAKE_SYSTEM_NAME Linux)

set(CMAKE_C_COMPILER /opt/apps/intel15/impi_5_0/tau/2.24.1/x86_64/bin/tau_cc.sh)
set(CMAKE_CXX_COMPILER /opt/apps/intel15/impi_5_0/tau/2.24.1/x86_64/bin/tau_cxx.sh)
set(CMAKE_Fortran_COMPILER /opt/apps/intel15/impi_5_0/tau/2.24.1/x86_64/bin/tau_f90.sh)
set(CMAKE_AR /usr/bin/ar CACHE STRING "" FORCE)

set(intel "/opt/apps/intel/15/composer_xe_2015.2.164")
set(impi "/opt/apps/intel15/impi/5.0.2.044")
set(compilermic "${intel}/compiler/lib/mic")
set(cxxflags "-mmic -Wl,-rpath-link=${compilermic} -Wl,--as-needed ")
set(opt "-opt-assume-safe-padding -opt-streaming-stores -opt-streaming-cache-evict=0")

set(CMAKE_C_FLAGS "-optCompInst -mmic -std=c99" CACHE STRING "" FORCE)
set(CMAKE_CXX_FLAGS "-optCompInst ${cxxflags}" CACHE STRING "" FORCE)
set(CMAKE_Fortran_FLAGS "-optCompInst -mmic" CACHE STRING "" FORCE)

set(CMAKE_C_FLAGS_RELEASE "-optCompInst -mmic ${opt} " CACHE STRING "" FORCE)
set(CMAKE_CXX_FLAGS_RELEASE "-optCompInst -mmic ${opt} " CACHE STRING "" FORCE)
set(CMAKE_Fortran_FLAGS_RELEASE "-optCompInst -mmic ${opt} -align array64byte " CACHE STRING "" FORCE)

set(CMAKE_FIND_ROOT_PATH 
    /usr/linux-k1om-4.7/linux-k1om/usr/lib64/
    /usr/linux-k1om-4.7/linux-k1om/usr/
    ${impi}/mic/lib/release_mt
    ${impi}/mic/lib/
    ${impi}/mic/)

LINK_DIRECTORIES(/opt/apps/papi/5.3.0/k1om/lib)
LINK_DIRECTORIES(/home1/apps/intel/itac/9.0.3.049/mic/slib)
LINK_DIRECTORIES(/opt/apps/intel15/impi_5_0/tau/2.24.1/mic_linux/lib)
LINK_DIRECTORIES(/opt/apps/intel/15/composer_xe_2015.2.164/mkl/lib/mic)
LINK_DIRECTORIES(/opt/apps/intel/15/composer_xe_2015.2.164/tbb/lib/mic)
LINK_DIRECTORIES(/opt/intel/mic/coi/device-linux-release/lib)
LINK_DIRECTORIES(/opt/intel/mic/myo/lib:/opt/apps/intel/15/composer_xe_2015.2.164/compiler/lib/mic)

CMake Command 

cmake \
-DCMAKE_TOOLCHAIN_FILE=../StampedePhiTau.cmake \
-DCMAKE_BUILD_TYPE=Release \
-DPHASTA_INCOMPRESSIBLE=OFF \
-DPHASTA_COMPRESSIBLE=ON \
-DPHASTA_USE_SVLS=OFF \
-DPHASTA_USE_PETSC=OFF \
-DPHASTA_TESTING=ON \
-DCASES=/path/to/phastaChefTests \
\
-DPCU_COMPRESS=ON \
-DENABLE_ZOLTAN=ON \
-DIS_TESTING=True \
-DMESHES=$d/meshes \
\
-DMPIRUN=~/bin/micrun.sh \
..

Example Run Script

Below is a run script using MPI-IO ROMIO hints to run compressible phasta on a two process case. 

d=/work/02422/cwsmith/phastaShared/phastaChef/buildPhiTau/
export PHASTA_CONFIG=$d
export TAU_METRICS=LINUXTIMERS
export TAU_PROFILE_FORMAT="merged"
echo 0 > 2-procs_case/numstart.dat
/home1/02422/cwsmith/bin/micrun.sh 2 $d/phasta/bin/phastaC.exe
Add a custom footer
Add a custom sidebar
Clone this wiki locally