#!/bin/sh
# BEGIN_ICS_COPYRIGHT8 ****************************************
#
# Copyright (c) 2015, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of Intel Corporation nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# END_ICS_COPYRIGHT8 ****************************************
# [ICS VERSION STRING: unknown]
# This is a template to allow generic applications to be run in a test and tune
# mode where statistics and output is desirable to be collected per run for
# later analysis
# edit the assignments below as need to define the application to run
# you may also want to use runmyapp (or a similar script) as the actual
# APP_CMD in which case you can put the actual command to run and any setup for
# the command in runmyapp on each compute node see runmyapp for an example
# of running HPL
# Also you can create a file mpi_chassis listing all the relevant SilverStorm
# chassis IP addresses/names for which switch statistics should be cleared and
# fetched for this run.
# If its not present the default fast fabric chassis file will be used.
# BASE PATH TO MPI EXECUTABLES: To use an alternate location,
# either edit this line or set MPICH_PREFIX in your environment.
# default to MPI used for build
MPICH_PREFIX=${MPICH_PREFIX:-`cat .prefix 2>/dev/null`}
# MPI_HOSTS - mpi_hosts file to use (default is $PWD/mpi_hosts)
MPI_HOSTS=${MPI_HOSTS:-$PWD/mpi_hosts}
if [ $(echo $MPI_HOSTS|cut -c1) != '/' ]
then
MPI_HOSTS="$PWD/$MPI_HOSTS"
fi
# temporary file name to use when collecting statistics
# This file will be created on each node in the job
# and deleted again after the data has been collected.
OPASTATS_FILE="$(mktemp --tmpdir 'opastats.XXXXXXXX')"
trap "rm -f $OPASTATS_FILE; exit 1" SIGHUP SIGTERM SIGINT
trap "rm -f $OPASTATS_FILE" EXIT
# the application to run along with any necessary arguments
# APP_CMD can be ./runmyapp or the actual application name.
# When using runmyapp, you can edit runmyapp with the
# correct application name and arguments. runmyapp gives you an opportunity to
# control the environment such as library paths, core file options, etc
APP_CMD="./runmyapp"
#APP_CMD="PMB2.2.1/SRC_PMB/PMB-MPI1"
# Application executable name, used to kill application before each run to
# ensure no remnants from a previous run exist
# also used as part of log directory name and included in runlog
# if this is not set, basename of APP_CMD is used instead
APP=xhpl
# the param file to use (only applicable to MPICH 1 runs)
PARAM_FILE=
# any relevant input data files which should be saved per run as part of record
# this should be files which you plan to change between runs as you tune
INPUT_FILES=hpl/bin/ICS.$(uname -s).$(./get_mpi_cc.sh)/HPL.dat
# any output files generated which should be saved per run as part of record
OUTPUT_FILES=
# an single output file which should be monitored and displayed when running
# for example: hpccoutf.txt
# the file specified must appear on the head node (possibly in a shared
# filesystem which other ranks in the job will write to)
# leave this blank if application primary progress/status output is to screen
MONITOR_FILE=
# any relevant code files which should be saved per run as part of record
# this should be files which might change between runs as you tune
# can also include any scripts or Makefiles which might be used to build the
# app or directory of any source code for app which might be changing
#CODE_FILES=hpl/bin/ICS.Linux.gcc/xhpl
CODE_FILES=
loop=n
if [ x"$1" = "x-l" ]
then
loop=y
shift
fi
if [ -z "$1" ]
then
echo " Usage: run_app [-l] number_processes ['reason for run']"
echo " For example: ./run_app 2 'run with 2t problem size'"
echo "number_processes may be 'all' in which case one rank will be started for"
echo "every process in the mpi_hosts file"
exit 1
else
NUM_PROCESSES=$1
fi
if [ -s mpi_chassis ]
then
chassis_arg="-F mpi_chassis"
else
chassis_arg=
fi
if [ "x$APP" = "x" ]
then
# get command name part of APP_CMD (eg. remove arguments)
app=`(set $APP_CMD; echo $1)`
APP=`basename $app`
fi
CURTIME=`date +%d%b%y%H%M%S`
dir=$PWD/logs/$APP.$CURTIME$LOGSUFFIX
LOGFILE=$dir/log
mkdir -p $dir
# keep some temp local files with info about the job
echo "$CURTIME" > curtime
echo "$APP" > app
echo "$APP.$CURTIME$LOGSUFFIX - $2" >> logs/runlog
echo " Running MPI app ($APP_CMD) with $NUM_PROCESSES processes"
echo " logfile $LOGFILE"
> $LOGFILE
. ./prepare_run
# clear output
# remove debug output file
echo " Removing debug and core files..."
rm -f MVICH-DBG.* pid core.* $OUTPUT_FILES $MONITOR_FILE
opacmdall -p -f ff.hosts -u `whoami` "cd $PWD;"'rm -f MVICH-DBG.* core.*' > /dev/null 2>&1
if [ x"$MONITOR_FILE" != x ]
then
> $MONITOR_FILE
fi
# copy data before we start the run
echo " Copying files to $dir..."
cp -r $PARAM_FILE $INPUT_FILES $CODE_FILES $MPI_HOSTS run_app runmyapp $dir
if [ -f mpi_chassis ]
then
cp mpi_chassis $dir
fi
# kill any existing instances of app, just to be safe
echo " Killing $APP..."
pkill $APP >/dev/null 2>&1
opacmdall -p -f ff.hosts -u `whoami` "pkill $APP 2>/dev/null" >/dev/null 2>&1
# fetch host stats before
echo " Fetching host stats..."
opacmdall -p -f ff.hosts -u `whoami` "opainfo -o stats > $OPASTATS_FILE 2>/dev/null" > /dev/null
opauploadall -p -f ff.hosts -u `whoami` -d $dir $OPASTATS_FILE opastats-before > /dev/null
opacmdall -p -f ff.hosts -u `whoami` "rm -f $OPASTATS_FILE" > /dev/null
# fetch switch stats
echo " Fetching switch stats..."
opacmdall -C $chassis_arg "ismPortStats -noprompt" > $dir/switch_stats-before
#iba_report -C -o none
echo "$msg" | tee -i -a $LOGFILE
if [ $loop = y ]
then
echo " Looping $APP_CMD ..."
echo " Looping $APP_CMD ..." >> $LOGFILE
else
echo " Running $APP_CMD ..."
echo " Running $APP_CMD ..." >> $LOGFILE
fi
show_mpi_hosts | tee -a $LOGFILE
while true
do
date
date >> $LOGFILE
echo "$MPI_RUN_CMD $APP_CMD" >> $LOGFILE
if [ x"$MONITOR_FILE" = x ]
then
# simple run, nothing to monitor
set -x
$MPI_RUN_CMD $APP_CMD 2>&1|tee -i -a $LOGFILE
res=$?
set +x
else
# run both a tail and app so we can see the results while it runs and
# stop the tail when app finishes
( $MPI_RUN_CMD $APP_CMD 2>&1|tee -i -a $LOGFILE ) &
pid=$!
set +x
echo "$pid" > pid
tail -f $MONITOR_FILE &
tail=$!
while true
do
if kill -s SIGCONT $pid >/dev/null 2>&1 # see if pid still running
then
sleep 10
else
break
fi
done
kill -9 $tail
wait $pid
res=$?
if [ $res = 127 ]
then
echo "Unable to get exit status of application"
res=0
fi
wait
fi
# check for both exit status and typical app failure messages
if [ ! $res ] || cat $LOGFILE $MONITOR_FILE| egrep 'ERROR|disconnect|truncated|IRECV|STOP|FAIL'
then
echo " $APP FAILED!! See log file"
break
fi
if [ $loop = y ]
then
echo "########################################### " >> $LOGFILE
# uncomment this if global id is part of MVICH-DBG filename
#rm -f MVICH-DBG.*
#opacmdall -p -f ff.hosts -u `whoami` "cd $PWD;"'rm -f MVICH-DBG.*' > /dev/null 2>&1
sleep 5 # a little delay so user could cntl-C us
else
break
fi
done
echo " Copying files to $dir..."
cp MVICH-DBG.* core.* $OUTPUT_FILES $MONITOR_FILE $dir
# fetch host stats
echo " Fetching host stats..."
opacmdall -p -f ff.hosts -u `whoami` "opainfo -o stats > $OPASTATS_FILE 2>/dev/null" > /dev/null
opauploadall -p -f ff.hosts -u `whoami` -d $dir $OPASTATS_FILE opastats-after > /dev/null
opacmdall -p -f ff.hosts -u `whoami` "rm -f $OPASTATS_FILE" > /dev/null
# fetch switch stats
echo " Fetching switch stats..."
opacmdall -C $chassis_arg "ismPortStats -noprompt" > $dir/switch_stats-after
#echo " Fetching fabric errors..."
#iba_report -o errors > $dir/iba_report_errors
echo " Fetching debug and core to $dir..."
opacmdall -p -f ff.hosts -u `whoami` "cd $PWD;"'rm -f debug.tgz; tar cfz debug.tgz MVICH-DBG.* core.*' > /dev/null 2>/dev/null
opauploadall -p -f ff.hosts -u `whoami` -d $dir $PWD/debug.tgz debug.tgz > /dev/null 2>/dev/null
rm -f pid curtime app ff.hosts