#!/bin/bash
# BEGIN_ICS_COPYRIGHT8 ****************************************
#
# Copyright (c) 2015, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of Intel Corporation nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# END_ICS_COPYRIGHT8 ****************************************
# [ICS VERSION STRING: unknown]
temp1="$(mktemp)"
temp2="$(mktemp)"
tmpdir=$PWD/tmp
trap "rm -f $temp1 $temp2; exit 1" SIGHUP SIGTERM SIGINT
trap "rm -f $temp1 $temp2" EXIT
BATCH_SIZE=${BATCH_SIZE:-18} # now many hosts per mpi job
MIN_BATCH_SIZE=${MIN_BATCH_SIZE:-2} # now many hosts per mpi job
# start the given run_* script as a series of small BATCH_SIZE node jobs.
# This approach limits the impact should a few hosts
# crash and also allows for quicker job startup.
# jobs are nohup'ed so they can run for long duration in the background
Usage() {
echo "Usage: ./run_batch_script [-e] run_script [args]" >&2
echo " or" >&2
echo " ./run_batch_script --help" >&2
echo " -e - force an even number of hosts in final batch by skipping the last one" >&2
echo " run_script - a run_* script from this directory" >&2
echo " args - arguments for the run_script" >&2
echo " if the 1st arg is 'NP' it will be replaced with the process count" >&2
echo >&2
echo " This will build a set of mpi_hosts.# files with no" >&2
echo " more than BATCH_SIZE hosts each. If -e is specified and an odd number of" >&2
echo " hosts appear in mpi_hosts, the last one is skipped" >&2
echo " Each run_script MPI job will have its output saved to a corresponding" >&2
echo " ./tmp/nohup.#.out file" >&2
echo >&2
echo " This may only used for scripts which do use MPI_HOSTS." >&2
echo " To run run_cabletest use run_batch_cabletest" >&2
echo "Environment:" >&2
echo " MPI_HOSTS - mpi_hosts file to use, default is mpi_hosts" >&2
echo " BATCH_SIZE - max hosts per MPI job, default is 18, if -e must be even" >&2
echo " MIN_BATCH_SIZE - min hosts per MPI job, default is 2, if -e must be even" >&2
echo "Examples:" >&2
echo " ./run_batch_script run_deviation NP ff" >&2
echo " BATCH_SIZE=2 MPI_HOSTS=good ./run_batch_script run_lat2" >&2
echo " BATCH_SIZE=16 MPI_HOSTS=good ./run_batch_script run_deviation ff" >&2
echo " MIN_BATCH_SIZE=16 BATCH_SIZE=16 ./run_batch_script run_hpl2 16" >&2
exit 2
}
if [ x"$1" = x"--help" ]
then
Usage
fi
even=n
if [ x"$1" = x"-e" ]
then
even=y
shift
fi
if [ $# -lt 1 ]
then
Usage
fi
script=$1
shift
if [ $(basename $script) = 'run_cabletest' ]
then
echo "To execute run_cabletest use run_batch_cabletest instead" >&2
exit 1
fi
np=n
if [ x"$1" = x"NP" ]
then
np=y
shift
fi
export MPI_HOSTS=${MPI_HOSTS:-mpi_hosts}
if [ ! -e $MPI_HOSTS ]
then
echo "MPI_HOSTS='$MPI_HOSTS': Not Found" >&2
Usage
fi
if ! [ $BATCH_SIZE -gt 0 ] 2>/dev/null
then
echo "Invalid BATCH_SIZE='$BATCH_SIZE': must be a number > 0" >&2
Usage
fi
if [ $even = y -a $(($BATCH_SIZE % 2)) -eq 1 ]
then
echo "Invalid BATCH_SIZE='$BATCH_SIZE': must be an even number" >&2
Usage
fi
if ! [ $MIN_BATCH_SIZE -gt 0 ] 2>/dev/null
then
echo "Invalid MIN_BATCH_SIZE='$MIN_BATCH_SIZE': must be a number > 0" >&2
Usage
fi
if [ $even = y -a $(($MIN_BATCH_SIZE % 2)) -eq 1 ]
then
echo "Invalid MIN_BATCH_SIZE='$MIN_BATCH_SIZE': must be an even number" >&2
Usage
fi
if [ $MIN_BATCH_SIZE -gt $BATCH_SIZE ] 2>/dev/null
then
echo "Invalid MIN_BATCH_SIZE='$MIN_BATCH_SIZE': must be < BATCH_SIZE of $BATCH_SIZE" >&2
Usage
fi
# ignore blank and comment lines in mpi_hosts
cat $MPI_HOSTS|egrep -v '^[[:space:]]*#'|egrep -v '^[[:space:]]*$' > $temp1
echo "Starting $script jobs using $MPI_HOSTS: $(cat $temp1|wc -l) hosts"
echo " Max batch size $BATCH_SIZE, Min batch size $MIN_BATCH_SIZE"
#set -x
mkdir -p $tmpdir
loop=1
while [ $(cat $temp1|wc -l) -ge $MIN_BATCH_SIZE ]
do
head -$BATCH_SIZE $temp1 > $tmpdir/mpi_hosts.$loop
if [ $even = y -a $(($(cat $tmpdir/mpi_hosts.$loop|wc -l) % 2)) -eq 1 ]
then
# need an even number, drop the last one
echo "Skipping odd host:" `tail -1 $temp1`
head -$(($(cat $temp1|wc -l) - 1)) $temp1 > $tmpdir/mpi_hosts.$loop
fi
tail -n +$(($BATCH_SIZE + 1)) $temp1 > $temp2; mv $temp2 $temp1
NP=$(cat $tmpdir/mpi_hosts.$loop|wc -l)
echo "starting $script job $loop on $NP hosts"
[ $np = n ] && NP= # don't insert process count
export MPI_HOSTS=$tmpdir/mpi_hosts.$loop
export LOGSUFFIX=".$loop"
nohup ./$script $NP "$@" > $tmpdir/nohup.$loop.out 2>&1 &
loop=$(($loop + 1))
#echo "Enter to Continue"
#read trash
done
if [ $(cat $temp1|wc -l) -gt 0 ]
then
echo "Skipping odd hosts:" `cat $temp1`
fi
rm -f $temp1 $temp2
echo "Started $(($loop - 1 )) MPI $script jobs"