#!/bin/bash

# BEGIN_ICS_COPYRIGHT8 ****************************************

# 

# Copyright (c) 2015, Intel Corporation

# 

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are met:

# 

#     * Redistributions of source code must retain the above copyright notice,

#       this list of conditions and the following disclaimer.

#     * Redistributions in binary form must reproduce the above copyright

#       notice, this list of conditions and the following disclaimer in the

#       documentation and/or other materials provided with the distribution.

#     * Neither the name of Intel Corporation nor the names of its contributors

#       may be used to endorse or promote products derived from this software

#       without specific prior written permission.

# 

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE

# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# 

# END_ICS_COPYRIGHT8   ****************************************

# [ICS VERSION STRING: unknown]

temp1="$(mktemp)"

temp2="$(mktemp)"

tmpdir=$PWD/tmp

trap "rm -f $temp1 $temp2; exit 1" SIGHUP SIGTERM SIGINT

trap "rm -f $temp1 $temp2" EXIT

BATCH_SIZE=${BATCH_SIZE:-18}	# now many hosts per mpi job

MIN_BATCH_SIZE=${MIN_BATCH_SIZE:-2}	# now many hosts per mpi job

# start the given run_* script as a series of small BATCH_SIZE node jobs.

# This approach limits the impact should a few hosts

# crash and also allows for quicker job startup.

# jobs are nohup'ed so they can run for long duration in the background

Usage() {

	echo "Usage: ./run_batch_script [-e] run_script [args]" >&2

	echo "           or" >&2

	echo "       ./run_batch_script --help" >&2

	echo "    -e - force an even number of hosts in final batch by skipping the last one" >&2

  	echo "    run_script - a run_* script from this directory" >&2

  	echo "    args - arguments for the run_script" >&2

  	echo "           if the 1st arg is 'NP' it will be replaced with the process count" >&2

	echo >&2

	echo "    This will build a set of mpi_hosts.# files with no" >&2

	echo "    more than BATCH_SIZE hosts each. If -e is specified and an odd number of" >&2

  	echo "    hosts appear in mpi_hosts, the last one is skipped" >&2

	echo "    Each run_script MPI job will have its output saved to a corresponding" >&2

	echo "    ./tmp/nohup.#.out file" >&2

	echo >&2

	echo "    This may only used for scripts which do use MPI_HOSTS." >&2

	echo "    To run run_cabletest use run_batch_cabletest" >&2

	echo "Environment:" >&2

	echo "    MPI_HOSTS - mpi_hosts file to use, default is mpi_hosts" >&2

	echo "    BATCH_SIZE - max hosts per MPI job, default is 18, if -e must be even" >&2

	echo "    MIN_BATCH_SIZE - min hosts per MPI job, default is 2, if -e must be even" >&2

	echo "Examples:" >&2

	echo "    ./run_batch_script run_deviation NP ff" >&2

	echo "    BATCH_SIZE=2 MPI_HOSTS=good ./run_batch_script run_lat2" >&2

	echo "    BATCH_SIZE=16 MPI_HOSTS=good ./run_batch_script run_deviation ff" >&2

	echo "    MIN_BATCH_SIZE=16 BATCH_SIZE=16 ./run_batch_script run_hpl2 16" >&2

	exit 2

}

if [ x"$1" = x"--help" ]

then

	Usage

fi

even=n

if [ x"$1" = x"-e" ]

then

	even=y

	shift

fi

if [ $# -lt 1 ]

then

	Usage

fi

script=$1

shift

if [ $(basename $script) = 'run_cabletest' ]

then

	echo "To execute run_cabletest use run_batch_cabletest instead" >&2

	exit 1

fi

np=n

if [ x"$1" = x"NP" ]

then

	np=y

	shift

fi

export MPI_HOSTS=${MPI_HOSTS:-mpi_hosts}

if [ ! -e $MPI_HOSTS ]

then

	echo "MPI_HOSTS='$MPI_HOSTS': Not Found" >&2

	Usage

fi

if ! [ $BATCH_SIZE -gt 0 ] 2>/dev/null

then

	echo "Invalid BATCH_SIZE='$BATCH_SIZE': must be a number > 0" >&2

	Usage

fi

if [ $even = y -a $(($BATCH_SIZE % 2)) -eq 1 ]

then

	echo "Invalid BATCH_SIZE='$BATCH_SIZE': must be an even number" >&2

	Usage

fi

if ! [ $MIN_BATCH_SIZE -gt 0 ] 2>/dev/null

then

	echo "Invalid MIN_BATCH_SIZE='$MIN_BATCH_SIZE': must be a number > 0" >&2

	Usage

fi

if [ $even = y -a $(($MIN_BATCH_SIZE % 2)) -eq 1 ]

then

	echo "Invalid MIN_BATCH_SIZE='$MIN_BATCH_SIZE': must be an even number" >&2

	Usage

fi

if [ $MIN_BATCH_SIZE -gt $BATCH_SIZE ] 2>/dev/null

then

	echo "Invalid MIN_BATCH_SIZE='$MIN_BATCH_SIZE': must be < BATCH_SIZE of $BATCH_SIZE" >&2

	Usage

fi

# ignore blank and comment lines in mpi_hosts

cat $MPI_HOSTS|egrep -v '^[[:space:]]*#'|egrep -v '^[[:space:]]*$' > $temp1

echo "Starting $script jobs using $MPI_HOSTS: $(cat $temp1|wc -l) hosts"

echo "  Max batch size $BATCH_SIZE, Min batch size $MIN_BATCH_SIZE"

#set -x

mkdir -p $tmpdir

loop=1

while [ $(cat $temp1|wc -l) -ge $MIN_BATCH_SIZE ]

do

	head -$BATCH_SIZE $temp1 > $tmpdir/mpi_hosts.$loop

	if [ $even = y -a $(($(cat $tmpdir/mpi_hosts.$loop|wc -l) % 2)) -eq 1 ]

	then

		# need an even number, drop the last one

		echo "Skipping odd host:" `tail -1 $temp1`

		head -$(($(cat $temp1|wc -l) - 1)) $temp1 > $tmpdir/mpi_hosts.$loop

	fi

	tail -n +$(($BATCH_SIZE + 1)) $temp1 > $temp2; mv $temp2 $temp1

	NP=$(cat $tmpdir/mpi_hosts.$loop|wc -l)

	echo "starting $script job $loop on $NP hosts"

	[ $np = n ] && NP=	# don't insert process count

	export MPI_HOSTS=$tmpdir/mpi_hosts.$loop

	export LOGSUFFIX=".$loop"

	nohup ./$script $NP "$@" > $tmpdir/nohup.$loop.out 2>&1 &

	loop=$(($loop + 1))

	#echo "Enter to Continue"

	#read trash

done

if [ $(cat $temp1|wc -l) -gt 0 ]

then

	echo "Skipping odd hosts:" `cat $temp1`

fi

rm -f $temp1 $temp2

echo "Started $(($loop - 1 )) MPI $script jobs"