#!/bin/bash

# BEGIN_ICS_COPYRIGHT8 ****************************************

# 

# Copyright (c) 2015, Intel Corporation

# 

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are met:

# 

#     * Redistributions of source code must retain the above copyright notice,

#       this list of conditions and the following disclaimer.

#     * Redistributions in binary form must reproduce the above copyright

#       notice, this list of conditions and the following disclaimer in the

#       documentation and/or other materials provided with the distribution.

#     * Neither the name of Intel Corporation nor the names of its contributors

#       may be used to endorse or promote products derived from this software

#       without specific prior written permission.

# 

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE

# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# 

# END_ICS_COPYRIGHT8   ****************************************

# [ICS VERSION STRING: unknown]

# Might want to tune this later.

NBs=168

# Generate some reasonable defaults based on the current node.

default_numcores=$(lscpu | grep "^CPU(s):" | awk '{print $2}')

bytesram=$(cat /proc/meminfo | grep MemTotal | awk '{print $2}')

default_megsram=$(( $bytesram / 1024 ))

default_pressure=0.3

# If we have an mpi_hosts file, use that for the default # of hosts.

if [ -e /usr/src/opa/mpi_hosts ]; then 

	default_numnodes=$(sort -u mpi_hosts | wc -l)

else

	default_numnodes=1

fi

numcores=$default_numcores

megsram=$default_megsram

pressure=$default_pressure

numnodes=$default_numnodes

USAGE() {

	echo >&2

	echo >&2 "Usage:"

	echo >&2 "	./hpl_dat_gen.sh [-2|-1][-n <nodes>][-c <cores>][-r <ram>][-p <pressure>][-d]"

	echo >&2 

	echo >&2 "  This tool generates an HPL.dat file and copies it to the appropriate hpl"

	echo >&2 "  executable directory (hpl-2.2/bin/ICS.Linux.*)"

	echo >&2 

	echo >&2 "  The HPL.dat file can be tuned for different numbers of nodes, cores per node,"

	echo >&2 "  and the approximate amount of RAM per node to use. The script chooses"

	echo >&2 "  defaults based on the characteristics of the node the script is run on, but"

	echo >&2 "  these values can be overridden either by command line options or by"

	echo >&2 "  prompting the user to enter new values."

	echo >&2 

	echo >&2 "Options:"

	echo >&2 

	echo >&2 "	--nodes <nodes>"

	echo >&2 "	-n <nodes>      # of nodes in the fabric."

	echo >&2 

	echo >&2 "	--cores <cores>" 

	echo >&2 "	-c <cores>      # of cores per node."

	echo >&2 

	echo >&2 "	--ram <ram>"

	echo >&2 "	-r <ram>        Amount of RAM per node in MB."

	echo >&2 

	echo >&2 "	--pressure 
"

	echo >&2 "	-p 
          Total memory pressure, expressed as a number"

	echo >&2 "	                between 0.1 and 0.9. This value scales the"

	echo >&2 "	                problem size against the amount of RAM available."

	echo >&2 "	                (For example, a pressure of 0.5 will use about"

	echo >&2 "	                half the available RAM on each node.)"

	echo >&2 

	echo >&2 "	--use-defaults"

	echo >&2 "	-d              Use defaults for unspecified values."

	echo >&2 "	                (otherwise prompt for the missing values.)"

	echo >&2 

	echo >&2 "Defaults for this system are:"

	echo >&2 " 	-2"

	echo >&2 " 	-n $default_numnodes"

	echo >&2 "	-c $default_numcores"

	echo >&2 "	-r $default_megsram"

	echo >&2 "	-p $default_pressure"

}

P=0

Q=0

HPL_PQ() {

	local cores=$1

	local pp

	local qq

	# Find values of P and Q that use every available core and are as close

	# in value as possible, with Q being slightly larger than P.

	# For example, if the # of cores is 36, the optimal (P,Q) is (4,9) rather

	# than (6,6).

	for pp in $(seq 1 $cores); do

		qq=$(( $cores / $pp))

		ll=$(( $pp * $qq))

		if [ $pp -ge $qq ]; then

			 break

		fi

		if [ $ll -eq $cores ]; then 

			P=$pp;

			Q=$qq;

		fi

	done

}

Ns=""

CALC_NS() {

	local numnodes=$1

	local megspernode=$2

	local pressure=$3

	# This calculation adjusts the size of the HPL problem to a percentage

	# of the available RAM. The formula is:

	#

	# "Take the square root of the total amount of memory in the cluster

	# divided by the number of bytes in a double precision number, then 

	# multiply by the requested memory pressure and round up to a multiple

	# of the block size."

	Ns=$(echo "sqrt($megspernode * 1024 * 1024 * $numnodes / 8) * $pressure / $NBs * $NBs" | bc)

}

inputval=""

GET_INPUT() {

	local defval=""

	local prompt="Input"

	local value=""

	if [ $# -ge 1 ]; then

		prompt=$1

	fi

	if [ $# -ge 2 ]; then

		defval=$2

	fi

	while [ -z $value ]; do

		if [ ! -z $defval ]; then

			echo -n "$prompt [$defval]? "

		else

			echo -n "$prompt? "

		fi

		read value

		if [ -z $value ]; then

			value=$defval

		fi

	done

	inputval=$value

}

TEMP_FILE="$(mktemp)"

trap "rm -rf $TEMP_FILE; exit 1" SIGINT SIGHUP SIGTERM

trap "rm -rf $TEMP_FILE" EXIT

arch=ICS.`uname -s`.`./get_mpi_cc.sh`

got_nodes=0

got_cores=0

got_ram=0

got_pressure=0

use_defaults=0

OPTIONS=$(getopt -o "n:c:r:p:hd12" --long "nodes:,cores:,ram:,pressure:,help,use-defaults" -- "$@")

if [ $? -ne 0 ]; then

	USAGE; exit 1

fi

eval set -- "$OPTIONS"

while true; do

	case "$1" in

		-n | --nodes ) 

			numnodes=$2; got_nodes=1;

			shift; shift;;

		-c | --cores ) 

			numcores=$2; got_cores=1;

			shift; shift;;

		-r | --ram ) 

			megsram=$2; got_ram=1;

			shift; shift;;

		-p | --pressure ) 

			pressure=$2; got_pressure=1;

			shift; shift;;

		-d | --use-defaults ) 

			use_defaults=1;

			shift;;

		-- ) 

			# End of arguments list.

			break;;

		-h | --help ) 

			USAGE; exit 0;;

		* )

			USAGE; exit 1;;

	esac

done

if ! [[ $numnodes =~ ^[0-9]+$ ]]; then

	echo >&2 "\"$numnodes\" is not a valid # of nodes"

	USAGE; exit 1

elif ! [[ $numcores =~ ^[0-9]+$ ]]; then

	echo >&2 "\"$numcores\" is not a valid # of cores"

	USAGE; exit 1

elif ! [[ $megsram =~ ^[0-9]+$ ]]; then

	echo >&2 "\"$megsram\" is not a valid amount of RAM"

	USAGE; exit 1

elif ! [[ $pressure =~ ^0\.[1-9][0-9]*$ ]]; then

	echo >&2 "\"$pressure\" is not a valid amount memory pressure."

	USAGE; exit 1

fi

while [ $got_nodes -eq 0 -a $use_defaults -eq 0 ]

do

	GET_INPUT "# of compute nodes" $default_numnodes

	numnodes=$inputval

	if [[ $numnodes =~ ^[0-9]+$ ]]; then

		got_nodes=1

	fi

done

while [ $got_cores -eq 0 -a $use_defaults -eq 0 ]

do

	GET_INPUT "# of cores per node" $default_numcores

	numcores=$inputval

	if [[ $numcores =~ ^[0-9]+$ ]]; then

		got_cores=1

	fi

done

while [ $got_ram -eq 0 -a $use_defaults -eq 0 ]

do

	GET_INPUT "# of RAM per node (in MB)" $default_megsram

	megsram=$inputval

	if [[ $megsram =~ ^[0-9]+$ ]]; then

		got_ram=1

	fi

done

while [ $got_pressure -eq 0 -a $use_defaults -eq 0 ]

do

	GET_INPUT "Memory pressure (range between 0.1 and 0.9)" $default_pressure

	pressure=$inputval

	if [[ $pressure =~ ^0\.[1-9][0-9]*$ ]]; then

		got_pressure=1

	fi

done

totnumcores=$(( $numcores * $numnodes ))

HPL_PQ $totnumcores

CALC_NS $numnodes $megsram $pressure

echo

echo "Use \"./run_hpl2 $totnumcores\" to use this configuration."

echo

echo

DATFILE=$PWD/hpl-2.2/bin/$arch/HPL.dat

cat <

HPLinpack benchmark input file - $totnumcores processes, $pressure memory size

Generated by hpl_dat_gen.sh.

HPL.out		output file name (if any)

6		device out (6=stdout,7=stderr,file)

1		# of problems sizes (N)

$Ns		Ns

1		# of NBs

$NBs		NBs

0		PMAP process mapping (0=Row-,1=Column-major)

1		# of process grids (P x Q)

$P		Ps

$Q		Qs

16.0		threshold

1		# of panel fact

1		PFACTs (0=left, 1=Crout, 2=Right)

1		# of recursive stopping criterium

4		NBMINs (>= 1)

1		# of panels in recursion

2		NDIVs

1		# of recursive panel fact.

2		RFACTs (0=left, 1=Crout, 2=Right)

1		# of broadcast

1		BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM)

1		# of lookahead depth

1		DEPTHs (>=0)

2		SWAP (0=bin-exch,1=long,2=mix)

$NBs		swapping threshold

0		L1 in (0=transposed,1=no-transposed) form

0		U  in (0=transposed,1=no-transposed) form

1		Equilibration (0=no,1=yes)

8		memory alignment in double (> 0)

EOF