#!/bin/bash

# BEGIN_ICS_COPYRIGHT8 ****************************************

# 

# Copyright (c) 2015-2017, Intel Corporation

# 

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are met:

# 

#     * Redistributions of source code must retain the above copyright notice,

#       this list of conditions and the following disclaimer.

#     * Redistributions in binary form must reproduce the above copyright

#       notice, this list of conditions and the following disclaimer in the

#       documentation and/or other materials provided with the distribution.

#     * Neither the name of Intel Corporation nor the names of its contributors

#       may be used to endorse or promote products derived from this software

#       without specific prior written permission.

# 

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE

# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# 

# END_ICS_COPYRIGHT8   ****************************************

# [ICS VERSION STRING: unknown]

# verify hosts basic single node configuration and performance via hostverify.sh

# prior to using this, copy /usr/share/opa/samples/hostverify.sh to FF_HOSTVERIFY_DIR,

# and edit to set proper expectations for node configuration and performance, 

# optional override of defaults

if [ -f /etc/opa/opafastfabric.conf ]

then

	. /etc/opa/opafastfabric.conf

fi

. /usr/lib/opa/tools/opafastfabric.conf.def

. /usr/lib/opa/tools/ff_funcs

trap "exit 1" SIGHUP SIGTERM SIGINT

punchlist=$FF_RESULT_DIR/punchlist.csv

del=';' # TBD what will work best for import

Usage_full()

{

	echo "Usage: opaverifyhosts [-kc] [-f hostfile] [-u upload_file] [-d upload_dir]" >&2

	echo "                         [-h 'hosts'] [-T timelimit] [test ...]" >&2

	echo "              or" >&2

	echo "       opaverifyhosts --help" >&2

	echo "   --help - produce full help text" >&2

	echo "   -k - at start and end of verification, kill any existing hostverify" >&2

	echo "        or xhpl jobs on the hosts" >&2

	echo "   -c - copy hostverify.sh to hosts first, useful if you have edited it" >&2

	echo "   -f hostfile - file with hosts in cluster, default is $CONFIG_DIR/opa/hosts" >&2

	echo "   -h hosts - list of hosts to ping" >&2

	echo "   -u upload_file - filename to upload hostverify.res to after verification" >&2

	echo "                    to allow backup and review of the detailed results" >&2

	echo "                    for each node" >&2

	echo "                    The default upload destination file is hostverify.res" >&2

	echo "                    If -u '' is specified, no upload will occur" >&2

	echo "   -d upload_dir - directory to upload result from each host to" >&2

	echo "                   default is uploads" >&2

	echo "   -T timelimit - timelimit in seconds for host to complete tests" >&2

	echo "                  default of 300 seconds (5 minutes)" >&2

	echo "   -F filename - filename of hostverify script to use. Default is $FF_HOSTVERIFY_DIR/hostverify.sh" >&2

	echo "	 test - one or more specific tests to run" >&2

	echo "	        see /usr/share/opa/samples/hostverify.sh for a list of available tests" >&2

	echo "This verifies basic node configuration and performance by running" >&2

	echo "FF_HOSTVERIFY_DIR/hostverify.sh on all specified hosts" >&2

	echo >&2

	echo "Prior to using this, copy /usr/share/opa/samples/hostverify.sh to FF_HOSTVERIFY_DIR" >&2

	echo "and edit to set proper expectations for node configuration and performance" >&2

	echo "Then be sure to use the -c option on first run for a given node" >&2

	echo "so that hostverify.sh gets copied to each node." >&2

	echo "FF_HOSTVERIFY_DIR is configured in /etc/opa/opafastfabric.conf" >&2

	echo >&2

	echo "A summary of results is appended to FF_RESULT_DIR/verifyhosts.res." >&2

	echo "A punchlist of failures is also appended to FF_RESULT_DIR/punchlist.csv" >&2

	echo "Only failures are shown on stdout" >&2

	echo >&2

	echo " Environment:" >&2

	echo "   HOSTS - list of hosts, used if -h option not supplied" >&2

	echo "   HOSTS_FILE - file containing list of hosts, used in absence of -f and -h" >&2

	echo "   UPLOADS_DIR - directory to upload to, used in absence of -d" >&2

	echo "   FF_MAX_PARALLEL - maximum concurrent operations" >&2

	echo "example:">&2

	echo "   opaverifyhosts -c" >&2

	echo "   opaverifyhosts -h 'arwen elrond'" >&2

	echo "   HOSTS='arwen elrond' opaverifyhosts" >&2

	exit 0

}

Usage()

{

	echo "Usage: opaverifyhosts [-kc] [-f hostfile] [-u upload_file]" >&2

	echo "              or" >&2

	echo "       opaverifyhosts --help" >&2

	echo "   --help - produce full help text" >&2

	echo "   -k - at start and end of verification, kill any existing hostverify" >&2

	echo "        or xhpl jobs on the hosts" >&2

	echo "   -c - copy hostverify.sh to hosts first, useful if you have edited it" >&2

	echo "   -f hostfile - file with hosts in cluster, default is $CONFIG_DIR/opa/hosts" >&2

	echo "   -u upload_file - filename to upload hostverify.res to after verification" >&2

	echo "                    to allow backup and review of the detailed results" >&2

	echo "                    for each node" >&2

	echo "                    The default upload destination file is hostverify.res" >&2

	echo "                    If -u '' is specified, no upload will occur" >&2

	echo >&2

	echo "This verifies basic node configuration and performance by running" >&2

	echo "FF_HOSTVERIFY_DIR/hostverify.sh on all specified hosts" >&2

	echo >&2

	echo "Prior to using this, copy /usr/share/opa/samples/hostverify.sh to FF_HOSTVERIFY_DIR" >&2

	echo "and edit to set proper expectations for node configuration and performance" >&2

	echo "Then be sure to use the -c option on first run for a given node" >&2

	echo "so that hostverify.sh gets copied to each node." >&2

	echo "FF_HOSTVERIFY_DIR is configured in /etc/opa/opafastfabric.conf" >&2

	echo >&2

	echo "A summary of results is appended to FF_RESULT_DIR/verifyhosts.res." >&2

	echo "A punchlist of failures is also appended to FF_RESULT_DIR/punchlist.csv" >&2

	echo "Only failures are shown on stdout" >&2

	echo >&2

	echo "example:">&2

	echo "   opaverifyhosts" >&2

	echo "   opaverifyhosts -c" >&2

	exit 2

}

if [ x"$1" = "x--help" ]

then

	Usage_full

fi

append_punchlist()

# stdin has verifyhosts.res failure lines generated in this run

{

	(

	export IFS=':'

	opasorthosts | while read host failinfo

	do

		echo "$timestamp$del$host$del$failinfo"

	done >> $punchlist

	)

}

job_cleanup()

{

	# HOSTS is exported

	if [ "$do_kill" = y ]

	then

		# we don't log this

		echo "Killing hostverify and xhpl on hosts..."

		# we use patterns so the pkill doesn't kill this script or opacmdall itself

		# use an echo at end so exit status is good

		opacmdall -p -T 60 "pkill -9 -f -x 'host[v]erify.*.sh'; pkill -9 '[x]hpl'; echo -n"

	fi

}

do_copy=n

upload_file=hostverify.res

timelimit=300

do_kill=n

filename=$FF_HOSTVERIFY_DIR/hostverify.sh

while getopts kcf:h:d:u:T:F: param

do

	case $param in

	k)

		do_kill=y;;

	c)

		do_copy=y;;

	h)

		HOSTS="$OPTARG";;

	f)

		HOSTS_FILE="$OPTARG";;

	d)

		export UPLOADS_DIR="$OPTARG";;

	u)

		upload_file="$OPTARG";;

	T)

		timelimit="$OPTARG";;

	F)

		filename="$OPTARG";;

	?)

		Usage;;

	esac

done

shift $((OPTIND -1))

check_host_args opaverifyhosts

# HOSTS now lists all the hosts, pass it along to the commands below via env

export HOSTS

unset HOSTS_FILE

job_cleanup

echo "=============================================================================" >> $FF_RESULT_DIR/verifyhosts.res

date >> $FF_RESULT_DIR/verifyhosts.res

echo "$(echo "$HOSTS"|tr -s ' ' '\n'|sed -e '/^$/d'|sort -u| wc -l) hosts will be verified" | tee -a $FF_RESULT_DIR/verifyhosts.res

if [ "$do_copy" = y ]

then

	echo "SCPing $filename to $FF_HOSTVERIFY_DIR/hostverify.sh ..."| tee -a $FF_RESULT_DIR/verifyhosts.res

	opascpall -p "$filename" "$FF_HOSTVERIFY_DIR/hostverify.sh" 2>&1|tee -a $FF_RESULT_DIR/verifyhosts.res

	date >> $FF_RESULT_DIR/verifyhosts.res

fi

timestamp=$(date +"%Y/%m/%d %T")

echo "Running $FF_HOSTVERIFY_DIR/hostverify.sh -d $FF_HOSTVERIFY_DIR $* ..."

resultlineno=$(cat $FF_RESULT_DIR/verifyhosts.res|wc -l)	# for punchlist

opacmdall -p -T $timelimit "bash $FF_HOSTVERIFY_DIR/hostverify.sh -d $FF_HOSTVERIFY_DIR $*" 2>&1|tee -a $FF_RESULT_DIR/verifyhosts.res|egrep 'FAIL'

# update punchlist using new failures

tail -n +$resultlineno $FF_RESULT_DIR/verifyhosts.res| egrep 'FAIL'|append_punchlist

date >> $FF_RESULT_DIR/verifyhosts.res

job_cleanup

# upload the result file from each host

if [ z"$upload_file" != z ]

then

	echo "Uploading $FF_HOSTVERIFY_DIR/hostverify.res to $UPLOADS_DIR/$upload_file ..."| tee -a $FF_RESULT_DIR/verifyhosts.res

	opauploadall -p $FF_HOSTVERIFY_DIR/hostverify.res $upload_file 2>&1|tee -a $FF_RESULT_DIR/verifyhosts.res

	date >> $FF_RESULT_DIR/verifyhosts.res

fi