#!/bin/bash

. $(dirname $0)/../include.rc
. $(dirname $0)/../cluster.rc

function check_peers {
        $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
}

function volume_count {
        local cli=$1;
        if [ $cli -eq '1' ] ; then
                $CLI_1 volume info | grep 'Volume Name' | wc -l;
        else
                $CLI_2 volume info | grep 'Volume Name' | wc -l;
        fi
}

function volinfo_field()
{
    local vol=$1;
    local field=$2;

    $CLI_1 volume info $vol | grep "^$field: " | sed 's/.*: //';
}

function two_diff_vols_create {
        # Both volume creates should be successful
        $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 $H3:$B3/$V0 &
        PID_1=$!

        $CLI_2 volume create $V1 $H1:$B1/$V1 $H2:$B2/$V1 $H3:$B3/$V1 &
        PID_2=$!

        wait $PID_1 $PID_2
}

function two_diff_vols_start {
        # Both volume starts should be successful
        $CLI_1 volume start $V0 &
        PID_1=$!

        $CLI_2 volume start $V1 &
        PID_2=$!

        wait $PID_1 $PID_2
}

function two_diff_vols_stop_force {
        # Force stop, so that if rebalance from the
        # remove bricks is in progress, stop can
        # still go ahead. Both volume stops should
        # be successful
        $CLI_1 volume stop $V0 force &
        PID_1=$!

        $CLI_2 volume stop $V1 force &
        PID_2=$!

        wait $PID_1 $PID_2
}

function same_vol_remove_brick {

        # Running two same vol commands at the same time can result in
        # two success', two failures, or one success and one failure, all
        # of which are valid. The only thing that shouldn't happen is a
        # glusterd crash.

        local vol=$1
        local brick=$2
        $CLI_1 volume remove-brick $1 $2 start &
        PID=$!
        $CLI_2 volume remove-brick $1 $2 start
        wait $PID
}

cleanup;

TEST launch_cluster 3;
TEST $CLI_1 peer probe $H2;
TEST $CLI_1 peer probe $H3;

EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers

two_diff_vols_create
EXPECT 'Created' volinfo_field $V0 'Status';
EXPECT 'Created' volinfo_field $V1 'Status';

two_diff_vols_start
EXPECT 'Started' volinfo_field $V0 'Status';
EXPECT 'Started' volinfo_field $V1 'Status';

same_vol_remove_brick $V0 $H2:$B2/$V0
# Checking glusterd crashed or not after same volume remove brick
# on both nodes.
EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers

same_vol_remove_brick $V1 $H2:$B2/$V1
# Checking glusterd crashed or not after same volume remove brick
# on both nodes.
EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers

$CLI_1 volume set $V0 diagnostics.client-log-level DEBUG &
PID=$!
$CLI_1 volume set $V1 diagnostics.client-log-level DEBUG
wait $PID
kill_glusterd 3
$CLI_1 volume status $V0
$CLI_2 volume status $V1
$CLI_1 peer status
EXPECT_WITHIN $PROBE_TIMEOUT 1 check_peers
EXPECT 'Started' volinfo_field $V0 'Status';
EXPECT 'Started' volinfo_field $V1 'Status';

TEST $glusterd_3
$CLI_1 volume status $V0
$CLI_2 volume status $V1
$CLI_1 peer status
cleanup;