#!/usr/bin/env bash
#
# Copyright 2018, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the
#       distribution.
#
#     * Neither the name of the copyright holder nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#
# pmempool_sync_remote/TEST35 -- pmempool sync with remote replication
#                                and bad blocks in the remote replica:
#                                - the pool consists of 1 part
#                                - offset of bad block: 1000
#

# standard unit test setup
. ../unittest/unittest.sh

require_test_type medium
require_fs_type non-pmem
require_build_type debug nondebug
require_linked_with_ndctl $PMEMPOOL$EXESUFFIX

require_nfit_tests_enabled
require_nodes 2
require_sudo_allowed_node 1
require_kernel_module_node 1 nfit_test
require_command_node 1 ndctl

setup

require_node_libfabric 0 $RPMEM_PROVIDER
require_node_libfabric 1 $RPMEM_PROVIDER

init_rpmem_on_node 0 1

require_node_log_files 0 pmemobj$UNITTEST_NUM.log
require_node_log_files 0 pmempool$UNITTEST_NUM.log
require_node_log_files 1 pmemobj$UNITTEST_NUM.log
require_node_log_files 1 pmempool$UNITTEST_NUM.log

. ../common_badblock.sh

ndctl_nfit_test_init_node 1

DEVICE=$(ndctl_nfit_test_get_block_device_node 1)
NAMESPACE=$(ndctl_nfit_test_get_namespace_of_device_node 1 $DEVICE)
FULLDEV="/dev/$DEVICE"
MOUNT_DIR="${NODE_DIR[1]}/mnt-pmem"
LAYOUT="layout"

ndctl_nfit_test_mount_pmem_node 1 $FULLDEV $MOUNT_DIR

LOG=out${UNITTEST_NUM}.log
rm -f $LOG && touch $LOG

POOLSET_LOCAL="local_pool.set"
POOLSET_REMOTE="remote_pool.set"

create_poolset $DIR/$POOLSET_LOCAL \
	8M:${NODE_DIR[0]}/pool.local:x \
	M \
	${NODE_ADDR[1]}:$POOLSET_REMOTE

create_poolset $DIR/$POOLSET_REMOTE \
	8M:$MOUNT_DIR/pool.remote:x

copy_files_to_node 0 ${NODE_DIR[0]} $DIR/$POOLSET_LOCAL
copy_files_to_node 1 ${NODE_DIR[1]} $DIR/$POOLSET_REMOTE

expect_normal_exit run_on_node 0 ../pmempool rm -sf ${NODE_DIR[0]}$POOLSET_LOCAL
expect_normal_exit run_on_node 0 "../obj_verify ${NODE_DIR[0]}$POOLSET_LOCAL $LAYOUT c v &>> $LOG"

###############################################################################
# Enabling features in poolsets with remote replicas is not supported now,
# so we have to use the following workaround to enable
# the POOL_FEAT_CHECK_BAD_BLOCKS compat feature in such poolset.
# We create the auxiliary poolset without the remote replica
# and enable this compat feature on all nodes separately.

POOLSET_LOCAL_AUX=${POOLSET_LOCAL}_aux

create_poolset $DIR/${POOLSET_LOCAL_AUX} \
	8M:${NODE_DIR[0]}/pool.local:x

copy_files_to_node 0 ${NODE_DIR[0]} $DIR/${POOLSET_LOCAL_AUX}

turn_on_checking_bad_blocks_node 0 ${NODE_DIR[0]}/${POOLSET_LOCAL_AUX}
turn_on_checking_bad_blocks_node 1 ${NODE_DIR[1]}/${POOLSET_REMOTE}

###############################################################################

# inject bad block:
FILE=$MOUNT_DIR/pool.remote
FIRST_SECTOR=$(expect_normal_exit run_on_node 1 ../extents $FILE -l 1000)

expect_normal_exit run_on_node 1 "sudo ndctl inject-error --block=$FIRST_SECTOR --count=1 $NAMESPACE &>> $PREP_LOG_FILE"
expect_normal_exit run_on_node 1 "sudo ndctl start-scrub &>> $PREP_LOG_FILE"
expect_normal_exit run_on_node 1 "sudo ndctl wait-scrub &>> $PREP_LOG_FILE"

expect_bad_blocks_node 1

# verification should fail
expect_abnormal_exit run_on_node 0 "../obj_verify ${NODE_DIR[0]}/$POOLSET_LOCAL $LAYOUT v &>> $LOG"

expect_bad_blocks_node 1

# the remote replica contains bad blocks, so it cannot be opened and will be recreated
expect_normal_exit run_on_node 0 "../pmempool sync -b -v ${NODE_DIR[0]}/$POOLSET_LOCAL &>> $LOG"

# verification should succeed
expect_normal_exit run_on_node 0 "../obj_verify ${NODE_DIR[0]}/$POOLSET_LOCAL $LAYOUT v &>> $LOG"

ndctl_nfit_test_fini_node 1 $MOUNT_DIR

pass