#!/bin/bash
#
# Copyright 2011-2018 the Pacemaker project contributors
#
# The version control history for this file may have further details.
#
# This source code is licensed under the GNU General Public License version 2
# or later (GPLv2+) WITHOUT ANY WARRANTY.
#
accept_defaults=0
do_raw=0
ETCHOSTS=0
pcmk_ver=11
nodelist=0
limit=0
pkgs="corosync xinetd nmap abrt-cli fence-agents perl-TimeDate gdb"
transport="multicast"
inaddr_any="no"
INSTALL=
cs_conf=
fence_conf=
rpm_repo=
distro=
dsh_group=0
if [ ! -z $cluster_name ]; then
cluster=$cluster_name
else
cluster=dummy0
fi
# Corosync Settings
cs_port=666
# Settings that work great on nXX
join=60
#token=3000
consensus=1500
# Official settings
join=2000
token=5000
consensus=2500
# Testing
join=1000
consensus=7500
do_debug=off
function ip_for_node() {
ping -c 1 $1 | grep "bytes from" | head -n 1 | sed -e 's/.*bytes from//' -e 's/: icmp.*//' | awk '{print $NF}' | sed 's:(::' | sed 's:)::'
# if [ $do_raw = 1 ]; then
# echo $1
# else
# #host $1 | grep "has address" | head -n 1 | awk '{print $NF}' | sed 's:(::' | sed 's:)::'
# fi
}
function id_for_node() {
ip_for_node $* | tr '.' ' ' | awk '{print $4}'
}
function name_for_node() {
echo $1 | awk -F. '{print $1}'
}
function helptext() {
echo "cluster-init - Configure cluster communication for the infrastructures supported by Pacemaker"
echo ""
echo "-g, --group Specify the group to operate on/with"
echo "-w, --host Specify a host to operate on/with. May be specified multiple times"
echo "-r, --raw-ip Supplied nodes were listed as their IP addresses"
echo ""
echo "-c, --corosync configure for corosync"
echo "-C, --nodelist configure for corosync with a node list"
echo "-u, --unicast configure point-to-point communication instead of multicast"
echo ""
echo "-I, --install Install packages"
echo "-R, --repo name Setup and update/install Pacemaker from the named clusterlabs.org repo"
echo " Known values: rpm, rpm-test, rpm-next, rpm-test-next, rpm-test-rhel"
echo "-D, --distro The distro within the --repo. Defaults to fedora-15"
echo ""
echo "-d, --debug Enable debug logging for the cluster"
echo "-10 install stable-1.0 packages, implies: -p 0 -R rpm-test -I"
echo "--hosts Copy /etc/hosts from the test master to the nodes"
echo "-e, --extra list Whitespace separated list of extra packages to install"
echo "-l, --limit N Use the first N hosts from the named group"
echo " Extra packages to install"
exit $1
}
host_input=""
while true; do
case "$1" in
-g) cluster=$2;
shift; shift;;
-w|--host)
for h in $2; do
host_input="$host_input -w $h";
done
shift; shift;;
-w) host_input="$host_input -w $2"
shift; shift;;
-r|--raw-ip) do_raw=1; shift;;
-D) distro=$2; shift; shift;;
-d|--debug) do_debug=on; shift;;
-R|--repo) rpm_repo=$2; shift; shift;;
-I|--install) INSTALL=Yes; shift;;
--hosts) ETCHOSTS=1; shift;;
-c|--corosync) CTYPE=corosync; shift;;
-C|--nodelist) CTYPE=corosync; nodelist=1; shift;;
-u|--unicast) nodelist=1; transport=udpu; inaddr_any="yes"; shift;;
-e|--extra) pkgs="$pkgs $2"; shift; shift;;
-t|--test) rpm_repo=rpm-test-next; pkgs="$pkgs valgrind"; shift;;
-l|--limit) limit=$2; shift; shift;;
r*[0-9])
rhel=`echo $1 | sed -e s/rhel// -e s/-// -e s/r//`
distro="rhel-$rhel";
pkgs="$pkgs qarsh-server";
case $rhel in
7) CTYPE=corosync;;
esac
shift
;;
f*[0-9][0-9])
distro="fedora-`echo $1 | sed -e s/fedora// -e s/-// -e s/f//`";
CTYPE=corosync;
shift
;;
p0|10) pcmk_ver=10; rpm_repo="rpm-test"; install=1; shift;;
-y|--yes|--defaults) accept_defaults=1; shift;;
-x) set -x; shift;;
-\?|--help) helptext 0; shift;;
"") break;;
*) echo "unknown option: $1"; exit 1;;
esac
done
if [ ! -z $cluster ]; then
host_input="-g $cluster"
# use the last digit present in the variable (if any)
dsh_group=`echo $cluster | sed 's/[^0-9][^0-9]*//g;s/.*\([0-9]\)$/\1/'`
fi
if [ -z $dsh_group ]; then
dsh_group=1
fi
if [ x = "x$host_input" -a x = "x$cluster" ]; then
if [ -d $HOME/.dsh/group ]; then
read -p "Please specify a dsh group you'd like to configure as a cluster? [] " -t 60 cluster
else
read -p "Please specify a whitespace delimetered list of nodes you'd like to configure as a cluster? [] " -t 60 host_list
for h in $2; do
host_input="$host_input -w $h";
done
fi
fi
if [ -z "$host_input" ]; then
echo "You didn't specify any nodes or groups to configure"
exit 1
fi
if [ $limit -gt 0 ]; then
echo "Using only the first $limit hosts in $cluster group"
host_list=`cluster-helper --list bullet $host_input | head -n $limit | tr '\n*' ' '`
else
host_list=`cluster-helper --list short $host_input`
fi
num_hosts=`echo $host_list | wc -w`
if [ $num_hosts -gt 9 ]; then
cs_port=66
fi
for h in $host_list; do
ping -c 1 -q $h
if [ $? != 0 ]; then
echo "Using long names..."
host_list=`cluster-helper --list long $host_input`
break
fi
done
if [ -z $CTYPE ]; then
echo ""
read -p "Where should Pacemaker obtain membership and quorum from? [corosync] (corosync) " -t 60 CTYPE
fi
case $CTYPE in
corosync) cs_conf=/etc/corosync/corosync.conf;;
esac
function get_defaults()
{
if [ -z $SSH ]; then
SSH="No"
fi
if [ -z $SELINUX ]; then
SELINUX="No"
fi
if [ -z $IPTABLES ]; then
IPTABLES="Yes"
fi
if [ -z $DOMAIN ]; then
DOMAIN="No"
fi
if [ -z $INSTALL ]; then
INSTALL="Yes"
fi
if [ -z $DATE ]; then
DATE="No"
fi
}
get_defaults
if [ $accept_defaults = 0 ]; then
echo ""
read -p "Shall I install an ssh key to cluster nodes? [$SSH] " -t 60 SSH
echo ""
echo "SELinux prevent many things, including password-less ssh logins"
read -p "Shall I disable selinux? [$SELINUX] " -t 60 SELINUX
echo ""
echo "Incorrectly configured firewalls will prevent corosync from starting up"
read -p "Shall I disable iptables? [$IPTABLES] " -t 60 IPTABLES
if [ $pcmk_ver = 10 ]; then
echo ""
echo "Without a default domain, external/ssh fencing probably won't work because it can't find its peers"
read -p "Shall I set one? [No] (domain.name) " -t 60 DOMAIN
fi
echo ""
read -p "Shall I install/update the relevant packages? [$INSTALL] " -t 60 INSTALL
case $INSTALL in
[Yy][Ee][Ss]|[Yy]|"")
if [ -z $rpm_repo ]; then
echo ""
read -p "Would you like to install packages from ClusterLabs.org? [No] (rpm, rpm-next, rpm-test-next) " -t 60 rpm_repo
fi
if [ ! -z $rpm_repo ]; then
if [ -z $distro ]; then
distro=fedora-18
echo ""
read -p "Which distro are you installing for? [$distro] (eg. fedora-17, rhel-6) " -t 60 distro
fi
fi
;;
esac
echo ""
read -p "Shall I sync the date/time? [$DATE] " -t 60 DATE
fi
get_defaults
echo ""
echo "Detecting possible fencing options"
if [ -e /etc/cluster/fence_xvm.key ]; then
echo "* Found fence_xvm"
fence_conf=/etc/cluster/fence_xvm.key
pkgs="$pkgs fence-virt"
fi
if [ ! -z ${OS_AUTH_URL} ]; then
echo "* Found openstack credentials"
fence_conf=/sbin/fence_openstack
pkgs="$pkgs python-novaclient"
fi
echo ""
echo "Beginning cluster configuration"
echo ""
case $SSH in
[Yy][Ee][Ss]|[Yy])
for host in $host_list; do
echo "Installing our ssh key on ${host}"
ssh-copy-id root@${host} >/dev/null 2>&1
# Fix selinux labeling
ssh -l root ${host} -- restorecon -R -v .
done
;;
esac
case $DATE in
[Yy][Ee][Ss]|[Yy])
for host in $host_list; do
echo "Setting time on ${host}"
scp /etc/localtime root@${host}:/etc
now=`date +%s`
ssh -l root ${host} -- date -s @$now
echo ""
done
;;
esac
REPO=
if [ ! -z $rpm_repo ]; then
REPO=$rpm_repo/$distro
fi
init=`mktemp`
cat<<-END>$init
verbose=0
pkgs="$pkgs"
lhost=\`uname -n\`
lshort=\`echo \$lhost | awk -F. '{print \$1}'\`
log() {
printf "%-10s \$*\n" "\$lshort:" 1>&2
}
debug() {
if [ \$verbose -gt 0 ]; then
log "Debug: \$*"
fi
}
info() {
log "\$*"
}
warning() {
log "WARN: \$*"
}
fatal() {
log "ERROR: \$*"
exit 1
}
case $SELINUX in
[Yy][Ee][Ss]|[Yy])
sed -i.sed "s/enforcing/disabled/g" /etc/selinux/config
;;
esac
case $IPTABLES in
[Yy][Ee][Ss]|[Yy]|"")
service iptables stop
chkconfig iptables off
service firewalld stop
chkconfig firewalld off
;;
esac
case $DOMAIN in
[Nn][Oo]|"")
;;
*.*)
if
! grep domain /etc/resolv.conf
then
sed -i.sed "s/nameserver/domain\ $DOMAIN\\\nnameserver/g" /etc/resolv.conf
fi
;;
*) echo "Unknown domain: $DOMAIN";;
esac
case $INSTALL in
[Yy][Ee][Ss]|[Yy]|"")
if [ ! -z $REPO ]; then
info Configuring Clusterlabs repo: $REPO
yum install -y wget
rm -f /etc/yum.repos.d/clusterlabs.repo
wget -O /etc/yum.repos.d/clusterlabs.repo http://www.clusterlabs.org/$REPO/clusterlabs.repo &>/dev/null
yum clean all
fi
info Installing cluster software
if [ $pcmk_ver = 10 ]; then
yum install -y $pkgs at
service atd start
systemctl enable atd.service
yum install -y "pacemaker < 1.1"
else
yum install -y $pkgs pacemaker
fi
;;
esac
info "Configuring services"
chkconfig xinetd on
service xinetd start &>/dev/null
chkconfig corosync off &> /dev/null
mkdir -p /etc/cluster
info "Turning on core files"
grep -q "unlimited" /etc/bashrc
if [ $? = 1 ]; then
sed -i.sed "s/bashrc/bashrc\\\nulimit\ -c\ unlimited/g" /etc/bashrc
fi
function patch_cs_config() {
test $num_hosts != 2
two_node=$?
priority="info"
if [ $do_debug = 1 ]; then
priority="debug"
fi
ssh -l root ${host} -- sed -i.sed "s/.*mcastaddr:.*/mcastaddr:\ 226.94.1.1/g" $cs_conf
ssh -l root ${host} -- sed -i.sed "s/.*mcastport:.*/mcastport:\ $cs_port$dsh_group/g" $cs_conf
ssh -l root ${host} -- sed -i.sed "s/.*bindnetaddr:.*/bindnetaddr:\ $ip/g" $cs_conf
ssh -l root ${host} -- sed -i.sed "s/.*syslog_facility:.*/syslog_facility:\ daemon/g" $cs_conf
ssh -l root ${host} -- sed -i.sed "s/.*logfile_priority:.*/logfile_priority:\ $priority/g" $cs_conf
if [ ! -z $token ]; then
ssh -l root ${host} -- sed -i.sed "s/.*token:.*/token:\ $token/g" $cs_conf
fi
if [ ! -z $consensus ]; then
ssh -l root ${host} -- sed -i.sed "s/.*consensus:.*/consensus:\ $consensus/g" $cs_conf
fi
if [ ! -z $join ]; then
ssh -l root ${host} -- sed -i.sed "s/^join:.*/join:\ $join/g" $cs_conf
ssh -l root ${host} -- sed -i.sed "s/\\\Wjoin:.*/join:\ $join/g" $cs_conf
fi
ssh -l root ${host} -- grep -q "corosync_votequorum" $cs_conf 2>&1 > /dev/null
if [ $? -eq 0 ]; then
ssh -l root ${host} -- sed -i.sed "s/\\\Wexpected_votes:.*/expected_votes:\ $num_hosts/g" $cs_conf
ssh -l root ${host} -- sed -i.sed "s/\\\Wtwo_node:.*/two_node:\ $two_node/g" $cs_conf
else
printf "%-10s Wrong quorum provider: installing $cs_conf for corosync instead\n" ${host}
create_cs_config
fi
}
function create_cs_config() {
cs_tmp=/tmp/cs_conf.$$
test $num_hosts != 2
two_node=$?
# Base config
priority="info"
if [ $do_debug = 1 ]; then
priority="debug"
fi
cat <<-END >$cs_tmp
# Please read the corosync.conf.5 manual page
totem {
version: 2
# cypto_cipher and crypto_hash: Used for mutual node authentication.
# If you choose to enable this, then do remember to create a shared
# secret with "corosync-keygen".
crypto_cipher: none
crypto_hash: none
# Assign a fixed node id
nodeid: $id
# Disable encryption
secauth: off
transport: $transport
inaddr_any: $inaddr_any
# interface: define at least one interface to communicate
# over. If you define more than one interface stanza, you must
# also set rrp_mode.
interface {
# Rings must be consecutively numbered, starting at 0.
ringnumber: 0
# This is normally the *network* address of the
# interface to bind to. This ensures that you can use
# identical instances of this configuration file
# across all your cluster nodes, without having to
# modify this option.
bindnetaddr: $ip
# However, if you have multiple physical network
# interfaces configured for the same subnet, then the
# network address alone is not sufficient to identify
# the interface Corosync should bind to. In that case,
# configure the *host* address of the interface
# instead:
# bindnetaddr: 192.168.1.1
# When selecting a multicast address, consider RFC
# 2365 (which, among other things, specifies that
# 239.255.x.x addresses are left to the discretion of
# the network administrator). Do not reuse multicast
# addresses across multiple Corosync clusters sharing
# the same network.
# Corosync uses the port you specify here for UDP
# messaging, and also the immediately preceding
# port. Thus if you set this to 5405, Corosync sends
# messages over UDP ports 5405 and 5404.
mcastport: $cs_port$dsh_group
# Time-to-live for cluster communication packets. The
# number of hops (routers) that this ring will allow
# itself to pass. Note that multicast routing must be
# specifically enabled on most network routers.
ttl: 1
mcastaddr: 226.94.1.1
}
}
logging {
debug: off
fileline: off
to_syslog: yes
to_stderr: no
syslog_facility: daemon
timestamp: on
to_logfile: yes
logfile: /var/log/corosync.log
logfile_priority: $priority
}
amf {
mode: disabled
}
quorum {
provider: corosync_votequorum
expected_votes: $num_hosts
votes: 1
two_node: $two_node
wait_for_all: 0
last_man_standing: 0
auto_tie_breaker: 0
}
END
scp -q $cs_tmp root@${host}:$cs_conf
rm -f $cs_tmp
}
for host in $host_list; do
echo ""
echo ""
echo "* Configuring $host"
cs_short_host=`name_for_node $host`
ip=`ip_for_node $host`
id=`id_for_node $host`
echo $ip | grep -qis NXDOMAIN
if [ $? = 0 ]; then
echo "Couldn't find resolve $host to an IP address"
exit 1
fi
if [ `uname -n` = $host ]; then
bash $init
else
cat $init | ssh -l root -T $host -- "cat > $init; bash $init"
fi
if [ "x$fence_conf" != x ]; then
if [ -e $fence_conf ]; then
scp $fence_conf root@${host}:$fence_conf
fi
fi
if [ $ETCHOSTS = 1 ]; then
scp /etc/hosts root@${host}:/etc/hosts
fi
if [ $pcmk_ver = 10 ]; then
scp /etc/hosts root@${host}:/etc/hosts
scp ~/.ssh/id_dsa.suse root@${host}:.ssh/id_dsa
scp ~/.ssh/known_hosts root@${host}:.ssh/known_hosts
fi
ssh -l root ${host} -- grep -q "token:" $cs_conf 2>&1 > /dev/null
new_config=$?
new_config=1
if [ $new_config = 0 ]; then
printf "%-10s Updating $cs_conf\n" ${host}:
patch_cs_config
else
printf "%-10s Installing $cs_conf\n" ${host}:
create_cs_config
fi
done