#!@BASH_PATH@
#
# ocf:pacemaker:ifspeed resource agent
#
# Copyright 2011-2019 the Pacemaker project contributors
#
# The version control history for this file may have further details.
#
# This source code is licensed under the GNU General Public License version 2
# or later (GPLv2+) WITHOUT ANY WARRANTY.
#

#
# OCF resource agent which monitors state of network interface and records it
# as a node attribute in the CIB based on the sum of speeds of its active (up,
# link detected, not blocked) underlying interfaces.
#
# Partially based on 'ping' RA by Andrew Beekhof
#
# Change on 2017 by Tomer Azran <tomerazran@gmail.com>:
#     Add "ip" parameter to detect network interface name by ip address:
#     http://lists.clusterlabs.org/pipermail/users/2017-August/006224.html
#
# OCF instance parameters:
#    OCF_RESKEY_name:         name of attribute to set in CIB
#    OCF_RESKEY_ip            ip address to check
#    OCF_RESKEY_iface:        network interface to monitor
#    OCF_RESKEY_bridge_ports: if not null and OCF_RESKEY_iface is a bridge, list of
#                             bridge ports to consider.
#                             Default is all ports which have designated_bridge=root_id
#    OCF_RESKEY_weight_base:  Relative weight of 1Gbps. This can be used to tune
#                             value of resulting CIB attribute.
#
# Initialization:

: ${OCF_FUNCTIONS:="${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs"}

# If these aren't available, we can still show help,
# which is all that is needed to build the man pages.
[ -r "${OCF_FUNCTIONS}" ] && . "${OCF_FUNCTIONS}"
[ -r "${OCF_FUNCTIONS_DIR}/findif.sh" ] && . "${OCF_FUNCTIONS_DIR}/findif.sh"
: ${OCF_SUCCESS:=0}

: ${__OCF_ACTION:=$1}

FINDIF=findif

# Defaults
OCF_RESKEY_name_default="ifspeed"
OCF_RESKEY_bridge_ports_default="detect"
OCF_RESKEY_weight_base_default=1000
OCF_RESKEY_dampen_default=5

: ${OCF_RESKEY_name:=${OCF_RESKEY_name_default}}
: ${OCF_RESKEY_bridge_ports:=${OCF_RESKEY_bridge_ports_default}}
: ${OCF_RESKEY_weight_base:=${OCF_RESKEY_weight_base_default}}
: ${OCF_RESKEY_dampen:=${OCF_RESKEY_dampen_default}}

meta_data() {
        cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="ifspeed" version="1.0">
<version>1.0</version>

<longdesc lang="en">
Every time the monitor action is run, this resource agent records (in the CIB)
(relative) speed of network interface it monitors.

This RA can monitor physical interfaces, bonds, bridges, vlans and (hopefully)
any combination of them.

Examples:
*) Bridge on top of one 10Gbps interface (eth2) and 802.3ad bonding (bond0) built
   on two 1Gbps interfaces (eth0 and eth1).
*) Active-backup bonding built on top of one physical interface and one vlan on
   another interface.

For STP-enabled bridges this RA tries to some-how guess network topology and by
default looks only on ports which are connected to upstream switch. This can be
overridden by 'bridge_ports' parameter. Active interfaces in this case are those
in "forwarding" state.

For balancing bonds this RA summs speeds of underlying "up" slave interfaces
(and applies coefficient 0.8 to result).

For non-balancing bonds ('active-backup' and probably 'broadcast'), only the
speed of the currently active slave is used.
</longdesc>
<shortdesc lang="en">Network interface speed monitor</shortdesc>

<parameters>

<parameter name="name" unique="1">
<longdesc lang="en">
The name of the attribute to set.  This is the name to be used in the constraints.
</longdesc>
<shortdesc lang="en">Attribute name</shortdesc>
<content type="string" default="${OCF_RESKEY_name_default}"/>
</parameter>

<parameter name="iface" unique="0" required="1">
<longdesc lang="en">
Network interface to monitor.
</longdesc>
<shortdesc lang="en">Network interface</shortdesc>
<content type="string" default=""/>
</parameter>

<parameter name="ip" unique="0" required="0">
<longdesc lang="en">
Try to detect interface name by detecting the interface that holds the IP address.
The IPv4 (dotted quad notation) or IPv6 address (colon hexadecimal notation)
example IPv4 "192.168.1.1".
example IPv6 "2001:db8:DC28:0:0:FC57:D4C8:1FFF".
</longdesc>
<shortdesc lang="en">IPv4 or IPv6 address</shortdesc>
<content type="string" default="" />
</parameter>

<parameter name="bridge_ports" unique="0">
<longdesc lang="en">
If not null and OCF_RESKEY_iface is a bridge, list of bridge ports to consider.
Default is all ports which have designated_bridge=root_id.
</longdesc>
<shortdesc lang="en">Bridge ports</shortdesc>
<content type="string" default="${OCF_RESKEY_bridge_ports_default}"/>
</parameter>

<parameter name="weight_base" unique="0">
<longdesc lang="en">
Relative weight of 1Gbps in interface speed.
Can be used to tune how big attribute value will be.
</longdesc>
<shortdesc lang="en">Weight of 1Gbps</shortdesc>
<content type="integer" default="${OCF_RESKEY_weight_base_default}"/>
</parameter>

<parameter name="dampen" unique="0">
<longdesc lang="en">
The time to wait (dampening) for further changes to occur.
</longdesc>
<shortdesc lang="en">Dampening interval</shortdesc>
<content type="integer" default="${OCF_RESKEY_dampen_default}"/>
</parameter>

<parameter name="debug" unique="0">
<longdesc lang="en">
Log what have been done more verbosely.
</longdesc>
<shortdesc lang="en">Verbose logging</shortdesc>
<content type="string" default="false"/>
</parameter>

</parameters>

<actions>
<action name="start"   timeout="30s" />
<action name="stop"    timeout="30s" />
<action name="monitor" depth="0"  timeout="30s" interval="10s"/>
<action name="meta-data"  timeout="5s" />
<action name="validate-all"  timeout="30s" />
</actions>
</resource-agent>
END
}

usage() {
    cat <<END
usage: $0 {start|stop|monitor|validate-all|meta-data}

Expects to have a fully populated OCF RA-compliant environment set.
END
}

start() {
    monitor
    if [ $? -eq $OCF_SUCCESS ]; then
        return $OCF_SUCCESS
    fi
    ha_pseudo_resource ${ha_pseudo_resource_name} start
    update
    return $?
}

stop() {
    ha_pseudo_resource "${ha_pseudo_resource_name}" stop
    attrd_updater -D -n "${OCF_RESKEY_name}" -d "${OCF_RESKEY_dampen}" ${attrd_options}
    return $OCF_SUCCESS
}

monitor() {
    local ret
    ha_pseudo_resource "${ha_pseudo_resource_name}" monitor
    ret=$?
    if [ ${ret} -eq $OCF_SUCCESS ] ; then
        update
    fi
    return ${ret}
}

# This function tries to guess nic interface by IP
get_nic_name_by_ip(){
    # $FINDIF takes its parameters from the environment.
    # Its output is as follows:
    # [NIC_NAME] netmask [NETMASK] broadcast [BROADCAST}
    NICINFO=$( "${FINDIF}" )
    rc=$?
    if [ $rc -eq 0 ];then
        # Get NIC_NAME part of findif function output.
        echo "${NICINFO%% *}"
    else
        echo ""
    fi
}

validate() {
    # Check the interface parameter
    if [ -z "${OCF_RESKEY_iface}" ]; then
        if [ -z "${OCF_RESKEY_ip}" ]; then
            ocf_log err "Empty iface and ip parameters. Please specify either an interface name or valid ip address."
            exit $OCF_ERR_CONFIGURED
        else
            ipcheck_ipv4 "${OCF_RESKEY_ip}"
            if [ $? -eq 1 ] ; then
                ipcheck_ipv6 "${OCF_RESKEY_ip}"
                if [ $? -eq 1 ] ; then
                  ocf_exit_reason "ip parameter [${OCF_RESKEY_ip}] is not a valid ip address."
                  exit $OCF_ERR_CONFIGURED
                fi
            fi
        fi
    fi
    return $OCF_SUCCESS
}

iface_get_speed() {
    local iface="$1"
    local operstate
    local carrier
    local speed

    if [ ! -e "/sys/class/net/${iface}" ] ; then
        echo "0"
    elif iface_is_bridge "${iface}" ; then # bridges do not have operstate
        read carrier < "/sys/class/net/${iface}/carrier"

        if [ "${carrier}" != "1" ] ; then
            echo "0"
        else
            bridge_get_speed "${iface}"
        fi
    else
        read operstate < "/sys/class/net/${iface}/operstate"
        read carrier < "/sys/class/net/${iface}/carrier"

        if [ "${operstate}" != "up" ] || [ "${carrier}" != "1" ] ; then
            echo "0"
        elif iface_is_bond "${iface}" ; then
            bond_get_speed "${iface}"
        elif iface_is_vlan "${iface}" ; then
            iface_get_speed "$(vlan_get_phy "${iface}")"
        elif iface_is_hfi1 "${iface}" ; then
            hfi1_get_speed "${iface}"
        else
            read speed < "/sys/class/net/${iface}/speed"
            echo "${speed}"
        fi
    fi
}

iface_is_vlan() {
    local iface="$1"
    [ -e "/proc/net/vlan/${iface}" ] && return 0 || return 1
}

iface_is_bridge() {
    local iface="$1"
    [ -e "/sys/class/net/${iface}/bridge" ] && return 0 || return 1
}

iface_is_bond() {
    local iface="$1"
    [ -e "/sys/class/net/${iface}/bonding" ] && return 0 || return 1
}

iface_is_hfi1() {
    local iface="$1"
    driver=$(readlink "/sys/class/net/${iface}/device/driver")
    [[ $(basename "${driver}") =~ "hfi1" ]] && return 0 || return 1
}

vlan_get_phy() {
    local iface="$1"
    sed -ne "s/^${iface} .*| *//p" < /proc/net/vlan/config
}

bridge_is_stp_enabled() {
    local iface="$1"
    local stp
    read stp < "/sys/class/net/${iface}/bridge/stp_state"
    [ "${stp}" = "1" ] && return 0 || return 1
}

bridge_get_root_ports() {
    local bridge="$1"
    local root_id
    local root_ports=""
    local bridge_id

    read root_id < "/sys/class/net/${bridge}/bridge/root_id"

    for port in /sys/class/net/${bridge}/brif/* ; do
        read bridge_id < "${port}/designated_bridge"
        if [ "${bridge_id}" = "${root_id}" ] ; then
            root_ports="${root_ports} ${port##*/}"
        fi
    done

    root_ports=${root_ports# }

    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${root_ports}"
    else # Expect sub-shell
        echo ${root_ports}
    fi
}

# From /inlude/linux/if_bridge.h:
#define BR_STATE_DISABLED 0
#define BR_STATE_LISTENING 1
#define BR_STATE_LEARNING 2
#define BR_STATE_FORWARDING 3
#define BR_STATE_BLOCKING 4

bridge_get_active_ports() {
    local bridge="$1"
    shift 1
    local ports="$*"
    local active_ports=""
    local port_state
    local stp_state
    local warn=0

    bridge_is_stp_enabled "${bridge}"
    stp_state=$?

    if [ -z "${ports}" ] || [ "${ports}" = "detect" ] ; then
        bridge_get_root_ports "${bridge}" ports
    fi

    for port in $ports ; do
        if [ ! -e "/sys/class/net/${bridge}/brif/${port}" ] ; then
            ocf_log warning "Port ${port} doesn't belong to bridge ${bridge}"
            continue
        fi
        read port_state < "/sys/class/net/${bridge}/brif/${port}/state"
        if [ "${port_state}" = "3" ] ; then
            if [ -n "${active_ports}" ] && ${stp_state} ; then
                warn=1
            fi
            active_ports="${active_ports} ${port}"
        fi
    done
    if [ ${warn} -eq 1 ] ; then
        ocf_log warning "More then one upstream port in bridge '${bridge}' is in forwarding state while STP is enabled: ${active_ports}"
    fi
    echo "${active_ports# }"
}

bridge_get_speed() {
    local iface="$1"
    local aggregate_speed=0

    if ! iface_is_bridge "${iface}" ; then
        echo 0
        return
    fi

    BGS_PORTS=$( bridge_get_active_ports "${iface}" "${OCF_RESKEY_bridge_ports}" )
    for port in ${BGS_PORTS} ; do
        : $(( aggregate_speed += $( iface_get_speed "${port}" ) ))
    done
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${aggregate_speed}"
    else # Expect sub-shell
        echo ${aggregate_speed}
    fi
}

hfi1_get_speed() {
    local iface="$1"
    local hfi1_speed
    local hfi1_value
    local hfi1_desc

    # Currently (9/14/2017 Intel Omni Path v10.5.0.0.155), Intel doesn't have
    # dual- or multiple-port Host Channel Adapters, and it's safe to use this
    # method to get the speed. Example output:
    # [root@es-host0 ~]# cat /sys/class/net/ib0/device/infiniband/*/ports/*/rate
    # 100 Gb/sec (4X EDR)
    read hfi1_speed hfi1_value hfi1_desc < "/sys/class/net/${iface}/device/infiniband"/*/ports/*/rate
    ocf_is_true "${OCF_RESKEY_debug}" && ocf_log debug "Detected speed $hfi1_speed $hfi1_value $hfi1_desc"

    # hfi1_value always in Gb/sec, so we need to convert hfi1_speed in Mb/sec
    echo $(( hfi1_speed * 1000 ))
}

bond_get_slaves() {
    local iface="$1"
    local slaves
    read slaves < "/sys/class/net/${iface}/bonding/slaves"
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${slaves}"
    else # Expect sub-shell
        echo ${slaves}
    fi
}

bond_get_active_iface() {
    local iface="$1"
    local active
    read active < "/sys/class/net/${iface}/bonding/active_slave"
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${active}"
    else # Expect sub-shell
        echo ${active}
    fi
}

bond_is_balancing() {
    local iface="$1"
    read mode mode_index < "/sys/class/net/${iface}/bonding/mode"
    ocf_is_true "${OCF_RESKEY_debug}" && ocf_log debug "Detected balancing $mode $mode_index"
    case "${mode}" in
        "balance-rr"|"balance-xor"|"802.3ad"|"balance-tlb"|"balance-alb")
            return 0
            ;;
        *)
            return 1
            ;;
    esac
}

bond_get_speed() {
    local iface="$1"
    local aggregate_speed=0
    local active_iface
    local bond_slaves

    if ! iface_is_bond "${iface}" ; then
        echo 0
        return
    fi

    bond_get_slaves "${iface}" bond_slaves

    if bond_is_balancing "${iface}" ; then
        for slave in ${bond_slaves} ; do
            : $(( aggregate_speed += $( iface_get_speed "${slave}" ) ))
        done
        # Bonding is unable to get speed*n
        : $(( aggregate_speed = aggregate_speed * 8 / 10 ))
    else
        bond_get_active_iface "${iface}" "active_iface"
        aggregate_speed=$( iface_get_speed "$active_iface" )
    fi
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${aggregate_speed}"
    else # Expect sub-shell
        echo ${aggregate_speed}
    fi
}

update() {
    local speed;
    local nic="${OCF_RESKEY_iface}";
    if [ -z "${OCF_RESKEY_iface}" ]; then
        nic=$( get_nic_name_by_ip )
        if [ -z "${nic}" ];then
            ocf_log err "Could not retrieve network interface name from ip address (${OCF_RESKEY_ip})"
            exit $OCF_ERR_GENERIC
        fi
    fi
    speed=$( iface_get_speed "${nic}" )

    : $(( score = speed * ${OCF_RESKEY_weight_base} / 1000 ))
    if [ "$__OCF_ACTION" = "start" ] ; then
        attrd_updater -n "${OCF_RESKEY_name}" -B "${score}" -d "${OCF_RESKEY_dampen}" ${attrd_options}
    else
        attrd_updater -n "${OCF_RESKEY_name}" -v "${score}" -d "${OCF_RESKEY_dampen}" ${attrd_options}
    fi
    rc=$?
    case ${rc} in
        0)
            ocf_is_true "${OCF_RESKEY_debug}" && ocf_log debug "Updated ${OCF_RESKEY_name} = ${score}"
            ;;
        *)
            ocf_log warn "Could not update ${OCF_RESKEY_name} = ${score}: rc=${rc}"
            ;;
    esac
    return ${rc}
}

case $__OCF_ACTION in
    meta-data)
        meta_data
        exit $OCF_SUCCESS
        ;;
    usage|help)
        usage
        exit $OCF_SUCCESS
        ;;
esac

if [ "$(uname)" != "Linux" ] ; then
    ocf_log err "This RA works only on linux."
    exit $OCF_ERR_INSTALLED
fi

: ${ha_pseudo_resource_name:="ifspeed-${OCF_RESOURCE_INSTANCE}"}

attrd_options='-q'
if ocf_is_true "${OCF_RESKEY_debug}" ; then
    attrd_options=''
fi

validate || exit $?

case "$__OCF_ACTION" in
    start)
        start
        ;;
    stop)
        stop
        ;;
    monitor)
        monitor
        ;;
    validate-all)
        ;;
    *)
        usage
        exit $OCF_ERR_UNIMPLEMENTED
        ;;
esac

exit $?

# vim: set filetype=sh expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=80: