Blob Blame History Raw
/*
 * Soft:        Keepalived is a failover program for the LVS project
 *              <www.linuxvirtualserver.org>. It monitor & manipulate
 *              a loadbalanced server pool using multi-layer checks.
 *
 * Part:        Checkers registration.
 *
 * Author:      Alexandre Cassen, <acassen@linux-vs.org>
 *
 *              This program is distributed in the hope that it will be useful,
 *              but WITHOUT ANY WARRANTY; without even the implied warranty of
 *              MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *              See the GNU General Public License for more details.
 *
 *              This program is free software; you can redistribute it and/or
 *              modify it under the terms of the GNU General Public License
 *              as published by the Free Software Foundation; either version
 *              2 of the License, or (at your option) any later version.
 *
 * Copyright (C) 2001-2017 Alexandre Cassen, <acassen@gmail.com>
 */

#include "config.h"

#include <dlfcn.h>
#include <stdint.h>
#include <stdio.h>
#include <arpa/inet.h>

#include "check_api.h"
#include "main.h"
#include "parser.h"
#include "utils.h"
#include "logger.h"
#include "bitops.h"
#include "global_data.h"
#include "keepalived_netlink.h"
#include "check_misc.h"
#include "check_smtp.h"
#include "check_tcp.h"
#include "check_http.h"
#include "check_ssl.h"
#include "check_dns.h"
#include "check_ping.h"
#include "check_udp.h"
#include "check_file.h"
#include "ipwrapper.h"
#include "check_daemon.h"
#ifdef _WITH_BFD_
#include "check_bfd.h"
#include "bfd_event.h"
#include "bfd_daemon.h"
#endif
#include "track_file.h"

/* Global vars */
list_head_t checkers_queue;
#ifdef _CHECKER_DEBUG_
bool do_checker_debug;
#endif

/* free checker data */
void
free_checker(checker_t *checker)
{
	list_del_init(&checker->e_list);
	(*checker->free_func) (checker);
}
void
free_checker_list(list_head_t *l)
{
	checker_t *checker, *checker_tmp;

	list_for_each_entry_safe(checker, checker_tmp, l, e_list)
		free_checker(checker);
}

/* dump checker data */
static void
dump_checker(FILE *fp, const checker_t *checker)
{
	conf_write(fp, " %s -> %s", FMT_VS(checker->vs), FMT_CHK(checker));
	conf_write(fp, "   Enabled = %s", checker->enabled ? "yes" : "no");
	conf_write(fp, "   Up = %s", checker->is_up ? "yes" : "no");
	conf_write(fp, "   Has run = %s", checker->has_run ? "yes" : "no");
	conf_write(fp, "   Alpha = %s", checker->has_run ? "yes" : "no");
	conf_write(fp, "   Delay loop = %lu us", checker->delay_loop);
	conf_write(fp, "   Warmup = %lu us", checker->warmup);
	conf_write(fp, "   Retries = %u", checker->retry);
	conf_write(fp, "   Delay before retry = %lu us", checker->delay_before_retry);
	conf_write(fp, "   Retries iterations = %u", checker->retry_it);
	conf_write(fp, "   Default delay before retry = %lu us", checker->default_delay_before_retry);
	conf_write(fp, "   Log all failures = %s", checker->log_all_failures ? "yes" : "no");

	(*checker->dump_func) (fp, checker);
}
static void
dump_checker_list(FILE *fp, const list_head_t *l)
{
	checker_t *checker;

	list_for_each_entry(checker, l, e_list)
		dump_checker(fp, checker);
}

void
dump_connection_opts(FILE *fp, const void *data)
{
	const conn_opts_t *conn = data;

	conf_write(fp, "     Dest = %s", inet_sockaddrtopair(&conn->dst));
	if (conn->bindto.ss_family)
		conf_write(fp, "     Bind to = %s", inet_sockaddrtopair(&conn->bindto));
	if (conn->bind_if[0])
		conf_write(fp, "     Bind i/f = %s", conn->bind_if);
#ifdef _WITH_SO_MARK_
	if (conn->fwmark != 0)
		conf_write(fp, "     Mark = %u", conn->fwmark);
#endif
	conf_write(fp, "     Timeout = %f", (double)conn->connection_to / TIMER_HZ);
	if (conn->last_errno)
		conf_write(fp, "     Last errno = %d", conn->last_errno);
}

void
dump_checker_opts(FILE *fp, const void *data)
{
	const checker_t *checker = data;
	const conn_opts_t *conn = checker->co;

	if (conn) {
		conf_write(fp, "   Connection");
		dump_connection_opts(fp, conn);
	}

	conf_write(fp, "   Alpha is %s", checker->alpha ? "ON" : "OFF");
	conf_write(fp, "   Log all failures %s", checker->log_all_failures ? "ON" : "OFF");
	conf_write(fp, "   Delay loop = %f" , (double)checker->delay_loop / TIMER_HZ);
	if (checker->retry) {
		conf_write(fp, "   Retry count = %u" , checker->retry);
		conf_write(fp, "   Retry delay = %f" , (double)checker->delay_before_retry / TIMER_HZ);
	}
	conf_write(fp, "   Warmup = %f", (double)checker->warmup / TIMER_HZ);

	conf_write(fp, "   Enabled = %d", checker->enabled);
	conf_write(fp, "   Is up = %d", checker->is_up);
	conf_write(fp, "   Has run = %d", checker->has_run);
	conf_write(fp, "   Retries left before fail = %u", checker->retry_it);
	conf_write(fp, "   Delay before retry = %f", (double)checker->default_delay_before_retry / TIMER_HZ);
}

/* Queue a checker into the checkers_queue */
checker_t *
queue_checker(void (*free_func) (checker_t *), void (*dump_func) (FILE *, const checker_t *)
	      , thread_func_t launch
	      , bool (*compare) (const checker_t *, checker_t *)
	      , void *data
	      , conn_opts_t *co
	      , bool fd_required)
{
	virtual_server_t *vs = list_last_entry(&check_data->vs, virtual_server_t, e_list);
	real_server_t *rs = list_last_entry(&vs->rs, real_server_t, e_list);
	checker_t *checker;

	/* Set default dst = RS, timeout = default */
	if (co) {
		co->dst = rs->addr;
		co->connection_to = UINT_MAX;
	}

	PMALLOC(checker);
	INIT_LIST_HEAD(&checker->e_list);
	checker->free_func = free_func;
	checker->dump_func = dump_func;
	checker->launch = launch;
	checker->compare = compare;
	checker->vs = vs;
	checker->rs = rs;
	checker->data = data;
	checker->co = co;
	checker->enabled = true;
	checker->alpha = -1;
	checker->delay_loop = ULONG_MAX;
	checker->warmup = ULONG_MAX;
	checker->retry = UINT_MAX;
	checker->delay_before_retry = ULONG_MAX;
	checker->retry_it = 0;
	checker->is_up = true;
	checker->default_delay_before_retry = 1 * TIMER_HZ;
	checker->default_retry = 1 ;

	/* queue the checker */
	list_add_tail(&checker->e_list, &checkers_queue);

	if (fd_required)
		check_data->num_checker_fd_required++;

	return checker;
}

void
dequeue_new_checker(void)
{
	checker_t *checker = CHECKER_GET_CURRENT();

	if (!checker->is_up)
		set_checker_state(checker, true);

	free_checker(checker);
}

bool
check_conn_opts(conn_opts_t *co)
{
	if (co->dst.ss_family == AF_INET6 &&
	    IN6_IS_ADDR_LINKLOCAL(&((struct sockaddr_in6*)&co->dst)->sin6_addr) &&
	    !co->bind_if[0]) {
		report_config_error(CONFIG_GENERAL_ERROR, "Checker link local address %s requires a bind_if", inet_sockaddrtos(&co->dst));
		return false;
	}

	return true;
}

bool __attribute__ ((pure))
compare_conn_opts(const conn_opts_t *a, const conn_opts_t *b)
{
	if (a == b)
		return true;

	if (!a || !b)
		return false;
	if (!sockstorage_equal(&a->dst, &b->dst))
		return false;
	if (!sockstorage_equal(&a->bindto, &b->bindto))
		return false;
	if (strcmp(a->bind_if, b->bind_if))
		return false;
	if (a->connection_to != b->connection_to)
		return false;
#ifdef _WITH_SO_MARK_
	if (a->fwmark != b->fwmark)
		return false;
#endif

	return true;
}

void
checker_set_dst_port(struct sockaddr_storage *dst, uint16_t port)
{
	/* NOTE: we are relying on the offset of sin_port and sin6_port being
	 * the same if an IPv6 address is specified after the port */
	if (dst->ss_family == AF_INET6) {
		struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *) dst;
		addr6->sin6_port = port;
	} else if (dst->ss_family == AF_UNSPEC &&
		   offsetof(struct sockaddr_in6, sin6_port) != offsetof(struct sockaddr_in, sin_port)) {
		log_message(LOG_INFO, "BUG: checker_set_dst_port() in/in6 port offsets differ");
	} else {
		struct sockaddr_in *addr4 = (struct sockaddr_in *) dst;
		addr4->sin_port = port;
	}
}

/* "connect_ip" keyword */
static void
co_ip_handler(const vector_t *strvec)
{
	conn_opts_t *co = CHECKER_GET_CO();

	if (inet_stosockaddr(strvec_slot(strvec, 1), NULL, &co->dst))
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid connect_ip address %s - ignoring", strvec_slot(strvec, 1));
	else if (co->bindto.ss_family != AF_UNSPEC &&
		 co->bindto.ss_family != co->dst.ss_family) {
		report_config_error(CONFIG_GENERAL_ERROR, "connect_ip address %s does not match address family of bindto - skipping", strvec_slot(strvec, 1));
		co->dst.ss_family = AF_UNSPEC;
	}
}

/* "connect_port" keyword */
static void
co_port_handler(const vector_t *strvec)
{
	conn_opts_t *co = CHECKER_GET_CO();
	unsigned port;

	if (!read_unsigned_strvec(strvec, 1, &port, 1, 65535, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid checker connect_port '%s'", strvec_slot(strvec, 1));
		return;
	}

	checker_set_dst_port(&co->dst, htons(port));
}

/* "bindto" keyword */
static void
co_srcip_handler(const vector_t *strvec)
{
	conn_opts_t *co = CHECKER_GET_CO();
	if (inet_stosockaddr(strvec_slot(strvec, 1), NULL, &co->bindto))
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid bindto address %s - ignoring", strvec_slot(strvec, 1));
	else if (co->dst.ss_family != AF_UNSPEC &&
		 co->dst.ss_family != co->bindto.ss_family) {
		report_config_error(CONFIG_GENERAL_ERROR, "bindto address %s does not match address family of connect_ip - skipping", strvec_slot(strvec, 1));
		co->bindto.ss_family = AF_UNSPEC;
	}
}

/* "bind_port" keyword */
static void
co_srcport_handler(const vector_t *strvec)
{
	conn_opts_t *co = CHECKER_GET_CO();
	unsigned port;

	if (!read_unsigned_strvec(strvec, 1, &port, 1, 65535, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid checker bind_port '%s'", strvec_slot(strvec, 1));
		return;
	}

	checker_set_dst_port(&co->bindto, htons(port));
}

/* "bind_if" keyword */
static void
co_srcif_handler(const vector_t *strvec)
{
	// This is needed for link local IPv6 bindto address
	conn_opts_t *co = CHECKER_GET_CO();

	if (strlen(strvec_slot(strvec, 1)) > sizeof(co->bind_if) - 1) {
		report_config_error(CONFIG_GENERAL_ERROR, "Interface name %s is too long - ignoring", strvec_slot(strvec, 1));
		return;
	}
	strcpy(co->bind_if, strvec_slot(strvec, 1));
}

/* "connect_timeout" keyword */
static void
co_timeout_handler(const vector_t *strvec)
{
	conn_opts_t *co = CHECKER_GET_CO();
	unsigned long timer;

	if (!read_timer(strvec, 1, &timer, 1, UINT_MAX, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "connect_timeout %s invalid - ignoring", strvec_slot(strvec, 1));
		return;
	}
	co->connection_to = timer;
}

#ifdef _WITH_SO_MARK_
/* "fwmark" keyword */
static void
co_fwmark_handler(const vector_t *strvec)
{
	conn_opts_t *co = CHECKER_GET_CO();
	unsigned fwmark;

	if (!read_unsigned_strvec(strvec, 1, &fwmark, 0, UINT_MAX, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid fwmark connection value '%s'", strvec_slot(strvec, 1));
		return;
	}
	co->fwmark = fwmark;
}
#endif

static void
retry_handler(const vector_t *strvec)
{
	checker_t *checker = CHECKER_GET_CURRENT();
	unsigned retry;

	if (!read_unsigned_strvec(strvec, 1, &retry, 0, UINT_MAX, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid retry connection value '%s'", strvec_slot(strvec, 1));
		return;
	}

	checker->retry = retry;
}

static void
delay_before_retry_handler(const vector_t *strvec)
{
	checker_t *checker = CHECKER_GET_CURRENT();
	unsigned long delay;

	if (!read_timer(strvec, 1, &delay, 0, 0, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid delay_before_retry connection value '%s'", strvec_slot(strvec, 1));
		return;
	}

	checker->delay_before_retry = delay;
}

/* "warmup" keyword */
static void
warmup_handler(const vector_t *strvec)
{
	checker_t *checker = CHECKER_GET_CURRENT();
	unsigned long warmup;

	if (!read_timer(strvec, 1, &warmup, 0, 0, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "Invalid warmup connection value '%s'", strvec_slot(strvec, 1));
		return;
	}

	checker->warmup = warmup;
}

static void
delay_handler(const vector_t *strvec)
{
	checker_t *checker = CHECKER_GET_CURRENT();
	unsigned long delay_loop;

	if (!read_timer(strvec, 1, &delay_loop, 1, 0, true)) {
		report_config_error(CONFIG_GENERAL_ERROR, "delay_loop '%s' is invalid - ignoring", strvec_slot(strvec, 1));
		return;
	}

	checker->delay_loop = delay_loop;
}

static void
alpha_handler(const vector_t *strvec)
{
	checker_t *checker = CHECKER_GET_CURRENT();
	int res = true;

	if (vector_size(strvec) >= 2) {
		res = check_true_false(strvec_slot(strvec, 1));
		if (res == -1) {
			report_config_error(CONFIG_GENERAL_ERROR, "Invalid alpha parameter %s", strvec_slot(strvec, 1));
			return;
		}
	}
	checker->alpha = res;
}
static void
log_all_failures_handler(const vector_t *strvec)
{
	checker_t *checker = CHECKER_GET_CURRENT();
	int res = true;

	if (vector_size(strvec) >= 2) {
		res = check_true_false(strvec_slot(strvec, 1));
		if (res == -1) {
			report_config_error(CONFIG_GENERAL_ERROR, "Invalid log_all_failures parameter %s", strvec_slot(strvec, 1));
			return;
		}
	}
	checker->log_all_failures = res;
}
void
install_checker_common_keywords(bool connection_keywords)
{
	if (connection_keywords) {
		install_keyword("connect_ip", &co_ip_handler);
		install_keyword("connect_port", &co_port_handler);
		install_keyword("bindto", &co_srcip_handler);
		install_keyword("bind_port", &co_srcport_handler);
		install_keyword("bind_if", &co_srcif_handler);
		install_keyword("connect_timeout", &co_timeout_handler);
#ifdef _WITH_SO_MARK_
		install_keyword("fwmark", &co_fwmark_handler);
#endif
	}
	install_keyword("retry", &retry_handler);
	install_keyword("delay_before_retry", &delay_before_retry_handler);
	install_keyword("warmup", &warmup_handler);
	install_keyword("delay_loop", &delay_handler);
	install_keyword("alpha", &alpha_handler);
	install_keyword("log_all_failures", &log_all_failures_handler);
}

/* dump the checkers_queue */
void
dump_checkers_queue(FILE *fp)
{
	if (!list_empty(&checkers_queue)) {
		conf_write(fp, "------< Health checkers >------");
		dump_checker_list(fp, &checkers_queue);
	}
}

/* init the global checkers queue */
void
init_checkers_queue(void)
{
	INIT_LIST_HEAD(&checkers_queue);
}

/* release the checkers for a virtual server */
void
free_vs_checkers(const virtual_server_t *vs)
{
	checker_t *checker, *checker_tmp;

	list_for_each_entry_safe(checker, checker_tmp, &checkers_queue, e_list) {
		if (checker->vs != vs)
			continue;

		free_checker(checker);
	}
}

/* release the checkers for a virtual server */
void
free_rs_checkers(const real_server_t *rs)
{
	checker_t *checker, *checker_tmp;

	list_for_each_entry_safe(checker, checker_tmp, &checkers_queue, e_list) {
		if (checker->rs != rs)
			continue;

		free_checker(checker);
	}
}

/* release the checkers_queue */
void
free_checkers_queue(void)
{
	free_checker_list(&checkers_queue);
}

/* register checkers to the global I/O scheduler */
void
register_checkers_thread(void)
{
	checker_t *checker;
	unsigned long warmup;

	list_for_each_entry(checker, &checkers_queue, e_list) {
		if (checker->launch)
		{
			if (checker->vs->ha_suspend && !checker->vs->ha_suspend_addr_count)
				checker->enabled = false;

			log_message(LOG_INFO, "%sctivating healthchecker for service %s for VS %s"
					    , checker->enabled ? "A" : "Dea", FMT_RS(checker->rs, checker->vs), FMT_VS(checker->vs));

			/* wait for a random timeout to begin checker thread.
			   It helps avoiding multiple simultaneous checks to
			   the same RS.
			*/
			warmup = checker->warmup;
			if (warmup) {
				/* coverity[dont_call] */
				warmup = warmup * (unsigned)random() / RAND_MAX;
			}
			thread_add_timer(master, checker->launch, checker,
					 BOOTSTRAP_DELAY + warmup);
		}
	}

#ifdef _WITH_BFD_
	log_message(LOG_INFO, "Activating BFD healthchecker");

	/* We need to always enable this, since the bfd process may write to the pipe, and we
	 * need to ensure that messages are stripped out. */
	start_bfd_monitoring(master);
#endif
}

/* Sync checkers activity with netlink kernel reflection */
static bool __attribute__ ((pure))
addr_matches(const virtual_server_t *vs, void *address)
{
	virtual_server_group_entry_t *vsg_entry;
	struct in_addr mask_addr = {0};
	struct in6_addr mask_addr6 = {{{0}}};
	unsigned addr_base;
	const void *addr;

	if (vs->addr.ss_family != AF_UNSPEC) {
		if (vs->addr.ss_family == AF_INET6)
			addr = (const void *) &((const struct sockaddr_in6 *)&vs->addr)->sin6_addr;
		else
			addr = (const void *) &((const struct sockaddr_in *)&vs->addr)->sin_addr;

		return inaddr_equal(vs->addr.ss_family, addr, address);
	}

	if (!vs->vsg || list_empty(&vs->vsg->addr_range))
		return false;

	if (vs->af == AF_INET) {
		mask_addr = *(struct in_addr*)address;
		addr_base = ntohl(mask_addr.s_addr) & 0xFF;
		mask_addr.s_addr &= htonl(0xFFFFFF00);
	} else {
		mask_addr6 = *(struct in6_addr*)address;
		addr_base = ntohs(mask_addr6.s6_addr16[7]);
		mask_addr6.s6_addr16[7] = 0;
	}

	list_for_each_entry(vsg_entry, &vs->vsg->addr_range, e_list) {
		struct sockaddr_storage range_addr = vsg_entry->addr;
		uint32_t ra_base;

		if (!vsg_entry->range) {
			if (vsg_entry->addr.ss_family == AF_INET6)
				addr = (void *) &((struct sockaddr_in6 *)&vsg_entry->addr)->sin6_addr;
			else
				addr = (void *) &((struct sockaddr_in *)&vsg_entry->addr)->sin_addr;

			if (inaddr_equal(vsg_entry->addr.ss_family, addr, address))
				return true;

			continue;
		}

		if (range_addr.ss_family == AF_INET) {
			struct in_addr ra;

			ra = ((struct sockaddr_in *)&range_addr)->sin_addr;
			ra_base = ntohl(ra.s_addr) & 0xFF;

			if (addr_base < ra_base || addr_base > ra_base + vsg_entry->range)
				continue;

			ra.s_addr &= htonl(0xFFFFFF00);
			if (ra.s_addr != mask_addr.s_addr)
				continue;
		} else {
			struct in6_addr ra = ((struct sockaddr_in6 *)&range_addr)->sin6_addr;
			ra_base = ntohs(ra.s6_addr16[7]);

			if (addr_base < ra_base || addr_base > ra_base + vsg_entry->range)
				continue;

			ra.s6_addr16[7] = 0;
			if (!inaddr_equal(AF_INET6, &ra, &mask_addr6))
				continue;
		}

		return true;
	}

	return false;
}

void
update_checker_activity(sa_family_t family, void *address, bool enable)
{
	checker_t *checker;
	virtual_server_t *vs;
	char addr_str[INET6_ADDRSTRLEN];
	bool address_logged = false;

	if (__test_bit(LOG_ADDRESS_CHANGES, &debug)) {
		inet_ntop(family, address, addr_str, sizeof(addr_str));
		log_message(LOG_INFO, "Netlink reflector reports IP %s %s"
				    , addr_str, (enable) ? "added" : "removed");
		address_logged = true;
	}

	if (!using_ha_suspend)
		return;

	if (list_empty(&checkers_queue))
		return;

	/* Check if any of the virtual servers are using this address, and have ha_suspend */
	list_for_each_entry(vs, &check_data->vs, e_list) {
		if (!vs->ha_suspend)
			continue;

		/* If there is no address configured, the family will be AF_UNSPEC */
		if (vs->af != family)
			continue;

		if (!addr_matches(vs, address))
			continue;

		if (!address_logged &&
		    __test_bit(LOG_DETAIL_BIT, &debug)) {
			inet_ntop(family, address, addr_str, sizeof(addr_str));
			log_message(LOG_INFO, "Netlink reflector reports IP %s %s"
					    , addr_str, (enable) ? "added" : "removed");
		}
		address_logged = true;

		/* If we have that same address (IPv6 link local) on multiple interfaces,
		 * we want to count them multiple times so that we only suspend the checkers
		 * if they are all deleted */
		if (enable)
			vs->ha_suspend_addr_count++;
		else
			vs->ha_suspend_addr_count--;

		/* Processing Healthcheckers queue for this vs */
		list_for_each_entry(checker, &checkers_queue, e_list) {
			if (checker->vs != vs)
				continue;

			if (enable != checker->enabled &&
			    (enable || vs->ha_suspend_addr_count == 0)) {
				log_message(LOG_INFO, "%sing healthchecker for service %s for VS %s",
							!checker->enabled ? "Activat" : "Suspend",
							FMT_RS(checker->rs, checker->vs), FMT_VS(checker->vs));
				checker->enabled = enable;
			}
		}
	}
}

/* Install checkers keywords */
void
install_checkers_keyword(void)
{
	install_misc_check_keyword();
	install_smtp_check_keyword();
	install_tcp_check_keyword();
	install_ping_check_keyword();
	install_udp_check_keyword();
	install_http_check_keyword();
	install_ssl_check_keyword();
	install_dns_check_keyword();
	install_file_check_keyword();
#ifdef _WITH_BFD_
	install_bfd_check_keyword();
#endif
}