/*
Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
This file is part of GlusterFS.
This file is licensed to you under your choice of the GNU Lesser
General Public License, version 3 or any later version (LGPLv3 or
later), or the GNU General Public License, version 2 (GPLv2), in all
cases as published by the Free Software Foundation.
*/
#include <glusterfs/common-utils.h>
#include "cli1-xdr.h"
#include "xdr-generic.h"
#include "glusterd.h"
#include "glusterd-op-sm.h"
#include "glusterd-store.h"
#include "glusterd-utils.h"
#include "glusterd-volgen.h"
#include <glusterfs/run.h>
#include <glusterfs/syscall.h>
#include <glusterfs/byte-order.h>
#include <glusterfs/compat-errno.h>
#include "glusterd-scrub-svc.h"
#include "glusterd-messages.h"
#include <sys/wait.h>
#include <dlfcn.h>
const char *gd_bitrot_op_list[GF_BITROT_OPTION_TYPE_MAX] = {
[GF_BITROT_OPTION_TYPE_NONE] = "none",
[GF_BITROT_OPTION_TYPE_ENABLE] = "enable",
[GF_BITROT_OPTION_TYPE_DISABLE] = "disable",
[GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE] = "scrub-throttle",
[GF_BITROT_OPTION_TYPE_SCRUB_FREQ] = "scrub-frequency",
[GF_BITROT_OPTION_TYPE_SCRUB] = "scrub",
[GF_BITROT_OPTION_TYPE_EXPIRY_TIME] = "expiry-time",
[GF_BITROT_OPTION_TYPE_SIGNER_THREADS] = "signer-threads",
};
int
__glusterd_handle_bitrot(rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{
0,
}};
dict_t *dict = NULL;
glusterd_op_t cli_op = GD_OP_BITROT;
char *volname = NULL;
char *scrub = NULL;
int32_t type = 0;
char msg[256] = {
0,
};
xlator_t *this = NULL;
glusterd_conf_t *conf = NULL;
GF_ASSERT(req);
this = THIS;
GF_ASSERT(this);
conf = this->private;
GF_ASSERT(conf);
ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
if (ret < 0) {
req->rpc_err = GARBAGE_ARGS;
goto out;
}
if (cli_req.dict.dict_len) {
/* Unserialize the dictionary */
dict = dict_new();
ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
&dict);
if (ret < 0) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
"failed to "
"unserialize req-buffer to dictionary");
snprintf(msg, sizeof(msg),
"Unable to decode the "
"command");
goto out;
} else {
dict->extra_stdfree = cli_req.dict.dict_val;
}
}
ret = dict_get_str(dict, "volname", &volname);
if (ret) {
snprintf(msg, sizeof(msg), "Unable to get volume name");
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get volume name, "
"while handling bitrot command");
goto out;
}
ret = dict_get_int32(dict, "type", &type);
if (ret) {
snprintf(msg, sizeof(msg), "Unable to get type of command");
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get type of cmd, "
"while handling bitrot command");
goto out;
}
if (conf->op_version < GD_OP_VERSION_3_7_0) {
snprintf(msg, sizeof(msg),
"Cannot execute command. The "
"cluster is operating at version %d. Bitrot command "
"%s is unavailable in this version",
conf->op_version, gd_bitrot_op_list[type]);
ret = -1;
goto out;
}
if (type == GF_BITROT_CMD_SCRUB_STATUS) {
/* Backward compatibility handling for scrub status command*/
if (conf->op_version < GD_OP_VERSION_3_7_7) {
snprintf(msg, sizeof(msg),
"Cannot execute command. "
"The cluster is operating at version %d. "
"Bitrot scrub status command unavailable in "
"this version",
conf->op_version);
ret = -1;
goto out;
}
ret = dict_get_str(dict, "scrub-value", &scrub);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
"Failed to get scrub value.");
ret = -1;
goto out;
}
if (!strncmp(scrub, "status", SLEN("status"))) {
ret = glusterd_op_begin_synctask(req, GD_OP_SCRUB_STATUS, dict);
goto out;
}
}
if (type == GF_BITROT_CMD_SCRUB_ONDEMAND) {
/* Backward compatibility handling for scrub status command*/
if (conf->op_version < GD_OP_VERSION_3_9_0) {
snprintf(msg, sizeof(msg),
"Cannot execute command. "
"The cluster is operating at version %d. "
"Bitrot scrub ondemand command unavailable in "
"this version",
conf->op_version);
ret = -1;
goto out;
}
ret = dict_get_str(dict, "scrub-value", &scrub);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
"Failed to get scrub value.");
ret = -1;
goto out;
}
if (!strncmp(scrub, "ondemand", SLEN("ondemand"))) {
ret = glusterd_op_begin_synctask(req, GD_OP_SCRUB_ONDEMAND, dict);
goto out;
}
}
ret = glusterd_op_begin_synctask(req, GD_OP_BITROT, dict);
out:
if (ret) {
if (msg[0] == '\0')
snprintf(msg, sizeof(msg), "Bitrot operation failed");
ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, msg);
}
return ret;
}
int
glusterd_handle_bitrot(rpcsvc_request_t *req)
{
return glusterd_big_locked_handler(req, __glusterd_handle_bitrot);
}
static int
glusterd_bitrot_scrub_throttle(glusterd_volinfo_t *volinfo, dict_t *dict,
char *key, char **op_errstr)
{
int32_t ret = -1;
char *scrub_throttle = NULL;
char *option = NULL;
xlator_t *this = NULL;
this = THIS;
GF_ASSERT(this);
ret = dict_get_str(dict, "scrub-throttle-value", &scrub_throttle);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to fetch scrub-"
"throttle value");
goto out;
}
option = gf_strdup(scrub_throttle);
ret = dict_set_dynstr(volinfo->dict, key, option);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"Failed to set option %s", key);
goto out;
}
ret = glusterd_scrubsvc_reconfigure();
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
"Failed to reconfigure scrub "
"services");
goto out;
}
out:
return ret;
}
static int
glusterd_bitrot_scrub_freq(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
char **op_errstr)
{
int32_t ret = -1;
char *scrub_freq = NULL;
xlator_t *this = NULL;
char *option = NULL;
this = THIS;
GF_ASSERT(this);
ret = dict_get_str(dict, "scrub-frequency-value", &scrub_freq);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to fetch scrub-"
"freq value");
goto out;
}
option = gf_strdup(scrub_freq);
ret = dict_set_dynstr(volinfo->dict, key, option);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"Failed to set option %s", key);
goto out;
}
ret = glusterd_scrubsvc_reconfigure();
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
"Failed to reconfigure scrub "
"services");
goto out;
}
out:
return ret;
}
static int
glusterd_bitrot_scrub(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
char **op_errstr)
{
int32_t ret = -1;
char *scrub_value = NULL;
xlator_t *this = NULL;
char *option = NULL;
this = THIS;
GF_ASSERT(this);
ret = dict_get_str(dict, "scrub-value", &scrub_value);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
"Unable to fetch scrub"
"value");
goto out;
}
if (!strcmp(scrub_value, "resume")) {
option = gf_strdup("Active");
} else {
option = gf_strdup(scrub_value);
}
ret = dict_set_dynstr(volinfo->dict, key, option);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"Failed to set option %s", key);
goto out;
}
ret = glusterd_scrubsvc_reconfigure();
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
"Failed to reconfigure scrub "
"services");
goto out;
}
out:
return ret;
}
static int
glusterd_bitrot_expiry_time(glusterd_volinfo_t *volinfo, dict_t *dict,
char *key, char **op_errstr)
{
int32_t ret = -1;
uint32_t expiry_time = 0;
xlator_t *this = NULL;
char dkey[1024] = {
0,
};
this = THIS;
GF_ASSERT(this);
ret = dict_get_uint32(dict, "expiry-time", &expiry_time);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get bitrot expiry"
" timer value.");
goto out;
}
snprintf(dkey, sizeof(dkey), "%d", expiry_time);
ret = dict_set_dynstr_with_alloc(volinfo->dict, key, dkey);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"Failed to set option %s", key);
goto out;
}
ret = glusterd_bitdsvc_reconfigure();
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITDSVC_RECONF_FAIL,
"Failed to reconfigure bitrot"
"services");
goto out;
}
out:
return ret;
}
static gf_boolean_t
is_bitd_configure_noop(xlator_t *this, glusterd_volinfo_t *volinfo)
{
gf_boolean_t noop = _gf_true;
glusterd_brickinfo_t *brickinfo = NULL;
if (!glusterd_is_bitrot_enabled(volinfo))
goto out;
else if (volinfo->status != GLUSTERD_STATUS_STARTED)
goto out;
else {
cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
{
if (!glusterd_is_local_brick(this, volinfo, brickinfo))
continue;
noop = _gf_false;
return noop;
}
}
out:
return noop;
}
static int
glusterd_bitrot_signer_threads(glusterd_volinfo_t *volinfo, dict_t *dict,
char *key, char **op_errstr)
{
int32_t ret = -1;
uint32_t signer_th_count = 0;
uint32_t existing_th_count = 0;
xlator_t *this = NULL;
glusterd_conf_t *priv = NULL;
char dkey[32] = {
0,
};
this = THIS;
GF_ASSERT(this);
priv = this->private;
GF_VALIDATE_OR_GOTO(this->name, priv, out);
ret = dict_get_uint32(dict, "signer-threads", &signer_th_count);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get bitrot signer thread count.");
goto out;
}
ret = dict_get_uint32(volinfo->dict, key, &existing_th_count);
if (ret == 0 && signer_th_count == existing_th_count) {
goto out;
}
snprintf(dkey, sizeof(dkey), "%d", signer_th_count);
ret = dict_set_dynstr_with_alloc(volinfo->dict, key, dkey);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"Failed to set option %s", key);
goto out;
}
if (!is_bitd_configure_noop(this, volinfo)) {
ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL,
PROC_START_NO_WAIT);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITDSVC_RECONF_FAIL,
"Failed to reconfigure bitrot services");
goto out;
}
}
out:
return ret;
}
static int
glusterd_bitrot_enable(glusterd_volinfo_t *volinfo, char **op_errstr)
{
int32_t ret = -1;
xlator_t *this = NULL;
this = THIS;
GF_ASSERT(this);
GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
if (glusterd_is_volume_started(volinfo) == 0) {
*op_errstr = gf_strdup(
"Volume is stopped, start volume "
"to enable bitrot.");
ret = -1;
goto out;
}
ret = glusterd_is_bitrot_enabled(volinfo);
if (ret) {
*op_errstr = gf_strdup("Bitrot is already enabled");
ret = -1;
goto out;
}
ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_BITROT, "on");
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"dict set failed");
goto out;
}
/*Once bitrot is enable scrubber should be in Active state*/
ret = dict_set_dynstr_with_alloc(volinfo->dict, "features.scrub", "Active");
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"Failed to set option "
"features.scrub value");
goto out;
}
ret = 0;
out:
if (ret && op_errstr && !*op_errstr)
gf_asprintf(op_errstr,
"Enabling bitrot on volume %s has been "
"unsuccessful",
volinfo->volname);
return ret;
}
static int
glusterd_bitrot_disable(glusterd_volinfo_t *volinfo, char **op_errstr)
{
int32_t ret = -1;
xlator_t *this = NULL;
this = THIS;
GF_VALIDATE_OR_GOTO("glusterd", this, out);
GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_BITROT,
"off");
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"dict set failed");
goto out;
}
/*Once bitrot disabled scrubber should be Inactive state*/
ret = dict_set_dynstr_with_alloc(volinfo->dict, "features.scrub",
"Inactive");
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
"Failed to set "
"features.scrub value");
goto out;
}
ret = 0;
out:
if (ret && op_errstr && !*op_errstr)
gf_asprintf(op_errstr,
"Disabling bitrot on volume %s has "
"been unsuccessful",
volinfo->volname);
return ret;
}
gf_boolean_t
glusterd_should_i_stop_bitd()
{
glusterd_conf_t *conf = THIS->private;
glusterd_volinfo_t *volinfo = NULL;
gf_boolean_t stopped = _gf_true;
glusterd_brickinfo_t *brickinfo = NULL;
xlator_t *this = NULL;
this = THIS;
GF_ASSERT(this);
cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
{
if (!glusterd_is_bitrot_enabled(volinfo))
continue;
else if (volinfo->status != GLUSTERD_STATUS_STARTED)
continue;
else {
cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
{
if (!glusterd_is_local_brick(this, volinfo, brickinfo))
continue;
stopped = _gf_false;
return stopped;
}
/* Before stopping bitrot/scrubber daemon check
* other volume also whether respective volume
* host a brick from this node or not.*/
continue;
}
}
return stopped;
}
static int
glusterd_manage_bitrot(int opcode)
{
int ret = -1;
xlator_t *this = NULL;
glusterd_conf_t *priv = NULL;
this = THIS;
GF_ASSERT(this);
priv = this->private;
GF_ASSERT(priv);
switch (opcode) {
case GF_BITROT_OPTION_TYPE_ENABLE:
case GF_BITROT_OPTION_TYPE_DISABLE:
ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL,
PROC_START_NO_WAIT);
if (ret)
break;
ret = priv->scrub_svc.manager(&(priv->scrub_svc), NULL,
PROC_START_NO_WAIT);
break;
default:
ret = 0;
break;
}
return ret;
}
int
glusterd_op_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
{
glusterd_volinfo_t *volinfo = NULL;
int32_t ret = -1;
char *volname = NULL;
int type = -1;
glusterd_conf_t *priv = NULL;
xlator_t *this = NULL;
GF_ASSERT(dict);
GF_ASSERT(op_errstr);
this = THIS;
GF_ASSERT(this);
priv = this->private;
GF_ASSERT(priv);
ret = dict_get_str(dict, "volname", &volname);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get volume name");
goto out;
}
ret = glusterd_volinfo_find(volname, &volinfo);
if (ret) {
gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
goto out;
}
ret = dict_get_int32(dict, "type", &type);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get type from "
"dict");
goto out;
}
switch (type) {
case GF_BITROT_OPTION_TYPE_ENABLE:
ret = glusterd_bitrot_enable(volinfo, op_errstr);
if (ret < 0)
goto out;
break;
case GF_BITROT_OPTION_TYPE_DISABLE:
ret = glusterd_bitrot_disable(volinfo, op_errstr);
if (ret < 0)
goto out;
break;
case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
ret = glusterd_bitrot_scrub_throttle(
volinfo, dict, "features.scrub-throttle", op_errstr);
if (ret)
goto out;
break;
case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
ret = glusterd_bitrot_scrub_freq(volinfo, dict,
"features.scrub-freq", op_errstr);
if (ret)
goto out;
break;
case GF_BITROT_OPTION_TYPE_SCRUB:
ret = glusterd_bitrot_scrub(volinfo, dict, "features.scrub",
op_errstr);
if (ret)
goto out;
break;
case GF_BITROT_OPTION_TYPE_EXPIRY_TIME:
ret = glusterd_bitrot_expiry_time(
volinfo, dict, "features.expiry-time", op_errstr);
if (ret)
goto out;
break;
case GF_BITROT_OPTION_TYPE_SIGNER_THREADS:
ret = glusterd_bitrot_signer_threads(
volinfo, dict, "features.signer-threads", op_errstr);
if (ret)
goto out;
break;
case GF_BITROT_CMD_SCRUB_STATUS:
case GF_BITROT_CMD_SCRUB_ONDEMAND:
break;
default:
gf_asprintf(op_errstr,
"Bitrot command failed. Invalid "
"opcode");
ret = -1;
goto out;
}
ret = glusterd_manage_bitrot(type);
if (ret)
goto out;
ret = glusterd_create_volfiles_and_notify_services(volinfo);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
"Unable to re-create "
"volfiles");
ret = -1;
goto out;
}
ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
if (ret) {
gf_msg_debug(this->name, 0,
"Failed to store volinfo for "
"bitrot");
goto out;
}
out:
return ret;
}
int
glusterd_op_stage_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
{
int ret = 0;
char *volname = NULL;
char *scrub_cmd = NULL;
char *scrub_cmd_from_dict = NULL;
char msg[2048] = {
0,
};
int type = 0;
xlator_t *this = NULL;
glusterd_conf_t *priv = NULL;
glusterd_volinfo_t *volinfo = NULL;
this = THIS;
GF_ASSERT(this);
priv = this->private;
GF_ASSERT(priv);
GF_ASSERT(dict);
GF_ASSERT(op_errstr);
ret = dict_get_str(dict, "volname", &volname);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get volume name");
goto out;
}
ret = glusterd_volinfo_find(volname, &volinfo);
if (ret) {
gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
goto out;
}
if (!glusterd_is_volume_started(volinfo)) {
*op_errstr = gf_strdup(
"Volume is stopped, start volume "
"before executing bit rot command.");
ret = -1;
goto out;
}
ret = dict_get_int32(dict, "type", &type);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to get type for "
"operation");
*op_errstr = gf_strdup(
"Staging stage failed for bitrot "
"operation.");
goto out;
}
if ((GF_BITROT_OPTION_TYPE_ENABLE != type) &&
(glusterd_is_bitrot_enabled(volinfo) == 0)) {
ret = -1;
gf_asprintf(op_errstr, "Bitrot is not enabled on volume %s", volname);
goto out;
}
if ((GF_BITROT_OPTION_TYPE_SCRUB == type)) {
ret = dict_get_str(volinfo->dict, "features.scrub",
&scrub_cmd_from_dict);
if (!ret) {
ret = dict_get_str(dict, "scrub-value", &scrub_cmd);
if (ret) {
gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
"Unable to "
"get scrub-value");
*op_errstr = gf_strdup(
"Staging failed for "
"bitrot operation. "
"Please check log file"
" for more details.");
goto out;
}
/* If scrubber is resume then value of scrubber will be
* "Active" in the dictionary. */
if (!strcmp(scrub_cmd_from_dict, scrub_cmd) ||
(!strncmp("Active", scrub_cmd_from_dict, SLEN("Active")) &&
!strncmp("resume", scrub_cmd, SLEN("resume")))) {
snprintf(msg, sizeof(msg),
"Scrub is already"
" %sd for volume %s",
scrub_cmd, volinfo->volname);
*op_errstr = gf_strdup(msg);
ret = -1;
goto out;
}
}
ret = 0;
}
out:
if (ret && op_errstr && *op_errstr)
gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_BITROT_FAIL, "%s",
*op_errstr);
gf_msg_debug(this->name, 0, "Returning %d", ret);
return ret;
}