/* BEGIN_ICS_COPYRIGHT7 ****************************************
Copyright (c) 2015-2020, Intel Corporation
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Intel Corporation nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
** END_ICS_COPYRIGHT7 ****************************************/
/* [ICS VERSION STRING: unknown] */
//===========================================================================//
// //
// FILE NAME //
// sm_fsm.c //
// //
// DESCRIPTION //
// This thread implements the SM state machine. The details can //
// be found in Volume 1, Section 14.4.1 //
// //
// DATA STRUCTURES //
// None //
// //
// FUNCTIONS //
// state_thread main //
// //
// DEPENDENCIES //
// ib_mad.h //
// ib_status.h //
// //
// //
//===========================================================================//
// JSY - do we need to check the SM_Key for every packet received?
#include "os_g.h"
#include "ib_mad.h"
#include "ib_status.h"
#include "cs_g.h"
#include "sm_l.h"
#include "sm_dbsync.h"
#ifdef IB_STACK_OPENIB
#include "mal_g.h"
#endif
#include "time.h"
#ifdef __VXWORKS__
#include "UiUtil.h"
#endif
#define LOCAL_MOD_ID VIEO_SM_MOD_ID
extern SmMaiHandle_t *fd_async;
extern SmMaiHandle_t *fd_sminfo;
extern uint64_t topology_wakeup_time;
extern Status_t sm_fsm_standby(Mai_t *, char *);
extern Status_t sm_fsm_master(Mai_t *, char *);
extern Status_t sm_fsm_notactive(Mai_t *, char *);
extern Status_t sm_fsm_discovering(Mai_t *, char *);
extern Status_t sm_fsm_default(Mai_t *, char *);
//---------------------------------------------------------------------------//
static char *getAmod (uint32_t amod) {
switch(amod) {
case SM_AMOD_HANDOVER:
return("HANDOVER");
case SM_AMOD_ACKNOWLEDGE:
return("HANDOVER_ACK");
case SM_AMOD_DISABLE:
return("DISABLE");
case SM_AMOD_STANDBY:
return("STANDBY");
case SM_AMOD_DISCOVER:
return("DISCOVER");
default:
return("INVALID AMOD");
}
}
Status_t
state_event_timeout(void) {
uint64_t now;
Status_t status;
static uint64_t lastStandbyCheck=0;
IB_ENTER(__func__, 0, 0, 0, 0);
//
// If this is the STANDBY state, then we need to heartbeat the Master.
// If this is the MASTER or DISCOVERING state, then we need to do a topology sweep.
//
// Note that sweeping in DISCOVERING is required to avoid deadlocks when we
// abort a sweep before transitioning out of DISCOVERING.
//
if (lastStandbyCheck == 0) {
(void)vs_time_get(&lastStandbyCheck);
}
if (sm_state == SM_STATE_STANDBY) {
#if defined(IB_STACK_OPENIB)
status = ib_refresh_devport();
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("cannot refresh sm pkeys rc:", status);
}
#endif
status = sm_check_Master();
if (status != VSTATUS_OK) {
/* Master in trouble, start discovery */
(void)sm_transition(SM_STATE_DISCOVERING);
sm_trigger_sweep(SM_SWEEP_REASON_MASTER_TROUBLE);
}
} else if (sm_state == SM_STATE_DISCOVERING || sm_state == SM_STATE_MASTER) {
(void)vs_time_get(&now);
if ((topology_wakeup_time != 0ull) && (now >= topology_wakeup_time)) {
topology_wakeup_time = 0ull;
lastStandbyCheck = now;
sm_trigger_sweep(SM_SWEEP_REASON_SCHEDULED);
} else if (sm_config.monitor_standby_enable &&
topology_wakeup_time != 0ull &&
sm_state == SM_STATE_MASTER &&
((now - lastStandbyCheck) > sm_masterCheckInterval * sm_config.master_ping_max_fail)) {
sm_dbsync_standbyCheck();
lastStandbyCheck = now;
}
} else if (sm_state == SM_STATE_NOTACTIVE) {
// Notify master sm that we are going away. Most HSMs and the X switches
// don't currently support capabilityMask trap indicating IsSm off.
status = sm_check_Master();
}
IB_EXIT(__func__, VSTATUS_OK);
return(VSTATUS_OK);
}
//---------------------------------------------------------------------------//
Status_t
state_event_mad(Mai_t *maip) {
Status_t status, mkeyCheckStatus, dbsyncCheckStat=VSTATUS_OK;
STL_SM_INFO theirSmInfo, ourSmInfo;
uint64_t mkey=0, portguid=0;
Node_t *nodep=NULL;
Port_t *portp=NULL;
char nodescription[64]={0};
int identified=0;
boolean uptodate = 0;
SmRecp smrecp = NULL;
IB_ENTER(__func__, maip, 0, 0, 0);
status = VSTATUS_OK;
//
// Redundancy check on the AID.
//
if (maip->base.aid != MAD_SMA_SMINFO) {
IB_LOG_ERROR("not an SMInfo MAD aid:", maip->base.aid);
IB_EXIT(__func__, VSTATUS_BAD);
return(VSTATUS_BAD);
}
/*
* get requester's sm info
* ONLY OUR SM (as far as I know) provides its smInfo in the LR GET request
* we'll use that to find the standby sm node in topology (only if passcount > 0)
*/
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)stl_mai_get_smp_data(maip), &theirSmInfo);
/* get requester's nodeInfo using the portGuid from the smInfo request */
portguid = theirSmInfo.PortGUID;
if (sm_state == SM_STATE_MASTER && topology_passcount > 0) {
(void)vs_rdlock(&old_topology_lock);
if (portguid != 0) {
if ((portp = sm_find_port_guid(&old_topology, portguid)) != NULL)
nodep = sm_find_port_node(&old_topology, portp);
}
if (nodep == NULL) {
if (maip->addrInfo.slid != STL_LID_PERMISSIVE) {
/* try find by lid */
portp = sm_find_node_and_port_lid(&old_topology, maip->addrInfo.slid, &nodep);
}
} else if (portguid != 0 && portguid != sm_smInfo.PortGUID) {
identified = 1; /* remote included his identity */
}
if (nodep) {
/* get all data that we need before releasing topology lock */
(void)memcpy(nodescription, nodep->nodeDesc.NodeString, 64);
if (portp->portData) portguid = portp->portData->guid;
}
(void)vs_rwunlock(&old_topology_lock);
if ((identified || (portguid != 0)) && sm_smInfo.PortGUID != portguid) {
/* see if it's time to do a full sync of this SM */
dbsyncCheckStat = sm_dbsync_fullSyncCheck(portguid);
/* get it's current dbsync state */
uptodate = sm_dbsync_isUpToDate(portguid, NULL);
}
}
/*
* For 1.1 (C14-53), SM in the NOTACTIVE state, replies to SubnGet/Set of SMInfo
* Do an MKey check
*/
if ((mkeyCheckStatus = sm_mkey_check(maip, &mkey)) == VSTATUS_MISMATCH) {
IB_LOG_WARN_FMT(__func__, "Mismatch mKey["FMT_U64"] SMInfo from node %s with , lid[0x%x], portguid "FMT_U64", TID="FMT_U64,
mkey, nodescription, maip->addrInfo.slid, portguid, maip->base.tid);
IB_EXIT(__func__, VSTATUS_BAD);
return(VSTATUS_BAD);
}
/*
* If this is a Get method, then send the requestor our SMInfo.
*/
if (maip->base.method == MAD_CM_GET) {
if (smDebugPerf) {
IB_LOG_INFINI_INFO_FMT(__func__, "got SMInfo GET request from node %s, lid[0x%x], portguid "FMT_U64", TID="FMT_U64,
nodescription, maip->addrInfo.slid, portguid, maip->base.tid);
}
sm_smInfo.ActCount++;
/*
* Do not return our SMkey if the requester fails the Mkey check
*/
memcpy((void *)&ourSmInfo, (void *)&sm_smInfo, sizeof(sm_smInfo));
if (mkeyCheckStatus == VSTATUS_CONDITIONAL) {
ourSmInfo.SM_Key = 0;
}
// If this SM is Master, then fill in the elapsed time.
if (ourSmInfo.u.s.SMStateCurrent == SM_STATE_MASTER) {
ourSmInfo.ElapsedTime = (uint32_t)difftime(time(NULL),sm_masterStartTime);
} else {
ourSmInfo.ElapsedTime = 0;
}
BSWAPCOPY_STL_SM_INFO(&ourSmInfo, (STL_SM_INFO *)stl_mai_get_smp_data(maip));
if ((status = mai_stl_reply(fd_async->fdMai, maip, sizeof(STL_SM_INFO))) != VSTATUS_OK) {
IB_LOG_WARN_FMT(__func__, "failed to send reply [status=%d] to SMInfo GET request from node %s, lid[0x%x], portguid "FMT_U64", TID="FMT_U64,
status, nodescription, maip->addrInfo.slid, portguid, maip->base.tid);
}
/*
* THIS IS NOT IN THE SPEC! GET request does not really include the requester's own smInfo
* We made our SM work this way so that the handover would happen right away instead of waiting
* until the next sweep. If sweep is turned off, you may never handover!
*
* If we're master and get is from higher priority SM in standby or higher, force sweep so we'll handover
*/
if (identified && sm_smInfo.u.s.SMStateCurrent == SM_STATE_MASTER)
switch (theirSmInfo.u.s.SMStateCurrent) {
case SM_STATE_MASTER:
case SM_STATE_STANDBY:
/* If this is an SM that is not in our list, trigger a sweep to add it */
/* lock out SM record table */
if ((status = vs_lock(&smRecords.smLock)) != VSTATUS_OK) {
IB_LOG_ERROR("Can't lock SM Record table, rc:", status);
} else {
if ((smrecp = (SmRecp)cs_hashtable_search(smRecords.smMap, &theirSmInfo.PortGUID)) != NULL) {
//trigger sweep if theirSmInfo.u.s.SMStateCurrent changed
if (smrecp->smInfoRec.SMInfo.u.s.SMStateCurrent != theirSmInfo.u.s.SMStateCurrent) {
IB_LOG_INFINI_INFO_FMT(__func__,
"triggering a sweep, remote SM lid[0x%x] state changed from %s to %s",
maip->addrInfo.dlid, sm_getStateText(smrecp->smInfoRec.SMInfo.u.s.SMStateCurrent),
sm_getStateText(theirSmInfo.u.s.SMStateCurrent));
sm_trigger_sweep(SM_SWEEP_REASON_UPDATED_STANDBY);
}
} else {
sm_trigger_sweep(SM_SWEEP_REASON_UNEXPECTED_SM);
}
(void)vs_unlock(&smRecords.smLock);
}
if ((sm_smInfo.u.s.Priority < theirSmInfo.u.s.Priority) ||
(sm_smInfo.u.s.Priority == theirSmInfo.u.s.Priority && sm_smInfo.PortGUID > theirSmInfo.PortGUID)) {
if (uptodate) {
(void)vs_lock(&handover_sent_lock);
/* clear flags if a handover has been sent*/
if (handover_sent) {
handover_sent = 0;
triggered_handover = 0;
}
(void)vs_unlock(&handover_sent_lock);
/* Prevent multiple triggers for handover sweep while the topo thread
* is stll sweeping to do a handover due to our previous trigger.
* With this even if we get multiple Get SmInfo requests (due to
* standby SM pings) while topo thread is still processing our
* previous trigger, we do not cause multiple triggers.
*/
if (!triggered_handover) {
IB_LOG_INFINI_INFO_FMT(__func__,
"triggering a sweep to hand over to node %s, lid[0x%x], portguid "FMT_U64", TID="FMT_U64,
nodescription, maip->addrInfo.dlid, theirSmInfo.PortGUID, maip->base.tid); // lids got swapped by mai_reply
sm_trigger_sweep(SM_SWEEP_REASON_HANDOFF);
triggered_handover = 1;
}
}
} else {
sm_dbsync_standbyHello(theirSmInfo.PortGUID);
}
break;
case SM_STATE_DISCOVERING:
if (dbsyncCheckStat != VSTATUS_NOT_FOUND) {
IB_LOG_INFINI_INFO_FMT(__func__,
"Standby SM in Discovery, may have been restarted, node %s, lid[0x%x], portguid "FMT_U64,
nodescription, maip->addrInfo.dlid, theirSmInfo.PortGUID);
/* If it restarted, forget what we think we know about it */
sm_dbsync_deleteSm(theirSmInfo.PortGUID);
}
break;
case SM_STATE_NOTACTIVE:
IB_LOG_INFINI_INFO_FMT(__func__,
"standby SM indicating no longer active, node %s, lid[0x%x], portguid "FMT_U64,
nodescription, maip->addrInfo.dlid, theirSmInfo.PortGUID);
sm_dbsync_deleteSm(theirSmInfo.PortGUID);
break;
} // end of case
IB_EXIT(__func__, VSTATUS_OK);
return(VSTATUS_OK);
} // end if MAD_CM_GET
if (smDebugPerf) {
IB_LOG_INFINI_INFO_FMT(__func__, "got SMInfo SET request from node %s, Lid [0x%x], portguid "FMT_U64", TID="FMT_U64,
nodescription, maip->addrInfo.slid, portguid, maip->base.tid);
}
/*
* This is now a Set method. Let's check the transitions. C14-38.1.1
*/
if ( (maip->base.amod != SM_AMOD_ACKNOWLEDGE && theirSmInfo.u.s.SMStateCurrent != SM_STATE_MASTER) ) {
maip->base.status = MAD_STATUS_BAD_FIELD;
IB_LOG_WARN_FMT(__func__,
"SmInfo SET control packet not from a Master SM on node %s, lid [0x%x], portguid "FMT_U64", TID="FMT_U64,
nodescription, maip->addrInfo.slid, portguid, maip->base.tid);
if ((status = mai_stl_reply(fd_async->fdMai, maip, sizeof(STL_SM_INFO))) != VSTATUS_OK)
IB_LOG_WARN_FMT(__func__,
"failed to send reply [status=%d] to SMInfo SET request from node %s, lid [0x%x], portguid "FMT_U64", TID="FMT_U64,
status, nodescription, maip->addrInfo.slid, portguid, maip->base.tid);
IB_EXIT(__func__, VSTATUS_OK);
return(VSTATUS_OK);
}
switch (sm_state) {
case SM_STATE_STANDBY:
(void)sm_fsm_standby(maip, nodescription);
break;
case SM_STATE_NOTACTIVE:
(void)sm_fsm_notactive(maip, nodescription);
break;
case SM_STATE_MASTER:
(void)sm_fsm_master(maip, nodescription);
break;
case SM_STATE_DISCOVERING:
(void)sm_fsm_discovering(maip, nodescription);
break;
default:
(void)sm_fsm_default(maip, nodescription);
break;
}
IB_EXIT(__func__, status);
return(status);
}
Status_t
sm_fsm_standby(Mai_t *maip, char *nodename)
{
int i;
uint8_t ipath[64];
uint8_t *rpath;
uint8_t *path;
Status_t status;
long new_state=-1;
STL_SM_INFO theirSmInfo;
STL_SM_INFO smInfoCopy;
STL_LID slid=maip->addrInfo.slid; // original source lid; mai_reply swaps slid-dlid
STL_LID dlid=maip->addrInfo.dlid; // original destination lid(us); mai_reply swaps slid-dlid
IB_ENTER(__func__, maip->base.amod, 0, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)stl_mai_get_smp_data(maip), &theirSmInfo);
switch (maip->base.amod) {
case SM_AMOD_DISCOVER: // C14-48
new_state = SM_STATE_DISCOVERING;
break;
case SM_AMOD_DISABLE: // C14-19
new_state = SM_STATE_NOTACTIVE;
break;
case SM_AMOD_HANDOVER: // C14-50
new_state = SM_STATE_MASTER;
break;
case SM_AMOD_ACKNOWLEDGE: // from previous HANDOVER
break;
default:
maip->base.status = MAD_STATUS_BAD_ATTR;
break;
}
if (maip->base.status > MAD_STATUS_OK) {
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid AMOD[%d] from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), maip->base.amod, nodename,
maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
} else {
IB_LOG_INFINI_INFO_FMT(__func__,
"SM in [%s] state after processing %s smInfo control from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
(new_state < 0 ? sm_getStateText(sm_smInfo.u.s.SMStateCurrent):sm_getStateText(new_state)), getAmod(maip->base.amod),
nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
}
/*
* Reply to this Set(SMInfo).
*/
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)stl_mai_get_smp_data(maip));
status = mai_stl_reply(fd_async->fdMai, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_standby - bad mai_reply rc:", status);
}
(void)vs_lock(&new_topology_lock);
/* make appropriate none master transition if necessary */
if (new_state >= SM_STATE_NOTACTIVE && new_state != SM_STATE_MASTER) {
(void)sm_transition(new_state);
if (new_state == SM_STATE_DISCOVERING) {
sm_trigger_sweep(SM_SWEEP_REASON_STATE_TRANSITION); // wakeup the topology thread to start sweep
}
}
/*
* If this was a HANDOVER, we need to ACK back.
*/
if (maip->base.amod == SM_AMOD_HANDOVER) {
maip->addrInfo.destqp &= 0x00ffffff;
if (maip->base.mclass == MAD_CV_SUBN_LR) {
path = NULL;
IB_LOG_INFINI_INFO_FMT(__func__, "sending LR HANDOVER ACK to node %s, Lid [0x%x], portguid "FMT_U64,
nodename, slid, theirSmInfo.PortGUID);
} else {
DRStlSmp_t *drsmp = (DRStlSmp_t *)maip->data;
path = ipath;
memset((void *)ipath, 0, 64);
ipath[0] = maip->base.hopCount;
rpath = drsmp->RetPath;
for (i = 1; i <= maip->base.hopCount; i++) {
ipath[i] = rpath[maip->base.hopCount + 1 - i];
}
IB_LOG_INFINI_INFO_FMT(__func__, "sending DR HANDOVER ACK to node %s, portguid "FMT_U64,
nodename, theirSmInfo.PortGUID);
}
smInfoCopy = sm_smInfo;
// slid: this is original sender of SmInfo
// dlid: this is us
SmpAddr_t addr = SMP_ADDR_CREATE(path, dlid, slid);
status = SM_Set_SMInfo(fd_sminfo, SM_AMOD_ACKNOWLEDGE, &addr, &smInfoCopy, sm_config.mkey);
if (status != VSTATUS_OK) {
IB_LOG_WARN_FMT(__func__,
"[%s] SM did not receive response to Handover Acknowledgement from SM node %s, LID [0x%x], portguid ["FMT_U64"]",
sm_getStateText(new_state), nodename, slid, theirSmInfo.PortGUID);
}
/* make transition to MASTER state */
(void)sm_transition(new_state);
sm_trigger_sweep(SM_SWEEP_REASON_STATE_TRANSITION);
}
(void)vs_unlock(&new_topology_lock);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
Status_t
sm_fsm_notactive(Mai_t *maip, char *nodename)
{
Status_t status;
long new_state=-1;
STL_SM_INFO theirSmInfo;
IB_ENTER(__func__, maip->base.amod, sm_state, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)stl_mai_get_smp_data(maip), &theirSmInfo);
switch (maip->base.amod) {
case SM_AMOD_STANDBY: // C14-54.1.1
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM received request to transition to STANDBY from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
new_state = SM_STATE_STANDBY;
break;
case SM_AMOD_DISABLE:
IB_LOG_WARN_FMT(__func__,
"[%s] SM received request to transition to NOTACTIVE from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
new_state = SM_STATE_NOTACTIVE;
break;
default:
maip->base.status = MAD_STATUS_BAD_ATTR;
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid transition request %s (%u) from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), maip->base.amod,
nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
break;
}
/*
* Reply to this Set(SMInfo).
*/
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)stl_mai_get_smp_data(maip));
status = mai_stl_reply(fd_async->fdMai, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_notactive - bad mai_reply rc:", status);
}
/* make appropriate transition if necessary */
if (new_state >= SM_STATE_NOTACTIVE)
(void)sm_transition(new_state);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
Status_t
sm_fsm_master(Mai_t *maip, char *nodename)
{
Status_t status;
long new_state=-1;
STL_SM_INFO theirSmInfo;
uint8_t ipath[64];
uint8_t *rpath;
uint8_t *path;
int i, wakeTpThread=0;
STL_SM_INFO smInfoCopy;
STL_LID slid=maip->addrInfo.slid; // original source lid; mai_reply swaps slid-dlid
STL_LID dlid=maip->addrInfo.dlid; // original destination lid(us); mai_reply swaps slid-dlid
IB_ENTER(__func__, maip->base.amod, sm_state, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)stl_mai_get_smp_data(maip), &theirSmInfo);
switch (maip->base.amod) {
case SM_AMOD_HANDOVER: // C14-61
new_state = SM_STATE_MASTER;
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM received '%s' from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
break;
case SM_AMOD_ACKNOWLEDGE:
if (theirSmInfo.u.s.SMStateCurrent < SM_STATE_STANDBY) { // C14-38.1.1 almost
maip->base.status = MAD_STATUS_BAD_FIELD;
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid Handover Ack from remote SM %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64"; remote not in STANDBY state [%s]",
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid, sm_getStateText(theirSmInfo.u.s.SMStateCurrent));
} else {
new_state = SM_STATE_STANDBY;
wakeTpThread = 1;
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM received '%s' from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
}
break;
default:
maip->base.status = MAD_STATUS_BAD_ATTR;
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid MASTER transition [%s] from remote [%s] SM %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), sm_getStateText(theirSmInfo.u.s.SMStateCurrent), nodename,
maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
break;
}
/*
* Reply to this Set(SMInfo).
*/
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)stl_mai_get_smp_data(maip));
status = mai_stl_reply(fd_async->fdMai, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_master - bad mai_reply rc:", status);
}
(void)vs_lock(&new_topology_lock);
/* make appropriate transition if necessary */
if (new_state >= SM_STATE_NOTACTIVE && new_state != sm_state)
(void)sm_transition(new_state);
if (wakeTpThread) {
/* wakeup the topology thread to clean up SA tables */
sm_trigger_sweep(SM_SWEEP_REASON_HANDOFF);
}
/*
* If this was a HANDOVER, we need to ACK back.
*/
if (maip->base.amod == SM_AMOD_HANDOVER) {
maip->addrInfo.destqp &= 0x00ffffff;
if (maip->base.mclass == MAD_CV_SUBN_LR) {
path = NULL;
IB_LOG_INFINI_INFO_FMT(__func__, "sending LR HANDOVER ACK to node %s, Lid [0x%x], portguid "FMT_U64,
nodename, slid, theirSmInfo.PortGUID);
} else {
DRStlSmp_t *drsmp = (DRStlSmp_t *)maip->data;
path = ipath;
memset((void *)ipath, 0, 64);
ipath[0] = maip->base.hopCount;
rpath = drsmp->RetPath;
for (i = 1; i <= maip->base.hopCount; i++) {
ipath[i] = rpath[maip->base.hopCount + 1 - i];
}
IB_LOG_INFINI_INFO_FMT(__func__, "sending DR HANDOVER ACK to node %s, portguid "FMT_U64,
nodename, theirSmInfo.PortGUID);
}
smInfoCopy = sm_smInfo;
// slid: this is original sender of SmInfo
// dlid: this is us
SmpAddr_t addr = SMP_ADDR_CREATE(path, dlid, slid);
status = SM_Set_SMInfo(fd_sminfo, SM_AMOD_ACKNOWLEDGE, &addr, &smInfoCopy, sm_config.mkey);
if (status != VSTATUS_OK) {
IB_LOG_WARN_FMT(__func__,
"[%s] SM did not receive response to Handover Acknowledgement from [%s] SM node %s, LID [0x%x], portguid ["FMT_U64"]",
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), sm_getStateText(theirSmInfo.u.s.SMStateCurrent), nodename, slid, theirSmInfo.PortGUID);
} else {
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM successfully acknowledged Handover from remote SM node %s, LID [0x%x], portguid ["FMT_U64"]",
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, slid, theirSmInfo.PortGUID);
}
/* make transition to MASTER state */
(void)sm_transition(new_state);
sm_trigger_sweep(SM_SWEEP_REASON_STATE_TRANSITION); // wakeup the topology thread
}
(void)vs_unlock(&new_topology_lock);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
Status_t
sm_fsm_discovering(Mai_t *maip, char *nodename)
{
Status_t status;
STL_SM_INFO theirSmInfo;
STL_LID slid=maip->addrInfo.slid; // original source lid; mai_reply swaps slid-dlid
IB_ENTER(__func__, maip->base.amod, sm_state, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)stl_mai_get_smp_data(maip), &theirSmInfo);
//
// The SM in the DISCOVERING state does not have any valid transitions via
// the Set(SMInfo) method.
//
maip->base.status = MAD_STATUS_BAD_ATTR;
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)stl_mai_get_smp_data(maip));
status = mai_stl_reply(fd_async->fdMai, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_discovering - bad mai_reply rc:", status);
}
IB_LOG_WARN_FMT(__func__,
"No transitions allowed from DISCOVERING state; Got %s request from [%s] SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
getAmod(maip->base.amod), sm_getStateText(theirSmInfo.u.s.SMStateCurrent), nodename, slid, theirSmInfo.PortGUID, maip->base.tid);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
Status_t
sm_fsm_default(Mai_t *maip, char *nodename)
{
Status_t status;
IB_ENTER(__func__, maip->base.amod, sm_state, 0, 0);
//
// Send our reply to the requester.
//
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)stl_mai_get_smp_data(maip));
maip->base.status = MAD_STATUS_BAD_ATTR;
status = mai_stl_reply(fd_async->fdMai, maip, sizeof(STL_SM_INFO));
IB_EXIT(__func__, status);
(void)(status); // fix "unused" warning
return(VSTATUS_OK);
}
// -------------------------------------------------------------------------- //
// Fetch the master SMInfo and check the activity counter.
// If returns != VSTATUS_OK, this standby SM should enter DISCOVERING state
Status_t
sm_check_Master() {
Status_t status;
STL_SM_INFO theirSmInfo;
STL_PORT_INFO portInfo;
uint8_t path[64];
SmpAddr_t addr;
static uint32_t fsmCheckMasterFailed=0; // count of fails to check master
static uint32_t fsmMultMaxFail = 1; // multiplier for sm_config.master_ping_max_fail
IB_ENTER(__func__, 0, 0, 0, 0);
(void)memset((void *)path, 0, 64);
SMP_ADDR_SET_DR(&addr,path);
if ((status = SM_Get_PortInfo(fd_sminfo, 1<<24, &addr, &portInfo)) != VSTATUS_OK) {
IB_LOG_ERRORRC("failed to get master SM Lid from my PortInfo, rc:", status);
// having a local problem
// reset count, must be healthy before we can consider becoming master
fsmCheckMasterFailed = 0;
goto stay_standby;
}
if (portInfo.LID == 0 || portInfo.MasterSMLID == 0
|| portInfo.PortStates.s.PortState != IB_PORT_ACTIVE
) {
if (smDebugPerf) {
if (portInfo.PortStates.s.PortState != IB_PORT_ACTIVE) {
IB_LOG_INFINI_INFO("our portInfo indicates state not active; portState=", (int)portInfo.PortStates.s.PortState);
} else
if (portInfo.MasterSMLID == 0) {
IB_LOG_INFINI_INFOX("our portInfo smLid is not set yet; Lid=", portInfo.LID);
}
}
// stay in standby until link comes up
if (portInfo.PortStates.s.PortState > IB_PORT_DOWN) {
IB_LOG_WARN0("Switching to DISCOVERY state; Local port uninitialized");
goto discovering;
}
goto stay_standby;
}
// make sure we aren't trying to talk to ourself during the handover window
if (portInfo.LID == portInfo.MasterSMLID) {
// we're talking to ourself. if a master SM doesn't come along and
// reprogram our SM LID in the timeout period, attempt a discovery.
// Since for large fabrics, the master can take some time to program
// our LID, use a higher upper limit for failure count to give the
// master enough time to program our SM LID.
fsmMultMaxFail = 2;
if (++fsmCheckMasterFailed >= fsmMultMaxFail * sm_config.master_ping_max_fail) {
IB_LOG_WARN0("Switching to DISCOVERY state; Timed out waiting for SM LID to get reprogrammed");
goto discovering;
}
if (smDebugPerf) {
IB_LOG_INFINI_INFOX("our portInfo smLid is not set yet; smLid=", portInfo.MasterSMLID);
}
goto stay_standby; // not yet at threshold
}
SMP_ADDR_SET_LR(&addr,portInfo.LID, portInfo.MasterSMLID);
if ((status = SM_Get_SMInfo(fd_sminfo, 0, &addr, &theirSmInfo)) != VSTATUS_OK) {
if (++fsmCheckMasterFailed >= fsmMultMaxFail * sm_config.master_ping_max_fail) {
IB_LOG_WARNX("Switching to DISCOVERY state; Failed to get SmInfo from master SM at LID:", portInfo.MasterSMLID);
goto discovering;
} else {
IB_LOG_INFINI_INFO_FMT(__func__,
"failed to get SmInfo from master SM at LID[0x%X], retry count=%d",
portInfo.MasterSMLID, fsmCheckMasterFailed);
goto stay_standby; // not yet at threshold
}
}
sm_saw_another_sm = TRUE;
/*
* PR 105313 - restart results in 2 standby SMs
* Must check the state we get back to make sure master is still master
*/
if (theirSmInfo.u.s.SMStateCurrent != SM_STATE_MASTER) {
IB_LOG_WARN_FMT(__func__,
"SmInfo from SM at SMLID[0x%X] indicates SM is no longer master, switching to DISCOVERY state",
portInfo.MasterSMLID);
goto discovering;
}
// all OK, save data about master, reset fail count and threshold
sm_topop->sm_count = theirSmInfo.ActCount;
sm_topop->sm_key = theirSmInfo.SM_Key;
fsmCheckMasterFailed = 0;
fsmMultMaxFail = 1;
if (smDebugPerf) {
IB_LOG_INFINI_INFO_FMT(__func__,
"Master SM["FMT_U64"] at LID=0x%x has priority[%d] and smKey ["FMT_U64"]",
theirSmInfo.PortGUID, portInfo.MasterSMLID, theirSmInfo.u.s.Priority, theirSmInfo.SM_Key);
}
stay_standby:
status = VSTATUS_OK;
done:
IB_EXIT(__func__, status);
return(status);
discovering:
// reset count and threshold in case stay standby
fsmCheckMasterFailed = 0;
fsmMultMaxFail = 1;
status = VSTATUS_BAD;
goto done;
}