/* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2015 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * Copyright (c) 2013 Oracle and/or its affiliates. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ /* * Abstract: * Implementation of osm_drop_mgr_t. * This object represents the Drop Manager object. * This object is part of the opensm family of objects. */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #include #include #include #include #include #include #include #include #define FILE_ID OSM_FILE_DROP_MGR_C #include #include #include #include #include #include #include #include #include #include static void drop_mgr_remove_router(osm_sm_t * sm, IN const ib_net64_t portguid) { osm_router_t *p_rtr; cl_qmap_t *p_rtr_guid_tbl; p_rtr_guid_tbl = &sm->p_subn->rtr_guid_tbl; p_rtr = (osm_router_t *) cl_qmap_remove(p_rtr_guid_tbl, portguid); if (p_rtr != (osm_router_t *) cl_qmap_end(p_rtr_guid_tbl)) { OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Cleaned router for port guid 0x%016" PRIx64 "\n", cl_ntoh64(portguid)); osm_router_delete(&p_rtr); } } static void drop_mgr_clean_physp(osm_sm_t * sm, IN osm_physp_t * p_physp) { osm_physp_t *p_remote_physp; osm_port_t *p_remote_port; p_remote_physp = osm_physp_get_remote(p_physp); if (p_remote_physp) { p_remote_port = osm_get_port_by_guid(sm->p_subn, p_remote_physp->port_guid); if (p_remote_port) { /* Let's check if this is a case of link that is lost (both ports weren't recognized), or a "hiccup" in the subnet - in which case the remote port was recognized, and its state is ACTIVE. If this is just a "hiccup" - force a heavy sweep in the next sweep. We don't want to lose that part of the subnet. */ if (p_remote_port->discovery_count && osm_physp_get_port_state(p_remote_physp) == IB_LINK_ACTIVE) { OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Forcing new heavy sweep. Remote " "port 0x%016" PRIx64 " port num: %u " "was recognized in ACTIVE state\n", cl_ntoh64(p_remote_physp->port_guid), p_remote_physp->port_num); sm->p_subn->force_heavy_sweep = TRUE; } /* If the remote node is ca or router - need to remove the remote port, since it is no longer reachable. This can be done if we reset the discovery count of the remote port. */ if (!p_remote_physp->p_node->sw && p_remote_physp->port_guid != sm->p_subn->sm_port_guid) { p_remote_port->discovery_count = 0; OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Resetting discovery count of node: " "0x%016" PRIx64 " port num:%u\n", cl_ntoh64(osm_node_get_node_guid (p_remote_physp->p_node)), p_remote_physp->port_num); } } OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Unlinking local node 0x%016" PRIx64 ", port %u" "\n\t\t\t\tand remote node 0x%016" PRIx64 ", port %u\n", cl_ntoh64(osm_node_get_node_guid(p_physp->p_node)), p_physp->port_num, cl_ntoh64(osm_node_get_node_guid (p_remote_physp->p_node)), p_remote_physp->port_num); if (sm->ucast_mgr.cache_valid) osm_ucast_cache_add_link(&sm->ucast_mgr, p_physp, p_remote_physp); osm_physp_unlink(p_physp, p_remote_physp); } /* Make port as undiscovered */ p_physp->p_node->physp_discovered[p_physp->port_num] = 0; OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Clearing node 0x%016" PRIx64 " physical port number %u\n", cl_ntoh64(osm_node_get_node_guid(p_physp->p_node)), p_physp->port_num); osm_physp_destroy(p_physp); } static void drop_mgr_remove_port(osm_sm_t * sm, IN osm_port_t * p_port) { ib_net64_t port_guid; osm_port_t *p_port_check; cl_qmap_t *p_alias_guid_tbl; cl_qmap_t *p_sm_guid_tbl; osm_mcm_port_t *mcm_port; cl_ptr_vector_t *p_port_lid_tbl; uint16_t min_lid_ho; uint16_t max_lid_ho; uint16_t lid_ho; osm_node_t *p_node; osm_remote_sm_t *p_sm; osm_alias_guid_t *p_alias_guid, *p_alias_guid_check; osm_guidinfo_work_obj_t *wobj; cl_list_item_t *item, *next_item; ib_gid_t port_gid; ib_mad_notice_attr_t notice; ib_api_status_t status; OSM_LOG_ENTER(sm->p_log); port_guid = osm_port_get_guid(p_port); OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Unreachable port 0x%016" PRIx64 "\n", cl_ntoh64(port_guid)); p_port_check = (osm_port_t *) cl_qmap_get(&sm->p_subn->port_guid_tbl, port_guid); if (p_port_check != p_port) { OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0101: " "Port 0x%016" PRIx64 " not in guid table\n", cl_ntoh64(port_guid)); goto Exit; } /* issue a notice - trap 65 (SM_GID_OUT_OF_SERVICE_TRAP) */ /* details of the notice */ notice.generic_type = 0x80 | IB_NOTICE_TYPE_SUBN_MGMT; /* is generic subn mgt type */ ib_notice_set_prod_type_ho(¬ice, 4); /* A class manager generator */ /* endport ceases to be reachable */ notice.g_or_v.generic.trap_num = CL_HTON16(SM_GID_OUT_OF_SERVICE_TRAP); /* 65 */ /* The sm_base_lid is saved in network order already. */ notice.issuer_lid = sm->p_subn->sm_base_lid; /* following C14-72.1.2 and table 119 p725 */ /* we need to provide the GID */ port_gid.unicast.prefix = sm->p_subn->opt.subnet_prefix; port_gid.unicast.interface_id = port_guid; memcpy(&(notice.data_details.ntc_64_67.gid), &(port_gid), sizeof(ib_gid_t)); /* According to page 653 - the issuer gid in this case of trap is the SM gid, since the SM is the initiator of this trap. */ notice.issuer_gid.unicast.prefix = sm->p_subn->opt.subnet_prefix; notice.issuer_gid.unicast.interface_id = sm->p_subn->sm_port_guid; status = osm_report_notice(sm->p_log, sm->p_subn, ¬ice); if (status != IB_SUCCESS) { OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0103: " "Error sending trap reports (%s)\n", ib_get_err_str(status)); } next_item = cl_qlist_head(&sm->p_subn->alias_guid_list); while (next_item != cl_qlist_end(&sm->p_subn->alias_guid_list)) { item = next_item; next_item = cl_qlist_next(item); wobj = cl_item_obj(item, wobj, list_item); if (wobj->p_port == p_port) { cl_qlist_remove_item(&sm->p_subn->alias_guid_list, &wobj->list_item); osm_guid_work_obj_delete(wobj); } } while (!cl_is_qlist_empty(&p_port->mcm_list)) { mcm_port = cl_item_obj(cl_qlist_head(&p_port->mcm_list), mcm_port, list_item); osm_mgrp_delete_port(sm->p_subn, sm->p_log, mcm_port->mgrp, p_port); } p_alias_guid_tbl = &sm->p_subn->alias_port_guid_tbl; p_alias_guid_check = (osm_alias_guid_t *) cl_qmap_head(p_alias_guid_tbl); while (p_alias_guid_check != (osm_alias_guid_t *) cl_qmap_end(p_alias_guid_tbl)) { if (p_alias_guid_check->p_base_port == p_port) p_alias_guid = p_alias_guid_check; else p_alias_guid = NULL; p_alias_guid_check = (osm_alias_guid_t *) cl_qmap_next(&p_alias_guid_check->map_item); if (p_alias_guid) { cl_qmap_remove_item(p_alias_guid_tbl, &p_alias_guid->map_item); osm_alias_guid_delete(&p_alias_guid); } } cl_qmap_remove(&sm->p_subn->port_guid_tbl, port_guid); p_sm_guid_tbl = &sm->p_subn->sm_guid_tbl; p_sm = (osm_remote_sm_t *) cl_qmap_remove(p_sm_guid_tbl, port_guid); if (p_sm != (osm_remote_sm_t *) cl_qmap_end(p_sm_guid_tbl)) { /* need to remove this item */ OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Cleaned SM for port guid 0x%016" PRIx64 "\n", cl_ntoh64(port_guid)); free(p_sm); } drop_mgr_remove_router(sm, port_guid); osm_port_get_lid_range_ho(p_port, &min_lid_ho, &max_lid_ho); OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Clearing abandoned LID range [%u,%u]\n", min_lid_ho, max_lid_ho); p_port_lid_tbl = &sm->p_subn->port_lid_tbl; for (lid_ho = min_lid_ho; lid_ho <= max_lid_ho; lid_ho++) cl_ptr_vector_set(p_port_lid_tbl, lid_ho, NULL); drop_mgr_clean_physp(sm, p_port->p_physp); /* Delete event forwarding subscriptions */ if (sm->p_subn->opt.drop_event_subscriptions) { if (osm_infr_remove_subscriptions(sm->p_subn, sm->p_log, port_guid) == CL_SUCCESS) OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Removed event subscriptions for port 0x%016" PRIx64 "\n", cl_ntoh64(port_guid)); } /* initialize the p_node - may need to get node_desc later */ p_node = p_port->p_node; osm_port_delete(&p_port); OSM_LOG(sm->p_log, OSM_LOG_INFO, "Removed port with GUID:0x%016" PRIx64 " LID range [%u, %u] of node:%s\n", cl_ntoh64(port_gid.unicast.interface_id), min_lid_ho, max_lid_ho, p_node ? p_node->print_desc : "UNKNOWN"); Exit: OSM_LOG_EXIT(sm->p_log); } static void drop_mgr_remove_switch(osm_sm_t * sm, IN osm_node_t * p_node) { osm_switch_t *p_sw; cl_qmap_t *p_sw_guid_tbl; ib_net64_t node_guid; OSM_LOG_ENTER(sm->p_log); node_guid = osm_node_get_node_guid(p_node); p_sw_guid_tbl = &sm->p_subn->sw_guid_tbl; p_sw = (osm_switch_t *) cl_qmap_remove(p_sw_guid_tbl, node_guid); if (p_sw == (osm_switch_t *) cl_qmap_end(p_sw_guid_tbl)) { OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0102: " "Node 0x%016" PRIx64 " not in switch table\n", cl_ntoh64(osm_node_get_node_guid(p_node))); } else { p_node->sw = NULL; osm_switch_delete(&p_sw); } OSM_LOG_EXIT(sm->p_log); } static boolean_t drop_mgr_process_node(osm_sm_t * sm, IN osm_node_t * p_node) { osm_physp_t *p_physp; osm_port_t *p_port; osm_node_t *p_node_check; uint32_t port_num; uint32_t max_ports; ib_net64_t port_guid; boolean_t return_val = FALSE; OSM_LOG_ENTER(sm->p_log); OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Unreachable node 0x%016" PRIx64 "\n", cl_ntoh64(osm_node_get_node_guid(p_node))); if (sm->ucast_mgr.cache_valid) osm_ucast_cache_add_node(&sm->ucast_mgr, p_node); /* Delete all the logical and physical port objects associated with this node. */ max_ports = osm_node_get_num_physp(p_node); for (port_num = 0; port_num < max_ports; port_num++) { p_physp = osm_node_get_physp_ptr(p_node, port_num); if (p_physp) { port_guid = osm_physp_get_port_guid(p_physp); p_port = osm_get_port_by_guid(sm->p_subn, port_guid); if (p_port) drop_mgr_remove_port(sm, p_port); else drop_mgr_clean_physp(sm, p_physp); } } return_val = TRUE; if (p_node->sw) drop_mgr_remove_switch(sm, p_node); p_node_check = (osm_node_t *) cl_qmap_remove(&sm->p_subn->node_guid_tbl, osm_node_get_node_guid(p_node)); if (p_node_check != p_node) { OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0105: " "Node 0x%016" PRIx64 " not in guid table\n", cl_ntoh64(osm_node_get_node_guid(p_node))); } /* free memory allocated to node */ osm_node_delete(&p_node); OSM_LOG_EXIT(sm->p_log); return return_val; } static void drop_mgr_check_switch_node(osm_sm_t * sm, IN osm_node_t * p_node) { ib_net64_t node_guid; osm_physp_t *p_physp, *p_remote_physp; osm_node_t *p_remote_node; osm_port_t *p_port; ib_net64_t port_guid; uint8_t port_num, remote_port_num; OSM_LOG_ENTER(sm->p_log); node_guid = osm_node_get_node_guid(p_node); /* Make sure we have a switch object for this node */ if (!p_node->sw) { /* We do not have switch info for this node */ OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Node 0x%016" PRIx64 " no switch in table\n", cl_ntoh64(node_guid)); drop_mgr_process_node(sm, p_node); goto Exit; } /* Make sure we have a port object for port zero */ p_physp = osm_node_get_physp_ptr(p_node, 0); if (!p_physp) { OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Node 0x%016" PRIx64 " no valid physical port 0\n", cl_ntoh64(node_guid)); drop_mgr_process_node(sm, p_node); goto Exit; } port_guid = osm_physp_get_port_guid(p_physp); p_port = osm_get_port_by_guid(sm->p_subn, port_guid); if (!p_port) { OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Node 0x%016" PRIx64 " has no port object\n", cl_ntoh64(node_guid)); drop_mgr_process_node(sm, p_node); goto Exit; } if (!p_node->physp_discovered[0]) { OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Node 0x%016" PRIx64 " port has discovery count zero\n", cl_ntoh64(node_guid)); drop_mgr_process_node(sm, p_node); goto Exit; } /* * Unlink all ports that havn't been discovered during the last sweep. * Optimization: Skip the check if discovered all the ports of the switch. */ if (p_port->discovery_count < p_node->physp_tbl_size) { for (port_num = 1; port_num < p_node->physp_tbl_size; port_num++) { if (!p_node->physp_discovered[port_num]) { p_physp = osm_node_get_physp_ptr(p_node, port_num); if (!p_physp) continue; p_remote_physp = osm_physp_get_remote(p_physp); if (!p_remote_physp) continue; p_remote_node = osm_physp_get_node_ptr(p_remote_physp); remote_port_num = osm_physp_get_port_num(p_remote_physp); OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "Unlinking local node 0x%" PRIx64 ", port %u" "\n\t\t\t\tand remote node 0x%" PRIx64 ", port %u due to missing PortInfo\n", cl_ntoh64(osm_node_get_node_guid (p_node)), port_num, cl_ntoh64(osm_node_get_node_guid (p_remote_node)), remote_port_num); if (sm->ucast_mgr.cache_valid) osm_ucast_cache_add_link(&sm->ucast_mgr, p_physp, p_remote_physp); osm_node_unlink(p_node, (uint8_t) port_num, p_remote_node, (uint8_t) remote_port_num); } } } Exit: OSM_LOG_EXIT(sm->p_log); return; } void osm_drop_mgr_process(osm_sm_t * sm) { cl_qmap_t *p_node_guid_tbl, *p_port_guid_tbl; osm_port_t *p_port, *p_next_port; osm_node_t *p_node, *p_next_node; int max_ports, port_num; osm_physp_t *p_physp; ib_net64_t port_guid; CL_ASSERT(sm); OSM_LOG_ENTER(sm->p_log); p_node_guid_tbl = &sm->p_subn->node_guid_tbl; p_port_guid_tbl = &sm->p_subn->port_guid_tbl; CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); p_next_node = (osm_node_t *) cl_qmap_head(p_node_guid_tbl); while (p_next_node != (osm_node_t *) cl_qmap_end(p_node_guid_tbl)) { p_node = p_next_node; p_next_node = (osm_node_t *) cl_qmap_next(&p_next_node->map_item); CL_ASSERT(cl_qmap_key(&p_node->map_item) == osm_node_get_node_guid(p_node)); OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Checking node 0x%016" PRIx64 "\n", cl_ntoh64(osm_node_get_node_guid(p_node))); /* Check if this node was discovered during the last sweep. If not, it is unreachable in the current subnet, and should therefore be removed from the subnet object. */ if (p_node->discovery_count == 0) drop_mgr_process_node(sm, p_node); else { /* * We want to preserve the configured pkey indexes, * so if we don't receive GetResp P_KeyTable for some block, * do the following: * 1. Drop node if the node is sw and got timeout for port 0. * 2. Drop node if node is HCA/RTR. * 3. Drop only physp if got timeout for sw when the port isn't 0. * We'll set error during initialization in order to * cause an immediate heavy sweep and try to get the * configured P_KeyTable again. */ if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH) port_num = 0; else port_num = 1; max_ports = osm_node_get_num_physp(p_node); for (; port_num < max_ports; port_num++) { p_physp = osm_node_get_physp_ptr(p_node, port_num); if (!p_physp || p_physp->pkeys.rcv_blocks_cnt == 0) continue; p_physp->pkeys.rcv_blocks_cnt = 0; p_physp->need_update = 2; sm->p_subn->subnet_initialization_error = TRUE; port_guid = osm_physp_get_port_guid(p_physp); p_port = osm_get_port_by_guid(sm->p_subn, port_guid); CL_ASSERT(p_port); if (p_node->physp_discovered[port_num]) { p_node->physp_discovered[port_num] = 0; p_port->discovery_count--; } } } } /* Go over all the nodes. If the node is a switch - make sure there is also a switch record for it, and a portInfo record for port zero of of the node. If not - this means that there was some error in getting the data of this node. Drop the node. */ p_next_node = (osm_node_t *) cl_qmap_head(p_node_guid_tbl); while (p_next_node != (osm_node_t *) cl_qmap_end(p_node_guid_tbl)) { p_node = p_next_node; p_next_node = (osm_node_t *) cl_qmap_next(&p_next_node->map_item); OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Checking full discovery of node 0x%016" PRIx64 "\n", cl_ntoh64(osm_node_get_node_guid(p_node))); if (osm_node_get_type(p_node) != IB_NODE_TYPE_SWITCH) continue; /* We are handling a switch node */ drop_mgr_check_switch_node(sm, p_node); } p_next_port = (osm_port_t *) cl_qmap_head(p_port_guid_tbl); while (p_next_port != (osm_port_t *) cl_qmap_end(p_port_guid_tbl)) { p_port = p_next_port; p_next_port = (osm_port_t *) cl_qmap_next(&p_next_port->map_item); CL_ASSERT(cl_qmap_key(&p_port->map_item) == osm_port_get_guid(p_port)); OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Checking port 0x%016" PRIx64 "\n", cl_ntoh64(osm_port_get_guid(p_port))); /* If the port is unreachable, remove it from the guid table. */ if (p_port->discovery_count == 0) drop_mgr_remove_port(sm, p_port); } CL_PLOCK_RELEASE(sm->p_lock); OSM_LOG_EXIT(sm->p_log); }