Blame psm_hal_gen1/opa_service_gen1.c

Packit 961e70
/*
Packit 961e70
Packit 961e70
  This file is provided under a dual BSD/GPLv2 license.  When using or
Packit 961e70
  redistributing this file, you may do so under either license.
Packit 961e70
Packit 961e70
  GPL LICENSE SUMMARY
Packit 961e70
Packit 961e70
  Copyright(c) 2018 Intel Corporation.
Packit 961e70
Packit 961e70
  This program is free software; you can redistribute it and/or modify
Packit 961e70
  it under the terms of version 2 of the GNU General Public License as
Packit 961e70
  published by the Free Software Foundation.
Packit 961e70
Packit 961e70
  This program is distributed in the hope that it will be useful, but
Packit 961e70
  WITHOUT ANY WARRANTY; without even the implied warranty of
Packit 961e70
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Packit 961e70
  General Public License for more details.
Packit 961e70
Packit 961e70
  Contact Information:
Packit 961e70
  Intel Corporation, www.intel.com
Packit 961e70
Packit 961e70
  BSD LICENSE
Packit 961e70
Packit 961e70
  Copyright(c) 2018 Intel Corporation.
Packit 961e70
Packit 961e70
  Redistribution and use in source and binary forms, with or without
Packit 961e70
  modification, are permitted provided that the following conditions
Packit 961e70
  are met:
Packit 961e70
Packit 961e70
    * Redistributions of source code must retain the above copyright
Packit 961e70
      notice, this list of conditions and the following disclaimer.
Packit 961e70
    * Redistributions in binary form must reproduce the above copyright
Packit 961e70
      notice, this list of conditions and the following disclaimer in
Packit 961e70
      the documentation and/or other materials provided with the
Packit 961e70
      distribution.
Packit 961e70
    * Neither the name of Intel Corporation nor the names of its
Packit 961e70
      contributors may be used to endorse or promote products derived
Packit 961e70
      from this software without specific prior written permission.
Packit 961e70
Packit 961e70
  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
Packit 961e70
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
Packit 961e70
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
Packit 961e70
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
Packit 961e70
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
Packit 961e70
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
Packit 961e70
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
Packit 961e70
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
Packit 961e70
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
Packit 961e70
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
Packit 961e70
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Packit 961e70
Packit 961e70
*/
Packit 961e70
Packit 961e70
/* This file contains hfi service routine interface used by the low
Packit 961e70
   level hfi protocol code. */
Packit 961e70
Packit 961e70
#include <sys/types.h>
Packit 961e70
#include <sys/stat.h>
Packit 961e70
#include <sys/mman.h>
Packit 961e70
#include <ctype.h>
Packit 961e70
#include <dirent.h>
Packit 961e70
#include <fcntl.h>
Packit 961e70
#include <unistd.h>
Packit 961e70
#include <errno.h>
Packit 961e70
#include <string.h>
Packit 961e70
#include <stdarg.h>
Packit 961e70
#include <stdlib.h>
Packit 961e70
#include <stdio.h>
Packit 961e70
#include <time.h>
Packit 961e70
#include <poll.h>
Packit 961e70
#include "opa_service_gen1.h"
Packit 961e70
#include "psmi_wrappers.h"
Packit 961e70
Packit 961e70
typedef union
Packit 961e70
{
Packit 961e70
	struct
Packit 961e70
	{
Packit 961e70
		uint16_t minor;
Packit 961e70
		uint16_t major;
Packit 961e70
	};
Packit 961e70
	uint32_t version;
Packit 961e70
} sw_version_t;
Packit 961e70
Packit 961e70
static sw_version_t sw_version =
Packit 961e70
{
Packit 961e70
	{
Packit 961e70
	.major = HFI1_USER_SWMAJOR,
Packit 961e70
	.minor = HFI1_USER_SWMINOR
Packit 961e70
	}
Packit 961e70
};
Packit 961e70
Packit 961e70
/* fwd declaration */
Packit 961e70
ustatic int _hfi_cmd_write(int fd, struct hfi1_cmd *cmd, size_t count);
Packit 961e70
Packit 961e70
#ifdef PSM2_SUPPORT_IW_CMD_API
Packit 961e70
Packit 961e70
/* fwd declaration */
Packit 961e70
ustatic int _hfi_cmd_ioctl(int fd, struct hfi1_cmd *cmd, size_t count);
Packit 961e70
Packit 961e70
/* Function pointer. */
Packit 961e70
static int (*_hfi_cmd_send)(int fd, struct hfi1_cmd *cmd, size_t count) = _hfi_cmd_ioctl;
Packit 961e70
Packit 961e70
#else
Packit 961e70
/* Function pointer. */
Packit 961e70
static int (*const _hfi_cmd_send)(int fd, struct hfi1_cmd *cmd, size_t count) = _hfi_cmd_write;
Packit 961e70
#endif
Packit 961e70
Packit 961e70
uint16_t hfi_get_user_major_version(void)
Packit 961e70
{
Packit 961e70
	return sw_version.major;
Packit 961e70
}
Packit 961e70
Packit 961e70
void hfi_set_user_major_version(uint16_t major_version)
Packit 961e70
{
Packit 961e70
	sw_version.major = major_version;
Packit 961e70
}
Packit 961e70
Packit 961e70
uint16_t hfi_get_user_minor_version(void)
Packit 961e70
{
Packit 961e70
	return sw_version.minor;
Packit 961e70
}
Packit 961e70
Packit 961e70
void hfi_set_user_version(uint32_t version)
Packit 961e70
{
Packit 961e70
	sw_version.version = version;
Packit 961e70
}
Packit 961e70
Packit 961e70
int hfi_context_open(int unit, int port, uint64_t open_timeout)
Packit 961e70
{
Packit 961e70
	char dev_name_ignored[256];
Packit 961e70
Packit 961e70
	return hfi_context_open_ex(unit, port, open_timeout,
Packit 961e70
				   dev_name_ignored, sizeof(dev_name_ignored));
Packit 961e70
}
Packit 961e70
Packit 961e70
int hfi_context_open_ex(int unit, int port, uint64_t open_timeout,
Packit 961e70
		     char *dev_name,size_t dev_name_len)
Packit 961e70
{
Packit 961e70
	int fd;
Packit 961e70
Packit 961e70
	if (unit != HFI_UNIT_ID_ANY && unit >= 0)
Packit 961e70
		snprintf(dev_name, dev_name_len, "%s_%u", HFI_DEVICE_PATH_GEN1,
Packit 961e70
			 unit);
Packit 961e70
	else
Packit 961e70
		snprintf(dev_name, dev_name_len, "%s_%u", HFI_DEVICE_PATH_GEN1,
Packit 961e70
			 0);
Packit 961e70
Packit 961e70
	if ((fd = open(dev_name, O_RDWR)) == -1) {
Packit 961e70
		_HFI_DBG("(host:Can't open %s for reading and writing",
Packit 961e70
			 dev_name);
Packit 961e70
		return -1;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	if (fcntl(fd, F_SETFD, FD_CLOEXEC))
Packit 961e70
		_HFI_INFO("Failed to set close on exec for device: %s\n",
Packit 961e70
			  strerror(errno));
Packit 961e70
Packit 961e70
#ifdef PSM2_SUPPORT_IW_CMD_API
Packit 961e70
	{
Packit 961e70
		/* if hfi1DriverMajor == -1, then we are potentially talking to a new driver.
Packit 961e70
		   Let's confirm by issuing an ioctl version request: */
Packit 961e70
		struct hfi1_cmd c;
Packit 961e70
Packit 961e70
		memset(&c, 0, sizeof(struct hfi1_cmd));
Packit 961e70
		c.type = PSMI_HFI_CMD_GET_VERS;
Packit 961e70
		c.len  = 0;
Packit 961e70
		c.addr = 0;
Packit 961e70
Packit 961e70
		if (hfi_cmd_write(fd, &c, sizeof(c)) == -1) {
Packit 961e70
			/* Let's assume that the driver is the old driver */
Packit 961e70
			hfi_set_user_major_version(IOCTL_CMD_API_MODULE_MAJOR - 1);
Packit 961e70
			/* the old driver uses write() for its command interface: */
Packit 961e70
			_hfi_cmd_send = _hfi_cmd_write;
Packit 961e70
		}
Packit 961e70
		else
Packit 961e70
		{
Packit 961e70
			int major = c.addr >> HFI1_SWMAJOR_SHIFT;
Packit 961e70
			if (major != hfi_get_user_major_version()) {
Packit 961e70
					/* If there is a skew between the major version of the driver
Packit 961e70
					   that is executing and the major version which was used during
Packit 961e70
					   compilation of PSM, we treat that is a fatal error. */
Packit 961e70
					_HFI_INFO("PSM2 and driver version mismatch: (%d != %d)\n",
Packit 961e70
						  major, hfi_get_user_major_version());
Packit 961e70
				close(fd);
Packit 961e70
				return -1;
Packit 961e70
			}
Packit 961e70
		}
Packit 961e70
	}
Packit 961e70
Packit 961e70
#endif
Packit 961e70
	return fd;
Packit 961e70
}
Packit 961e70
Packit 961e70
/*
Packit 961e70
 * Check if non-double word multiple message size for SDMA is allowed to be
Packit 961e70
 * pass to the driver. Starting from 6.2 driver version, PSM is able to pass
Packit 961e70
 * to the driver message which size is not a multiple of double word for SDMA.
Packit 961e70
 */
Packit 961e70
uint32_t hfi_check_non_dw_mul_sdma(void)
Packit 961e70
{
Packit 961e70
	uint16_t major = hfi_get_user_major_version();
Packit 961e70
	uint16_t minor = hfi_get_user_minor_version();
Packit 961e70
Packit 961e70
	if ((major > HFI1_USER_SWMAJOR_NON_DW_MUL_MSG_SIZE_ALLOWED) ||
Packit 961e70
		((major == HFI1_USER_SWMAJOR_NON_DW_MUL_MSG_SIZE_ALLOWED) &&
Packit 961e70
		 (minor >= HFI1_USER_SWMINOR_NON_DW_MUL_MSG_SIZE_ALLOWED)))
Packit 961e70
		return 1;
Packit 961e70
Packit 961e70
	return 0;
Packit 961e70
}
Packit 961e70
Packit 961e70
void hfi_context_close(int fd)
Packit 961e70
{
Packit 961e70
	(void)close(fd);
Packit 961e70
}
Packit 961e70
Packit 961e70
int hfi_cmd_writev(int fd, const struct iovec *iov, int iovcnt)
Packit 961e70
{
Packit 961e70
	return writev(fd, iov, iovcnt);
Packit 961e70
}
Packit 961e70
Packit 961e70
int hfi_cmd_write(int fd, struct hfi1_cmd *cmd, size_t count)
Packit 961e70
{
Packit 961e70
	return _hfi_cmd_send(fd, cmd, count);
Packit 961e70
}
Packit 961e70
Packit 961e70
ustatic
Packit 961e70
int _hfi_cmd_write(int fd, struct hfi1_cmd *cmd, size_t count)
Packit 961e70
{
Packit 961e70
    const static unsigned int cmdTypeToWriteNum[PSMI_HFI_CMD_LAST] = {
Packit 961e70
        [PSMI_HFI_CMD_ASSIGN_CTXT]      = LEGACY_HFI1_CMD_ASSIGN_CTXT,
Packit 961e70
        [PSMI_HFI_CMD_CTXT_INFO]        = LEGACY_HFI1_CMD_CTXT_INFO,
Packit 961e70
        [PSMI_HFI_CMD_USER_INFO]        = LEGACY_HFI1_CMD_USER_INFO,
Packit 961e70
        [PSMI_HFI_CMD_TID_UPDATE]       = LEGACY_HFI1_CMD_TID_UPDATE,
Packit 961e70
        [PSMI_HFI_CMD_TID_FREE]         = LEGACY_HFI1_CMD_TID_FREE,
Packit 961e70
        [PSMI_HFI_CMD_CREDIT_UPD]       = LEGACY_HFI1_CMD_CREDIT_UPD,
Packit 961e70
        [PSMI_HFI_CMD_RECV_CTRL]        = LEGACY_HFI1_CMD_RECV_CTRL,
Packit 961e70
        [PSMI_HFI_CMD_POLL_TYPE]        = LEGACY_HFI1_CMD_POLL_TYPE,
Packit 961e70
        [PSMI_HFI_CMD_ACK_EVENT]        = LEGACY_HFI1_CMD_ACK_EVENT,
Packit 961e70
        [PSMI_HFI_CMD_SET_PKEY]         = LEGACY_HFI1_CMD_SET_PKEY,
Packit 961e70
        [PSMI_HFI_CMD_CTXT_RESET]       = LEGACY_HFI1_CMD_CTXT_RESET,
Packit 961e70
        [PSMI_HFI_CMD_TID_INVAL_READ]   = LEGACY_HFI1_CMD_TID_INVAL_READ,
Packit 961e70
        [PSMI_HFI_CMD_GET_VERS]         = LEGACY_HFI1_CMD_GET_VERS,
Packit 961e70
    };
Packit 961e70
Packit 961e70
    if (cmd->type < PSMI_HFI_CMD_LAST) {
Packit 961e70
        cmd->type = cmdTypeToWriteNum[cmd->type];
Packit 961e70
Packit 961e70
	    return psmi_write(fd, cmd, count);
Packit 961e70
    } else {
Packit 961e70
        errno = EINVAL;
Packit 961e70
        return -1;
Packit 961e70
    }
Packit 961e70
}
Packit 961e70
Packit 961e70
#ifdef PSM2_SUPPORT_IW_CMD_API
Packit 961e70
ustatic
Packit 961e70
int _hfi_cmd_ioctl(int fd, struct hfi1_cmd *cmd, size_t count)
Packit 961e70
{
Packit 961e70
	uint64_t addrOrLiteral[2] = { (uint64_t)cmd->addr, (uint64_t)&cmd->addr };
Packit 961e70
	const static struct
Packit 961e70
	{
Packit 961e70
		unsigned int ioctlCmd;
Packit 961e70
		unsigned int addrOrLiteralIdx;
Packit 961e70
	} cmdTypeToIoctlNum[PSMI_HFI_CMD_LAST] = {
Packit 961e70
        [PSMI_HFI_CMD_ASSIGN_CTXT]      = {HFI1_IOCTL_ASSIGN_CTXT   , 0},
Packit 961e70
        [PSMI_HFI_CMD_CTXT_INFO]        = {HFI1_IOCTL_CTXT_INFO     , 0},
Packit 961e70
        [PSMI_HFI_CMD_USER_INFO]        = {HFI1_IOCTL_USER_INFO     , 0},
Packit 961e70
        [PSMI_HFI_CMD_TID_UPDATE]       = {HFI1_IOCTL_TID_UPDATE    , 0},
Packit 961e70
        [PSMI_HFI_CMD_TID_FREE]         = {HFI1_IOCTL_TID_FREE      , 0},
Packit 961e70
        [PSMI_HFI_CMD_CREDIT_UPD]       = {HFI1_IOCTL_CREDIT_UPD    , 1},
Packit 961e70
        [PSMI_HFI_CMD_RECV_CTRL]        = {HFI1_IOCTL_RECV_CTRL     , 1},
Packit 961e70
        [PSMI_HFI_CMD_POLL_TYPE]        = {HFI1_IOCTL_POLL_TYPE     , 1},
Packit 961e70
        [PSMI_HFI_CMD_ACK_EVENT]        = {HFI1_IOCTL_ACK_EVENT     , 1},
Packit 961e70
        [PSMI_HFI_CMD_SET_PKEY]         = {HFI1_IOCTL_SET_PKEY      , 1},
Packit 961e70
        [PSMI_HFI_CMD_CTXT_RESET]       = {HFI1_IOCTL_CTXT_RESET    , 1},
Packit 961e70
        [PSMI_HFI_CMD_TID_INVAL_READ]   = {HFI1_IOCTL_TID_INVAL_READ, 0},
Packit 961e70
        [PSMI_HFI_CMD_GET_VERS]         = {HFI1_IOCTL_GET_VERS      , 1},
Packit 961e70
#ifdef PSM_CUDA
Packit 961e70
	[PSMI_HFI_CMD_TID_UPDATE_V2]	= {HFI1_IOCTL_TID_UPDATE_V2 , 0},
Packit 961e70
#endif
Packit 961e70
    };
Packit 961e70
Packit 961e70
	if (cmd->type < PSMI_HFI_CMD_LAST)
Packit 961e70
		return psmi_ioctl(fd,
Packit 961e70
			     cmdTypeToIoctlNum[cmd->type].ioctlCmd,
Packit 961e70
			     addrOrLiteral[cmdTypeToIoctlNum[cmd->type].addrOrLiteralIdx]);
Packit 961e70
	else
Packit 961e70
	{
Packit 961e70
		errno = EINVAL;
Packit 961e70
		return -1;
Packit 961e70
	}
Packit 961e70
}
Packit 961e70
#endif /* #ifdef PSM2_SUPPORT_IW_CMD_API */
Packit 961e70
Packit 961e70
/* we use mmap64() because we compile in both 32 and 64 bit mode,
Packit 961e70
   and we have to map physical addresses that are > 32 bits long.
Packit 961e70
   While linux implements mmap64, it doesn't have a man page,
Packit 961e70
   and isn't declared in any header file, so we declare it here ourselves.
Packit 961e70
Packit 961e70
   We'd like to just use -D_LARGEFILE64_SOURCE, to make off_t 64 bits and
Packit 961e70
   redirects mmap to mmap64 for us, but at least through suse10 and fc4,
Packit 961e70
   it doesn't work when the address being mapped is > 32 bits.  It chips
Packit 961e70
   off bits 32 and above.   So we stay with mmap64. */
Packit 961e70
void *hfi_mmap64(void *addr, size_t length, int prot, int flags, int fd,
Packit 961e70
		 __off64_t offset)
Packit 961e70
{
Packit 961e70
	return mmap64(addr, length, prot, flags, fd, offset);
Packit 961e70
}
Packit 961e70
Packit 961e70
/* get the number of units supported by the driver.  Does not guarantee */
Packit 961e70
/* that a working chip has been found for each possible unit #. */
Packit 961e70
/* number of units >=0 (0 means none found). */
Packit 961e70
/* formerly used sysfs file "num_units" */
Packit Service 7ed5cc
int hfi_get_num_units(void)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
Packit 961e70
	for (ret = 0;; ret++) {
Packit 961e70
		char pathname[PATH_MAX];
Packit 961e70
		struct stat st;
Packit 961e70
		int r;
Packit 961e70
Packit 961e70
		snprintf(pathname, sizeof(pathname), HFI_DEVICE_PATH_GEN1 "_%d", ret);
Packit Service 7ed5cc
		r = stat(pathname, &st);
Packit 961e70
		if (!r)
Packit 961e70
			continue;
Packit 961e70
		else
Packit 961e70
			break;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given a unit number, returns 1 if any port on the unit is active.
Packit 961e70
   returns 0 if no port on the unit is active.
Packit 961e70
   returns -1 when an error occurred. */
Packit 961e70
int hfi_get_unit_active(int unit)
Packit 961e70
{
Packit 961e70
	int p,rv;
Packit 961e70
Packit 961e70
	for (p = HFI_MIN_PORT; p <= HFI_MAX_PORT; p++)
Packit 961e70
		if ((rv=hfi_get_port_lid(unit, p)) > 0)
Packit 961e70
			break;
Packit 961e70
Packit 961e70
	if (p <= HFI_MAX_PORT)
Packit 961e70
	{
Packit 961e70
		return 1;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return rv;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* get the number of contexts from the unit id. */
Packit 961e70
/* Returns 0 if no unit or no match. */
Packit Service 7ed5cc
int hfi_get_num_contexts(int unit_id)
Packit 961e70
{
Packit 961e70
	int n = 0;
Packit 961e70
	int units;
Packit 961e70
	int64_t val;
Packit 961e70
	uint32_t p = HFI_MIN_PORT;
Packit 961e70
Packit Service 7ed5cc
	units = hfi_get_num_units();
Packit 961e70
Packit 961e70
	if_pf(units <=  0)
Packit 961e70
		return 0;
Packit 961e70
Packit 961e70
	if (unit_id == HFI_UNIT_ID_ANY) {
Packit 961e70
		uint32_t u;
Packit 961e70
Packit 961e70
		for (u = 0; u < units; u++) {
Packit 961e70
			for (p = HFI_MIN_PORT; p <= HFI_MAX_PORT; p++)
Packit 961e70
				if (hfi_get_port_lid(u, p) > 0)
Packit 961e70
					break;
Packit 961e70
Packit 961e70
			if (p <= HFI_MAX_PORT &&
Packit 961e70
			    !hfi_sysfs_unit_read_s64(u, "nctxts", &val, 0))
Packit 961e70
				n += (uint32_t) val;
Packit 961e70
		}
Packit 961e70
	} else {
Packit 961e70
		for (; p <= HFI_MAX_PORT; p++)
Packit 961e70
			if (hfi_get_port_lid(unit_id, p) > 0)
Packit 961e70
				break;
Packit 961e70
Packit 961e70
		if (p <= HFI_MAX_PORT &&
Packit 961e70
		    !hfi_sysfs_unit_read_s64(unit_id, "nctxts", &val, 0))
Packit 961e70
			n += (uint32_t) val;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return n;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given a unit number and port number, returns 1 if the unit and port are active.
Packit 961e70
   returns 0 if the unit and port are not active.
Packit 961e70
   returns -1 when an error occurred. */
Packit 961e70
int hfi_get_port_active(int unit, int port)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	char *state;
Packit 961e70
Packit 961e70
	ret = hfi_sysfs_port_read(unit, port, "phys_state", &state);
Packit 961e70
	if (ret == -1) {
Packit 961e70
		if (errno == ENODEV)
Packit 961e70
			/* this is "normal" for port != 1, on single port chips */
Packit 961e70
			_HFI_VDBG
Packit 961e70
			    ("Failed to get phys_state for unit %u:%u: %s\n",
Packit 961e70
			     unit, port, strerror(errno));
Packit 961e70
		else
Packit 961e70
			_HFI_DBG
Packit 961e70
			    ("Failed to get phys_state for unit %u:%u: %s\n",
Packit 961e70
			     unit, port, strerror(errno));
Packit 961e70
		return -1;
Packit 961e70
	} else {
Packit 961e70
		if (strncmp(state, "5: LinkUp", 9)) {
Packit 961e70
			_HFI_DBG("Link is not Up for unit %u:%u\n", unit, port);
Packit 961e70
			free(state);
Packit 961e70
			return 0;
Packit 961e70
		}
Packit 961e70
		free(state);
Packit 961e70
		return 1;
Packit 961e70
	}
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given the unit number, return an error, or the corresponding LID
Packit 961e70
   For now, it's used only so the MPI code can determine it's own
Packit 961e70
   LID, and which other LIDs (if any) are also assigned to this node
Packit 961e70
   Returns an int, so -1 indicates an error.  0 may indicate that
Packit 961e70
   the unit is valid, but no LID has been assigned.
Packit 961e70
   No error print because we call this for both potential
Packit 961e70
   ports without knowing if both ports exist (or are connected) */
Packit 961e70
int hfi_get_port_lid(int unit, int port)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	int64_t val;
Packit 961e70
Packit 961e70
	if (hfi_get_port_active(unit,port) != 1)
Packit 961e70
		return -2;
Packit 961e70
	ret = hfi_sysfs_port_read_s64(unit, port, "lid", &val, 0);
Packit 961e70
	_HFI_VDBG("hfi_get_port_lid: ret %d, unit %d port %d\n", ret, unit,
Packit 961e70
		  port);
Packit 961e70
Packit 961e70
	if (ret == -1) {
Packit 961e70
		if (errno == ENODEV)
Packit 961e70
			/* this is "normal" for port != 1, on single port chips */
Packit 961e70
			_HFI_VDBG("Failed to get LID for unit %u:%u: %s\n",
Packit 961e70
				  unit, port, strerror(errno));
Packit 961e70
		else
Packit 961e70
			_HFI_DBG("Failed to get LID for unit %u:%u: %s\n",
Packit 961e70
				 unit, port, strerror(errno));
Packit 961e70
	} else {
Packit 961e70
		ret = val;
Packit 961e70
Packit 961e70
/* disable this feature since we don't have a way to provide
Packit 961e70
   file descriptor in multiple context case. */
Packit 961e70
#if 0
Packit 961e70
		if (getenv("HFI_DIAG_LID_LOOP")) {
Packit 961e70
			/* provides diagnostic ability to run MPI, etc. even */
Packit 961e70
			/* on loopback, by claiming a different LID for each context */
Packit 961e70
			struct hfi1_ctxt_info info;
Packit 961e70
			struct hfi1_cmd cmd;
Packit 961e70
			cmd.type = PSMI_HFI_CMD_CTXT_INFO;
Packit 961e70
			cmd.cmd.ctxt_info = (uintptr_t) &info;
Packit 961e70
			if (__hfi_lastfd == -1)
Packit 961e70
				_HFI_INFO
Packit 961e70
				    ("Can't run CONTEXT_INFO for lid_loop, fd not set\n");
Packit 961e70
			else if (write(__hfi_lastfd, &cmd, sizeof(cmd)) == -1)
Packit 961e70
				_HFI_INFO("CONTEXT_INFO command failed: %s\n",
Packit 961e70
					  strerror(errno));
Packit 961e70
			else if (!info.context)
Packit 961e70
				_HFI_INFO("CONTEXT_INFO returned context 0!\n");
Packit 961e70
			else {
Packit 961e70
				_HFI_PRDBG
Packit 961e70
				    ("Using lid 0x%x, base %x, context %x\n",
Packit 961e70
				     ret + info.context, ret, info.context);
Packit 961e70
				ret += info.context;
Packit 961e70
			}
Packit 961e70
		}
Packit 961e70
#endif
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given the unit number, return an error, or the corresponding GID
Packit 961e70
   For now, it's used only so the MPI code can determine its fabric ID.
Packit 961e70
   Returns an int, so -1 indicates an error.
Packit 961e70
   No error print because we call this for both potential
Packit 961e70
   ports without knowing if both ports exist (or are connected) */
Packit 961e70
int hfi_get_port_gid(int unit, int port, uint64_t *hi, uint64_t *lo)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	char *gid_str = NULL;
Packit 961e70
Packit 961e70
	ret = hfi_sysfs_port_read(unit, port, "gids/0", &gid_str);
Packit 961e70
Packit 961e70
	if (ret == -1) {
Packit 961e70
		if (errno == ENODEV)
Packit 961e70
			/* this is "normal" for port != 1, on single
Packit 961e70
			 * port chips */
Packit 961e70
			_HFI_VDBG("Failed to get GID for unit %u:%u: %s\n",
Packit 961e70
				  unit, port, strerror(errno));
Packit 961e70
		else
Packit 961e70
			_HFI_DBG("Failed to get GID for unit %u:%u: %s\n",
Packit 961e70
				 unit, port, strerror(errno));
Packit 961e70
	} else {
Packit 961e70
		uint32_t gid[8];
Packit 961e70
		if (sscanf(gid_str, "%4x:%4x:%4x:%4x:%4x:%4x:%4x:%4x",
Packit 961e70
			   &gid[0], &gid[1], &gid[2], &gid[3],
Packit 961e70
			   &gid[4], &gid[5], &gid[6], &gid[7]) != 8) {
Packit 961e70
			_HFI_DBG("Failed to parse GID for unit %u:%u: %s\n",
Packit 961e70
				 unit, port, gid_str);
Packit 961e70
			ret = -1;
Packit 961e70
		} else {
Packit 961e70
			*hi = (((uint64_t) gid[0]) << 48) | (((uint64_t) gid[1])
Packit 961e70
							     << 32) |
Packit 961e70
			    (((uint64_t)
Packit 961e70
			      gid[2]) << 16) | (((uint64_t) gid[3]) << 0);
Packit 961e70
			*lo = (((uint64_t) gid[4]) << 48) | (((uint64_t) gid[5])
Packit 961e70
							     << 32) |
Packit 961e70
			    (((uint64_t)
Packit 961e70
			      gid[6]) << 16) | (((uint64_t) gid[7]) << 0);
Packit 961e70
		}
Packit 961e70
		free(gid_str);
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given the unit number, return an error, or the corresponding LMC value
Packit 961e70
   for the port */
Packit 961e70
/* Returns an int, so -1 indicates an error.  0 */
Packit 961e70
int hfi_get_port_lmc(int unit, int port)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	int64_t val;
Packit 961e70
Packit 961e70
	ret = hfi_sysfs_port_read_s64(unit, port, "lid_mask_count", &val, 0);
Packit 961e70
Packit 961e70
	if (ret == -1) {
Packit 961e70
		_HFI_INFO("Failed to get LMC for unit %u:%u: %s\n",
Packit 961e70
			  unit, port, strerror(errno));
Packit 961e70
	} else
Packit 961e70
		ret = val;
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given the unit number, return an error, or the corresponding link rate
Packit 961e70
   for the port */
Packit 961e70
/* Returns an int, so -1 indicates an error. */
Packit 961e70
int hfi_get_port_rate(int unit, int port)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	double rate;
Packit 961e70
	char *data_rate = NULL, *newptr;
Packit 961e70
Packit 961e70
	ret = hfi_sysfs_port_read(unit, port, "rate", &data_rate);
Packit 961e70
	if (ret == -1)
Packit 961e70
		goto get_port_rate_error;
Packit 961e70
	else {
Packit 961e70
		rate = strtod(data_rate, &newptr);
Packit 961e70
		if ((rate == 0) && (data_rate == newptr))
Packit 961e70
			goto get_port_rate_error;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	free(data_rate);
Packit 961e70
	return ((int)(rate * 2) >> 1);
Packit 961e70
Packit 961e70
get_port_rate_error:
Packit 961e70
	_HFI_INFO("Failed to get link rate for unit %u:%u: %s\n",
Packit 961e70
		  unit, port, strerror(errno));
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given a unit, port and SL, return an error, or the corresponding SC for the
Packit 961e70
   SL as programmed by the SM */
Packit 961e70
/* Returns an int, so -1 indicates an error. */
Packit 961e70
int hfi_get_port_sl2sc(int unit, int port, int sl)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	int64_t val;
Packit 961e70
	char sl2scpath[16];
Packit 961e70
Packit 961e70
	snprintf(sl2scpath, sizeof(sl2scpath), "sl2sc/%d", sl);
Packit 961e70
	ret = hfi_sysfs_port_read_s64(unit, port, sl2scpath, &val, 0);
Packit 961e70
Packit 961e70
	if (ret == -1) {
Packit 961e70
		_HFI_DBG
Packit 961e70
		    ("Failed to get SL2SC mapping for SL %d unit %u:%u: %s\n",
Packit 961e70
		     sl, unit, port, strerror(errno));
Packit 961e70
	} else
Packit 961e70
		ret = val;
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given a unit, port and SC, return an error, or the corresponding VL for the
Packit 961e70
   SC as programmed by the SM */
Packit 961e70
/* Returns an int, so -1 indicates an error. */
Packit 961e70
int hfi_get_port_sc2vl(int unit, int port, int sc)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	int64_t val;
Packit 961e70
	char sc2vlpath[16];
Packit 961e70
Packit 961e70
	snprintf(sc2vlpath, sizeof(sc2vlpath), "sc2vl/%d", sc);
Packit 961e70
	ret = hfi_sysfs_port_read_s64(unit, port, sc2vlpath, &val, 0);
Packit 961e70
Packit 961e70
	if (ret == -1) {
Packit 961e70
		_HFI_DBG
Packit 961e70
		    ("Failed to get SC2VL mapping for SC %d unit %u:%u: %s\n",
Packit 961e70
		     sc, unit, port, strerror(errno));
Packit 961e70
	} else
Packit 961e70
		ret = val;
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given a unit, port and VL, return an error, or the corresponding MTU for the
Packit 961e70
   VL as programmed by the SM */
Packit 961e70
/* Returns an int, so -1 indicates an error. */
Packit 961e70
int hfi_get_port_vl2mtu(int unit, int port, int vl)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	int64_t val;
Packit 961e70
	char vl2mtupath[16];
Packit 961e70
Packit 961e70
	snprintf(vl2mtupath, sizeof(vl2mtupath), "vl2mtu/%d", vl);
Packit 961e70
	ret = hfi_sysfs_port_read_s64(unit, port, vl2mtupath, &val, 0);
Packit 961e70
Packit 961e70
	if (ret == -1) {
Packit 961e70
		_HFI_DBG
Packit 961e70
		    ("Failed to get VL2MTU mapping for VL %d unit %u:%u: %s\n",
Packit 961e70
		     vl, unit, port, strerror(errno));
Packit 961e70
	} else
Packit 961e70
		ret = val;
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Given a unit, port and index, return an error, or the corresponding pkey
Packit 961e70
   value for the index as programmed by the SM */
Packit 961e70
/* Returns an int, so -1 indicates an error. */
Packit 961e70
int hfi_get_port_index2pkey(int unit, int port, int index)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	int64_t val;
Packit 961e70
	char index2pkeypath[16];
Packit 961e70
Packit 961e70
	snprintf(index2pkeypath, sizeof(index2pkeypath), "pkeys/%d", index);
Packit 961e70
	ret = hfi_sysfs_port_read_s64(unit, port, index2pkeypath, &val, 0);
Packit 961e70
Packit 961e70
	if (ret == -1) {
Packit 961e70
		_HFI_DBG
Packit 961e70
		    ("Failed to get index2pkey mapping for index %d unit %u:%u: %s\n",
Packit 961e70
		     index, unit, port, strerror(errno));
Packit 961e70
	} else
Packit 961e70
		ret = val;
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
int hfi_get_cc_settings_bin(int unit, int port, char *ccabuf, size_t len_ccabuf)
Packit 961e70
{
Packit 961e70
	int fd;
Packit 961e70
Packit 961e70
	/*
Packit 961e70
	 * 4 bytes for 'control map'
Packit 961e70
	 * 2 bytes 'port control'
Packit 961e70
	 * 32 (#SLs) * 6 bytes 'congestion setting' (per-SL)
Packit 961e70
	 */
Packit 961e70
	const size_t count = 4 + 2 + (32 * 6);
Packit 961e70
Packit 961e70
	if (count > len_ccabuf)
Packit 961e70
		return -2;
Packit 961e70
/*
Packit 961e70
 * Check qib driver CCA setting, and try to use it if available.
Packit 961e70
 * Fall to self CCA setting if errors.
Packit 961e70
 */
Packit 961e70
	if (snprintf(ccabuf, len_ccabuf, "%s%d/ports/%d/CCMgtA/cc_settings_bin",
Packit 961e70
		     hfi_sysfs_path(), unit, port) >= (len_ccabuf-1))
Packit 961e70
		return -1;
Packit 961e70
Packit 961e70
	fd = open(ccabuf, O_RDONLY);
Packit 961e70
	if (fd < 0) {
Packit 961e70
		return 0;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	if (read(fd, ccabuf, count) != count) {
Packit 961e70
		_HFI_CCADBG("Read cc_settings_bin failed. using static CCA\n");
Packit 961e70
		close(fd);
Packit 961e70
		return 0;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	close(fd);
Packit 961e70
Packit 961e70
	return 1;
Packit 961e70
}
Packit 961e70
Packit 961e70
int hfi_get_cc_table_bin(int unit, int port, uint16_t **cctp)
Packit 961e70
{
Packit 961e70
	int i;
Packit 961e70
	unsigned short ccti_limit;
Packit 961e70
	uint16_t *cct;
Packit 961e70
	int fd;
Packit 961e70
	char pathname[256];
Packit 961e70
	*cctp = NULL;
Packit 961e70
Packit 961e70
	if (snprintf(pathname,sizeof(pathname), "%s%d/ports/%d/CCMgtA/cc_table_bin",
Packit 961e70
		     hfi_sysfs_path(), unit, port) >= (sizeof(pathname)-1))
Packit 961e70
		return -1;
Packit 961e70
Packit 961e70
	fd = open(pathname, O_RDONLY);
Packit 961e70
	if (fd < 0) {
Packit 961e70
		_HFI_CCADBG("Open cc_table_bin failed. using static CCA\n");
Packit 961e70
		return 0;
Packit 961e70
	}
Packit 961e70
	if (read(fd, &ccti_limit, sizeof(ccti_limit)) != sizeof(ccti_limit)) {
Packit 961e70
		_HFI_CCADBG("Read ccti_limit failed. using static CCA\n");
Packit 961e70
		close(fd);
Packit 961e70
		return 0;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	_HFI_CCADBG("ccti_limit = %d\n", ccti_limit);
Packit 961e70
Packit 961e70
	if (ccti_limit < 63) {
Packit 961e70
		_HFI_CCADBG("Read ccti_limit %d not in range [63, 65535], "
Packit 961e70
			    "using static CCA.\n", ccti_limit);
Packit 961e70
		close(fd);
Packit 961e70
		return 0;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	i = (ccti_limit + 1) * sizeof(uint16_t);
Packit 961e70
	cct = malloc(i);
Packit 961e70
	if (!cct) {
Packit 961e70
		close(fd);
Packit 961e70
		return -1;
Packit 961e70
	}
Packit 961e70
	if (read(fd, cct, i) != i) {
Packit 961e70
		_HFI_CCADBG("Read ccti_entry_list, using static CCA\n");
Packit 961e70
		free(cct);
Packit 961e70
		close(fd);
Packit 961e70
		return 0;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	close(fd);
Packit 961e70
Packit 961e70
	_HFI_CCADBG("cct[0] = 0x%04x\n", cct[0]);
Packit 961e70
Packit 961e70
	*cctp = cct;
Packit 961e70
	return ccti_limit;
Packit 961e70
}
Packit 961e70
Packit 961e70
/*
Packit 961e70
 * This is for diag function hfi_wait_for_packet() only
Packit 961e70
 */
Packit 961e70
int hfi_cmd_wait_for_packet(int fd)
Packit 961e70
{
Packit 961e70
	int ret;
Packit 961e70
	struct pollfd pfd;
Packit 961e70
Packit 961e70
	pfd.fd = fd;
Packit 961e70
	pfd.events = POLLIN;
Packit 961e70
Packit 961e70
	ret = poll(&pfd, 1, 500 /* ms */);
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}