/*

 * Copyright (c) 2009 Voltaire, Inc. All rights reserved.

 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.

 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.

 *

 * This software is available to you under a choice of one of two

 * licenses.  You may choose to be licensed under the terms of the GNU

 * General Public License (GPL) Version 2, available from the file

 * COPYING in the main directory of this source tree, or the

 * OpenIB.org BSD license below:

 *

 *     Redistribution and use in source and binary forms, with or

 *     without modification, are permitted provided that the following

 *     conditions are met:

 *

 *      - Redistributions of source code must retain the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer.

 *

 *      - Redistributions in binary form must reproduce the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer in the documentation and/or other materials

 *        provided with the distribution.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS

 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN

 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN

 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

 * SOFTWARE.

 *

 */

#ifdef OSM_VENDOR_INTF_MTL

/*                  - Mellanox Confidential and Proprietary -

 *

 *  Copyright (C) Jul. 2001, Mellanox Technologies Ltd.  ALL RIGHTS RESERVED.

 *

 *  Except as specifically permitted herein, no portion of the information,

 *  including but not limited to object code and source code, may be reproduced,

 *  modified, distributed, republished or otherwise exploited in any form or by

 *  any means for any purpose without the prior written permission of Mellanox

 *  Technologies Ltd. Use of software subject to the terms and conditions

 *  detailed in the file "LICENSE.txt".

 *

 *  End of legal section ......................................................

 *

 *  osmt_mtl_regular_qp.c -

 *    Provide Simple Interface for Sending and Receiving MADS through a regular QP

 *

 *  Creation date:

 *

 *  Version: $Id$

 *

 *  Authors:

 *    Eitan Zahavi

 *

 *  Changes:

 */

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#include <signal.h>

#include <mtl_common.h>

#include <vapi.h>

#include <evapi.h>

#include <vapi_common.h>

#include <ib_defs.h>

#include <osmt_mtl_regular_qp.h>

#include <complib/cl_types.h>

/*

 * Initialize the QP etc.

 * Given in res: port_num, max_outs_sq, max_outs_rq

 */

VAPI_ret_t osmt_mtl_get_qp_resources(IN OUT osmt_mtl_mad_res_t * res)

{

	VAPI_ret_t ret;

	VAPI_hca_port_t hca_port_info;

	VAPI_qp_init_attr_t qp_init_attr;

	VAPI_qp_prop_t qp_prop;

	VAPI_cqe_num_t act_num;

	/* Get HCA LID */

	ret =

	    VAPI_query_hca_port_prop(res->hca_hndl, res->port_num,

				     &hca_port_info);

	VAPI_CHECK_RET;

	res->slid = hca_port_info.lid;

	/* Get a PD */

	ret = VAPI_alloc_pd(res->hca_hndl, &(res->pd_hndl));

	VAPI_CHECK_RET;

	/* Create CQ for RQ and SQ *//* TBD - Check we have enough act nums */

	ret =

	    VAPI_create_cq(res->hca_hndl, res->max_outs_sq + 1,

			   &(res->sq_cq_hndl), &act_num);

	VAPI_CHECK_RET;

	ret =

	    VAPI_create_cq(res->hca_hndl, res->max_outs_rq + 1,

			   &(res->rq_cq_hndl), &act_num);

	VAPI_CHECK_RET;

	/* register event handlers for polling(block mode) internal use */

	/* ret= EVAPI_set_comp_eventh(res->hca_hndl,res->rq_cq_hndl, */

	/*                            EVAPI_POLL_CQ_UNBLOCK_HANDLER,NULL,&(res->rq_cq_eventh)); */

	/* VAPI_CHECK_RET; */

	/* ret= EVAPI_set_comp_eventh(res->hca_hndl,res->sq_cq_hndl, */

	/*                            EVAPI_POLL_CQ_UNBLOCK_HANDLER,NULL,&(res->sq_cq_eventh)); */

	/* VAPI_CHECK_RET; */

	/* Create QP */

	qp_init_attr.cap.max_oust_wr_sq = res->max_outs_sq + 1;

	qp_init_attr.cap.max_oust_wr_rq = res->max_outs_rq + 1;

	qp_init_attr.cap.max_sg_size_sq = 4;

	qp_init_attr.cap.max_sg_size_rq = 4;

	qp_init_attr.pd_hndl = res->pd_hndl;

	qp_init_attr.rdd_hndl = 0;

	qp_init_attr.rq_cq_hndl = res->rq_cq_hndl;

	qp_init_attr.rq_sig_type = VAPI_SIGNAL_ALL_WR;	/* That's default for IB */

	qp_init_attr.sq_cq_hndl = res->sq_cq_hndl;

	qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;

	qp_init_attr.ts_type = VAPI_TS_UD;

	ret =

	    VAPI_create_qp(res->hca_hndl, &qp_init_attr, &(res->qp_hndl),

			   &qp_prop);

	VAPI_CHECK_RET;

	res->qp_id.qp_num = qp_prop.qp_num;

	return (VAPI_OK);

}

VAPI_ret_t osmt_mtl_qp_init(osmt_mtl_mad_res_t * res)

{

	VAPI_ret_t ret;

	VAPI_qp_attr_t qp_attr;

	VAPI_qp_attr_mask_t qp_attr_mask;

	VAPI_qp_cap_t qp_cap;

	/*

	 * Change QP to INIT

	 *

	 */

	QP_ATTR_MASK_CLR_ALL(qp_attr_mask);

	qp_attr.qp_state = VAPI_INIT;

	QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);

	qp_attr.pkey_ix = 0;

	QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);

	qp_attr.port = res->port_num;

	QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);

	qp_attr.qkey = res->qkey;

	QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QKEY);

	/* If I do not set this mask, I get an error from HH. QPM should catch it */

	ret =

	    VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,

			   &qp_cap);

	VAPI_CHECK_RET;

	return (ret);

}

VAPI_ret_t osmt_mtl_qp_2_rtr_rts(osmt_mtl_mad_res_t * res)

{

	VAPI_ret_t ret;

	VAPI_qp_attr_t qp_attr;

	VAPI_qp_attr_mask_t qp_attr_mask;

	VAPI_qp_cap_t qp_cap;

	/*

	 *  Change QP to RTR

	 *

	 */

	QP_ATTR_MASK_CLR_ALL(qp_attr_mask);

	qp_attr.qp_state = VAPI_RTR;

	QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);

	/*   qp_attr.rq_psn   = 0;                */

	/*   QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RQ_PSN); */

	ret =

	    VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,

			   &qp_cap);

	VAPI_CHECK_RET;

	/*

	 * Change QP to RTS

	 *

	 */

	QP_ATTR_MASK_CLR_ALL(qp_attr_mask);

	qp_attr.qp_state = VAPI_RTS;

	QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);

	qp_attr.sq_psn = 0;

	QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);

	ret =

	    VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,

			   &qp_cap);

	VAPI_CHECK_RET;

	return (ret);

}

VAPI_ret_t osmt_mtl_mad_create_mr(osmt_mtl_mad_res_t * res)

{

	VAPI_ret_t ret;

	VAPI_mrw_t mr_in, mr_out;

	res->buf_size =

	    (MAD_SIZE + GRH_LEN) * (res->max_outs_sq + res->max_outs_rq + 1);

	/* Register single memory address region for all buffers */

	res->buf_ptr = VMALLOC(res->buf_size);

	if (res->buf_ptr == ((VAPI_virt_addr_t) NULL)) {

		ret = VAPI_EAGAIN;

		VAPI_CHECK_RET;

	}

	/* Enable local and remote access to memory region */

	mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;

	mr_in.l_key = 0;

	mr_in.pd_hndl = res->pd_hndl;

	mr_in.r_key = 0;

	mr_in.size = res->buf_size;

	ASSERT_VOIDP2UINTN(res->buf_ptr);

	mr_in.start = (VAPI_virt_addr_t) (res->buf_ptr);

	mr_in.type = VAPI_MR;

	ret = VAPI_register_mr(res->hca_hndl, &mr_in, &(res->mr_hndl), &mr_out);

	VAPI_CHECK_RET;

	res->l_key = mr_out.l_key;

	return (ret);

}

VAPI_ret_t osmt_mtl_init_opened_hca(osmt_mtl_mad_res_t * res)

{

	VAPI_ret_t ret;

	res->pd_hndl = VAPI_INVAL_HNDL;

	res->rq_cq_hndl = VAPI_INVAL_HNDL;

	res->sq_cq_hndl = VAPI_INVAL_HNDL;

	res->sq_cq_eventh = VAPI_INVAL_HNDL;

	res->rq_cq_eventh = VAPI_INVAL_HNDL;

	res->qp_hndl = VAPI_INVAL_HNDL;

	res->mr_hndl = VAPI_INVAL_HNDL;

	/*

	 * Create QP

	 *

	 */

	ret = osmt_mtl_get_qp_resources(res);

	if (ret != VAPI_OK) {

		return ret;

	}

	/*

	 * Move to init

	 *

	 */

	ret = osmt_mtl_qp_init(res);

	if (ret != VAPI_OK) {

		return ret;

	}

	/*

	 * Initialize memory regions

	 *

	 */

	ret = osmt_mtl_mad_create_mr(res);

	if (ret != VAPI_OK) {

		return ret;

	}

	/* only now move to RTR and RTS */

	ret = osmt_mtl_qp_2_rtr_rts(res);

	if (ret != VAPI_OK) {

		return ret;

	}

	return VAPI_OK;

}

VAPI_ret_t osmt_mtl_mad_cleanup(osmt_mtl_mad_res_t * res)

{

	if (res->qp_hndl != VAPI_INVAL_HNDL) {

		VAPI_destroy_qp(res->hca_hndl, res->qp_hndl);

	}

	if (res->sq_cq_eventh != VAPI_INVAL_HNDL) {

		EVAPI_clear_comp_eventh(res->hca_hndl, res->sq_cq_eventh);

	}

	if (res->rq_cq_eventh != VAPI_INVAL_HNDL) {

		EVAPI_clear_comp_eventh(res->hca_hndl, res->rq_cq_eventh);

	}

	if (res->rq_cq_hndl != VAPI_INVAL_HNDL) {

		VAPI_destroy_cq(res->hca_hndl, res->rq_cq_hndl);

	}

	if (res->sq_cq_hndl != VAPI_INVAL_HNDL) {

		VAPI_destroy_cq(res->hca_hndl, res->sq_cq_hndl);

	}

	if (res->mr_hndl != VAPI_INVAL_HNDL) {

		VAPI_deregister_mr(res->hca_hndl, res->mr_hndl);

	}

	if (res->pd_hndl != VAPI_INVAL_HNDL) {

		VAPI_dealloc_pd(res->hca_hndl, res->pd_hndl);

	}

#if 0

	/* open/close of HCA should be done system wide - not per application */

	if (res->hca_hndl != VAPI_INVAL_HNDL) {

		VAPI_close_hca(res->hca_hndl);	/* TBD: HCA_open/close should be done on a system wide basis */

	}

#endif

	return VAPI_OK;

}

VAPI_ret_t osmt_mtl_create_av(osmt_mtl_mad_res_t * res, int16_t dlid,

			      VAPI_ud_av_hndl_t * avh_p)

{

	VAPI_ud_av_t av;

	VAPI_ret_t ret;

	av.dlid = dlid;

	av.port = res->port_num;

	av.sl = 0;		/* dest->sl; */

	av.src_path_bits = 0;	/*  dest->ee_dlid.dst_path_bits; */

	av.static_rate = 0;

	/* GRH ? */

	av.grh_flag = 0;

	ret = VAPI_create_addr_hndl(res->hca_hndl, res->pd_hndl, &av, avh_p);

	if (ret != VAPI_OK) {

		MTL_ERROR1("%s: failed VAPI_create_addr_hndl (%s)\n", __func__,

			   VAPI_strerror_sym(ret));

		return ret;

	}

	return VAPI_OK;

}

VAPI_ret_t osmt_mtl_mad_send(osmt_mtl_mad_res_t * res, VAPI_wr_id_t id,

			     void *mad, VAPI_qp_num_t dest_qp, IB_sl_t sl,

			     u_int32_t dest_qkey, VAPI_ud_av_hndl_t avh)

{

	VAPI_sr_desc_t sr;

	VAPI_sg_lst_entry_t sg_entry;

	VAPI_ret_t ret;

	/* building SEND request */

	sr.opcode = VAPI_SEND;

	sr.remote_ah = avh;

	sr.remote_qp = dest_qp;

	sr.remote_qkey = dest_qkey;

	sr.id = id;

	sr.set_se = FALSE;

	sr.fence = FALSE;

	sr.comp_type = VAPI_SIGNALED;

	sr.sg_lst_len = 1;

	sr.sg_lst_p = &sg_entry;

	ASSERT_VOIDP2UINTN(mad);

	sg_entry.addr = (VAPI_virt_addr_t) (mad);

	sg_entry.len = MAD_SIZE;

	sg_entry.lkey = res->l_key;

	ret = VAPI_post_sr(res->hca_hndl, res->qp_hndl, &sr);

	if (ret != VAPI_OK) {

		MTL_ERROR1(__FUNCTION__ ": failed VAPI_post_sr (%s)\n",

			   VAPI_strerror_sym(ret));

		return ret;

	}

	return VAPI_OK;

}

int osmt_mtl_mad_post_recv_bufs(osmt_mtl_mad_res_t * res, void *buf_array,

				u_int32_t num_o_bufs, u_int32_t size,

				VAPI_wr_id_t start_id)

{

	uint32_t i;

	void *cur_buf;

	VAPI_rr_desc_t rr;

	VAPI_sg_lst_entry_t sg_entry;

	VAPI_ret_t ret;

	rr.opcode = VAPI_RECEIVE;

	rr.comp_type = VAPI_SIGNALED;	/* All with CQE (IB compliant) */

	rr.sg_lst_len = 1;	/* single buffers */

	rr.sg_lst_p = &sg_entry;

	sg_entry.lkey = res->l_key;

	cur_buf = buf_array;

	for (i = 0; i < num_o_bufs; i++) {

		rr.id = start_id + i;	/* WQE id used is the index to buffers ptr array */

		ASSERT_VOIDP2UINTN(cur_buf);

		sg_entry.addr = (VAPI_virt_addr_t) cur_buf;

		sg_entry.len = size;

		memset(cur_buf, 0x00, size);	/* fill with 0 */

		ret = VAPI_post_rr(res->hca_hndl, res->qp_hndl, &rr);

		if (ret != VAPI_OK) {

			MTL_ERROR1(__FUNCTION__

				   ": failed posting RQ WQE (%s)\n",

				   VAPI_strerror_sym(ret));

			return i;

		}

		MTL_DEBUG4(__FUNCTION__ ": posted buf at %p\n", cur_buf);

		cur_buf += size;

	}

	return i;		/* num of buffers posted */

}

VAPI_ret_t osmt_mtl_mad_poll4cqe(VAPI_hca_hndl_t hca, VAPI_cq_hndl_t cq,

				 VAPI_wc_desc_t * wc_desc_p,

				 u_int32_t max_poll, u_int32_t poll_sleep,

				 VAPI_ud_av_hndl_t * avh_p)

{

	VAPI_ret_t ret = VAPI_CQ_EMPTY;

	u_int32_t poll_cnt = 0;

	/* wait for something to arrive */

	while ((ret == VAPI_CQ_EMPTY) && (poll_cnt < max_poll)) {

		ret = VAPI_poll_cq(hca, cq, wc_desc_p);

		/* don't sleep if we already succeeded) */

		if (ret != VAPI_CQ_EMPTY) {

			break;

		}

		usleep(poll_sleep);

		poll_cnt++;

	}

	/* if passed an AVH to destory - do it */

	if (avh_p != NULL) {

		VAPI_destroy_addr_hndl(hca, *avh_p);

	}

	if ((poll_cnt == max_poll) && (ret == VAPI_CQ_EMPTY)) {

		MTL_DEBUG1(__FUNCTION__

			   ": Failed to get completion on wq after %d polls.\n",

			   max_poll);

		return VAPI_CQ_EMPTY;

	}

	if (ret != VAPI_OK) {

		MTL_DEBUG1(__FUNCTION__

			   ": VAPI_poll_cq failed with ret=%s on sq_cq\n",

			   mtl_strerror_sym(ret));

		return ret;

	}

	if (wc_desc_p->status != VAPI_SUCCESS) {

		MTL_DEBUG1(__FUNCTION__ ": completion error (%d) detected\n",

			   wc_desc_p->status);

	}

	return VAPI_OK;

}

#endif				/*  OSM_VENDOR_INTF_MTL */