Blame ptl_ips/ips_proto_am.c

Packit 961e70
/*
Packit 961e70
Packit 961e70
  This file is provided under a dual BSD/GPLv2 license.  When using or
Packit 961e70
  redistributing this file, you may do so under either license.
Packit 961e70
Packit 961e70
  GPL LICENSE SUMMARY
Packit 961e70
Packit 961e70
  Copyright(c) 2015 Intel Corporation.
Packit 961e70
Packit 961e70
  This program is free software; you can redistribute it and/or modify
Packit 961e70
  it under the terms of version 2 of the GNU General Public License as
Packit 961e70
  published by the Free Software Foundation.
Packit 961e70
Packit 961e70
  This program is distributed in the hope that it will be useful, but
Packit 961e70
  WITHOUT ANY WARRANTY; without even the implied warranty of
Packit 961e70
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Packit 961e70
  General Public License for more details.
Packit 961e70
Packit 961e70
  Contact Information:
Packit 961e70
  Intel Corporation, www.intel.com
Packit 961e70
Packit 961e70
  BSD LICENSE
Packit 961e70
Packit 961e70
  Copyright(c) 2015 Intel Corporation.
Packit 961e70
Packit 961e70
  Redistribution and use in source and binary forms, with or without
Packit 961e70
  modification, are permitted provided that the following conditions
Packit 961e70
  are met:
Packit 961e70
Packit 961e70
    * Redistributions of source code must retain the above copyright
Packit 961e70
      notice, this list of conditions and the following disclaimer.
Packit 961e70
    * Redistributions in binary form must reproduce the above copyright
Packit 961e70
      notice, this list of conditions and the following disclaimer in
Packit 961e70
      the documentation and/or other materials provided with the
Packit 961e70
      distribution.
Packit 961e70
    * Neither the name of Intel Corporation nor the names of its
Packit 961e70
      contributors may be used to endorse or promote products derived
Packit 961e70
      from this software without specific prior written permission.
Packit 961e70
Packit 961e70
  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
Packit 961e70
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
Packit 961e70
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
Packit 961e70
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
Packit 961e70
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
Packit 961e70
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
Packit 961e70
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
Packit 961e70
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
Packit 961e70
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
Packit 961e70
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
Packit 961e70
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Packit 961e70
Packit 961e70
*/
Packit 961e70
Packit 961e70
/* Copyright (c) 2003-2014 Intel Corporation. All rights reserved. */
Packit 961e70
Packit 961e70
#include "psm_user.h"
Packit 961e70
#include "psm2_hal.h"
Packit 961e70
#include "psm2_am.h"
Packit 961e70
#include "psm_am_internal.h"
Packit 961e70
#include "psm_mq_internal.h"
Packit 961e70
#include "ips_proto.h"
Packit 961e70
#include "ips_expected_proto.h"
Packit 961e70
#include "ips_proto_help.h"
Packit 961e70
Packit 961e70
struct ips_am_token {
Packit 961e70
	struct psmi_am_token tok;
Packit 961e70
Packit 961e70
	/* ptl-specific token stuff */
Packit 961e70
	struct ips_epaddr *epaddr_rail;
Packit 961e70
	struct ips_proto_am *proto_am;
Packit 961e70
};
Packit 961e70
Packit 961e70
struct ips_am_message {
Packit 961e70
	struct ips_message_header p_hdr;
Packit 961e70
	struct ips_am_message *next;
Packit 961e70
	struct ips_epaddr *ipsaddr;
Packit 961e70
	struct ips_proto_am *proto_am;
Packit 961e70
	uint64_t *payload;
Packit 961e70
	uint32_t paylen;
Packit 961e70
	uint16_t seqnum;
Packit 961e70
};
Packit 961e70
Packit 961e70
/* These variables are shared for all packet flows in a PSM process; they are
Packit 961e70
 * shared across multiple rails.  There is no single AM object to hang these
Packit 961e70
 * off of, so they are declared here as globals. */
Packit 961e70
static struct {
Packit 961e70
	struct ips_am_message head;
Packit 961e70
	struct ips_am_message *tail;
Packit 961e70
} ips_am_outoforder_q;
Packit 961e70
Packit 961e70
static mpool_t ips_am_msg_pool;
Packit 961e70
Packit 961e70
/* This calculation ensures that the number of reply slots will always be at
Packit 961e70
 * least twice as large + 1 as the number of request slots. This is optimal: the
Packit 961e70
 * minimum amount required is actually only twice as many, but it is much
Packit 961e70
 * slower. */
Packit 961e70
#define calc_optimal_num_reply_slots(nslots) (((nslots)*2 / 3) + 1)
Packit 961e70
Packit 961e70
psm2_error_t
Packit 961e70
MOCKABLE(ips_proto_am_init)(struct ips_proto *proto,
Packit 961e70
		  int num_send_slots,
Packit 961e70
		  uint32_t imm_size,
Packit 961e70
		  struct ips_proto_am *proto_am)
Packit 961e70
{
Packit 961e70
	psm2_error_t err = PSM2_OK;
Packit 961e70
	int send_buf_size = psmi_hal_get_pio_size(proto->ep->context.psm_hw_ctxt);
Packit 961e70
	int num_rep_slots = calc_optimal_num_reply_slots(num_send_slots);
Packit 961e70
	int num_req_slots = num_send_slots - num_rep_slots;
Packit 961e70
Packit 961e70
	proto_am->proto = proto;
Packit 961e70
Packit 961e70
	/* In a node pair, the number of reply send buffers on at least one of
Packit 961e70
	 * the nodes must be at least double the number (optimal: double + 1) of
Packit 961e70
	 * send descriptors on the other node. While this constraint applies
Packit 961e70
	 * only to the reply send buffers, allowing the caller to tune only the
Packit 961e70
	 * number of request send buffers would be awkward, as they have no
Packit 961e70
	 * knowledge of the subdivision of the memory into separate mempools for
Packit 961e70
	 * requests and replies. It's an internal concern at this point. */
Packit 961e70
	if ((err = ips_scbctrl_init(&proto->ep->context,
Packit 961e70
				    num_req_slots,
Packit 961e70
				    num_req_slots,
Packit 961e70
				    imm_size,
Packit 961e70
				    send_buf_size,
Packit 961e70
				    NULL,
Packit 961e70
				    NULL,
Packit 961e70
				    &proto_am->scbc_request)))
Packit 961e70
		goto fail;
Packit 961e70
Packit 961e70
	if ((err = ips_scbctrl_init(&proto->ep->context,
Packit 961e70
				    num_rep_slots,
Packit 961e70
				    num_rep_slots,
Packit 961e70
				    imm_size,
Packit 961e70
				    send_buf_size,
Packit 961e70
				    NULL,
Packit 961e70
				    NULL,
Packit 961e70
				    &proto_am->scbc_reply)))
Packit 961e70
		goto fail;
Packit 961e70
Packit 961e70
	if (ips_am_msg_pool == NULL) {
Packit 961e70
		union psmi_envvar_val max_msgs;
Packit 961e70
Packit 961e70
		ips_am_outoforder_q.head.next = NULL;
Packit 961e70
		ips_am_outoforder_q.tail = &ips_am_outoforder_q.head;
Packit 961e70
Packit 961e70
		psmi_getenv("PSM2_AM_MAX_OOO_MSGS",
Packit 961e70
			"Maximum number of OOO Active Messages to queue before dropping.",
Packit 961e70
			PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT,
Packit 961e70
			(union psmi_envvar_val)1024, &max_msgs);
Packit 961e70
Packit 961e70
		ips_am_msg_pool = psmi_mpool_create(
Packit 961e70
				sizeof(struct ips_am_message),
Packit 961e70
				32, max_msgs.e_uint, 0, UNDEFINED, NULL, NULL);
Packit 961e70
	}
Packit 961e70
fail:
Packit 961e70
	return err;
Packit 961e70
}
Packit 961e70
MOCK_DEF_EPILOGUE(ips_proto_am_init);
Packit 961e70
Packit 961e70
psm2_error_t ips_proto_am_fini(struct ips_proto_am *proto_am)
Packit 961e70
{
Packit 961e70
	ips_scbctrl_fini(&proto_am->scbc_request);
Packit 961e70
	ips_scbctrl_fini(&proto_am->scbc_reply);
Packit 961e70
	if (ips_am_msg_pool != NULL) {
Packit 961e70
		psmi_mpool_destroy(ips_am_msg_pool);
Packit 961e70
		ips_am_msg_pool = NULL;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return PSM2_OK;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Fill in AM capabilities parameters */
Packit 961e70
psm2_error_t
Packit 961e70
ips_am_get_parameters(psm2_ep_t ep, struct psm2_am_parameters *parameters)
Packit 961e70
{
Packit 961e70
	int max_nargs = min(1 << IPS_AM_HDR_NARGS_BITS, PSMI_AM_MAX_ARGS);
Packit 961e70
	int max_payload =
Packit 961e70
		psmi_hal_get_pio_size(ep->context.psm_hw_ctxt) -
Packit 961e70
		((max_nargs - IPS_AM_HDR_NARGS) * sizeof(psm2_amarg_t));
Packit 961e70
Packit 961e70
	if (parameters == NULL) {
Packit 961e70
		return PSM2_PARAM_ERR;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	parameters->max_handlers = 1 << IPS_AM_HDR_HIDX_BITS;
Packit 961e70
	parameters->max_nargs = max_nargs;
Packit 961e70
	parameters->max_request_short = max_payload;
Packit 961e70
	parameters->max_reply_short = max_payload;
Packit 961e70
Packit 961e70
	return PSM2_OK;
Packit 961e70
}
Packit 961e70
Packit 961e70
static
Packit 961e70
psm2_error_t
Packit 961e70
am_short_reqrep(ips_scb_t *scb, struct ips_epaddr *ipsaddr,
Packit 961e70
		psm2_amarg_t *args, int nargs, uint8_t opcode,
Packit 961e70
		void *src, size_t len, int flags, int pad_bytes)
Packit 961e70
{
Packit 961e70
	int i, hdr_qwords = IPS_AM_HDR_NARGS;
Packit 961e70
	struct ips_proto *proto = ((psm2_epaddr_t)ipsaddr)->proto;
Packit 961e70
Packit 961e70
	psmi_assert(proto->msgflowid < EP_FLOW_LAST);
Packit 961e70
Packit 961e70
	struct ips_flow *flow = &ipsaddr->flows[proto->msgflowid];
Packit 961e70
Packit 961e70
	/* There are a limited number of bits for nargs in the header, making
Packit 961e70
	   overflow very easy.  Make sure the values match. */
Packit 961e70
	psmi_assert(nargs == scb->ips_lrh.amhdr_nargs);
Packit 961e70
Packit 961e70
	_HFI_VDBG("%s src=%p len=%d, nargs=%d\n",
Packit 961e70
		  ((opcode == OPCODE_AM_REQUEST) ||
Packit 961e70
		   (opcode == OPCODE_AM_REQUEST_NOREPLY)) ? "req" : "rep",
Packit 961e70
		  src, (int)len, nargs);
Packit 961e70
Packit 961e70
	if (nargs == 1) {	/* fastpath */
Packit 961e70
		scb->ips_lrh.data[0].u64w0 = args[0].u64w0;
Packit 961e70
		hdr_qwords--;
Packit 961e70
	} else if (nargs > 1) {
Packit 961e70
		/* Easily unrollable but leave as is in case we can increase
Packit 961e70
		 * qwords on the chip in the near future */
Packit 961e70
		for (i = 0; i < IPS_AM_HDR_NARGS; i++, hdr_qwords--)
Packit 961e70
			scb->ips_lrh.data[i].u64w0 = args[i].u64w0;
Packit 961e70
Packit 961e70
		if (nargs > IPS_AM_HDR_NARGS) {
Packit 961e70
			/* Slow case -- we don't have iovec and not enough
Packit 961e70
			 * space in the message header, so we have to copy the
Packit 961e70
			 * user's arguments even if the payload is marked ASYNC
Packit 961e70
			 */
Packit 961e70
			uintptr_t bufp = (uintptr_t) ips_scb_buffer(scb);
Packit 961e70
			size_t arg_payload_len =
Packit 961e70
			    sizeof(psm2_amarg_t) * (nargs - IPS_AM_HDR_NARGS);
Packit 961e70
Packit 961e70
			psmi_mq_mtucpy((void *)bufp,
Packit 961e70
				       &args[IPS_AM_HDR_NARGS],
Packit 961e70
				       arg_payload_len);
Packit 961e70
			bufp += arg_payload_len;
Packit 961e70
			scb->payload_size = arg_payload_len;
Packit 961e70
Packit 961e70
			if (src != NULL && len > 0) {
Packit 961e70
				psmi_mq_mtucpy((void *)bufp, src, len);
Packit 961e70
				scb->payload_size += len;
Packit 961e70
			}
Packit 961e70
Packit 961e70
			psmi_assert(pad_bytes < (1 << IPS_AM_HDR_LEN_BITS));
Packit 961e70
			scb->payload_size += pad_bytes;
Packit 961e70
			scb->ips_lrh.amhdr_len = pad_bytes;
Packit 961e70
			goto send_scb;
Packit 961e70
		}
Packit 961e70
	}
Packit 961e70
Packit 961e70
	if (len == 0) {
Packit 961e70
		scb->payload_size = 0;
Packit 961e70
		scb->ips_lrh.amhdr_len = 0;
Packit 961e70
	} else if (len <= (hdr_qwords << 3)) {
Packit 961e70
		/* Inline the payload into the header. */
Packit 961e70
		/* This path CANNOT handle length = 0 due to limited space
Packit 961e70
		   in the header.  If IPS_SEND_FLAG_AMISTINY is set, an
Packit 961e70
		   amhdr_len value of 0 means a full payload, i.e.
Packit 961e70
		   1 << IPS_AM_HDR_LEN_BITS bytes of packed payload. */
Packit 961e70
		psmi_assert(len > 0);
Packit 961e70
Packit 961e70
		psmi_mq_mtucpy(&scb->ips_lrh.
Packit 961e70
			       data[IPS_AM_HDR_NARGS - hdr_qwords], src, len);
Packit 961e70
		scb->payload_size = 0;
Packit 961e70
		psmi_assert(len <= (1 << IPS_AM_HDR_LEN_BITS));
Packit 961e70
		scb->ips_lrh.amhdr_len = len & ((1 << IPS_AM_HDR_LEN_BITS) - 1);
Packit 961e70
		scb->scb_flags |= IPS_SEND_FLAG_AMISTINY;
Packit 961e70
	} else { /* Whatever's left requires a separate payload */
Packit 961e70
		if (ips_scb_buffer(scb) == NULL) /* Just attach the buffer */
Packit 961e70
			ips_scb_buffer(scb) = src;
Packit 961e70
		else /* May need to re-xmit user data, keep it around */
Packit 961e70
			psmi_mq_mtucpy(ips_scb_buffer(scb), src, len);
Packit 961e70
Packit 961e70
		psmi_assert(pad_bytes < (1 << IPS_AM_HDR_LEN_BITS));
Packit 961e70
		scb->payload_size = len + pad_bytes;
Packit 961e70
		scb->ips_lrh.amhdr_len = pad_bytes;
Packit 961e70
	}
Packit 961e70
Packit 961e70
send_scb:
Packit 961e70
	ips_scb_opcode(scb) = opcode;
Packit 961e70
	scb->ips_lrh.khdr.kdeth0 = ipsaddr->msgctl->am_send_seqnum++;
Packit 961e70
	ips_proto_flow_enqueue(flow, scb);
Packit 961e70
	flow->flush(flow, NULL);
Packit 961e70
Packit 961e70
	return PSM2_OK;
Packit 961e70
}
Packit 961e70
Packit 961e70
static inline int
Packit 961e70
calculate_pad_bytes(size_t len)
Packit 961e70
{
Packit 961e70
	/* Align to dword (4 bytes) */
Packit 961e70
	size_t dword_aligned_len = (len + 3) & ~3;
Packit 961e70
	return dword_aligned_len - len;
Packit 961e70
}
Packit 961e70
Packit 961e70
static inline
Packit 961e70
void
Packit 961e70
ips_am_scb_init(ips_scb_t *scb, uint8_t handler, int nargs,
Packit 961e70
		int pad_bytes,
Packit 961e70
		psm2_am_completion_fn_t completion_fn, void *completion_ctxt)
Packit 961e70
{
Packit 961e70
	psmi_assert(pad_bytes < (1 << IPS_AM_HDR_LEN_BITS));
Packit 961e70
Packit 961e70
	scb->completion_am = completion_fn;
Packit 961e70
	scb->cb_param = completion_ctxt;
Packit 961e70
	scb->ips_lrh.amhdr_hidx = handler;
Packit 961e70
	scb->ips_lrh.amhdr_len = pad_bytes;
Packit 961e70
	scb->ips_lrh.amhdr_nargs = nargs;
Packit 961e70
	scb->ips_lrh.flags = 0;
Packit 961e70
	if (completion_fn)
Packit 961e70
		scb->scb_flags |= IPS_SEND_FLAG_ACKREQ;
Packit 961e70
	return;
Packit 961e70
}
Packit 961e70
Packit 961e70
psm2_error_t
Packit 961e70
ips_am_short_request(psm2_epaddr_t epaddr,
Packit 961e70
		     psm2_handler_t handler, psm2_amarg_t *args, int nargs,
Packit 961e70
		     void *src, size_t len, int flags,
Packit 961e70
		     psm2_am_completion_fn_t completion_fn,
Packit 961e70
		     void *completion_ctxt)
Packit 961e70
{
Packit 961e70
	struct ips_proto_am *proto_am = &epaddr->proto->proto_am;
Packit 961e70
	psm2_error_t err;
Packit 961e70
	ips_scb_t *scb;
Packit 961e70
	ips_epaddr_t *ipsaddr;
Packit 961e70
	int pad_bytes = calculate_pad_bytes(len);
Packit 961e70
	int payload_sz = (nargs << 3);
Packit 961e70
Packit 961e70
	if_pt(!(flags & PSM2_AM_FLAG_ASYNC))
Packit 961e70
	    payload_sz += len;
Packit 961e70
Packit 961e70
	if (payload_sz > (IPS_AM_HDR_NARGS << 3)) {
Packit 961e70
		/* Payload can't fit in header, allocate buffer to carry data */
Packit 961e70
		int arg_sz = (nargs > IPS_AM_HDR_NARGS) ?
Packit 961e70
		    ((nargs - IPS_AM_HDR_NARGS) << 3) : 0;
Packit 961e70
Packit 961e70
		/* len + pad_bytes + overflow_args */
Packit 961e70
		PSMI_BLOCKUNTIL(epaddr->ptlctl->ep,
Packit 961e70
				err,
Packit 961e70
				((scb = ips_scbctrl_alloc(
Packit 961e70
				      &proto_am->scbc_request,
Packit 961e70
				      1,
Packit 961e70
				      len + pad_bytes + arg_sz,
Packit 961e70
				      IPS_SCB_FLAG_ADD_BUFFER)) != NULL));
Packit 961e70
	} else {
Packit 961e70
		PSMI_BLOCKUNTIL(epaddr->ptlctl->ep,
Packit 961e70
				err,
Packit 961e70
				((scb = ips_scbctrl_alloc_tiny(
Packit 961e70
				      &proto_am->scbc_request)) != NULL));
Packit 961e70
	}
Packit 961e70
Packit 961e70
	psmi_assert_always(scb != NULL);
Packit 961e70
	ips_am_scb_init(scb, handler, nargs, pad_bytes,
Packit 961e70
			completion_fn, completion_ctxt);
Packit 961e70
Packit 961e70
	/* Select the next ipsaddr for multi-rail */
Packit 961e70
	ipsaddr = ((ips_epaddr_t *)epaddr)->msgctl->ipsaddr_next;
Packit 961e70
	ipsaddr->msgctl->ipsaddr_next = ipsaddr->next;
Packit 961e70
Packit 961e70
	return am_short_reqrep(scb, ipsaddr, args,
Packit 961e70
			       nargs,
Packit 961e70
			       (flags & PSM2_AM_FLAG_NOREPLY) ?
Packit 961e70
			       OPCODE_AM_REQUEST_NOREPLY : OPCODE_AM_REQUEST,
Packit 961e70
			       src, len, flags, pad_bytes);
Packit 961e70
}
Packit 961e70
Packit 961e70
psm2_error_t
Packit 961e70
ips_am_short_reply(psm2_am_token_t tok,
Packit 961e70
		   psm2_handler_t handler, psm2_amarg_t *args, int nargs,
Packit 961e70
		   void *src, size_t len, int flags,
Packit 961e70
		   psm2_am_completion_fn_t completion_fn, void *completion_ctxt)
Packit 961e70
{
Packit 961e70
	struct ips_am_token *token = (struct ips_am_token *)tok;
Packit 961e70
	struct ips_proto_am *proto_am = token->proto_am;
Packit 961e70
	struct ips_epaddr *ipsaddr = token->epaddr_rail;
Packit 961e70
	int pad_bytes = calculate_pad_bytes(len);
Packit 961e70
	int scb_flags = 0;
Packit 961e70
	ips_scb_t *scb;
Packit 961e70
Packit 961e70
	if (!token->tok.can_reply) {
Packit 961e70
		_HFI_ERROR("Invalid AM reply for request!");
Packit 961e70
		return PSM2_AM_INVALID_REPLY;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	psmi_assert(ips_scbctrl_avail(&proto_am->scbc_reply));
Packit 961e70
Packit 961e70
	if ((nargs << 3) + len <= (IPS_AM_HDR_NARGS << 3)) {
Packit 961e70
		scb = ips_scbctrl_alloc_tiny(&proto_am->scbc_reply);
Packit 961e70
	} else {
Packit 961e70
		int payload_sz = (nargs << 3);
Packit 961e70
Packit 961e70
		payload_sz += (flags & PSM2_AM_FLAG_ASYNC) ?
Packit 961e70
			      0 : (len + pad_bytes);
Packit 961e70
		scb_flags |= (payload_sz > (IPS_AM_HDR_NARGS << 3)) ?
Packit 961e70
		    IPS_SCB_FLAG_ADD_BUFFER : 0;
Packit 961e70
Packit 961e70
		scb =
Packit 961e70
		    ips_scbctrl_alloc(&proto_am->scbc_reply, 1, payload_sz,
Packit 961e70
				      scb_flags);
Packit 961e70
	}
Packit 961e70
Packit 961e70
	psmi_assert_always(scb != NULL);
Packit 961e70
	ips_am_scb_init(scb, handler, nargs, pad_bytes,
Packit 961e70
			completion_fn, completion_ctxt);
Packit 961e70
	am_short_reqrep(scb, ipsaddr, args, nargs, OPCODE_AM_REPLY,
Packit 961e70
			src, len, flags, pad_bytes);
Packit 961e70
	return PSM2_OK;
Packit 961e70
}
Packit 961e70
Packit 961e70
/* Prepares and runs a handler from a receive event. */
Packit 961e70
static int
Packit 961e70
ips_am_run_handler(const struct ips_message_header *p_hdr,
Packit 961e70
		struct ips_epaddr *ipsaddr, struct ips_proto_am *proto_am,
Packit 961e70
		uint64_t *payload,
Packit 961e70
		uint32_t paylen)
Packit 961e70
{
Packit 961e70
	struct ips_am_token token;
Packit 961e70
	int nargs = p_hdr->amhdr_nargs;
Packit 961e70
	int ret;
Packit 961e70
	struct psm2_ep_am_handle_entry *hentry;
Packit 961e70
	psm2_amarg_t *args = (psm2_amarg_t *)p_hdr->data;
Packit 961e70
Packit 961e70
	token.tok.flags = p_hdr->flags;
Packit 961e70
	token.tok.epaddr_incoming = (psm2_epaddr_t)&ipsaddr->msgctl->master_epaddr;
Packit 961e70
	token.tok.can_reply =
Packit 961e70
		(_get_proto_hfi_opcode(p_hdr) == OPCODE_AM_REQUEST);
Packit 961e70
	token.epaddr_rail = ipsaddr;
Packit 961e70
	token.proto_am = proto_am;
Packit 961e70
Packit 961e70
	if (token.tok.flags & IPS_SEND_FLAG_AMISTINY) {
Packit 961e70
		/* Payload is packed into header after args */
Packit 961e70
		payload = (uint64_t *)&p_hdr->data[nargs].u64;
Packit 961e70
		paylen = p_hdr->amhdr_len;
Packit 961e70
		/* Interpret amhdr_len == 0 as 16 bytes of payload */
Packit 961e70
		if (paylen == 0)
Packit 961e70
			paylen = 1 << IPS_AM_HDR_LEN_BITS;
Packit 961e70
	} else {
Packit 961e70
		if (nargs > IPS_AM_HDR_NARGS) {
Packit 961e70
			/* Args are split across header and payload */
Packit 961e70
			int payload_args_len =
Packit 961e70
				(nargs - IPS_AM_HDR_NARGS) *
Packit 961e70
				sizeof(psm2_amarg_t);
Packit 961e70
Packit 961e70
			args = alloca(PSMI_AM_MAX_ARGS * sizeof(psm2_amarg_t));
Packit 961e70
Packit 961e70
			args[0].u64 = p_hdr->data[0].u64;
Packit 961e70
			args[1].u64 = p_hdr->data[1].u64;
Packit 961e70
Packit 961e70
			memcpy(&args[2], payload, payload_args_len);
Packit 961e70
Packit 961e70
			payload += nargs - IPS_AM_HDR_NARGS;
Packit 961e70
			paylen -= payload_args_len;
Packit 961e70
		}
Packit 961e70
Packit 961e70
		/* Subtract off padding bytes (dword padding) for non-TINY. */
Packit 961e70
		paylen -= p_hdr->amhdr_len;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	hentry = psm_am_get_handler_function(proto_am->proto->ep,
Packit 961e70
			p_hdr->amhdr_hidx);
Packit 961e70
Packit 961e70
	/* Note a guard here for hentry != NULL is not needed because at
Packit 961e70
	 * initialization, a psmi_assert_always() assure the entry will be
Packit 961e70
	 * non-NULL. */
Packit 961e70
Packit 961e70
	if (likely(hentry->version == PSM2_AM_HANDLER_V2)) {
Packit 961e70
		psm2_am_handler_2_fn_t hfn2 =
Packit 961e70
				(psm2_am_handler_2_fn_t)hentry->hfn;
Packit 961e70
		ret = hfn2(&token, args, nargs, payload, paylen, hentry->hctx);
Packit 961e70
	} else {
Packit 961e70
		psm2_am_handler_fn_t hfn1 =
Packit 961e70
				(psm2_am_handler_fn_t)hentry->hfn;
Packit 961e70
		ret = hfn1(&token, args, nargs, payload, paylen);
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
static int
Packit 961e70
ips_proto_am_handle_outoforder_queue()
Packit 961e70
{
Packit 961e70
	struct ips_am_message *msg, *prev;
Packit 961e70
	int ret = IPS_RECVHDRQ_CONTINUE;
Packit 961e70
Packit 961e70
	prev = &ips_am_outoforder_q.head;
Packit 961e70
	msg = ips_am_outoforder_q.head.next;
Packit 961e70
Packit 961e70
	while (msg != NULL) {
Packit 961e70
		struct ips_epaddr *ipsaddr = msg->ipsaddr;
Packit 961e70
		if (ipsaddr->msgctl->am_recv_seqnum != msg->seqnum) {
Packit 961e70
			prev = msg;
Packit 961e70
			msg = msg->next;
Packit 961e70
			continue;
Packit 961e70
		}
Packit 961e70
Packit 961e70
		ipsaddr->msgctl->am_recv_seqnum++;
Packit 961e70
Packit 961e70
		if (ips_am_run_handler(&msg->p_hdr,
Packit 961e70
					ipsaddr, msg->proto_am,
Packit 961e70
					msg->payload, msg->paylen))
Packit 961e70
			ret = IPS_RECVHDRQ_BREAK;
Packit 961e70
Packit 961e70
		prev->next = msg->next;
Packit 961e70
		if (prev->next == NULL)
Packit 961e70
			ips_am_outoforder_q.tail = prev;
Packit 961e70
Packit 961e70
		psmi_mq_sysbuf_free(msg->proto_am->proto->mq, msg->payload);
Packit 961e70
		psmi_mpool_put(msg);
Packit 961e70
Packit 961e70
		msg = prev->next;
Packit 961e70
	}
Packit 961e70
Packit 961e70
	return ret;
Packit 961e70
}
Packit 961e70
Packit 961e70
static void
Packit 961e70
ips_proto_am_queue_msg(struct ips_am_message *msg)
Packit 961e70
{
Packit 961e70
	msg->next = NULL;
Packit 961e70
	ips_am_outoforder_q.tail->next = msg;
Packit 961e70
	ips_am_outoforder_q.tail = msg;
Packit 961e70
}
Packit 961e70
Packit 961e70
int ips_proto_am(struct ips_recvhdrq_event *rcv_ev)
Packit 961e70
{
Packit 961e70
	struct ips_message_header *p_hdr = rcv_ev->p_hdr;
Packit 961e70
	struct ips_epaddr *ipsaddr = rcv_ev->ipsaddr;
Packit 961e70
	struct ips_proto_am *proto_am = &rcv_ev->proto->proto_am;
Packit 961e70
	ips_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
Packit 961e70
	struct ips_flow *flow;
Packit 961e70
	struct ips_am_message *msg = NULL;
Packit 961e70
	int ret = IPS_RECVHDRQ_CONTINUE;
Packit 961e70
	enum ips_msg_order msgorder;
Packit 961e70
Packit 961e70
	psmi_assert(flowid < EP_FLOW_LAST);
Packit 961e70
	flow = &ipsaddr->flows[flowid];
Packit 961e70
	/*
Packit 961e70
	 * Based on AM request/reply traffic pattern, if we don't have a reply
Packit 961e70
	 * scb slot then we can't process the request packet, we just silently
Packit 961e70
	 * drop it.  Otherwise, it will be a deadlock.  note:
Packit 961e70
	 * ips_proto_is_expected_or_nak() can not be called in this case.
Packit 961e70
	 */
Packit 961e70
	if (_get_proto_hfi_opcode(p_hdr) == OPCODE_AM_REQUEST &&
Packit 961e70
	    !ips_scbctrl_avail(&proto_am->scbc_reply))
Packit 961e70
		return IPS_RECVHDRQ_CONTINUE;
Packit 961e70
Packit 961e70
	if (!ips_proto_is_expected_or_nak(rcv_ev))
Packit 961e70
		return IPS_RECVHDRQ_CONTINUE;
Packit 961e70
Packit 961e70
	uint16_t send_msgseq =
Packit 961e70
	    __le32_to_cpu(p_hdr->khdr.kdeth0) & HFI_KHDR_MSGSEQ_MASK;
Packit 961e70
	msgorder = ips_proto_check_msg_order(ipsaddr, flow, send_msgseq,
Packit 961e70
			&ipsaddr->msgctl->am_recv_seqnum);
Packit 961e70
Packit 961e70
	if (msgorder == IPS_MSG_ORDER_FUTURE)
Packit 961e70
		return IPS_RECVHDRQ_REVISIT;
Packit 961e70
	else if (msgorder == IPS_MSG_ORDER_FUTURE_RECV) {
Packit 961e70
		uint64_t *msg_payload;
Packit 961e70
		uint64_t *payload = ips_recvhdrq_event_payload(rcv_ev);
Packit 961e70
		uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev);
Packit 961e70
Packit 961e70
		psmi_assert(paylen == 0 || payload);
Packit 961e70
		msg = psmi_mpool_get(ips_am_msg_pool);
Packit 961e70
		if (unlikely(msg == NULL)) {
Packit 961e70
			/* Out of memory, drop the packet. */
Packit 961e70
			flow->recv_seq_num.psn_num =
Packit 961e70
				(flow->recv_seq_num.psn_num - 1) &
Packit 961e70
				rcv_ev->proto->psn_mask;
Packit 961e70
			return IPS_RECVHDRQ_BREAK;
Packit 961e70
		}
Packit 961e70
		msg_payload = psmi_mq_sysbuf_alloc(
Packit 961e70
				proto_am->proto->mq,
Packit 961e70
				ips_recvhdrq_event_paylen(rcv_ev));
Packit 961e70
		if (unlikely(msg_payload == NULL)) {
Packit 961e70
			/* Out of memory, drop the packet. */
Packit 961e70
			flow->recv_seq_num.psn_num =
Packit 961e70
				(flow->recv_seq_num.psn_num - 1) &
Packit 961e70
				rcv_ev->proto->psn_mask;
Packit 961e70
			psmi_mpool_put(msg);
Packit 961e70
			return IPS_RECVHDRQ_BREAK;
Packit 961e70
		}
Packit 961e70
Packit 961e70
		memcpy(&msg->p_hdr, p_hdr, sizeof(struct ips_message_header));
Packit 961e70
		memcpy(msg_payload, payload, paylen);
Packit 961e70
Packit 961e70
		msg->payload = msg_payload;
Packit 961e70
		msg->ipsaddr = ipsaddr;
Packit 961e70
		msg->proto_am = proto_am;
Packit 961e70
		msg->paylen = paylen;
Packit 961e70
		msg->seqnum =
Packit 961e70
			__le32_to_cpu(p_hdr->khdr.kdeth0) &
Packit 961e70
			HFI_KHDR_MSGSEQ_MASK;
Packit 961e70
Packit 961e70
		ips_proto_am_queue_msg(msg);
Packit 961e70
	} else if ((msgorder == IPS_MSG_ORDER_EXPECTED) ||
Packit 961e70
		   (msgorder == IPS_MSG_ORDER_EXPECTED_MATCH)) {
Packit 961e70
		uint64_t *payload = ips_recvhdrq_event_payload(rcv_ev);
Packit 961e70
		uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev);
Packit 961e70
Packit 961e70
		psmi_assert(paylen == 0 || payload);
Packit 961e70
		if (ips_am_run_handler(p_hdr, ipsaddr, proto_am,
Packit 961e70
					payload, paylen))
Packit 961e70
			ret = IPS_RECVHDRQ_BREAK;
Packit 961e70
Packit 961e70
		ips_proto_am_handle_outoforder_queue();
Packit 961e70
	}
Packit 961e70
Packit 961e70
	/* Look if the handler replied, if it didn't, ack the request */
Packit 961e70
	if ((__be32_to_cpu(p_hdr->bth[2]) & IPS_SEND_FLAG_ACKREQ) ||
Packit 961e70
	    (flow->flags & IPS_FLOW_FLAG_GEN_BECN))
Packit 961e70
		ips_proto_send_ack((struct ips_recvhdrq *)rcv_ev->recvq, flow);
Packit 961e70
Packit 961e70
	ips_proto_process_ack(rcv_ev);
Packit 961e70
	return ret;
Packit 961e70
}