|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
This file is provided under a dual BSD/GPLv2 license. When using or
|
|
Packit |
961e70 |
redistributing this file, you may do so under either license.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
GPL LICENSE SUMMARY
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
This program is free software; you can redistribute it and/or modify
|
|
Packit |
961e70 |
it under the terms of version 2 of the GNU General Public License as
|
|
Packit |
961e70 |
published by the Free Software Foundation.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
This program is distributed in the hope that it will be useful, but
|
|
Packit |
961e70 |
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit |
961e70 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit |
961e70 |
General Public License for more details.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Contact Information:
|
|
Packit |
961e70 |
Intel Corporation, www.intel.com
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
BSD LICENSE
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Redistribution and use in source and binary forms, with or without
|
|
Packit |
961e70 |
modification, are permitted provided that the following conditions
|
|
Packit |
961e70 |
are met:
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
* Redistributions of source code must retain the above copyright
|
|
Packit |
961e70 |
notice, this list of conditions and the following disclaimer.
|
|
Packit |
961e70 |
* Redistributions in binary form must reproduce the above copyright
|
|
Packit |
961e70 |
notice, this list of conditions and the following disclaimer in
|
|
Packit |
961e70 |
the documentation and/or other materials provided with the
|
|
Packit |
961e70 |
distribution.
|
|
Packit |
961e70 |
* Neither the name of Intel Corporation nor the names of its
|
|
Packit |
961e70 |
contributors may be used to endorse or promote products derived
|
|
Packit |
961e70 |
from this software without specific prior written permission.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
Packit |
961e70 |
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
Packit |
961e70 |
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
Packit |
961e70 |
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
Packit |
961e70 |
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
Packit |
961e70 |
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit |
961e70 |
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
Packit |
961e70 |
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
Packit |
961e70 |
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
Packit |
961e70 |
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
Packit |
961e70 |
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Copyright (c) 2003-2014 Intel Corporation. All rights reserved. */
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
#include "psm_user.h"
|
|
Packit |
961e70 |
#include "psm2_hal.h"
|
|
Packit |
961e70 |
#include "psm2_am.h"
|
|
Packit |
961e70 |
#include "psm_am_internal.h"
|
|
Packit |
961e70 |
#include "psm_mq_internal.h"
|
|
Packit |
961e70 |
#include "ips_proto.h"
|
|
Packit |
961e70 |
#include "ips_expected_proto.h"
|
|
Packit |
961e70 |
#include "ips_proto_help.h"
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
struct ips_am_token {
|
|
Packit |
961e70 |
struct psmi_am_token tok;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* ptl-specific token stuff */
|
|
Packit |
961e70 |
struct ips_epaddr *epaddr_rail;
|
|
Packit |
961e70 |
struct ips_proto_am *proto_am;
|
|
Packit |
961e70 |
};
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
struct ips_am_message {
|
|
Packit |
961e70 |
struct ips_message_header p_hdr;
|
|
Packit |
961e70 |
struct ips_am_message *next;
|
|
Packit |
961e70 |
struct ips_epaddr *ipsaddr;
|
|
Packit |
961e70 |
struct ips_proto_am *proto_am;
|
|
Packit |
961e70 |
uint64_t *payload;
|
|
Packit |
961e70 |
uint32_t paylen;
|
|
Packit |
961e70 |
uint16_t seqnum;
|
|
Packit |
961e70 |
};
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* These variables are shared for all packet flows in a PSM process; they are
|
|
Packit |
961e70 |
* shared across multiple rails. There is no single AM object to hang these
|
|
Packit |
961e70 |
* off of, so they are declared here as globals. */
|
|
Packit |
961e70 |
static struct {
|
|
Packit |
961e70 |
struct ips_am_message head;
|
|
Packit |
961e70 |
struct ips_am_message *tail;
|
|
Packit |
961e70 |
} ips_am_outoforder_q;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
static mpool_t ips_am_msg_pool;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* This calculation ensures that the number of reply slots will always be at
|
|
Packit |
961e70 |
* least twice as large + 1 as the number of request slots. This is optimal: the
|
|
Packit |
961e70 |
* minimum amount required is actually only twice as many, but it is much
|
|
Packit |
961e70 |
* slower. */
|
|
Packit |
961e70 |
#define calc_optimal_num_reply_slots(nslots) (((nslots)*2 / 3) + 1)
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_error_t
|
|
Packit |
961e70 |
MOCKABLE(ips_proto_am_init)(struct ips_proto *proto,
|
|
Packit |
961e70 |
int num_send_slots,
|
|
Packit |
961e70 |
uint32_t imm_size,
|
|
Packit |
961e70 |
struct ips_proto_am *proto_am)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
psm2_error_t err = PSM2_OK;
|
|
Packit |
961e70 |
int send_buf_size = psmi_hal_get_pio_size(proto->ep->context.psm_hw_ctxt);
|
|
Packit |
961e70 |
int num_rep_slots = calc_optimal_num_reply_slots(num_send_slots);
|
|
Packit |
961e70 |
int num_req_slots = num_send_slots - num_rep_slots;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
proto_am->proto = proto;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* In a node pair, the number of reply send buffers on at least one of
|
|
Packit |
961e70 |
* the nodes must be at least double the number (optimal: double + 1) of
|
|
Packit |
961e70 |
* send descriptors on the other node. While this constraint applies
|
|
Packit |
961e70 |
* only to the reply send buffers, allowing the caller to tune only the
|
|
Packit |
961e70 |
* number of request send buffers would be awkward, as they have no
|
|
Packit |
961e70 |
* knowledge of the subdivision of the memory into separate mempools for
|
|
Packit |
961e70 |
* requests and replies. It's an internal concern at this point. */
|
|
Packit |
961e70 |
if ((err = ips_scbctrl_init(&proto->ep->context,
|
|
Packit |
961e70 |
num_req_slots,
|
|
Packit |
961e70 |
num_req_slots,
|
|
Packit |
961e70 |
imm_size,
|
|
Packit |
961e70 |
send_buf_size,
|
|
Packit |
961e70 |
NULL,
|
|
Packit |
961e70 |
NULL,
|
|
Packit |
961e70 |
&proto_am->scbc_request)))
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if ((err = ips_scbctrl_init(&proto->ep->context,
|
|
Packit |
961e70 |
num_rep_slots,
|
|
Packit |
961e70 |
num_rep_slots,
|
|
Packit |
961e70 |
imm_size,
|
|
Packit |
961e70 |
send_buf_size,
|
|
Packit |
961e70 |
NULL,
|
|
Packit |
961e70 |
NULL,
|
|
Packit |
961e70 |
&proto_am->scbc_reply)))
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (ips_am_msg_pool == NULL) {
|
|
Packit |
961e70 |
union psmi_envvar_val max_msgs;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
ips_am_outoforder_q.head.next = NULL;
|
|
Packit |
961e70 |
ips_am_outoforder_q.tail = &ips_am_outoforder_q.head;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_getenv("PSM2_AM_MAX_OOO_MSGS",
|
|
Packit |
961e70 |
"Maximum number of OOO Active Messages to queue before dropping.",
|
|
Packit |
961e70 |
PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT,
|
|
Packit |
961e70 |
(union psmi_envvar_val)1024, &max_msgs);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
ips_am_msg_pool = psmi_mpool_create(
|
|
Packit |
961e70 |
sizeof(struct ips_am_message),
|
|
Packit |
961e70 |
32, max_msgs.e_uint, 0, UNDEFINED, NULL, NULL);
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
fail:
|
|
Packit |
961e70 |
return err;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
MOCK_DEF_EPILOGUE(ips_proto_am_init);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_error_t ips_proto_am_fini(struct ips_proto_am *proto_am)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
ips_scbctrl_fini(&proto_am->scbc_request);
|
|
Packit |
961e70 |
ips_scbctrl_fini(&proto_am->scbc_reply);
|
|
Packit |
961e70 |
if (ips_am_msg_pool != NULL) {
|
|
Packit |
961e70 |
psmi_mpool_destroy(ips_am_msg_pool);
|
|
Packit |
961e70 |
ips_am_msg_pool = NULL;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
return PSM2_OK;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Fill in AM capabilities parameters */
|
|
Packit |
961e70 |
psm2_error_t
|
|
Packit |
961e70 |
ips_am_get_parameters(psm2_ep_t ep, struct psm2_am_parameters *parameters)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
int max_nargs = min(1 << IPS_AM_HDR_NARGS_BITS, PSMI_AM_MAX_ARGS);
|
|
Packit |
961e70 |
int max_payload =
|
|
Packit |
961e70 |
psmi_hal_get_pio_size(ep->context.psm_hw_ctxt) -
|
|
Packit |
961e70 |
((max_nargs - IPS_AM_HDR_NARGS) * sizeof(psm2_amarg_t));
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (parameters == NULL) {
|
|
Packit |
961e70 |
return PSM2_PARAM_ERR;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
parameters->max_handlers = 1 << IPS_AM_HDR_HIDX_BITS;
|
|
Packit |
961e70 |
parameters->max_nargs = max_nargs;
|
|
Packit |
961e70 |
parameters->max_request_short = max_payload;
|
|
Packit |
961e70 |
parameters->max_reply_short = max_payload;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
return PSM2_OK;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
static
|
|
Packit |
961e70 |
psm2_error_t
|
|
Packit |
961e70 |
am_short_reqrep(ips_scb_t *scb, struct ips_epaddr *ipsaddr,
|
|
Packit |
961e70 |
psm2_amarg_t *args, int nargs, uint8_t opcode,
|
|
Packit |
961e70 |
void *src, size_t len, int flags, int pad_bytes)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
int i, hdr_qwords = IPS_AM_HDR_NARGS;
|
|
Packit |
961e70 |
struct ips_proto *proto = ((psm2_epaddr_t)ipsaddr)->proto;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(proto->msgflowid < EP_FLOW_LAST);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
struct ips_flow *flow = &ipsaddr->flows[proto->msgflowid];
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* There are a limited number of bits for nargs in the header, making
|
|
Packit |
961e70 |
overflow very easy. Make sure the values match. */
|
|
Packit |
961e70 |
psmi_assert(nargs == scb->ips_lrh.amhdr_nargs);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
_HFI_VDBG("%s src=%p len=%d, nargs=%d\n",
|
|
Packit |
961e70 |
((opcode == OPCODE_AM_REQUEST) ||
|
|
Packit |
961e70 |
(opcode == OPCODE_AM_REQUEST_NOREPLY)) ? "req" : "rep",
|
|
Packit |
961e70 |
src, (int)len, nargs);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (nargs == 1) { /* fastpath */
|
|
Packit |
961e70 |
scb->ips_lrh.data[0].u64w0 = args[0].u64w0;
|
|
Packit |
961e70 |
hdr_qwords--;
|
|
Packit |
961e70 |
} else if (nargs > 1) {
|
|
Packit |
961e70 |
/* Easily unrollable but leave as is in case we can increase
|
|
Packit |
961e70 |
* qwords on the chip in the near future */
|
|
Packit |
961e70 |
for (i = 0; i < IPS_AM_HDR_NARGS; i++, hdr_qwords--)
|
|
Packit |
961e70 |
scb->ips_lrh.data[i].u64w0 = args[i].u64w0;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (nargs > IPS_AM_HDR_NARGS) {
|
|
Packit |
961e70 |
/* Slow case -- we don't have iovec and not enough
|
|
Packit |
961e70 |
* space in the message header, so we have to copy the
|
|
Packit |
961e70 |
* user's arguments even if the payload is marked ASYNC
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
uintptr_t bufp = (uintptr_t) ips_scb_buffer(scb);
|
|
Packit |
961e70 |
size_t arg_payload_len =
|
|
Packit |
961e70 |
sizeof(psm2_amarg_t) * (nargs - IPS_AM_HDR_NARGS);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_mq_mtucpy((void *)bufp,
|
|
Packit |
961e70 |
&args[IPS_AM_HDR_NARGS],
|
|
Packit |
961e70 |
arg_payload_len);
|
|
Packit |
961e70 |
bufp += arg_payload_len;
|
|
Packit |
961e70 |
scb->payload_size = arg_payload_len;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (src != NULL && len > 0) {
|
|
Packit |
961e70 |
psmi_mq_mtucpy((void *)bufp, src, len);
|
|
Packit |
961e70 |
scb->payload_size += len;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(pad_bytes < (1 << IPS_AM_HDR_LEN_BITS));
|
|
Packit |
961e70 |
scb->payload_size += pad_bytes;
|
|
Packit |
961e70 |
scb->ips_lrh.amhdr_len = pad_bytes;
|
|
Packit |
961e70 |
goto send_scb;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (len == 0) {
|
|
Packit |
961e70 |
scb->payload_size = 0;
|
|
Packit |
961e70 |
scb->ips_lrh.amhdr_len = 0;
|
|
Packit |
961e70 |
} else if (len <= (hdr_qwords << 3)) {
|
|
Packit |
961e70 |
/* Inline the payload into the header. */
|
|
Packit |
961e70 |
/* This path CANNOT handle length = 0 due to limited space
|
|
Packit |
961e70 |
in the header. If IPS_SEND_FLAG_AMISTINY is set, an
|
|
Packit |
961e70 |
amhdr_len value of 0 means a full payload, i.e.
|
|
Packit |
961e70 |
1 << IPS_AM_HDR_LEN_BITS bytes of packed payload. */
|
|
Packit |
961e70 |
psmi_assert(len > 0);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_mq_mtucpy(&scb->ips_lrh.
|
|
Packit |
961e70 |
data[IPS_AM_HDR_NARGS - hdr_qwords], src, len);
|
|
Packit |
961e70 |
scb->payload_size = 0;
|
|
Packit |
961e70 |
psmi_assert(len <= (1 << IPS_AM_HDR_LEN_BITS));
|
|
Packit |
961e70 |
scb->ips_lrh.amhdr_len = len & ((1 << IPS_AM_HDR_LEN_BITS) - 1);
|
|
Packit |
961e70 |
scb->scb_flags |= IPS_SEND_FLAG_AMISTINY;
|
|
Packit |
961e70 |
} else { /* Whatever's left requires a separate payload */
|
|
Packit |
961e70 |
if (ips_scb_buffer(scb) == NULL) /* Just attach the buffer */
|
|
Packit |
961e70 |
ips_scb_buffer(scb) = src;
|
|
Packit |
961e70 |
else /* May need to re-xmit user data, keep it around */
|
|
Packit |
961e70 |
psmi_mq_mtucpy(ips_scb_buffer(scb), src, len);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(pad_bytes < (1 << IPS_AM_HDR_LEN_BITS));
|
|
Packit |
961e70 |
scb->payload_size = len + pad_bytes;
|
|
Packit |
961e70 |
scb->ips_lrh.amhdr_len = pad_bytes;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
send_scb:
|
|
Packit |
961e70 |
ips_scb_opcode(scb) = opcode;
|
|
Packit |
961e70 |
scb->ips_lrh.khdr.kdeth0 = ipsaddr->msgctl->am_send_seqnum++;
|
|
Packit |
961e70 |
ips_proto_flow_enqueue(flow, scb);
|
|
Packit |
961e70 |
flow->flush(flow, NULL);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
return PSM2_OK;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
static inline int
|
|
Packit |
961e70 |
calculate_pad_bytes(size_t len)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
/* Align to dword (4 bytes) */
|
|
Packit |
961e70 |
size_t dword_aligned_len = (len + 3) & ~3;
|
|
Packit |
961e70 |
return dword_aligned_len - len;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
static inline
|
|
Packit |
961e70 |
void
|
|
Packit |
961e70 |
ips_am_scb_init(ips_scb_t *scb, uint8_t handler, int nargs,
|
|
Packit |
961e70 |
int pad_bytes,
|
|
Packit |
961e70 |
psm2_am_completion_fn_t completion_fn, void *completion_ctxt)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
psmi_assert(pad_bytes < (1 << IPS_AM_HDR_LEN_BITS));
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
scb->completion_am = completion_fn;
|
|
Packit |
961e70 |
scb->cb_param = completion_ctxt;
|
|
Packit |
961e70 |
scb->ips_lrh.amhdr_hidx = handler;
|
|
Packit |
961e70 |
scb->ips_lrh.amhdr_len = pad_bytes;
|
|
Packit |
961e70 |
scb->ips_lrh.amhdr_nargs = nargs;
|
|
Packit |
961e70 |
scb->ips_lrh.flags = 0;
|
|
Packit |
961e70 |
if (completion_fn)
|
|
Packit |
961e70 |
scb->scb_flags |= IPS_SEND_FLAG_ACKREQ;
|
|
Packit |
961e70 |
return;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_error_t
|
|
Packit |
961e70 |
ips_am_short_request(psm2_epaddr_t epaddr,
|
|
Packit |
961e70 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit |
961e70 |
void *src, size_t len, int flags,
|
|
Packit |
961e70 |
psm2_am_completion_fn_t completion_fn,
|
|
Packit |
961e70 |
void *completion_ctxt)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
struct ips_proto_am *proto_am = &epaddr->proto->proto_am;
|
|
Packit |
961e70 |
psm2_error_t err;
|
|
Packit |
961e70 |
ips_scb_t *scb;
|
|
Packit |
961e70 |
ips_epaddr_t *ipsaddr;
|
|
Packit |
961e70 |
int pad_bytes = calculate_pad_bytes(len);
|
|
Packit |
961e70 |
int payload_sz = (nargs << 3);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if_pt(!(flags & PSM2_AM_FLAG_ASYNC))
|
|
Packit |
961e70 |
payload_sz += len;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (payload_sz > (IPS_AM_HDR_NARGS << 3)) {
|
|
Packit |
961e70 |
/* Payload can't fit in header, allocate buffer to carry data */
|
|
Packit |
961e70 |
int arg_sz = (nargs > IPS_AM_HDR_NARGS) ?
|
|
Packit |
961e70 |
((nargs - IPS_AM_HDR_NARGS) << 3) : 0;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* len + pad_bytes + overflow_args */
|
|
Packit |
961e70 |
PSMI_BLOCKUNTIL(epaddr->ptlctl->ep,
|
|
Packit |
961e70 |
err,
|
|
Packit |
961e70 |
((scb = ips_scbctrl_alloc(
|
|
Packit |
961e70 |
&proto_am->scbc_request,
|
|
Packit |
961e70 |
1,
|
|
Packit |
961e70 |
len + pad_bytes + arg_sz,
|
|
Packit |
961e70 |
IPS_SCB_FLAG_ADD_BUFFER)) != NULL));
|
|
Packit |
961e70 |
} else {
|
|
Packit |
961e70 |
PSMI_BLOCKUNTIL(epaddr->ptlctl->ep,
|
|
Packit |
961e70 |
err,
|
|
Packit |
961e70 |
((scb = ips_scbctrl_alloc_tiny(
|
|
Packit |
961e70 |
&proto_am->scbc_request)) != NULL));
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert_always(scb != NULL);
|
|
Packit |
961e70 |
ips_am_scb_init(scb, handler, nargs, pad_bytes,
|
|
Packit |
961e70 |
completion_fn, completion_ctxt);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Select the next ipsaddr for multi-rail */
|
|
Packit |
961e70 |
ipsaddr = ((ips_epaddr_t *)epaddr)->msgctl->ipsaddr_next;
|
|
Packit |
961e70 |
ipsaddr->msgctl->ipsaddr_next = ipsaddr->next;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
return am_short_reqrep(scb, ipsaddr, args,
|
|
Packit |
961e70 |
nargs,
|
|
Packit |
961e70 |
(flags & PSM2_AM_FLAG_NOREPLY) ?
|
|
Packit |
961e70 |
OPCODE_AM_REQUEST_NOREPLY : OPCODE_AM_REQUEST,
|
|
Packit |
961e70 |
src, len, flags, pad_bytes);
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_error_t
|
|
Packit |
961e70 |
ips_am_short_reply(psm2_am_token_t tok,
|
|
Packit |
961e70 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit |
961e70 |
void *src, size_t len, int flags,
|
|
Packit |
961e70 |
psm2_am_completion_fn_t completion_fn, void *completion_ctxt)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
struct ips_am_token *token = (struct ips_am_token *)tok;
|
|
Packit |
961e70 |
struct ips_proto_am *proto_am = token->proto_am;
|
|
Packit |
961e70 |
struct ips_epaddr *ipsaddr = token->epaddr_rail;
|
|
Packit |
961e70 |
int pad_bytes = calculate_pad_bytes(len);
|
|
Packit |
961e70 |
int scb_flags = 0;
|
|
Packit |
961e70 |
ips_scb_t *scb;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (!token->tok.can_reply) {
|
|
Packit |
961e70 |
_HFI_ERROR("Invalid AM reply for request!");
|
|
Packit |
961e70 |
return PSM2_AM_INVALID_REPLY;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(ips_scbctrl_avail(&proto_am->scbc_reply));
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if ((nargs << 3) + len <= (IPS_AM_HDR_NARGS << 3)) {
|
|
Packit |
961e70 |
scb = ips_scbctrl_alloc_tiny(&proto_am->scbc_reply);
|
|
Packit |
961e70 |
} else {
|
|
Packit |
961e70 |
int payload_sz = (nargs << 3);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
payload_sz += (flags & PSM2_AM_FLAG_ASYNC) ?
|
|
Packit |
961e70 |
0 : (len + pad_bytes);
|
|
Packit |
961e70 |
scb_flags |= (payload_sz > (IPS_AM_HDR_NARGS << 3)) ?
|
|
Packit |
961e70 |
IPS_SCB_FLAG_ADD_BUFFER : 0;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
scb =
|
|
Packit |
961e70 |
ips_scbctrl_alloc(&proto_am->scbc_reply, 1, payload_sz,
|
|
Packit |
961e70 |
scb_flags);
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert_always(scb != NULL);
|
|
Packit |
961e70 |
ips_am_scb_init(scb, handler, nargs, pad_bytes,
|
|
Packit |
961e70 |
completion_fn, completion_ctxt);
|
|
Packit |
961e70 |
am_short_reqrep(scb, ipsaddr, args, nargs, OPCODE_AM_REPLY,
|
|
Packit |
961e70 |
src, len, flags, pad_bytes);
|
|
Packit |
961e70 |
return PSM2_OK;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Prepares and runs a handler from a receive event. */
|
|
Packit |
961e70 |
static int
|
|
Packit |
961e70 |
ips_am_run_handler(const struct ips_message_header *p_hdr,
|
|
Packit |
961e70 |
struct ips_epaddr *ipsaddr, struct ips_proto_am *proto_am,
|
|
Packit |
961e70 |
uint64_t *payload,
|
|
Packit |
961e70 |
uint32_t paylen)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
struct ips_am_token token;
|
|
Packit |
961e70 |
int nargs = p_hdr->amhdr_nargs;
|
|
Packit |
961e70 |
int ret;
|
|
Packit |
961e70 |
struct psm2_ep_am_handle_entry *hentry;
|
|
Packit |
961e70 |
psm2_amarg_t *args = (psm2_amarg_t *)p_hdr->data;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
token.tok.flags = p_hdr->flags;
|
|
Packit |
961e70 |
token.tok.epaddr_incoming = (psm2_epaddr_t)&ipsaddr->msgctl->master_epaddr;
|
|
Packit |
961e70 |
token.tok.can_reply =
|
|
Packit |
961e70 |
(_get_proto_hfi_opcode(p_hdr) == OPCODE_AM_REQUEST);
|
|
Packit |
961e70 |
token.epaddr_rail = ipsaddr;
|
|
Packit |
961e70 |
token.proto_am = proto_am;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (token.tok.flags & IPS_SEND_FLAG_AMISTINY) {
|
|
Packit |
961e70 |
/* Payload is packed into header after args */
|
|
Packit |
961e70 |
payload = (uint64_t *)&p_hdr->data[nargs].u64;
|
|
Packit |
961e70 |
paylen = p_hdr->amhdr_len;
|
|
Packit |
961e70 |
/* Interpret amhdr_len == 0 as 16 bytes of payload */
|
|
Packit |
961e70 |
if (paylen == 0)
|
|
Packit |
961e70 |
paylen = 1 << IPS_AM_HDR_LEN_BITS;
|
|
Packit |
961e70 |
} else {
|
|
Packit |
961e70 |
if (nargs > IPS_AM_HDR_NARGS) {
|
|
Packit |
961e70 |
/* Args are split across header and payload */
|
|
Packit |
961e70 |
int payload_args_len =
|
|
Packit |
961e70 |
(nargs - IPS_AM_HDR_NARGS) *
|
|
Packit |
961e70 |
sizeof(psm2_amarg_t);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
args = alloca(PSMI_AM_MAX_ARGS * sizeof(psm2_amarg_t));
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
args[0].u64 = p_hdr->data[0].u64;
|
|
Packit |
961e70 |
args[1].u64 = p_hdr->data[1].u64;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
memcpy(&args[2], payload, payload_args_len);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
payload += nargs - IPS_AM_HDR_NARGS;
|
|
Packit |
961e70 |
paylen -= payload_args_len;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Subtract off padding bytes (dword padding) for non-TINY. */
|
|
Packit |
961e70 |
paylen -= p_hdr->amhdr_len;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
hentry = psm_am_get_handler_function(proto_am->proto->ep,
|
|
Packit |
961e70 |
p_hdr->amhdr_hidx);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Note a guard here for hentry != NULL is not needed because at
|
|
Packit |
961e70 |
* initialization, a psmi_assert_always() assure the entry will be
|
|
Packit |
961e70 |
* non-NULL. */
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (likely(hentry->version == PSM2_AM_HANDLER_V2)) {
|
|
Packit |
961e70 |
psm2_am_handler_2_fn_t hfn2 =
|
|
Packit |
961e70 |
(psm2_am_handler_2_fn_t)hentry->hfn;
|
|
Packit |
961e70 |
ret = hfn2(&token, args, nargs, payload, paylen, hentry->hctx);
|
|
Packit |
961e70 |
} else {
|
|
Packit |
961e70 |
psm2_am_handler_fn_t hfn1 =
|
|
Packit |
961e70 |
(psm2_am_handler_fn_t)hentry->hfn;
|
|
Packit |
961e70 |
ret = hfn1(&token, args, nargs, payload, paylen);
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
return ret;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
static int
|
|
Packit |
961e70 |
ips_proto_am_handle_outoforder_queue()
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
struct ips_am_message *msg, *prev;
|
|
Packit |
961e70 |
int ret = IPS_RECVHDRQ_CONTINUE;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
prev = &ips_am_outoforder_q.head;
|
|
Packit |
961e70 |
msg = ips_am_outoforder_q.head.next;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
while (msg != NULL) {
|
|
Packit |
961e70 |
struct ips_epaddr *ipsaddr = msg->ipsaddr;
|
|
Packit |
961e70 |
if (ipsaddr->msgctl->am_recv_seqnum != msg->seqnum) {
|
|
Packit |
961e70 |
prev = msg;
|
|
Packit |
961e70 |
msg = msg->next;
|
|
Packit |
961e70 |
continue;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
ipsaddr->msgctl->am_recv_seqnum++;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (ips_am_run_handler(&msg->p_hdr,
|
|
Packit |
961e70 |
ipsaddr, msg->proto_am,
|
|
Packit |
961e70 |
msg->payload, msg->paylen))
|
|
Packit |
961e70 |
ret = IPS_RECVHDRQ_BREAK;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
prev->next = msg->next;
|
|
Packit |
961e70 |
if (prev->next == NULL)
|
|
Packit |
961e70 |
ips_am_outoforder_q.tail = prev;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_mq_sysbuf_free(msg->proto_am->proto->mq, msg->payload);
|
|
Packit |
961e70 |
psmi_mpool_put(msg);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
msg = prev->next;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
return ret;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
static void
|
|
Packit |
961e70 |
ips_proto_am_queue_msg(struct ips_am_message *msg)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
msg->next = NULL;
|
|
Packit |
961e70 |
ips_am_outoforder_q.tail->next = msg;
|
|
Packit |
961e70 |
ips_am_outoforder_q.tail = msg;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
int ips_proto_am(struct ips_recvhdrq_event *rcv_ev)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
struct ips_message_header *p_hdr = rcv_ev->p_hdr;
|
|
Packit |
961e70 |
struct ips_epaddr *ipsaddr = rcv_ev->ipsaddr;
|
|
Packit |
961e70 |
struct ips_proto_am *proto_am = &rcv_ev->proto->proto_am;
|
|
Packit |
961e70 |
ips_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
|
|
Packit |
961e70 |
struct ips_flow *flow;
|
|
Packit |
961e70 |
struct ips_am_message *msg = NULL;
|
|
Packit |
961e70 |
int ret = IPS_RECVHDRQ_CONTINUE;
|
|
Packit |
961e70 |
enum ips_msg_order msgorder;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(flowid < EP_FLOW_LAST);
|
|
Packit |
961e70 |
flow = &ipsaddr->flows[flowid];
|
|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
* Based on AM request/reply traffic pattern, if we don't have a reply
|
|
Packit |
961e70 |
* scb slot then we can't process the request packet, we just silently
|
|
Packit |
961e70 |
* drop it. Otherwise, it will be a deadlock. note:
|
|
Packit |
961e70 |
* ips_proto_is_expected_or_nak() can not be called in this case.
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
if (_get_proto_hfi_opcode(p_hdr) == OPCODE_AM_REQUEST &&
|
|
Packit |
961e70 |
!ips_scbctrl_avail(&proto_am->scbc_reply))
|
|
Packit |
961e70 |
return IPS_RECVHDRQ_CONTINUE;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (!ips_proto_is_expected_or_nak(rcv_ev))
|
|
Packit |
961e70 |
return IPS_RECVHDRQ_CONTINUE;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
uint16_t send_msgseq =
|
|
Packit |
961e70 |
__le32_to_cpu(p_hdr->khdr.kdeth0) & HFI_KHDR_MSGSEQ_MASK;
|
|
Packit |
961e70 |
msgorder = ips_proto_check_msg_order(ipsaddr, flow, send_msgseq,
|
|
Packit |
961e70 |
&ipsaddr->msgctl->am_recv_seqnum);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (msgorder == IPS_MSG_ORDER_FUTURE)
|
|
Packit |
961e70 |
return IPS_RECVHDRQ_REVISIT;
|
|
Packit |
961e70 |
else if (msgorder == IPS_MSG_ORDER_FUTURE_RECV) {
|
|
Packit |
961e70 |
uint64_t *msg_payload;
|
|
Packit |
961e70 |
uint64_t *payload = ips_recvhdrq_event_payload(rcv_ev);
|
|
Packit |
961e70 |
uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(paylen == 0 || payload);
|
|
Packit |
961e70 |
msg = psmi_mpool_get(ips_am_msg_pool);
|
|
Packit |
961e70 |
if (unlikely(msg == NULL)) {
|
|
Packit |
961e70 |
/* Out of memory, drop the packet. */
|
|
Packit |
961e70 |
flow->recv_seq_num.psn_num =
|
|
Packit |
961e70 |
(flow->recv_seq_num.psn_num - 1) &
|
|
Packit |
961e70 |
rcv_ev->proto->psn_mask;
|
|
Packit |
961e70 |
return IPS_RECVHDRQ_BREAK;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
msg_payload = psmi_mq_sysbuf_alloc(
|
|
Packit |
961e70 |
proto_am->proto->mq,
|
|
Packit |
961e70 |
ips_recvhdrq_event_paylen(rcv_ev));
|
|
Packit |
961e70 |
if (unlikely(msg_payload == NULL)) {
|
|
Packit |
961e70 |
/* Out of memory, drop the packet. */
|
|
Packit |
961e70 |
flow->recv_seq_num.psn_num =
|
|
Packit |
961e70 |
(flow->recv_seq_num.psn_num - 1) &
|
|
Packit |
961e70 |
rcv_ev->proto->psn_mask;
|
|
Packit |
961e70 |
psmi_mpool_put(msg);
|
|
Packit |
961e70 |
return IPS_RECVHDRQ_BREAK;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
memcpy(&msg->p_hdr, p_hdr, sizeof(struct ips_message_header));
|
|
Packit |
961e70 |
memcpy(msg_payload, payload, paylen);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
msg->payload = msg_payload;
|
|
Packit |
961e70 |
msg->ipsaddr = ipsaddr;
|
|
Packit |
961e70 |
msg->proto_am = proto_am;
|
|
Packit |
961e70 |
msg->paylen = paylen;
|
|
Packit |
961e70 |
msg->seqnum =
|
|
Packit |
961e70 |
__le32_to_cpu(p_hdr->khdr.kdeth0) &
|
|
Packit |
961e70 |
HFI_KHDR_MSGSEQ_MASK;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
ips_proto_am_queue_msg(msg);
|
|
Packit |
961e70 |
} else if ((msgorder == IPS_MSG_ORDER_EXPECTED) ||
|
|
Packit |
961e70 |
(msgorder == IPS_MSG_ORDER_EXPECTED_MATCH)) {
|
|
Packit |
961e70 |
uint64_t *payload = ips_recvhdrq_event_payload(rcv_ev);
|
|
Packit |
961e70 |
uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(paylen == 0 || payload);
|
|
Packit |
961e70 |
if (ips_am_run_handler(p_hdr, ipsaddr, proto_am,
|
|
Packit |
961e70 |
payload, paylen))
|
|
Packit |
961e70 |
ret = IPS_RECVHDRQ_BREAK;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
ips_proto_am_handle_outoforder_queue();
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Look if the handler replied, if it didn't, ack the request */
|
|
Packit |
961e70 |
if ((__be32_to_cpu(p_hdr->bth[2]) & IPS_SEND_FLAG_ACKREQ) ||
|
|
Packit |
961e70 |
(flow->flags & IPS_FLOW_FLAG_GEN_BECN))
|
|
Packit |
961e70 |
ips_proto_send_ack((struct ips_recvhdrq *)rcv_ev->recvq, flow);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
ips_proto_process_ack(rcv_ev);
|
|
Packit |
961e70 |
return ret;
|
|
Packit |
961e70 |
}
|