|
Packit Service |
aa3af4 |
/*
|
|
Packit Service |
aa3af4 |
* Copyright (c) 2001-2020 Mellanox Technologies, Ltd. All rights reserved.
|
|
Packit Service |
aa3af4 |
*
|
|
Packit Service |
aa3af4 |
* This software is available to you under a choice of one of two
|
|
Packit Service |
aa3af4 |
* licenses. You may choose to be licensed under the terms of the GNU
|
|
Packit Service |
aa3af4 |
* General Public License (GPL) Version 2, available from the file
|
|
Packit Service |
aa3af4 |
* COPYING in the main directory of this source tree, or the
|
|
Packit Service |
aa3af4 |
* BSD license below:
|
|
Packit Service |
aa3af4 |
*
|
|
Packit Service |
aa3af4 |
* Redistribution and use in source and binary forms, with or
|
|
Packit Service |
aa3af4 |
* without modification, are permitted provided that the following
|
|
Packit Service |
aa3af4 |
* conditions are met:
|
|
Packit Service |
aa3af4 |
*
|
|
Packit Service |
aa3af4 |
* - Redistributions of source code must retain the above
|
|
Packit Service |
aa3af4 |
* copyright notice, this list of conditions and the following
|
|
Packit Service |
aa3af4 |
* disclaimer.
|
|
Packit Service |
aa3af4 |
*
|
|
Packit Service |
aa3af4 |
* - Redistributions in binary form must reproduce the above
|
|
Packit Service |
aa3af4 |
* copyright notice, this list of conditions and the following
|
|
Packit Service |
aa3af4 |
* disclaimer in the documentation and/or other materials
|
|
Packit Service |
aa3af4 |
* provided with the distribution.
|
|
Packit Service |
aa3af4 |
*
|
|
Packit Service |
aa3af4 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
Packit Service |
aa3af4 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
Packit Service |
aa3af4 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
Packit Service |
aa3af4 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
|
Packit Service |
aa3af4 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
Packit Service |
aa3af4 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
Packit Service |
aa3af4 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
Packit Service |
aa3af4 |
* SOFTWARE.
|
|
Packit Service |
aa3af4 |
*/
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
#include <dev/ring_eth_cb.h>
|
|
Packit Service |
aa3af4 |
#include <dev/qp_mgr_mp.h>
|
|
Packit Service |
aa3af4 |
#include <dev/cq_mgr_mp.h>
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
#undef MODULE_NAME
|
|
Packit Service |
aa3af4 |
#define MODULE_NAME "ring_eth_cb"
|
|
Packit Service |
aa3af4 |
#undef MODULE_HDR
|
|
Packit Service |
aa3af4 |
#define MODULE_HDR MODULE_NAME "%d:%s() "
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
#ifdef HAVE_MP_RQ
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
#define DUMP_LKEY (0x700)
|
|
Packit Service |
aa3af4 |
#define VMA_MP_MIN_LOG_STRIDES (10)
|
|
Packit Service |
aa3af4 |
#define MAX_MP_WQES (20) // limit max used memory
|
|
Packit Service |
aa3af4 |
#define MIN_MP_WQES (4)
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
ring_eth_cb::ring_eth_cb(int if_index, vma_cyclic_buffer_ring_attr *cb_ring,
|
|
Packit Service |
aa3af4 |
iovec *mem_desc, ring *parent):
|
|
Packit Service |
aa3af4 |
ring_eth(if_index, parent, RING_ETH_CB, false)
|
|
Packit Service |
aa3af4 |
,m_curr_wqe_used_strides(0)
|
|
Packit Service |
aa3af4 |
,m_curr_packets(0)
|
|
Packit Service |
aa3af4 |
,m_padd_mode_used_strides(0)
|
|
Packit Service |
aa3af4 |
,m_all_wqes_used_strides(0)
|
|
Packit Service |
aa3af4 |
,m_packet_receive_mode(cb_ring->packet_receive_mode)
|
|
Packit Service |
aa3af4 |
,m_curr_wq(0)
|
|
Packit Service |
aa3af4 |
,m_curr_payload_addr(NULL)
|
|
Packit Service |
aa3af4 |
,m_curr_hdr_ptr(NULL)
|
|
Packit Service |
aa3af4 |
,m_res_domain(NULL)
|
|
Packit Service |
aa3af4 |
,m_external_mem(cb_ring->comp_mask & VMA_CB_EXTERNAL_MEM)
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
struct ibv_exp_res_domain_init_attr res_domain_attr;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
// check MP capabilities currently all caps are 0 due to a buf
|
|
Packit Service |
aa3af4 |
vma_ibv_device_attr* r_ibv_dev_attr = m_p_ib_ctx->get_ibv_device_attr();
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
memset(&m_umr_wr, 0, sizeof(m_umr_wr));
|
|
Packit Service |
aa3af4 |
memset(m_sge_ptrs, 0, sizeof(m_sge_ptrs));
|
|
Packit Service |
aa3af4 |
m_p_umr_mr = NULL;
|
|
Packit Service |
aa3af4 |
m_hdr_len = 0;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
if (!r_ibv_dev_attr->max_ctx_res_domain) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("device doesn't support resource domain");
|
|
Packit Service |
aa3af4 |
throw_vma_exception("device doesn't support resource domain");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
struct ibv_exp_mp_rq_caps *mp_rq_caps = &r_ibv_dev_attr->mp_rq_caps;
|
|
Packit Service |
aa3af4 |
if (!(mp_rq_caps->supported_qps & IBV_EXP_QPT_RAW_PACKET)) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("mp_rq is not supported");
|
|
Packit Service |
aa3af4 |
throw_vma_exception("device doesn't support RC QP");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
res_domain_attr.comp_mask = IBV_EXP_RES_DOMAIN_THREAD_MODEL |
|
|
Packit Service |
aa3af4 |
IBV_EXP_RES_DOMAIN_MSG_MODEL;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
// driver is in charge of locks
|
|
Packit Service |
aa3af4 |
res_domain_attr.thread_model = IBV_EXP_THREAD_SAFE;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
// currently have no affect
|
|
Packit Service |
aa3af4 |
res_domain_attr.msg_model = IBV_EXP_MSG_HIGH_BW;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
m_res_domain = ibv_exp_create_res_domain(m_p_ib_ctx->get_ibv_context(),
|
|
Packit Service |
aa3af4 |
&res_domain_attr);
|
|
Packit Service |
aa3af4 |
if (!m_res_domain) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("could not create resource domain");
|
|
Packit Service |
aa3af4 |
throw_vma_exception("failed creating resource domain");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
// stride size is headers + user payload aligned to power of 2
|
|
Packit Service |
aa3af4 |
uint16_t net_len = 0;
|
|
Packit Service |
aa3af4 |
if (m_partition) {
|
|
Packit Service |
aa3af4 |
net_len = ETH_VLAN_HDR_LEN + sizeof(struct iphdr) + sizeof(struct udphdr);
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
net_len = ETH_HDR_LEN + sizeof(struct iphdr) + sizeof(struct udphdr);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_single_stride_log_num_of_bytes = ilog_2(align32pow2(
|
|
Packit Service |
aa3af4 |
cb_ring->stride_bytes + cb_ring->hdr_bytes + net_len));
|
|
Packit Service |
aa3af4 |
if (m_single_stride_log_num_of_bytes < mp_rq_caps->min_single_stride_log_num_of_bytes) {
|
|
Packit Service |
aa3af4 |
m_single_stride_log_num_of_bytes = mp_rq_caps->min_single_stride_log_num_of_bytes;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (m_single_stride_log_num_of_bytes > mp_rq_caps->max_single_stride_log_num_of_bytes) {
|
|
Packit Service |
aa3af4 |
m_single_stride_log_num_of_bytes = mp_rq_caps->max_single_stride_log_num_of_bytes;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_stride_size = 1 << m_single_stride_log_num_of_bytes;
|
|
Packit Service |
aa3af4 |
uint32_t max_wqe_size = 1 << mp_rq_caps->max_single_wqe_log_num_of_strides;
|
|
Packit Service |
aa3af4 |
uint32_t user_req_wq = cb_ring->num / max_wqe_size;
|
|
Packit Service |
aa3af4 |
if (user_req_wq > MIN_MP_WQES) {
|
|
Packit Service |
aa3af4 |
m_wq_count = std::min<uint32_t>(user_req_wq, MAX_MP_WQES);
|
|
Packit Service |
aa3af4 |
m_single_wqe_log_num_of_strides = mp_rq_caps->max_single_wqe_log_num_of_strides;
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
m_wq_count = MIN_MP_WQES;
|
|
Packit Service |
aa3af4 |
m_single_wqe_log_num_of_strides = ilog_2(align32pow2(cb_ring->num) / m_wq_count);
|
|
Packit Service |
aa3af4 |
if (m_single_wqe_log_num_of_strides < VMA_MP_MIN_LOG_STRIDES) {
|
|
Packit Service |
aa3af4 |
m_single_wqe_log_num_of_strides = VMA_MP_MIN_LOG_STRIDES;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (m_single_wqe_log_num_of_strides > mp_rq_caps->max_single_wqe_log_num_of_strides) {
|
|
Packit Service |
aa3af4 |
m_single_wqe_log_num_of_strides = mp_rq_caps->max_single_wqe_log_num_of_strides;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_strides_num = 1 << m_single_wqe_log_num_of_strides;
|
|
Packit Service |
aa3af4 |
ring_logdbg("using strides_num %d stride size %d, wqe_count %d stride_bytes "
|
|
Packit Service |
aa3af4 |
"%d, hdr_bytes %d num %d rec mode %d", m_strides_num, m_stride_size,
|
|
Packit Service |
aa3af4 |
m_wq_count, cb_ring->stride_bytes, cb_ring->hdr_bytes, cb_ring->num,
|
|
Packit Service |
aa3af4 |
m_packet_receive_mode);
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
memset(&m_curr_hw_timestamp, 0, sizeof(m_curr_hw_timestamp));
|
|
Packit Service |
aa3af4 |
if (m_packet_receive_mode == PADDED_PACKET) {
|
|
Packit Service |
aa3af4 |
size_t buffer_size = m_stride_size * m_strides_num * m_wq_count;
|
|
Packit Service |
aa3af4 |
m_sge_ptrs[CB_UMR_PAYLOAD] = (uint64_t)allocate_memory(mem_desc, buffer_size);
|
|
Packit Service |
aa3af4 |
if (unlikely(!m_sge_ptrs[CB_UMR_PAYLOAD])) {
|
|
Packit Service |
aa3af4 |
throw_vma_exception("user provided to small memory");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_buff_data.addr = m_sge_ptrs[CB_UMR_PAYLOAD];
|
|
Packit Service |
aa3af4 |
m_buff_data.length = m_stride_size * m_strides_num;
|
|
Packit Service |
aa3af4 |
m_buff_data.lkey = get_mem_lkey(m_p_ib_ctx);
|
|
Packit Service |
aa3af4 |
m_packet_size = cb_ring->stride_bytes + net_len;
|
|
Packit Service |
aa3af4 |
m_payload_len = m_stride_size;
|
|
Packit Service |
aa3af4 |
if (unlikely(m_buff_data.lkey == (uint32_t)(-1))) {
|
|
Packit Service |
aa3af4 |
ring_logerr("got invalid lkey for memory %p size %zd",
|
|
Packit Service |
aa3af4 |
mem_desc->iov_base, mem_desc->iov_len);
|
|
Packit Service |
aa3af4 |
throw_vma_exception("failed retrieving lkey");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
ring_logdbg("using buffer size %zd", buffer_size);
|
|
Packit Service |
aa3af4 |
} else if (allocate_umr_mem(cb_ring, mem_desc, net_len)) {
|
|
Packit Service |
aa3af4 |
ring_logerr("failed creating UMR QP");
|
|
Packit Service |
aa3af4 |
throw_vma_exception("failed creating UMR QP");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
/* Complete resources initialization */
|
|
Packit Service |
aa3af4 |
ring_simple::create_resources();
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
void* ring_eth_cb::allocate_memory(iovec *mem_desc, size_t buffer_size)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
if (mem_desc && mem_desc->iov_len) {
|
|
Packit Service |
aa3af4 |
if (unlikely(mem_desc->iov_len < buffer_size)) {
|
|
Packit Service |
aa3af4 |
ring_logerr("user provided to small memory "
|
|
Packit Service |
aa3af4 |
"expected %zd but got %zd",
|
|
Packit Service |
aa3af4 |
buffer_size, mem_desc->iov_len);
|
|
Packit Service |
aa3af4 |
errno = EINVAL;
|
|
Packit Service |
aa3af4 |
return NULL;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
return m_alloc.alloc_and_reg_mr(mem_desc->iov_len, m_p_ib_ctx,
|
|
Packit Service |
aa3af4 |
mem_desc->iov_base);
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
return m_alloc.alloc_and_reg_mr(buffer_size, m_p_ib_ctx);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
qp_mgr* ring_eth_cb::create_qp_mgr(const ib_ctx_handler *ib_ctx,
|
|
Packit Service |
aa3af4 |
uint8_t port_num,
|
|
Packit Service |
aa3af4 |
struct ibv_comp_channel *p_rx_comp_event_channel)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
return new qp_mgr_mp(this, ib_ctx, port_num, p_rx_comp_event_channel,
|
|
Packit Service |
aa3af4 |
get_tx_num_wr(), m_partition, m_buff_data,
|
|
Packit Service |
aa3af4 |
m_external_mem);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
int ring_eth_cb::get_mem_info(ibv_sge &mem_info)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
if (!m_buff_data.addr) {
|
|
Packit Service |
aa3af4 |
ring_logwarn("no valid memory to return");
|
|
Packit Service |
aa3af4 |
return -1;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
mem_info.addr = m_buff_data.addr;
|
|
Packit Service |
aa3af4 |
mem_info.length = m_buff_data.length;
|
|
Packit Service |
aa3af4 |
mem_info.lkey = m_buff_data.lkey;
|
|
Packit Service |
aa3af4 |
ring_logdbg("returning ptr %p, legnth %zd, lkey %u", mem_info.addr,
|
|
Packit Service |
aa3af4 |
mem_info.length, mem_info.lkey);
|
|
Packit Service |
aa3af4 |
return 0;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
/**
|
|
Packit Service |
aa3af4 |
* allocate and set UMR addresses
|
|
Packit Service |
aa3af4 |
* @return 0 on success -1 on failure
|
|
Packit Service |
aa3af4 |
* @note when using UMR memory appears in VMA as follows
|
|
Packit Service |
aa3af4 |
* +----------------------------+
|
|
Packit Service |
aa3af4 |
* | WQE0 network headers |
|
|
Packit Service |
aa3af4 |
* | WQE1 network headers |
|
|
Packit Service |
aa3af4 |
* | ... |
|
|
Packit Service |
aa3af4 |
* | WQE0 user headers |
|
|
Packit Service |
aa3af4 |
* | WQE1 user headers |
|
|
Packit Service |
aa3af4 |
* | ... |
|
|
Packit Service |
aa3af4 |
* | WQE0 payload |
|
|
Packit Service |
aa3af4 |
* | WQE1 payload |
|
|
Packit Service |
aa3af4 |
* | ... |
|
|
Packit Service |
aa3af4 |
* | WQE0 padding |
|
|
Packit Service |
aa3af4 |
* | WQE1 padding |
|
|
Packit Service |
aa3af4 |
* | ... |
|
|
Packit Service |
aa3af4 |
* +----------------------------+
|
|
Packit Service |
aa3af4 |
*/
|
|
Packit Service |
aa3af4 |
int ring_eth_cb::allocate_umr_mem(vma_cyclic_buffer_ring_attr *cb_ring,
|
|
Packit Service |
aa3af4 |
iovec *mem_desc,
|
|
Packit Service |
aa3af4 |
uint16_t net_len)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
ibv_exp_create_mr_in mrin;
|
|
Packit Service |
aa3af4 |
ibv_exp_mem_repeat_block* p_mem_rep_list = NULL;
|
|
Packit Service |
aa3af4 |
ibv_mr* mr = NULL, *dump_mr;
|
|
Packit Service |
aa3af4 |
size_t curr_data_len = 0, packet_len, pad_len, buffer_size;
|
|
Packit Service |
aa3af4 |
size_t packets_num = m_strides_num * m_wq_count;
|
|
Packit Service |
aa3af4 |
uint64_t base_ptr, prev_addr, pad_addr;
|
|
Packit Service |
aa3af4 |
int index = 0, count = 1, umr_blocks;
|
|
Packit Service |
aa3af4 |
const int ndim = 1; // we only use one dimension see UMR docs
|
|
Packit Service |
aa3af4 |
int retval = 0;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
// the min mr is two one for padding and one for data
|
|
Packit Service |
aa3af4 |
umr_blocks = 2;
|
|
Packit Service |
aa3af4 |
if ((cb_ring->comp_mask & VMA_CB_HDR_BYTE) && cb_ring->hdr_bytes &&
|
|
Packit Service |
aa3af4 |
m_packet_receive_mode == RAW_PACKET) {
|
|
Packit Service |
aa3af4 |
ring_logwarn("bad parameters!, you cannot choose "
|
|
Packit Service |
aa3af4 |
"RAW_PACKET and define user header "
|
|
Packit Service |
aa3af4 |
"the header\n");
|
|
Packit Service |
aa3af4 |
return -1;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
if (m_packet_receive_mode != RAW_PACKET) {
|
|
Packit Service |
aa3af4 |
umr_blocks++; // add user_hd\netwrok_hdr
|
|
Packit Service |
aa3af4 |
if ((cb_ring->comp_mask & VMA_CB_HDR_BYTE) &&
|
|
Packit Service |
aa3af4 |
cb_ring->hdr_bytes &&
|
|
Packit Service |
aa3af4 |
m_packet_receive_mode == STRIP_NETWORK_HDRS) {
|
|
Packit Service |
aa3af4 |
umr_blocks++; // strip network hdr
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
p_mem_rep_list = new(std::nothrow) ibv_exp_mem_repeat_block[umr_blocks]();
|
|
Packit Service |
aa3af4 |
if (p_mem_rep_list == NULL) {
|
|
Packit Service |
aa3af4 |
ring_logwarn("failed allocating memory");
|
|
Packit Service |
aa3af4 |
errno = ENOMEM;
|
|
Packit Service |
aa3af4 |
return -1;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
for (int i = 0; i < umr_blocks; i++) {
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[i].byte_count = new(std::nothrow) size_t[ndim];
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[i].stride = new(std::nothrow) size_t[ndim];
|
|
Packit Service |
aa3af4 |
if (p_mem_rep_list[i].byte_count == NULL ||
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[i].stride == NULL) {
|
|
Packit Service |
aa3af4 |
ring_logwarn("failed allocating memory");
|
|
Packit Service |
aa3af4 |
errno = ENOMEM;
|
|
Packit Service |
aa3af4 |
retval = -1;
|
|
Packit Service |
aa3af4 |
goto cleanup;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
m_payload_len = cb_ring->stride_bytes;
|
|
Packit Service |
aa3af4 |
m_hdr_len = cb_ring->hdr_bytes;
|
|
Packit Service |
aa3af4 |
m_packet_size = m_payload_len + m_hdr_len + net_len;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
// in case stride smaller then packet size
|
|
Packit Service |
aa3af4 |
while ((m_stride_size * count) <= m_packet_size) {
|
|
Packit Service |
aa3af4 |
++count;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
// no need to allocate padding
|
|
Packit Service |
aa3af4 |
pad_len = (m_stride_size * count) - m_packet_size;
|
|
Packit Service |
aa3af4 |
// allocate buffer
|
|
Packit Service |
aa3af4 |
if (m_packet_receive_mode == STRIP_NETWORK_HDRS) {
|
|
Packit Service |
aa3af4 |
buffer_size = (m_packet_size - net_len) * packets_num;
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
buffer_size = m_packet_size * packets_num;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
// will raise an exception on failure
|
|
Packit Service |
aa3af4 |
base_ptr = (uint64_t)allocate_memory(mem_desc, buffer_size);
|
|
Packit Service |
aa3af4 |
if (unlikely(!base_ptr)) {
|
|
Packit Service |
aa3af4 |
goto cleanup;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
ring_logdbg("using buffer parameters, buffer_size %zd "
|
|
Packit Service |
aa3af4 |
"pad len %d packet size %d stride size %d",
|
|
Packit Service |
aa3af4 |
buffer_size, pad_len, m_packet_size, m_stride_size);
|
|
Packit Service |
aa3af4 |
prev_addr = base_ptr;
|
|
Packit Service |
aa3af4 |
mr = m_alloc.find_ibv_mr_by_ib_ctx(m_p_ib_ctx);
|
|
Packit Service |
aa3af4 |
// redmine.mellanox.com/issues/1379468
|
|
Packit Service |
aa3af4 |
pad_addr = (uint64_t)m_dump_mr.alloc_and_reg_mr(128, m_p_ib_ctx);
|
|
Packit Service |
aa3af4 |
dump_mr = m_dump_mr.find_ibv_mr_by_ib_ctx(m_p_ib_ctx);
|
|
Packit Service |
aa3af4 |
BULLSEYE_EXCLUDE_BLOCK_START
|
|
Packit Service |
aa3af4 |
if (unlikely(mr == NULL || dump_mr == NULL)) {
|
|
Packit Service |
aa3af4 |
ring_logerr("could not find mr %p, dump mr %p", mr, dump_mr);
|
|
Packit Service |
aa3af4 |
retval = -1;
|
|
Packit Service |
aa3af4 |
goto cleanup;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
BULLSEYE_EXCLUDE_BLOCK_END
|
|
Packit Service |
aa3af4 |
// no problem overriding lkey since deregmr is not using it
|
|
Packit Service |
aa3af4 |
dump_mr->lkey = DUMP_LKEY;
|
|
Packit Service |
aa3af4 |
packet_len = net_len;
|
|
Packit Service |
aa3af4 |
switch (m_packet_receive_mode) {
|
|
Packit Service |
aa3af4 |
case RAW_PACKET:
|
|
Packit Service |
aa3af4 |
packet_len += m_payload_len;
|
|
Packit Service |
aa3af4 |
// for size calculation in read_cyclic
|
|
Packit Service |
aa3af4 |
m_payload_len = packet_len;
|
|
Packit Service |
aa3af4 |
m_sge_ptrs[CB_UMR_PAYLOAD] = base_ptr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].base_addr = base_ptr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].byte_count[0] = packet_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].stride[0] = packet_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].mr = mr;
|
|
Packit Service |
aa3af4 |
index++;
|
|
Packit Service |
aa3af4 |
break;
|
|
Packit Service |
aa3af4 |
case STRIP_NETWORK_HDRS:
|
|
Packit Service |
aa3af4 |
// network not accessible to application
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].base_addr = pad_addr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].byte_count[0] = net_len;
|
|
Packit Service |
aa3af4 |
// optimize write header to the same physical address
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].stride[0] = 0;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].mr = dump_mr;
|
|
Packit Service |
aa3af4 |
index++;
|
|
Packit Service |
aa3af4 |
if (m_hdr_len) {
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].base_addr = base_ptr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].byte_count[0] = m_hdr_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].stride[0] = m_hdr_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].mr = mr;
|
|
Packit Service |
aa3af4 |
m_sge_ptrs[CB_UMR_HDR] = base_ptr;
|
|
Packit Service |
aa3af4 |
curr_data_len = packets_num * m_hdr_len;
|
|
Packit Service |
aa3af4 |
prev_addr += curr_data_len;
|
|
Packit Service |
aa3af4 |
index++;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].base_addr = prev_addr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].byte_count[0] = m_payload_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].stride[0] = m_payload_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].mr = mr;
|
|
Packit Service |
aa3af4 |
m_sge_ptrs[CB_UMR_PAYLOAD] = prev_addr;
|
|
Packit Service |
aa3af4 |
index++;
|
|
Packit Service |
aa3af4 |
break;
|
|
Packit Service |
aa3af4 |
case SEPERATE_NETWORK_HDRS:
|
|
Packit Service |
aa3af4 |
if (m_hdr_len) {
|
|
Packit Service |
aa3af4 |
packet_len += m_hdr_len;
|
|
Packit Service |
aa3af4 |
// for size calculation in read_cyclic
|
|
Packit Service |
aa3af4 |
m_hdr_len = packet_len;
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
m_hdr_len = net_len;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].base_addr = base_ptr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].byte_count[0] = packet_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].stride[0] = packet_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].mr = mr;
|
|
Packit Service |
aa3af4 |
m_sge_ptrs[CB_UMR_HDR] = base_ptr;
|
|
Packit Service |
aa3af4 |
curr_data_len = packets_num * packet_len;
|
|
Packit Service |
aa3af4 |
prev_addr += curr_data_len;
|
|
Packit Service |
aa3af4 |
index++;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].base_addr = prev_addr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].byte_count[0] = m_payload_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].stride[0] = m_payload_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].mr = mr;
|
|
Packit Service |
aa3af4 |
m_sge_ptrs[CB_UMR_PAYLOAD] = prev_addr;
|
|
Packit Service |
aa3af4 |
index++;
|
|
Packit Service |
aa3af4 |
break;
|
|
Packit Service |
aa3af4 |
default:
|
|
Packit Service |
aa3af4 |
ring_logpanic("bad packet_receive_mode\n");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
// use base_ptr as base_addr to corrupt user data and prevent stack
|
|
Packit Service |
aa3af4 |
// corruption in case of unexpected big packet
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].base_addr = pad_addr;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].byte_count[0] = pad_len;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].stride[0] = 0;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[index].mr = dump_mr;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
// allocate empty lkey
|
|
Packit Service |
aa3af4 |
memset(&mrin, 0, sizeof(mrin));
|
|
Packit Service |
aa3af4 |
mrin.pd = m_p_ib_ctx->get_ibv_pd();
|
|
Packit Service |
aa3af4 |
mrin.attr.create_flags = IBV_EXP_MR_INDIRECT_KLMS;
|
|
Packit Service |
aa3af4 |
mrin.attr.exp_access_flags = IBV_EXP_ACCESS_LOCAL_WRITE;
|
|
Packit Service |
aa3af4 |
mrin.attr.max_klm_list_size = umr_blocks;
|
|
Packit Service |
aa3af4 |
m_p_umr_mr = ibv_exp_create_mr(&mrin);
|
|
Packit Service |
aa3af4 |
BULLSEYE_EXCLUDE_BLOCK_START
|
|
Packit Service |
aa3af4 |
if (!m_p_umr_mr) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("Failed creating mr %m", errno);
|
|
Packit Service |
aa3af4 |
retval = -1;
|
|
Packit Service |
aa3af4 |
goto cleanup;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
BULLSEYE_EXCLUDE_BLOCK_END
|
|
Packit Service |
aa3af4 |
memset(&m_umr_wr, 0, sizeof(m_umr_wr));
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.umr_type = IBV_EXP_UMR_REPEAT;
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.mem_list.rb.mem_repeat_block_list = p_mem_rep_list;
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.mem_list.rb.stride_dim = ndim;
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.mem_list.rb.repeat_count = &packets_num;
|
|
Packit Service |
aa3af4 |
m_umr_wr.exp_send_flags = IBV_EXP_SEND_INLINE;
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.exp_access = IBV_EXP_ACCESS_LOCAL_WRITE;
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.modified_mr = m_p_umr_mr;
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.base_addr = (uint64_t)mr->addr;
|
|
Packit Service |
aa3af4 |
m_umr_wr.ext_op.umr.num_mrs = umr_blocks;
|
|
Packit Service |
aa3af4 |
m_umr_wr.exp_send_flags |= IBV_EXP_SEND_SIGNALED;
|
|
Packit Service |
aa3af4 |
m_umr_wr.exp_opcode = IBV_EXP_WR_UMR_FILL;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
if (!m_p_ib_ctx->post_umr_wr(m_umr_wr)) {
|
|
Packit Service |
aa3af4 |
ring_logerr("Failed in ibv_exp_post_send IBV_EXP_WR_UMR_FILL\n");
|
|
Packit Service |
aa3af4 |
// prevent removal
|
|
Packit Service |
aa3af4 |
m_umr_wr.exp_opcode = IBV_EXP_WR_NOP;
|
|
Packit Service |
aa3af4 |
retval = -1;
|
|
Packit Service |
aa3af4 |
goto cleanup;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
m_buff_data.addr = m_umr_wr.ext_op.umr.base_addr;
|
|
Packit Service |
aa3af4 |
m_buff_data.length = m_stride_size * m_strides_num;
|
|
Packit Service |
aa3af4 |
m_buff_data.lkey = m_p_umr_mr->lkey;
|
|
Packit Service |
aa3af4 |
cleanup:
|
|
Packit Service |
aa3af4 |
for (int i = 0; i < umr_blocks; i++) {
|
|
Packit Service |
aa3af4 |
if (p_mem_rep_list[i].stride) {
|
|
Packit Service |
aa3af4 |
delete[] p_mem_rep_list[i].stride;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[i].stride = NULL;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (p_mem_rep_list[i].byte_count) {
|
|
Packit Service |
aa3af4 |
delete[] p_mem_rep_list[i].byte_count;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list[i].byte_count = NULL;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
delete[] p_mem_rep_list;
|
|
Packit Service |
aa3af4 |
p_mem_rep_list = NULL;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
if (retval == -1) {
|
|
Packit Service |
aa3af4 |
remove_umr_res();
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
return retval;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
void ring_eth_cb::remove_umr_res()
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
if (m_umr_wr.exp_opcode == IBV_EXP_WR_UMR_FILL) {
|
|
Packit Service |
aa3af4 |
m_umr_wr.exp_opcode = IBV_EXP_WR_UMR_INVALIDATE;
|
|
Packit Service |
aa3af4 |
if (m_p_ib_ctx->post_umr_wr(m_umr_wr)) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("Releasing UMR failed\n");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
if (m_p_umr_mr) {
|
|
Packit Service |
aa3af4 |
ibv_dereg_mr(m_p_umr_mr);
|
|
Packit Service |
aa3af4 |
m_p_umr_mr = NULL;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
ring_logdbg("UMR resources removed\n");
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
int ring_eth_cb::drain_and_proccess()
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
return 0;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
int ring_eth_cb::poll_and_process_element_rx(uint64_t* p_cq_poll_sn,
|
|
Packit Service |
aa3af4 |
void* pv_fd_ready_array)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
NOT_IN_USE(p_cq_poll_sn);
|
|
Packit Service |
aa3af4 |
NOT_IN_USE(pv_fd_ready_array);
|
|
Packit Service |
aa3af4 |
return 0;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
/**
|
|
Packit Service |
aa3af4 |
* loop poll_cq
|
|
Packit Service |
aa3af4 |
* @param limit
|
|
Packit Service |
aa3af4 |
* @return TBD about -1 on error,
|
|
Packit Service |
aa3af4 |
* 0 if cq is empty
|
|
Packit Service |
aa3af4 |
* 1 if done looping
|
|
Packit Service |
aa3af4 |
* 2 if need to return due to WQ or filler
|
|
Packit Service |
aa3af4 |
*/
|
|
Packit Service |
aa3af4 |
inline mp_loop_result ring_eth_cb::mp_loop_padded(size_t limit)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
struct mlx5_cqe64 *cqe64;
|
|
Packit Service |
aa3af4 |
uint16_t size = 0;
|
|
Packit Service |
aa3af4 |
uint32_t flags = 0, used_strides = 0;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
while (m_curr_packets < limit) {
|
|
Packit Service |
aa3af4 |
int ret = ((cq_mgr_mp *)m_p_cq_mgr_rx)->poll_mp_cq(size, used_strides,
|
|
Packit Service |
aa3af4 |
flags, cqe64);
|
|
Packit Service |
aa3af4 |
if (size == 0) {
|
|
Packit Service |
aa3af4 |
ring_logfine("no packet found");
|
|
Packit Service |
aa3af4 |
return MP_LOOP_DRAINED;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (unlikely(ret == -1)) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("poll_mp_cq failed with errno %m", errno);
|
|
Packit Service |
aa3af4 |
return MP_LOOP_RETURN_TO_APP;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_curr_wqe_used_strides += used_strides;
|
|
Packit Service |
aa3af4 |
if (unlikely(flags & VMA_MP_RQ_BAD_PACKET)) {
|
|
Packit Service |
aa3af4 |
if (m_curr_wqe_used_strides >= m_strides_num) {
|
|
Packit Service |
aa3af4 |
reload_wq();
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
return MP_LOOP_RETURN_TO_APP;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_padd_mode_used_strides += used_strides;
|
|
Packit Service |
aa3af4 |
m_p_ring_stat->n_rx_pkt_count++;
|
|
Packit Service |
aa3af4 |
m_p_ring_stat->n_rx_byte_count += size;
|
|
Packit Service |
aa3af4 |
++m_curr_packets;
|
|
Packit Service |
aa3af4 |
if (unlikely(m_curr_wqe_used_strides >= m_strides_num)) {
|
|
Packit Service |
aa3af4 |
if (reload_wq()) {
|
|
Packit Service |
aa3af4 |
return MP_LOOP_RETURN_TO_APP;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
ring_logfine("mp_loop finished all iterations");
|
|
Packit Service |
aa3af4 |
return MP_LOOP_LIMIT;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
/**
|
|
Packit Service |
aa3af4 |
* loop poll_cq
|
|
Packit Service |
aa3af4 |
* @param limit
|
|
Packit Service |
aa3af4 |
* @return TBD about -1 on error,
|
|
Packit Service |
aa3af4 |
* 0 if cq is empty
|
|
Packit Service |
aa3af4 |
* 1 if done looping
|
|
Packit Service |
aa3af4 |
* 2 if need to return due to WQ or filler
|
|
Packit Service |
aa3af4 |
*/
|
|
Packit Service |
aa3af4 |
inline mp_loop_result ring_eth_cb::mp_loop(size_t limit)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
struct mlx5_cqe64 *cqe64;
|
|
Packit Service |
aa3af4 |
uint16_t size = 0;
|
|
Packit Service |
aa3af4 |
uint32_t flags = 0, used_strides = 0;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
while (m_curr_packets < limit) {
|
|
Packit Service |
aa3af4 |
int ret = ((cq_mgr_mp *)m_p_cq_mgr_rx)->poll_mp_cq(size, used_strides,
|
|
Packit Service |
aa3af4 |
flags, cqe64);
|
|
Packit Service |
aa3af4 |
if (size == 0) {
|
|
Packit Service |
aa3af4 |
ring_logfine("no packet found");
|
|
Packit Service |
aa3af4 |
return MP_LOOP_DRAINED;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (unlikely(ret == -1)) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("poll_mp_cq failed with errno %m", errno);
|
|
Packit Service |
aa3af4 |
return MP_LOOP_RETURN_TO_APP;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_curr_wqe_used_strides += used_strides;
|
|
Packit Service |
aa3af4 |
if (unlikely(size > m_packet_size)) {
|
|
Packit Service |
aa3af4 |
errno = EMSGSIZE;
|
|
Packit Service |
aa3af4 |
ring_logerr("got unexpected packet size, expected "
|
|
Packit Service |
aa3af4 |
"packet size %u but got %d, user data is "
|
|
Packit Service |
aa3af4 |
"corrupted", m_packet_size, size);
|
|
Packit Service |
aa3af4 |
return MP_LOOP_RETURN_TO_APP;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (unlikely(flags & VMA_MP_RQ_BAD_PACKET)) {
|
|
Packit Service |
aa3af4 |
if (m_curr_wqe_used_strides >= m_strides_num) {
|
|
Packit Service |
aa3af4 |
reload_wq();
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
return MP_LOOP_RETURN_TO_APP;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_p_ring_stat->n_rx_pkt_count++;
|
|
Packit Service |
aa3af4 |
m_p_ring_stat->n_rx_byte_count += size;
|
|
Packit Service |
aa3af4 |
++m_curr_packets;
|
|
Packit Service |
aa3af4 |
if (unlikely(m_curr_wqe_used_strides >= m_strides_num)) {
|
|
Packit Service |
aa3af4 |
if (reload_wq()) {
|
|
Packit Service |
aa3af4 |
return MP_LOOP_RETURN_TO_APP;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
ring_logfine("mp_loop finished all iterations");
|
|
Packit Service |
aa3af4 |
return MP_LOOP_LIMIT;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
/*
|
|
Packit Service |
aa3af4 |
* all WQE are contagious in memory so we need to return to the user
|
|
Packit Service |
aa3af4 |
* true if last WQE was posted so we're at the end of the buffer
|
|
Packit Service |
aa3af4 |
*
|
|
Packit Service |
aa3af4 |
*/
|
|
Packit Service |
aa3af4 |
inline bool ring_eth_cb::reload_wq()
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
// in current implementation after each WQe is used by the HW
|
|
Packit Service |
aa3af4 |
// the ring reloads it to the HW again that why 1 is used
|
|
Packit Service |
aa3af4 |
((cq_mgr_mp *)m_p_cq_mgr_rx)->update_dbell();
|
|
Packit Service |
aa3af4 |
((qp_mgr_mp *)m_p_qp_mgr)->post_recv(m_curr_wq, 1);
|
|
Packit Service |
aa3af4 |
m_curr_wq = (m_curr_wq + 1) % m_wq_count;
|
|
Packit Service |
aa3af4 |
m_curr_wqe_used_strides = 0;
|
|
Packit Service |
aa3af4 |
if (m_curr_wq == 0) {
|
|
Packit Service |
aa3af4 |
m_all_wqes_used_strides = 0;
|
|
Packit Service |
aa3af4 |
return true;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_all_wqes_used_strides += m_strides_num;
|
|
Packit Service |
aa3af4 |
return false;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
int ring_eth_cb::cyclic_buffer_read(vma_completion_cb_t &completion,
|
|
Packit Service |
aa3af4 |
size_t min, size_t max, int flags)
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
uint32_t poll_flags = 0, used_strides = 0;
|
|
Packit Service |
aa3af4 |
uint16_t size;
|
|
Packit Service |
aa3af4 |
struct mlx5_cqe64 *cqe64;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
// sanity check
|
|
Packit Service |
aa3af4 |
if (unlikely(min > max || max == 0 || flags != MSG_DONTWAIT)) {
|
|
Packit Service |
aa3af4 |
errno = EINVAL;
|
|
Packit Service |
aa3af4 |
ring_logdbg("Illegal values, got min: %d, max: %d, flags %d",
|
|
Packit Service |
aa3af4 |
min, max, flags);
|
|
Packit Service |
aa3af4 |
if (flags != MSG_DONTWAIT) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("only %d flag is currently supported",
|
|
Packit Service |
aa3af4 |
MSG_DONTWAIT);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
return -1;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
int prev_used_strides = m_curr_wqe_used_strides;
|
|
Packit Service |
aa3af4 |
int ret = ((cq_mgr_mp *)m_p_cq_mgr_rx)->poll_mp_cq(size, used_strides,
|
|
Packit Service |
aa3af4 |
poll_flags, cqe64);
|
|
Packit Service |
aa3af4 |
// empty
|
|
Packit Service |
aa3af4 |
if (size == 0) {
|
|
Packit Service |
aa3af4 |
return 0;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
if (m_packet_receive_mode != PADDED_PACKET &&
|
|
Packit Service |
aa3af4 |
unlikely(size > m_packet_size)) {
|
|
Packit Service |
aa3af4 |
errno = EMSGSIZE;
|
|
Packit Service |
aa3af4 |
ring_logerr("got unexpected packet size, expected "
|
|
Packit Service |
aa3af4 |
"packet size %u but got %d, user data is "
|
|
Packit Service |
aa3af4 |
"corrupted", m_packet_size, size);
|
|
Packit Service |
aa3af4 |
return -1;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (unlikely(ret == -1)) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("poll_mp_cq failed with errno %m", errno);
|
|
Packit Service |
aa3af4 |
return -1;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_curr_wqe_used_strides += used_strides;
|
|
Packit Service |
aa3af4 |
m_padd_mode_used_strides += used_strides;
|
|
Packit Service |
aa3af4 |
// set it here because we might not have min packets avail in this run
|
|
Packit Service |
aa3af4 |
if (likely(!(poll_flags & VMA_MP_RQ_BAD_PACKET))) {
|
|
Packit Service |
aa3af4 |
m_p_ring_stat->n_rx_pkt_count++;
|
|
Packit Service |
aa3af4 |
m_p_ring_stat->n_rx_byte_count += size;
|
|
Packit Service |
aa3af4 |
if (unlikely(m_curr_payload_addr == NULL)) {
|
|
Packit Service |
aa3af4 |
// data is in calculated UMR location array +
|
|
Packit Service |
aa3af4 |
// number of strides in old WQEs (e.g. first WQE that was already consumed) +
|
|
Packit Service |
aa3af4 |
// number of used strides in current WQE
|
|
Packit Service |
aa3af4 |
prev_used_strides += m_all_wqes_used_strides;
|
|
Packit Service |
aa3af4 |
m_curr_payload_addr = (void *)(m_sge_ptrs[CB_UMR_PAYLOAD] +
|
|
Packit Service |
aa3af4 |
(uint32_t)m_payload_len * prev_used_strides);
|
|
Packit Service |
aa3af4 |
m_curr_hdr_ptr = (void *)(m_sge_ptrs[CB_UMR_HDR] +
|
|
Packit Service |
aa3af4 |
(uint32_t)m_hdr_len * prev_used_strides);
|
|
Packit Service |
aa3af4 |
if (completion.comp_mask & VMA_CB_MASK_TIMESTAMP) {
|
|
Packit Service |
aa3af4 |
convert_hw_time_to_system_time(ntohll(cqe64->timestamp),
|
|
Packit Service |
aa3af4 |
&m_curr_hw_timestamp);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
m_curr_packets = 1;
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
m_curr_packets++;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
bool return_to_app = false;
|
|
Packit Service |
aa3af4 |
if (unlikely(m_curr_wqe_used_strides >= m_strides_num)) {
|
|
Packit Service |
aa3af4 |
return_to_app = reload_wq();
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (!return_to_app) {
|
|
Packit Service |
aa3af4 |
if (m_packet_receive_mode == PADDED_PACKET) {
|
|
Packit Service |
aa3af4 |
ret = mp_loop_padded(min);
|
|
Packit Service |
aa3af4 |
if (ret == MP_LOOP_LIMIT) { // there might be more to drain
|
|
Packit Service |
aa3af4 |
mp_loop_padded(max);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
ret = mp_loop(min);
|
|
Packit Service |
aa3af4 |
if (ret == MP_LOOP_LIMIT) { // there might be more to drain
|
|
Packit Service |
aa3af4 |
mp_loop(max);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
if (ret == MP_LOOP_DRAINED) { // no packets left
|
|
Packit Service |
aa3af4 |
((cq_mgr_mp *)m_p_cq_mgr_rx)->update_max_drain(m_curr_packets);
|
|
Packit Service |
aa3af4 |
return 0;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
((cq_mgr_mp *)m_p_cq_mgr_rx)->update_max_drain(m_curr_packets);
|
|
Packit Service |
aa3af4 |
completion.payload_ptr = m_curr_payload_addr;
|
|
Packit Service |
aa3af4 |
if (m_packet_receive_mode == PADDED_PACKET) {
|
|
Packit Service |
aa3af4 |
// support packet taking more then one stride
|
|
Packit Service |
aa3af4 |
completion.payload_length = m_padd_mode_used_strides * m_stride_size;
|
|
Packit Service |
aa3af4 |
} else {
|
|
Packit Service |
aa3af4 |
completion.payload_length = m_payload_len * m_curr_packets;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
completion.packets = m_curr_packets;
|
|
Packit Service |
aa3af4 |
completion.usr_hdr_ptr = m_curr_hdr_ptr;
|
|
Packit Service |
aa3af4 |
completion.usr_hdr_ptr_length = m_hdr_len * m_curr_packets;
|
|
Packit Service |
aa3af4 |
// hw_timestamp of first packet in batch
|
|
Packit Service |
aa3af4 |
completion.hw_timestamp = m_curr_hw_timestamp;
|
|
Packit Service |
aa3af4 |
m_curr_payload_addr = 0;
|
|
Packit Service |
aa3af4 |
m_padd_mode_used_strides = 0;
|
|
Packit Service |
aa3af4 |
ring_logdbg("Returning completion, buffer ptr %p, data size %zd, "
|
|
Packit Service |
aa3af4 |
"usr hdr ptr %p usr hdr size %zd, number of packets %zd curr wqe idx %d",
|
|
Packit Service |
aa3af4 |
completion.payload_ptr, completion.payload_length,
|
|
Packit Service |
aa3af4 |
completion.usr_hdr_ptr, completion.usr_hdr_ptr_length,
|
|
Packit Service |
aa3af4 |
m_curr_packets, m_curr_wq);
|
|
Packit Service |
aa3af4 |
return 0;
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
ring_eth_cb::~ring_eth_cb()
|
|
Packit Service |
aa3af4 |
{
|
|
Packit Service |
aa3af4 |
struct ibv_exp_destroy_res_domain_attr attr;
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
m_lock_ring_rx.lock();
|
|
Packit Service |
aa3af4 |
flow_udp_del_all();
|
|
Packit Service |
aa3af4 |
flow_tcp_del_all();
|
|
Packit Service |
aa3af4 |
m_lock_ring_rx.unlock();
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
memset(&attr, 0, sizeof(attr));
|
|
Packit Service |
aa3af4 |
int res = ibv_exp_destroy_res_domain(m_p_ib_ctx->get_ibv_context(),
|
|
Packit Service |
aa3af4 |
m_res_domain, &attr);
|
|
Packit Service |
aa3af4 |
if (res) {
|
|
Packit Service |
aa3af4 |
ring_logdbg("call to ibv_exp_destroy_res_domain returned %d", res);
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
|
|
Packit Service |
aa3af4 |
remove_umr_res();
|
|
Packit Service |
aa3af4 |
}
|
|
Packit Service |
aa3af4 |
#endif /* HAVE_MP_RQ */
|
|
Packit Service |
aa3af4 |
|