|
Packit |
6d2c1b |
/*
|
|
Packit |
6d2c1b |
* Copyright (c) 2001-2020 Mellanox Technologies, Ltd. All rights reserved.
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* This software is available to you under a choice of one of two
|
|
Packit |
6d2c1b |
* licenses. You may choose to be licensed under the terms of the GNU
|
|
Packit |
6d2c1b |
* General Public License (GPL) Version 2, available from the file
|
|
Packit |
6d2c1b |
* COPYING in the main directory of this source tree, or the
|
|
Packit |
6d2c1b |
* BSD license below:
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* Redistribution and use in source and binary forms, with or
|
|
Packit |
6d2c1b |
* without modification, are permitted provided that the following
|
|
Packit |
6d2c1b |
* conditions are met:
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* - Redistributions of source code must retain the above
|
|
Packit |
6d2c1b |
* copyright notice, this list of conditions and the following
|
|
Packit |
6d2c1b |
* disclaimer.
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* - Redistributions in binary form must reproduce the above
|
|
Packit |
6d2c1b |
* copyright notice, this list of conditions and the following
|
|
Packit |
6d2c1b |
* disclaimer in the documentation and/or other materials
|
|
Packit |
6d2c1b |
* provided with the distribution.
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
Packit |
6d2c1b |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
Packit |
6d2c1b |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
Packit |
6d2c1b |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
|
Packit |
6d2c1b |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
Packit |
6d2c1b |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
Packit |
6d2c1b |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
Packit |
6d2c1b |
* SOFTWARE.
|
|
Packit |
6d2c1b |
*/
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
#include "dm_mgr.h"
|
|
Packit |
6d2c1b |
#include "vlogger/vlogger.h"
|
|
Packit |
6d2c1b |
#include "vma/proto/mem_buf_desc.h"
|
|
Packit |
6d2c1b |
#include "vma/dev/ib_ctx_handler.h"
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
#if defined(DEFINED_DIRECT_VERBS)
|
|
Packit |
6d2c1b |
#if defined(DEFINED_IBV_DM)
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
#define DM_MEMORY_MASK_8 7
|
|
Packit |
6d2c1b |
#define DM_MEMORY_MASK_64 63
|
|
Packit |
6d2c1b |
#define DM_ALIGN_SIZE(size, mask) ((size + mask) & (~mask))
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
#undef MODULE_NAME
|
|
Packit |
6d2c1b |
#define MODULE_NAME "dm_mgr"
|
|
Packit |
6d2c1b |
#undef MODULE_HDR
|
|
Packit |
6d2c1b |
#define MODULE_HDR MODULE_NAME "%d:%s() "
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
#define dm_logerr __log_info_err
|
|
Packit |
6d2c1b |
#define dm_logwarn __log_info_warn
|
|
Packit |
6d2c1b |
#define dm_logdbg __log_info_dbg
|
|
Packit |
6d2c1b |
#define dm_logfunc __log_info_func
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
dm_mgr::dm_mgr() :
|
|
Packit |
6d2c1b |
m_p_dm_mr(NULL),
|
|
Packit |
6d2c1b |
m_p_ibv_dm(NULL),
|
|
Packit |
6d2c1b |
m_p_ring_stat(NULL),
|
|
Packit |
6d2c1b |
m_allocation(0),
|
|
Packit |
6d2c1b |
m_used(0),
|
|
Packit |
6d2c1b |
m_head(0)
|
|
Packit |
6d2c1b |
{};
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
/*
|
|
Packit |
6d2c1b |
* Allocate dev_mem resources
|
|
Packit |
6d2c1b |
*/
|
|
Packit |
6d2c1b |
bool dm_mgr::allocate_resources(ib_ctx_handler* ib_ctx, ring_stats_t* ring_stats)
|
|
Packit |
6d2c1b |
{
|
|
Packit |
6d2c1b |
size_t allocation_size = DM_ALIGN_SIZE(safe_mce_sys().ring_dev_mem_tx, DM_MEMORY_MASK_64);
|
|
Packit |
6d2c1b |
vma_ibv_alloc_dm_attr dm_attr;
|
|
Packit |
6d2c1b |
vma_ibv_reg_mr_in mr_in;
|
|
Packit |
6d2c1b |
m_p_ring_stat = ring_stats;
|
|
Packit |
6d2c1b |
if (!allocation_size) {
|
|
Packit |
6d2c1b |
// On Device Memory usage was disabled by the user
|
|
Packit |
6d2c1b |
return false;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
if (!ib_ctx->get_on_device_memory_size()) {
|
|
Packit |
6d2c1b |
// On Device Memory usage is not supported
|
|
Packit |
6d2c1b |
return false;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Allocate on device memory buffer
|
|
Packit |
6d2c1b |
memset(&dm_attr, 0, sizeof(dm_attr));
|
|
Packit |
6d2c1b |
dm_attr.length = allocation_size;
|
|
Packit |
6d2c1b |
m_p_ibv_dm = vma_ibv_alloc_dm(ib_ctx->get_ibv_context(), &dm_attr);
|
|
Packit |
6d2c1b |
if (!m_p_ibv_dm) {
|
|
Packit |
6d2c1b |
// Memory allocation can fail if we have already allocated the maximum possible.
|
|
Packit |
6d2c1b |
dm_logdbg("ibv_alloc_dm error - On Device Memory allocation failed, %d %m", errno);
|
|
Packit |
6d2c1b |
errno = 0;
|
|
Packit |
6d2c1b |
return false;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Initialize MR attributes
|
|
Packit |
6d2c1b |
memset(&mr_in, 0, sizeof(mr_in));
|
|
Packit |
6d2c1b |
vma_ibv_init_dm_mr(mr_in, ib_ctx->get_ibv_pd(), allocation_size, m_p_ibv_dm);
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Register On Device Memory MR
|
|
Packit |
6d2c1b |
m_p_dm_mr = vma_ibv_reg_dm_mr(&mr_in);
|
|
Packit |
6d2c1b |
if (!m_p_dm_mr) {
|
|
Packit |
6d2c1b |
vma_ibv_free_dm(m_p_ibv_dm);
|
|
Packit |
6d2c1b |
m_p_ibv_dm = NULL;
|
|
Packit |
6d2c1b |
dm_logerr("ibv_free_dm error - dm_mr registration failed, %d %m", errno);
|
|
Packit |
6d2c1b |
return false;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
m_allocation = allocation_size;
|
|
Packit |
6d2c1b |
m_p_ring_stat->simple.n_tx_dev_mem_allocated = m_allocation;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
dm_logdbg("Device memory allocation completed successfully! device[%s] bytes[%zu] dm_mr handle[%d] dm_mr lkey[%d]",
|
|
Packit |
6d2c1b |
ib_ctx->get_ibv_device()->name, dm_attr.length, m_p_dm_mr->handle, m_p_dm_mr->lkey);
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
return true;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
/*
|
|
Packit |
6d2c1b |
* Release dev_mem resources
|
|
Packit |
6d2c1b |
*/
|
|
Packit |
6d2c1b |
void dm_mgr::release_resources()
|
|
Packit |
6d2c1b |
{
|
|
Packit |
6d2c1b |
if (m_p_dm_mr) {
|
|
Packit |
6d2c1b |
if (ibv_dereg_mr(m_p_dm_mr)) {
|
|
Packit |
6d2c1b |
dm_logerr("ibv_dereg_mr failed, %d %m", errno);
|
|
Packit |
6d2c1b |
} else {
|
|
Packit |
6d2c1b |
dm_logdbg("ibv_dereg_mr success");
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
m_p_dm_mr = NULL;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
if (m_p_ibv_dm) {
|
|
Packit |
6d2c1b |
if (vma_ibv_free_dm(m_p_ibv_dm)) {
|
|
Packit |
6d2c1b |
dm_logerr("ibv_free_dm failed %d %m", errno);
|
|
Packit |
6d2c1b |
} else {
|
|
Packit |
6d2c1b |
dm_logdbg("ibv_free_dm success");
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
m_p_ibv_dm = NULL;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
m_p_ring_stat = NULL;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
dm_logdbg("Device memory release completed!");
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
/*
|
|
Packit |
6d2c1b |
* Copy data into the On Device Memory buffer.
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* On Device Memory buffer is implemented in a cycle way using two variables :
|
|
Packit |
6d2c1b |
* m_head - index of the next offset to be written.
|
|
Packit |
6d2c1b |
* m_used - amount of used bytes within the On Device Memory buffer (which also used to calculate the tail of the buffer).
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* In order to maintain a proper order of allocation and release, we must distinguish between three possible cases:
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* First case:
|
|
Packit |
6d2c1b |
* Free space exists in the beginning and in the end of the array.
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* | |XXXXXXXXXX| |
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* tail head
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* Second case:
|
|
Packit |
6d2c1b |
* There is not enough free space at the end of the array.
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* | |XXXXXXXXXX| |
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* tail head
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* In the case above, we will move the head to the beginning of the array.
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* | |XXXXXXXXXXXXX|
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* head tail
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* Third case:
|
|
Packit |
6d2c1b |
* Free space exists in the middle of the array
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* |XXXXXXXXXXXXXX| |XXXXXX|
|
|
Packit |
6d2c1b |
* |-------------------------------------------|
|
|
Packit |
6d2c1b |
* head tail
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* Due to hardware limitations:
|
|
Packit |
6d2c1b |
* 1. Data should be written to 4bytes aligned addresses.
|
|
Packit |
6d2c1b |
* 2. Data length should be aligned to 4bytes.
|
|
Packit |
6d2c1b |
*
|
|
Packit |
6d2c1b |
* Due to performance reasons:
|
|
Packit |
6d2c1b |
* 1. Data should be written to a continuous memory area.
|
|
Packit |
6d2c1b |
* 2. Data will be written to 8bytes aligned addresses.
|
|
Packit |
6d2c1b |
*/
|
|
Packit |
6d2c1b |
bool dm_mgr::copy_data(struct mlx5_wqe_data_seg* seg, uint8_t* src, uint32_t length, mem_buf_desc_t* buff)
|
|
Packit |
6d2c1b |
{
|
|
Packit |
6d2c1b |
vma_ibv_memcpy_dm_attr memcpy_attr;
|
|
Packit |
6d2c1b |
uint32_t length_aligned_8 = DM_ALIGN_SIZE(length, DM_MEMORY_MASK_8);
|
|
Packit |
6d2c1b |
size_t continuous_left = 0;
|
|
Packit |
6d2c1b |
size_t &dev_mem_length = buff->tx.dev_mem_length = 0;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Check if On Device Memory buffer is full
|
|
Packit |
6d2c1b |
if (m_used >= m_allocation) {
|
|
Packit |
6d2c1b |
goto dev_mem_oob;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Check for a continuous space to write
|
|
Packit |
6d2c1b |
if (m_head >= m_used) { // First case
|
|
Packit |
6d2c1b |
if ((continuous_left = m_allocation - m_head) < length_aligned_8) { // Second case
|
|
Packit |
6d2c1b |
if (m_head - m_used >= length_aligned_8) {
|
|
Packit |
6d2c1b |
// There is enough space at the beginning of the buffer.
|
|
Packit |
6d2c1b |
m_head = 0;
|
|
Packit |
6d2c1b |
dev_mem_length = continuous_left;
|
|
Packit |
6d2c1b |
} else {
|
|
Packit |
6d2c1b |
// There no enough space at the beginning of the buffer.
|
|
Packit |
6d2c1b |
goto dev_mem_oob;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
} else if ((continuous_left = m_allocation - m_used) < length_aligned_8) { // Third case
|
|
Packit |
6d2c1b |
goto dev_mem_oob;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Initialize memcopy attributes
|
|
Packit |
6d2c1b |
memset(&memcpy_attr, 0, sizeof(memcpy_attr));
|
|
Packit |
6d2c1b |
vma_ibv_init_memcpy_dm(memcpy_attr, src, m_head, length_aligned_8);
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Copy data into the On Device Memory buffer.
|
|
Packit |
6d2c1b |
if (vma_ibv_memcpy_dm(m_p_ibv_dm, &memcpy_attr)) {
|
|
Packit |
6d2c1b |
dm_logfunc("Failed to memcopy data into the memic buffer %m");
|
|
Packit |
6d2c1b |
return false;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Update values
|
|
Packit |
6d2c1b |
seg->lkey = htonl(m_p_dm_mr->lkey);
|
|
Packit |
6d2c1b |
seg->addr = htonll(m_head);
|
|
Packit |
6d2c1b |
m_head = (m_head + length_aligned_8) % m_allocation;
|
|
Packit |
6d2c1b |
dev_mem_length += length_aligned_8;
|
|
Packit |
6d2c1b |
m_used += dev_mem_length;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
// Update On Device Memory statistics
|
|
Packit |
6d2c1b |
m_p_ring_stat->simple.n_tx_dev_mem_pkt_count++;
|
|
Packit |
6d2c1b |
m_p_ring_stat->simple.n_tx_dev_mem_byte_count += length;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
dm_logfunc("Send completed successfully! Buffer[%p] length[%d] length_aligned_8[%d] continuous_left[%zu] head[%zu] used[%zu]",
|
|
Packit |
6d2c1b |
buff, length, length_aligned_8, continuous_left, m_head, m_used);
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
return true;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
dev_mem_oob:
|
|
Packit |
6d2c1b |
dm_logfunc("Send OOB! Buffer[%p] length[%d] length_aligned_8[%d] continuous_left[%zu] head[%zu] used[%zu]",
|
|
Packit |
6d2c1b |
buff, length, length_aligned_8, continuous_left, m_head, m_used);
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
m_p_ring_stat->simple.n_tx_dev_mem_oob++;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
return false;
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
/*
|
|
Packit |
6d2c1b |
* Release On Device Memory buffer.
|
|
Packit |
6d2c1b |
* This method should be called after completion was received.
|
|
Packit |
6d2c1b |
*/
|
|
Packit |
6d2c1b |
void dm_mgr::release_data(mem_buf_desc_t* buff)
|
|
Packit |
6d2c1b |
{
|
|
Packit |
6d2c1b |
m_used -= buff->tx.dev_mem_length;
|
|
Packit |
6d2c1b |
buff->tx.dev_mem_length = 0;
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
dm_logfunc("Device memory release! buffer[%p] buffer_dev_mem_length[%zu] head[%zu] used[%zu]",
|
|
Packit |
6d2c1b |
buff, buff->tx.dev_mem_length, m_head, m_used);
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
}
|
|
Packit |
6d2c1b |
|
|
Packit |
6d2c1b |
#endif /* DEFINED_IBV_DM */
|
|
Packit |
6d2c1b |
#endif /* DEFINED_DIRECT_VERBS */
|