Blame src/vma/dev/dm_mgr.cpp

Packit 6d2c1b
/*
Packit 6d2c1b
 * Copyright (c) 2001-2020 Mellanox Technologies, Ltd. All rights reserved.
Packit 6d2c1b
 *
Packit 6d2c1b
 * This software is available to you under a choice of one of two
Packit 6d2c1b
 * licenses.  You may choose to be licensed under the terms of the GNU
Packit 6d2c1b
 * General Public License (GPL) Version 2, available from the file
Packit 6d2c1b
 * COPYING in the main directory of this source tree, or the
Packit 6d2c1b
 * BSD license below:
Packit 6d2c1b
 *
Packit 6d2c1b
 *     Redistribution and use in source and binary forms, with or
Packit 6d2c1b
 *     without modification, are permitted provided that the following
Packit 6d2c1b
 *     conditions are met:
Packit 6d2c1b
 *
Packit 6d2c1b
 *      - Redistributions of source code must retain the above
Packit 6d2c1b
 *        copyright notice, this list of conditions and the following
Packit 6d2c1b
 *        disclaimer.
Packit 6d2c1b
 *
Packit 6d2c1b
 *      - Redistributions in binary form must reproduce the above
Packit 6d2c1b
 *        copyright notice, this list of conditions and the following
Packit 6d2c1b
 *        disclaimer in the documentation and/or other materials
Packit 6d2c1b
 *        provided with the distribution.
Packit 6d2c1b
 *
Packit 6d2c1b
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
Packit 6d2c1b
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
Packit 6d2c1b
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
Packit 6d2c1b
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
Packit 6d2c1b
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
Packit 6d2c1b
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
Packit 6d2c1b
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
Packit 6d2c1b
 * SOFTWARE.
Packit 6d2c1b
 */
Packit 6d2c1b
Packit 6d2c1b
#include "dm_mgr.h"
Packit 6d2c1b
#include "vlogger/vlogger.h"
Packit 6d2c1b
#include "vma/proto/mem_buf_desc.h"
Packit 6d2c1b
#include "vma/dev/ib_ctx_handler.h"
Packit 6d2c1b
Packit 6d2c1b
#if defined(DEFINED_DIRECT_VERBS)
Packit 6d2c1b
#if defined(DEFINED_IBV_DM)
Packit 6d2c1b
Packit 6d2c1b
#define DM_MEMORY_MASK_8  7
Packit 6d2c1b
#define DM_MEMORY_MASK_64 63
Packit 6d2c1b
#define DM_ALIGN_SIZE(size, mask) ((size + mask) & (~mask))
Packit 6d2c1b
Packit 6d2c1b
#undef  MODULE_NAME
Packit 6d2c1b
#define MODULE_NAME 		"dm_mgr"
Packit 6d2c1b
#undef  MODULE_HDR
Packit 6d2c1b
#define MODULE_HDR MODULE_NAME "%d:%s() "
Packit 6d2c1b
Packit 6d2c1b
#define dm_logerr	__log_info_err
Packit 6d2c1b
#define dm_logwarn	__log_info_warn
Packit 6d2c1b
#define dm_logdbg	__log_info_dbg
Packit 6d2c1b
#define dm_logfunc	__log_info_func
Packit 6d2c1b
Packit 6d2c1b
dm_mgr::dm_mgr() :
Packit 6d2c1b
	m_p_dm_mr(NULL),
Packit 6d2c1b
	m_p_ibv_dm(NULL),
Packit 6d2c1b
	m_p_ring_stat(NULL),
Packit 6d2c1b
	m_allocation(0),
Packit 6d2c1b
	m_used(0),
Packit 6d2c1b
	m_head(0)
Packit 6d2c1b
{};
Packit 6d2c1b
Packit 6d2c1b
/*
Packit 6d2c1b
 * Allocate dev_mem resources
Packit 6d2c1b
 */
Packit 6d2c1b
bool dm_mgr::allocate_resources(ib_ctx_handler* ib_ctx, ring_stats_t* ring_stats)
Packit 6d2c1b
{
Packit 6d2c1b
	size_t allocation_size = DM_ALIGN_SIZE(safe_mce_sys().ring_dev_mem_tx, DM_MEMORY_MASK_64);
Packit 6d2c1b
	vma_ibv_alloc_dm_attr dm_attr;
Packit 6d2c1b
	vma_ibv_reg_mr_in mr_in;
Packit 6d2c1b
	m_p_ring_stat = ring_stats;
Packit 6d2c1b
	if (!allocation_size) {
Packit 6d2c1b
		// On Device Memory usage was disabled by the user
Packit 6d2c1b
		return false;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	if (!ib_ctx->get_on_device_memory_size()) {
Packit 6d2c1b
		// On Device Memory usage is not supported
Packit 6d2c1b
		return false;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	// Allocate on device memory buffer
Packit 6d2c1b
	memset(&dm_attr, 0, sizeof(dm_attr));
Packit 6d2c1b
	dm_attr.length = allocation_size;
Packit 6d2c1b
	m_p_ibv_dm = vma_ibv_alloc_dm(ib_ctx->get_ibv_context(), &dm_attr);
Packit 6d2c1b
	if (!m_p_ibv_dm) {
Packit 6d2c1b
		// Memory allocation can fail if we have already allocated the maximum possible.
Packit 6d2c1b
		dm_logdbg("ibv_alloc_dm error - On Device Memory allocation failed, %d %m", errno);
Packit 6d2c1b
		errno = 0;
Packit 6d2c1b
		return false;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	// Initialize MR attributes
Packit 6d2c1b
	memset(&mr_in, 0, sizeof(mr_in));
Packit 6d2c1b
	vma_ibv_init_dm_mr(mr_in, ib_ctx->get_ibv_pd(), allocation_size, m_p_ibv_dm);
Packit 6d2c1b
Packit 6d2c1b
	// Register On Device Memory MR
Packit 6d2c1b
	m_p_dm_mr = vma_ibv_reg_dm_mr(&mr_in);
Packit 6d2c1b
	if (!m_p_dm_mr) {
Packit 6d2c1b
		vma_ibv_free_dm(m_p_ibv_dm);
Packit 6d2c1b
		m_p_ibv_dm = NULL;
Packit 6d2c1b
		dm_logerr("ibv_free_dm error - dm_mr registration failed, %d %m", errno);
Packit 6d2c1b
		return false;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	m_allocation = allocation_size;
Packit 6d2c1b
	m_p_ring_stat->simple.n_tx_dev_mem_allocated = m_allocation;
Packit 6d2c1b
Packit 6d2c1b
	dm_logdbg("Device memory allocation completed successfully! device[%s] bytes[%zu] dm_mr handle[%d] dm_mr lkey[%d]",
Packit 6d2c1b
			ib_ctx->get_ibv_device()->name, dm_attr.length, m_p_dm_mr->handle, m_p_dm_mr->lkey);
Packit 6d2c1b
Packit 6d2c1b
	return true;
Packit 6d2c1b
}
Packit 6d2c1b
Packit 6d2c1b
/*
Packit 6d2c1b
 * Release dev_mem resources
Packit 6d2c1b
 */
Packit 6d2c1b
void dm_mgr::release_resources()
Packit 6d2c1b
{
Packit 6d2c1b
	if (m_p_dm_mr) {
Packit 6d2c1b
		if (ibv_dereg_mr(m_p_dm_mr)) {
Packit 6d2c1b
			dm_logerr("ibv_dereg_mr failed, %d %m", errno);
Packit 6d2c1b
		} else {
Packit 6d2c1b
			dm_logdbg("ibv_dereg_mr success");
Packit 6d2c1b
		}
Packit 6d2c1b
		m_p_dm_mr = NULL;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	if (m_p_ibv_dm) {
Packit 6d2c1b
		if (vma_ibv_free_dm(m_p_ibv_dm)) {
Packit 6d2c1b
			dm_logerr("ibv_free_dm failed %d %m", errno);
Packit 6d2c1b
		} else {
Packit 6d2c1b
			dm_logdbg("ibv_free_dm success");
Packit 6d2c1b
		}
Packit 6d2c1b
		m_p_ibv_dm = NULL;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	m_p_ring_stat = NULL;
Packit 6d2c1b
Packit 6d2c1b
	dm_logdbg("Device memory release completed!");
Packit 6d2c1b
}
Packit 6d2c1b
Packit 6d2c1b
/*
Packit 6d2c1b
 * Copy data into the On Device Memory buffer.
Packit 6d2c1b
 *
Packit 6d2c1b
 * On Device Memory buffer is implemented in a cycle way using two variables :
Packit 6d2c1b
 * m_head - index of the next offset to be written.
Packit 6d2c1b
 * m_used - amount of used bytes within the On Device Memory buffer (which also used to calculate the tail of the buffer).
Packit 6d2c1b
 *
Packit 6d2c1b
 * In order to maintain a proper order of allocation and release, we must distinguish between three possible cases:
Packit 6d2c1b
 *
Packit 6d2c1b
 * First case:
Packit 6d2c1b
 *   Free space exists in the beginning and in the end of the array.
Packit 6d2c1b
 *
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *   |    |XXXXXXXXXX|                           |
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *       tail     head
Packit 6d2c1b
 *
Packit 6d2c1b
 * Second case:
Packit 6d2c1b
 *   There is not enough free space at the end of the array.
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *   |                             |XXXXXXXXXX|  |
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *                                tail     head
Packit 6d2c1b
 *
Packit 6d2c1b
 *   In the case above, we will move the head to the beginning of the array.
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *   |                             |XXXXXXXXXXXXX|
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *   head                         tail
Packit 6d2c1b
 *
Packit 6d2c1b
 * Third case:
Packit 6d2c1b
 *   Free space exists in the middle of the array
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *   |XXXXXXXXXXXXXX|                     |XXXXXX|
Packit 6d2c1b
 *   |-------------------------------------------|
Packit 6d2c1b
 *                 head                 tail
Packit 6d2c1b
 *
Packit 6d2c1b
 * Due to hardware limitations:
Packit 6d2c1b
 * 1. Data should be written to 4bytes aligned addresses.
Packit 6d2c1b
 * 2. Data length should be aligned to 4bytes.
Packit 6d2c1b
 *
Packit 6d2c1b
 * Due to performance reasons:
Packit 6d2c1b
 *  1. Data should be written to a continuous memory area.
Packit 6d2c1b
 *  2. Data will be written to 8bytes aligned addresses.
Packit 6d2c1b
 */
Packit 6d2c1b
bool dm_mgr::copy_data(struct mlx5_wqe_data_seg* seg, uint8_t* src, uint32_t length, mem_buf_desc_t* buff)
Packit 6d2c1b
{
Packit 6d2c1b
	vma_ibv_memcpy_dm_attr memcpy_attr;
Packit 6d2c1b
	uint32_t length_aligned_8 = DM_ALIGN_SIZE(length, DM_MEMORY_MASK_8);
Packit 6d2c1b
	size_t continuous_left = 0;
Packit 6d2c1b
	size_t &dev_mem_length = buff->tx.dev_mem_length = 0;
Packit 6d2c1b
Packit 6d2c1b
	// Check if On Device Memory buffer is full
Packit 6d2c1b
	if (m_used >= m_allocation) {
Packit 6d2c1b
		goto dev_mem_oob;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	// Check for a continuous space to write
Packit 6d2c1b
	if (m_head >= m_used) {	// First case
Packit 6d2c1b
		if ((continuous_left = m_allocation - m_head) < length_aligned_8) {	// Second case
Packit 6d2c1b
			if (m_head - m_used >= length_aligned_8) {
Packit 6d2c1b
				// There is enough space at the beginning of the buffer.
Packit 6d2c1b
				m_head  = 0;
Packit 6d2c1b
				dev_mem_length = continuous_left;
Packit 6d2c1b
			} else {
Packit 6d2c1b
				// There no enough space at the beginning of the buffer.
Packit 6d2c1b
				goto dev_mem_oob;
Packit 6d2c1b
			}
Packit 6d2c1b
		}
Packit 6d2c1b
	} else if ((continuous_left = m_allocation - m_used) < length_aligned_8) {	// Third case
Packit 6d2c1b
		goto dev_mem_oob;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	// Initialize memcopy attributes
Packit 6d2c1b
	memset(&memcpy_attr, 0, sizeof(memcpy_attr));
Packit 6d2c1b
	vma_ibv_init_memcpy_dm(memcpy_attr, src, m_head, length_aligned_8);
Packit 6d2c1b
Packit 6d2c1b
	// Copy data into the On Device Memory buffer.
Packit 6d2c1b
	if (vma_ibv_memcpy_dm(m_p_ibv_dm, &memcpy_attr)) {
Packit 6d2c1b
		dm_logfunc("Failed to memcopy data into the memic buffer %m");
Packit 6d2c1b
		return false;
Packit 6d2c1b
	}
Packit 6d2c1b
Packit 6d2c1b
	// Update values
Packit 6d2c1b
	seg->lkey = htonl(m_p_dm_mr->lkey);
Packit 6d2c1b
	seg->addr = htonll(m_head);
Packit 6d2c1b
	m_head = (m_head + length_aligned_8) % m_allocation;
Packit 6d2c1b
	dev_mem_length += length_aligned_8;
Packit 6d2c1b
	m_used += dev_mem_length;
Packit 6d2c1b
Packit 6d2c1b
	// Update On Device Memory statistics
Packit 6d2c1b
	m_p_ring_stat->simple.n_tx_dev_mem_pkt_count++;
Packit 6d2c1b
	m_p_ring_stat->simple.n_tx_dev_mem_byte_count += length;
Packit 6d2c1b
Packit 6d2c1b
	dm_logfunc("Send completed successfully! Buffer[%p] length[%d] length_aligned_8[%d] continuous_left[%zu] head[%zu] used[%zu]",
Packit 6d2c1b
			buff, length, length_aligned_8, continuous_left, m_head, m_used);
Packit 6d2c1b
Packit 6d2c1b
	return true;
Packit 6d2c1b
Packit 6d2c1b
dev_mem_oob:
Packit 6d2c1b
	dm_logfunc("Send OOB! Buffer[%p] length[%d] length_aligned_8[%d] continuous_left[%zu] head[%zu] used[%zu]",
Packit 6d2c1b
			buff, length, length_aligned_8, continuous_left, m_head, m_used);
Packit 6d2c1b
Packit 6d2c1b
	m_p_ring_stat->simple.n_tx_dev_mem_oob++;
Packit 6d2c1b
Packit 6d2c1b
	return false;
Packit 6d2c1b
}
Packit 6d2c1b
Packit 6d2c1b
/*
Packit 6d2c1b
 * Release On Device Memory buffer.
Packit 6d2c1b
 * This method should be called after completion was received.
Packit 6d2c1b
 */
Packit 6d2c1b
void dm_mgr::release_data(mem_buf_desc_t* buff)
Packit 6d2c1b
{
Packit 6d2c1b
	m_used -= buff->tx.dev_mem_length;
Packit 6d2c1b
	buff->tx.dev_mem_length = 0;
Packit 6d2c1b
Packit 6d2c1b
	dm_logfunc("Device memory release! buffer[%p] buffer_dev_mem_length[%zu] head[%zu] used[%zu]",
Packit 6d2c1b
			buff, buff->tx.dev_mem_length, m_head, m_used);
Packit 6d2c1b
Packit 6d2c1b
}
Packit 6d2c1b
Packit 6d2c1b
#endif /* DEFINED_IBV_DM */
Packit 6d2c1b
#endif /* DEFINED_DIRECT_VERBS */