/*
 * Copyright (c) 2001-2020 Mellanox Technologies, Ltd. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "dm_mgr.h"
#include "vlogger/vlogger.h"
#include "vma/proto/mem_buf_desc.h"
#include "vma/dev/ib_ctx_handler.h"

#if defined(DEFINED_DIRECT_VERBS)
#if defined(DEFINED_IBV_DM)

#define DM_MEMORY_MASK_8  7
#define DM_MEMORY_MASK_64 63
#define DM_ALIGN_SIZE(size, mask) ((size + mask) & (~mask))

#undef  MODULE_NAME
#define MODULE_NAME 		"dm_mgr"
#undef  MODULE_HDR
#define MODULE_HDR MODULE_NAME "%d:%s() "

#define dm_logerr	__log_info_err
#define dm_logwarn	__log_info_warn
#define dm_logdbg	__log_info_dbg
#define dm_logfunc	__log_info_func

dm_mgr::dm_mgr() :
	m_p_dm_mr(NULL),
	m_p_ibv_dm(NULL),
	m_p_ring_stat(NULL),
	m_allocation(0),
	m_used(0),
	m_head(0)
{};

/*
 * Allocate dev_mem resources
 */
bool dm_mgr::allocate_resources(ib_ctx_handler* ib_ctx, ring_stats_t* ring_stats)
{
	size_t allocation_size = DM_ALIGN_SIZE(safe_mce_sys().ring_dev_mem_tx, DM_MEMORY_MASK_64);
	vma_ibv_alloc_dm_attr dm_attr;
	vma_ibv_reg_mr_in mr_in;
	m_p_ring_stat = ring_stats;
	if (!allocation_size) {
		// On Device Memory usage was disabled by the user
		return false;
	}

	if (!ib_ctx->get_on_device_memory_size()) {
		// On Device Memory usage is not supported
		return false;
	}

	// Allocate on device memory buffer
	memset(&dm_attr, 0, sizeof(dm_attr));
	dm_attr.length = allocation_size;
	m_p_ibv_dm = vma_ibv_alloc_dm(ib_ctx->get_ibv_context(), &dm_attr);
	if (!m_p_ibv_dm) {
		// Memory allocation can fail if we have already allocated the maximum possible.
		dm_logdbg("ibv_alloc_dm error - On Device Memory allocation failed, %d %m", errno);
		errno = 0;
		return false;
	}

	// Initialize MR attributes
	memset(&mr_in, 0, sizeof(mr_in));
	vma_ibv_init_dm_mr(mr_in, ib_ctx->get_ibv_pd(), allocation_size, m_p_ibv_dm);

	// Register On Device Memory MR
	m_p_dm_mr = vma_ibv_reg_dm_mr(&mr_in);
	if (!m_p_dm_mr) {
		vma_ibv_free_dm(m_p_ibv_dm);
		m_p_ibv_dm = NULL;
		dm_logerr("ibv_free_dm error - dm_mr registration failed, %d %m", errno);
		return false;
	}

	m_allocation = allocation_size;
	m_p_ring_stat->simple.n_tx_dev_mem_allocated = m_allocation;

	dm_logdbg("Device memory allocation completed successfully! device[%s] bytes[%zu] dm_mr handle[%d] dm_mr lkey[%d]",
			ib_ctx->get_ibv_device()->name, dm_attr.length, m_p_dm_mr->handle, m_p_dm_mr->lkey);

	return true;
}

/*
 * Release dev_mem resources
 */
void dm_mgr::release_resources()
{
	if (m_p_dm_mr) {
		if (ibv_dereg_mr(m_p_dm_mr)) {
			dm_logerr("ibv_dereg_mr failed, %d %m", errno);
		} else {
			dm_logdbg("ibv_dereg_mr success");
		}
		m_p_dm_mr = NULL;
	}

	if (m_p_ibv_dm) {
		if (vma_ibv_free_dm(m_p_ibv_dm)) {
			dm_logerr("ibv_free_dm failed %d %m", errno);
		} else {
			dm_logdbg("ibv_free_dm success");
		}
		m_p_ibv_dm = NULL;
	}

	m_p_ring_stat = NULL;

	dm_logdbg("Device memory release completed!");
}

/*
 * Copy data into the On Device Memory buffer.
 *
 * On Device Memory buffer is implemented in a cycle way using two variables :
 * m_head - index of the next offset to be written.
 * m_used - amount of used bytes within the On Device Memory buffer (which also used to calculate the tail of the buffer).
 *
 * In order to maintain a proper order of allocation and release, we must distinguish between three possible cases:
 *
 * First case:
 *   Free space exists in the beginning and in the end of the array.
 *
 *   |-------------------------------------------|
 *   |    |XXXXXXXXXX|                           |
 *   |-------------------------------------------|
 *       tail     head
 *
 * Second case:
 *   There is not enough free space at the end of the array.
 *   |-------------------------------------------|
 *   |                             |XXXXXXXXXX|  |
 *   |-------------------------------------------|
 *                                tail     head
 *
 *   In the case above, we will move the head to the beginning of the array.
 *   |-------------------------------------------|
 *   |                             |XXXXXXXXXXXXX|
 *   |-------------------------------------------|
 *   head                         tail
 *
 * Third case:
 *   Free space exists in the middle of the array
 *   |-------------------------------------------|
 *   |XXXXXXXXXXXXXX|                     |XXXXXX|
 *   |-------------------------------------------|
 *                 head                 tail
 *
 * Due to hardware limitations:
 * 1. Data should be written to 4bytes aligned addresses.
 * 2. Data length should be aligned to 4bytes.
 *
 * Due to performance reasons:
 *  1. Data should be written to a continuous memory area.
 *  2. Data will be written to 8bytes aligned addresses.
 */
bool dm_mgr::copy_data(struct mlx5_wqe_data_seg* seg, uint8_t* src, uint32_t length, mem_buf_desc_t* buff)
{
	vma_ibv_memcpy_dm_attr memcpy_attr;
	uint32_t length_aligned_8 = DM_ALIGN_SIZE(length, DM_MEMORY_MASK_8);
	size_t continuous_left = 0;
	size_t &dev_mem_length = buff->tx.dev_mem_length = 0;

	// Check if On Device Memory buffer is full
	if (m_used >= m_allocation) {
		goto dev_mem_oob;
	}

	// Check for a continuous space to write
	if (m_head >= m_used) {	// First case
		if ((continuous_left = m_allocation - m_head) < length_aligned_8) {	// Second case
			if (m_head - m_used >= length_aligned_8) {
				// There is enough space at the beginning of the buffer.
				m_head  = 0;
				dev_mem_length = continuous_left;
			} else {
				// There no enough space at the beginning of the buffer.
				goto dev_mem_oob;
			}
		}
	} else if ((continuous_left = m_allocation - m_used) < length_aligned_8) {	// Third case
		goto dev_mem_oob;
	}

	// Initialize memcopy attributes
	memset(&memcpy_attr, 0, sizeof(memcpy_attr));
	vma_ibv_init_memcpy_dm(memcpy_attr, src, m_head, length_aligned_8);

	// Copy data into the On Device Memory buffer.
	if (vma_ibv_memcpy_dm(m_p_ibv_dm, &memcpy_attr)) {
		dm_logfunc("Failed to memcopy data into the memic buffer %m");
		return false;
	}

	// Update values
	seg->lkey = htonl(m_p_dm_mr->lkey);
	seg->addr = htonll(m_head);
	m_head = (m_head + length_aligned_8) % m_allocation;
	dev_mem_length += length_aligned_8;
	m_used += dev_mem_length;

	// Update On Device Memory statistics
	m_p_ring_stat->simple.n_tx_dev_mem_pkt_count++;
	m_p_ring_stat->simple.n_tx_dev_mem_byte_count += length;

	dm_logfunc("Send completed successfully! Buffer[%p] length[%d] length_aligned_8[%d] continuous_left[%zu] head[%zu] used[%zu]",
			buff, length, length_aligned_8, continuous_left, m_head, m_used);

	return true;

dev_mem_oob:
	dm_logfunc("Send OOB! Buffer[%p] length[%d] length_aligned_8[%d] continuous_left[%zu] head[%zu] used[%zu]",
			buff, length, length_aligned_8, continuous_left, m_head, m_used);

	m_p_ring_stat->simple.n_tx_dev_mem_oob++;

	return false;
}

/*
 * Release On Device Memory buffer.
 * This method should be called after completion was received.
 */
void dm_mgr::release_data(mem_buf_desc_t* buff)
{
	m_used -= buff->tx.dev_mem_length;
	buff->tx.dev_mem_length = 0;

	dm_logfunc("Device memory release! buffer[%p] buffer_dev_mem_length[%zu] head[%zu] used[%zu]",
			buff, buff->tx.dev_mem_length, m_head, m_used);

}

#endif /* DEFINED_IBV_DM */
#endif /* DEFINED_DIRECT_VERBS */