.\" Automatically generated by Pandoc 1.19.2.4
.\"
.TH "MLX5DV_WR" "3" "2019\-02\-24" "mlx5" "mlx5 Programmer\[aq]s Manual"
.hy
.SH NAME
.PP
mlx5dv_wr_set_dc_addr \- Attach a DC info to the last work request
.SH SYNOPSIS
.IP
.nf
\f[C]
#include\ <infiniband/mlx5dv.h>

static\ inline\ void\ mlx5dv_wr_set_dc_addr(struct\ mlx5dv_qp_ex\ *mqp,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ struct\ ibv_ah\ *ah,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint32_t\ remote_dctn,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint64_t\ remote_dc_key);

struct\ mlx5dv_mr_interleaved\ {
\ \ \ \ uint64_t\ \ \ \ \ \ \ \ addr;
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ bytes_count;
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ bytes_skip;
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ lkey;
};

static\ inline\ void\ mlx5dv_wr_mr_interleaved(struct\ mlx5dv_qp_ex\ *mqp,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ struct\ mlx5dv_mkey\ *mkey,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint32_t\ access_flags,\ /*\ use\ enum\ ibv_access_flags\ */
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint32_t\ repeat_count,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint16_t\ num_interleaved,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ struct\ mlx5dv_mr_interleaved\ *data);

static\ inline\ void\ mlx5dv_wr_mr_list(struct\ mlx5dv_qp_ex\ *mqp,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ struct\ mlx5dv_mkey\ *mkey,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint32_t\ access_flags,\ /*\ use\ enum\ ibv_access_flags\ */
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint16_t\ num_sges,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ struct\ ibv_sge\ *sge);
\f[]
.fi
.SH DESCRIPTION
.PP
The MLX5DV work request APIs (mlx5dv_wr_*) is an extension for IBV work
request API (ibv_wr_*) with mlx5 specific features for send work
request.
This may be used together with or without ibv_wr_* calls.
.SH USAGE
.PP
To use these APIs a QP must be created using mlx5dv_create_qp() with
\f[I]send_ops_flags\f[] of struct ibv_qp_init_attr_ex set.
.PP
If the QP does not support all the requested work request types then QP
creation will fail.
.PP
The mlx5dv_qp_ex is extracted from the IBV_QP by ibv_qp_to_qp_ex() and
mlx5dv_qp_ex_from_ibv_qp_ex().
This should be used to apply the mlx5 specific features on the posted
WR.
.PP
A work request creation requires to use the ibv_qp_ex as described in
the man for ibv_wr_post and mlx5dv_qp with its available builders and
setters.
.SS QP Specific builders
.TP
.B \f[I]RC\f[] QPs
\f[I]mlx5dv_wr_mr_interleaved()\f[]
.RS
.PP
registers an interleaved memory layout by using an indirect mkey and
some interleaved data.
The layout of the memory pointed by the mkey after its registration will
be the \f[I]data\f[] representation for the \f[I]num_interleaved\f[]
entries.
This single layout representation is repeated by \f[I]repeat_count\f[].
.PP
The \f[I]data\f[] as described by struct mlx5dv_mr_interleaved will hold
real data defined by \f[I]bytes_count\f[] and then a padding of
\f[I]bytes_skip\f[].
Post a successful registration, RDMA operations can use this
\f[I]mkey\f[].
The hardware will scatter the data according to the pattern.
The \f[I]mkey\f[] should be used in a zero\-based mode.
The \f[I]addr\f[] field in its \f[I]ibv_sge\f[] is an offset in the
total data.
To create this \f[I]mkey\f[] mlx5dv_create_mkey() should be used.
.PP
Current implementation requires the IBV_SEND_INLINE option to be on in
\f[I]ibv_qp_ex\->wr_flags\f[] field.
To be able to have more than 3 \f[I]num_interleaved\f[] entries, the QP
should be created with a larger WQE size that may fit it.
This should be done using the \f[I]max_inline_data\f[] attribute of
\f[I]struct ibv_qp_cap\f[] upon its creation.
.PP
As one entry will be consumed for strided header, the \f[I]mkey\f[]
should be created with one more entry than the required
\f[I]num_interleaved\f[].
.PP
In case \f[I]ibv_qp_ex\->wr_flags\f[] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_UMR.
Unregister the \f[I]mkey\f[] to enable another pattern registration
should be done via ibv_post_send with IBV_WR_LOCAL_INV opcode.
.RE
\f[I]mlx5dv_wr_mr_list()\f[]
.RS
.PP
registers a memory layout based on list of ibv_sge.
The layout of the memory pointed by the \f[I]mkey\f[] after its
registration will be based on the list of \f[I]sge\f[] counted by
\f[I]num_sges\f[].
Post a successful registration RDMA operations can use this
\f[I]mkey\f[], the hardware will scatter the data according to the
pattern.
The \f[I]mkey\f[] should be used in a zero\-based mode, the
\f[I]addr\f[] field in its \f[I]ibv_sge\f[] is an offset in the total
data.
.PP
Current implementation requires the IBV_SEND_INLINE option to be on in
\f[I]ibv_qp_ex\->wr_flags\f[] field.
To be able to have more than 4 \f[I]num_sge\f[] entries, the QP should
be created with a larger WQE size that may fit it.
This should be done using the \f[I]max_inline_data\f[] attribute of
\f[I]struct ibv_qp_cap\f[] upon its creation.
.PP
In case \f[I]ibv_qp_ex\->wr_flags\f[] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_UMR.
Unregister the \f[I]mkey\f[] to enable other pattern registration should
be done via ibv_post_send with IBV_WR_LOCAL_INV opcode.
.RE
.SS QP Specific setters
.TP
.B \f[I]DCI\f[] QPs
\f[I]mlx5dv_wr_set_dc_addr()\f[] must be called to set the DCI WR
properties.
The destination address of the work is specified by \f[I]ah\f[], the
remote DCT number is specified by \f[I]remote_dctn\f[] and the DC key is
specified by \f[I]remote_dc_key\f[].
This setter is available when the QP transport is DCI and send_ops_flags
in struct ibv_qp_init_attr_ex is set.
The available builders and setters for DCI QP are the same as RC QP.
.RS
.RE
.SH EXAMPLE
.IP
.nf
\f[C]
/*\ create\ DC\ QP\ type\ and\ specify\ the\ required\ send\ opcodes\ */
attr_ex.qp_type\ =\ IBV_QPT_DRIVER;
attr_ex.comp_mask\ |=\ IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
attr_ex.send_ops_flags\ |=\ IBV_QP_EX_WITH_RDMA_WRITE;

attr_dv.comp_mask\ |=\ MLX5DV_QP_INIT_ATTR_MASK_DC;
attr_dv.dc_init_attr.dc_type\ =\ MLX5DV_DCTYPE_DCI;

ibv_qp\ *qp\ =\ mlx5dv_create_qp(ctx,\ attr_ex,\ attr_dv);
ibv_qp_ex\ *qpx\ =\ ibv_qp_to_qp_ex(qp);
mlx5dv_qp_ex\ *mqpx\ =\ mlx5dv_qp_ex_from_ibv_qp_ex(qpx);

ibv_wr_start(qpx);

/*\ Use\ ibv_qp_ex\ object\ to\ set\ WR\ generic\ attributes\ */
qpx\->wr_id\ =\ my_wr_id_1;
qpx\->wr_flags\ =\ IBV_SEND_SIGNALED;
ibv_wr_rdma_write(qpx,\ rkey,\ remote_addr_1);
ibv_wr_set_sge(qpx,\ lkey,\ local_addr_1,\ length_1);

/*\ Use\ mlx5\ DC\ setter\ using\ mlx5dv_qp_ex\ object\ */
mlx5dv_wr_set_wr_dc_addr(mqpx,\ ah,\ remote_dctn,\ remote_dc_key);

ret\ =\ ibv_wr_complete(qpx);
\f[]
.fi
.SH SEE ALSO
.PP
\f[B]ibv_post_send\f[](3), \f[B]ibv_create_qp_ex(3)\f[],
\f[B]ibv_wr_post(3)\f[], \f[B]mlx5dv_create_mkey(3)\f[].
.SH AUTHOR
.PP
Guy Levi <guyle@mellanox.com>