UCT/IB/MLX5: Track RC send PSN with path MTU #11821
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -452,6 +452,35 @@ uct_rc_mlx5_ep_fm_cq_update(uct_rc_mlx5_iface_common_t *iface, | |
| return fm_ce_se; | ||
|
roiedanino marked this conversation as resolved.
|
||
| } | ||
|
|
||
| static UCS_F_ALWAYS_INLINE uint32_t | ||
| uct_rc_mlx5_num_packets(const uct_ib_mlx5_txwq_t *txwq, | ||
| size_t message_length) | ||
| { | ||
| ucs_assert(txwq->path_mtu_shift > 0); | ||
|
roiedanino marked this conversation as resolved.
|
||
|
|
||
| return (message_length + txwq->path_mtu_mask) >> txwq->path_mtu_shift; | ||
| } | ||
|
|
||
| static UCS_F_ALWAYS_INLINE void | ||
| uct_rc_mlx5_txwq_add_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, | ||
| uint32_t num_packets) | ||
| { | ||
| if (qp_type == IBV_QPT_RC) { | ||
| txwq->next_wqe_psn += num_packets; | ||
| } | ||
| } | ||
|
|
||
| static UCS_F_ALWAYS_INLINE void | ||
| uct_rc_mlx5_txwq_update_psn(uct_ib_mlx5_txwq_t *txwq, int qp_type, | ||
| size_t message_length) | ||
| { | ||
| if (qp_type == IBV_QPT_RC) { | ||
| uct_rc_mlx5_txwq_add_psn(txwq, qp_type, | ||
| uct_rc_mlx5_num_packets(txwq, | ||
| message_length)); | ||
| } | ||
| } | ||
|
evgeny-leksikov marked this conversation as resolved.
|
||
|
|
||
| static UCS_F_ALWAYS_INLINE void | ||
| uct_rc_mlx5_common_post_send(uct_rc_mlx5_iface_common_t *iface, int qp_type, | ||
| uct_rc_txqp_t *txqp, uct_ib_mlx5_txwq_t *txwq, | ||
|
|
@@ -532,6 +561,7 @@ static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txqp_inline_iov_post( | |
| uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, MLX5_OPCODE_SEND, | ||
| 0, fm_ce_se, dci_channel, wqe_size, 0, | ||
| INT_MAX, NULL); | ||
| uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); | ||
| } | ||
|
|
||
| /* | ||
|
|
@@ -627,6 +657,10 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, | |
| uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0, fm_ce_se, | ||
| dci_channel, wqe_size, imm_val_be, | ||
| max_log_sge, NULL); | ||
| if (opcode != MLX5_OPCODE_NOP) { | ||
| /* Inline short operations always fit in one RC packet. */ | ||
| uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); | ||
| } | ||
| } | ||
|
|
||
| /* | ||
|
|
@@ -707,6 +741,7 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, | |
| /* Data segment */ | ||
| if (length == 0) { | ||
| wqe_size = ctrl_av_size + sizeof(*raddr); | ||
| uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); | ||
| } else { | ||
| /* dptr cannot wrap, because ctrl+av could be either 2 or 4 segs */ | ||
| dptr = uct_ib_mlx5_txwq_wrap_none(txwq, raddr + 1); | ||
|
|
@@ -810,6 +845,7 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, | |
| (opcode_flags & UCT_RC_MLX5_OPCODE_MASK), opmod, | ||
| fm_ce_se, dci_channel, wqe_size, imm_val_be, | ||
| max_log_sge, log_sge); | ||
| uct_rc_mlx5_txwq_update_psn(txwq, qp_type, length); | ||
| } | ||
|
|
||
| static UCS_F_ALWAYS_INLINE | ||
|
|
@@ -830,6 +866,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty | |
| struct mlx5_wqe_inl_data_seg *inl; | ||
| uct_rc_mlx5_hdr_t *rch; | ||
| unsigned wqe_size, inl_seg_size, ctrl_av_size; | ||
| size_t iov_length, message_length; | ||
| void *next_seg; | ||
| uint8_t opmod; | ||
| #if HAVE_MLX5_MMO | ||
|
|
@@ -852,9 +889,6 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty | |
| inl_seg_size = ucs_align_up_pow2(sizeof(*inl) + sizeof(*rch) + am_hdr_len, | ||
| UCT_IB_MLX5_WQE_SEG_SIZE); | ||
|
|
||
| /* Inline segment with AM ID and header */ | ||
| inl = next_seg; | ||
| inl->byte_count = htonl((sizeof(*rch) + am_hdr_len) | MLX5_INLINE_SEG); | ||
|
|
@@ -866,9 +900,12 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty | |
| /* Data segment with payload */ | ||
| dptr = (struct mlx5_wqe_data_seg *)((char *)inl + inl_seg_size); | ||
| wqe_size = ctrl_av_size + inl_seg_size + | ||
| uct_ib_mlx5_set_data_seg_iov( | ||
| txwq, dptr, iov, iovcnt, &iov_length); | ||
| opmod = 0; | ||
| message_length = iov_length + sizeof(*rch) + am_hdr_len; | ||
|
|
||
| ucs_assert(message_length <= iface->super.super.config.seg_size); | ||
| ucs_assert(wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE); | ||
| break; | ||
|
|
||
|
|
@@ -881,8 +918,10 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty | |
| inl->byte_count = htonl(sizeof(struct ibv_tmh) | MLX5_INLINE_SEG); | ||
| dptr = uct_ib_mlx5_txwq_wrap_exact(txwq, (char *)inl + inl_seg_size); | ||
| wqe_size = ctrl_av_size + inl_seg_size + | ||
| uct_ib_mlx5_set_data_seg_iov( | ||
| txwq, dptr, iov, iovcnt, &iov_length); | ||
| opmod = 0; | ||
| message_length = iov_length + sizeof(struct ibv_tmh); | ||
|
|
||
| uct_rc_mlx5_fill_tmh((struct ibv_tmh*)(inl + 1), tag, app_ctx, | ||
| IBV_TMH_EAGER); | ||
|
|
@@ -895,16 +934,16 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty | |
| /* Fall through */ | ||
| case MLX5_OPCODE_RDMA_WRITE: | ||
| /* Set RDMA segment */ | ||
| raddr = next_seg; | ||
| uct_ib_mlx5_ep_set_rdma_seg(raddr, remote_addr, rkey); | ||
|
|
||
| /* Data segment */ | ||
| wqe_size = ctrl_av_size + sizeof(*raddr) + | ||
| uct_ib_mlx5_set_data_seg_iov( | ||
| txwq, (void*)(raddr + 1), iov, iovcnt, | ||
| &message_length); | ||
| opmod = 0; | ||
| ucs_assert(message_length <= UCT_IB_MAX_MESSAGE_SIZE); | ||
| break; | ||
|
|
||
| #if HAVE_MLX5_MMO | ||
|
|
@@ -938,8 +977,10 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty | |
| uct_ib_mlx5_set_data_seg(dptr /* scatter segment */, | ||
| dma_dst_buf, iov[0].length, dma_dst_key); | ||
|
|
||
| wqe_size = sizeof(*ctrl) + sizeof(*dma_seg) + | ||
| (2 * sizeof(*dptr)); | ||
| opmod = UCT_IB_MLX5_OPMOD_MMO_DMA; | ||
| message_length = 0; | ||
| break; | ||
| #endif | ||
| default: | ||
|
|
@@ -950,6 +991,18 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty | |
| opcode_flags & UCT_RC_MLX5_OPCODE_MASK, opmod, | ||
| fm_ce_se, dci_channel, wqe_size, ib_imm_be, | ||
| max_log_sge, NULL); | ||
| #if HAVE_MLX5_MMO | ||
| if ((opcode_flags & UCT_RC_MLX5_OPCODE_MASK) == MLX5_OPCODE_MMO) { | ||
| return; | ||
| } | ||
| #endif | ||
|
evgeny-leksikov marked this conversation as resolved.
|
||
|
|
||
| uct_rc_mlx5_txwq_update_psn(txwq, qp_type, message_length); | ||
| if (opcode_flags == MLX5_OPCODE_RDMA_WRITE) { | ||
|
There was a problem hiding this comment. Zero-length RDMA_READ with message_length==0 would get update_psn(0)=0 PSN added because the special case only checks RDMA_WRITE, not RDMA_READ. This doesn't affect real paths (get_zcopy has a min length check and zero-length reads via dptr_post_iov don't occur), but it is a slight inconsistency with the dptr_post (bcopy) path which adds 1 for both READ and WRITE on the zero-length case. |
||
| /* opcode_flags is constant after inlining, so only zero-length | ||
| * PUT_ZCOPY gets the branchless one-PSN adjustment. */ | ||
| uct_rc_mlx5_txwq_add_psn(txwq, qp_type, !message_length); | ||
| } | ||
| } | ||
|
|
||
| /* | ||
|
|
@@ -1099,6 +1152,8 @@ uct_rc_mlx5_txqp_tag_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type, | |
| uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0, | ||
| fm_ce_se, dci_channel, wqe_size, imm_val_be, | ||
| INT_MAX, NULL); | ||
| /* Inline tag operations always fit in one RC packet. */ | ||
| uct_rc_mlx5_txwq_add_psn(txwq, qp_type, 1); | ||
| } | ||
|
|
||
| static UCS_F_ALWAYS_INLINE void | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -212,6 +212,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, | |
| uct_ib_mlx5_txwq_t *txwq = &ep->tx.wq; | ||
|
There was a problem hiding this comment. The DM short paths (
Contributor
Author
There was a problem hiding this comment. The DM path is already covered indirectly through uct_rc_mlx5_common_txqp_bcopy_post() -> uct_rc_mlx5_txqp_dptr_post(), which updates PSN using hdr_len + iov_length. RC passes IBV_QPT_RC, while the DC update is compiled out, so an additional bump would double-count. |
||
| size_t total = 0; | ||
| struct mlx5_wqe_ctrl_seg *ctrl = NULL; | ||
| uint32_t num_packets = 0; | ||
| struct mlx5_wqe_raddr_seg *raddr; | ||
| struct mlx5_wqe_data_seg *dptr; | ||
| size_t wqe_size, i; | ||
|
|
@@ -288,7 +289,8 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, | |
| curr = UCS_PTR_BYTE_OFFSET(ctrl, MLX5_SEND_WQE_BB); | ||
| curr = uct_ib_mlx5_txwq_wrap_exact(txwq, curr); | ||
| pi++; | ||
| total += lengths[i]; | ||
| num_packets += uct_rc_mlx5_num_packets(txwq, lengths[i]); | ||
| } | ||
|
|
||
| res_count = pi - 1 - txwq->prev_sw_pi; | ||
|
|
@@ -299,6 +301,7 @@ uct_rc_mlx5_base_ep_put_sgl_zcopy(uct_ep_h tl_ep, void * const *buffers, | |
|
|
||
| uct_rc_txqp_posted(&ep->super.txqp, &iface->super, res_count, 1); | ||
| uct_ib_mlx5_txwq_ring_doorbell(txwq, ctrl, txwq->sw_pi, 1); | ||
| uct_rc_mlx5_txwq_add_psn(txwq, IBV_QPT_RC, num_packets); | ||
|
|
||
| uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, | ||
| uct_rc_ep_send_op_completion_handler, comp, sn, | ||
|
|
@@ -877,6 +880,18 @@ void uct_rc_mlx5_common_packet_dump(uct_base_iface_t *iface, uct_am_trace_type_t | |
| valid_length, buffer, max); | ||
| } | ||
|
|
||
| void uct_rc_mlx5_txwq_set_path_mtu(uct_ib_mlx5_txwq_t *txwq, | ||
|
There was a problem hiding this comment. minor — misaligned continuation line. The |
||
| enum ibv_mtu path_mtu) | ||
| { | ||
| size_t mtu = uct_ib_mtu_value(path_mtu); | ||
|
|
||
| ucs_assert(mtu <= UINT16_MAX); | ||
| ucs_assert(ucs_is_pow2(mtu)); | ||
|
|
||
| txwq->path_mtu_mask = mtu - 1; | ||
| txwq->path_mtu_shift = ucs_ilog2(mtu); | ||
| } | ||
|
|
||
| ucs_status_t | ||
| uct_rc_mlx5_ep_connect_qp(uct_rc_mlx5_iface_common_t *iface, | ||
| uct_ib_mlx5_qp_t *qp, uint32_t qp_num, | ||
|
|
@@ -974,6 +989,8 @@ uct_rc_mlx5_ep_connect_to_ep_v2(uct_ep_h tl_ep, | |
| return status; | ||
| } | ||
|
|
||
| uct_rc_mlx5_txwq_set_path_mtu(&ep->super.tx.wq, path_mtu); | ||
|
evgeny-leksikov marked this conversation as resolved.
|
||
|
|
||
| ep->super.super.atomic_mr_offset = uct_ib_md_atomic_offset( | ||
| rc_addr->atomic_mr_id); | ||
| ep->super.super.flags |= UCT_RC_EP_FLAG_CONNECTED; | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -320,9 +320,9 @@ static UCS_F_ALWAYS_INLINE ucs_status_t uct_ud_mlx5_ep_inline_iov_post( | |
| /* set iov to dptr */ | ||
| if (iovcnt > 0) { | ||
| wqe_size = ucs_align_up_pow2(wqe_size, UCT_IB_MLX5_WQE_SEG_SIZE); | ||
| wqe_size += uct_ib_mlx5_set_data_seg_iov( | ||
| &iface->tx.wq, UCS_PTR_BYTE_OFFSET(ctrl, wqe_size), iov, | ||
| iovcnt, NULL); | ||
| } | ||
|
|
||
| uct_ud_mlx5_post_send(iface, ep, 0, ctrl, wqe_size, neth, | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Non-blocking (previously noted): next_wqe_psn is stored unmasked as uint32_t while the accessor masks to 24 bits.