UCP: Tune Alltoall pairwise posting depth

src/components/tl/ucp/Makefile.am

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 
 if TL_UCP_ENABLED
@@ -30,6 +30,7 @@ alltoall =                       \
 	alltoall/alltoall.c          \
 	alltoall/alltoall_onesided.c \
 	alltoall/alltoall_pairwise.c \
+	alltoall/alltoall_pairwise_num_posts.h \
 	alltoall/alltoall_bruck.c
 
 alltoallv =                        \

src/components/tl/ucp/alltoall/alltoall_pairwise.c

@@ -1,20 +1,17 @@
 /**
- * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  *
  * See file LICENSE for terms.
  */
 
 #include "config.h"
 #include "tl_ucp.h"
 #include "alltoall.h"
+#include "alltoall_pairwise_num_posts.h"
 #include "core/ucc_progress_queue.h"
 #include "utils/ucc_math.h"
 #include "tl_ucp_sendrecv.h"
 
-/* TODO: add as parameters */
-#define MSG_MEDIUM 66000
-#define NP_THRESH 32
-
 static inline ucc_rank_t get_recv_peer(ucc_rank_t rank, ucc_rank_t size,
                                        ucc_rank_t step)
 {
@@ -32,18 +29,14 @@ static ucc_rank_t get_num_posts(const ucc_tl_ucp_team_t *team,
 {
     unsigned long posts = UCC_TL_UCP_TEAM_LIB(team)->cfg.alltoall_pairwise_num_posts;
     ucc_rank_t    tsize = UCC_TL_TEAM_SIZE(team);
-    size_t data_size;
+    size_t        dt_size, data_size, peer_size;
 
-    data_size = (size_t)args->src.info.count *
-                ucc_dt_size(args->src.info.datatype);
+    dt_size   = ucc_dt_size(args->src.info.datatype);
+    data_size = (size_t)args->src.info.count * dt_size;
+    peer_size = (size_t)(args->src.info.count / tsize) * dt_size;
     if (posts == UCC_ULUNITS_AUTO) {
-        if ((data_size > MSG_MEDIUM) && (tsize > NP_THRESH)) {
-            /* use pairwise algorithm */
-            posts = 1;
-        } else {
-            /* use linear algorithm */
-            posts = 0;
-        }
+        posts = ucc_tl_ucp_alltoall_pairwise_auto_num_posts(
+            tsize, data_size, peer_size);
     }
 
     posts = (posts > tsize || posts == 0) ? tsize: posts;

src/components/tl/ucp/alltoall/alltoall_pairwise_num_posts.h

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *
+ * See file LICENSE for terms.
+ */
+
+#ifndef ALLTOALL_PAIRWISE_NUM_POSTS_H_
+#define ALLTOALL_PAIRWISE_NUM_POSTS_H_
+
+#include "utils/ucc_math.h"
+
+#define UCC_TL_UCP_ALLTOALL_TOTAL_SMALL 66000
+#define UCC_TL_UCP_ALLTOALL_PEER_64K    (64 * 1024)
+#define UCC_TL_UCP_ALLTOALL_PEER_1M     (1 * 1024 * 1024)
+#define UCC_TL_UCP_ALLTOALL_PEER_4M     (4 * 1024 * 1024)
+#define UCC_TL_UCP_ALLTOALL_PEER_8M     (8 * 1024 * 1024)
+
+/*
+ * Preserve the existing tiny-message and small-team behavior, then use coarse,
+ * benchmark-informed per-peer bands to limit outstanding sends and receives.
+ */
+static inline ucc_rank_t
+ucc_tl_ucp_alltoall_pairwise_auto_num_posts(ucc_rank_t tsize,
+                                             size_t total_size,
+                                             size_t peer_size)
+{
+    if (total_size <= UCC_TL_UCP_ALLTOALL_TOTAL_SMALL || tsize <= 8) {
+        return tsize;
+    }
+
+    if (peer_size <= UCC_TL_UCP_ALLTOALL_PEER_64K) {
+        return ucc_min(tsize, 32);
+    }
+
+    if (peer_size <= UCC_TL_UCP_ALLTOALL_PEER_1M) {
+        return ucc_min(tsize, 16);
+    }
+
+    if (peer_size <= UCC_TL_UCP_ALLTOALL_PEER_4M) {
+        return tsize <= 16 ? tsize : 8;
+    }
+
+    if (peer_size <= UCC_TL_UCP_ALLTOALL_PEER_8M) {
+        return ucc_min(tsize, 4);
+    }
+
+    return tsize <= 32 ? 4 : 1;
+}
+
+#endif

test/gtest/Makefile.am

@@ -101,6 +101,7 @@ gtest_SOURCES =                           \
 	utils/test_string.cc                  \
 	utils/test_ep_map.cc                  \
 	utils/test_lock_free_queue.cc         \
+	utils/test_alltoall_pairwise_num_posts.cc \
 	utils/test_math.cc                    \
 	utils/test_cfg_file.cc                \
 	utils/test_parser.cc                  \

test/gtest/utils/test_alltoall_pairwise_num_posts.cc

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *
+ * See file LICENSE for terms.
+ */
+
+#include "common/gtest.h"
+#include "components/tl/ucp/alltoall/alltoall_pairwise_num_posts.h"
+
+TEST(alltoall_pairwise_num_posts, automatic_boundaries)
+{
+    struct test_case {
+        ucc_rank_t tsize;
+        size_t     total_size;
+        size_t     peer_size;
+        ucc_rank_t expected;
+    };
+    const test_case cases[] = {
+        {64, 66000, 66000 / 64, 64},
+        {64, 66001, 66001 / 64, 32},
+        {64, 64 * 64 * 1024, 64 * 1024, 32},
+        {64, 64 * (64 * 1024 + 1), 64 * 1024 + 1, 16},
+        {64, 64 * 1024 * 1024, 1024 * 1024, 16},
+        {16, 16 * (1024 * 1024 + 1), 1024 * 1024 + 1, 16},
+        {17, 17 * (1024 * 1024 + 1), 1024 * 1024 + 1, 8},
+        {64, 64 * (4 * 1024 * 1024), 4 * 1024 * 1024, 8},
+        {64, 64 * (4 * 1024 * 1024 + 1), 4 * 1024 * 1024 + 1, 4},
+        {64, 64 * (8 * 1024 * 1024), 8 * 1024 * 1024, 4},
+        {8, 8 * (4 * 1024 * 1024 + 1), 4 * 1024 * 1024 + 1, 8},
+        {9, 9 * (4 * 1024 * 1024 + 1), 4 * 1024 * 1024 + 1, 4},
+        {8, 8 * (8 * 1024 * 1024 + 1), 8 * 1024 * 1024 + 1, 8},
+        {9, 9 * (8 * 1024 * 1024 + 1), 8 * 1024 * 1024 + 1, 4},
+        {32, 32 * (8 * 1024 * 1024 + 1), 8 * 1024 * 1024 + 1, 4},
+        {33, 33 * (8 * 1024 * 1024 + 1), 8 * 1024 * 1024 + 1, 1},
+    };
+
+    for (const auto &c : cases) {
+        EXPECT_EQ(c.expected, ucc_tl_ucp_alltoall_pairwise_auto_num_posts(
+                                  c.tsize, c.total_size, c.peer_size));
+    }
+}
+
+TEST(alltoall_pairwise_num_posts, valid_and_nonincreasing)
+{
+    const size_t peer_sizes[] = {
+        0,
+        1,
+        64 * 1024,
+        64 * 1024 + 1,
+        1024 * 1024,
+        1024 * 1024 + 1,
+        4 * 1024 * 1024,
+        4 * 1024 * 1024 + 1,
+        8 * 1024 * 1024,
+        8 * 1024 * 1024 + 1,
+    };
+
+    for (ucc_rank_t tsize = 1; tsize <= 128; tsize++) {
+        ucc_rank_t previous = tsize;
+
+        for (size_t peer_size : peer_sizes) {
+            ucc_rank_t posts = ucc_tl_ucp_alltoall_pairwise_auto_num_posts(
+                tsize, (size_t)tsize * peer_size, peer_size);
+
+            EXPECT_GE(posts, 1);
+            EXPECT_LE(posts, tsize);
+            EXPECT_LE(posts, previous)
+                << "team size " << tsize << ", peer size " << peer_size;
+            previous = posts;
+        }
+    }
+}