part-2 cherry from: [Distributed] Opt nccl connection by lazy initial…

…ization (PaddlePaddle#55005)

python/paddle/distributed/fleet/base/topology.py

@@ -212,6 +212,19 @@ def __init__(self, topology):
             group=self._pp_comm_group,
         )
 
+        # create comm group for pipe parallel
+        self._pp_group, self._pp_comm_group = self._set_comm_group("pipe")
+        # NOTE(shenliang03): In pipeline parallel, we use batch_isend_irecv.
+        # if batch_isend_irecv is the first collective operation, all ranks of
+        # the pipeline group must participate in this call. In order to avoid
+        # this situation, we perform a collective communication in advance and
+        # create a communicator.
+        paddle.distributed.all_reduce(
+            paddle.zeros([1], dtype="int32"),
+            op=paddle.distributed.ReduceOp.SUM,
+            group=self._pp_comm_group,
+        )
+
         # create comm group for data parallel
         self._dp_group, self._dp_comm_group = self._set_comm_group("data")