Merge pull request PaddlePaddle#97 from shengxiangwang/xpu/all_reduce

add extra all_reduce for xpu

Author: nemonameless

ppdet/engine/trainer.py

@@ -422,6 +422,12 @@ def train(self, validate=False):
                     # model forward
                     outputs = model(data)
                     loss = outputs['loss']
+
+                    # avoid some all_reduce timeout due to computation progress differs between xpu cards
+                    if self._nranks > 1 and self.cfg.use_xpu:
+                        tensor_for_all_reduce = paddle.to_tensor(1.0)
+                        paddle.distributed.all_reduce(tensor_for_all_reduce)
+
                     # model backward
                     loss.backward()
                     self.optimizer.step()