Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

… check_gpups_on_haokanctr
PaddlePaddle · Mar 7, 2023 · 68672eb · 68672eb
2 parents 8e3c362 + c2c1cfc
commit 68672eb
Show file tree

Hide file tree

Showing 530 changed files with 10,802 additions and 8,035 deletions.
diff --git a/.gitignore b/.gitignore
@@ -94,4 +94,6 @@ paddle/fluid/pybind/op_function_impl.h
 paddle/fluid/pybind/*final_state_op_function_impl.h
 paddle/fluid/prim/api/generated/prim_api/*
 paddle/fluid/framework/__init__.py
+paddle/phi/api/profiler/__init__.py
 python/paddle/incubate/fleet/parameter_server/pslib/ps_pb2.py
+python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -52,6 +52,7 @@ option(WITH_MPI "Compile PaddlePaddle with MPI" OFF)
 option(WITH_TENSORRT "Compile PaddlePaddle with NVIDIA TensorRT" OFF)
 option(WITH_XPU "Compile PaddlePaddle with BAIDU KUNLUN XPU" OFF)
 option(WITH_XPU_KP "Compile PaddlePaddle with BAIDU XPU compiler " OFF)
+option(WITH_XPU_XFT "Compile PaddlePaddle with BAIDU XPU-XFT" OFF)
 option(WITH_MLU "Compile PaddlePaddle with CAMBRICON MLU" OFF)
 option(WITH_WIN_DUMP_DBG "Compile with windows core dump debug mode" OFF)
 option(WITH_ASCEND "Compile PaddlePaddle with ASCEND" OFF)
@@ -74,6 +75,9 @@ endif()
 if(WITH_GPU AND WITH_XPU_KP)
   message(FATAL_ERROR "Error when compile GPU and XPU2 at the same time")
 endif()
+if(WITH_GPU AND WITH_XPU_XFT)
+  message(FATAL_ERROR "Error when compile GPU and XPU-XFT at the same time")
+endif()
 if(WITH_GPU AND WITH_ASCEND)
   message(FATAL_ERROR "Error when compile GPU and ASCEND at the same time")
 endif()
@@ -419,6 +423,15 @@ if(WITH_XPU_KP AND NOT WITH_XPU)
       CACHE STRING "Enable WITH_XPU when compiling with WITH_XPU_KP" FORCE)
 endif()
 
+if(NOT WITH_XPU AND WITH_XPU_XFT)
+  message(
+    WARNING
+      "Enable WITH_XPU when compiling with WITH_XPU_XFT. Force WITH_XPU=ON.")
+  set(WITH_XPU
+      ON
+      CACHE STRING "Enable WITH_XPU when compiling with WITH_XPU_XFT" FORCE)
+endif()
+
 if(NOT WITH_XPU AND WITH_XPU_BKCL)
   message(
     WARNING "Disable BKCL when compiling without XPU. Force WITH_XPU_BKCL=OFF.")

diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake
@@ -47,7 +47,7 @@ ExternalProject_Add(
   extern_brpc
   ${EXTERNAL_PROJECT_LOG_ARGS}
   GIT_REPOSITORY "https://github.com/apache/incubator-brpc"
-  GIT_TAG 1.2.0
+  GIT_TAG 1.4.0
   PREFIX ${BRPC_PREFIX_DIR}
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake
@@ -42,6 +42,23 @@ if(WITH_GPU)
   endif()
 endif()
 
+if(CMAKE_COMPILER_IS_GNUCC)
+  execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpfullversion -dumpversion
+                  OUTPUT_VARIABLE GCC_VERSION)
+  string(REGEX MATCHALL "[0-9]+" GCC_VERSION_COMPONENTS ${GCC_VERSION})
+  list(GET GCC_VERSION_COMPONENTS 0 GCC_MAJOR)
+  list(GET GCC_VERSION_COMPONENTS 1 GCC_MINOR)
+  set(GCC_VERSION "${GCC_MAJOR}.${GCC_MINOR}")
+  if(GCC_VERSION GREATER_EQUAL "12.0")
+    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/device.cc.patch
+         native_dst)
+    set(GLOO_PATCH_COMMAND patch -d ${GLOO_SOURCE_DIR}/gloo/transport/tcp <
+                           ${native_dst})
+    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/types.h.patch
+         types_header)
+    set(GLOO_PATCH_COMMAND patch -d ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+  endif()
+endif()
 include_directories(${GLOO_INCLUDE_DIR})
 
 if(WITH_ASCEND OR WITH_ASCEND_CL)

diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake
@@ -6,9 +6,11 @@ include(ExternalProject)
 set(XPU_PROJECT "extern_xpu")
 set(XPU_API_LIB_NAME "libxpuapi.so")
 set(XPU_RT_LIB_NAME "libxpurt.so")
+set(XPU_XFT_LIB_NAME "libxft.so")
 
 set(XPU_BASE_DATE "20230227")
 set(XPU_XCCL_BASE_VERSION "1.0.10")
+set(XPU_XFT_BASE_VERSION "latest")
 
 if(NOT DEFINED XPU_BASE_URL)
   set(XPU_BASE_URL_WITHOUT_DATE
@@ -22,30 +24,42 @@ set(XPU_XCCL_BASE_URL
     "https://klx-sdk-release-public.su.bcebos.com/xccl/release/${XPU_XCCL_BASE_VERSION}"
 )
 
+if(NOT XPU_XFT_BASE_URL)
+  set(XPU_XFT_BASE_URL
+      "https://klx-sdk-release-public.su.bcebos.com/xft/dev/${XPU_XFT_BASE_VERSION}"
+  )
+endif()
+
 if(WITH_AARCH64)
   set(XPU_XRE_DIR_NAME "xre-kylin_aarch64")
   set(XPU_XDNN_DIR_NAME "xdnn-kylin_aarch64")
   set(XPU_XCCL_DIR_NAME "xccl-kylin_aarch64")
+  set(XPU_XFT_DIR_NAME "") # TODO: xft has no kylin output at now.
 elseif(WITH_SUNWAY)
   set(XPU_XRE_DIR_NAME "xre-deepin_sw6_64")
   set(XPU_XDNN_DIR_NAME "xdnn-deepin_sw6_64")
   set(XPU_XCCL_DIR_NAME "xccl-deepin_sw6_64")
+  set(XPU_XFT_DIR_NAME "") # TODO: xft has no deepin output at now.
 elseif(WITH_BDCENTOS)
   set(XPU_XRE_DIR_NAME "xre-bdcentos_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-bdcentos_x86_64")
   set(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64")
+  set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
 elseif(WITH_UBUNTU)
   set(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64")
   set(XPU_XCCL_DIR_NAME "xccl-ubuntu_x86_64")
+  set(XPU_XFT_DIR_NAME "xft_ubuntu1604_x86_64")
 elseif(WITH_CENTOS)
   set(XPU_XRE_DIR_NAME "xre-centos7_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-centos7_x86_64")
   set(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64")
+  set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
 else()
   set(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64")
   set(XPU_XCCL_DIR_NAME "xccl-ubuntu_x86_64")
+  set(XPU_XFT_DIR_NAME "xft_ubuntu1604_x86_64")
 endif()
 
 set(XPU_XRE_URL
@@ -57,12 +71,16 @@ set(XPU_XDNN_URL
 set(XPU_XCCL_URL
     "${XPU_XCCL_BASE_URL}/${XPU_XCCL_DIR_NAME}.tar.gz"
     CACHE STRING "" FORCE)
+set(XPU_XFT_URL "${XPU_XFT_BASE_URL}/${XPU_XFT_DIR_NAME}.tar.gz")
 set(XPU_PACK_DEPENCE_URL
     "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/pack_paddle_depence.sh"
     CACHE STRING "" FORCE)
 set(XPU_CHECK_DEPENCE_URL
     "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/check_xpu_dependence.sh"
     CACHE STRING "" FORCE)
+set(XPU_XFT_GET_DEPENCE_URL
+    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/get_xft_dependence.sh"
+    CACHE STRING "" FORCE)
 
 set(SNAPPY_PREFIX_DIR "${THIRD_PARTY_PATH}/xpu")
 set(XPU_DOWNLOAD_DIR "${SNAPPY_PREFIX_DIR}/src/${XPU_PROJECT}")
@@ -90,7 +108,9 @@ ExternalProject_Add(
     wget ${XPU_CHECK_DEPENCE_URL} && bash check_xpu_dependence.sh
     ${XPU_BASE_URL} ${XPU_XCCL_BASE_URL} && wget ${XPU_PACK_DEPENCE_URL} && bash
     pack_paddle_depence.sh ${XPU_XRE_URL} ${XPU_XRE_DIR_NAME} ${XPU_XDNN_URL}
-    ${XPU_XDNN_DIR_NAME} ${XPU_XCCL_URL} ${XPU_XCCL_DIR_NAME}
+    ${XPU_XDNN_DIR_NAME} ${XPU_XCCL_URL} ${XPU_XCCL_DIR_NAME} && wget
+    ${XPU_XFT_GET_DEPENCE_URL} && bash get_xft_dependence.sh ${XPU_XFT_URL}
+    ${XPU_XFT_DIR_NAME}
   DOWNLOAD_NO_PROGRESS 1
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${XPU_INSTALL_ROOT}
@@ -116,7 +136,22 @@ if(WITH_XPU_BKCL)
   set(XPU_BKCL_LIB "${XPU_LIB_DIR}/${XPU_BKCL_LIB_NAME}")
   set(XPU_BKCL_INC_DIR "${THIRD_PARTY_PATH}/install/xpu/include")
   include_directories(${XPU_BKCL_INC_DIR})
+endif()
+
+if(WITH_XPU_XFT)
+  message(STATUS "Compile with XPU XFT!")
+  add_definitions(-DPADDLE_WITH_XPU_XFT)
+
+  set(XPU_XFT_LIB "${XPU_LIB_DIR}/${XPU_XFT_LIB_NAME}")
+endif()
+
+if(WITH_XPU_BKCL AND WITH_XPU_XFT)
+  target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ${XPU_BKCL_LIB}
+                        ${XPU_XFT_LIB})
+elseif(WITH_XPU_BKCL)
   target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ${XPU_BKCL_LIB})
+elseif(WITH_XPU_XFT)
+  target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ${XPU_XFT_LIB})
 else()
   target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB})
 endif()

diff --git a/paddle/fluid/distributed/CMakeLists.txt b/paddle/fluid/distributed/CMakeLists.txt
@@ -52,7 +52,7 @@ endif()
 proto_library(ps_framework_proto SRCS the_one_ps.proto)
 
 set(DISTRIBUTE_COMPILE_FLAGS
-    "-Wno-error=unused-value -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result"
+    "-Wno-error=unused-value -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=parentheses -Wno-error=unused-result"
 )
 
 if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)

diff --git a/paddle/fluid/distributed/collective/CMakeLists.txt b/paddle/fluid/distributed/collective/CMakeLists.txt
@@ -61,5 +61,7 @@ if(WITH_CUSTOM_DEVICE)
          place
          enforce
          collective_helper
-         device_context)
+         device_context
+         comm_static_check
+         dense_tensor)
 endif()