Skip to content

Commit

Permalink
Intel(R) oneAPI Collective Communications Library (oneCCL) 2021.1-beta10
Browse files Browse the repository at this point in the history
  • Loading branch information
sazanovd committed Nov 5, 2020
1 parent af0c8c9 commit 20d86a0
Show file tree
Hide file tree
Showing 706 changed files with 41,023 additions and 24,478 deletions.
130 changes: 74 additions & 56 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,6 @@ endif()
#make build variable case insensitive
string( TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_CASE_INSENSITIVE)

#show build info
message(STATUS "Installation directory: ${CMAKE_INSTALL_PREFIX}")
message(STATUS "Build type: ${CMAKE_BUILD_TYPE_CASE_INSENSITIVE}")
message(STATUS "C compiler : ${CMAKE_C_COMPILER}")
message(STATUS "CXX compiler : ${CMAKE_CXX_COMPILER}")

#predefined options
option(USE_SECURITY_FLAGS "Use security flags" TRUE)
if (${CMAKE_BUILD_TYPE_CASE_INSENSITIVE} STREQUAL "debug")
Expand All @@ -58,10 +52,17 @@ option(WITH_ASAN "Use address sanitizer, can only be used in Debug build" FALSE)

#installation path variables
include(GNUInstallDirs)

if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/_install" CACHE PATH "Default install path" FORCE)
endif()

#show build info
message(STATUS "Installation directory: ${CMAKE_INSTALL_PREFIX}")
message(STATUS "Build type: ${CMAKE_BUILD_TYPE_CASE_INSENSITIVE}")
message(STATUS "C compiler : ${CMAKE_C_COMPILER}")
message(STATUS "CXX compiler : ${CMAKE_CXX_COMPILER}")

SET(MULTI_GPU_SUPPORT OFF CACHE BOOL "Enable Multi GPU extension support")

set(CCL_COMMON_INSTALL_PREFIX "intel64")
Expand All @@ -76,13 +77,24 @@ set(CCL_INSTALL_LICENSE "${CMAKE_INSTALL_PREFIX}/licensing")
set(CCL_INSTALL_MODULE "${CMAKE_INSTALL_PREFIX}/modulefiles")
set(CCL_INSTALL_EXAMPLES "${CMAKE_INSTALL_PREFIX}/examples")
set(CCL_INSTALL_TESTS "${CMAKE_INSTALL_PREFIX}/tests")


set(MPI_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/mpi/include/")
set(MPI_LIB_DIR "${PROJECT_SOURCE_DIR}/mpi/lib/")
if ( "${LIBFABRIC_DIR}" STREQUAL "")
set(LIBFABRIC_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/ofi/include")
set(LIBFABRIC_LIB_DIR "${PROJECT_SOURCE_DIR}/ofi/lib/")
else()
set(LIBFABRIC_INCLUDE_DIR "${LIBFABRIC_DIR}/include/")
set(LIBFABRIC_LIB_DIR "${LIBFABRIC_DIR}/lib")
endif()
include_directories(${MPI_INCLUDE_DIR})
include_directories(${LIBFABRIC_INCLUDE_DIR})
link_directories(${MPI_LIB_DIR})
link_directories(${LIBFABRIC_LIB_DIR})



set(CCL_INSTALL_UNIT_TESTS "${CMAKE_INSTALL_PREFIX}/tests/unit")

set(CMAKE_SKIP_INSTALL_RPATH TRUE)
Expand Down Expand Up @@ -134,7 +146,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)

if (COMPUTE_RUNTIME)
activate_compute_runtime("${CMAKE_CURRENT_LIST_DIR}/cmake" ${COMPUTE_RUNTIME})

set(PARENT_COMPUTE_RUNTIME_TARGET_NAME ${COMPUTE_RUNTIME_TARGET_NAME})
if (${CCL_ENABLE_SYCL_V} STREQUAL 1)
option (CCL_ENABLE_SYCL "Enable CCL SYCL runtime" ON)
message(STATUS "Enable CCL SYCL runtime")
Expand All @@ -156,93 +168,84 @@ if(WITH_ASAN AND ${CMAKE_BUILD_TYPE_CASE_INSENSITIVE} STREQUAL "debug")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lasan")
endif()

set(GCC_BFP16_MIN_SUPPORTED "4.9.0")
set(GCC_BFP16_AVX512BF_MIN_SUPPORTED "10.0.0")
set(ICC_BFP16_AVX512BF_MIN_SUPPORTED "19.1.0")
set(CLANG_BFP16_MIN_SUPPORTED "9.0.0")
set(CLANG_BFP16_AVX512BF_MIN_SUPPORTED "10.0.0")
set(GCC_BF16_MIN_SUPPORTED "4.9.0")
set(GCC_BF16_AVX512BF_MIN_SUPPORTED "10.0.0")
set(ICC_BF16_AVX512BF_MIN_SUPPORTED "19.1.0")
set(CLANG_BF16_MIN_SUPPORTED "9.0.0")
set(CLANG_BF16_AVX512BF_MIN_SUPPORTED "10.0.0")

if (${CMAKE_C_COMPILER_ID} STREQUAL "Intel"
OR (${CMAKE_C_COMPILER_ID} STREQUAL "Clang"
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${CLANG_BFP16_MIN_SUPPORTED})
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${CLANG_BF16_MIN_SUPPORTED})
OR (${CMAKE_C_COMPILER_ID} STREQUAL "GNU"
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${GCC_BFP16_MIN_SUPPORTED})
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${GCC_BF16_MIN_SUPPORTED})
)
add_definitions(-DCCL_BFP16_COMPILER)
set(CCL_BFP16_COMPILER ON)
message(STATUS "BFP16 compiler: yes")
add_definitions(-DCCL_BF16_COMPILER)
set(CCL_BF16_COMPILER ON)
message(STATUS "BF16 compiler: yes")
else()
set(CCL_BFP16_COMPILER OFF)
message(STATUS "BFP16 compiler: no")
set(CCL_BF16_COMPILER OFF)
message(STATUS "BF16 compiler: no")
endif()

if ((${CMAKE_C_COMPILER_ID} STREQUAL "Intel"
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${ICC_BFP16_AVX512BF_MIN_SUPPORTED})
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${ICC_BF16_AVX512BF_MIN_SUPPORTED})
OR (${CMAKE_C_COMPILER_ID} STREQUAL "Clang"
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${CLANG_BFP16_AVX512BF_MIN_SUPPORTED})
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${CLANG_BF16_AVX512BF_MIN_SUPPORTED})
OR (${CMAKE_C_COMPILER_ID} STREQUAL "GNU"
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${GCC_BFP16_AVX512BF_MIN_SUPPORTED})
AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${GCC_BF16_AVX512BF_MIN_SUPPORTED})
)
add_definitions(-DCCL_BFP16_AVX512BF_COMPILER)
message(STATUS "BFP16 AVX512BF compiler: yes")
add_definitions(-DCCL_BF16_AVX512BF_COMPILER)
message(STATUS "BF16 AVX512BF compiler: yes")
else()
message(STATUS "BFP16 AVX512BF compiler: no")
message(STATUS "BF16 AVX512BF compiler: no")
endif()

if (CCL_BFP16_COMPILER)
if (CCL_BF16_COMPILER)
if ((${CMAKE_C_COMPILER_ID} STREQUAL "Clang" OR ${CMAKE_C_COMPILER_ID} STREQUAL "GNU"))
add_definitions(-DCCL_BFP16_TARGET_ATTRIBUTES)
message(STATUS "BFP16 target attributes: yes")
add_definitions(-DCCL_BF16_TARGET_ATTRIBUTES)
message(STATUS "BF16 target attributes: yes")
else()
message(STATUS "BFP16 target attributes: no")
message(STATUS "BF16 target attributes: no")
endif()
endif()

set(CCL_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/src)

enable_testing()

set (EXTERNAL_LIBS "")
set(EXAMPLES_INC_DIRS ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/examples/include)
set(EXTERNAL_LIBS "")
set(EXAMPLES_INC_DIRS ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/examples/include ${PROJECT_SOURCE_DIR}/mpi/include)

# allow `deprecated`
set(CMAKE_CLANG_FLAGS "${CMAKE_CLANG_FLAGS}")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")

if (MULTI_GPU_SUPPORT)
if (NOT ((${COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "ze_loader")
OR (${COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "Intel::SYCL")))
message( FATAL_ERROR "MULTI_GPU_SUPPORT required COMPUTE_RUNTIME=L0 or DPC++")
if (COMPUTE_RUNTIME)
if ((${COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "ze_loader")
OR (${COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "Intel::SYCL"))
set(MULTI_GPU_SUPPORT ON)
activate_compute_runtime("${CMAKE_CURRENT_LIST_DIR}/cmake" L0)
message ("Enable multi GPU support: ${MULTI_GPU_SUPPORT}")
message ("COMPUTE_RUNTIME_TARGET_NAME: ${COMPUTE_RUNTIME_TARGET_NAME}")
endif()
endif(COMPUTE_RUNTIME)

if (MULTI_GPU_SUPPORT)
option(CCL_GPU_DEVICES_AFFINITY_ENABLE "Enable L0" ON)
if(CCL_GPU_DEVICES_AFFINITY_ENABLE)
set(CCL_GPU_DEVICES_AFFINITY_MASK_SIZE 4)
message ("Set L0 device mask affinity size: ${CCL_GPU_DEVICES_AFFINITY_MASK_SIZE}")
endif()

endif(MULTI_GPU_SUPPORT)

#include other CMakeLists
add_subdirectory(src)
add_subdirectory(examples/cpu)

if (CCL_ENABLE_SYCL)
if (${COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "Intel::SYCL")
set (CCL_ENABLCE_SYCL_CHECK_CONTRACT "#if defined(__cplusplus)\n#if !defined(__clang__) || __clang_major__ < 9 || !defined(CL_SYCL_LANGUAGE_VERSION)\n#error This version of CCL configured only for oneAPI DPC++ Compiler\n#endif\n#endif")
else(${COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "Codeplay::ComputeCpp")
set (CCL_ENABLCE_SYCL_CHECK_CONTRACT "#if defined(__cplusplus)\n#if !defined(__clang__) || __clang_major__ < 6\n#error This version of CCL configured only for oneAPI DPC++ Compiler\n#endif\n#endif")
if (${PARENT_COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "Intel::SYCL")
set (CCL_ENABLE_SYCL_CHECK_CONTRACT "#if defined(__cplusplus)\n#if !defined(__clang__) || __clang_major__ < 9 || !defined(CL_SYCL_LANGUAGE_VERSION)\n#error This version of CCL configured only for oneAPI DPC++ Compiler\n#endif\n#endif")
else(${PARENT_COMPUTE_RUNTIME_TARGET_NAME} STREQUAL "Codeplay::ComputeCpp")
set (CCL_ENABLE_SYCL_CHECK_CONTRACT "#if defined(__cplusplus)\n#if !defined(__clang__) || __clang_major__ < 6\n#error This version of CCL configured only for oneAPI DPC++ Compiler\n#endif\n#endif")
endif()
add_subdirectory(examples/sycl)
endif()

add_subdirectory(examples/benchmark)
add_subdirectory(examples/common)
add_subdirectory(tests/functional)
if(MULTI_GPU_SUPPORT)
add_subdirectory(examples/level_zero)
add_subdirectory(tests/unit)
endif()

#generate & install vars.sh
Expand All @@ -257,11 +260,26 @@ install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/third-party-programs.txt DESTINATIO
install(PROGRAMS ${PROJECT_SOURCE_DIR}/LICENSE DESTINATION ${CCL_INSTALL_LICENSE})

set(CCL_MAJOR_VERSION "0")
set(CCL_MINOR_VERSION "9")
set(CCL_MINOR_VERSION "10")
set(CCL_UPDATE_VERSION "0")
set(CCL_PRODUCT_STATUS "beta")
string(TIMESTAMP CCL_PRODUCT_BUILD_DATE "%Y-%m-%dT %H:%M:%SZ")
get_vcs_properties("git")
set(CCL_PRODUCT_FULL "${CCL_PRODUCT_STATUS}-${CCL_MAJOR_VERSION}.${CCL_MINOR_VERSION}.${CCL_UPDATE_VERSION} ${CCL_PRODUCT_BUILD_DATE} ${VCS_INFO}")
configure_file(${PROJECT_SOURCE_DIR}/include/ccl_config.h.in "${CMAKE_CURRENT_BINARY_DIR}/include/ccl_config.h")
file(COPY "${CMAKE_CURRENT_BINARY_DIR}/include/ccl_config.h" DESTINATION ${PROJECT_SOURCE_DIR}/include/)
configure_file(${PROJECT_SOURCE_DIR}/include/oneapi/ccl/ccl_config.h.in "${CMAKE_CURRENT_BINARY_DIR}/include/oneapi/ccl/ccl_config.h")
file(COPY "${CMAKE_CURRENT_BINARY_DIR}/include/oneapi/ccl/ccl_config.h" DESTINATION ${PROJECT_SOURCE_DIR}/include/oneapi/ccl)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)


#include other CMakeLists

add_subdirectory(src)
add_subdirectory(examples/benchmark)
add_subdirectory(examples/common)
add_subdirectory(examples/cpu)
if (CCL_ENABLE_SYCL)
add_subdirectory(examples/sycl)
endif()

add_subdirectory(tests/functional)
#add_subdirectory(tests/unit)
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
oneAPI Collective Communications Library (oneCCL) provides an efficient implementation of communication patterns used in deep learning.

oneCCL is integrated into:
* [Horovod\*](https://github.com/horovod/horovod) (distributed training framework). Refer to [Horovod with oneCCL](https://github.com/horovod/horovod/blob/master/docs/oneccl.rst) for details.
* [Horovod\*](https://github.com/horovod/horovod) (distributed training framework). Refer to [Horovod with oneCCL](https://github.com/horovod/horovod/blob/master/docs/oneccl.md) for details.
* [PyTorch\*](https://github.com/pytorch/pytorch) (machine learning framework). Refer to [PyTorch bindings for oneCCL](https://github.com/intel/torch-ccl) for details.

Release notes available by [link](https://software.intel.com/content/www/us/en/develop/articles/oneapi-collective-communication-library-ccl-release-notes.html).
Expand All @@ -13,6 +13,9 @@ Release notes available by [link](https://software.intel.com/content/www/us/en/d
Ubuntu* 18
GNU*: C, C++ 4.8.5 or higher.

### SYCL support
Intel(R) oneAPI DPC++ Compiler with L0 v1.0 support

## Installation
### General installation scenario

Expand Down
35 changes: 29 additions & 6 deletions cmake/ccl
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@
## Intel(R) oneAPI Collective Communications Library modulefile
##

set min_tcl_ver 8.4
if { $tcl_version < $min_tcl_ver } {
puts stderr " "
puts stderr "ERROR: This modulefile requires tcl $min_tcl_ver or greater."
puts stderr "Your system reports that tclsh version $tcl_version is installed."
exit 1
}

# get full pathname for this script file
set scriptpath "${ModulesCurrentModulefile}"

Expand Down Expand Up @@ -36,24 +44,39 @@ set oneapiroot "[file dirname "$componentroot"]"

# On load print component name and version being loaded
if { [ module-info mode load ] } {
puts stderr "Loading $modulefilename"
puts stderr "Loading $modulefilename version $modulefilever"
}

# On remove print component name and version being removed
if { [ module-info mode ] == "unload" || [ module-info mode ] == "remove" } {
puts stderr "Removing $modulefilename"
puts stderr "Removing $modulefilename version $modulefilever"
}


# ###### Component Specific env vars setup ###################################

module-whatis "Intel(R) oneAPI Collective Communications Library"

set ccl_conf "cpu_gpu_dpcpp"

if { [info exists ::env(CCL_CONFIGURATION) ] } {
switch "$::env(CCL_CONFIGURATION)" {
"cpu_gpu_dpcpp" -
"cpu_icc" {
set ccl_conf "$::env(CCL_CONFIGURATION)"
}
default {
}
}
}

set topdir "$componentroot/$modulefilever"

setenv CCL_ROOT "$topdir"
setenv CCL_ATL_TRANSPORT_PATH "$topdir/lib"
setenv CCL_CONFIGURATION "$ccl_conf"
setenv CCL_ATL_TRANSPORT_PATH "$topdir/lib/$ccl_conf"

prepend-path LD_LIBRARY_PATH "$topdir/lib/$ccl_conf"
prepend-path LIBRARY_PATH "$topdir/lib/$ccl_conf"
prepend-path CPATH "$topdir/include/$ccl_conf"

prepend-path LD_LIBRARY_PATH "$topdir/lib"
prepend-path LIBRARY_PATH "$topdir/lib"
prepend-path CPATH "$topdir/include"
69 changes: 62 additions & 7 deletions cmake/vars.sh.in
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
#!/bin/bash
#
# Copyright 2016-2020 Intel Corporation
#
Expand All @@ -14,12 +13,68 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
#!/bin/bash

get_script_path() (
script="$1"
while [ -L "$script" ] ; do
# combining next two lines fails in zsh shell
script_dir=$(command dirname -- "$script")
script_dir=$(cd "$script_dir" && command pwd -P)
script="$(readlink "$script")"
case $script in
(/*) ;;
(*) script="$script_dir/$script" ;;
esac
done
# combining next two lines fails in zsh shell
script_dir=$(command dirname -- "$script")
script_dir=$(cd "$script_dir" && command pwd -P)
echo "$script_dir"
)

usage() {
printf "%s\n" "ERROR: This script must be sourced."
printf "%s\n" "Usage: source $1"
exit 255
}

vars_script_name=""

if [ -n "${ZSH_VERSION+x}" ] ; then # only executed in zsh
# echo " ZSH version = \"$ZSH_VERSION\""
case $ZSH_EVAL_CONTEXT in (*:file*) vars_script_name="${(%):-%x}" ;; esac ;
if [ -n "$ZSH_EVAL_CONTEXT" ] && [ "" = "$vars_script_name" ] ; then
usage "${(%):-%x}" ;
fi
elif [ -n "${KSH_VERSION+x}" ] ; then # only executed in ksh
# echo " KSH version = \"$KSH_VERSION\""
if [ "$(cd "$(dirname -- "$0")" && pwd -P)/$(basename -- "$0")" \
!= "$(cd "$(dirname -- "${.sh.file}")" && pwd -P)/$(basename -- "${.sh.file}")" ] ; then
vars_script_name="${.sh.file}" || usage "$0" ;
fi
elif [ -n "${BASH_VERSION+x}" ] ; then # only executed in bash
# echo " BASH version = \"$BASH_VERSION\""
# shellcheck disable=2128,2015
(return 0 2>/dev/null) && vars_script_name="${BASH_SOURCE}" || usage "${BASH_SOURCE}"
else
case ${0##*/} in (sh|dash) vars_script_name="" ;; esac
# echo " Shell version = \"unknown\""
fi

if [ "" = "$vars_script_name" ] ; then
>&2 echo ":: ERROR: Unable to proceed: no support for sourcing from '[dash|sh]' shell." ;
>&2 echo " This script must be sourced. Did you execute or source this script?" ;
>&2 echo " Can be caused by sourcing from inside a \"shebang-less\" script." ;
>&2 echo " Can also be caused by sourcing from ZSH version 4.x or older." ;
return 1 2>/dev/null || exit 1
fi

WORK_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd)"
WORK_DIR=$(get_script_path "$vars_script_name")

export CCL_ROOT="$(cd ${WORK_DIR}/../; pwd -P)"
export CCL_ATL_TRANSPORT_PATH="${CCL_ROOT}/lib"
CCL_ROOT="$(cd "${WORK_DIR}"/../; pwd -P)"; export CCL_ROOT
CCL_ATL_TRANSPORT_PATH="${CCL_ROOT}/lib"; export CCL_ATL_TRANSPORT_PATH

export CPATH=${CCL_ROOT}/include/${CPATH+:${CPATH}}
export LIBRARY_PATH=${CCL_ROOT}/lib/${LIBRARY_PATH+:${LIBRARY_PATH}}
export LD_LIBRARY_PATH=${CCL_ROOT}/lib/${LD_LIBRARY_PATH+:${LD_LIBRARY_PATH}}
CPATH="${CCL_ROOT}/include/${CPATH+:${CPATH}}"; export CPATH
LIBRARY_PATH="${CCL_ROOT}/lib/${LIBRARY_PATH+:${LIBRARY_PATH}}"; export LIBRARY_PATH
LD_LIBRARY_PATH="${CCL_ROOT}/lib/${LD_LIBRARY_PATH+:${LD_LIBRARY_PATH}}"; export LD_LIBRARY_PATH
2 changes: 1 addition & 1 deletion doc/Doxyfile
Original file line number Diff line number Diff line change
Expand Up @@ -940,7 +940,7 @@ EXCLUDE_PATTERNS =
# Note that the wildcards are matched against the file with absolute path, so to
# exclude all test directories use the pattern */test/*

EXCLUDE_SYMBOLS = "ccl::ccl_type_info_export" "ccl::crequest_t" "ccl::native_type_info" "ccl::request_t" "ccl::type_info" "*CCL_TYPE_TRAITS*" "*CLASS_TYPE_TRAITS*" "COMMA"
EXCLUDE_SYMBOLS = "ccl::ccl_type_info_export" "ccl::crequest_t" "ccl::native_type_info" "ccl::event" "ccl::type_info" "*CCL_TYPE_TRAITS*" "*CLASS_TYPE_TRAITS*" "COMMA"

# The EXAMPLE_PATH tag can be used to specify one or more files or directories
# that contain example code fragments that are included (see the \include
Expand Down
Loading

0 comments on commit 20d86a0

Please sign in to comment.