HIP tests for GPU and other build improvements (Xilinx#8008)

* Added makefiles to exercise HIP tests on GPU Signed-off-by: Sonal Santan <[email protected]> * Add a switch to enable(default)/disable bundling of Alveo drivers with XRT Fix a build failure on Ubuntu 23.10 due to an illegal std::move of a temporary object returned by create_profiling_event() Signed-off-by: Sonal Santan <[email protected]> * Add check for optional alveo driver install in postinst script Signed-off-by: Sonal Santan <[email protected]> * Update copyright notice to include 2024 Signed-off-by: Sonal Santan <[email protected]> * When Alveo is disabled do not bundle ERT firmware and xbflash2 Signed-off-by: Sonal Santan <[email protected]> * Fix build error Signed-off-by: Sonal Santan <[email protected]> --------- Signed-off-by: Sonal Santan <[email protected]>
sonals · Mar 18, 2024 · 2b87027 · 2b87027
1 parent f149538
commit 2b87027
Show file tree

Hide file tree

Showing 8 changed files with 144 additions and 13 deletions.
diff --git a/build/build.sh b/build/build.sh
@@ -53,6 +53,7 @@ usage()
     echo "[-opt]                      Build optimized library only (default)"
     echo "[-edge]                     Build edge of x64.  Turns off opt and dbg"
     echo "[-hip]                      Enable hip bindings"
+    echo "[-noalveo]                  Disable bundling of Alveo Linux drivers"
     echo "[-disable-werror]           Disable compilation with warnings as error"
     echo "[-nocmake]                  Skip CMake call"
     echo "[-noert]                    Do not treat missing ERT FW as a build error"
@@ -107,6 +108,7 @@ static_boost=""
 ertbsp=""
 ertfw=""
 werror=1
+alveo=1
 xrt_install_prefix="/opt/xilinx"
 cmake_flags="-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
 
@@ -152,6 +154,10 @@ while [ $# -gt 0 ]; do
             shift
             cmake_flags+=" -DXRT_ENABLE_HIP=ON"
             ;;
+	-noalveo)
+            shift
+	    alveo=0
+            ;;
         -opt)
             dbg=0
             opt=1
@@ -248,6 +254,10 @@ cmake_flags+=" -DXRT_ENABLE_WERROR=$werror"
 # set CMAKE_INSTALL_PREFIX
 cmake_flags+=" -DCMAKE_INSTALL_PREFIX=$xrt_install_prefix -DXRT_INSTALL_PREFIX=$xrt_install_prefix"
 
+if [[ $alveo == 1 ]]; then
+    cmake_flags+=" -DXRT_DKMS_ALVEO=ON"
+fi
+
 here=$PWD
 cd $BUILDDIR
 

diff --git a/src/CMake/config/postinst.in b/src/CMake/config/postinst.in
@@ -43,6 +43,8 @@ rmmodules()
 
 installdir=@CMAKE_INSTALL_PREFIX@/xrt
 systemddir=/etc/systemd/system
+test -e /usr/src/xrt-@XRT_VERSION_STRING@/dkms.conf
+alveo=$?
 
 msd_active=`systemctl is-active msd`
 mpd_active=`systemctl is-active mpd`
@@ -78,11 +80,16 @@ done
 rmmodules
 
 DRACUT_CONF_PATH=/etc/dracut.conf.d
-if [ -e $DRACUT_CONF_PATH ]; then
+if [ -e $DRACUT_CONF_PATH ] && [ $alveo == 1]; then
     install -m 644 /usr/src/xrt-@XRT_VERSION_STRING@/driver/xocl/userpf/xocl.dracut.conf $DRACUT_CONF_PATH
     install -m 644 /usr/src/xrt-@XRT_VERSION_STRING@/driver/xocl/mgmtpf/xclmgmt.dracut.conf $DRACUT_CONF_PATH
 fi
 
+if [ $alveo == 0]; then
+    echo "Skipping XRT Alveo driver install"
+    exit 0
+fi
+
 echo "Invoking DKMS common.postinst for xrt"
 /usr/lib/dkms/common.postinst xrt @XRT_VERSION_STRING@ "" "" $2
 if [ $? -eq 0 ]; then

diff --git a/src/CMake/nativeLnx.cmake b/src/CMake/nativeLnx.cmake
@@ -208,12 +208,16 @@ message("-- XRT version: ${XRT_VERSION_STRING}")
 # -- CPack
 include (CMake/cpackLin.cmake)
 
-set (XRT_DKMS_DRIVER_SRC_BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/runtime_src/core")
-
-include (CMake/dkms.cmake)
-include (CMake/dkms-aws.cmake)
-include (CMake/dkms-azure.cmake)
-include (CMake/dkms-container.cmake)
+if (XRT_DKMS_ALVEO STREQUAL "ON")
+  message("-- XRT Alveo drivers will be bundled with the XRT package")
+  set (XRT_DKMS_DRIVER_SRC_BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/runtime_src/core")
+  include (CMake/dkms.cmake)
+  include (CMake/dkms-aws.cmake)
+  include (CMake/dkms-azure.cmake)
+  include (CMake/dkms-container.cmake)
+else()
+  message("-- Skipping bundling of XRT Alveo drivers with XRT package")
+endif()
 
 # --- ICD ---
 include (CMake/icd.cmake)

diff --git a/src/runtime_src/CMakeLists.txt b/src/runtime_src/CMakeLists.txt
@@ -35,7 +35,11 @@ xrt_add_subdirectory(xdp)
 xrt_add_subdirectory(tools/xclbinutil)
 xrt_add_subdirectory(xocl)
 xrt_add_subdirectory(xrt)
-xrt_add_subdirectory(ert)
+
+if (XRT_DKMS_ALVEO STREQUAL "ON")
+  xrt_add_subdirectory(ert)
+endif()
+
 if (XRT_ENABLE_HIP)
   xrt_add_subdirectory(hip)
 endif(XRT_ENABLE_HIP)

diff --git a/src/runtime_src/core/common/api/aie/xrt_graph.cpp b/src/runtime_src/core/common/api/aie/xrt_graph.cpp
@@ -139,7 +139,7 @@ class profiling_impl
     }
   }
 
-  handle 
+  handle
   start_profiling(int option, const std::string& port1_name, const std::string& port2_name, uint32_t value)
   {
     profiling_hdl = device->start_profiling(option, port1_name.c_str(), port2_name.c_str(), value);
@@ -153,7 +153,7 @@ class profiling_impl
       throw xrt_core::error(-EINVAL, "Not a valid profiling handle");
 
     return device->read_profiling(profiling_hdl);
-    
+
   }
 
   void
@@ -889,7 +889,7 @@ xrtAIEReadProfiling(xrtDeviceHandle /*handle*/, int pHandle)
     if (it != profiling_cache.end())
       return it->second->read_profiling();
     else
-      throw xrt_core::error(-EINVAL, "No such profiling handle"); 
+      throw xrt_core::error(-EINVAL, "No such profiling handle");
   }
   catch (const xrt_core::error& ex) {
     xrt_core::send_exception_message(ex.what());
@@ -921,7 +921,7 @@ xrtAIEStopProfiling(xrtDeviceHandle /*handle*/, int pHandle)
       profiling_cache.erase(pHandle);
     }
     else
-      throw xrt_core::error(-EINVAL, "No such profiling handle");   
+      throw xrt_core::error(-EINVAL, "No such profiling handle");
   }
   catch (const xrt_core::error& ex) {
     xrt_core::send_exception_message(ex.what());

diff --git a/src/runtime_src/core/tools/CMakeLists.txt b/src/runtime_src/core/tools/CMakeLists.txt
@@ -10,7 +10,9 @@ if (${XRT_NATIVE_BUILD} STREQUAL "yes")
   xrt_add_subdirectory(xbmgmt2)
   if(NOT WIN32)
     xrt_add_subdirectory(xbtop)
-    xrt_add_subdirectory(xbflash2)
+    if (XRT_DKMS_ALVEO STREQUAL "ON")
+      xrt_add_subdirectory(xbflash2)
+    endif()
     xrt_add_subdirectory(nagios)
   endif()
 endif()

diff --git a/tests/hip/vadd-stream/Makefile.gpu b/tests/hip/vadd-stream/Makefile.gpu
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache License 2.0 #
+# Copyright (c) 2021-2022 Xilinx, Inc. All rights reserved #
+# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. #
+
+ROCM_ROOT = /opt/rocm
+SRC = main.cpp
+OBJ = main.o
+HIPCC = $(ROCM_ROOT)/bin/hipcc
+HIPCCFLAGS= --rocm-device-lib-path=/usr/lib/x86_64-linux-gnu/amdgcn/bitcode
+CXX = g++
+CC = g++
+CXXFLAGS = -Wall -Werror -D__HIP_PLATFORM_HCC__= -D__HIP_PLATFORM_AMD__ -I$(ROCM_ROOT)/include -I$(ROCM_ROOT)/llvm/bin/../lib/clang/14.0.0 -I$(ROCM_ROOT)/hsa/include
+RPROF = $(ROCM_ROOT)/rocprof
+LDFLAGS = -L$(ROCM_ROOT)/hip/lib
+LDLIBS = -lamdhip64 -luuid -lm -lrt
+COMPILE_DB = compile_commands.json
+
+export LD_LIBRARY_PATH += :$(ROCM_ROOT)/hip/lib
+
+debug ?= 0
+ifeq ($(debug), 1)
+    CXXFLAGS +=-DDEBUG -g
+else
+    CXXFLAGS +=-DNDEBUG -O2
+endif
+
+all: main kernel.co nop.co
+
+main: main.o
+
+%.co: %.cpp
+	$(HIPCC) $(HIPCCFLAGS) --genco $< -o $@
+
+run: all
+	@echo "LD_LIBRARY_PATH = $(LD_LIBRARY_PATH)"
+	./main
+
+
+profile: all
+	$(RPROF) --hip-trace ./main
+	$(RPROF) --hsa-trace ./main
+	jq '.traceEvents[] | .name' results.json | sort | uniq
+	strace -e trace=ioctl -o strace.log ./main
+	grep AMDKFD strace.log | awk '-F,' '{print $$2}' | sort | uniq
+
+$(COMPILE_DB): main.cpp kernel.cpp nop.cpp Makefile
+	bear -- make debug=1 all
+
+compdb: $(COMPILE_DB)
+
+clean:
+	rm -f main *.co results.* *.o
diff --git a/tests/hip/vadd/Makefile.gpu b/tests/hip/vadd/Makefile.gpu
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache License 2.0 #
+# Copyright (c) 2021-2022 Xilinx, Inc. All rights reserved #
+# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. #
+
+ROCM_ROOT = /opt/rocm
+SRC = main.cpp
+OBJ = main.o
+HIPCC = $(ROCM_ROOT)/bin/hipcc
+HIPCCFLAGS= --rocm-device-lib-path=/usr/lib/x86_64-linux-gnu/amdgcn/bitcode
+CXX = g++
+CC = g++
+CXXFLAGS = -Wall -Werror -D__HIP_PLATFORM_HCC__= -D__HIP_PLATFORM_AMD__ -I$(ROCM_ROOT)/include -I$(ROCM_ROOT)/llvm/bin/../lib/clang/14.0.0 -I$(ROCM_ROOT)/hsa/include
+RPROF = $(ROCM_ROOT)/rocprof
+LDFLAGS = -L$(ROCM_ROOT)/hip/lib
+LDLIBS = -lamdhip64 -luuid -lm -lrt
+COMPILE_DB = compile_commands.json
+
+export LD_LIBRARY_PATH += :$(ROCM_ROOT)/hip/lib
+
+debug ?= 0
+ifeq ($(debug), 1)
+    CXXFLAGS +=-DDEBUG -g
+else
+    CXXFLAGS +=-DNDEBUG -O2
+endif
+
+all: main kernel.co nop.co
+
+main: main.o
+
+%.co: %.cpp
+	$(HIPCC) $(HIPCCFLAGS) --genco $< -o $@
+
+run: all
+	@echo "LD_LIBRARY_PATH = $(LD_LIBRARY_PATH)"
+	./main
+
+
+profile: all
+	$(RPROF) --hip-trace ./main
+	$(RPROF) --hsa-trace ./main
+	jq '.traceEvents[] | .name' results.json | sort | uniq
+	strace -e trace=ioctl -o strace.log ./main
+	grep AMDKFD strace.log | awk '-F,' '{print $$2}' | sort | uniq
+
+$(COMPILE_DB): main.cpp kernel.cpp nop.cpp Makefile
+	bear -- make debug=1 all
+
+compdb: $(COMPILE_DB)
+
+clean:
+	rm -f main *.co results.* *.o