Enable compile time generation of dynlink wrappers for nvml (NVIDIA#2463

)

* Enable compile time generation of dynlink wrappers for nvml

Signed-off-by: Janusz Lisiecki <[email protected]>

CMakeLists.txt

@@ -36,7 +36,7 @@ check_cxx_compiler_flag(-fopenmp-simd CXX_HAVE_OMP_SIMD)
 
 # Build options
 option(BUILD_DALI_NODEPS "Disable components that require extra external libraries to be present in the system. Effectively, it builds only the DALI core and kernel libraries")
-option(LINK_LIBCUDA "Links directly with libcuda.so instead of dlopen it at runtime" OFF)
+option(LINK_DRIVER "Links directly with libcuda.so instead of dlopen it at runtime" OFF)
 
 # Tests use OpenCV...
 cmake_dependent_option(BUILD_TEST "Build googletest test suite" ON
@@ -142,7 +142,7 @@ propagate_option(BUILD_NVJPEG2K)
 propagate_option(BUILD_NVOF)
 propagate_option(BUILD_NVDEC)
 propagate_option(BUILD_NVML)
-propagate_option(LINK_LIBCUDA)
+propagate_option(LINK_DRIVER)
 
 get_dali_version(${PROJECT_SOURCE_DIR}/VERSION DALI_VERSION)
 

cmake/Dependencies.cmake

@@ -68,6 +68,9 @@ list(APPEND DALI_EXCLUDES libculibos.a)
 if (LINK_LIBCUDA)
   CUDA_find_library_stub(CUDA_cuda_LIBRARY cuda)
   list(APPEND DALI_LIBS ${CUDA_cuda_LIBRARY})
+
+  CUDA_find_library_stub(CUDA_nvml_LIBRARY nvidia-ml)
+  list(APPEND DALI_LIBS ${CUDA_nvml_LIBRARY})
 endif()
 
 # NVTX for profiling

dali/CMakeLists.txt

@@ -57,8 +57,7 @@ collect_sources(DALI_SRCS PARENT_SCOPE)
 if (BUILD_PROTOBUF)
   set(DALI_PROTO_OBJ $<TARGET_OBJECTS:DALI_PROTO>)
   add_library(dali ${LIBTYPE} ${DALI_SRCS} ${DALI_PROTO_OBJ} ${CUDART_LIB})
-  set_target_properties(dali PROPERTIES
-      LIBRARY_OUTPUT_DIRECTORY "${DALI_LIBRARY_OUTPUT_DIR}")
+  set_target_properties(dali PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${DALI_LIBRARY_OUTPUT_DIR}")
 endif()
 
 if (BUILD_DALI_PIPELINE)
@@ -75,6 +74,10 @@ if (BUILD_DALI_PIPELINE)
   target_link_libraries(dali PRIVATE "-Wl,--exclude-libs,${exclude_libs}")
 endif()
 
+if (BUILD_NVML)
+  target_link_libraries(dali PRIVATE $<TARGET_OBJECTS:dynlink_nvml>)
+endif(BUILD_NVML)
+
 ################################################
 # Build test suite
 ################################################
@@ -84,6 +87,9 @@ if (BUILD_DALI_PIPELINE AND BUILD_TEST)
 
   target_link_libraries(dali_test PUBLIC dali dali_core dali_kernels dali_operators ${DALI_LIBS} gtest)
   target_link_libraries(dali_test PRIVATE dynlink_cuda ${CUDART_LIB})
+  if (BUILD_NVML)
+    target_link_libraries(dali_test PRIVATE $<TARGET_OBJECTS:dynlink_nvml>)
+  endif(BUILD_NVML)
   target_link_libraries(dali_test PRIVATE "-pie")
   set_target_properties(dali_test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${TEST_BINARY_DIR})
   set_target_properties(dali_test PROPERTIES POSITION_INDEPENDENT_CODE ON)

dali/benchmark/CMakeLists.txt

@@ -44,6 +44,9 @@ if (BUILD_BENCHMARK)
   add_executable(dali_benchmark "${DALI_BENCHMARK_SRCS}")
 
   target_link_libraries(dali_benchmark PRIVATE dali dali_operators benchmark ${DALI_LIBS})
+  if (BUILD_NVML)
+    target_link_libraries(dali_benchmark PRIVATE $<TARGET_OBJECTS:dynlink_nvml>)
+  endif(BUILD_NVML)
   target_link_libraries(dali_benchmark PRIVATE "-pie")
   set_target_properties(dali_benchmark PROPERTIES POSITION_INDEPENDENT_CODE ON)
   set_target_properties(dali_benchmark PROPERTIES OUTPUT_NAME "dali_benchmark.bin")

dali/core/CMakeLists.txt

@@ -27,7 +27,7 @@ foreach(incl_dir ${INFERED_COMPILER_INCLUDE})
 endforeach(incl_dir)
 separate_arguments(DEFAULT_COMPILER_INCLUDE UNIX_COMMAND  "${DEFAULT_COMPILER_INCLUDE}")
 
-if (NOT LINK_LIBCUDA)
+if (NOT LINK_DRIVER)
   set(CUDA_GENERATED_STUB "${CMAKE_CURRENT_BINARY_DIR}/dynlink_cuda_gen.cc")
   add_custom_command(
       OUTPUT ${CUDA_GENERATED_STUB}

dali/core/dynlink_cuda.cc

@@ -12,14 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <dlfcn.h>
 #include <stdio.h>
 #include <mutex>
 #include <string>
 #include <unordered_map>
 #include "dali/core/dynlink_cuda.h"
 
-#include <dlfcn.h>
-
 namespace {
 
 typedef void *CUDADRIVER;
@@ -48,7 +47,7 @@ void *LoadSymbol(const char *name) {
   return ret;
 }
 
-} // namespace
+}  // namespace
 
 // it is defined in the generated file
 typedef void *tLoadSymbol(const char *name);
@@ -67,7 +66,7 @@ bool cuInitChecked() {
       return true;
 
   // set symbol loader for this library
-#if !LINK_LIBCUDA_ENABLED
+#if !LINK_DRIVER_ENABLED
   CudaSetSymbolLoader(LoadSymbol);
 #endif
   static CUresult res = cuInit(0);

dali/operators/CMakeLists.txt

@@ -75,6 +75,9 @@ if (BUILD_TEST)
 
   target_link_libraries(dali_operator_test PUBLIC dali_operators)
   target_link_libraries(dali_operator_test PRIVATE gtest dynlink_cuda ${DALI_LIBS})
+  if (BUILD_NVML)
+    target_link_libraries(dali_operator_test PRIVATE $<TARGET_OBJECTS:dynlink_nvml>)
+  endif(BUILD_NVML)
   target_link_libraries(dali_operator_test PRIVATE "-Wl,--exclude-libs,${exclude_libs}")
   target_link_libraries(dali_operator_test PRIVATE "-pie")
   set_target_properties(dali_operator_test PROPERTIES POSITION_INDEPENDENT_CODE ON)

dali/operators/decoder/nvjpeg/decoupled_api/nvjpeg_decoder_decoupled_api_test.cc

@@ -209,7 +209,7 @@ class HwDecoderUtilizationTest : public ::testing::Test {
     if (!node->op->GetDiagnostic<bool>("using_hw_decoder")) {
       if (nvml::HasCuda11NvmlFunctions()) {
           unsigned int device_count;
-          DALI_CALL(nvml::wrapNvmlDeviceGetCount_v2(&device_count));
+          CUDA_CALL(nvmlDeviceGetCount_v2(&device_count));
           for (unsigned int device_idx = 0; device_idx < device_count; device_idx++) {
             auto info = nvml::GetDeviceInfo(device_idx);
             std::cerr << "Device " << device_idx

dali/operators/reader/nvdecoder/dynlink_nvcuvid.cc

@@ -12,14 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <dlfcn.h>
 #include <stdio.h>
 #include <mutex>
 #include <string>
 #include <unordered_map>
 #include "dali/operators/reader/nvdecoder/dynlink_nvcuvid.h"
 
-#include <dlfcn.h>
-
 namespace {
 
 static char __DriverLibName[] = "libnvcuvid.so";
@@ -50,7 +49,7 @@ void *LoadSymbol(const char *name) {
   return ret;
 }
 
-}
+}  // namespace
 
 // it is defined in the generated file
 typedef void *tLoadSymbol(const char *name);

dali/operators/reader/nvdecoder/dynlink_nvcuvid.h

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#if !defined(DALI_OPERATORS_READER_NVDECODER_DYNLINK_NVCUVID_H_)
+#ifndef DALI_OPERATORS_READER_NVDECODER_DYNLINK_NVCUVID_H_
 #define DALI_OPERATORS_READER_NVDECODER_DYNLINK_NVCUVID_H_
 
 #include <string>
@@ -24,4 +24,4 @@
 bool cuvidInitChecked(unsigned int Flags);
 bool cuvidIsSymbolAvailable(const char *name);
 
-#endif // DALI_OPERATORS_READER_NVDECODER_DYNLINK_NVCUVID_H_
+#endif  // DALI_OPERATORS_READER_NVDECODER_DYNLINK_NVCUVID_H_

dali/operators/reader/video_reader_op_test.cc

@@ -107,15 +107,12 @@ TEST_F(VideoReaderTest, MultipleVideoResolution) {
   float driverVersion = 0;
   char version[80];
 
-  if (nvml::wrapSymbols() != DALISuccess) {
-    FAIL() << "wrapSymbols() failed";
-  }
-  if (nvml::wrapNvmlInit() != DALISuccess) {
-    FAIL() << "wrapNvmlInit() failed";
+  if (nvmlInitChecked() != NVML_SUCCESS) {
+    FAIL() << "nvmlInitChecked() failed";
   }
 
-  if (nvml::wrapNvmlSystemGetDriverVersion(version, sizeof version) != DALISuccess) {
-    FAIL() << "wrapNvmlSystemGetDriverVersion failed!";
+  if (nvmlSystemGetDriverVersion(version, sizeof version) != NVML_SUCCESS) {
+    FAIL() << "nvmlSystemGetDriverVersion failed!";
   }
 
   driverVersion = std::stof(version);

dali/util/CMakeLists.txt

@@ -45,10 +45,28 @@ if(BUILD_NVML)
     "${CMAKE_CURRENT_SOURCE_DIR}/nvml.h"
     "${CMAKE_CURRENT_SOURCE_DIR}/nvml_wrap.h")
 
-  set(DALI_SRCS ${DALI_SRCS}
-    "${CMAKE_CURRENT_SOURCE_DIR}/nvml_wrap.cc")
-endif()
+  if (NOT LINK_DRIVER)
+    set(NVML_GENERATED_STUB "${CMAKE_CURRENT_BINARY_DIR}/dynlink_nvml_gen.cc")
+    add_custom_command(
+        OUTPUT ${NVML_GENERATED_STUB}
+        COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/../../tools/stub_generator/stub_codegen.py --unique_prefix=Nvml --
+                    "${CMAKE_CURRENT_SOURCE_DIR}/../../tools/stub_generator/nvml.json" ${NVML_GENERATED_STUB}
+                    "${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}/nvml.h" "-I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}"
+                    # for some reason QNX fails with 'too many errors emitted' is this is not set
+                    "-ferror-limit=0"
+                    ${DEFAULT_COMPILER_INCLUDE}
+        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../tools/stub_generator/stub_codegen.py
+                "${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}/nvml.h"
+                "${CMAKE_CURRENT_SOURCE_DIR}/../../tools/stub_generator/nvml.json"
+        COMMENT "Running nvml.h stub generator"
+        VERBATIM)
 
+    set_source_files_properties(${NVML_GENERATED_STUB} PROPERTIES GENERATED TRUE)
+    add_library(dynlink_nvml OBJECT nvml_wrap.cc ${NVML_GENERATED_STUB})
+  else()
+    add_library(dynlink_nvml OBJECT nvml_wrap.cc)
+  endif()
+endif()
 
 set(DALI_INST_HDRS ${DALI_INST_HDRS} PARENT_SCOPE)
 set(DALI_SRCS ${DALI_SRCS} PARENT_SCOPE)