Work-around indirect loading of libxrt_core during profiling (Xilinx#…

…2953)

CR1056184 triggers a problem during emulation with profiling enabled.
PR Xilinx#2949 enabled emulation with native XRT kernel APIs, but that
triggered a conflict with profiling which incorrectly and
uncoditionally loads libxrt_core.  Having two shim libraries loaded
with same symbols is not cool to begin with, but with the reference PR
it falt out doesn't work since shim libraries intialize a singleton
xrt_core::system variable.

This PR also fixes the historical weirdness where both libxrt_swemu
and libxrt_hwemu were loaded when emulation was specified.  Now only
one library is loaded depending on the value of XCL_EMULATION_MODE.

src/runtime_src/core/edge/hw_em/CMakeLists.txt

@@ -19,10 +19,6 @@ file(GLOB EM_SRC_FILES
   "${EM_SRC_DIR}/*.cpp"
   "${COMMON_SRC_DIR}/common/*.h"
   "${COMMON_SRC_DIR}/common/*.cpp"
-  "${COMMON_SRC_DIR}/../common/system.h"
-  "${COMMON_SRC_DIR}/../common/system.cpp"
-  "${COMMON_SRC_DIR}/../common/device.cpp"
-  "${COMMON_SRC_DIR}/../common/device.h"
   )
 
 add_definitions(-DXCLHAL_MAJOR_VER=1 -DXCLHAL_MINOR_VER=0 -D__HWEM__)

src/runtime_src/core/pcie/emulation/hw_em/generic_pcie_hal2/system_hwemu.cxx

@@ -30,6 +30,8 @@ namespace {
 static xrt_core::hwemu::system*
 singleton_instance()
 {
+  if (xrt_core::config::get_profile())
+    return nullptr; // waiting for fix to CR1056184
   static xrt_core::hwemu::system singleton;
   return &singleton;
 }

src/runtime_src/xocl/core/platform.cpp

@@ -50,19 +50,27 @@ get_env(const char* env)
   return value_or_empty(std::getenv(env));
 }
 
-XOCL_UNUSED
 static bool
-is_emulation_mode()
+is_emulation()
 {
-  static bool emulation_mode = false;
-  static bool initialized = false;
-  if (!initialized) {
-    std::string env = get_env("XCL_EMULATION_MODE");
-    if(!env.empty() && (env=="sw_emu" || env=="hw_emu") )
-      emulation_mode = true;
-    initialized = true;
-  }
-  return emulation_mode;
+  static bool val = (std::getenv("XCL_EMULATION_MODE") != nullptr);
+  return val;
+}
+
+static bool
+is_sw_emulation()
+{
+  static auto xem = std::getenv("XCL_EMULATION_MODE");
+  static bool swem = xem ? (std::strcmp(xem,"sw_emu")==0) : false;
+  return swem;
+}
+
+static bool
+is_hw_emulation()
+{
+  static auto xem = std::getenv("XCL_EMULATION_MODE");
+  static bool hwem = xem ? (std::strcmp(xem,"hw_emu")==0) : false;
+  return hwem;
 }
 
 static std::vector<char>
@@ -139,10 +147,6 @@ class platform::xrt_device_manager
     std::reverse(m_hw.begin(),m_hw.end());
     std::reverse(m_hwem.begin(),m_hwem.end());
     std::reverse(m_swem.begin(),m_swem.end());
-
-    // Sanity checks
-    if (m_hwem.size() != m_swem.size())
-      throw xocl::error(CL_DEVICE_NOT_FOUND,"Emulation device mismatch");
   }
 
   bool
@@ -223,20 +227,26 @@ platform()
 
   XOCL_DEBUG(std::cout,"xocl::platform::platform(",m_uid,")\n");
 
-  if (is_emulation_mode()) {
+  if (is_sw_emulation()) {
+    while (auto swem_device = m_device_mgr->get_swem_device()) {
+      auto udev = std::make_unique<xocl::device>(this,swem_device,nullptr);
+      auto dev = udev.release();
+      add_device(dev);
+      dev->release();
+    }
+  }
+
+  if (is_hw_emulation()) {
     while (auto hwem_device = m_device_mgr->get_hwem_device()) {
-      auto swem_device = m_device_mgr->get_swem_device();
-      auto udev = std::make_unique<xocl::device>(this,swem_device,hwem_device);
-#ifndef PMD_OCL
+      auto udev = std::make_unique<xocl::device>(this,nullptr,hwem_device);
       auto dev = udev.release();
       add_device(dev);
       dev->release();
-#endif
     }
   }
 
   //User can target either emulation or board. Not both at the same time.
-  if (!is_emulation_mode() && m_device_mgr->has_hw_devices()) {
+  if (!is_emulation() && m_device_mgr->has_hw_devices()) {
     while (xrt::device* hw_device = m_device_mgr->get_hw_device()) {
       auto udev = std::make_unique<xocl::device>(this,hw_device,nullptr,nullptr);
       auto dev = udev.release();

src/runtime_src/xrt/device/hal.cpp

@@ -87,12 +87,20 @@ dllpath(const boost::filesystem::path& root, const std::string& libnm)
 }
 
 static bool
-isEmulationMode()
+is_emulation()
 {
   static bool val = (std::getenv("XCL_EMULATION_MODE") != nullptr);
   return val;
 }
 
+static bool
+is_sw_emulation()
+{
+  static auto xem = std::getenv("XCL_EMULATION_MODE");
+  static bool swem = xem ? (std::strcmp(xem,"sw_emu")==0) : false;
+  return swem;
+}
+
 // Open the HAL implementation dll and construct a hal::device for
 // each board detected by the implementation
 static void
@@ -156,20 +164,20 @@ loadDevices()
   // xrt
   bfs::path xrt(emptyOrValue(getenv("XILINX_XRT")));
 
-  if (!xrt.empty() && !isEmulationMode()) {
+  if (!xrt.empty() && !is_emulation()) {
     directoryOrError(xrt);
     auto p = dllpath(xrt,"xrt_core");
     if (isDLL(p))
       createHalDevices(devices,p.string());
   }
 
-  if (devices.empty()) { // if failed libxrt_core load, try libxrt_aws
+  if (devices.empty() && !is_emulation()) { // if failed libxrt_core load, try libxrt_aws
     auto p = dllpath(xrt,"xrt_aws");
     if (isDLL(p))
       createHalDevices(devices,p.string());
   }
 
-  if (!xrt.empty() && isEmulationMode()) {
+  if (!xrt.empty() && is_emulation() && !is_sw_emulation()) {
     directoryOrError(xrt);
 
     auto hw_em_driver_path = xrt::config::get_hw_em_driver();
@@ -183,7 +191,7 @@ loadDevices()
       createHalDevices(devices,hw_em_driver_path);
   }
 
-  if (!xrt.empty() && isEmulationMode()) {
+  if (!xrt.empty() && is_emulation() && is_sw_emulation()) {
     directoryOrError(xrt);
 
     auto sw_em_driver_path = xrt::config::get_sw_em_driver();