Skip to content

Commit

Permalink
Use dlr binary from compilation artifact if avaliable (#197)
Browse files Browse the repository at this point in the history 
* Use dlr binary from compilation artifact if avaliable

* Avoid crashes when multiple dlrs are loaded.

* Fix setup to handle find_lib_path change

Co-authored-by: varunnag <[email protected]>
  • Loading branch information
@cnv1989
cnv1989 and varunnag authored Jun 18, 2020
1 parent acd1b8a commit ef33b66
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 110 deletions.
4 changes: 2 additions & 2 deletions python/dlr/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def _is_module_found(name):

# Wrapper class
class DLRModel(IDLRModel):
def __init__(self, model_path, dev_type=None, dev_id=None, error_log_file=None):
def __init__(self, model_path, dev_type=None, dev_id=None, error_log_file=None, use_default_dlr=False):
self.neo_logger = create_logger(log_file=error_log_file)
try:
# Find correct runtime implementation for the model
Expand Down Expand Up @@ -79,7 +79,7 @@ def __init__(self, model_path, dev_type=None, dev_id=None, error_log_file=None):
dev_type = 'cpu'
if dev_id is None:
dev_id = 0
self._impl = DLRModelImpl(model_path, dev_type, dev_id)
self._impl = DLRModelImpl(model_path, dev_type, dev_id, error_log_file, use_default_dlr)
except Exception as ex:
self.neo_logger.exception("error in DLRModel instantiation {}".format(ex))
raise ex
Expand Down
159 changes: 85 additions & 74 deletions python/dlr/dlr_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,13 @@
from ctypes import c_void_p, c_int, c_char_p, byref, POINTER, c_longlong
import numpy as np
import os
import sys
from pathlib import Path

from .api import IDLRModel
from .compatibility import check_tensorrt_compatibility

from .libpath import find_lib_path
from .neologger import create_logger

# Map from dtype string to ctype type.
# Equivalent to np.ctypeslib.as_ctypes_type which requires numpy>=1.16.1
Expand Down Expand Up @@ -38,57 +41,48 @@ class DLRError(Exception):
"""Error thrown by DLR"""
pass

def _check_call(ret):
"""
Check the return value of C API call
This function will raise exception when error occurs.
Wrap every API call with this function
Parameters
----------
ret : int
return value from API calls
"""
if ret != 0:
raise DLRError(_LIB.DLRGetLastError().decode('ascii'))

def _load_lib():
def _load_lib(lib_path):
"""Load DLR library."""
lib_paths = find_lib_path()
if len(lib_paths) == 0:
return None
try:
pathBackup = os.environ['PATH'].split(os.pathsep)
except KeyError:
pathBackup = []
lib_success = False
os_error_list = []
for lib_path in lib_paths:
try:
# needed when the lib is linked with non-system-available dependencies
os.environ['PATH'] = os.pathsep.join(pathBackup + [os.path.dirname(lib_path)])
lib = ctypes.cdll.LoadLibrary(lib_path)
lib_success = True
except OSError as e:
os_error_list.append(str(e))
continue
finally:
os.environ['PATH'] = os.pathsep.join(pathBackup)
if not lib_success:
libname = os.path.basename(lib_paths[0])

try:
# needed when the lib is linked with non-system-available dependencies
os.environ['PATH'] = os.pathsep.join(pathBackup + [os.path.dirname(lib_path)])
lib = ctypes.cdll.LoadLibrary(lib_path)
except Exception as e:
libname = os.path.basename(lib_path)
raise DLRError(
'DLR library ({}) could not be loaded.\n'.format(libname) +
'Likely causes:\n' +
' * OpenMP runtime is not installed ' +
'(vcomp140.dll or libgomp-1.dll for Windows, ' +
'libgomp.so for UNIX-like OSes)\n' +
' * You are running 32-bit Python on a 64-bit OS\n' +
'Error message(s): {}\n'.format(os_error_list))
'Error message(s): {}\n'.format(e))
finally:
os.environ['PATH'] = os.pathsep.join(pathBackup)

lib.DLRGetLastError.restype = ctypes.c_char_p
return lib

# load the DLR library globally
_LIB = _load_lib()

def _check_call(ret):
"""
Check the return value of C API call
This function will raise exception when error occurs.
Wrap every API call with this function
Parameters
----------
ret : int
return value from API calls
"""
if ret != 0:
raise DLRError(DLRModelImpl._LIB.DLRGetLastError().decode('ascii'))


class DLRModelImpl(IDLRModel):
"""
Expand All @@ -103,26 +97,16 @@ class DLRModelImpl(IDLRModel):
dev_id : int
Device ID
"""

def _lazy_init_output_shape(self):
self.output_shapes = []
self.output_size_dim = []
for i in range(self.num_outputs):
shape = self._get_output_shape(i)
self.output_shapes.append(shape)

def _parse_backend(self):
backend = c_char_p()
_check_call(_LIB.GetDLRBackend(byref(self.handle),
byref(backend)))
return backend.value.decode('ascii')

def _get_version(self):
version = c_char_p()
_check_call(_LIB.GetDLRVersion(byref(version)))
return version.value.decode('ascii')

def __init__(self, model_path, dev_type='cpu', dev_id=0):

# TVM crashes with `Check failed: override: Global PackedFunc runtime.module.loadfile_so is already registered`
# when we try to link muliple versions of the dlr withing the same process. In order to prevent this.
# libdlr.so will be loaded when the first `DLRModelImpl` instance is created.
# Once libdlr.so is loaded it will be used to run the first and all consequent models.
_LIB = None

def __init__(self, model_path, dev_type='cpu', dev_id=0, error_log_file=None, use_default_dlr=False):
self.logger = create_logger(log_file=error_log_file)

if not os.path.exists(model_path):
raise ValueError("model_path %s doesn't exist" % model_path)
# Backwards compatibility for .tensorrt artifacts.
Expand All @@ -135,7 +119,10 @@ def __init__(self, model_path, dev_type='cpu', dev_id=0):
'gpu': 2,
'opencl': 4,
}
_check_call(_LIB.CreateDLRModel(byref(self.handle),
self.model_path = model_path
self.use_default_dlr = use_default_dlr
self._init_libdlr()
_check_call(DLRModelImpl._LIB.CreateDLRModel(byref(self.handle),
c_char_p(model_path.encode()),
c_int(device_table[dev_type]),
c_int(dev_id)))
Expand Down Expand Up @@ -166,20 +153,44 @@ def __init__(self, model_path, dev_type='cpu', dev_id=0):
def __del__(self):
if getattr(self, "handle", None) is not None and self.handle is not None:
if getattr(self, "lib", None) is not None:
_check_call(_LIB.DeleteDLRModel(byref(self.handle)))
_check_call(DLRModelImpl._LIB.DeleteDLRModel(byref(self.handle)))
self.handle = None

def _lazy_init_output_shape(self):
self.output_shapes = []
self.output_size_dim = []
for i in range(self.num_outputs):
shape = self._get_output_shape(i)
self.output_shapes.append(shape)

def _parse_backend(self):
backend = c_char_p()
_check_call(DLRModelImpl._LIB.GetDLRBackend(byref(self.handle),
byref(backend)))
return backend.value.decode('ascii')

def _get_version(self):
version = c_char_p()
_check_call(DLRModelImpl._LIB.GetDLRVersion(byref(version)))
return version.value.decode('ascii')

def _init_libdlr(self):
if DLRModelImpl._LIB:
self.logger.info("{} is already loaded!".format(DLRModelImpl._LIB))
return
DLRModelImpl._LIB = _load_lib(find_lib_path(self.model_path, self.use_default_dlr, self.logger))

def _get_num_inputs(self):
"""Get the number of inputs of a network"""
num_inputs = c_int()
_check_call(_LIB.GetDLRNumInputs(byref(self.handle),
_check_call(DLRModelImpl._LIB.GetDLRNumInputs(byref(self.handle),
byref(num_inputs)))
return num_inputs.value

def _get_num_weights(self):
"""Get the number of weights of a network"""
num_weights = c_int()
_check_call(_LIB.GetDLRNumWeights(byref(self.handle),
_check_call(DLRModelImpl._LIB.GetDLRNumWeights(byref(self.handle),
byref(num_weights)))
return num_weights.value

Expand All @@ -195,15 +206,15 @@ def get_input_names(self):

def has_metadata(self) -> bool:
flag = ctypes.c_bool()
_check_call(_LIB.GetDLRHasMetadata(byref(self.handle), byref(flag)))
_check_call(DLRModelImpl._LIB.GetDLRHasMetadata(byref(self.handle), byref(flag)))
return flag.value

def _fetch_output_names(self):
self.output_names = []
try:
for i in range(self.num_outputs):
name = c_char_p()
_check_call(_LIB.GetDLROutputName(byref(self.handle), i, byref(name)))
_check_call(DLRModelImpl._LIB.GetDLROutputName(byref(self.handle), i, byref(name)))
self.output_names.append(name.value.decode('utf-8'))
except Exception:
"""
Expand All @@ -223,7 +234,7 @@ def _fetch_input_dtypes(self):
try:
for i in range(self.num_inputs):
dtype = c_char_p()
_check_call(_LIB.GetDLRInputType(byref(self.handle), i, byref(dtype)))
_check_call(DLRModelImpl._LIB.GetDLRInputType(byref(self.handle), i, byref(dtype)))
self.input_dtypes.append(dtype.value.decode('utf-8'))
except Exception:
"""
Expand All @@ -237,7 +248,7 @@ def _fetch_output_dtypes(self):
try:
for i in range(self.num_outputs):
dtype = c_char_p()
_check_call(_LIB.GetDLROutputType(byref(self.handle), i, byref(dtype)))
_check_call(DLRModelImpl._LIB.GetDLROutputType(byref(self.handle), i, byref(dtype)))
self.output_dtypes.append(dtype.value.decode('utf-8'))
except Exception:
"""
Expand Down Expand Up @@ -293,7 +304,7 @@ def get_version(self):

def _get_input_name(self, index):
name = ctypes.c_char_p()
_check_call(_LIB.GetDLRInputName(byref(self.handle),
_check_call(DLRModelImpl._LIB.GetDLRInputName(byref(self.handle),
c_int(index), byref(name)))
return name.value.decode("utf-8")

Expand All @@ -305,7 +316,7 @@ def _get_input_index(self, name) -> int:

def _get_weight_name(self, index):
name = ctypes.c_char_p()
_check_call(_LIB.GetDLRWeightName(byref(self.handle),
_check_call(DLRModelImpl._LIB.GetDLRWeightName(byref(self.handle),
c_int(index), byref(name)))
return name.value.decode("utf-8")

Expand Down Expand Up @@ -337,7 +348,7 @@ def _set_input(self, name, data):
in_data = np.ascontiguousarray(data, dtype=input_dtype)
shape = np.array(in_data.shape, dtype=np.int64)
self.input_shapes[name] = shape
_check_call(_LIB.SetDLRInput(byref(self.handle),
_check_call(DLRModelImpl._LIB.SetDLRInput(byref(self.handle),
c_char_p(name.encode('utf-8')),
shape.ctypes.data_as(POINTER(c_longlong)),
in_data.ctypes.data_as(POINTER(input_ctype)),
Expand All @@ -347,12 +358,12 @@ def _set_input(self, name, data):

def _run(self):
"""A light wrapper to call run in the DLR backend."""
_check_call(_LIB.RunDLRModel(byref(self.handle)))
_check_call(DLRModelImpl._LIB.RunDLRModel(byref(self.handle)))

def _get_num_outputs(self):
"""Get the number of outputs of a network"""
num_outputs = c_int()
_check_call(_LIB.GetDLRNumOutputs(byref(self.handle),
_check_call(DLRModelImpl._LIB.GetDLRNumOutputs(byref(self.handle),
byref(num_outputs)))
return num_outputs.value

Expand All @@ -374,7 +385,7 @@ def _get_output_size_dim(self, index):
idx = ctypes.c_int(index)
size = ctypes.c_longlong()
dim = ctypes.c_int()
_check_call(_LIB.GetDLROutputSizeDim(byref(self.handle), idx,
_check_call(DLRModelImpl._LIB.GetDLROutputSizeDim(byref(self.handle), idx,
byref(size), byref(dim)))
return size.value, dim.value

Expand All @@ -396,7 +407,7 @@ def _get_output_shape(self, index):
self.output_size_dim = [(0, 0)] * self._get_num_outputs()
self.output_size_dim[index] = (size, dim)
shape = np.zeros(dim, dtype=np.int64)
_check_call(_LIB.GetDLROutputShape(byref(self.handle),
_check_call(DLRModelImpl._LIB.GetDLROutputShape(byref(self.handle),
c_int(index),
shape.ctypes.data_as(ctypes.POINTER(ctypes.c_longlong))))
return shape
Expand All @@ -420,7 +431,7 @@ def _get_output(self, index):
output_dtype = self.get_output_dtype(index)
output_ctype = _get_ctype_from_dtype(output_dtype)
output = np.zeros(self.output_size_dim[index][0], dtype=output_dtype)
_check_call(_LIB.GetDLROutput(byref(self.handle), c_int(index),
_check_call(DLRModelImpl._LIB.GetDLROutput(byref(self.handle), c_int(index),
output.ctypes.data_as(ctypes.POINTER(output_ctype))))
out = output.reshape(self.output_shapes[index])
return out
Expand Down Expand Up @@ -493,7 +504,7 @@ def get_input(self, name, shape=None):
shape = self.input_shapes[name]
shape = np.array(shape)
out = np.zeros(shape.prod(), dtype=input_dtype)
_check_call(_LIB.GetDLRInput(byref(self.handle),
_check_call(DLRModelImpl._LIB.GetDLRInput(byref(self.handle),
c_char_p(name.encode('utf-8')),
out.ctypes.data_as(ctypes.POINTER(input_ctype))))
out = out.reshape(shape)
Expand Down
Loading

0 comments on commit ef33b66

Please sign in to comment.