Skip to content

Commit

Permalink
Initial code drop
Browse files Browse the repository at this point in the history
  • Loading branch information
ptrendx committed Jun 18, 2018
1 parent a5f24e2 commit f4087bc
Showing 335 changed files with 48,035 additions and 26 deletions.
13 changes: 13 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
*~
build
*.pyc
doxygen
html
latex
.vimrc
.ycm_extra_conf.py
**/imgui.ini
.*.swp
valgrind-out.txt
**/.ipynb_checkpoints
.dockerignore
9 changes: 9 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[submodule "third_party/googletest"]
path = third_party/googletest
url = https://github.com/google/googletest.git
[submodule "third_party/benchmark"]
path = third_party/benchmark
url = https://github.com/google/benchmark.git
[submodule "third_party/pybind11"]
path = third_party/pybind11
url = https://github.com/pybind/pybind11.git
782 changes: 782 additions & 0 deletions Acknowledgements.txt

Large diffs are not rendered by default.

81 changes: 81 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

cmake_minimum_required(VERSION 3.5)

project(DALI CXX)

# Build options
option(BUILD_TEST "Build googletest test suite" ON)
option(BUILD_BENCHMARK "Build benchmark suite" ON)
option(BUILD_NVTX "Build with NVTX profiling enabled" OFF)
option(BUILD_PYTHON "Build python bindings" ON)
option(BUILD_LMDB "Build LMDB readers" OFF)
option(BUILD_TENSORFLOW "Build TensorFlow plugin" OFF)

# Helper function to remove elements from a variable
function (remove TARGET INPUT)
foreach(ITEM ${ARGN})
list(REMOVE_ITEM INPUT "${ITEM}")
endforeach()
set(${TARGET} ${INPUT} PARENT_SCOPE)
endfunction(remove)

# Default to release build
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING
"Build type from [Debug, Release]. For perf testing, build Release" FORCE)
endif()

if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
message(STATUS "Building in DEBUG mode")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb -DDALI_DEBUG=1")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -g -G -DDALI_DEBUG=1")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDALI_DEBUG=0")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -DDALI_DEBUG=0")
endif()

# Cmake path
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules)

# Dependencies
include(cmake/Dependencies.cmake)

# CXX flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-unused-variable -Wno-unused-function -fno-strict-aliasing -O2 -fPIC")

# Add ptx & bin flags for cuda
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} \
-gencode arch=compute_35,code=sm_35 \
-gencode arch=compute_50,code=sm_50 \
-gencode arch=compute_52,code=sm_52 \
-gencode arch=compute_60,code=sm_60 \
-gencode arch=compute_61,code=sm_61 \
-gencode arch=compute_70,code=sm_70 \
-gencode arch=compute_70,code=compute_70")

# Project dir
include_directories(BEFORE ${PROJECT_SOURCE_DIR})
include_directories(BEFORE ${PROJECT_BINARY_DIR})
cuda_include_directories(${PROJECT_SOURCE_DIR})

# Project build
add_subdirectory(dali)

# HACK: Add __init__.pys as needed
file(WRITE ${CMAKE_BINARY_DIR}/dali/__init__.py "")

add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/lint.cmake)

13 changes: 13 additions & 0 deletions COPYRIGHT
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
45 changes: 45 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#########################################################################################
## Stage 2: build DALI wheels using manylinux1 (CentOS 5 derivative)
#########################################################################################
ARG DEPS_IMAGE_NAME
FROM ${DEPS_IMAGE_NAME}

ARG PYVER=2.7
ARG PYV=27

ENV PYVER=${PYVER} PYV=${PYV} PYTHONPATH=/opt/python/v

ENV PYBIN=${PYTHONPATH}/bin \
PYLIB=${PYTHONPATH}/lib

ENV PATH=${PYBIN}:${PATH} \
LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:/opt/dali/build:${PYLIB}:${LD_LIBRARY_PATH}

RUN ln -s /opt/python/cp${PYV}* /opt/python/v

RUN pip install future numpy setuptools wheel tensorflow-gpu && \
rm -rf /root/.cache/pip/

RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
ldconfig

WORKDIR /opt/dali

COPY . .

WORKDIR /opt/dali/build

RUN LD_LIBRARY_PATH="${PWD}:${LD_LIBRARY_PATH}" && \
cmake ../ -DCMAKE_INSTALL_PREFIX=. \
-DBUILD_TEST=ON -DBUILD_BENCHMARK=ON -DBUILD_PYTHON=ON \
-DBUILD_LMDB=ON -DBUILD_TENSORFLOW=ON && \
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install

ARG NVIDIA_BUILD_ID
ENV NVIDIA_BUILD_ID ${NVIDIA_BUILD_ID:-0}

RUN pip wheel -v dali/python \
--build-option --python-tag=$(basename /opt/python/cp${PYV}-*) \
--build-option --plat-name=manylinux1_x86_64 \
--build-option --build-number=${NVIDIA_BUILD_ID} && \
../dali/python/bundle-wheel.sh nvidia_dali-*.whl
86 changes: 86 additions & 0 deletions Dockerfile.deps
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#########################################################################################
## Stage 1: build DALI dependencies using manylinux1 (CentOS 5 derivative)
#########################################################################################
FROM quay.io/pypa/manylinux1_x86_64

# Install yum Dependencies
RUN yum install -y zip

# Don't want the short-unicode version for Python 2.7
RUN rm -f /opt/python/cp27-cp27m

# Boost
RUN BOOST_VERSION=1.66.0 && \
cd /usr/local && \
curl -L https://dl.bintray.com/boostorg/release/1.66.0/source/boost_${BOOST_VERSION//./_}.tar.gz | tar -xzf - && \
ln -s ../boost_${BOOST_VERSION//./_}/boost include/boost

# CMake
RUN CMAKE_VERSION=3.11 && \
CMAKE_BUILD=3.11.3 && \
curl -L https://cmake.org/files/v${CMAKE_VERSION}/cmake-${CMAKE_BUILD}.tar.gz | tar -xzf - && \
cd /cmake-${CMAKE_BUILD} && \
./bootstrap --parallel=$(grep ^processor /proc/cpuinfo | wc -l) && \
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
rm -rf /cmake-${CMAKE_BUILD}

# protobuf v3.5.1
RUN PROTOBUF_VERSION=3.5.1 && \
curl -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar -xzf - && \
cd /protobuf-${PROTOBUF_VERSION} && \
./autogen.sh && \
./configure --prefix=/usr/local 2>&1 > /dev/null && \
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install 2>&1 > /dev/null && \
rm -rf /protobuf-${PROTOBUF_VERSION}

# LMDB
RUN LMDB_VERSION=0.9.22 && \
git clone -b LMDB_${LMDB_VERSION} --single-branch https://github.com/LMDB/lmdb && \
cd /lmdb/libraries/liblmdb && \
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
rm -rf /lmdb

# OpenCV
RUN OPENCV_VERSION=3.1.0 && \
curl -L https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.tar.gz | tar -xzf - && \
cd /opencv-${OPENCV_VERSION} && \
cmake -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local \
-DWITH_CUDA=OFF -DWITH_1394=OFF -DWITH_IPP=OFF -DWITH_OPENCL=OFF -DWITH_GTK=OFF \
-DBUILD_DOCS=OFF -DBUILD_TESTS=OFF -DBUILD_PERF_TESTS=OFF \
-DBUILD_opencv_cudalegacy=OFF -DBUILD_opencv_stitching=OFF . && \
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
rm -rf /opencv-${OPENCV_VERSION}

# libjpeg-turbo
#
# Note: the preceding OpenCV installation intentionally does NOT use libjpeg-turbo.
# DALI will directly call libjpeg-turbo first, and if it fails, DALI will fall back
# to OpenCV, which in turn will call its bundled (built-from-source) libjpeg.
# To be extra sure OpenCV doesn't pick up libjpeg-turbo (in which case we'd have no
# fallback), libjpeg-turbo is built and installed _after_ OpenCV.
#
RUN JPEG_TURBO_VERSION=1.5.2 && \
curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${JPEG_TURBO_VERSION}.tar.gz | tar -xzf - && \
cd /libjpeg-turbo-${JPEG_TURBO_VERSION} && \
autoreconf -fiv && \
./configure --enable-shared --prefix=/usr/local 2>&1 >/dev/null && \
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install 2>&1 >/dev/null && \
rm -rf /libjpeg-turbo-${JPEG_TURBO_VERSION}

# CUDA
RUN CUDA_VERSION=9.0 && \
CUDA_BUILD=9.0.176_384.81 && \
curl -LO https://developer.nvidia.com/compute/cuda/${CUDA_VERSION}/Prod/local_installers/cuda_${CUDA_BUILD}_linux-run && \
chmod +x cuda_${CUDA_BUILD}_linux-run && \
./cuda_${CUDA_BUILD}_linux-run --silent --no-opengl-libs --toolkit && \
rm -f cuda_${CUDA_BUILD}_linux-run

# NVJPEG
RUN NVJPEG_VERSION=9.0 && \
curl -L https://developer.download.nvidia.com/compute/redist/libnvjpeg/cuda-linux64-nvjpeg-${NVJPEG_VERSION}.tar.gz | tar -xzf - && \
cd /cuda-linux64-nvjpeg/ && \
mv lib64/libnvjpeg.so* /usr/local/lib/ && \
mv include/nvjpeg.h /usr/local/include/ && \
rm -rf /cuda-linux64-nvjpeg


Loading

0 comments on commit f4087bc

Please sign in to comment.