forked from NVIDIA/DALI
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
335 changed files
with
48,035 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
*~ | ||
build | ||
*.pyc | ||
doxygen | ||
html | ||
latex | ||
.vimrc | ||
.ycm_extra_conf.py | ||
**/imgui.ini | ||
.*.swp | ||
valgrind-out.txt | ||
**/.ipynb_checkpoints | ||
.dockerignore |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
[submodule "third_party/googletest"] | ||
path = third_party/googletest | ||
url = https://github.com/google/googletest.git | ||
[submodule "third_party/benchmark"] | ||
path = third_party/benchmark | ||
url = https://github.com/google/benchmark.git | ||
[submodule "third_party/pybind11"] | ||
path = third_party/pybind11 | ||
url = https://github.com/pybind/pybind11.git |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
cmake_minimum_required(VERSION 3.5) | ||
|
||
project(DALI CXX) | ||
|
||
# Build options | ||
option(BUILD_TEST "Build googletest test suite" ON) | ||
option(BUILD_BENCHMARK "Build benchmark suite" ON) | ||
option(BUILD_NVTX "Build with NVTX profiling enabled" OFF) | ||
option(BUILD_PYTHON "Build python bindings" ON) | ||
option(BUILD_LMDB "Build LMDB readers" OFF) | ||
option(BUILD_TENSORFLOW "Build TensorFlow plugin" OFF) | ||
|
||
# Helper function to remove elements from a variable | ||
function (remove TARGET INPUT) | ||
foreach(ITEM ${ARGN}) | ||
list(REMOVE_ITEM INPUT "${ITEM}") | ||
endforeach() | ||
set(${TARGET} ${INPUT} PARENT_SCOPE) | ||
endfunction(remove) | ||
|
||
# Default to release build | ||
if (NOT CMAKE_BUILD_TYPE) | ||
set(CMAKE_BUILD_TYPE "Release" CACHE STRING | ||
"Build type from [Debug, Release]. For perf testing, build Release" FORCE) | ||
endif() | ||
|
||
if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") | ||
message(STATUS "Building in DEBUG mode") | ||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb -DDALI_DEBUG=1") | ||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -g -G -DDALI_DEBUG=1") | ||
else() | ||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDALI_DEBUG=0") | ||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -DDALI_DEBUG=0") | ||
endif() | ||
|
||
# Cmake path | ||
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules) | ||
|
||
# Dependencies | ||
include(cmake/Dependencies.cmake) | ||
|
||
# CXX flags | ||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-unused-variable -Wno-unused-function -fno-strict-aliasing -O2 -fPIC") | ||
|
||
# Add ptx & bin flags for cuda | ||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} \ | ||
-gencode arch=compute_35,code=sm_35 \ | ||
-gencode arch=compute_50,code=sm_50 \ | ||
-gencode arch=compute_52,code=sm_52 \ | ||
-gencode arch=compute_60,code=sm_60 \ | ||
-gencode arch=compute_61,code=sm_61 \ | ||
-gencode arch=compute_70,code=sm_70 \ | ||
-gencode arch=compute_70,code=compute_70") | ||
|
||
# Project dir | ||
include_directories(BEFORE ${PROJECT_SOURCE_DIR}) | ||
include_directories(BEFORE ${PROJECT_BINARY_DIR}) | ||
cuda_include_directories(${PROJECT_SOURCE_DIR}) | ||
|
||
# Project build | ||
add_subdirectory(dali) | ||
|
||
# HACK: Add __init__.pys as needed | ||
file(WRITE ${CMAKE_BINARY_DIR}/dali/__init__.py "") | ||
|
||
add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/lint.cmake) | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
######################################################################################### | ||
## Stage 2: build DALI wheels using manylinux1 (CentOS 5 derivative) | ||
######################################################################################### | ||
ARG DEPS_IMAGE_NAME | ||
FROM ${DEPS_IMAGE_NAME} | ||
|
||
ARG PYVER=2.7 | ||
ARG PYV=27 | ||
|
||
ENV PYVER=${PYVER} PYV=${PYV} PYTHONPATH=/opt/python/v | ||
|
||
ENV PYBIN=${PYTHONPATH}/bin \ | ||
PYLIB=${PYTHONPATH}/lib | ||
|
||
ENV PATH=${PYBIN}:${PATH} \ | ||
LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:/opt/dali/build:${PYLIB}:${LD_LIBRARY_PATH} | ||
|
||
RUN ln -s /opt/python/cp${PYV}* /opt/python/v | ||
|
||
RUN pip install future numpy setuptools wheel tensorflow-gpu && \ | ||
rm -rf /root/.cache/pip/ | ||
|
||
RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ | ||
ldconfig | ||
|
||
WORKDIR /opt/dali | ||
|
||
COPY . . | ||
|
||
WORKDIR /opt/dali/build | ||
|
||
RUN LD_LIBRARY_PATH="${PWD}:${LD_LIBRARY_PATH}" && \ | ||
cmake ../ -DCMAKE_INSTALL_PREFIX=. \ | ||
-DBUILD_TEST=ON -DBUILD_BENCHMARK=ON -DBUILD_PYTHON=ON \ | ||
-DBUILD_LMDB=ON -DBUILD_TENSORFLOW=ON && \ | ||
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install | ||
|
||
ARG NVIDIA_BUILD_ID | ||
ENV NVIDIA_BUILD_ID ${NVIDIA_BUILD_ID:-0} | ||
|
||
RUN pip wheel -v dali/python \ | ||
--build-option --python-tag=$(basename /opt/python/cp${PYV}-*) \ | ||
--build-option --plat-name=manylinux1_x86_64 \ | ||
--build-option --build-number=${NVIDIA_BUILD_ID} && \ | ||
../dali/python/bundle-wheel.sh nvidia_dali-*.whl |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
######################################################################################### | ||
## Stage 1: build DALI dependencies using manylinux1 (CentOS 5 derivative) | ||
######################################################################################### | ||
FROM quay.io/pypa/manylinux1_x86_64 | ||
|
||
# Install yum Dependencies | ||
RUN yum install -y zip | ||
|
||
# Don't want the short-unicode version for Python 2.7 | ||
RUN rm -f /opt/python/cp27-cp27m | ||
|
||
# Boost | ||
RUN BOOST_VERSION=1.66.0 && \ | ||
cd /usr/local && \ | ||
curl -L https://dl.bintray.com/boostorg/release/1.66.0/source/boost_${BOOST_VERSION//./_}.tar.gz | tar -xzf - && \ | ||
ln -s ../boost_${BOOST_VERSION//./_}/boost include/boost | ||
|
||
# CMake | ||
RUN CMAKE_VERSION=3.11 && \ | ||
CMAKE_BUILD=3.11.3 && \ | ||
curl -L https://cmake.org/files/v${CMAKE_VERSION}/cmake-${CMAKE_BUILD}.tar.gz | tar -xzf - && \ | ||
cd /cmake-${CMAKE_BUILD} && \ | ||
./bootstrap --parallel=$(grep ^processor /proc/cpuinfo | wc -l) && \ | ||
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \ | ||
rm -rf /cmake-${CMAKE_BUILD} | ||
|
||
# protobuf v3.5.1 | ||
RUN PROTOBUF_VERSION=3.5.1 && \ | ||
curl -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar -xzf - && \ | ||
cd /protobuf-${PROTOBUF_VERSION} && \ | ||
./autogen.sh && \ | ||
./configure --prefix=/usr/local 2>&1 > /dev/null && \ | ||
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install 2>&1 > /dev/null && \ | ||
rm -rf /protobuf-${PROTOBUF_VERSION} | ||
|
||
# LMDB | ||
RUN LMDB_VERSION=0.9.22 && \ | ||
git clone -b LMDB_${LMDB_VERSION} --single-branch https://github.com/LMDB/lmdb && \ | ||
cd /lmdb/libraries/liblmdb && \ | ||
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \ | ||
rm -rf /lmdb | ||
|
||
# OpenCV | ||
RUN OPENCV_VERSION=3.1.0 && \ | ||
curl -L https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.tar.gz | tar -xzf - && \ | ||
cd /opencv-${OPENCV_VERSION} && \ | ||
cmake -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local \ | ||
-DWITH_CUDA=OFF -DWITH_1394=OFF -DWITH_IPP=OFF -DWITH_OPENCL=OFF -DWITH_GTK=OFF \ | ||
-DBUILD_DOCS=OFF -DBUILD_TESTS=OFF -DBUILD_PERF_TESTS=OFF \ | ||
-DBUILD_opencv_cudalegacy=OFF -DBUILD_opencv_stitching=OFF . && \ | ||
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \ | ||
rm -rf /opencv-${OPENCV_VERSION} | ||
|
||
# libjpeg-turbo | ||
# | ||
# Note: the preceding OpenCV installation intentionally does NOT use libjpeg-turbo. | ||
# DALI will directly call libjpeg-turbo first, and if it fails, DALI will fall back | ||
# to OpenCV, which in turn will call its bundled (built-from-source) libjpeg. | ||
# To be extra sure OpenCV doesn't pick up libjpeg-turbo (in which case we'd have no | ||
# fallback), libjpeg-turbo is built and installed _after_ OpenCV. | ||
# | ||
RUN JPEG_TURBO_VERSION=1.5.2 && \ | ||
curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${JPEG_TURBO_VERSION}.tar.gz | tar -xzf - && \ | ||
cd /libjpeg-turbo-${JPEG_TURBO_VERSION} && \ | ||
autoreconf -fiv && \ | ||
./configure --enable-shared --prefix=/usr/local 2>&1 >/dev/null && \ | ||
make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install 2>&1 >/dev/null && \ | ||
rm -rf /libjpeg-turbo-${JPEG_TURBO_VERSION} | ||
|
||
# CUDA | ||
RUN CUDA_VERSION=9.0 && \ | ||
CUDA_BUILD=9.0.176_384.81 && \ | ||
curl -LO https://developer.nvidia.com/compute/cuda/${CUDA_VERSION}/Prod/local_installers/cuda_${CUDA_BUILD}_linux-run && \ | ||
chmod +x cuda_${CUDA_BUILD}_linux-run && \ | ||
./cuda_${CUDA_BUILD}_linux-run --silent --no-opengl-libs --toolkit && \ | ||
rm -f cuda_${CUDA_BUILD}_linux-run | ||
|
||
# NVJPEG | ||
RUN NVJPEG_VERSION=9.0 && \ | ||
curl -L https://developer.download.nvidia.com/compute/redist/libnvjpeg/cuda-linux64-nvjpeg-${NVJPEG_VERSION}.tar.gz | tar -xzf - && \ | ||
cd /cuda-linux64-nvjpeg/ && \ | ||
mv lib64/libnvjpeg.so* /usr/local/lib/ && \ | ||
mv include/nvjpeg.h /usr/local/include/ && \ | ||
rm -rf /cuda-linux64-nvjpeg | ||
|
||
|
Oops, something went wrong.