Initial code drop

Author: ptrendx

.gitignore

@@ -0,0 +1,13 @@
+*~
+build
+*.pyc
+doxygen
+html
+latex
+.vimrc
+.ycm_extra_conf.py
+**/imgui.ini
+.*.swp
+valgrind-out.txt
+**/.ipynb_checkpoints
+.dockerignore

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://github.com/google/googletest.git
+[submodule "third_party/benchmark"]
+	path = third_party/benchmark
+	url = https://github.com/google/benchmark.git
+[submodule "third_party/pybind11"]
+	path = third_party/pybind11
+	url = https://github.com/pybind/pybind11.git

Acknowledgements.txt

@@ -0,0 +1,81 @@
+# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.5)
+
+project(DALI CXX)
+
+# Build options
+option(BUILD_TEST "Build googletest test suite" ON)
+option(BUILD_BENCHMARK "Build benchmark suite" ON)
+option(BUILD_NVTX "Build with NVTX profiling enabled" OFF)
+option(BUILD_PYTHON "Build python bindings" ON)
+option(BUILD_LMDB "Build LMDB readers" OFF)
+option(BUILD_TENSORFLOW "Build TensorFlow plugin" OFF)
+
+# Helper function to remove elements from a variable
+function (remove TARGET INPUT)
+  foreach(ITEM ${ARGN})
+    list(REMOVE_ITEM INPUT "${ITEM}")
+  endforeach()
+  set(${TARGET} ${INPUT} PARENT_SCOPE)
+endfunction(remove)
+
+# Default to release build
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING
+    "Build type from [Debug, Release]. For perf testing, build Release" FORCE)
+endif()
+
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+  message(STATUS "Building in DEBUG mode")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb -DDALI_DEBUG=1")
+  set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -g -G -DDALI_DEBUG=1")
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDALI_DEBUG=0")
+  set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -DDALI_DEBUG=0")
+endif()
+
+# Cmake path
+list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules)
+
+# Dependencies
+include(cmake/Dependencies.cmake)
+
+# CXX flags
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-unused-variable -Wno-unused-function -fno-strict-aliasing -O2 -fPIC")
+
+# Add ptx & bin flags for cuda
+set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} \
+  -gencode arch=compute_35,code=sm_35 \
+  -gencode arch=compute_50,code=sm_50 \
+  -gencode arch=compute_52,code=sm_52 \
+  -gencode arch=compute_60,code=sm_60 \
+  -gencode arch=compute_61,code=sm_61 \
+  -gencode arch=compute_70,code=sm_70 \
+  -gencode arch=compute_70,code=compute_70")
+
+# Project dir
+include_directories(BEFORE ${PROJECT_SOURCE_DIR})
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+cuda_include_directories(${PROJECT_SOURCE_DIR})
+
+# Project build
+add_subdirectory(dali)
+
+# HACK: Add __init__.pys as needed
+file(WRITE ${CMAKE_BINARY_DIR}/dali/__init__.py "")
+
+add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/lint.cmake)
+

CMakeLists.txt

@@ -0,0 +1,13 @@
+   Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

COPYRIGHT

@@ -0,0 +1,45 @@
+#########################################################################################
+##  Stage 2: build DALI wheels using manylinux1 (CentOS 5 derivative)
+#########################################################################################
+ARG DEPS_IMAGE_NAME
+FROM ${DEPS_IMAGE_NAME}
+
+ARG PYVER=2.7
+ARG PYV=27
+
+ENV PYVER=${PYVER} PYV=${PYV} PYTHONPATH=/opt/python/v
+
+ENV PYBIN=${PYTHONPATH}/bin \
+    PYLIB=${PYTHONPATH}/lib
+
+ENV PATH=${PYBIN}:${PATH} \
+    LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:/opt/dali/build:${PYLIB}:${LD_LIBRARY_PATH}
+
+RUN ln -s /opt/python/cp${PYV}* /opt/python/v
+
+RUN pip install future numpy setuptools wheel tensorflow-gpu && \
+    rm -rf /root/.cache/pip/
+
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
+    ldconfig
+
+WORKDIR /opt/dali
+
+COPY . .
+
+WORKDIR /opt/dali/build
+
+RUN LD_LIBRARY_PATH="${PWD}:${LD_LIBRARY_PATH}" && \
+    cmake ../ -DCMAKE_INSTALL_PREFIX=. \
+          -DBUILD_TEST=ON -DBUILD_BENCHMARK=ON -DBUILD_PYTHON=ON \
+          -DBUILD_LMDB=ON -DBUILD_TENSORFLOW=ON && \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install
+
+ARG NVIDIA_BUILD_ID
+ENV NVIDIA_BUILD_ID ${NVIDIA_BUILD_ID:-0}
+
+RUN pip wheel -v dali/python \
+        --build-option --python-tag=$(basename /opt/python/cp${PYV}-*) \
+        --build-option --plat-name=manylinux1_x86_64 \
+        --build-option --build-number=${NVIDIA_BUILD_ID} && \
+    ../dali/python/bundle-wheel.sh nvidia_dali-*.whl

Dockerfile

@@ -0,0 +1,86 @@
+#########################################################################################
+##  Stage 1: build DALI dependencies using manylinux1 (CentOS 5 derivative)
+#########################################################################################
+FROM quay.io/pypa/manylinux1_x86_64
+
+# Install yum Dependencies
+RUN yum install -y zip
+
+# Don't want the short-unicode version for Python 2.7
+RUN rm -f /opt/python/cp27-cp27m
+
+# Boost
+RUN BOOST_VERSION=1.66.0 && \
+    cd /usr/local && \
+    curl -L https://dl.bintray.com/boostorg/release/1.66.0/source/boost_${BOOST_VERSION//./_}.tar.gz | tar -xzf - && \
+    ln -s ../boost_${BOOST_VERSION//./_}/boost include/boost
+
+# CMake
+RUN CMAKE_VERSION=3.11 && \
+    CMAKE_BUILD=3.11.3 && \
+    curl -L https://cmake.org/files/v${CMAKE_VERSION}/cmake-${CMAKE_BUILD}.tar.gz | tar -xzf - && \
+    cd /cmake-${CMAKE_BUILD} && \
+    ./bootstrap --parallel=$(grep ^processor /proc/cpuinfo | wc -l) && \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
+    rm -rf /cmake-${CMAKE_BUILD}
+
+# protobuf v3.5.1
+RUN PROTOBUF_VERSION=3.5.1 && \
+    curl -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar -xzf - && \
+    cd /protobuf-${PROTOBUF_VERSION} && \
+    ./autogen.sh && \
+    ./configure --prefix=/usr/local 2>&1 > /dev/null && \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install 2>&1 > /dev/null && \
+    rm -rf /protobuf-${PROTOBUF_VERSION}
+
+# LMDB
+RUN LMDB_VERSION=0.9.22 && \
+    git clone -b LMDB_${LMDB_VERSION} --single-branch https://github.com/LMDB/lmdb && \
+    cd /lmdb/libraries/liblmdb && \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
+    rm -rf /lmdb
+
+# OpenCV
+RUN OPENCV_VERSION=3.1.0 && \
+    curl -L https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.tar.gz | tar -xzf - && \
+    cd /opencv-${OPENCV_VERSION} && \
+    cmake -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local \
+          -DWITH_CUDA=OFF -DWITH_1394=OFF -DWITH_IPP=OFF -DWITH_OPENCL=OFF -DWITH_GTK=OFF \
+          -DBUILD_DOCS=OFF -DBUILD_TESTS=OFF -DBUILD_PERF_TESTS=OFF \
+          -DBUILD_opencv_cudalegacy=OFF -DBUILD_opencv_stitching=OFF . && \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
+    rm -rf /opencv-${OPENCV_VERSION}
+
+# libjpeg-turbo
+#
+# Note: the preceding OpenCV installation intentionally does NOT use libjpeg-turbo.
+# DALI will directly call libjpeg-turbo first, and if it fails, DALI will fall back
+# to OpenCV, which in turn will call its bundled (built-from-source) libjpeg.
+# To be extra sure OpenCV doesn't pick up libjpeg-turbo (in which case we'd have no
+# fallback), libjpeg-turbo is built and installed _after_ OpenCV.
+#
+RUN JPEG_TURBO_VERSION=1.5.2 && \
+    curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${JPEG_TURBO_VERSION}.tar.gz | tar -xzf - && \
+    cd /libjpeg-turbo-${JPEG_TURBO_VERSION} && \
+    autoreconf -fiv && \
+    ./configure --enable-shared --prefix=/usr/local 2>&1 >/dev/null && \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install 2>&1 >/dev/null && \
+    rm -rf /libjpeg-turbo-${JPEG_TURBO_VERSION}
+
+# CUDA
+RUN CUDA_VERSION=9.0 && \
+    CUDA_BUILD=9.0.176_384.81 && \
+    curl -LO https://developer.nvidia.com/compute/cuda/${CUDA_VERSION}/Prod/local_installers/cuda_${CUDA_BUILD}_linux-run && \
+    chmod +x cuda_${CUDA_BUILD}_linux-run && \
+    ./cuda_${CUDA_BUILD}_linux-run --silent --no-opengl-libs --toolkit && \
+    rm -f cuda_${CUDA_BUILD}_linux-run
+
+# NVJPEG
+RUN NVJPEG_VERSION=9.0 && \
+    curl -L https://developer.download.nvidia.com/compute/redist/libnvjpeg/cuda-linux64-nvjpeg-${NVJPEG_VERSION}.tar.gz | tar -xzf - && \
+    cd /cuda-linux64-nvjpeg/ && \
+    mv lib64/libnvjpeg.so* /usr/local/lib/ && \
+    mv include/nvjpeg.h /usr/local/include/ && \
+    rm -rf /cuda-linux64-nvjpeg
+
+