Enabled convo unit tests. Added batch normalization sample.

ExampleSetups/Image/CIFAR-10/02_BatchNormConv.config

@@ -0,0 +1,81 @@
+WorkDir=.
+ModelDir=$WorkDir$/_out/$ConfigName$
+stderr=$WorkDir$/_out/$ConfigName$
+
+ndlMacros=$WorkDir$/Macros.ndl
+
+precision=float
+deviceId=Auto
+prefetch=true
+parallelTrain=false
+
+command=Train:AddBNEval:Test
+
+Train=[
+    action=train
+    modelPath=$ModelDir$/02_BatchNormConv
+
+     NDLNetworkBuilder=[
+        networkDescription=$WorkDir$/02_BatchNormConv.ndl
+    ]
+
+    SGD=[
+        epochSize=49984
+        minibatchSize=64
+        learningRatesPerMB=0.03*7:0.01*8:0.003
+        #momentumPerMB=0.9*10:0.99
+        maxEpochs=10
+        #L2RegWeight=0.03
+        dropoutRate=0*1:0.5
+    ]
+
+    reader=[
+        readerType=UCIFastReader
+        file=$WorkDir$/Train.txt
+        randomize=None
+        features=[
+            dim=3072
+            start=1
+        ]
+        labels=[
+            dim=1
+            start=0
+            labelDim=10
+            labelMappingFile=$WorkDir$/labelsmap.txt
+        ]
+    ]    
+]
+
+AddBNEval=[    
+    action=edit
+    CurModel=$ModelDir$/02_BatchNormConv
+    NewModel=$ModelDir$/02_BatchNormConv.Eval
+    editPath=$WorkDir$/02_BatchNormConv.mel
+]
+
+Test=[
+    action=test
+    modelPath=$ModelDir$/02_BatchNormConv.Eval
+    # Set minibatch size for testing.
+    minibatchSize=128
+
+     NDLNetworkBuilder=[
+        networkDescription=$WorkDir$/02_BatchNormConv.ndl
+    ]
+
+    reader=[
+        readerType=UCIFastReader
+        file=$WorkDir$/Test.txt
+        randomize=None
+        features=[
+            dim=3072
+            start=1
+        ]
+        labels=[
+            dim=1
+            start=0
+            labelDim=10
+            labelMappingFile=$WorkDir$/labelsmap.txt
+        ]
+    ]    
+]

ExampleSetups/Image/CIFAR-10/02_BatchNormConv.mel

@@ -0,0 +1,16 @@
+m=LoadModel($CurModel$, format=cntk)
+SetDefaultModel(m)
+
+ibn_e = BatchNormalization(featScaled, isc, ib, im, iisd, eval = true, spatial = true)
+SetNodeInput(conv1.c, 1, ibn_e)
+
+conv2.bn_e = BatchNormalization(pool1, conv2.sc, conv2.b, conv2.m, conv2.isd, eval = true, spatial = true)
+SetNodeInput(conv2.c, 1, conv2.bn_e)
+
+conv3.bn_e = BatchNormalization(pool2, conv3.sc, conv3.b, conv3.m, conv3.isd, eval = true, spatial = true)
+SetNodeInput(conv3.c, 1, conv3.bn_e)
+
+h1.bn_e = BatchNormalization(pool3, h1.sc, h1.b, h1.m, h1.isd, eval = true, spatial = false)
+SetNodeInput(h1.t, 1, h1.bn_e)
+
+SaveModel(m, $NewModel$, format=cntk)

ExampleSetups/Image/CIFAR-10/02_BatchNormConv.ndl

@@ -0,0 +1,91 @@
+load=ndlMnistMacros
+run=DNN
+
+ndlMnistMacros = [
+    ImageW = 32
+    ImageH = 32
+    ImageC = 3
+    LabelDim = 10
+
+    features = ImageInput(ImageW, ImageH, ImageC, tag = feature)
+    featOffs = Const(128, rows = 3072)
+    featScaled = Minus(features, featOffs)
+    labels = Input(LabelDim, tag = label)
+
+    conv1WScale = 0.0043
+    conv1BValue = 0
+    conv2WScale = 1.414
+    conv2BValue = 0
+    conv3WScale = 1.414
+    conv3BValue = 0
+    fc1WScale = 12
+    fc1BValue = 0
+    fc2WScale = 1.5
+    fc2BValue = 0
+]
+
+DNN=[
+    ib = Parameter(ImageC, 1, init = Uniform, initValueScale = 100)
+    isc = Parameter(ImageC, 1, init = Uniform, initValueScale = 100)
+    im = Parameter(ImageC, 1, init = fixedValue, value = 0, needGradient = false)
+    iisd = Parameter(ImageC, 1, init = fixedValue, value = 0, needGradient = false)
+    ibn = BatchNormalization(featScaled, isc, ib, im, iisd, eval = false, spatial = true)
+
+    # conv1
+    kW1 = 5
+    kH1 = 5
+    cMap1 = 32
+    hStride1 = 1
+    vStride1 = 1
+    # weight[cMap1, kW1 * kH1 * ImageC]
+    conv1 = ConvReLULayer(ibn, cMap1, 75, kW1, kH1, hStride1, vStride1, conv1WScale, conv1BValue)
+
+    # pool1
+    pool1W = 3
+    pool1H = 3
+    pool1hStride = 2
+    pool1vStride = 2
+    pool1 = MaxPooling(conv1, pool1W, pool1H, pool1hStride, pool1vStride)
+
+    # conv2
+    kW2 = 5
+    kH2 = 5
+    cMap2 = 32
+    hStride2 = 1
+    vStride2 = 1
+    # weight[cMap2, kW2 * kH2 * cMap1]
+    conv2 = ConvBNReLULayer(pool1, cMap1, cMap2, 800, kW2, kH2, hStride2, vStride2, conv2WScale, conv2BValue)
+
+    # pool2
+    pool2W = 3
+    pool2H = 3
+    pool2hStride = 2
+    pool2vStride = 2
+    pool2 = MaxPooling(conv2, pool2W, pool2H, pool2hStride, pool2vStride)
+
+    # conv3
+    kW3 = 5
+    kH3 = 5
+    cMap3 = 64
+    hStride3 = 1
+    vStride3 = 1
+    # weight[cMap3, kW3 * kH3 * cMap2]
+    conv3 = ConvBNReLULayer(pool2, cMap2, cMap3, 800, kW3, kH3, hStride3, vStride3, conv3WScale, conv3BValue)
+
+    # pool3
+    pool3W = 3
+    pool3H = 3
+    pool3hStride = 2
+    pool3vStride = 2
+    pool3 = MaxPooling(conv3, pool3W, pool3H, pool3hStride, pool3vStride)
+
+    hiddenDim = 64
+    h1 = DnnBNReLULayer(576, hiddenDim, pool3, fc1WScale, fc1BValue)
+    h1_d = Dropout(h1)
+    ol = DNNLastLayer(hiddenDim, labelDim, h1_d, fc2WScale, fc2BValue)
+
+    CE = CrossEntropyWithSoftmax(labels, ol, tag = Criteria)
+    Err = ErrorPrediction(labels, ol, tag = Eval)
+    OutputNodes = ol
+]
+

ExampleSetups/Image/CIFAR-10/Macros.ndl

@@ -1,21 +1,45 @@
 ConvReLULayer(inp, outMap, inWCount, kW, kH, hStride, vStride, wScale, bValue)
 {
-    convW = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
-    conv = Convolution(convW, inp, kW, kH, outMap, hStride, vStride, zeroPadding = true)
-    convB = Parameter(outMap, 1, init = fixedValue, value = bValue)
-    convPlusB = Plus(conv, convB);
-    act = RectifiedLinear(convPlusB);
+    W = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
+    b = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    c = Convolution(W, inp, kW, kH, outMap, hStride, vStride, zeroPadding = true)
+    p = Plus(c, b);
+    y = RectifiedLinear(p);
 }
 
-DNNReLULayer(inDim, outDim, x, wScale, bValue)
+ConvBNReLULayer(inp, inMap, outMap, inWCount, kW, kH, hStride, vStride, wScale, bValue)
+{
+    W = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
+    b = Parameter(inMap, 1, init = Gaussian, initValueScale = 0.03)
+    sc = Parameter(inMap, 1, init = Gaussian, initValueScale = 0.03)
+    m = Parameter(inMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd = Parameter(inMap, 1, init = fixedValue, value = 0, needGradient = false)
+    bn = BatchNormalization(inp, sc, b, m, isd, eval = false, spatial = true)
+    c = Convolution(W, bn, kW, kH, outMap, hStride, vStride, zeroPadding = true)
+    y = RectifiedLinear(c);
+}
+
+DnnReLULayer(inDim, outDim, x, wScale, bValue)
 {
     W = Parameter(outDim, inDim, init = Gaussian, initValueScale = wScale) 
-    b = Parameter(outDim, init = fixedValue, value = bValue) 
+    b = Parameter(outDim, 1, init = fixedValue, value = bValue) 
     t = Times(W, x)
     z = Plus(t, b)
     y = RectifiedLinear(z)
 }
 
+DnnBNReLULayer(inDim, outDim, x, wScale, bValue)
+{
+    W = Parameter(outDim, inDim, init = Gaussian, initValueScale = wScale) 
+    b = Parameter(inDim, 1, init = fixedValue, value = bValue) 
+    sc = Parameter(inDim, 1, init = Gaussian, initValueScale = 0.01)
+    m = Parameter(inDim, 1, init = fixedValue, value = 0, needGradient = false)
+    isd = Parameter(inDim, 1, init = fixedValue, value = 0, needGradient = false)
+    bn = BatchNormalization(x, sc, b, m, isd, eval = false, spatial = false)
+    t = Times(W, bn)
+    y = RectifiedLinear(t)
+}
+
 DNNLastLayer(hiddenDim, labelDim, x, wScale, bValue)
 {
     W = Parameter(labelDim, hiddenDim, init = Gaussian, initValueScale = wScale)

MachineLearning/CNTKComputationNetworkLib/ConvolutionalNodes.h

@@ -109,7 +109,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             Matrix<ElemType> sliceOutputGrad = GradientFor(fr);
             Matrix<ElemType> sliceInput1Value = Input(1)->ValueFor(fr);
 
-            size_t batchSize = m_pMBLayout->GetNumParallelSequences();
+            size_t batchSize = sliceInput1Value.GetNumCols();
             m_inT->setN(batchSize);
             m_outT->setN(batchSize);
             assert(m_convEng != nullptr);
@@ -140,7 +140,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             Matrix<ElemType> sliceOutputValue = ValueFor(fr);
 
             // REVIEW alexeyk: setting batch size, can it be done elsewhere in a single place?
-            size_t batchSize = m_pMBLayout->GetNumParallelSequences();
+            size_t batchSize = sliceInput1Value.GetNumCols();
             m_inT->setN(batchSize);
             m_outT->setN(batchSize);
             assert(m_convEng != nullptr);
@@ -362,7 +362,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             Matrix<ElemType> sliceInput0Value = Input(0)->ValueFor(fr);
             Matrix<ElemType> sliceOutputValue = ValueFor(fr);
 
-            size_t batchSize = m_pMBLayout->GetNumParallelSequences();
+            size_t batchSize = sliceInput0Value.GetNumCols();
             m_inT->setN(batchSize);
             m_outT->setN(batchSize);
             assert(m_poolEng != nullptr);
@@ -375,7 +375,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             Matrix<ElemType> sliceInput0Value = Input(0)->ValueFor(fr);
             Matrix<ElemType> sliceOutputValue = ValueFor(fr);
 
-            size_t batchSize = m_pMBLayout->GetNumParallelSequences();
+            size_t batchSize = sliceInput0Value.GetNumCols();
             m_inT->setN(batchSize);
             m_outT->setN(batchSize);
             assert(m_poolEng != nullptr);
@@ -618,7 +618,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
                 const Matrix<ElemType>& scale = Input(1)->Value();
                 const Matrix<ElemType>& bias = Input(2)->Value();
 
-                size_t batchSize = m_pMBLayout->GetNumParallelSequences();
+                size_t batchSize = sliceInputValue.GetNumCols();
                 m_inT->setN(batchSize);
                 assert(m_convEng != nullptr);
 
@@ -661,7 +661,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
 
             Matrix<ElemType> sliceOutputValue = ValueFor(fr);
 
-            size_t batchSize = m_pMBLayout->GetNumParallelSequences();
+            size_t batchSize = sliceInputValue.GetNumCols();
             m_inT->setN(batchSize);
             assert(m_convEng != nullptr);
 #if NANCHECK

Math/Math/CNTKMathCUDA.vcxproj

@@ -215,6 +215,9 @@ if exist "$(CuDnnDll)" (xcopy /Y "$(CuDnnDll)" $(OutputPath))
     <Import Project="$(VCTargetsPath)\BuildCustomizations\CUDA 7.0.targets" />
   </ImportGroup>
   <Target Name="CheckDependencies">
-    <Error Condition="!Exists('$(CUB_PATH)')" Text="CNTK requires NVIDIA CUB library v1.4.1 to build. Please download the library from https://nvlabs.github.io/cub/ and set CUB_PATH environment variable to CUB root path (e.g. c:\src\cub-1.4.1)." />
+    <Error Condition="!Exists('$(CUB_PATH)')" 
+             Text="CNTK requires NVIDIA CUB library v1.4.1 to build. Please download the library from https://nvlabs.github.io/cub/ and set CUB_PATH environment variable to CUB root path (e.g. c:\src\cub-1.4.1)." />
+    <Warning Condition="!Exists('$(CUDNN_PATH)')"
+             Text="cuDNN library was not found. Please download the cuDNN v4 from https://developer.nvidia.com/cudnn, unzip it to a folder (e.g. C:\NVIDIA\cudnn-4.0) and set CUDNN_PATH to cuda directory inside that folder (e.g. run: setx CUDNN_PATH C:\NVIDIA\cudnn-4.0\cuda)."/>
   </Target>
 </Project>

Math/Math/ConvolutionEngine.cpp

@@ -410,12 +410,25 @@ namespace Microsoft { namespace MSR { namespace CNTK {
     };
 
     template<class ElemType>
-    std::unique_ptr<ConvolutionEngineFactory<ElemType>> ConvolutionEngineFactory<ElemType>::Create(DEVICEID_TYPE deviceId)
+    std::unique_ptr<ConvolutionEngineFactory<ElemType>> ConvolutionEngineFactory<ElemType>::Create(DEVICEID_TYPE deviceId, EngineType engType)
     {
-        // REVIEW alexeyk: make cuDNN default when running on GPU and compiled with cuDNN, add config parameter to enable runtime switch between implementations.
-        if (deviceId >= 0 && CuDnnConvolutionEngineFactory<ElemType>::IsSupported(deviceId))
-            return std::make_unique<CuDnnConvolutionEngineFactory<ElemType>>();
-        return std::make_unique<DefaultConvolutionEngineFactory<ElemType>>();
+        if (engType == EngineType::Auto)
+        {
+            // REVIEW alexeyk: make cuDNN default when running on GPU and compiled with cuDNN, add config parameter to enable runtime switch between implementations.
+            if (deviceId >= 0 && CuDnnConvolutionEngineFactory<ElemType>::IsSupported(deviceId))
+                return std::make_unique<CuDnnConvolutionEngineFactory<ElemType>>();
+            return std::make_unique<DefaultConvolutionEngineFactory<ElemType>>();
+        }
+        else if (engType == EngineType::CuDnn)
+        {
+            if (deviceId >= 0 && CuDnnConvolutionEngineFactory<ElemType>::IsSupported(deviceId))
+                return std::make_unique<CuDnnConvolutionEngineFactory<ElemType>>();
+            RuntimeError("cuDNN convolution engine is not supported, check the device id and whether the code was compiled with cuDNN.");
+        }
+        else if (engType == EngineType::Legacy)
+            return std::make_unique<DefaultConvolutionEngineFactory<ElemType>>();
+
+        RuntimeError("Not supported convolution engine type: %d.", engType);
     }
 
     template class ConvolutionEngineFactory<float>;

Math/Math/ConvolutionEngine.h

@@ -251,7 +251,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         virtual ConvEnginePtr CreateConvEngine(DEVICEID_TYPE deviceId, size_t maxTempMemSizeInSamples) = 0;
         virtual PoolEnginePtr CreatePoolEngine(DEVICEID_TYPE deviceId) = 0;
 
-        static std::unique_ptr<ConvolutionEngineFactory<ElemType>> Create(DEVICEID_TYPE deviceId);
+        enum class EngineType { Auto, CuDnn, Legacy };
+        static std::unique_ptr<ConvolutionEngineFactory<ElemType>> Create(DEVICEID_TYPE deviceId, EngineType engType = EngineType::Auto);
 
     public:
         ConvolutionEngineFactory(const ConvolutionEngineFactory&) = delete;