Fix for non-spatial BN dimensions. Updated samples.

tjt7a · Jan 7, 2016 · de6ac08 · de6ac08
1 parent dda0524
commit de6ac08
Show file tree

Hide file tree

Showing 11 changed files with 53 additions and 70 deletions.
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/Macros.ndl b/Examples/Image/Miscellaneous/ImageNet/ResNet/Macros.ndl
@@ -6,8 +6,8 @@ ConvBNReLULayer(inp, outMap, inWCount, kW, kH, hStride, vStride, wScale, bValue,
     m = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c = Convolution(W, inp, kW, kH, outMap, hStride, vStride, zeroPadding = true)
-    bn = BatchNormalization(c, sc, b, m, isd, eval = false, spatial = true, expAvgFactor = 1.0)
+    c = Convolution(W, inp, kW, kH, outMap, hStride, vStride, zeroPadding = true, imageLayout = "cudnn")
+    bn = BatchNormalization(c, sc, b, m, isd, eval = false, spatial = true, expAvgFactor = 1.0, imageLayout = "cudnn")
     y = RectifiedLinear(bn);
 }
 
@@ -20,8 +20,8 @@ ResNetNode2(inp, outMap, inWCount, kW, kH, wScale, bValue, scValue)
     m1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c1 = Convolution(W1, inp, kW, kH, outMap, 1, 1, zeroPadding = true)
-    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0)
+    c1 = Convolution(W1, inp, kW, kH, outMap, 1, 1, zeroPadding = true, imageLayout = "cudnn")
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0, imageLayout = "cudnn")
     y1 = RectifiedLinear(bn1);
 
     W2 = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
@@ -30,8 +30,8 @@ ResNetNode2(inp, outMap, inWCount, kW, kH, wScale, bValue, scValue)
     m2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true)
-    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+    c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true, imageLayout = "cudnn")
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0, imageLayout = "cudnn")
     p = Plus(bn2, inp)
     y2 = RectifiedLinear(p);
 }
@@ -45,8 +45,8 @@ ResNetNode2Conv(inp, outMap, inWCount, wCount, kW, kH, wScale, bValue, scValue,
     m1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c1 = Convolution(W1, inp, kW, kH, outMap, 2, 2, zeroPadding = true)
-    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0)
+    c1 = Convolution(W1, inp, kW, kH, outMap, 2, 2, zeroPadding = true, imageLayout = "cudnn")
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0, imageLayout = "cudnn")
     y1 = RectifiedLinear(bn1);
 
     W2 = Parameter(outMap, wCount, init = Gaussian, initValueScale = wScale)
@@ -55,10 +55,10 @@ ResNetNode2Conv(inp, outMap, inWCount, wCount, kW, kH, wScale, bValue, scValue,
     m2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true)
-    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+    c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true, imageLayout = "cudnn")
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0, imageLayout = "cudnn")
 
-    cproj = Convolution(Wproj, inp, 1, 1, outMap, 2, 2, zeroPadding = false)
+    cproj = Convolution(Wproj, inp, 1, 1, outMap, 2, 2, zeroPadding = false, imageLayout = "cudnn")
     p = Plus(bn2, cproj)
     y2 = RectifiedLinear(p);
 }
@@ -73,8 +73,8 @@ ResNetNode3(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue)
     m1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c1 = Convolution(W1, inp, 1, 1, convMap, 1, 1, zeroPadding = false)
-    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true)
+    c1 = Convolution(W1, inp, 1, 1, convMap, 1, 1, zeroPadding = false, imageLayout = "cudnn")
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, imageLayout = "cudnn")
     y1 = RectifiedLinear(bn1);
 
     # 3x3 convolution.
@@ -84,8 +84,8 @@ ResNetNode3(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue)
     m2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c2 = Convolution(W2, y1, 3, 3, convMap, 1, 1, zeroPadding = true)
-    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+    c2 = Convolution(W2, y1, 3, 3, convMap, 1, 1, zeroPadding = true, imageLayout = "cudnn")
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0, imageLayout = "cudnn")
     y2 = RectifiedLinear(bn2);
 
     # 1x1 expanding convolution.
@@ -95,8 +95,8 @@ ResNetNode3(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue)
     m3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c3 = Convolution(W3, y2, 1, 1, outMap, 1, 1, zeroPadding = false)
-    bn3 = BatchNormalization(c3, sc3, b3, m3, isd3, eval = false, spatial = true)
+    c3 = Convolution(W3, y2, 1, 1, outMap, 1, 1, zeroPadding = false, imageLayout = "cudnn")
+    bn3 = BatchNormalization(c3, sc3, b3, m3, isd3, eval = false, spatial = true, imageLayout = "cudnn")
 
     p = Plus(bn3, inp)
     y3 = RectifiedLinear(p);
@@ -111,8 +111,8 @@ ResNetNode3Inc(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue,
     m1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c1 = Convolution(W1, inp, 1, 1, convMap, 1, 1, zeroPadding = false)
-    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true)
+    c1 = Convolution(W1, inp, 1, 1, convMap, 1, 1, zeroPadding = false, imageLayout = "cudnn")
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, imageLayout = "cudnn")
     y1 = RectifiedLinear(bn1);
 
     # 3x3 convolution.
@@ -122,8 +122,8 @@ ResNetNode3Inc(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue,
     m2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c2 = Convolution(W2, y1, 3, 3, convMap, 2, 2, zeroPadding = true)
-    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+    c2 = Convolution(W2, y1, 3, 3, convMap, 2, 2, zeroPadding = true, imageLayout = "cudnn")
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0, imageLayout = "cudnn")
     y2 = RectifiedLinear(bn2);
 
     # 1x1 expanding convolution.
@@ -133,11 +133,11 @@ ResNetNode3Inc(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue,
     m3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
     isd3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
-    c3 = Convolution(W3, y2, 1, 1, outMap, 1, 1, zeroPadding = false)
-    bn3 = BatchNormalization(c3, sc3, b3, m3, isd3, eval = false, spatial = true)
+    c3 = Convolution(W3, y2, 1, 1, outMap, 1, 1, zeroPadding = false, imageLayout = "cudnn")
+    bn3 = BatchNormalization(c3, sc3, b3, m3, isd3, eval = false, spatial = true, imageLayout = "cudnn")
 
     # Increasing input dimension convolution
-    cProj = Convolution(wProj, inp, 1, 1, outMap, 2, 2, zeroPadding = false)
+    cProj = Convolution(wProj, inp, 1, 1, outMap, 2, 2, zeroPadding = false, imageLayout = "cudnn")
 
     p = Plus(bn3, cProj)
     y3 = RectifiedLinear(p);

diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_152.config b/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_152.config
@@ -32,10 +32,10 @@ Train=[
 
     SGD=[
         epochSize=0
-        minibatchSize=2
-        learningRatesPerMB=0.1*20:0.03*10:0.01*30:0.003
+        minibatchSize=32
+        learningRatesPerMB=0.1*30:0.03*25:0.01*25:0.003*25:0.001
         momentumPerMB=0.9
-        maxEpochs=100
+        maxEpochs=120
         gradUpdateType=None
         L2RegWeight=0.0001
         dropoutRate=0
@@ -72,7 +72,7 @@ Train=[
             # Horizontal random flip, will be enabled by default if cropType=Random
             #hflip=0
             # Crop scale ratio. Examples: cropRatio=0.9, cropRatio=0.7:0.9. Default: 1.
-            cropRatio=0.875
+            cropRatio=0.46666:0.875
             # Crop scale ratio jitter type.
             # Possible values: None, UniRatio, UniLength, UniArea. Default: UniRatio
             jitterType=UniRatio
@@ -99,7 +99,7 @@ Test=[
     action=test
     modelPath=$ModelDir$/ResNet_152.Top5
     # Set minibatch size for testing.
-    minibatchSize=128
+    minibatchSize=32
 
      NDLNetworkBuilder=[
         networkDescription=$ConfigDir$/ResNet_152.ndl

diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_152.ndl b/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_152.ndl
@@ -7,7 +7,7 @@ ndlMacros = [
     ImageC = 3
     LabelDim = 1000
 
-    features = ImageInput(ImageW, ImageH, ImageC, tag = feature)
+    features = ImageInput(ImageW, ImageH, ImageC, tag = feature, imageLayout = "cudnn")
     featOffs = Const(0, rows = 150528)
     featScaled = Plus(features, featOffs)
     labels = Input(LabelDim, tag = label)
@@ -42,7 +42,7 @@ DNN=[
     cMap6 = 2048
 
     conv1 = ConvBNReLULayer(featScaled, cMap1, 147, 7, 7, 2, 2, convWScale, convBValue, scValue)
-    pool1 = MaxPooling(conv1, poolW, poolH, poolhs, poolvs)
+    pool1 = MaxPooling(conv1, poolW, poolH, poolhs, poolvs, imageLayout = "cudnn")
 
     rn1_1_Wproj = Parameter(cMap3, cMap1, init = fromFile, initFromFilePath = "$Proj64to256Filename$", needGradient = false)
     rn1_1 = ResNetNode3Inc(pool1, cMap1, cMap1, cMap3, 576, convWScale, convBValue, scValue, rn1_1_Wproj)
@@ -102,7 +102,7 @@ DNN=[
     rn4_2 = ResNetNode3(rn4_1, cMap6, cMap4, cMap6, 4608, convWScale, convBValue, scValue)
     rn4_3 = ResNetNode3(rn4_2, cMap6, cMap4, cMap6, 4608, convWScale, convBValue, scValue)
 
-    pool5 = AveragePooling(rn4_3, poolW, poolH, poolhs, poolvs)
+    pool5 = AveragePooling(rn4_3, poolW, poolH, poolhs, poolvs, imageLayout = "cudnn")
 
     ol = DnnLayer(8192, labelDim, pool5, fcWScale, fcBValue)
 

diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_34.config b/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_34.config
@@ -32,9 +32,9 @@ Train=[
     SGD=[
         epochSize=0
         minibatchSize=64
-        learningRatesPerMB=0.1*20:0.03*10:0.01*30:0.003
+        learningRatesPerMB=0.1*30:0.03*25:0.01*25:0.003*25:0.001
         momentumPerMB=0.9
-        maxEpochs=100
+        maxEpochs=120
         gradUpdateType=None
         L2RegWeight=0.0001
         dropoutRate=0
@@ -71,7 +71,7 @@ Train=[
             # Horizontal random flip, will be enabled by default if cropType=Random
             #hflip=0
             # Crop scale ratio. Examples: cropRatio=0.9, cropRatio=0.7:0.9. Default: 1.
-            cropRatio=0.875
+            cropRatio=0.46666:0.875
             # Crop scale ratio jitter type.
             # Possible values: None, UniRatio, UniLength, UniArea. Default: UniRatio
             jitterType=UniRatio
@@ -98,7 +98,7 @@ Test=[
     action=test
     modelPath=$ModelDir$/ResNet_34.Top5
     # Set minibatch size for testing.
-    minibatchSize=128
+    minibatchSize=64
 
      NDLNetworkBuilder=[
         networkDescription=$ConfigDir$/ResNet_34.ndl

diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_34.ndl b/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_34.ndl
@@ -7,9 +7,7 @@ ndlMacros = [
     ImageC = 3
     LabelDim = 1000
 
-    features = ImageInput(ImageW, ImageH, ImageC, tag = feature)
-    featOffs = Const(0, rows = 150528)
-    featScaled = Plus(features, featOffs)
+    features = ImageInput(ImageW, ImageH, ImageC, tag = feature, imageLayout = "cudnn")
     labels = Input(LabelDim, tag = label)
 
     # Kernels width and height.
@@ -35,8 +33,8 @@ ndlMacros = [
 
 DNN=[
     cMap1 = 64
-    conv1 = ConvBNReLULayer(featScaled, cMap1, 147, 7, 7, 2, 2, convWScale, convBValue, scValue)
-    pool1 = MaxPooling(conv1, poolW, poolH, poolhs, poolvs)
+    conv1 = ConvBNReLULayer(features, cMap1, 147, 7, 7, 2, 2, convWScale, convBValue, scValue)
+    pool1 = MaxPooling(conv1, poolW, poolH, poolhs, poolvs, imageLayout = "cudnn")
 
     rn1_1 = ResNetNode2(pool1, cMap1, 576, kW, kH, convWScale, convBValue, scValue)
     rn1_2 = ResNetNode2(rn1_1, cMap1, 576, kW, kH, convWScale, convBValue, scValue)
@@ -64,7 +62,7 @@ DNN=[
     rn4_2 = ResNetNode2(rn4_1, cMap4, 4608, kW, kH, convWScale, convBValue, scValue)
     rn4_3 = ResNetNode2(rn4_2, cMap4, 4608, kW, kH, convWScale, convBValue, scValue)
 
-    pool5 = AveragePooling(rn4_3, poolW, poolH, poolhs, poolvs)
+    pool5 = AveragePooling(rn4_3, poolW, poolH, poolhs, poolvs, imageLayout = "cudnn")
 
     ol = DnnLayer(4608, labelDim, pool5, fcWScale, fcBValue)
 

diff --git a/Examples/Image/Miscellaneous/ImageNet/VGG/Macros.ndl b/Examples/Image/Miscellaneous/ImageNet/VGG/Macros.ndl
@@ -12,14 +12,13 @@ DnnReLULayer(inDim, outDim, x, wScale, bValue)
 DnnBNReLULayer(inDim, outDim, x, wScale, bValue)
 {
     W = Parameter(outDim, inDim, init = Gaussian, initValueScale = wScale) 
-    b = Parameter(inDim, 1, init = fixedValue, value = bValue) 
-    sc = Parameter(inDim, 1, init = Gaussian, initValueScale = 0.01)
-    m = Parameter(inDim, 1, init = fixedValue, value = 0, needGradient = false)
-    isd = Parameter(inDim, 1, init = fixedValue, value = 0, needGradient = false)
-
-    bn = BatchNormalization(x, sc, b, m, isd, eval = false, spatial = false)
-    t = Times(W, bn)
-    y = RectifiedLinear(t)
+    b = Parameter(outDim, 1, init = fixedValue, value = bValue) 
+    sc = Parameter(outDim, 1, init = Gaussian, initValueScale = 0.01)
+    m = Parameter(outDim, 1, init = fixedValue, value = 0, needGradient = false)
+    isd = Parameter(outDim, 1, init = fixedValue, value = 0, needGradient = false)
+    t = Times(W, x)
+    bn = BatchNormalization(t, sc, b, m, isd, eval = false, spatial = false)
+    y = RectifiedLinear(bn)
 }
 
 # Fully-connected layer.

diff --git a/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_A.config b/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_A.config
@@ -85,7 +85,7 @@ Test=[
     action=test
     modelPath=$ModelDir$/VGG_A.Top5
     # Set minibatch size for testing.
-    minibatchSize=128
+    minibatchSize=32
 
      NDLNetworkBuilder=[
         networkDescription=$ConfigDir$/VGG_A.ndl

diff --git a/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_E.config b/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_E.config
@@ -94,7 +94,7 @@ Test=[
     action=test
     modelPath=$ModelDir$/VGG_E.Top5
     # Set minibatch size for testing.
-    minibatchSize=128
+    minibatchSize=16
 
      NDLNetworkBuilder=[
         networkDescription=$ConfigDir$/VGG_E.ndl

diff --git a/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_E_BN.config b/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_E_BN.config
@@ -94,7 +94,7 @@ Test=[
     action=test
     modelPath=$ModelDir$/VGG_E_BN.Top5
     # Set minibatch size for testing.
-    minibatchSize=128
+    minibatchSize=16
 
      NDLNetworkBuilder=[
         networkDescription=$ConfigDir$/VGG_E_BN.ndl

diff --git a/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_E_BN.ndl b/Examples/Image/Miscellaneous/ImageNet/VGG/VGG_E_BN.ndl
@@ -8,8 +8,6 @@ ndlMacros = [
     LabelDim = 1000
 
     features = ImageInput(ImageW, ImageH, ImageC, tag = feature, imageLayout = "cudnn")
-    featOffs = Const(0)
-    featScaled = Plus(features, featOffs)
     labels = Input(LabelDim, tag = label)
 
     # Kernels width and height.
@@ -39,7 +37,7 @@ ndlMacros = [
 
 DNN=[
     cMap1 = 64
-    conv1 = ConvBNReLULayer(featScaled, cMap1, 27, kW, kH, hs, vs, convWScale, convBValue, scValue)
+    conv1 = ConvBNReLULayer(features, cMap1, 27, kW, kH, hs, vs, convWScale, convBValue, scValue)
     conv2 = ConvBNReLULayer(conv1, cMap1, 576, kW, kH, hs, vs, convWScale, convBValue, scValue)
 
     pool1 = MaxPooling(conv2, poolW, poolH, poolhs, poolvs, imageLayout = "cudnn")

diff --git a/Source/ComputationNetworkLib/ConvolutionalNodes.h b/Source/ComputationNetworkLib/ConvolutionalNodes.h
@@ -751,23 +751,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
                 }
                 else
                 {
-                    assert(shape.GetRank() == 1);
-                    if (shape.GetRank() != 1)
-                        InvalidArgument("Batch normalization with spatial=false expects 1D tensor as an input.");
                     if (m_inT == nullptr)
-                        m_inT = m_factory->CreateTensor(shape[0], 1, 1, 1);
+                        m_inT = m_factory->CreateTensor(shape.GetNumElements(), 1, 1, 1);
                     if (m_scaleBiasT == nullptr)
-                        m_scaleBiasT = m_factory->CreateTensor(shape[0], 1, 1, 1);
+                        m_scaleBiasT = m_factory->CreateTensor(shape.GetNumElements(), 1, 1, 1);
                 }
-                //if (m_inT == nullptr)
-                //    m_inT = m_factory->CreateTensor(dims.m_width, dims.m_height, dims.m_numChannels, 1);
-                //if (m_scaleBiasT == nullptr)
-                //{
-                //    if (m_spatial)
-                //        m_scaleBiasT = m_factory->CreateTensor(1, 1, dims.m_numChannels, 1);
-                //    else
-                //        m_scaleBiasT = m_factory->CreateTensor(dims.m_width, dims.m_height, dims.m_numChannels, 1);
-                //}
             }
         }