Merge branch 'master' of https://git01.codeplex.com/cntk into amitaga…

…/separate1bitDataParallelSGD

Examples/Image/MNIST/Config/01_OneHidden.config

@@ -19,6 +19,8 @@ ndlMacros = "$ConfigDir$/Macros.ndl"
 
 # comment the following line to write logs to the console 
 stderr = "$OutputDir$/01_OneHidden_out"
+traceLevel=1
+numMBsToShowResult=500
 
 #######################################
 #  TRAINING CONFIG                    #
@@ -63,6 +65,7 @@ train = [
 
 test = [
     action = "test"
+    minibatchSize = 16
 
     NDLNetworkBuilder=[
         networkDescription = "$ConfigDir$/01_OneHidden.ndl"

Examples/Image/MNIST/Config/02_Convolution.config

@@ -19,6 +19,10 @@ ndlMacros = "$ConfigDir$/Macros.ndl"
 
 # comment the following line to write logs to the console 
 stderr = "$OutputDir$/02_Convolution_out"
+traceLevel=1
+numMBsToShowResult=500
+
+prefetch=true
 
 #######################################
 #  TRAINING CONFIG                    #
@@ -63,6 +67,7 @@ train = [
 
 test = [
     action = test
+    minibatchSize = 16
 
     NDLNetworkBuilder = [
         networkDescription = "$ConfigDir$/02_Convolution.ndl"

Examples/Image/Miscellaneous/CIFAR-10/01_Conv.config

@@ -1,20 +1,28 @@
-WorkDir=.
-ModelDir=$WorkDir$/_out/$ConfigName$
-stderr=$WorkDir$/_out/$ConfigName$
+RootDir = "."
 
-ndlMacros=$WorkDir$/Macros.ndl
+ConfigDir = "$RootDir$"
+DataDir = "$RootDir$"
+OutputDir = "$RootDir$/Output"
+ModelDir = "$OutputDir$/Models"
+
+ndlMacros=$ConfigDir$/Macros.ndl
 
 precision=float
 deviceId=Auto
+prefetch=true
 
 command=Train:Test
 
+stderr=$OutputDir$/01_Conv
+traceLevel=1
+numMBsToShowResult=500
+
 Train=[
     action=train
     modelPath=$ModelDir$/01_Convolution
 
      NDLNetworkBuilder=[
-        networkDescription=$WorkDir$/01_Convolution.ndl
+        networkDescription=$ConfigDir$/01_Convolution.ndl
     ]
 
     SGD=[
@@ -29,7 +37,7 @@ Train=[
 
     reader=[
         readerType=UCIFastReader
-        file=$WorkDir$/Train.txt
+        file=$DataDir$/Train.txt
         randomize=None
         features=[
             dim=3072
@@ -39,7 +47,7 @@ Train=[
             dim=1
             start=0
             labelDim=10
-            labelMappingFile=$WorkDir$/labelsmap.txt
+            labelMappingFile=$DataDir$/labelsmap.txt
         ]
     ]    
 ]
@@ -48,15 +56,15 @@ Test=[
     action=test
     modelPath=$ModelDir$/01_Convolution
     # Set minibatch size for testing.
-    minibatchSize=128
+    minibatchSize=16
 
      NDLNetworkBuilder=[
-        networkDescription=$WorkDir$/01_Convolution.ndl
+        networkDescription=$ConfigDir$/01_Convolution.ndl
     ]
 
     reader=[
         readerType=UCIFastReader
-        file=$WorkDir$/Test.txt
+        file=$DataDir$/Test.txt
         randomize=None
         features=[
             dim=3072
@@ -66,7 +74,7 @@ Test=[
             dim=1
             start=0
             labelDim=10
-            labelMappingFile=$WorkDir$/labelsmap.txt
+            labelMappingFile=$DataDir$/labelsmap.txt
         ]
     ]    
 ]

Examples/Image/Miscellaneous/CIFAR-10/01_Convolution.ndl

@@ -7,8 +7,8 @@ ndlMnistMacros = [
     ImageC = 3
     LabelDim = 10
 
-    features = ImageInput(ImageW, ImageH, ImageC, tag = feature)
-    featOffs = Const(128, rows = 3072)
+    features = ImageInput(ImageW, ImageH, ImageC, tag = feature, imageLayout = "cudnn")
+    featOffs = Const(128)
     featScaled = Minus(features, featOffs)
     labels = Input(LabelDim, tag = label)
 
@@ -39,7 +39,7 @@ DNN=[
     pool1H = 3
     pool1hStride = 2
     pool1vStride = 2
-    pool1 = MaxPooling(conv1_act, pool1W, pool1H, pool1hStride, pool1vStride)
+    pool1 = MaxPooling(conv1_act, pool1W, pool1H, pool1hStride, pool1vStride, imageLayout = "cudnn")
 
     # conv2
     kW2 = 5
@@ -55,7 +55,7 @@ DNN=[
     pool2H = 3
     pool2hStride = 2
     pool2vStride = 2
-    pool2 = MaxPooling(conv2_act, pool2W, pool2H, pool2hStride, pool2vStride)
+    pool2 = MaxPooling(conv2_act, pool2W, pool2H, pool2hStride, pool2vStride, imageLayout = "cudnn")
 
     # conv3
     kW3 = 5
@@ -71,7 +71,7 @@ DNN=[
     pool3H = 3
     pool3hStride = 2
     pool3vStride = 2
-    pool3 = MaxPooling(conv3_act, pool3W, pool3H, pool3hStride, pool3vStride)
+    pool3 = MaxPooling(conv3_act, pool3W, pool3H, pool3hStride, pool3vStride, imageLayout = "cudnn")
 
     hiddenDim = 64
     h1 = DNNReLULayer(576, hiddenDim, pool3, fc1WScale, fc1BValue)

Examples/Image/Miscellaneous/CIFAR-10/02_BatchNormConv.config

@@ -1,37 +1,43 @@
-WorkDir=.
-ModelDir=$WorkDir$/_out/$ConfigName$
-stderr=$WorkDir$/_out/$ConfigName$
+RootDir = "."
 
-ndlMacros=$WorkDir$/Macros.ndl
+ConfigDir = "$RootDir$"
+DataDir = "$RootDir$"
+OutputDir = "$RootDir$/Output"
+ModelDir = "$OutputDir$/Models"
+
+ndlMacros=$ConfigDir$/Macros.ndl
 
 precision=float
 deviceId=Auto
 prefetch=true
-parallelTrain=false
 
 command=Train:AddBNEval:Test
 
+stderr=$OutputDir$/02_BatchNormConv
+traceLevel=1
+numMBsToShowResult=500
+
 Train=[
     action=train
     modelPath=$ModelDir$/02_BatchNormConv
 
      NDLNetworkBuilder=[
-        networkDescription=$WorkDir$/02_BatchNormConv.ndl
+        networkDescription=$ConfigDir$/02_BatchNormConv.ndl
     ]
 
     SGD=[
         epochSize=49984
         minibatchSize=64
-        learningRatesPerMB=0.03*7:0.01*8:0.003
-        #momentumPerMB=0.9*10:0.99
+        learningRatesPerMB=0.03*7:0.01
+        momentumPerMB=0
         maxEpochs=10
-        #L2RegWeight=0.03
-        dropoutRate=0*1:0.5
+        L2RegWeight=0
+        dropoutRate=0
     ]
 
     reader=[
         readerType=UCIFastReader
-        file=$WorkDir$/Train.txt
+        file=$DataDir$/Train.txt
         randomize=None
         features=[
             dim=3072
@@ -41,7 +47,7 @@ Train=[
             dim=1
             start=0
             labelDim=10
-            labelMappingFile=$WorkDir$/labelsmap.txt
+            labelMappingFile=$DataDir$/labelsmap.txt
         ]
     ]    
 ]
@@ -50,22 +56,22 @@ AddBNEval=[
     action=edit
     CurModel=$ModelDir$/02_BatchNormConv
     NewModel=$ModelDir$/02_BatchNormConv.Eval
-    editPath=$WorkDir$/02_BatchNormConv.mel
+    editPath=$ConfigDir$/02_BatchNormConv.mel
 ]
 
 Test=[
     action=test
     modelPath=$ModelDir$/02_BatchNormConv.Eval
     # Set minibatch size for testing.
-    minibatchSize=128
+    minibatchSize=16
 
      NDLNetworkBuilder=[
-        networkDescription=$WorkDir$/02_BatchNormConv.ndl
+        networkDescription=$ConfigDir$/02_BatchNormConv.ndl
     ]
 
     reader=[
         readerType=UCIFastReader
-        file=$WorkDir$/Test.txt
+        file=$DataDir$/Test.txt
         randomize=None
         features=[
             dim=3072
@@ -75,7 +81,7 @@ Test=[
             dim=1
             start=0
             labelDim=10
-            labelMappingFile=$WorkDir$/labelsmap.txt
+            labelMappingFile=$DataDir$/labelsmap.txt
         ]
     ]    
 ]

Examples/Image/Miscellaneous/CIFAR-10/02_BatchNormConv.mel

@@ -1,16 +1,6 @@
 m=LoadModel($CurModel$, format=cntk)
 SetDefaultModel(m)
 
-ibn_e = BatchNormalization(featScaled, isc, ib, im, iisd, eval = true, spatial = true)
-SetNodeInput(conv1.c, 1, ibn_e)
-
-conv2.bn_e = BatchNormalization(pool1, conv2.sc, conv2.b, conv2.m, conv2.isd, eval = true, spatial = true)
-SetNodeInput(conv2.c, 1, conv2.bn_e)
-
-conv3.bn_e = BatchNormalization(pool2, conv3.sc, conv3.b, conv3.m, conv3.isd, eval = true, spatial = true)
-SetNodeInput(conv3.c, 1, conv3.bn_e)
-
-h1.bn_e = BatchNormalization(pool3, h1.sc, h1.b, h1.m, h1.isd, eval = true, spatial = false)
-SetNodeInput(h1.t, 1, h1.bn_e)
+SetPropertyForSubTree(CE, batchNormEvalMode, true)
 
 SaveModel(m, $NewModel$, format=cntk)

Examples/Image/Miscellaneous/CIFAR-10/02_BatchNormConv.ndl

@@ -7,8 +7,8 @@ ndlMnistMacros = [
     ImageC = 3
     LabelDim = 10
 
-    features = ImageInput(ImageW, ImageH, ImageC, tag = feature)
-    featOffs = Const(128, rows = 3072)
+    features = ImageInput(ImageW, ImageH, ImageC, tag = feature, imageLayout = "cudnn")
+    featOffs = Const(128)
     featScaled = Minus(features, featOffs)
     labels = Input(LabelDim, tag = label)
 
@@ -18,34 +18,33 @@ ndlMnistMacros = [
     conv2BValue = 0
     conv3WScale = 1.414
     conv3BValue = 0
+
+    scValue = 1
+
+    expAvg = 1
+
     fc1WScale = 12
     fc1BValue = 0
     fc2WScale = 1.5
     fc2BValue = 0
 ]
 
 DNN=[
-    ib = Parameter(ImageC, 1, init = Uniform, initValueScale = 100)
-    isc = Parameter(ImageC, 1, init = Uniform, initValueScale = 100)
-    im = Parameter(ImageC, 1, init = fixedValue, value = 0, needGradient = false)
-    iisd = Parameter(ImageC, 1, init = fixedValue, value = 0, needGradient = false)
-    ibn = BatchNormalization(featScaled, isc, ib, im, iisd, eval = false, spatial = true)
-
     # conv1
     kW1 = 5
     kH1 = 5
     cMap1 = 32
     hStride1 = 1
     vStride1 = 1
     # weight[cMap1, kW1 * kH1 * ImageC]
-    conv1 = ConvReLULayer(ibn, cMap1, 75, kW1, kH1, hStride1, vStride1, conv1WScale, conv1BValue)
+    conv1 = ConvBNReLULayer(featScaled, cMap1, 75, kW1, kH1, hStride1, vStride1, conv1WScale, conv1BValue, scValue, expAvg)
 
     # pool1
     pool1W = 3
     pool1H = 3
     pool1hStride = 2
     pool1vStride = 2
-    pool1 = MaxPooling(conv1, pool1W, pool1H, pool1hStride, pool1vStride)
+    pool1 = MaxPooling(conv1, pool1W, pool1H, pool1hStride, pool1vStride, imageLayout = "cudnn")
 
     # conv2
     kW2 = 5
@@ -54,14 +53,14 @@ DNN=[
     hStride2 = 1
     vStride2 = 1
     # weight[cMap2, kW2 * kH2 * cMap1]
-    conv2 = ConvBNReLULayer(pool1, cMap1, cMap2, 800, kW2, kH2, hStride2, vStride2, conv2WScale, conv2BValue)
+    conv2 = ConvBNReLULayer(pool1, cMap2, 800, kW2, kH2, hStride2, vStride2, conv2WScale, conv2BValue, scValue, expAvg)
 
     # pool2
     pool2W = 3
     pool2H = 3
     pool2hStride = 2
     pool2vStride = 2
-    pool2 = MaxPooling(conv2, pool2W, pool2H, pool2hStride, pool2vStride)
+    pool2 = MaxPooling(conv2, pool2W, pool2H, pool2hStride, pool2vStride, imageLayout = "cudnn")
 
     # conv3
     kW3 = 5
@@ -70,19 +69,18 @@ DNN=[
     hStride3 = 1
     vStride3 = 1
     # weight[cMap3, kW3 * kH3 * cMap2]
-    conv3 = ConvBNReLULayer(pool2, cMap2, cMap3, 800, kW3, kH3, hStride3, vStride3, conv3WScale, conv3BValue)
+    conv3 = ConvBNReLULayer(pool2, cMap3, 800, kW3, kH3, hStride3, vStride3, conv3WScale, conv3BValue, scValue, expAvg)
 
     # pool3
     pool3W = 3
     pool3H = 3
     pool3hStride = 2
     pool3vStride = 2
-    pool3 = MaxPooling(conv3, pool3W, pool3H, pool3hStride, pool3vStride)
+    pool3 = MaxPooling(conv3, pool3W, pool3H, pool3hStride, pool3vStride, imageLayout = "cudnn")
 
     hiddenDim = 64
-    h1 = DnnBNReLULayer(576, hiddenDim, pool3, fc1WScale, fc1BValue)
-    h1_d = Dropout(h1)
-    ol = DNNLastLayer(hiddenDim, labelDim, h1_d, fc2WScale, fc2BValue)
+    h1 = DnnBNReLULayer(576, hiddenDim, pool3, fc1WScale, fc1BValue, scValue, expAvg)
+    ol = DNNLastLayer(hiddenDim, labelDim, h1, fc2WScale, fc2BValue)
 
     CE = CrossEntropyWithSoftmax(labels, ol, tag = Criteria)
     Err = ErrorPrediction(labels, ol, tag = Eval)