Added ResNet ImageNet samples.

tjt7a · Dec 24, 2015 · 87096d4 · 87096d4
1 parent fd9e792
commit 87096d4
Show file tree

Hide file tree

Showing 14 changed files with 4,830 additions and 5 deletions.
diff --git a/Examples/Image/Miscellaneous/CIFAR-10/Macros.ndl b/Examples/Image/Miscellaneous/CIFAR-10/Macros.ndl
@@ -28,7 +28,7 @@ ConvBNReLULayer2(inp, outMap, inWCount, kW, kH, hStride, vStride, wScale, bValue
     isd = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
     c = Convolution(W, inp, kW, kH, outMap, hStride, vStride, zeroPadding = true)
-    bn = BatchNormalization(c, sc, b, m, isd, eval = false, spatial = true)
+    bn = BatchNormalization(c, sc, b, m, isd, eval = false, spatial = true, expAvgFactor = 1.0)
     y = RectifiedLinear(bn);
 }
 
@@ -41,7 +41,7 @@ ResNetNode2(inp, outMap, inWCount, kW, kH, wScale, bValue, scValue)
     isd1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
     c1 = Convolution(W1, inp, kW, kH, outMap, 1, 1, zeroPadding = true)
-    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 0.9)
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0)
     y1 = RectifiedLinear(bn1);
 
     W2 = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
@@ -51,7 +51,7 @@ ResNetNode2(inp, outMap, inWCount, kW, kH, wScale, bValue, scValue)
     isd2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
     c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true)
-    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 0.9)
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
     p = Plus(bn2, inp)
     y2 = RectifiedLinear(p);
 }
@@ -65,7 +65,7 @@ ResNetNode2Conv(inp, outMap, inWCount, wCount, kW, kH, wScale, bValue, scValue,
     isd1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
     c1 = Convolution(W1, inp, kW, kH, outMap, 2, 2, zeroPadding = true)
-    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 0.9)
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0)
     y1 = RectifiedLinear(bn1);
 
     W2 = Parameter(outMap, wCount, init = Gaussian, initValueScale = wScale)
@@ -75,7 +75,7 @@ ResNetNode2Conv(inp, outMap, inWCount, wCount, kW, kH, wScale, bValue, scValue,
     isd2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
 
     c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true)
-    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 0.9)
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
 
     cproj = Convolution(Wproj, inp, 1, 1, outMap, 2, 2, zeroPadding = false)
     p = Plus(bn2, cproj)

diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/1024to2048.txt b/Examples/Image/Miscellaneous/ImageNet/ResNet/1024to2048.txt
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/128to256.txt b/Examples/Image/Miscellaneous/ImageNet/ResNet/128to256.txt
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/256to512.txt b/Examples/Image/Miscellaneous/ImageNet/ResNet/256to512.txt
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/512to1024.txt b/Examples/Image/Miscellaneous/ImageNet/ResNet/512to1024.txt
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/64to128.txt b/Examples/Image/Miscellaneous/ImageNet/ResNet/64to128.txt
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/64to256.txt b/Examples/Image/Miscellaneous/ImageNet/ResNet/64to256.txt
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/Macros.ndl b/Examples/Image/Miscellaneous/ImageNet/ResNet/Macros.ndl
@@ -0,0 +1,152 @@
+ConvBNReLULayer(inp, outMap, inWCount, kW, kH, hStride, vStride, wScale, bValue, scValue)
+{
+    W = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
+    b = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    sc = Parameter(outMap, 1, init = Gaussian, initValueScale = scValue)
+    m = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c = Convolution(W, inp, kW, kH, outMap, hStride, vStride, zeroPadding = true)
+    bn = BatchNormalization(c, sc, b, m, isd, eval = false, spatial = true, expAvgFactor = 1.0)
+    y = RectifiedLinear(bn);
+}
+
+# Standard building block for ResNet.
+ResNetNode2(inp, outMap, inWCount, kW, kH, wScale, bValue, scValue)
+{
+    W1 = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
+    b1 = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    sc1 = Parameter(outMap, 1, init = Gaussian, initValueScale = scValue)
+    m1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c1 = Convolution(W1, inp, kW, kH, outMap, 1, 1, zeroPadding = true)
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0)
+    y1 = RectifiedLinear(bn1);
+
+    W2 = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
+    b2 = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    sc2 = Parameter(outMap, 1, init = Gaussian, initValueScale = scValue)
+    m2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true)
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+    p = Plus(bn2, inp)
+    y2 = RectifiedLinear(p);
+}
+
+# Standard building block for ResNet with padding.
+ResNetNode2Conv(inp, outMap, inWCount, wCount, kW, kH, wScale, bValue, scValue, Wproj)
+{
+    W1 = Parameter(outMap, inWCount, init = Gaussian, initValueScale = wScale)
+    b1 = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    sc1 = Parameter(outMap, 1, init = Gaussian, initValueScale = scValue)
+    m1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd1 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c1 = Convolution(W1, inp, kW, kH, outMap, 2, 2, zeroPadding = true)
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true, expAvgFactor = 1.0)
+    y1 = RectifiedLinear(bn1);
+
+    W2 = Parameter(outMap, wCount, init = Gaussian, initValueScale = wScale)
+    b2 = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    sc2 = Parameter(outMap, 1, init = Gaussian, initValueScale = scValue)
+    m2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd2 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c2 = Convolution(W2, y1, kW, kH, outMap, 1, 1, zeroPadding = true)
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+
+    cproj = Convolution(Wproj, inp, 1, 1, outMap, 2, 2, zeroPadding = false)
+    p = Plus(bn2, cproj)
+    y2 = RectifiedLinear(p);
+}
+
+# Bottleneck building block for ResNet.
+ResNetNode3(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue)
+{
+    # 1x1 reducing convolution.
+    W1 = Parameter(convMap, inMap, init = Gaussian, initValueScale = wScale)
+    b1 = Parameter(convMap, 1, init = fixedValue, value = bValue)
+    sc1 = Parameter(convMap, 1, init = Gaussian, initValueScale = scValue)
+    m1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c1 = Convolution(W1, inp, 1, 1, convMap, 1, 1, zeroPadding = false)
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true)
+    y1 = RectifiedLinear(bn1);
+
+    # 3x3 convolution.
+    W2 = Parameter(convMap, convWCount, init = Gaussian, initValueScale = wScale)
+    b2 = Parameter(convMap, 1, init = fixedValue, value = bValue)
+    sc2 = Parameter(convMap, 1, init = Gaussian, initValueScale = scValue)
+    m2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c2 = Convolution(W2, y1, 3, 3, convMap, 1, 1, zeroPadding = true)
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+    y2 = RectifiedLinear(bn2);
+
+    # 1x1 expanding convolution.
+    W3 = Parameter(outMap, convMap, init = Gaussian, initValueScale = wScale)
+    b3 = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    sc3 = Parameter(outMap, 1, init = Gaussian, initValueScale = scValue)
+    m3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c3 = Convolution(W3, y2, 1, 1, outMap, 1, 1, zeroPadding = false)
+    bn3 = BatchNormalization(c3, sc3, b3, m3, isd3, eval = false, spatial = true)
+
+    p = Plus(bn3, inp)
+    y3 = RectifiedLinear(p);
+}
+
+ResNetNode3Inc(inp, inMap, convMap, outMap, convWCount, wScale, bValue, scValue, wProj)
+{
+    # 1x1 reducing convolution.
+    W1 = Parameter(convMap, inMap, init = Gaussian, initValueScale = wScale)
+    b1 = Parameter(convMap, 1, init = fixedValue, value = bValue)
+    sc1 = Parameter(convMap, 1, init = Gaussian, initValueScale = scValue)
+    m1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd1 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c1 = Convolution(W1, inp, 1, 1, convMap, 1, 1, zeroPadding = false)
+    bn1 = BatchNormalization(c1, sc1, b1, m1, isd1, eval = false, spatial = true)
+    y1 = RectifiedLinear(bn1);
+
+    # 3x3 convolution.
+    W2 = Parameter(convMap, convWCount, init = Gaussian, initValueScale = wScale)
+    b2 = Parameter(convMap, 1, init = fixedValue, value = bValue)
+    sc2 = Parameter(convMap, 1, init = Gaussian, initValueScale = scValue)
+    m2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd2 = Parameter(convMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c2 = Convolution(W2, y1, 3, 3, convMap, 1, 1, zeroPadding = true)
+    bn2 = BatchNormalization(c2, sc2, b2, m2, isd2, eval = false, spatial = true, expAvgFactor = 1.0)
+    y2 = RectifiedLinear(bn2);
+
+    # 1x1 expanding convolution.
+    W3 = Parameter(outMap, convMap, init = Gaussian, initValueScale = wScale)
+    b3 = Parameter(outMap, 1, init = fixedValue, value = bValue)
+    sc3 = Parameter(outMap, 1, init = Gaussian, initValueScale = scValue)
+    m3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+    isd3 = Parameter(outMap, 1, init = fixedValue, value = 0, needGradient = false)
+
+    c3 = Convolution(W3, y2, 1, 1, outMap, 1, 1, zeroPadding = false)
+    bn3 = BatchNormalization(c3, sc3, b3, m3, isd3, eval = false, spatial = true)
+
+    # Increasing input dimension convolution
+    cProj = Convolution(wProj, inp, 1, 1, outMap, 1, 1, zeroPadding = false)
+
+    p = Plus(bn3, cProj)
+    y3 = RectifiedLinear(p);
+}
+
+DnnLayer(hiddenDim, labelDim, x, wScale, bValue)
+{
+    W = Parameter(labelDim, hiddenDim, init = Gaussian, initValueScale = wScale)
+    b = Parameter(labelDim, init = fixedValue, value = bValue)
+    t = Times(W, x)
+    z = Plus(t, b)
+}
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/ProjWeightsGen.py b/Examples/Image/Miscellaneous/ImageNet/ResNet/ProjWeightsGen.py
@@ -0,0 +1,12 @@
+import sys
+import numpy as np
+
+def writeConvWeights(fname, cmapIn, cmapOut):
+    w = np.eye(cmapOut, cmapIn)
+    np.savetxt(fname, w, fmt = '%d', delimiter = ' ')
+
+if __name__ == "__main__":
+    cmapIn = int(sys.argv[1])
+    cmapOut = int(sys.argv[2])
+    fname = sys.argv[3]
+    writeConvWeights(fname, cmapIn, cmapOut)
diff --git a/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_152.config b/Examples/Image/Miscellaneous/ImageNet/ResNet/ResNet_152.config
@@ -0,0 +1,123 @@
+RootDir = "."
+
+ConfigDir = "$RootDir$"
+DataDir = "$RootDir$"
+OutputDir = "$RootDir$/Output"
+ModelDir = "$OutputDir$/Models"
+
+ndlMacros=$ConfigDir$/Macros.ndl
+
+precision=float
+deviceId=Auto
+
+command=Train:AddTop5Eval:Test
+
+parallelTrain=false
+
+stderr=$OutputDir$/ResNet_152
+traceLevel=1
+
+Proj64to256Filename = $ConfigDir$/64to256.txt
+Proj256to512Filename = $ConfigDir$/256to512.txt
+Proj512to1024Filename = $ConfigDir$/512to1024.txt
+Proj1024to2048Filename = $ConfigDir$/1024to2048.txt
+
+Train=[
+    action=train
+    modelPath=$ModelDir$/ResNet_152
+
+     NDLNetworkBuilder=[
+        networkDescription=$ConfigDir$/ResNet_152.ndl
+    ]
+
+    SGD=[
+        epochSize=0
+        minibatchSize=1
+        learningRatesPerMB=0.1*20:0.03*10:0.01*30:0.003
+        momentumPerMB=0.9
+        maxEpochs=100
+        gradUpdateType=None
+        L2RegWeight=0.0001
+        dropoutRate=0
+
+        ParallelTrain=[
+            parallelizationMethod=DataParallelSGD
+            distributedMBReading=true
+            parallelizationStartEpoch=1
+            DataParallelSGD=[
+                gradientBits=1
+            ]
+        ]
+
+        numMBsToShowResult=100
+    ]
+
+    reader=[
+        readerType=ImageReader
+        # Map file which maps images to labels using the following format:
+        # <full path to image><tab><numerical label (0-based class id)>
+        # Example:
+        # C:\Data\ImageNet\2012\train\n01440764\n01440764_10026.JPEG<tab>0
+        file=$DataDir$/train_map.txt
+        # Randomize images before every epoch. Possible values: None, Auto. Default: Auto.
+        randomize=Auto
+        features=[
+            # Below are the required parameters.
+            width=224
+            height=224
+            channels=3
+            # Below are the optional parameters.
+            # Possible values: Center, Random. Default: Center
+            cropType=Random
+            # Horizontal random flip, will be enabled by default if cropType=Random
+            #hflip=0
+            # Crop scale ratio. Examples: cropRatio=0.9, cropRatio=0.7:0.9. Default: 1.
+            cropRatio=0.875
+            # Crop scale ratio jitter type.
+            # Possible values: None, UniRatio, UniLength, UniArea. Default: UniRatio
+            jitterType=UniRatio
+            # Interpolation to use when scaling image to width x height size.
+            # Possible values: nearest, linear, cubic, lanczos. Default: linear.
+            interpolations=Linear
+            # Stores mean values for each pixel in OpenCV matrix XML format.
+            meanFile=$ConfigDir$/ImageNet1K_mean.xml
+        ]
+        labels=[
+            labelDim=1000
+        ]
+    ]    
+]
+
+AddTop5Eval=[    
+    action=edit
+    CurModel=$ModelDir$/ResNet_152
+    NewModel=$ModelDir$/ResNet_152.Top5
+    editPath=$ConfigDir$/add_top5_layer.mel
+]
+
+Test=[
+    action=test
+    modelPath=$ModelDir$/ResNet_152.Top5
+    # Set minibatch size for testing.
+    minibatchSize=128
+
+     NDLNetworkBuilder=[
+        networkDescription=$ConfigDir$/ResNet_152.ndl
+    ]
+
+    reader=[
+        readerType=ImageReader
+        file=$DataDir$/val_map.txt
+        randomize=None
+        features=[
+            width=224
+            height=224
+            channels=3
+            cropType=Center
+            meanFile=$ConfigDir$/ImageNet1K_mean.xml
+        ]
+        labels=[
+            labelDim=1000
+        ]
+    ]    
+]