Merge branch 'master' of https://github.com/Microsoft/CNTK

inginx · Jan 25, 2016 · e8bda68 · e8bda68
2 parents fa939cc + 76aef06
commit e8bda68
Show file tree

Hide file tree

Showing 237 changed files with 8,582 additions and 17,782 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "Source/1BitSGD"]
 	path = Source/1BitSGD
-	url = https://github.com/Microsoft/CNTK-1bit-SGD.git
+	url = https://git.codeplex.com/cntk1bitsgd
diff --git a/CNTK.sln b/CNTK.sln
@@ -33,7 +33,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tutorial", "Tutorial", "{98
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CNTK-TechReport", "CNTK-TechReport", "{EA67F51F-1FE8-462D-9F3E-01161685AD59}"
 	ProjectSection(SolutionItems) = preProject
-		Documentation\CNTK-TechReport\lyx\CNTKBook-20151201.pdf = Documentation\CNTK-TechReport\lyx\CNTKBook-20151201.pdf
+		Documentation\CNTK-TechReport\lyx\CNTKBook-20160121.pdf = Documentation\CNTK-TechReport\lyx\CNTKBook-20160121.pdf
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Documents", "Documents", "{DE1A06BA-EC5C-4E0D-BCA8-3EA555310C58}"
@@ -499,11 +499,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CIFAR-10", "CIFAR-10", "{77
 		Examples\Image\Miscellaneous\CIFAR-10\03_ResNet.config = Examples\Image\Miscellaneous\CIFAR-10\03_ResNet.config
 		Examples\Image\Miscellaneous\CIFAR-10\03_ResNet.mel = Examples\Image\Miscellaneous\CIFAR-10\03_ResNet.mel
 		Examples\Image\Miscellaneous\CIFAR-10\03_ResNet.ndl = Examples\Image\Miscellaneous\CIFAR-10\03_ResNet.ndl
+		Examples\Image\Miscellaneous\CIFAR-10\04_ResNet_56.config = Examples\Image\Miscellaneous\CIFAR-10\04_ResNet_56.config
+		Examples\Image\Miscellaneous\CIFAR-10\04_ResNet_56.ndl = Examples\Image\Miscellaneous\CIFAR-10\04_ResNet_56.ndl
 		Examples\Image\Miscellaneous\CIFAR-10\16to32.txt = Examples\Image\Miscellaneous\CIFAR-10\16to32.txt
 		Examples\Image\Miscellaneous\CIFAR-10\32to64.txt = Examples\Image\Miscellaneous\CIFAR-10\32to64.txt
 		Examples\Image\Miscellaneous\CIFAR-10\CIFAR_convert.py = Examples\Image\Miscellaneous\CIFAR-10\CIFAR_convert.py
-		Examples\Image\Miscellaneous\CIFAR-10\labelsmap.txt = Examples\Image\Miscellaneous\CIFAR-10\labelsmap.txt
+		Examples\Image\Miscellaneous\CIFAR-10\CifarConverter.py = Examples\Image\Miscellaneous\CIFAR-10\CifarConverter.py
 		Examples\Image\Miscellaneous\CIFAR-10\Macros.ndl = Examples\Image\Miscellaneous\CIFAR-10\Macros.ndl
+		Examples\Image\Miscellaneous\CIFAR-10\labelsmap.txt = Examples\Image\Miscellaneous\CIFAR-10\labelsmap.txt
 		Examples\Image\Miscellaneous\CIFAR-10\readme.txt = Examples\Image\Miscellaneous\CIFAR-10\readme.txt
 	EndProjectSection
 EndProject
@@ -676,13 +679,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{83BFF5BF
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ResNet", "ResNet", "{06D2C644-AE5F-4C30-A1F6-C78E2845AAB1}"
 	ProjectSection(SolutionItems) = preProject
-		Examples\Image\Miscellaneous\ImageNet\ResNet\add_top5_layer.mel = Examples\Image\Miscellaneous\ImageNet\ResNet\add_top5_layer.mel
 		Examples\Image\Miscellaneous\ImageNet\ResNet\Macros.ndl = Examples\Image\Miscellaneous\ImageNet\ResNet\Macros.ndl
 		Examples\Image\Miscellaneous\ImageNet\ResNet\ProjWeightsGen.py = Examples\Image\Miscellaneous\ImageNet\ResNet\ProjWeightsGen.py
 		Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_152.config = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_152.config
 		Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_152.ndl = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_152.ndl
 		Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_34.config = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_34.config
 		Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_34.ndl = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_34.ndl
+		Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.config = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.config
+		Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.ndl = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.ndl
+		Examples\Image\Miscellaneous\ImageNet\ResNet\create_eval_model.mel = Examples\Image\Miscellaneous\ImageNet\ResNet\create_eval_model.mel
 	EndProjectSection
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EvalWrapper", "Source\EvalWrapper\EvalWrapper.vcxproj", "{EF766CAE-9CB1-494C-9153-0030631A6340}"

diff --git a/CppCntk.vssettings b/CppCntk.vssettings
@@ -18,7 +18,7 @@
         <PropertyValue name="ShowVerticalScrollBar">true</PropertyValue>
         <PropertyValue name="InsertTabs">false</PropertyValue>
         <PropertyValue name="WordWrapGlyphs">true</PropertyValue>
-        <PropertyValue name="EnableLeftClickForURLs">true</PropertyValue>
+        <PropertyValue name="EnableLeftClickForURLs">false</PropertyValue>
         <PropertyValue name="ShowErrors">true</PropertyValue>
         <PropertyValue name="OverviewWidth">58</PropertyValue>
         <PropertyValue name="ShowLineNumbers">false</PropertyValue>
@@ -35,21 +35,21 @@
         <PropertyValue name="NewlineFunctionBrace">0</PropertyValue>
         <PropertyValue name="IndentBlockContents">true</PropertyValue>
         <PropertyValue name="DisableAggressiveMemberList">false</PropertyValue>
-        <PropertyValue name="MaximumCachedTranslationUnits">7</PropertyValue>
+        <PropertyValue name="MaximumCachedTranslationUnits">15</PropertyValue>
         <PropertyValue name="SpaceBeforeInitListBrace">false</PropertyValue>
         <PropertyValue name="SpaceWithinCastParentheses">false</PropertyValue>
         <PropertyValue name="NewlineInitListBrace">0</PropertyValue>
         <PropertyValue name="IndentCaseContents">true</PropertyValue>
         <PropertyValue name="RecreateDatabase">false</PropertyValue>
         <PropertyValue name="SpaceAroundConditionalOperator">2</PropertyValue>
-        <PropertyValue name="SpaceWithinInitListBraces">true</PropertyValue>
+        <PropertyValue name="SpaceWithinInitListBraces">false</PropertyValue>
         <PropertyValue name="SpaceBeforeLambdaParenthesis">false</PropertyValue>
-        <PropertyValue name="NewlineEmptyFunctionCloseBrace">true</PropertyValue>
+        <PropertyValue name="NewlineEmptyFunctionCloseBrace">false</PropertyValue>
         <PropertyValue name="DisablePreLoadNavigateToCache">false</PropertyValue>
         <PropertyValue name="EnterOutliningModeWhenFilesOpen">true</PropertyValue>
-        <PropertyValue name="PreserveBlock">1</PropertyValue>
+        <PropertyValue name="PreserveBlock">0</PropertyValue>
         <PropertyValue name="SpaceBeforeConstructorColon">true</PropertyValue>
-        <PropertyValue name="NewlineKeywordWhile">false</PropertyValue>
+        <PropertyValue name="NewlineKeywordWhile">true</PropertyValue>
         <PropertyValue name="NewlineLambdaBrace">0</PropertyValue>
         <PropertyValue name="DisableSharedIntelliSense">false</PropertyValue>
         <PropertyValue name="AddSemicolonForClassTypes">true</PropertyValue>
@@ -58,7 +58,7 @@
         <PropertyValue name="DoNotWarnIfFallbackLocationUsed">false</PropertyValue>
         <PropertyValue name="DisableDatabaseImplicitAutoCleanup">false</PropertyValue>
         <PropertyValue name="SpaceBeforeFunctionParenthesis">false</PropertyValue>
-        <PropertyValue name="NewlineNamespaceBrace">0</PropertyValue>
+        <PropertyValue name="NewlineNamespaceBrace">1</PropertyValue>
         <PropertyValue name="AutoFormatOnSemicolon">true</PropertyValue>
         <PropertyValue name="AutoTuneMaximumCachedTranslationUnits">true</PropertyValue>
         <PropertyValue name="UseForwardSlashForIncludeAutoComplete">false</PropertyValue>
@@ -67,7 +67,7 @@
         <PropertyValue name="DisableDatabaseUpdates">false</PropertyValue>
         <PropertyValue name="DisableIntelliSenseUpdating">false</PropertyValue>
         <PropertyValue name="NewlineControlBlockBrace">0</PropertyValue>
-        <PropertyValue name="IndentNamespaceContents">true</PropertyValue>
+        <PropertyValue name="IndentNamespaceContents">false</PropertyValue>
         <PropertyValue name="CompleteSlashStar">true</PropertyValue>
         <PropertyValue name="FallbackLocation"/>
         <PropertyValue name="DisableErrorReporting">false</PropertyValue>
@@ -106,7 +106,7 @@
         <PropertyValue name="DisableSquiggles">false</PropertyValue>
         <PropertyValue name="EnableQuickInfoToolTips">true</PropertyValue>
         <PropertyValue name="IndentCaseBraces">false</PropertyValue>
-        <PropertyValue name="AlignParameters">false</PropertyValue>
+        <PropertyValue name="AlignParameters">true</PropertyValue>
         <PropertyValue name="AutoFormatOnBraceCompletion">true</PropertyValue>
         <PropertyValue name="AutoFormatOnClosingBrace">true</PropertyValue>
         <PropertyValue name="CompleteParensInRawString">true</PropertyValue>
@@ -134,7 +134,7 @@
         <PropertyValue name="DisableDatabase">false</PropertyValue>
         <PropertyValue name="SpaceBeforeBracket">false</PropertyValue>
         <PropertyValue name="SpaceWithinFunctionParentheses">false</PropertyValue>
-        <PropertyValue name="NewlineEmptyTypeCloseBrace">true</PropertyValue>
+        <PropertyValue name="NewlineEmptyTypeCloseBrace">false</PropertyValue>
         <PropertyValue name="DisableIncludeAutoComplete">false</PropertyValue>
         <PropertyValue name="EnableLogging">false</PropertyValue>
       </ToolsOptionsSubCategory>

diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook-20151201.pdf b/Documentation/CNTK-TechReport/lyx/CNTKBook-20151201.pdf
diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook-20160121.pdf b/Documentation/CNTK-TechReport/lyx/CNTKBook-20160121.pdf
diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook-master.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook-master.lyx
@@ -139,7 +139,7 @@ Yu Zhang, Geoffrey Zweig
 \end_layout
 
 \begin_layout Date
-MSR-TR-2014-112 (DRAFT v0.9: Nov 30, 2015)
+MSR-TR-2014-112 (DRAFT v1.0: Jan 21, 2016)
 \end_layout
 
 \begin_layout Standard
@@ -202,13 +202,22 @@ filename "CNTKBook_CNTK_Adv_Chapter.lyx"
 \end_layout
 
 \begin_layout Standard
+\begin_inset Note Comment
+status open
+
+\begin_layout Plain Layout
 \begin_inset CommandInset include
 LatexCommand include
 filename "CNTKBook_CNTK_Programmer_Chapter.lyx"
 
 \end_inset
 
 
+\end_layout
+
+\end_inset
+
+
 \end_layout
 
 \begin_layout Standard

diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Adv_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Adv_Chapter.lyx
@@ -669,7 +669,7 @@ status open
 
 \begin_layout Plain Layout
 
-FeatureNodes=(features)
+FeatureNodes=(features1, features2)
 \end_layout
 
 \begin_layout Plain Layout
@@ -733,7 +733,7 @@ Comments
 
 \end_inset
 
-:
+
 \end_layout
 
 \begin_layout Standard
@@ -2569,6 +2569,10 @@ m1 and m2 must have same dimension in ElementTimes.
  vcol must have same number of rows in ColumnElementTimes.
 \end_layout
 
+\begin_layout Standard
+\begin_inset Note Comment
+status open
+
 \begin_layout Subsubsection
 KhatriRaoProduct
 \begin_inset Index idx
@@ -2593,15 +2597,15 @@ ColumnwiseCrossProduct
 
 \end_layout
 
-\begin_layout Standard
+\begin_layout Plain Layout
 Compute the cross product of each column of two input matrices.
  These two functions mean the same thing but ColumnwiseCrossProduct is easier
  to understand for most people.
  The resulting matrix is a (m1.rows times m2.rows) by m1.cols matrix.
  The syntax is
 \end_layout
 
-\begin_layout Standard
+\begin_layout Plain Layout
 \begin_inset listings
 inline false
 status open
@@ -2650,7 +2654,7 @@ GMMLL
 
 \end_layout
 
-\begin_layout Standard
+\begin_layout Plain Layout
 Compute the Gaussian mixture model log likelihood of feature values given
  the unnormalized Gaussian mixture weights (i.e., priors), Gaussian means
  and log standard deviation.
@@ -2660,7 +2664,7 @@ Compute the Gaussian mixture model log likelihood of feature values given
  The syntax is
 \end_layout
 
-\begin_layout Standard
+\begin_layout Plain Layout
 \begin_inset listings
 inline false
 status open
@@ -2706,11 +2710,16 @@ FeatureValues - the feature values on which to compute the log likelihood.
 
 \end_layout
 
-\begin_layout Standard
+\begin_layout Plain Layout
 Note, UnnormedPrior, Means and LogStddev should have the same number of
  columns which should be either 1 or the same as the columns in FeatureValues.
 \end_layout
 
+\end_inset
+
+
+\end_layout
+
 \begin_layout Subsubsection
 SquareError
 \begin_inset Index idx
@@ -3517,7 +3526,7 @@ Convolution(w, image, kernelWidth, kernelHeight,
 
 \begin_layout Plain Layout
 
-            maxTempMemSizeInSamples=0])
+            maxTempMemSizeInSamples=0, imageLayout="HWC"|cudnn"])
 \end_layout
 
 \begin_layout Plain Layout
@@ -3537,7 +3546,7 @@ Convolve(w, image, kernelWidth, kernelHeight,
 
 \begin_layout Plain Layout
 
-            maxTempMemSizeInSamples=0]) #deprecated
+            maxTempMemSizeInSamples=0, imageLayout="HWC"|cudnn"]) #deprecated
 \end_layout
 
 \end_inset
@@ -3592,6 +3601,22 @@ maxTempMemSizeInSamples - [named optional] maximum amount of memory (in
  Default is 0 which means the same as the input samples.
 \end_layout
 
+\begin_layout Itemize
+imageLayout - [named optional] the storage format of each image.
+ By default it's 
+\begin_inset Quotes eld
+\end_inset
+
+HWC
+\begin_inset Quotes erd
+\end_inset
+
+, which means each image is stored as [channel, width, height] in column
+ major.
+ If you use cuDNN to speed up training, you should set it to cudnn, which
+ means each image is stored as [width, height, channel].
+\end_layout
+
 \begin_layout Subsubsection
 MaxPooling
 \begin_inset Index idx
@@ -3631,12 +3656,13 @@ status open
 
 \begin_layout Plain Layout
 
-MaxPooling(m, windowWidth, windowHeight, stepW, stepH)
+MaxPooling(m, windowWidth, windowHeight, stepW, stepH, imageLayout="HWC"|cudnn")
 \end_layout
 
 \begin_layout Plain Layout
 
-AveragePooling(m, windowWidth, windowHeight, stepW, stepH)
+AveragePooling(m, windowWidth, windowHeight, stepW, stepH, imageLayout="HWC"|cud
+nn")
 \end_layout
 
 \end_inset
@@ -3664,6 +3690,22 @@ stepW - step (or stride) used in the width direction
 stepH - step (or stride) used in the height direction
 \end_layout
 
+\begin_layout Itemize
+imageLayout - [named optional] the storage format of each image.
+ By default it's 
+\begin_inset Quotes eld
+\end_inset
+
+HWC
+\begin_inset Quotes erd
+\end_inset
+
+, which means each image is stored as [channel, width, height] in column
+ major.
+ If you use cuDNN to speed up training, you should set it to cudnn, which
+ means each image is stored as [width, height, channel].
+\end_layout
+
 \begin_layout Subsubsection
 PastValue (or Delay
 \begin_inset Index idx
@@ -3806,6 +3848,15 @@ key "DNN-SWB-seide+2011,FeatEngInDNN-Seide+2011"
 
  on DNNs by building shallow networks first and then inserting new layers
  one on top of another.
+ To use MEL, you need to use the 
+\begin_inset Quotes eld
+\end_inset
+
+edit
+\begin_inset Quotes erd
+\end_inset
+
+ command in the config file.
 \end_layout
 
 \begin_layout Standard