Merge remote-tracking branch 'origin/master' into linux-gcc

Conflicts:
	Common/Include/DataReader.h
	Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Chapter.lyx
	MachineLearning/CNTK/TrainingCriterionNodes.h
	Math/Math/Matrix.cpp

Common/Include/DataReader.h

@@ -29,9 +29,16 @@
 
 namespace Microsoft { namespace MSR { namespace CNTK {
 
-const size_t randomizeAuto = ((size_t)-1)>>2; // randomize range set automatically, parameter value for Init()
-const size_t randomizeNone = 0;  // don't randomize, parameter value for Init()
-const size_t requestDataSize = randomizeAuto;   // StartMinibatchLoop default parameter, sets number of requested frames equal to the number of frames in the dataset
+// randomize range set automatically, parameter value for Init()
+const size_t randomizeAuto = ((size_t) -1) >> 2;
+
+// don't randomize, parameter value for Init()
+const size_t randomizeNone = 0;
+
+// StartMinibatchLoop default parameter, sets number of requested
+// frames equal to the constant 3fffffffffffffff computed by ((size_t) -1) >> 2 above.
+// We use this constant as a stand in for the total number of frames in the dataset.
+const size_t requestDataSize = randomizeAuto;
 
 enum EndDataType
 {
@@ -48,7 +55,7 @@ class DATAREADER_API IDataReader
 {
 public:
     typedef std::string LabelType;
-    typedef unsigned LabelIdType;
+    typedef unsigned int LabelIdType;
     unsigned m_seed;
     size_t   mBlgSize;  /// number of utterances per minibatch
     bool     mDoRandomize; 

Common/Include/commandArgUtil.h

@@ -612,6 +612,7 @@ class ConfigParser
                 // pop out of content level
                 contentLevel = false;
             }
+
             if (quoteFound)
             {
                 // skip the closing quote
@@ -660,7 +661,7 @@ class ConfigParser
     std::string ReadConfigFiles(const std::string& filePaths);
     std::string ReadConfigFiles(const std::wstring& filePaths);
     std::string ResolveIncludeStatements(const std::string& configString, std::vector<std::string>& resolvedConfigFiles);
-    void LoadConfigFile(const std::wstring & filePath);
+    void LoadConfigFile(const std::wstring& filePath);
     void LoadConfigFileAndResolveVariables(const std::wstring& filePath, const ConfigParameters& config);
     void LoadConfigFiles(const std::wstring& filePaths, const std::string* configStringToAppend = nullptr);
 
@@ -873,17 +874,17 @@ class ConfigParameters: public ConfigParser, public ConfigDictionary
     }
 
     // Insert - insert an 'name=value' string into the dictionary
-    void Insert(const std::string &str)
+    void Insert(const std::string& str)
     {
         ParseValue(str, 0, str.length());
     }
 
-    bool Exists(const std::wstring & name) const
+    bool Exists(const std::wstring& name) const
     {
         return Exists(msra::strfun::utf8(name));
     }
 
-    bool Exists(const std::string & name) const
+    bool Exists(const std::string& name) const
     {
         if (find(name) != end())
         {
@@ -899,42 +900,42 @@ class ConfigParameters: public ConfigParser, public ConfigDictionary
     }
 
     // ExistsCurrent - check to see if a key exists in THIS config, don't check parent
-    bool ExistsCurrent(const std::string & name) const
+    bool ExistsCurrent(const std::string& name) const
     {
         return (find(name) != end());
     }
 
     // dict(name, default) for strings
-    ConfigValue operator()(const std::wstring & name,
-                           const wchar_t *defaultvalue) const
+    ConfigValue operator()(const std::wstring& name,
+                           const wchar_t* defaultvalue) const
     {
         return operator()(msra::strfun::utf8(name), defaultvalue);
     }
 
     // dict(name, default) for strings
-    ConfigValue operator()(const std::string & name,
-                           const wchar_t *defaultvalue) const
+    ConfigValue operator()(const std::string& name,
+                           const wchar_t* defaultvalue) const
     {
         return operator()(name, msra::strfun::utf8(defaultvalue).c_str());
     }
 
     // dict(name, default) for strings
-    ConfigValue operator()(const std::wstring & name,
-                           const char *defaultvalue) const
+    ConfigValue operator()(const std::wstring& name,
+                           const char* defaultvalue) const
     {
         return operator()(msra::strfun::utf8(name), defaultvalue);
     }
 
     // dict(name, default) for strings
-    ConfigValue operator()(const std::string & name,
-                           const char *defaultvalue) const
+    ConfigValue operator()(const std::string& name,
+                           const char* defaultvalue) const
     {
         ConfigValue value = Find(name, defaultvalue);
         return value;
     }
 
-    ConfigValue Find(const std::string & name,
-                     const char *defaultvalue = NULL) const
+    ConfigValue Find(const std::string& name,
+                     const char* defaultvalue = NULL) const
     {
         auto iter = find(name);
         ConfigValue result;
@@ -975,10 +976,11 @@ class ConfigParameters: public ConfigParser, public ConfigDictionary
     //     any whitespace characters.  If an opening "$" is found without a closing "$", an exception is thrown.
     // configString - the string that you would like to resolve variables in.
     // returns: A copy of 'configString' with all the variables resolved.
-    std::string ResolveVariablesInSingleLine(const std::string &configLine) const
+    std::string ResolveVariablesInSingleLine(const std::string& configLine) const
     {
         // ensure that this method was called on a single line (eg, no newline characters exist in 'configLine').
-        if (configLine.find_first_of("\n") != std::string::npos) {
+        if (configLine.find_first_of("\n") != std::string::npos)
+        {
             throw std::logic_error(
                 "\"ResolveVariablesInSingleLine\" shouldn't be called with a string containing a newline character");
         }
@@ -1053,7 +1055,7 @@ class ConfigParameters: public ConfigParser, public ConfigDictionary
     //     we shouldn't insert newlines where they didn't already exist.
     // configString - the string that you would like to resolve variables in.
     // returns: A copy of 'configString' with all the variables resolved.
-    std::string ResolveVariables(const std::string &configString) const
+    std::string ResolveVariables(const std::string& configString) const
     {
         std::string newConfigString;
         if (configString.find_first_of("\n") != std::string::npos)
@@ -1347,14 +1349,14 @@ class argvector: public std::vector<T>
             RuntimeError("argvector: invalid arg value");
         }
     }
-    static void parse(const std::wstring & in, std::wstring & val)
+    static void parse(const std::wstring& in, std::wstring& val)
     {
         val = in;
     }
 
 public:
     // constructor --construct empty, then assign a wstring from command-line argument
-    void operator=(const std::wstring & arg)
+    void operator=(const std::wstring& arg)
     {
         clear();
         // separate the arguments
@@ -1387,7 +1389,7 @@ class argvector: public std::vector<T>
     }
 
     // constructor --use this for setting default values
-    argvector(const std::wstring & arg)
+    argvector(const std::wstring& arg)
     {
         *this = arg;
     }
@@ -1438,7 +1440,7 @@ class argvector: public std::vector<T>
     }
 
     // we give full read access to the vector, so we can use it bounded as well
-    const std::vector<T> & tovector() const
+    const std::vector<T>& tovector() const
     {
         return *this;
     }

DataReader/LMSequenceReader/SequenceReader.cpp

@@ -2049,7 +2049,7 @@ void BatchSequenceReader<ElemType>::GetLabelOutput(std::map<std::wstring,
     }
     if (curDevId != CPUDEVICE)
     {
-        labels->TransferFromDeviceToDevice(CPUDEVICE, curDevId, true, false, false);
+        labels->TransferFromDeviceToDevice(CPUDEVICE, curDevId, false, false, false);
     }
 }
 

Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Chapter.lyx

@@ -1725,6 +1725,102 @@ numBestSearchEpoch
 
 \end_layout
 
+\begin_layout Standard
+Used in the Adaptive Minibatch Sizing mode.
+\end_layout
+
+\begin_layout Itemize
+
+\emph on
+numMiniBatch4LRSearch
+\emph default
+
+\begin_inset Index idx
+status open
+
+\begin_layout Plain Layout
+numMiniBatch4LRSearch
+\end_layout
+
+\end_inset
+
+: the number of minibatches used to search the minibatch size when
+in adaptive minibatch size mode.
+ Default value is 500.
+ It's typically set to 10-20% of the total minibatches in an epoch
+this is shared with the search for learning rate in 
+SearchBeforeEpoch mode.
+
+\end_layout
+
+\begin_layout Itemize
+
+\emph on
+autoAdjustMinibatch
+\emph default
+
+\begin_inset Index idx
+status open
+
+\begin_layout Plain Layout
+autoAdjustMinibatch
+\end_layout
+
+\end_inset
+
+: enable or disable whether minibatch size is adaptively adjusted.
+ Default value is false.
+Adapative minibatch sizing will begin on 
+epochs starting after user minbatch sizes expcitily
+specified are complete.  For example if the user
+specifed minibatchSize=256:1024, then 256 and 1024
+are used in the first 2 Epochs and adaptive minibatch
+sizing is used aferwards
+
+\end_layout
+
+\begin_layout Itemize
+
+\emph on
+minibatchSizeTuningFrequency
+\emph default
+
+\begin_inset Index idx
+status open
+
+\begin_layout Plain Layout
+minibatchSizeTuningFrequency
+\end_layout
+
+\end_inset
+
+: The number of epochs to skip, on a periodic basis, before
+dynamically adjusting the minibatch size.
+ Default value is 1.
+
+\end_layout
+
+\begin_layout Itemize
+
+\emph on
+minibatchSizeTuningMax
+\emph default
+
+\begin_inset Index idx
+status open
+
+\begin_layout Plain Layout
+minibatchSizeTuningMax
+\end_layout
+
+\end_inset
+
+: The maximum size allowed for an
+adaptively adjusted minibatch size.
+ Default value is 1048576.
+
+\end_layout
+
 \end_deeper
 \begin_layout Subsubsection
 Gradient control