Merge branch 'master' of https://github.com/Microsoft/CNTK into amita…

…ga/memshareFixes Conflicts: Source/ComputationNetworkLib/ComputationNetwork.h
BrunoBozza · Mar 18, 2016 · 747edc2 · 747edc2
2 parents 5f8f8b2 + 66d7aab
commit 747edc2
Show file tree

Hide file tree

Showing 52 changed files with 2,041 additions and 1,017 deletions.
diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Programmer_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Programmer_Chapter.lyx
@@ -3134,11 +3134,6 @@ virtual void Validate()
 {             
 \end_layout
 
-\begin_layout Plain Layout
-
-    PrintSelfBeforeValidation();
-\end_layout
-
 \begin_layout Plain Layout
 
 

diff --git a/Examples/Text/PennTreebank/Config/S2SAutoEncoder.cntk b/Examples/Text/PennTreebank/Config/S2SAutoEncoder.cntk
diff --git a/Examples/Text/PennTreebank/Config/rnn.cntk b/Examples/Text/PennTreebank/Config/rnn.cntk
@@ -54,11 +54,8 @@ writeWordAndClassInfo = [
 
 train = [
     action = "train"
-    minibatchSize = 128:256:512 # TODO: Why is this here and not inside SGD?
     traceLevel = 1
     epochSize = 0               # (for quick tests, this can be overridden with something small)
-    defaultHiddenActivity = 0.1 # default value for hidden states--is this used by SimpleNetworkBuilder?
-    useValidation = true
 
     SimpleNetworkBuilder = [
         rnnType = "CLASSLSTM"   # TODO: camelCase
@@ -70,6 +67,7 @@ train = [
         initValueScale = 6.0
         uniformInit = true
         layerSizes = "$confVocabSize$:150:200:10000"
+        defaultHiddenActivity = 0.1 # default value for hidden states
         addPrior = false
         addDropoutNodes = false
         applyMeanVarNorm = false
@@ -81,6 +79,7 @@ train = [
     ]
 
     SGD = [
+        minibatchSize = 128:256:512
         learningRatesPerSample = 0.1
         momentumPerMB = 0
         gradientClippingWithTruncation = true
@@ -90,6 +89,10 @@ train = [
         gradUpdateType = "none"
         loadBestModel = true
 
+        dropoutRate = 0.0
+
+        #traceNodeNamesReal = AutoName37 # this allows to track a node's value
+
         # settings for Auto Adjust Learning Rate
         AutoAdjust = [
             autoAdjustLR = "adjustAfterEpoch"
@@ -102,8 +105,6 @@ train = [
             numPrevLearnRates = 5
             numBestSearchEpoch = 1
         ]
-
-        dropoutRate = 0.0
     ]
 
     reader = [

diff --git a/Makefile b/Makefile
@@ -548,6 +548,7 @@ CNTK_SRC =\
 	$(SOURCEDIR)/CNTK/ModelEditLanguage.cpp \
 	$(SOURCEDIR)/CNTK/tests.cpp \
 	$(SOURCEDIR)/ComputationNetworkLib/ComputationNode.cpp \
+	$(SOURCEDIR)/ComputationNetworkLib/ReshapingNodes.cpp \
 	$(SOURCEDIR)/ComputationNetworkLib/ComputationNetwork.cpp \
 	$(SOURCEDIR)/ComputationNetworkLib/ComputationNetworkEvaluation.cpp \
 	$(SOURCEDIR)/ComputationNetworkLib/ComputationNetworkAnalysis.cpp \

diff --git a/Source/1BitSGD b/Source/1BitSGD
diff --git a/Source/ActionsLib/NDLNetworkBuilder.cpp b/Source/ActionsLib/NDLNetworkBuilder.cpp
@@ -206,7 +206,7 @@ void NDLNodeEvaluatorImpl<ElemType>::Evaluate(NDLNode<ElemType>* node, const wst
     else if (cnNodeType == OperationNameOf(RowRepeatNode))
     {
         if (parameter.size() != 2)
-            RuntimeError("RowRepeat should have two parameters. Usage: RowRepeat(origNodeName, numRepeats.");
+            RuntimeError("RowRepeat should have two parameters. Usage: RowRepeat(origNodeName, numRepeats).");
 
         nodeParamCount = 1;
         nodeParamStart = 0;
@@ -238,8 +238,8 @@ void NDLNodeEvaluatorImpl<ElemType>::Evaluate(NDLNode<ElemType>* node, const wst
     }
     else if (cnNodeType == L"Reshape" /*OperationNameOf(ReshapeNode)*/)
     {
-        if (parameter.size() < 2 || parameter.size() > 5)
-            RuntimeError("Reshape should have two to five parameters. Usage: Reshape(origNodeName, numRows, [imageWidth=], [imageHeight=], [imageChannels=].");
+        if (parameter.size() != 2)
+            RuntimeError("Reshape should have two parameters. Usage: Reshape(origNodeName, numRows, [imageWidth=], [imageHeight=], [imageChannels=].");
 
         nodeParamCount = 1;
         nodeParamStart = 0;
@@ -252,8 +252,9 @@ void NDLNodeEvaluatorImpl<ElemType>::Evaluate(NDLNode<ElemType>* node, const wst
             size_t img_width = node->GetOptionalParameter("imageWidth", "0");
             size_t img_height = node->GetOptionalParameter("imageHeight", "0");
             size_t img_channels = node->GetOptionalParameter("imageChannels", "0");
+            ImageLayoutKind imageLayoutKind = ImageLayoutKindFrom(node->GetOptionalParameter("imageLayout", "HWC"));
 
-            nodePtr = builder.LegacyReshape(NULL, num_rows, ImageDimensions::AsTensorShape(img_width, img_height, img_channels, ImageLayoutKind::HWC /*legacy*/), name); // BUGBUG: use a tensor descriptor instead
+            nodePtr = builder.LegacyReshape(NULL, num_rows, ImageDimensions::AsTensorShape(img_width, img_height, img_channels, imageLayoutKind), name);
         }
     }
     else if (cnNodeType == OperationNameOf(PastValueNode) ||

diff --git a/Source/ActionsLib/NetworkFactory.cpp b/Source/ActionsLib/NetworkFactory.cpp
@@ -75,12 +75,28 @@ function<ComputationNetworkPtr(DEVICEID_TYPE)> GetNetworkFactory(const ConfigRec
         // We interface with outer old CNTK config by taking the inner part, which we get as a string, as BrainScript.
         // We prepend a few standard definitions, and also definition of deviceId and precision, which all objects will pull out again when they are being constructed.
         // BUGBUG: We are not getting TextLocations right in this way! Do we need to inject location markers into the source? Moot once we fully switch to BS
-        wstring sourceCode = config.Exists(L"BrainScriptNetworkBuilder") ? config(L"BrainScriptNetworkBuilder") : config(L"ExperimentalNetworkBuilder");
-        auto configDirs = ConfigParameters::GetBrainScriptNetworkBuilderIncludePaths();
-        let expr = BS::ParseConfigDictFromString(L"include \'cntk.core.bs\'"     // Note: Using lowercase here to match the Linux name of the CNTK exe.
-                                                 + msra::strfun::wstrprintf(L"deviceId = %d ; precision = '%ls' ; network = new ComputationNetwork ", (int)deviceId, ElemTypeName<ElemType>())
-                                                 + sourceCode,      // source code has the form [ ... ] with brackets in the string
-                                                 move(configDirs)); // set include paths to all paths that configs were read from; no additional configurable include paths are supported by BrainScriptNetworkBuilder
+        wstring sourceOfNetwork = config.Exists(L"BrainScriptNetworkBuilder") ? config(L"BrainScriptNetworkBuilder") : config(L"ExperimentalNetworkBuilder");
+        if (sourceOfNetwork.find_first_of(L"([") != 0)
+            InvalidArgument("BrainScript network description must be either a BS expression in ( ) or a config record in [ ]");
+
+        // set the include paths to all paths that configs were read from; no additional configurable include paths are supported by BrainScriptNetworkBuilder
+        auto includePaths = ConfigParameters::GetBrainScriptNetworkBuilderIncludePaths();
+
+        // inject additional items into the source code
+        // We support two ways of specifying the network in BrainScript:
+        //  - BrainScriptNetworkBuilder = ( any BS expression that evaluates to a ComputationNetwork )
+        //  - BrainScriptNetworkBuilder = [ constructor parameters for a ComputationNetwork ]
+        if (sourceOfNetwork[0] == '[') // if [ ] form then we turn it into ComputationNetwork by constructing a ComputationNetwork from it
+            sourceOfNetwork = L"new ComputationNetwork " + sourceOfNetwork;
+        let sourceOfBS = msra::strfun::wstrprintf(L"include \'cntk.core.bs\'\n" // include our core lib. Note: Using lowercase here to match the Linux name of the CNTK exe.
+                                                  L"deviceId = %d\n"            // deviceId as passed in
+                                                  L"precision = '%ls'\n"        // 'float' or 'double'
+                                                  L"network = %ls",             // source code of expression that evaluates to a ComputationNetwork
+                                                  (int)deviceId, ElemTypeName<ElemType>(), sourceOfNetwork.c_str());
+        let expr = BS::ParseConfigDictFromString(sourceOfBS, move(includePaths));
+
+        // the rest is done in a lambda that is only evaluated when a virgin network is needed
+        // Note that evaluating the BrainScript *is* instantiating the network, so the evaluate call must be inside the lambda.
         return [expr](DEVICEID_TYPE /*deviceId*/)
         {
             // evaluate the parse tree, particularly the top-level field 'network'

diff --git a/Source/ActionsLib/OtherActions.cpp b/Source/ActionsLib/OtherActions.cpp
@@ -206,13 +206,18 @@ template void DoParameterSVD<double>(const ConfigParameters& config);
 // DoWriteWordAndClassInfo() - implements CNTK "writeWordAndClass" command
 // ===========================================================================
 
-// BUGBUG: This should compare both elements (first one is the word name). This current version leads to different sorting and thus class definitions with VS and gcc.
+// compare functor to for sorting by the second element of a pair
+// TODO: just use a lambda
 template <typename T>
 struct compare_second
 {
     bool operator()(const T& lhs, const T& rhs) const
     {
-        return lhs.second < rhs.second;
+        // BUGBUG: This should compare both elements (first one is the word name). This current version leads to different sorting and thus class definitions with VS and gcc.
+        //if (lhs.second == rhs.second) // if second element
+        //    return lhs.first < rhs.first;
+        //else
+            return lhs.second < rhs.second;
     }
 };
 
@@ -243,8 +248,9 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
     int cutoff = config(L"cutoff", "1");
 
     string inputFile = config(L"inputFile"); // training text file without <unk>
-    string outputVocabFile = config(L"outputVocabFile");
-    string outputWord2Cls  = nbrCls > 0 ? config(L"outputWord2Cls")  : string();
+    string outputMappingFile = config(L"outputMappingFile", ""); // if specified then write a regular mapping file
+    string outputVocabFile   = config(L"outputVocabFile");
+    string outputWord2Cls  = nbrCls > 0 ? config(L"outputWord2Cls") : string();
     string outputCls2Index = nbrCls > 0 ? config(L"outputCls2Index") : string();
 
     string unkWord       = config(L"unk", "<unk>");
@@ -255,14 +261,16 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
     if (beginSequence.empty() || endSequence.empty())
         InvalidArgument("Please specify parameters 'beginSequence' and 'endSequence'.");
 
-    std::cerr     << "Vocabulary file:    " << outputVocabFile << std::endl;
+    if (!outputMappingFile.empty())
+        cerr << "Mapping file       --> " << outputVocabFile << endl;
+    cerr     << "Vocabulary file    --> " << outputVocabFile << endl;
     if (nbrCls > 0)
     {
-        std::cerr << "Word-to-class map:  " << outputWord2Cls  << std::endl;
-        std::cerr << "Class-to-index map: " << outputCls2Index << std::endl;
+        cerr << "Word-to-class map  --> " << outputWord2Cls  << endl;
+        cerr << "Class-to-index map --> " << outputCls2Index << endl;
     }
-    std::cerr << std::endl;
-    
+    cerr << endl;
+
     // check whether we are already up-to-date
     bool makeMode = config(L"makeMode", true);
     if (makeMode)
@@ -275,7 +283,7 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
         }
         if (done)
         {
-            std::cerr << "All output files up to date.\n";
+            cerr << "All output files up to date.\n";
             return;
         }
     }
@@ -286,19 +294,12 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
     ifstream fp(inputFile.c_str()); // TODO: use class File, as to support pipes
     if (!fp)
         RuntimeError("Failed to open input file: %s", inputFile.c_str());
-    cerr << "Reading input file inputFile: " << inputFile << std::endl;
+    cerr << "Reading input file inputFile: " << inputFile << endl;
 
     if (nbrCls > 0)
         cls2idx.Resize(nbrCls, 1);
 
-#if 1
-    std::unordered_map<string, double> v_count;
-#else
-    // TODO: For unknown reasons, this gives a very different result (PPL of 500 instead of 190). Should be tracked down.
-    std::map<string, double> v_count;
-    v_count[beginSequence] = 0;  // get these into the table upfront into position 0 (and 1 if different)
-    v_count[endSequence]   = 0;
-#endif
+    unordered_map<string, double> v_count;
 
     // process input line by line
     string str;
@@ -324,18 +325,14 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
     }
     fp.close();
 
-    std::cerr << "Vocabulary size " << v_count.size() << ".\n";
-
-    std::vector<std::string> m_words;
-    std::set<std::string> m_remained_words;
-    std::unordered_map<std::string, size_t> m_index;
+    cerr << "Vocabulary size " << v_count.size() << ".\n";
 
-    std::vector<double> m_count;
-    std::vector<int> m_class; // class index of each word
+    vector<string> m_words;
+    set<string> m_remained_words;
+    unordered_map<string, size_t> m_index;
 
-    typedef std::pair<std::string, double> stringdouble;
-    std::priority_queue<stringdouble, std::vector<stringdouble>, compare_second<stringdouble>>
-        q(compare_second<stringdouble>(), std::vector<stringdouble>(v_count.begin(), v_count.end()));
+    vector<double> m_count;
+    vector<int> m_class; // class index of each word
 
     size_t wordCountLessCutoff = v_count.size();
     if (cutoff > 0)
@@ -349,24 +346,30 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
 
     if (vocabSize > wordCountLessCutoff)
     {
-        std::cerr << "Warning: actual vocabulary size is less than required." << endl;
-        std::cerr << "\t\tRequired vocabulary size:" << vocabSize << endl;
-        std::cerr << "\t\tActual vocabulary size:" << v_count.size() << endl;
-        std::cerr << "\t\tActual vocabulary size after cutoff:" << wordCountLessCutoff << endl;
-        std::cerr << "\t\tWe will change to actual vocabulary size: " << wordCountLessCutoff << endl;
+        cerr << "Warning: actual vocabulary size is less than required." << endl;
+        cerr << "\t\tRequired vocabulary size:" << vocabSize << endl;
+        cerr << "\t\tActual vocabulary size:" << v_count.size() << endl;
+        cerr << "\t\tActual vocabulary size after cutoff:" << wordCountLessCutoff << endl;
+        cerr << "\t\tWe will change to actual vocabulary size: " << wordCountLessCutoff << endl;
         vocabSize = wordCountLessCutoff;
     }
+
+    // form classes
+    // Implements an algorithm by Mikolov --TODO: get the reference
     wrd2cls.Resize(vocabSize, 1);
 
-    std::unordered_map<std::string, double> removed;
-    double unkCount = 0;
+    typedef pair<string, double> stringdouble;
+    unordered_map<string, double> removed; // note: std::map is supposedly faster
+    double unkCount = 0; // TODO: why double?
     size_t size = 0;
     size_t actual_vocab_size = vocabSize - 1;
-    while (size < actual_vocab_size && !q.empty())
+    priority_queue<stringdouble, vector<stringdouble>, compare_second<stringdouble>>
+        q(compare_second<stringdouble>(), vector<stringdouble>(v_count.begin(), v_count.end()));
+    while (size < actual_vocab_size && !q.empty()) // ==for (q=...; cond; q.pop())
     {
         size++;
-        std::string word = q.top().first;
-        double freq = q.top().second;
+        string word = q.top().first;
+        double freq = q.top().second; // TODO: why double?
         if (word == unkWord)
         {
             unkCount += freq;
@@ -381,8 +384,6 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
         q.pop();
     }
     removed[unkWord] = unkCount;
-    std::priority_queue<stringdouble, std::vector<stringdouble>, compare_second<stringdouble>>
-        p(compare_second<stringdouble>(), std::vector<stringdouble>(removed.begin(), removed.end()));
     m_count.resize(removed.size());
     double total = 0;
     double dd = 0;
@@ -397,11 +398,13 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
 
     double df = 0;
     size_t class_id = 0;
-    m_class.resize(p.size());
+    m_class.resize(removed.size());
 
+    priority_queue<stringdouble, vector<stringdouble>, compare_second<stringdouble>>
+        p(compare_second<stringdouble>(), vector<stringdouble>(removed.begin(), removed.end()));
     while (!p.empty())
     {
-        std::string word = p.top().first;
+        string word = p.top().first;
         double freq = p.top().second;
         if (nbrCls > 0)
         {
@@ -424,9 +427,19 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
         p.pop();
     }
 
-    std::ofstream ofvocab;
+    // write the files
+    if (!outputMappingFile.empty())
+    {
+        msra::files::make_intermediate_dirs(s2ws(outputMappingFile));
+        ofstream ofmapping(outputMappingFile.c_str());
+        for (size_t i = 0; i < m_index.size(); i++)
+            ofmapping << m_words[i] << endl;
+        ofmapping.close();
+        cerr << "Created label-mapping file with " << v_count.size() << " entries.\n";
+    }
+
     msra::files::make_intermediate_dirs(s2ws(outputVocabFile));
-    ofvocab.open(outputVocabFile.c_str());
+    ofstream ofvocab(outputVocabFile.c_str());
     for (size_t i = 0; i < m_index.size(); i++)
     {
         if (nbrCls > 0)
@@ -437,10 +450,10 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
             cls2idx(clsIdx, 0) = (ElemType) i; // the left boundary of clsIdx
             prevClsIdx = m_class[i];
         }
-        ofvocab << "     " << i << "\t     " << m_count[i] << "\t" << m_words[i] << "\t" << clsIdx << std::endl;
+        ofvocab << "     " << i << "\t     " << m_count[i] << "\t" << m_words[i] << "\t" << clsIdx << endl;
     }
     ofvocab.close();
-    std::cerr << "Created vocabulary file with " << v_count.size() << " entries.\n";
+    cerr << "Created vocabulary file with " << v_count.size() << " entries.\n";
 
     if (nbrCls > 0)
     {
@@ -453,7 +466,7 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
         for (size_t r = 0; r < wrd2cls.GetNumRows(); r++)
             owfp << (int) wrd2cls(r, 0) << endl;
         owfp.close();
-        std::cerr << "Created word-to-class map with " << wrd2cls.GetNumRows() << " entries.\n";
+        cerr << "Created word-to-class map with " << wrd2cls.GetNumRows() << " entries.\n";
 
         msra::files::make_intermediate_dirs(s2ws(outputCls2Index));
         ofstream ocfp(outputCls2Index.c_str());
@@ -462,7 +475,7 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
         for (size_t r = 0; r < cls2idx.GetNumRows(); r++)
             ocfp << (int) cls2idx(r, 0) << endl;
         ocfp.close();
-        std::cerr << "Created class-to-index map with " << cls2idx.GetNumRows() << " entries.\n";
+        cerr << "Created class-to-index map with " << cls2idx.GetNumRows() << " entries.\n";
     }
 }
 

diff --git a/Source/CNTK/BrainScript/BrainScriptEvaluator.cpp b/Source/CNTK/BrainScript/BrainScriptEvaluator.cpp
@@ -53,7 +53,7 @@ class EvaluationException : public ConfigException
 
 __declspec_noreturn static inline void EvaluationError(const wstring &msg, TextLocation where)
 {
-    Microsoft::MSR::CNTK::DebugUtil::PrintCallStack();
+    //Microsoft::MSR::CNTK::DebugUtil::PrintCallStack();
     throw EvaluationException(msg, where);
 }
 
@@ -768,7 +768,7 @@ void Do(ExpressionPtr e)
 
 shared_ptr<Object> EvaluateField(ExpressionPtr e, const wstring &id)
 {
-    return RecordLookup(e, id, e->location, nullptr /*scope for evaluating 'e'*/, L"$"); // we evaluate the member 'do'
+    return RecordLookup(e, id, e->location, nullptr /*scope for evaluating 'e'*/, L""); // we evaluate the member 'do'
 }
 
 ConfigValuePtr Evaluate(ExpressionPtr e)