Merge branch 'master' of https://github.com/Microsoft/CNTK into dongy…

…u/UCIFastReaderFix
aschi87 · Feb 11, 2016 · 8e560b1 · 8e560b1
2 parents 4ff8343 + 9386b8d
commit 8e560b1
Show file tree

Hide file tree

Showing 10 changed files with 437 additions and 159 deletions.
diff --git a/Source/CNTK/CNTK.cpp b/Source/CNTK/CNTK.cpp
@@ -619,12 +619,21 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[]) // called from wmain which is
     return EXIT_SUCCESS;
 }
 
+// new_handler to print call stack upon allocation failure
+void AllocationFailureHandler()
+{
+    Microsoft::MSR::CNTK::DebugUtil::PrintCallStack();
+    std::set_new_handler(nullptr);
+    throw std::bad_alloc();
+}
+
 // ---------------------------------------------------------------------------
 // main wrapper that catches C++ exceptions and prints them
 // ---------------------------------------------------------------------------
 
 int wmain1(int argc, wchar_t* argv[]) // called from wmain which is a wrapper that catches & reports Win32 exceptions
 {
+    std::set_new_handler(AllocationFailureHandler);
     try
     {
         PrintBuiltInfo(); // print build info directly in case that user provides zero argument (convenient for checking build type)

diff --git a/Source/CNTK/tests.cpp b/Source/CNTK/tests.cpp
@@ -115,8 +115,9 @@ void TestReader(const ConfigParameters& configBase)
     GetFileConfigNames(readerConfig, featureNames, labelNames);
 
     // setup minibatch matrices
-    Matrix<ElemType> featuresMatrix(0);
-    Matrix<ElemType> labelsMatrix(0);
+    int deviceId = 0;
+    Matrix<ElemType> featuresMatrix(deviceId);
+    Matrix<ElemType> labelsMatrix(deviceId);
     std::map<std::wstring, Matrix<ElemType>*> matrices;
     matrices[featureNames[0]] = &featuresMatrix;
     matrices[labelNames[0]] = &labelsMatrix;

diff --git a/Source/Common/Include/Basics.h b/Source/Common/Include/Basics.h
@@ -70,7 +70,6 @@ __declspec_noreturn static inline void ThrowFormatted(const char* format, ...)
 #define RuntimeError ThrowFormatted<std::runtime_error>
 #define LogicError ThrowFormatted<std::logic_error>
 #define InvalidArgument ThrowFormatted<std::invalid_argument>
-#define BadExceptionError(...) throw std::bad_exception() // ThrowFormatted<std::bad_exception> does not exist on gcc
 #else
 template <class... _Types>
 __declspec_noreturn static inline void RuntimeError(const char* format, _Types&&... _Args)
@@ -87,11 +86,6 @@ __declspec_noreturn static inline void InvalidArgument(const char* format, _Type
 {
     ThrowFormatted<std::invalid_argument>(format, forward<_Types>(_Args)...);
 }
-template <class... _Types>
-__declspec_noreturn static inline void BadExceptionError(const char* format, _Types&&... _Args)
-{
-    ThrowFormatted<std::bad_exception>(format, forward<_Types>(_Args)...);
-}
 #endif
 
 // Warning - warn with a formatted error string
@@ -130,7 +124,6 @@ using Microsoft::MSR::CNTK::ThrowFormatted;
 using Microsoft::MSR::CNTK::RuntimeError;
 using Microsoft::MSR::CNTK::LogicError;
 using Microsoft::MSR::CNTK::InvalidArgument;
-using Microsoft::MSR::CNTK::BadExceptionError;
 #endif
 
 #ifdef _MSC_VER

diff --git a/Source/Common/Include/ssematrix.h b/Source/Common/Include/ssematrix.h
@@ -1343,7 +1343,7 @@ class ssematrix : public ssematrixbase
     // helpers for SSE-compatible memory allocation
     static __declspec_noreturn void failed(size_t nbytes)
     {
-        BadExceptionError("allocation of SSE vector failed (%d bytes)", nbytes);
+        RuntimeError("allocation of SSE vector failed (%d bytes)", (int)nbytes);
     }
 #ifdef _WIN32
     template <typename T>

diff --git a/Source/Math/GPUMatrix.h b/Source/Math/GPUMatrix.h
@@ -520,7 +520,9 @@ class MATH_API GPUMatrix : public BaseMatrix<ElemType>
         ElemType* pArray = us.CopyToArray();
         for (size_t i = 0; i < us.GetNumElements(); ++i)
             stream << pArray[i];
+
         delete[] pArray;
+
         stream.PutMarker(fileMarkerEndSection, std::wstring(L"EMAT"));
         return stream;
     }

diff --git a/Source/Readers/HTKMLFReader/HTKMLFReader.cpp b/Source/Readers/HTKMLFReader/HTKMLFReader.cpp
@@ -302,6 +302,11 @@ void HTKMLFReader<ElemType>::PrepareForTrainingOrTesting(const ConfigRecordType&
     m_frameMode = readerConfig(L"frameMode", true);
     m_verbosity = readerConfig(L"verbosity", 2);
 
+    if (m_frameMode && m_truncated)
+    {
+        InvalidArgument("'Truncated' cannot be 'true' in frameMode (i.e. when 'frameMode' is 'true')");
+    }
+
     // determine if we partial minibatches are desired
     wstring minibatchMode(readerConfig(L"minibatchMode", L"partial"));
     m_partialMinibatch = EqualCI(minibatchMode, L"partial");
@@ -457,7 +462,8 @@ void HTKMLFReader<ElemType>::PrepareForTrainingOrTesting(const ConfigRecordType&
         m_lattices->setverbosity(m_verbosity);
 
         // now get the frame source. This has better randomization and doesn't create temp files
-        m_frameSource.reset(new msra::dbn::minibatchutterancesourcemulti(infilesmulti, labelsmulti, m_featDims, m_labelDims, numContextLeft, numContextRight, randomize, *m_lattices, m_latticeMap, m_frameMode));
+        bool minimizeReaderMemoryFootprint = readerConfig(L"minimizeReaderMemoryFootprint", true);
+        m_frameSource.reset(new msra::dbn::minibatchutterancesourcemulti(infilesmulti, labelsmulti, m_featDims, m_labelDims, numContextLeft, numContextRight, randomize, *m_lattices, m_latticeMap, m_frameMode, minimizeReaderMemoryFootprint));
         m_frameSource->setverbosity(m_verbosity);
     }
     else if (EqualCI(readMethod, L"rollingWindow"))

diff --git a/Source/Readers/HTKMLFReader/msra_mgram.h b/Source/Readers/HTKMLFReader/msra_mgram.h
@@ -143,7 +143,7 @@ class CSymbolSet : public std::unordered_map<const char *, int, std::hash<const
         // create
         const char *p = _strdup(key);
         if (!p)
-            BadExceptionError("CSymbolSet:id string allocation failure");
+            RuntimeError("CSymbolSet:id string allocation failure");
 
         try
         {