Added progress tracing for PreCompute phase - currently just printing…

… 0.0 instead of actual timeout which is sufficient for the current purpose
daihui · Jan 22, 2016 · c177d3a · c177d3a
1 parent 08e4b99
commit c177d3a
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 0 deletions.
diff --git a/Source/Common/Include/ProgressTracing.h b/Source/Common/Include/ProgressTracing.h
@@ -44,6 +44,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
     } // wrap static state in an accessor, so we won't need a CPP file
 
 public:
+
+    static bool IsEnabled()
+    {
+        return GetStaticInstance().m_enabled;
+    }
+
     // call TraceTotalNumberOfSteps() to set the total number of steps
     // Calling this with totalNumberOfSteps>0 will enable progress tracing.
     static void TraceTotalNumberOfSteps(size_t totalNumberOfSteps)

diff --git a/Source/SGDLib/SGD.cpp b/Source/SGDLib/SGD.cpp
@@ -1384,6 +1384,9 @@ template <class ElemType>
             auto node = static_pointer_cast<PreComputedNode<ElemType>>(*nodeIter);
         node->MarkComputed(false /*begin accumulating*/);
         }
+
+        const size_t numIterationsBeforePrintingProgress = 100;
+        size_t numItersSinceLastPrintOfProgress = 0;
         size_t actualMBSizeDummy;
         while (DataReaderHelpers::GetMinibatchIntoNetwork(*trainSetDataReader, net, nullptr, false, false, *inputMatrices, actualMBSizeDummy))
         {
@@ -1392,7 +1395,19 @@ template <class ElemType>
             ComputationNetwork::BumpEvalTimeStamp(labelNodes);
 
             net->ForwardProp(nodes);
+
+            if (ProgressTracing::IsEnabled())
+            {
+                numItersSinceLastPrintOfProgress++;
+                if (numItersSinceLastPrintOfProgress >= numIterationsBeforePrintingProgress)
+                {
+                    // TODO: For now just print 0.0 instead of calculating actual progress
+                    printf("PROGRESS: %.2f%%\n", 0.0f);
+                    numItersSinceLastPrintOfProgress = 0;
+                }
+            }
         }
+
         // finalize
         for (auto nodeIter = nodes.begin(); nodeIter != nodes.end(); nodeIter++)
         {