Add tracing flag and fix FUSE workaround

BrunoBozza · Feb 25, 2016 · 24acd99 · 24acd99
1 parent 18d65de
commit 24acd99
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 2 deletions.
diff --git a/Source/CNTK/CNTK.cpp b/Source/CNTK/CNTK.cpp
@@ -199,6 +199,7 @@ void DoCommands(const ConfigParameters& config)
     // set up progress tracing for compute cluster management
     if (progressTracing && ((g_mpi == nullptr) || g_mpi->IsMainNode()))
     {
+        ProgressTracing::SetTracingFlag();
         ProgressTracing::TraceTotalNumberOfSteps(fullTotalMaxEpochs); // enable tracing, using this as the total number of epochs
     }
 

diff --git a/Source/Common/Include/ProgressTracing.h b/Source/Common/Include/ProgressTracing.h
@@ -28,12 +28,13 @@ namespace Microsoft { namespace MSR { namespace CNTK {
 /*static*/ class ProgressTracing
 {
     bool m_enabled;
+    bool m_tracingFlag;
     size_t m_totalNumberOfSteps; // total number of epochs in entire training run
     size_t m_currentStepOffset;  // current offset
     Timer m_progressTracingTimer;
 
     ProgressTracing()
-        : m_enabled(false), m_totalNumberOfSteps(0), m_currentStepOffset(0)
+        : m_enabled(false), m_tracingFlag(false), m_totalNumberOfSteps(0), m_currentStepOffset(0)
     {
     }
 
@@ -49,6 +50,17 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         return GetStaticInstance().m_enabled;
     }
 
+    static bool GetTracingFlag()
+    {
+        return GetStaticInstance().m_tracingFlag;
+    }
+
+    static void SetTracingFlag()
+    {
+        auto& us = GetStaticInstance();
+        us.m_tracingFlag = true;
+    }
+
     // call TraceTotalNumberOfSteps() to set the total number of steps
     // Calling this with totalNumberOfSteps>0 will enable progress tracing.
     static void TraceTotalNumberOfSteps(size_t totalNumberOfSteps)

diff --git a/Source/Common/fileutil.cpp b/Source/Common/fileutil.cpp
@@ -615,7 +615,7 @@ void renameOrDie(const std::string& from, const std::string& to)
     }
 
     // workaround for FUSE rename when running on Philly
-    if (ProgressTracing::IsEnabled())
+    if (ProgressTracing::GetTracingFlag())
     {
         fprintf(stderr, "sleep for 120 sec as workaround for FUSE write.\n");
         sleep(120);