Addressed code review feedback.

aschi87 · Feb 12, 2016 · 19b0c82 · 19b0c82
1 parent fe997db
commit 19b0c82
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/Source/CNTK/SynchronousExecutionEngine.cpp b/Source/CNTK/SynchronousExecutionEngine.cpp
@@ -403,9 +403,9 @@ void SynchronousNodeEvaluator<ElemType>::Evaluate(NDLNode<ElemType>* node, const
             double epsilon = node->GetOptionalParameter("epsilon", "0.00001");
             std::wstring bnEngineS = node->GetOptionalParameter("engine", "cntk");
             bool useCntkEngine;
-            if (bnEngineS == L"cntk")
+            if (EqualCI(bnEngineS, L"cntk"))
                 useCntkEngine = true;
-            else if (bnEngineS == L"cudnn")
+            else if (EqualCI(bnEngineS, L"cudnn"))
                 useCntkEngine = false;
             else
                 InvalidArgument("Unsupported batch normalization engine, choose either \"cntk\"(default) or \"cudnn\".");

diff --git a/Source/ComputationNetworkLib/TrainingNodes.h b/Source/ComputationNetworkLib/TrainingNodes.h
@@ -1788,10 +1788,13 @@ class BatchNormalizationNode : public ComputationNode<ElemType>, public NumInput
             if (m_spatial && m_imageLayoutKind != CHW)
             {
                 InvalidArgument(
-                    "Batch normalization currently supports only cuDNN (CHW) data layout." 
+                    "Batch normalization currently supports only cuDNN (CHW) data layout. " 
                     "Please specify imageLayout=\"cudnn\" in BatchNormalization node in your NDL/BrainScript "
                     "and make sure your input data layout is CHW");
             }
+            double cudnnMinEps = 1e-5; // CUDNN_BN_MIN_EPSILON
+            if (!m_useCntkEngine && m_epsilon < cudnnMinEps) 
+                fprintf(stderr, "\nWARNING: cuDNN batch normalization requires epsilon >= %e. Epsilon will be reset to that value.\n", cudnnMinEps);
 
             auto shape = GetSampleLayout();
 

diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu
@@ -408,6 +408,7 @@ public:
         if (m_bnImpl == BatchNormImpl::CuDnn)
         {
             cudnnBatchNormMode_t mode = spatial ? CUDNN_BATCHNORM_SPATIAL : CUDNN_BATCHNORM_PER_ACTIVATION;
+            // cuDNN will fail with BAD_PARAM if epsilon < CUDNN_BN_MIN_EPSILON.
             epsilon = std::max(epsilon, CUDNN_BN_MIN_EPSILON);
             CUDNN_CALL(cudnnBatchNormalizationForwardTraining(m_cudnn, mode, &C::One, &C::Zero, t(inT), ptr(in), t(inT), ptr(out),
                 t(scaleBiasT), ptr(scale), ptr(bias), expAvgFactor, ptr(runMean), ptr(runInvStdDev), 
@@ -417,6 +418,8 @@ public:
         {
             // No support for exp averaging for now.
             assert(expAvgFactor == 1);
+            if (expAvgFactor != 1)
+                InvalidArgument("CNTK batch norm implementation currently supports expAvgFactor = 1 only.", m_bnImpl);
             epsilon = std::max(epsilon, 1e-9);
             CUDA_CALL(BatchNormalizationForwardTraining(inT, spatial, ptr(in), ptr(out), ptr(scale), ptr(bias),
                                                         ptr(runMean), ptr(runInvStdDev), epsilon,