diff --git a/Makefile b/Makefile index 3d7f9c5aed4c..479f64f7dbe2 100644 --- a/Makefile +++ b/Makefile @@ -265,6 +265,7 @@ READER_SRC =\ COMMON_SRC =\ $(SOURCEDIR)/Common/Config.cpp \ + $(SOURCEDIR)/Common/Globals.cpp \ $(SOURCEDIR)/Common/DataReader.cpp \ $(SOURCEDIR)/Common/DataWriter.cpp \ $(SOURCEDIR)/Common/ExceptionWithCallStack.cpp \ diff --git a/Source/CNTK/CNTK.cpp b/Source/CNTK/CNTK.cpp index 3f42a1d3399d..8e896352861c 100644 --- a/Source/CNTK/CNTK.cpp +++ b/Source/CNTK/CNTK.cpp @@ -13,6 +13,7 @@ #endif #include "Basics.h" +#include "Globals.h" #include "Actions.h" #include "ComputationNetwork.h" #include "ComputationNode.h" @@ -480,6 +481,9 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wrapp let valp = BS::Evaluate(expr); // evaluate parse into a dictionary let& config = valp.AsRef(); // this is the dictionary + if (config(L"forceDeterministicAlgorithms", false)) + Globals::ForceDeterministicAlgorithms(); + #ifndef CPUONLY auto valpp = config.Find(L"deviceId"); if (valpp) @@ -613,6 +617,9 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[]) ProgressTracing::SetTimestampingFlag(); } + if (config(L"forceDeterministicAlgorithms", false)) + Globals::ForceDeterministicAlgorithms(); + // get the command param set they want wstring logpath = config(L"stderr", L""); diff --git a/Source/CNTK/CNTK.vcxproj b/Source/CNTK/CNTK.vcxproj index eed60a80870c..5deb5f1ea51a 100644 --- a/Source/CNTK/CNTK.vcxproj +++ b/Source/CNTK/CNTK.vcxproj @@ -149,6 +149,7 @@ + diff --git a/Source/CNTK/CNTK.vcxproj.filters b/Source/CNTK/CNTK.vcxproj.filters index c6426d798652..f565939449ab 100644 --- a/Source/CNTK/CNTK.vcxproj.filters +++ b/Source/CNTK/CNTK.vcxproj.filters @@ -174,6 +174,9 @@ + + Common\Include + diff --git a/Source/Common/Common.vcxproj b/Source/Common/Common.vcxproj index 5f395786594f..6dce529d7ec2 100644 --- a/Source/Common/Common.vcxproj +++ b/Source/Common/Common.vcxproj @@ -65,9 +65,10 @@ + - + \ No newline at end of file diff --git a/Source/Common/Globals.cpp b/Source/Common/Globals.cpp new file mode 100644 index 000000000000..24edcdf386d0 --- /dev/null +++ b/Source/Common/Globals.cpp @@ -0,0 +1,14 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +#include "Globals.h" + +using namespace std; + +namespace Microsoft { namespace MSR { namespace CNTK { + + std::atomic Globals::m_forceDeterministicAlgorithms(false); + +}}} \ No newline at end of file diff --git a/Source/Common/Include/Globals.h b/Source/Common/Include/Globals.h new file mode 100644 index 000000000000..eddf803c869f --- /dev/null +++ b/Source/Common/Include/Globals.h @@ -0,0 +1,29 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +#pragma once + +#include + +namespace Microsoft { namespace MSR { namespace CNTK { + + // Class containing global configuration for CNTK. + class Globals + { + public: + static void ForceDeterministicAlgorithms() + { + m_forceDeterministicAlgorithms = true; + } + + static bool ShouldForceDeterministicAlgorithms() + { + return m_forceDeterministicAlgorithms; + } + + private: + static std::atomic m_forceDeterministicAlgorithms; + }; +}}} diff --git a/Source/ComputationNetworkLib/ConvolutionalNodes.h b/Source/ComputationNetworkLib/ConvolutionalNodes.h index 12d2be58fcad..c95ee11a8b1b 100644 --- a/Source/ComputationNetworkLib/ConvolutionalNodes.h +++ b/Source/ComputationNetworkLib/ConvolutionalNodes.h @@ -5,6 +5,7 @@ #pragma once #include "Basics.h" +#include "Globals.h" #include "Matrix.h" #include "ComputationNode.h" #include "ConvolutionEngine.h" @@ -452,7 +453,7 @@ class ConvolutionNode : public ConvolutionNodeBase, public NumInputs<2 m_sharing, m_autoPad, m_lowerPad, m_upperPad); m_convEng = ConvolutionEngine::Create(geometry, m_deviceId, m_imageLayout, m_maxTempMemSizeInSamples, m_poolKind, - ConvolutionEngineKind::All, NodeName()); + ConvolutionEngineKind::All, NodeName(), Globals::ShouldForceDeterministicAlgorithms()); } if (Input(0)->GetSampleLayout().GetNumElements() != m_kernelShape.GetNumElements() * m_convEng->Geometry()->KernelCount()) diff --git a/Source/Math/ConvolutionEngine.cpp b/Source/Math/ConvolutionEngine.cpp index 6dc56e7f1b17..5fb7eb22840b 100644 --- a/Source/Math/ConvolutionEngine.cpp +++ b/Source/Math/ConvolutionEngine.cpp @@ -850,7 +850,7 @@ class GemmConvolutionEngine : public ReferenceConvolutionEngine template std::unique_ptr> ConvolutionEngine::Create(ConvolveGeometryPtr geometry, DEVICEID_TYPE deviceId, ImageLayoutKind imageLayout, size_t maxTempMemSizeInSamples, PoolKind poolKind, - ConvolutionEngineKind enabledEngines, std::wstring logPrefix) + ConvolutionEngineKind enabledEngines, std::wstring logPrefix, bool forceDeterministicAlgorithms) { if (!logPrefix.empty()) logPrefix += L": "; @@ -875,7 +875,7 @@ std::unique_ptr> ConvolutionEngine::Create CuDnnConvolutionEngineFactory::IsSupported(deviceId, geometry, poolKind)) { fprintf(stderr, "%lsusing cuDNN convolution engine for geometry: %s.\n", logPrefix.c_str(), engStr.c_str()); - return CuDnnConvolutionEngineFactory::Create(geometry, deviceId, imageLayout, maxTempMemSizeInSamples, poolKind); + return CuDnnConvolutionEngineFactory::Create(geometry, deviceId, imageLayout, maxTempMemSizeInSamples, poolKind, forceDeterministicAlgorithms); } if (isEnabled(ConvolutionEngineKind::Gemm) && GemmConvolutionEngine::IsSupported(deviceId, geometry)) diff --git a/Source/Math/ConvolutionEngine.h b/Source/Math/ConvolutionEngine.h index 8a9bc67b3fbc..a903cce83b45 100644 --- a/Source/Math/ConvolutionEngine.h +++ b/Source/Math/ConvolutionEngine.h @@ -59,10 +59,10 @@ class MATH_API ConvolutionEngine std::shared_ptr Geometry() const { return m_geometry; } - static std::unique_ptr> Create(ConvolveGeometryPtr geometry, DEVICEID_TYPE deviceId, ImageLayoutKind imageLayout, - size_t maxTempMemSizeInSamples, PoolKind poolKind = PoolKind::None, + static std::unique_ptr> Create(ConvolveGeometryPtr geometry, DEVICEID_TYPE deviceId, + ImageLayoutKind imageLayout, size_t maxTempMemSizeInSamples, PoolKind poolKind = PoolKind::None, ConvolutionEngineKind enabledEngines = ConvolutionEngineKind::All, - std::wstring logPrefix = L""); + std::wstring logPrefix = L"", bool forceDeterministicAlgorithms = false); DISABLE_COPY_AND_MOVE(ConvolutionEngine); diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index 873a8eb25bf6..b6375d9976f0 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -117,7 +117,7 @@ private: class CuDnnPool { public: - CuDnnPool(const ConvolveGeometry& geometry, PoolKind kind) + CuDnnPool(const ConvolveGeometry& geometry, PoolKind kind, bool forceDeterministicAlgorithms) : m_pool(nullptr) { assert(kind == PoolKind::Max || kind == PoolKind::Average); @@ -139,7 +139,7 @@ public: // Must use CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING to get the same results as in reference engine. CUDNN_CALL(cudnnSetPoolingNdDescriptor(m_pool, - kind == PoolKind::Max ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, + kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, CUDNN_PROPAGATE_NAN, (int)dims.size(), dims.data(), pad.data(), stride.data())); } @@ -173,12 +173,13 @@ public: public: CuDnnConvolutionEngine(ConvolveGeometryPtr geometry, DEVICEID_TYPE deviceId, ImageLayoutKind imageLayout, - size_t maxTempMemSizeInSamples, PoolKind poolKind) + size_t maxTempMemSizeInSamples, PoolKind poolKind, bool forceDeterministicAlgorithms) : Base(geometry, deviceId, imageLayout, maxTempMemSizeInSamples, poolKind), m_cudnn(CuDnn::Instance()), m_dataType(CuDnnTensor::GetDataType()), m_inT(geometry->InputShape(), m_dataType), - m_outT(geometry->OutputShape(), m_dataType) + m_outT(geometry->OutputShape(), m_dataType), + m_forceDeterministicAlgorithms(forceDeterministicAlgorithms) { } @@ -212,7 +213,17 @@ protected: // Find best algo and allocate temp buffer, if needed. auto finder = [this](int& calgo, cudnnConvolutionFwdAlgoPerf_t algoPerf[MaxAlgoCount]) -> cudnnStatus_t { - return cudnnFindConvolutionForwardAlgorithm(*m_cudnn, m_inT, *m_kernelT, *m_conv, m_outT, MaxAlgoCount, &calgo, algoPerf); + auto result = cudnnFindConvolutionForwardAlgorithm(*m_cudnn, m_inT, *m_kernelT, *m_conv, m_outT, MaxAlgoCount, &calgo, algoPerf); + if (m_forceDeterministicAlgorithms) + { + auto found = std::find_if(algoPerf, algoPerf + calgo, + [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); + if (found == algoPerf + calgo) + RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); + calgo = 1; + algoPerf[0] = *found; + } + return result; }; auto staticFinder = [this](cudnnConvolutionFwdAlgo_t& algo) -> cudnnStatus_t { @@ -228,6 +239,8 @@ protected: // REVIEW alexeyk: NVIDIA is currently reviewing this issue. if (CUDNN_STATUS_INVALID_VALUE == err && m_fwdAlgo.Algo.memory > 0) { + if (m_forceDeterministicAlgorithms) + RuntimeError("Falling back of the algorithms is not allowed. Please set 'forceDeterministicAlgorithms=false'."); auto err2 = cudnnConvolutionForward(*m_cudnn, &C::One, m_inT, ptr(in), *m_kernelT, ptr(kernel), *m_conv, m_fwdAlgo.NoWorkspaceAlgo, nullptr, 0, &C::Zero, m_outT, ptr(out)); // Update original error in case of success. @@ -243,7 +256,17 @@ protected: // Find best algo and allocate temp buffer, if needed. auto finder = [this](int& calgo, cudnnConvolutionBwdDataAlgoPerf_t algoPerf[MaxAlgoCount]) -> cudnnStatus_t { - return cudnnFindConvolutionBackwardDataAlgorithm(*m_cudnn, *m_kernelT, m_outT, *m_conv, m_inT, MaxAlgoCount, &calgo, algoPerf); + auto result = cudnnFindConvolutionBackwardDataAlgorithm(*m_cudnn, *m_kernelT, m_outT, *m_conv, m_inT, MaxAlgoCount, &calgo, algoPerf); + if (m_forceDeterministicAlgorithms) + { + auto found = std::find_if(algoPerf, algoPerf + calgo, + [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + if (found == algoPerf + calgo) + RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); + calgo = 1; + algoPerf[0] = *found; + } + return result; }; auto staticFinder = [this](cudnnConvolutionBwdDataAlgo_t& algo) -> cudnnStatus_t { @@ -263,7 +286,17 @@ protected: // Find best algo and allocate temp buffer, if needed. auto finder = [this](int& calgo, cudnnConvolutionBwdFilterAlgoPerf_t algoPerf[MaxAlgoCount]) -> cudnnStatus_t { - return cudnnFindConvolutionBackwardFilterAlgorithm(*m_cudnn, m_inT, m_outT, *m_conv, *m_kernelT, MaxAlgoCount, &calgo, algoPerf); + auto result = cudnnFindConvolutionBackwardFilterAlgorithm(*m_cudnn, m_inT, m_outT, *m_conv, *m_kernelT, MaxAlgoCount, &calgo, algoPerf); + if (m_forceDeterministicAlgorithms) + { + auto found = std::find_if(algoPerf, algoPerf + calgo, + [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + if (found == algoPerf + calgo) + RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); + calgo = 1; + algoPerf[0] = *found; + } + return result; }; auto staticFinder = [this](cudnnConvolutionBwdFilterAlgo_t& algo) -> cudnnStatus_t { @@ -280,7 +313,7 @@ protected: void EnsurePoolingInitialized() override { if (m_pool == nullptr) - m_pool = std::make_unique(*m_geometry, m_poolKind); + m_pool = std::make_unique(*m_geometry, m_poolKind, m_forceDeterministicAlgorithms); } void ForwardPoolingCore(const Mat& in, Mat& out) override @@ -329,10 +362,14 @@ private: cudnnStatus_t err = finder(calgo, algoPerf); // Alloc failed - usually means cuDNN runtime auto-tuner could not allocate workspace. // In such case, use static auto-tuner with no workspace. + // This should never happen in the deterministic mode because we pick up algorithms with 0 memory workspace. if (err == CUDNN_STATUS_ALLOC_FAILED) { decltype(CuDnnAlgoT::algo) noMemAlgo; CUDNN_CALL(staticFinder(noMemAlgo)); + if (m_forceDeterministicAlgorithms) + RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); + algo.MaxAllowedMBSizeForCurrentAlgo = batchSize; algo.Algo = algoPerf[0]; algo.Algo.algo = noMemAlgo; @@ -347,20 +384,20 @@ private: size_t maxMem = m_maxTempMemSizeInSamples == 0 ? (std::numeric_limits::max)() : inputSampleSize * m_maxTempMemSizeInSamples * sizeof(ElemType); // Find best (fastest) algorithm which satisfies workspace requirements. auto res = std::find_if(algoPerf, algoPerf + calgo, - [=](const CuDnnAlgoT& cur) - { - return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; - }); + [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); + if (res == algoPerf + calgo) RuntimeError("cuDNN could not find suitable algorithm for the current convolution configuration."); algo.MaxAllowedMBSizeForCurrentAlgo = batchSize; algo.Algo = *res; + + if (m_forceDeterministicAlgorithms) // does not allow fallback. + return; + // Find fastest algorithm that does NOT require workspace. It is used as a fallback algo in Forward function. - res = std::find_if(algoPerf, algoPerf + calgo, - [](const CuDnnAlgoT& cur) - { - return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; - }); + // Currently all Forward algorithms are deterministic, so no need for checking. + res = std::find_if(algoPerf, algoPerf + calgo, + [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); if (res == algoPerf + calgo) { // In theory, this should never happen. @@ -423,14 +460,18 @@ private: ConvAlgoInfo m_fwdAlgo; ConvAlgoInfo m_backDataAlgo; ConvAlgoInfo m_backFiltAlgo; + + // Flag indicating whether only deterministic algorithms should be used. + bool m_forceDeterministicAlgorithms; }; template std::unique_ptr> CuDnnConvolutionEngineFactory::Create(ConvolveGeometryPtr geometry, DEVICEID_TYPE deviceId, ImageLayoutKind imageLayout, - size_t maxTempMemSizeInSamples, PoolKind poolKind) + size_t maxTempMemSizeInSamples, PoolKind poolKind, + bool forceDeterministicAlgorithms) { - return std::make_unique>(geometry, deviceId, imageLayout, maxTempMemSizeInSamples, poolKind); + return std::make_unique>(geometry, deviceId, imageLayout, maxTempMemSizeInSamples, poolKind, forceDeterministicAlgorithms); } template diff --git a/Source/Math/CuDnnFactories.h b/Source/Math/CuDnnFactories.h index df5138f29aa0..e62a7821324c 100644 --- a/Source/Math/CuDnnFactories.h +++ b/Source/Math/CuDnnFactories.h @@ -16,7 +16,7 @@ class CuDnnConvolutionEngineFactory public: static std::unique_ptr> Create(ConvolveGeometryPtr geometry, DEVICEID_TYPE deviceId, ImageLayoutKind imageLayout, size_t maxTempMemSizeInSamples, - PoolKind poolKind); + PoolKind poolKind, bool forceDeterministicAlgorithms); static bool IsSupported(DEVICEID_TYPE deviceId, ConvolveGeometryPtr geometry, PoolKind poolKind); }; diff --git a/Source/Math/NoGPU.cpp b/Source/Math/NoGPU.cpp index e1cb1bd62009..9b73078dc13e 100644 --- a/Source/Math/NoGPU.cpp +++ b/Source/Math/NoGPU.cpp @@ -2255,7 +2255,7 @@ void* GPUMatrix::s_curandGenerator = NULL; template std::unique_ptr> CuDnnConvolutionEngineFactory::Create(ConvolveGeometryPtr, DEVICEID_TYPE, - ImageLayoutKind, size_t, PoolKind) + ImageLayoutKind, size_t, PoolKind, bool) { RuntimeError("The code is compiled with CPUONLY macro."); } diff --git a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.linux.txt b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.linux.txt index c21ded0df4fe..c598f041b27d 100644 --- a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.linux.txt +++ b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.linux.txt @@ -3,12 +3,12 @@ CPU info: Hardware threads: 24 Total Memory: 264172964 kB ------------------------------------------------------------------- -=== Running /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/release/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config/02_Convolution.cntk currentDirectory=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData RunDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu DataDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config OutputDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" +=== Running /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/release/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config/02_Convolution.cntk currentDirectory=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData RunDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu DataDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config OutputDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true forceDeterministicAlgorithms=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" ------------------------------------------------------------------- Build info: - Built time: Aug 16 2016 09:41:56 - Last modified date: Fri Aug 12 07:32:43 2016 + Built time: Aug 26 2016 09:19:57 + Last modified date: Fri Aug 26 09:18:31 2016 Build type: release Build target: GPU With 1bit-SGD: no @@ -17,207 +17,64 @@ Build info: CUB_PATH: /usr/local/cub-1.4.1 CUDNN_PATH: /usr/local/cudnn-4.0 Build Branch: HEAD - Build SHA1: 026b1e772b963461e189f8f00aa7ed6951298f84 - Built by philly on f67b30a647de + Build SHA1: a718f2f213d1a045d5bbfa88f90c18fe15bd22f2 + Built by philly on 643085f7f8c2 Build Path: /home/philly/jenkins/workspace/CNTK-Build-Linux ------------------------------------------------------------------- -Changed current directory to /tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData -08/16/2016 10:49:50: ------------------------------------------------------------------- -08/16/2016 10:49:50: Build info: - -08/16/2016 10:49:50: Built time: Aug 16 2016 09:41:56 -08/16/2016 10:49:50: Last modified date: Fri Aug 12 07:32:43 2016 -08/16/2016 10:49:50: Build type: release -08/16/2016 10:49:50: Build target: GPU -08/16/2016 10:49:50: With 1bit-SGD: no -08/16/2016 10:49:50: Math lib: mkl -08/16/2016 10:49:50: CUDA_PATH: /usr/local/cuda-7.5 -08/16/2016 10:49:50: CUB_PATH: /usr/local/cub-1.4.1 -08/16/2016 10:49:50: CUDNN_PATH: /usr/local/cudnn-4.0 -08/16/2016 10:49:50: Build Branch: HEAD -08/16/2016 10:49:50: Build SHA1: 026b1e772b963461e189f8f00aa7ed6951298f84 -08/16/2016 10:49:50: Built by philly on f67b30a647de -08/16/2016 10:49:50: Build Path: /home/philly/jenkins/workspace/CNTK-Build-Linux -08/16/2016 10:49:50: ------------------------------------------------------------------- -08/16/2016 10:49:51: ------------------------------------------------------------------- -08/16/2016 10:49:51: GPU info: - -08/16/2016 10:49:51: Device[0]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB -08/16/2016 10:49:51: Device[1]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB -08/16/2016 10:49:51: Device[2]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB -08/16/2016 10:49:51: Device[3]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB -08/16/2016 10:49:51: ------------------------------------------------------------------- - -08/16/2016 10:49:51: Running on localhost at 2016/08/16 10:49:51 -08/16/2016 10:49:51: Command line: -/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/release/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config/02_Convolution.cntk currentDirectory=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData RunDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu DataDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config OutputDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" - - - -08/16/2016 10:49:51: >>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>> -08/16/2016 10:49:51: rootDir = ".." -configDir = "$rootDir$/Config" -dataDir = "$rootDir$/Data" -outputDir = "$rootDir$/Output" -modelDir = "$outputDir$/Models" -deviceId = 0 -command = train:test -precision = "float" -modelPath = "$modelDir$/02_Convolution" -traceLevel = 1 -numMBsToShowResult = 500 -train = [ - action = "train" - NDLNetworkBuilder = [ - imageLayout = "cudnn" - initOnCPUOnly = true - ndlMacros = "$configDir$/Macros.ndl" - networkDescription = "$ConfigDir$/02_Convolution.ndl" - ] - SGD = [ - epochSize = 60000 - minibatchSize = 32 - learningRatesPerMB = 0.1*5:0.3 - momentumPerMB = 0*10:0.7 - maxEpochs = 15 - ] - reader = [ - readerType = "CNTKTextFormatReader" - file = "$DataDir$/Train-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -test = [ - action = test - minibatchSize = 1024 - reader = [ - readerType = "CNTKTextFormatReader" - file = "$DataDir$/Test-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -currentDirectory=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData -RunDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu -DataDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData -ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config -OutputDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu -DeviceId=0 -timestamping=true -train=[SGD=[maxEpochs=3]] -imageLayout="cudnn" - -08/16/2016 10:49:51: <<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<< - -08/16/2016 10:49:51: >>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> -08/16/2016 10:49:51: rootDir = ".." -configDir = "../Config" -dataDir = "../Data" -outputDir = "../Output" -modelDir = "/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models" -deviceId = 0 -command = train:test -precision = "float" -modelPath = "/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution" -traceLevel = 1 -numMBsToShowResult = 500 -train = [ - action = "train" - NDLNetworkBuilder = [ - imageLayout = "cudnn" - initOnCPUOnly = true - ndlMacros = "/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config/Macros.ndl" - networkDescription = "/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config/02_Convolution.ndl" - ] - SGD = [ - epochSize = 60000 - minibatchSize = 32 - learningRatesPerMB = 0.1*5:0.3 - momentumPerMB = 0*10:0.7 - maxEpochs = 15 - ] - reader = [ - readerType = "CNTKTextFormatReader" - file = "/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData/Train-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -test = [ - action = test - minibatchSize = 1024 - reader = [ - readerType = "CNTKTextFormatReader" - file = "/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData/Test-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -currentDirectory=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData -RunDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu -DataDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData -ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config -OutputDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu -DeviceId=0 -timestamping=true -train=[SGD=[maxEpochs=3]] -imageLayout="cudnn" - -08/16/2016 10:49:51: <<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< - -08/16/2016 10:49:51: >>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> +Changed current directory to /tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData +08/26/2016 11:16:48: ------------------------------------------------------------------- +08/26/2016 11:16:48: Build info: + +08/26/2016 11:16:48: Built time: Aug 26 2016 09:19:57 +08/26/2016 11:16:48: Last modified date: Fri Aug 26 09:18:31 2016 +08/26/2016 11:16:48: Build type: release +08/26/2016 11:16:48: Build target: GPU +08/26/2016 11:16:48: With 1bit-SGD: no +08/26/2016 11:16:48: Math lib: mkl +08/26/2016 11:16:48: CUDA_PATH: /usr/local/cuda-7.5 +08/26/2016 11:16:48: CUB_PATH: /usr/local/cub-1.4.1 +08/26/2016 11:16:48: CUDNN_PATH: /usr/local/cudnn-4.0 +08/26/2016 11:16:48: Build Branch: HEAD +08/26/2016 11:16:48: Build SHA1: a718f2f213d1a045d5bbfa88f90c18fe15bd22f2 +08/26/2016 11:16:48: Built by philly on 643085f7f8c2 +08/26/2016 11:16:48: Build Path: /home/philly/jenkins/workspace/CNTK-Build-Linux +08/26/2016 11:16:48: ------------------------------------------------------------------- +08/26/2016 11:16:49: ------------------------------------------------------------------- +08/26/2016 11:16:49: GPU info: + +08/26/2016 11:16:49: Device[0]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB +08/26/2016 11:16:49: Device[1]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB +08/26/2016 11:16:49: Device[2]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB +08/26/2016 11:16:49: Device[3]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3071 MB +08/26/2016 11:16:49: ------------------------------------------------------------------- + +08/26/2016 11:16:49: Running on localhost at 2016/08/26 11:16:49 +08/26/2016 11:16:49: Command line: +/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/release/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config/02_Convolution.cntk currentDirectory=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData RunDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu DataDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config OutputDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true forceDeterministicAlgorithms=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" + + +Configuration After Processing and Variable Resolution: + configparameters: 02_Convolution.cntk:command=train:test configparameters: 02_Convolution.cntk:configDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/../../../../../../Examples/Image/MNIST/Config -configparameters: 02_Convolution.cntk:currentDirectory=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData -configparameters: 02_Convolution.cntk:dataDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData +configparameters: 02_Convolution.cntk:currentDirectory=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData +configparameters: 02_Convolution.cntk:dataDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData configparameters: 02_Convolution.cntk:deviceId=0 +configparameters: 02_Convolution.cntk:forceDeterministicAlgorithms=true configparameters: 02_Convolution.cntk:imageLayout=cudnn -configparameters: 02_Convolution.cntk:modelDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models -configparameters: 02_Convolution.cntk:modelPath=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution +configparameters: 02_Convolution.cntk:modelDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/Models +configparameters: 02_Convolution.cntk:modelPath=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution configparameters: 02_Convolution.cntk:numMBsToShowResult=500 -configparameters: 02_Convolution.cntk:outputDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu +configparameters: 02_Convolution.cntk:outputDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu configparameters: 02_Convolution.cntk:precision=float configparameters: 02_Convolution.cntk:rootDir=.. -configparameters: 02_Convolution.cntk:RunDir=/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu +configparameters: 02_Convolution.cntk:RunDir=/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu configparameters: 02_Convolution.cntk:test=[ action = test minibatchSize = 1024 reader = [ readerType = "CNTKTextFormatReader" - file = "/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData/Test-28x28_cntk_text.txt" + file = "/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData/Test-28x28_cntk_text.txt" input = [ features = [ dim = 784 @@ -250,7 +107,7 @@ configparameters: 02_Convolution.cntk:train=[ ] reader = [ readerType = "CNTKTextFormatReader" - file = "/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/TestData/Train-28x28_cntk_text.txt" + file = "/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/TestData/Train-28x28_cntk_text.txt" input = [ features = [ dim = 784 @@ -264,23 +121,22 @@ configparameters: 02_Convolution.cntk:train=[ ] ] [SGD=[maxEpochs=3]] -08/16/2016 10:49:51: <<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< -08/16/2016 10:49:51: Commands: train test -08/16/2016 10:49:51: Precision = "float" -08/16/2016 10:49:51: CNTKModelPath: /tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution -08/16/2016 10:49:51: CNTKCommandTrainInfo: train : 3 -08/16/2016 10:49:51: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 3 +08/26/2016 11:16:49: Commands: train test +08/26/2016 11:16:49: Precision = "float" +08/26/2016 11:16:49: CNTKModelPath: /tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution +08/26/2016 11:16:49: CNTKCommandTrainInfo: train : 3 +08/26/2016 11:16:49: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 3 -08/16/2016 10:49:51: ############################################################################## -08/16/2016 10:49:51: # # -08/16/2016 10:49:51: # Action "train" # -08/16/2016 10:49:51: # # -08/16/2016 10:49:51: ############################################################################## +08/26/2016 11:16:49: ############################################################################## +08/26/2016 11:16:49: # # +08/26/2016 11:16:49: # Action "train" # +08/26/2016 11:16:49: # # +08/26/2016 11:16:49: ############################################################################## -08/16/2016 10:49:51: CNTKCommandTrainBegin: train -NDLBuilder Using GPU 0 +08/26/2016 11:16:49: CNTKCommandTrainBegin: train -08/16/2016 10:49:51: Creating virgin network. +08/26/2016 11:16:49: Creating virgin network. +NDLBuilder Using GPU 0 Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.000000. Node 'conv1.w.W' (LearnableParameter operation): Initializing Parameter[16 x 25] <- 0.000000. Node 'conv1.b.b' (LearnableParameter operation): Initializing Parameter[1 x 1 x 16] <- 0.000000. @@ -293,14 +149,14 @@ Node 'ol.b' (LearnableParameter operation): Initializing Parameter[10 x 1] <- 0. Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.003906. Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.003906. Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.003906. -Node 'conv1.w.W' (LearnableParameter operation): Initializing Parameter[16 x 25] <- uniform(seed=1, range=0.050000*10.000000, onCPU=true). +Node 'conv1.w.W' (LearnableParameter operation): Initializing Parameter[16 x 25] <- uniform(seed=1, init dims=[16 x 25], range=0.050000*10.000000, onCPU=true). Node 'conv1.b.b' (LearnableParameter operation): Initializing Parameter[1 x 1 x 16] <- 1.000000. -Node 'conv2.w.W' (LearnableParameter operation): Initializing Parameter[32 x 400] <- uniform(seed=2, range=0.050000*10.000000, onCPU=true). +Node 'conv2.w.W' (LearnableParameter operation): Initializing Parameter[32 x 400] <- uniform(seed=2, init dims=[32 x 400], range=0.050000*10.000000, onCPU=true). Node 'conv2.b.b' (LearnableParameter operation): Initializing Parameter[1 x 1 x 32] <- 1.000000. -Node 'h1.W' (LearnableParameter operation): Initializing Parameter[128 x 7 x 7 x 32] <- uniform(seed=3, range=0.050000*1.000000, onCPU=true). -Node 'h1.b' (LearnableParameter operation): Initializing Parameter[128 x 1] <- uniform(seed=4, range=0.050000*1.000000, onCPU=true). -Node 'ol.W' (LearnableParameter operation): Initializing Parameter[10 x 128] <- uniform(seed=5, range=0.050000*1.000000, onCPU=true). -Node 'ol.b' (LearnableParameter operation): Initializing Parameter[10 x 1] <- uniform(seed=6, range=0.050000*1.000000, onCPU=true). +Node 'h1.W' (LearnableParameter operation): Initializing Parameter[128 x 7 x 7 x 32] <- uniform(seed=3, init dims=[128 x 1568], range=0.050000*1.000000, onCPU=true). +Node 'h1.b' (LearnableParameter operation): Initializing Parameter[128 x 1] <- uniform(seed=4, init dims=[128 x 1], range=0.050000*1.000000, onCPU=true). +Node 'ol.W' (LearnableParameter operation): Initializing Parameter[10 x 128] <- uniform(seed=5, init dims=[10 x 128], range=0.050000*1.000000, onCPU=true). +Node 'ol.b' (LearnableParameter operation): Initializing Parameter[10 x 1] <- uniform(seed=6, init dims=[10 x 1], range=0.050000*1.000000, onCPU=true). Post-processing network... @@ -344,13 +200,9 @@ Validating network. 16 nodes to process in pass 2. Validating network, final pass. - conv1.c.c: using cuDNN convolution engine for geometry: Input: 28 x 28 x 1, Output: 28 x 28 x 16, Kernel: 5 x 5 x 1, Map: 1 x 1 x 16, Stride: 1 x 1 x 1, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. - pool1: using cuDNN convolution engine for geometry: Input: 28 x 28 x 16, Output: 14 x 14 x 16, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. - conv2.c.c: using cuDNN convolution engine for geometry: Input: 14 x 14 x 16, Output: 14 x 14 x 32, Kernel: 5 x 5 x 16, Map: 32, Stride: 1 x 1 x 16, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. - pool2.p: using cuDNN convolution engine for geometry: Input: 14 x 14 x 32, Output: 7 x 7 x 32, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. @@ -358,13 +210,13 @@ pool2.p: using cuDNN convolution engine for geometry: Input: 14 x 14 x 32, Outpu Post-processing network complete. -08/16/2016 10:49:52: Created model with 27 nodes on GPU 0. +08/26/2016 11:16:50: Created model with 27 nodes on GPU 0. -08/16/2016 10:49:52: Training criterion node(s): -08/16/2016 10:49:52: ce = CrossEntropyWithSoftmax +08/26/2016 11:16:50: Training criterion node(s): +08/26/2016 11:16:50: ce = CrossEntropyWithSoftmax -08/16/2016 10:49:52: Evaluation criterion node(s): -08/16/2016 10:49:52: errs = ClassificationError +08/26/2016 11:16:50: Evaluation criterion node(s): +08/26/2016 11:16:50: errs = ClassificationError Allocating matrices for forward and/or backward propagation. @@ -402,58 +254,55 @@ Memory Sharing: Out of 49 matrices, 29 are shared as 13, and 20 are not shared. h1.y : [128 x 1 x *] (gradient) } -08/16/2016 10:49:52: Training 215370 parameters in 8 out of 8 parameter tensors and 22 nodes with gradient: +08/26/2016 11:16:50: Training 215370 parameters in 8 out of 8 parameter tensors and 22 nodes with gradient: -08/16/2016 10:49:52: Node 'conv1.b.b' (LearnableParameter operation) : [1 x 1 x 16] -08/16/2016 10:49:52: Node 'conv1.w.W' (LearnableParameter operation) : [16 x 25] -08/16/2016 10:49:52: Node 'conv2.b.b' (LearnableParameter operation) : [1 x 1 x 32] -08/16/2016 10:49:52: Node 'conv2.w.W' (LearnableParameter operation) : [32 x 400] -08/16/2016 10:49:52: Node 'h1.W' (LearnableParameter operation) : [128 x 7 x 7 x 32] -08/16/2016 10:49:52: Node 'h1.b' (LearnableParameter operation) : [128 x 1] -08/16/2016 10:49:52: Node 'ol.W' (LearnableParameter operation) : [10 x 128] -08/16/2016 10:49:52: Node 'ol.b' (LearnableParameter operation) : [10 x 1] +08/26/2016 11:16:50: Node 'conv1.b.b' (LearnableParameter operation) : [1 x 1 x 16] +08/26/2016 11:16:50: Node 'conv1.w.W' (LearnableParameter operation) : [16 x 25] +08/26/2016 11:16:50: Node 'conv2.b.b' (LearnableParameter operation) : [1 x 1 x 32] +08/26/2016 11:16:50: Node 'conv2.w.W' (LearnableParameter operation) : [32 x 400] +08/26/2016 11:16:50: Node 'h1.W' (LearnableParameter operation) : [128 x 7 x 7 x 32] +08/26/2016 11:16:50: Node 'h1.b' (LearnableParameter operation) : [128 x 1] +08/26/2016 11:16:50: Node 'ol.W' (LearnableParameter operation) : [10 x 128] +08/26/2016 11:16:50: Node 'ol.b' (LearnableParameter operation) : [10 x 1] -08/16/2016 10:49:52: No PreCompute nodes found, or all already computed. Skipping pre-computation step. +08/26/2016 11:16:50: No PreCompute nodes found, or all already computed. Skipping pre-computation step. -08/16/2016 10:49:52: Starting Epoch 1: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 0: frames [0..60000] (first sequence at sample 0), data subset 0 of 1 +08/26/2016 11:16:50: Starting Epoch 1: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/16/2016 10:49:52: Starting minibatch loop. -08/16/2016 10:49:54: Epoch[ 1 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 1.04692261 * 16000; errs = 35.156% * 16000; time = 2.0092s; samplesPerSecond = 7963.4 -08/16/2016 10:49:55: Epoch[ 1 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.17001367 * 16000; errs = 4.913% * 16000; time = 0.9653s; samplesPerSecond = 16575.7 -08/16/2016 10:49:56: Epoch[ 1 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.10910559 * 16000; errs = 3.169% * 16000; time = 0.9622s; samplesPerSecond = 16627.8 -08/16/2016 10:49:56: Finished Epoch[ 1 of 3]: [Training] ce = 0.37089482 * 60000; errs = 12.038% * 60000; totalSamplesSeen = 60000; learningRatePerSample = 0.003125; epochTime=4.67495s -08/16/2016 10:49:56: SGD: Saving checkpoint model '/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution.1' +08/26/2016 11:16:50: Starting minibatch loop. +08/26/2016 11:16:53: Epoch[ 1 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.98925934 * 16000; errs = 33.087% * 16000; time = 3.3535s; samplesPerSecond = 4771.1 +08/26/2016 11:16:55: Epoch[ 1 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.16534613 * 16000; errs = 4.787% * 16000; time = 2.3541s; samplesPerSecond = 6796.6 +08/26/2016 11:16:58: Epoch[ 1 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.10741089 * 16000; errs = 3.181% * 16000; time = 2.3508s; samplesPerSecond = 6806.3 +08/26/2016 11:17:00: Finished Epoch[ 1 of 3]: [Training] ce = 0.35380680 * 60000; errs = 11.483% * 60000; totalSamplesSeen = 60000; learningRatePerSample = 0.003125; epochTime=9.83185s +08/26/2016 11:17:00: SGD: Saving checkpoint model '/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution.1' -08/16/2016 10:49:56: Starting Epoch 2: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 1: frames [60000..120000] (first sequence at sample 60000), data subset 0 of 1 +08/26/2016 11:17:00: Starting Epoch 2: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/16/2016 10:49:56: Starting minibatch loop. -08/16/2016 10:49:57: Epoch[ 2 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.07433912 * 16000; errs = 2.369% * 16000; time = 0.9731s; samplesPerSecond = 16442.2 -08/16/2016 10:49:58: Epoch[ 2 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.06223948 * 16000; errs = 1.875% * 16000; time = 0.9738s; samplesPerSecond = 16430.7 -08/16/2016 10:49:59: Epoch[ 2 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.06269952 * 16000; errs = 1.812% * 16000; time = 0.9724s; samplesPerSecond = 16453.8 -08/16/2016 10:50:00: Finished Epoch[ 2 of 3]: [Training] ce = 0.06625302 * 60000; errs = 2.018% * 60000; totalSamplesSeen = 120000; learningRatePerSample = 0.003125; epochTime=3.6549s -08/16/2016 10:50:00: SGD: Saving checkpoint model '/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution.2' +08/26/2016 11:17:00: Starting minibatch loop. +08/26/2016 11:17:02: Epoch[ 2 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.07252145 * 16000; errs = 2.219% * 16000; time = 2.3573s; samplesPerSecond = 6787.4 +08/26/2016 11:17:04: Epoch[ 2 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.06217503 * 16000; errs = 1.912% * 16000; time = 2.3627s; samplesPerSecond = 6772.0 +08/26/2016 11:17:07: Epoch[ 2 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.06354430 * 16000; errs = 1.844% * 16000; time = 2.3633s; samplesPerSecond = 6770.1 +08/26/2016 11:17:08: Finished Epoch[ 2 of 3]: [Training] ce = 0.06565070 * 60000; errs = 1.978% * 60000; totalSamplesSeen = 120000; learningRatePerSample = 0.003125; epochTime=8.85624s +08/26/2016 11:17:08: SGD: Saving checkpoint model '/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution.2' -08/16/2016 10:50:00: Starting Epoch 3: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 2: frames [120000..180000] (first sequence at sample 120000), data subset 0 of 1 +08/26/2016 11:17:08: Starting Epoch 3: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/16/2016 10:50:00: Starting minibatch loop. -08/16/2016 10:50:01: Epoch[ 3 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.04532548 * 16000; errs = 1.319% * 16000; time = 0.9745s; samplesPerSecond = 16419.1 -08/16/2016 10:50:02: Epoch[ 3 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.04296139 * 16000; errs = 1.256% * 16000; time = 0.9719s; samplesPerSecond = 16463.3 -08/16/2016 10:50:03: Epoch[ 3 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.04916875 * 16000; errs = 1.456% * 16000; time = 0.9710s; samplesPerSecond = 16477.7 -08/16/2016 10:50:04: Finished Epoch[ 3 of 3]: [Training] ce = 0.04531107 * 60000; errs = 1.337% * 60000; totalSamplesSeen = 180000; learningRatePerSample = 0.003125; epochTime=3.65691s -08/16/2016 10:50:04: SGD: Saving checkpoint model '/tmp/cntk-test-20160816095502.258817/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution' -08/16/2016 10:50:04: CNTKCommandTrainEnd: train +08/26/2016 11:17:08: Starting minibatch loop. +08/26/2016 11:17:11: Epoch[ 3 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.04610618 * 16000; errs = 1.400% * 16000; time = 2.3607s; samplesPerSecond = 6777.7 +08/26/2016 11:17:13: Epoch[ 3 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.04150417 * 16000; errs = 1.200% * 16000; time = 2.3537s; samplesPerSecond = 6797.7 +08/26/2016 11:17:16: Epoch[ 3 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.04880609 * 16000; errs = 1.387% * 16000; time = 2.3638s; samplesPerSecond = 6768.8 +08/26/2016 11:17:17: Finished Epoch[ 3 of 3]: [Training] ce = 0.04505229 * 60000; errs = 1.328% * 60000; totalSamplesSeen = 180000; learningRatePerSample = 0.003125; epochTime=8.85535s +08/26/2016 11:17:17: SGD: Saving checkpoint model '/tmp/cntk-test-20160826102023.187877/Examples/Image/MNIST_02_Convolution@release_gpu/Models/02_Convolution' +08/26/2016 11:17:17: CNTKCommandTrainEnd: train -08/16/2016 10:50:04: Action "train" complete. +08/26/2016 11:17:17: Action "train" complete. -08/16/2016 10:50:04: ############################################################################## -08/16/2016 10:50:04: # # -08/16/2016 10:50:04: # Action "test" # -08/16/2016 10:50:04: # # -08/16/2016 10:50:04: ############################################################################## +08/26/2016 11:17:17: ############################################################################## +08/26/2016 11:17:17: # # +08/26/2016 11:17:17: # Action "test" # +08/26/2016 11:17:17: # # +08/26/2016 11:17:17: ############################################################################## Post-processing network... @@ -498,13 +347,9 @@ Validating network. 16 nodes to process in pass 2. Validating network, final pass. - conv1.c.c: using cuDNN convolution engine for geometry: Input: 28 x 28 x 1, Output: 28 x 28 x 16, Kernel: 5 x 5 x 1, Map: 1 x 1 x 16, Stride: 1 x 1 x 1, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. - pool1: using cuDNN convolution engine for geometry: Input: 28 x 28 x 16, Output: 14 x 14 x 16, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. - conv2.c.c: using cuDNN convolution engine for geometry: Input: 14 x 14 x 16, Output: 14 x 14 x 32, Kernel: 5 x 5 x 16, Map: 32, Stride: 1 x 1 x 16, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. - pool2.p: using cuDNN convolution engine for geometry: Input: 14 x 14 x 32, Output: 7 x 7 x 32, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. @@ -520,10 +365,9 @@ Allocating matrices for forward and/or backward propagation. Memory Sharing: Out of 27 matrices, 0 are shared as 0, and 27 are not shared. -BlockRandomizer::StartEpoch: epoch 0: frames [0..10000] (first sequence at sample 0), data subset 0 of 1 -08/16/2016 10:50:04: Minibatch[1-10]: errs = 1.580% * 10000; ce = 0.05054137 * 10000 -08/16/2016 10:50:04: Final Results: Minibatch[1-10]: errs = 1.580% * 10000; ce = 0.05054137 * 10000; perplexity = 1.05184038 +08/26/2016 11:17:18: Minibatch[1-10]: errs = 1.400% * 10000; ce = 0.04776463 * 10000 +08/26/2016 11:17:18: Final Results: Minibatch[1-10]: errs = 1.400% * 10000; ce = 0.04776463 * 10000; perplexity = 1.04892374 -08/16/2016 10:50:04: Action "test" complete. +08/26/2016 11:17:18: Action "test" complete. -08/16/2016 10:50:04: __COMPLETED__ \ No newline at end of file +08/26/2016 11:17:18: __COMPLETED__ diff --git a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.windows.txt b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.windows.txt index 9516431e5e5f..3b6f6613721a 100644 --- a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.windows.txt +++ b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/baseline.windows.txt @@ -3,12 +3,12 @@ CPU info: Hardware threads: 24 Total Memory: 268381192 kB ------------------------------------------------------------------- -=== Running /cygdrive/c/jenkins/workspace/CNTK-Test-Windows-W1/x64/release/cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config/02_Convolution.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" +=== Running /cygdrive/c/jenkins/workspace/CNTK-Test-Windows-W1/x64/release/cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config/02_Convolution.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true forceDeterministicAlgorithms=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" ------------------------------------------------------------------- Build info: - Built time: Aug 16 2016 02:54:53 - Last modified date: Fri Aug 12 05:31:21 2016 + Built time: Aug 26 2016 02:30:43 + Last modified date: Fri Aug 26 02:16:23 2016 Build type: Release Build target: GPU With 1bit-SGD: no @@ -17,207 +17,64 @@ Build info: CUB_PATH: c:\src\cub-1.4.1 CUDNN_PATH: c:\NVIDIA\cudnn-4.0\cuda Build Branch: HEAD - Build SHA1: 026b1e772b963461e189f8f00aa7ed6951298f84 - Built by svcphil on Philly-Pool3 - Build Path: c:\Jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ + Build SHA1: a718f2f213d1a045d5bbfa88f90c18fe15bd22f2 + Built by svcphil on PHILLY-POOL2 + Build Path: c:\jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ ------------------------------------------------------------------- -Changed current directory to C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData -08/16/2016 03:01:04: ------------------------------------------------------------------- -08/16/2016 03:01:04: Build info: - -08/16/2016 03:01:04: Built time: Aug 16 2016 02:54:53 -08/16/2016 03:01:04: Last modified date: Fri Aug 12 05:31:21 2016 -08/16/2016 03:01:04: Build type: Release -08/16/2016 03:01:04: Build target: GPU -08/16/2016 03:01:04: With 1bit-SGD: no -08/16/2016 03:01:04: Math lib: mkl -08/16/2016 03:01:04: CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5 -08/16/2016 03:01:04: CUB_PATH: c:\src\cub-1.4.1 -08/16/2016 03:01:04: CUDNN_PATH: c:\NVIDIA\cudnn-4.0\cuda -08/16/2016 03:01:04: Build Branch: HEAD -08/16/2016 03:01:04: Build SHA1: 026b1e772b963461e189f8f00aa7ed6951298f84 -08/16/2016 03:01:04: Built by svcphil on Philly-Pool3 -08/16/2016 03:01:04: Build Path: c:\Jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ -08/16/2016 03:01:04: ------------------------------------------------------------------- -08/16/2016 03:01:07: ------------------------------------------------------------------- -08/16/2016 03:01:07: GPU info: - -08/16/2016 03:01:07: Device[0]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/16/2016 03:01:07: Device[1]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/16/2016 03:01:07: Device[2]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/16/2016 03:01:07: Device[3]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/16/2016 03:01:07: ------------------------------------------------------------------- - -08/16/2016 03:01:07: Running on DPHAIM-24 at 2016/08/16 03:01:07 -08/16/2016 03:01:07: Command line: -C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\release\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config/02_Convolution.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" - - - -08/16/2016 03:01:07: >>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>> -08/16/2016 03:01:07: rootDir = ".." -configDir = "$rootDir$/Config" -dataDir = "$rootDir$/Data" -outputDir = "$rootDir$/Output" -modelDir = "$outputDir$/Models" -deviceId = 0 -command = train:test -precision = "float" -modelPath = "$modelDir$/02_Convolution" -traceLevel = 1 -numMBsToShowResult = 500 -train = [ - action = "train" - NDLNetworkBuilder = [ - imageLayout = "cudnn" - initOnCPUOnly = true - ndlMacros = "$configDir$/Macros.ndl" - networkDescription = "$ConfigDir$/02_Convolution.ndl" - ] - SGD = [ - epochSize = 60000 - minibatchSize = 32 - learningRatesPerMB = 0.1*5:0.3 - momentumPerMB = 0*10:0.7 - maxEpochs = 15 - ] - reader = [ - readerType = "CNTKTextFormatReader" - file = "$DataDir$/Train-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -test = [ - action = test - minibatchSize = 1024 - reader = [ - readerType = "CNTKTextFormatReader" - file = "$DataDir$/Test-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData -RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu -DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData -ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config -OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu -DeviceId=0 -timestamping=true -train=[SGD=[maxEpochs=3]] -imageLayout="cudnn" - -08/16/2016 03:01:07: <<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<< - -08/16/2016 03:01:07: >>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> -08/16/2016 03:01:07: rootDir = ".." -configDir = "../Config" -dataDir = "../Data" -outputDir = "../Output" -modelDir = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models" -deviceId = 0 -command = train:test -precision = "float" -modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution" -traceLevel = 1 -numMBsToShowResult = 500 -train = [ - action = "train" - NDLNetworkBuilder = [ - imageLayout = "cudnn" - initOnCPUOnly = true - ndlMacros = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config/Macros.ndl" - networkDescription = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config/02_Convolution.ndl" - ] - SGD = [ - epochSize = 60000 - minibatchSize = 32 - learningRatesPerMB = 0.1*5:0.3 - momentumPerMB = 0*10:0.7 - maxEpochs = 15 - ] - reader = [ - readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData/Train-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -test = [ - action = test - minibatchSize = 1024 - reader = [ - readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData/Test-28x28_cntk_text.txt" - input = [ - features = [ - dim = 784 - format = "dense" - ] - labels = [ - dim = 10 - format = "dense" - ] - ] - ] -] -currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData -RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu -DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData -ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config -OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu -DeviceId=0 -timestamping=true -train=[SGD=[maxEpochs=3]] -imageLayout="cudnn" - -08/16/2016 03:01:07: <<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< - -08/16/2016 03:01:07: >>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> +Changed current directory to C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData +08/26/2016 03:41:56: ------------------------------------------------------------------- +08/26/2016 03:41:56: Build info: + +08/26/2016 03:41:56: Built time: Aug 26 2016 02:30:43 +08/26/2016 03:41:56: Last modified date: Fri Aug 26 02:16:23 2016 +08/26/2016 03:41:56: Build type: Release +08/26/2016 03:41:56: Build target: GPU +08/26/2016 03:41:56: With 1bit-SGD: no +08/26/2016 03:41:56: Math lib: mkl +08/26/2016 03:41:56: CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5 +08/26/2016 03:41:56: CUB_PATH: c:\src\cub-1.4.1 +08/26/2016 03:41:56: CUDNN_PATH: c:\NVIDIA\cudnn-4.0\cuda +08/26/2016 03:41:56: Build Branch: HEAD +08/26/2016 03:41:56: Build SHA1: a718f2f213d1a045d5bbfa88f90c18fe15bd22f2 +08/26/2016 03:41:56: Built by svcphil on PHILLY-POOL2 +08/26/2016 03:41:56: Build Path: c:\jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ +08/26/2016 03:41:56: ------------------------------------------------------------------- +08/26/2016 03:41:58: ------------------------------------------------------------------- +08/26/2016 03:41:58: GPU info: + +08/26/2016 03:41:58: Device[0]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB +08/26/2016 03:41:58: Device[1]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB +08/26/2016 03:41:58: Device[2]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB +08/26/2016 03:41:58: Device[3]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB +08/26/2016 03:41:58: ------------------------------------------------------------------- + +08/26/2016 03:41:58: Running on DPHAIM-24 at 2016/08/26 03:41:58 +08/26/2016 03:41:58: Command line: +C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\release\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config/02_Convolution.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu DeviceId=0 timestamping=true forceDeterministicAlgorithms=true train=[SGD=[maxEpochs=3]] imageLayout="cudnn" + + +Configuration After Processing and Variable Resolution: + configparameters: 02_Convolution.cntk:command=train:test configparameters: 02_Convolution.cntk:configDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Examples\Image\MNIST\Config -configparameters: 02_Convolution.cntk:currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData -configparameters: 02_Convolution.cntk:dataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData +configparameters: 02_Convolution.cntk:currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData +configparameters: 02_Convolution.cntk:dataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData configparameters: 02_Convolution.cntk:deviceId=0 +configparameters: 02_Convolution.cntk:forceDeterministicAlgorithms=true configparameters: 02_Convolution.cntk:imageLayout=cudnn -configparameters: 02_Convolution.cntk:modelDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models -configparameters: 02_Convolution.cntk:modelPath=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution +configparameters: 02_Convolution.cntk:modelDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu/Models +configparameters: 02_Convolution.cntk:modelPath=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution configparameters: 02_Convolution.cntk:numMBsToShowResult=500 -configparameters: 02_Convolution.cntk:outputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu +configparameters: 02_Convolution.cntk:outputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu configparameters: 02_Convolution.cntk:precision=float configparameters: 02_Convolution.cntk:rootDir=.. -configparameters: 02_Convolution.cntk:RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu +configparameters: 02_Convolution.cntk:RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu configparameters: 02_Convolution.cntk:test=[ action = test minibatchSize = 1024 reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData/Test-28x28_cntk_text.txt" + file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData/Test-28x28_cntk_text.txt" input = [ features = [ dim = 784 @@ -250,7 +107,7 @@ configparameters: 02_Convolution.cntk:train=[ ] reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu\TestData/Train-28x28_cntk_text.txt" + file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu\TestData/Train-28x28_cntk_text.txt" input = [ features = [ dim = 784 @@ -264,23 +121,22 @@ configparameters: 02_Convolution.cntk:train=[ ] ] [SGD=[maxEpochs=3]] -08/16/2016 03:01:07: <<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< -08/16/2016 03:01:07: Commands: train test -08/16/2016 03:01:07: Precision = "float" -08/16/2016 03:01:07: CNTKModelPath: C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution -08/16/2016 03:01:07: CNTKCommandTrainInfo: train : 3 -08/16/2016 03:01:07: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 3 +08/26/2016 03:41:58: Commands: train test +08/26/2016 03:41:58: Precision = "float" +08/26/2016 03:41:58: CNTKModelPath: C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution +08/26/2016 03:41:58: CNTKCommandTrainInfo: train : 3 +08/26/2016 03:41:58: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 3 -08/16/2016 03:01:07: ############################################################################## -08/16/2016 03:01:07: # # -08/16/2016 03:01:07: # Action "train" # -08/16/2016 03:01:07: # # -08/16/2016 03:01:07: ############################################################################## +08/26/2016 03:41:58: ############################################################################## +08/26/2016 03:41:58: # # +08/26/2016 03:41:58: # Action "train" # +08/26/2016 03:41:58: # # +08/26/2016 03:41:58: ############################################################################## -08/16/2016 03:01:07: CNTKCommandTrainBegin: train -NDLBuilder Using GPU 0 +08/26/2016 03:41:58: CNTKCommandTrainBegin: train -08/16/2016 03:01:07: Creating virgin network. +08/26/2016 03:41:58: Creating virgin network. +NDLBuilder Using GPU 0 Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.000000. Node 'conv1.w.W' (LearnableParameter operation): Initializing Parameter[16 x 25] <- 0.000000. Node 'conv1.b.b' (LearnableParameter operation): Initializing Parameter[1 x 1 x 16] <- 0.000000. @@ -293,14 +149,14 @@ Node 'ol.b' (LearnableParameter operation): Initializing Parameter[10 x 1] <- 0. Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.003906. Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.003906. Node 'featScale' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.003906. -Node 'conv1.w.W' (LearnableParameter operation): Initializing Parameter[16 x 25] <- uniform(seed=1, range=0.050000*10.000000, onCPU=true). +Node 'conv1.w.W' (LearnableParameter operation): Initializing Parameter[16 x 25] <- uniform(seed=1, init dims=[16 x 25], range=0.050000*10.000000, onCPU=true). Node 'conv1.b.b' (LearnableParameter operation): Initializing Parameter[1 x 1 x 16] <- 1.000000. -Node 'conv2.w.W' (LearnableParameter operation): Initializing Parameter[32 x 400] <- uniform(seed=2, range=0.050000*10.000000, onCPU=true). +Node 'conv2.w.W' (LearnableParameter operation): Initializing Parameter[32 x 400] <- uniform(seed=2, init dims=[32 x 400], range=0.050000*10.000000, onCPU=true). Node 'conv2.b.b' (LearnableParameter operation): Initializing Parameter[1 x 1 x 32] <- 1.000000. -Node 'h1.W' (LearnableParameter operation): Initializing Parameter[128 x 7 x 7 x 32] <- uniform(seed=3, range=0.050000*1.000000, onCPU=true). -Node 'h1.b' (LearnableParameter operation): Initializing Parameter[128 x 1] <- uniform(seed=4, range=0.050000*1.000000, onCPU=true). -Node 'ol.W' (LearnableParameter operation): Initializing Parameter[10 x 128] <- uniform(seed=5, range=0.050000*1.000000, onCPU=true). -Node 'ol.b' (LearnableParameter operation): Initializing Parameter[10 x 1] <- uniform(seed=6, range=0.050000*1.000000, onCPU=true). +Node 'h1.W' (LearnableParameter operation): Initializing Parameter[128 x 7 x 7 x 32] <- uniform(seed=3, init dims=[128 x 1568], range=0.050000*1.000000, onCPU=true). +Node 'h1.b' (LearnableParameter operation): Initializing Parameter[128 x 1] <- uniform(seed=4, init dims=[128 x 1], range=0.050000*1.000000, onCPU=true). +Node 'ol.W' (LearnableParameter operation): Initializing Parameter[10 x 128] <- uniform(seed=5, init dims=[10 x 128], range=0.050000*1.000000, onCPU=true). +Node 'ol.b' (LearnableParameter operation): Initializing Parameter[10 x 1] <- uniform(seed=6, init dims=[10 x 1], range=0.050000*1.000000, onCPU=true). Post-processing network... @@ -344,13 +200,9 @@ Validating network. 16 nodes to process in pass 2. Validating network, final pass. - conv1.c.c: using cuDNN convolution engine for geometry: Input: 28 x 28 x 1, Output: 28 x 28 x 16, Kernel: 5 x 5 x 1, Map: 1 x 1 x 16, Stride: 1 x 1 x 1, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. - pool1: using cuDNN convolution engine for geometry: Input: 28 x 28 x 16, Output: 14 x 14 x 16, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. - conv2.c.c: using cuDNN convolution engine for geometry: Input: 14 x 14 x 16, Output: 14 x 14 x 32, Kernel: 5 x 5 x 16, Map: 32, Stride: 1 x 1 x 16, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. - pool2.p: using cuDNN convolution engine for geometry: Input: 14 x 14 x 32, Output: 7 x 7 x 32, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. @@ -358,102 +210,99 @@ pool2.p: using cuDNN convolution engine for geometry: Input: 14 x 14 x 32, Outpu Post-processing network complete. -08/16/2016 03:01:08: Created model with 27 nodes on GPU 0. +08/26/2016 03:41:59: Created model with 27 nodes on GPU 0. -08/16/2016 03:01:08: Training criterion node(s): -08/16/2016 03:01:08: ce = CrossEntropyWithSoftmax +08/26/2016 03:41:59: Training criterion node(s): +08/26/2016 03:41:59: ce = CrossEntropyWithSoftmax -08/16/2016 03:01:08: Evaluation criterion node(s): -08/16/2016 03:01:08: errs = ClassificationError +08/26/2016 03:41:59: Evaluation criterion node(s): +08/26/2016 03:41:59: errs = ClassificationError Allocating matrices for forward and/or backward propagation. Memory Sharing: Out of 49 matrices, 29 are shared as 13, and 20 are not shared. - { conv2.c.c : [14 x 14 x 32 x *] (gradient) - conv2.out : [14 x 14 x 32 x *] } + { conv1.cpb : [28 x 28 x 16 x *] (gradient) + pool1 : [14 x 14 x 16 x *] } { h1.W : [128 x 7 x 7 x 32] (gradient) h1.z : [128 x 1 x *] } { h1.t : [128 x *] (gradient) h1.y : [128 x 1 x *] } + { conv2.b.b : [1 x 1 x 32] (gradient) + conv2.out : [14 x 14 x 32 x *] (gradient) + h1.t : [128 x *] } + { conv1.c.c : [28 x 28 x 16 x *] (gradient) + conv1.out : [28 x 28 x 16 x *] } { h1.z : [128 x 1 x *] (gradient) ol.t : [10 x 1 x *] pool2.p : [7 x 7 x 32 x *] (gradient) } - { ol.W : [10 x 128] (gradient) - ol.z : [10 x 1 x *] (gradient) } { conv1.b.b : [1 x 1 x 16] (gradient) conv1.out : [28 x 28 x 16 x *] (gradient) } - { conv2.b.b : [1 x 1 x 32] (gradient) - conv2.out : [14 x 14 x 32 x *] (gradient) - h1.t : [128 x *] } + { conv1.cpb : [28 x 28 x 16 x *] + conv1.w.W : [16 x 25] (gradient) } { conv2.cpb : [14 x 14 x 32 x *] conv2.w.W : [32 x 400] (gradient) } + { conv2.c.c : [14 x 14 x 32 x *] (gradient) + conv2.out : [14 x 14 x 32 x *] } { conv2.cpb : [14 x 14 x 32 x *] (gradient) pool1 : [14 x 14 x 16 x *] (gradient) pool2.p : [7 x 7 x 32 x *] } - { conv1.c.c : [28 x 28 x 16 x *] (gradient) - conv1.out : [28 x 28 x 16 x *] } { h1.b : [128 x 1] (gradient) h1.y : [128 x 1 x *] (gradient) } - { conv1.cpb : [28 x 28 x 16 x *] - conv1.w.W : [16 x 25] (gradient) } - { conv1.cpb : [28 x 28 x 16 x *] (gradient) - pool1 : [14 x 14 x 16 x *] } + { ol.W : [10 x 128] (gradient) + ol.z : [10 x 1 x *] (gradient) } -08/16/2016 03:01:08: Training 215370 parameters in 8 out of 8 parameter tensors and 22 nodes with gradient: +08/26/2016 03:42:00: Training 215370 parameters in 8 out of 8 parameter tensors and 22 nodes with gradient: -08/16/2016 03:01:08: Node 'conv1.b.b' (LearnableParameter operation) : [1 x 1 x 16] -08/16/2016 03:01:08: Node 'conv1.w.W' (LearnableParameter operation) : [16 x 25] -08/16/2016 03:01:08: Node 'conv2.b.b' (LearnableParameter operation) : [1 x 1 x 32] -08/16/2016 03:01:08: Node 'conv2.w.W' (LearnableParameter operation) : [32 x 400] -08/16/2016 03:01:08: Node 'h1.W' (LearnableParameter operation) : [128 x 7 x 7 x 32] -08/16/2016 03:01:08: Node 'h1.b' (LearnableParameter operation) : [128 x 1] -08/16/2016 03:01:08: Node 'ol.W' (LearnableParameter operation) : [10 x 128] -08/16/2016 03:01:08: Node 'ol.b' (LearnableParameter operation) : [10 x 1] +08/26/2016 03:42:00: Node 'conv1.b.b' (LearnableParameter operation) : [1 x 1 x 16] +08/26/2016 03:42:00: Node 'conv1.w.W' (LearnableParameter operation) : [16 x 25] +08/26/2016 03:42:00: Node 'conv2.b.b' (LearnableParameter operation) : [1 x 1 x 32] +08/26/2016 03:42:00: Node 'conv2.w.W' (LearnableParameter operation) : [32 x 400] +08/26/2016 03:42:00: Node 'h1.W' (LearnableParameter operation) : [128 x 7 x 7 x 32] +08/26/2016 03:42:00: Node 'h1.b' (LearnableParameter operation) : [128 x 1] +08/26/2016 03:42:00: Node 'ol.W' (LearnableParameter operation) : [10 x 128] +08/26/2016 03:42:00: Node 'ol.b' (LearnableParameter operation) : [10 x 1] -08/16/2016 03:01:08: No PreCompute nodes found, or all already computed. Skipping pre-computation step. +08/26/2016 03:42:00: No PreCompute nodes found, or all already computed. Skipping pre-computation step. -08/16/2016 03:01:08: Starting Epoch 1: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 0: frames [0..60000] (first sequence at sample 0), data subset 0 of 1 +08/26/2016 03:42:00: Starting Epoch 1: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/16/2016 03:01:08: Starting minibatch loop. -08/16/2016 03:01:12: Epoch[ 1 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 1.27430457 * 16000; errs = 44.075% * 16000; time = 3.3942s; samplesPerSecond = 4714.0 -08/16/2016 03:01:13: Epoch[ 1 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.19224133 * 16000; errs = 5.400% * 16000; time = 1.7150s; samplesPerSecond = 9329.3 -08/16/2016 03:01:15: Epoch[ 1 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.11038135 * 16000; errs = 3.231% * 16000; time = 1.7153s; samplesPerSecond = 9327.6 -08/16/2016 03:01:16: Finished Epoch[ 1 of 3]: [Training] ce = 0.43859844 * 60000; errs = 14.615% * 60000; totalSamplesSeen = 60000; learningRatePerSample = 0.003125; epochTime=8.13585s -08/16/2016 03:01:16: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution.1' +08/26/2016 03:42:00: Starting minibatch loop. +08/26/2016 03:42:04: Epoch[ 1 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 1.29986121 * 16000; errs = 44.825% * 16000; time = 4.5059s; samplesPerSecond = 3550.9 +08/26/2016 03:42:07: Epoch[ 1 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.19019495 * 16000; errs = 5.319% * 16000; time = 2.8366s; samplesPerSecond = 5640.5 +08/26/2016 03:42:10: Epoch[ 1 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.11099622 * 16000; errs = 3.281% * 16000; time = 2.8383s; samplesPerSecond = 5637.3 +08/26/2016 03:42:12: Finished Epoch[ 1 of 3]: [Training] ce = 0.44494518 * 60000; errs = 14.805% * 60000; totalSamplesSeen = 60000; learningRatePerSample = 0.003125; epochTime=12.3306s +08/26/2016 03:42:12: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution.1' -08/16/2016 03:01:16: Starting Epoch 2: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 1: frames [60000..120000] (first sequence at sample 60000), data subset 0 of 1 +08/26/2016 03:42:12: Starting Epoch 2: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/16/2016 03:01:16: Starting minibatch loop. -08/16/2016 03:01:18: Epoch[ 2 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.07473590 * 16000; errs = 2.250% * 16000; time = 1.7130s; samplesPerSecond = 9340.4 -08/16/2016 03:01:20: Epoch[ 2 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.06083897 * 16000; errs = 1.825% * 16000; time = 1.7090s; samplesPerSecond = 9362.3 -08/16/2016 03:01:21: Epoch[ 2 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.06430908 * 16000; errs = 1.881% * 16000; time = 1.7094s; samplesPerSecond = 9359.8 -08/16/2016 03:01:23: Finished Epoch[ 2 of 3]: [Training] ce = 0.06608532 * 60000; errs = 1.973% * 60000; totalSamplesSeen = 120000; learningRatePerSample = 0.003125; epochTime=6.42906s -08/16/2016 03:01:23: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution.2' +08/26/2016 03:42:12: Starting minibatch loop. +08/26/2016 03:42:15: Epoch[ 2 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.07434936 * 16000; errs = 2.288% * 16000; time = 2.8353s; samplesPerSecond = 5643.1 +08/26/2016 03:42:18: Epoch[ 2 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.06040998 * 16000; errs = 1.825% * 16000; time = 2.8357s; samplesPerSecond = 5642.3 +08/26/2016 03:42:20: Epoch[ 2 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.06375415 * 16000; errs = 1.975% * 16000; time = 2.8379s; samplesPerSecond = 5637.9 +08/26/2016 03:42:23: Finished Epoch[ 2 of 3]: [Training] ce = 0.06537900 * 60000; errs = 1.988% * 60000; totalSamplesSeen = 120000; learningRatePerSample = 0.003125; epochTime=10.6486s +08/26/2016 03:42:23: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution.2' -08/16/2016 03:01:23: Starting Epoch 3: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 2: frames [120000..180000] (first sequence at sample 120000), data subset 0 of 1 +08/26/2016 03:42:23: Starting Epoch 3: learning rate per sample = 0.003125 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/16/2016 03:01:23: Starting minibatch loop. -08/16/2016 03:01:25: Epoch[ 3 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.04609646 * 16000; errs = 1.450% * 16000; time = 1.7107s; samplesPerSecond = 9352.7 -08/16/2016 03:01:26: Epoch[ 3 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.04193843 * 16000; errs = 1.256% * 16000; time = 1.7151s; samplesPerSecond = 9328.9 -08/16/2016 03:01:28: Epoch[ 3 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.04465855 * 16000; errs = 1.300% * 16000; time = 1.6923s; samplesPerSecond = 9454.4 -08/16/2016 03:01:29: Finished Epoch[ 3 of 3]: [Training] ce = 0.04399961 * 60000; errs = 1.310% * 60000; totalSamplesSeen = 180000; learningRatePerSample = 0.003125; epochTime=6.40462s -08/16/2016 03:01:29: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160816030038.674053\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution' -08/16/2016 03:01:29: CNTKCommandTrainEnd: train +08/26/2016 03:42:23: Starting minibatch loop. +08/26/2016 03:42:25: Epoch[ 3 of 3]-Minibatch[ 1- 500, 26.67%]: ce = 0.04688654 * 16000; errs = 1.475% * 16000; time = 2.8352s; samplesPerSecond = 5643.3 +08/26/2016 03:42:28: Epoch[ 3 of 3]-Minibatch[ 501-1000, 53.33%]: ce = 0.04112575 * 16000; errs = 1.250% * 16000; time = 2.8346s; samplesPerSecond = 5644.6 +08/26/2016 03:42:31: Epoch[ 3 of 3]-Minibatch[1001-1500, 80.00%]: ce = 0.04371809 * 16000; errs = 1.281% * 16000; time = 2.8338s; samplesPerSecond = 5646.1 +08/26/2016 03:42:33: Finished Epoch[ 3 of 3]: [Training] ce = 0.04367646 * 60000; errs = 1.347% * 60000; totalSamplesSeen = 180000; learningRatePerSample = 0.003125; epochTime=10.6409s +08/26/2016 03:42:33: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160826034130.867909\Examples\Image\MNIST_02_Convolution@release_gpu/Models/02_Convolution' +08/26/2016 03:42:33: CNTKCommandTrainEnd: train -08/16/2016 03:01:29: Action "train" complete. +08/26/2016 03:42:33: Action "train" complete. -08/16/2016 03:01:29: ############################################################################## -08/16/2016 03:01:29: # # -08/16/2016 03:01:29: # Action "test" # -08/16/2016 03:01:29: # # -08/16/2016 03:01:29: ############################################################################## +08/26/2016 03:42:33: ############################################################################## +08/26/2016 03:42:33: # # +08/26/2016 03:42:33: # Action "test" # +08/26/2016 03:42:33: # # +08/26/2016 03:42:33: ############################################################################## Post-processing network... @@ -498,13 +347,9 @@ Validating network. 16 nodes to process in pass 2. Validating network, final pass. - conv1.c.c: using cuDNN convolution engine for geometry: Input: 28 x 28 x 1, Output: 28 x 28 x 16, Kernel: 5 x 5 x 1, Map: 1 x 1 x 16, Stride: 1 x 1 x 1, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. - pool1: using cuDNN convolution engine for geometry: Input: 28 x 28 x 16, Output: 14 x 14 x 16, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. - conv2.c.c: using cuDNN convolution engine for geometry: Input: 14 x 14 x 16, Output: 14 x 14 x 32, Kernel: 5 x 5 x 16, Map: 32, Stride: 1 x 1 x 16, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. - pool2.p: using cuDNN convolution engine for geometry: Input: 14 x 14 x 32, Output: 7 x 7 x 32, Kernel: 2 x 2 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1, 1, 1), AutoPad: (1, 1, 0), LowerPad: 0 x 0 x 0, UpperPad: 0 x 0 x 0. @@ -520,10 +365,9 @@ Allocating matrices for forward and/or backward propagation. Memory Sharing: Out of 27 matrices, 0 are shared as 0, and 27 are not shared. -BlockRandomizer::StartEpoch: epoch 0: frames [0..10000] (first sequence at sample 0), data subset 0 of 1 -08/16/2016 03:01:30: Minibatch[1-10]: errs = 1.380% * 10000; ce = 0.04422099 * 10000 -08/16/2016 03:01:30: Final Results: Minibatch[1-10]: errs = 1.380% * 10000; ce = 0.04422099 * 10000; perplexity = 1.04521331 +08/26/2016 03:42:34: Minibatch[1-10]: errs = 1.320% * 10000; ce = 0.04335406 * 10000 +08/26/2016 03:42:34: Final Results: Minibatch[1-10]: errs = 1.320% * 10000; ce = 0.04335406 * 10000; perplexity = 1.04430758 -08/16/2016 03:01:30: Action "test" complete. +08/26/2016 03:42:34: Action "test" complete. -08/16/2016 03:01:30: __COMPLETED__ \ No newline at end of file +08/26/2016 03:42:34: __COMPLETED__ diff --git a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/run-test b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/run-test index 1da902cb1d71..41f72d6db329 100755 --- a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/run-test +++ b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/run-test @@ -2,7 +2,8 @@ . $TEST_DIR/../run-test-common -cntkrun 02_Convolution_ndl_deprecated.cntk "train=[SGD=[maxEpochs=3]] imageLayout=\"$imageLayout\"" +cntkrun 02_Convolution_ndl_deprecated.cntk "forceDeterministicAlgorithms=true train=[SGD=[maxEpochs=3]] imageLayout=\"$imageLayout\"" + ExitCode=$? # Delete the test data if copied diff --git a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/testcases.yml b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/testcases.yml index 736ec8e3fee2..b2959ee861dc 100644 --- a/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/testcases.yml +++ b/Tests/EndToEndTests/Examples/Image/MNIST/02_Convolution/testcases.yml @@ -1,9 +1,7 @@ dataDir: ../../../../../../Examples/Image/MNIST/Data tags: - # In BVT, run Release GPU - bvt-e (build_sku=='gpu') and (device=='gpu') and (flavor=='release') - # In Nightly on Linux, run Debug GPU in addition. - nightly-e (build_sku=='gpu') and (device=='gpu') testCases: @@ -21,8 +19,8 @@ testCases: # Epochs (with low train loss) must be finished with expected results: # patterns: # - Finished Epoch[{{integer}} of {{integer}}] -# - ce = {{float,tolerance=0.05}} * {{integer}} -# - errs = {{float,tolerance=0.5}}% * {{integer}} +# - ce = {{float,tolerance=0}} * {{integer}} +# - errs = {{float,tolerance=0}}% * {{integer}} # - totalSamplesSeen = {{integer}} # - learningRatePerSample = {{float,tolerance=0.1%}}