From fc78c6ba20db461816cb1ac8455e1bbe39e4e13d Mon Sep 17 00:00:00 2001 From: Nguyen Quy Hy Date: Thu, 26 Mar 2015 16:23:52 +0800 Subject: [PATCH 1/7] Allow ElementTimesNode to RepMat column vector. Add SumColumnElementsNode. --- MachineLearning/CNTK/LinearAlgebraNodes.h | 175 +++++++++++++++++- .../CNTK/NetworkDescriptionLanguage.cpp | 2 + 2 files changed, 173 insertions(+), 4 deletions(-) diff --git a/MachineLearning/CNTK/LinearAlgebraNodes.h b/MachineLearning/CNTK/LinearAlgebraNodes.h index 0f20e50afde4..9dbc4fa455d4 100644 --- a/MachineLearning/CNTK/LinearAlgebraNodes.h +++ b/MachineLearning/CNTK/LinearAlgebraNodes.h @@ -241,6 +241,136 @@ namespace Microsoft { namespace MSR { namespace CNTK { template class SumElementsNode; template class SumElementsNode; + template + class SumColumnElementsNode : public ComputationNode + { + UsingComputationNodeMembers; + public: + SumColumnElementsNode(const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"") + : ComputationNode(deviceId), m_sumValue(deviceId) + { + m_nodeName = (name == L"" ? CreateUniqNodeName() : name); + m_deviceId = deviceId; + MoveMatricesToDevice(deviceId); + InitRecurrentNode(); + } + + SumColumnElementsNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"") + : ComputationNode(deviceId), m_sumValue(deviceId) + { + m_nodeName = (name == L"" ? CreateUniqNodeName() : name); + LoadFromFile(fstream, modelVersion, deviceId); + } + + // copy constructor + SumColumnElementsNode(const SumColumnElementsNode* node, const std::wstring& newName, const CopyNodeFlags flags) + : ComputationNode(node->m_deviceId), m_sumValue(node->m_deviceId) + { + node->CopyTo(this, newName, flags); + } + + virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const + { + const std::wstring& name = (newName == L"") ? NodeName() : newName; + + ComputationNodePtr node = new SumColumnElementsNode(this, name, flags); + return node; + } + + virtual const std::wstring OperationName() const { return TypeName(); } + static const std::wstring TypeName() { return L"SumColumnElements"; } + + virtual void ComputeInputPartial(const size_t inputIndex) + { + if (inputIndex != 0) + throw std::invalid_argument("SumColumnElements only has one input."); + ComputeInputPartialS(Inputs(0)->GradientValues(), GradientValues()); + } + + virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq) + { + if (inputIndex != 0) + throw std::invalid_argument("SumColumnElements only has one input."); + + Matrix sliceInputGrad = Inputs(0)->GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + Matrix sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + + ComputeInputPartialS(sliceInputGrad, sliceOutputGrad); + } + + static void WINAPI ComputeInputPartialS(Matrix& inputGradientValues, const Matrix& gradientValues) + { + inputGradientValues += gradientValues; //here the assumption is that gradientValues are 1x1 matrix + } + + virtual void EvaluateThisNode() + { + EvaluateThisNodeS(m_functionValues, Inputs(0)->FunctionValues()); + } + + virtual void EvaluateThisNode(const size_t timeIdxInSeq) + { + Matrix sliceInputValue = Inputs(0)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + Matrix sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + + EvaluateThisNodeS(sliceOutputValue, sliceInputValue); + } + + static void WINAPI EvaluateThisNodeS(Matrix& functionValues, const Matrix& inputFunctionValues) + { + Matrix::VectorSum(inputFunctionValues, functionValues, true); +#if NANCHECK + functionValues.HasNan("SumColumnElements"); +#endif + } + + virtual void Validate() + { + PrintSelfBeforeValidation(); + + if (m_children.size() != 1) + throw std::logic_error("SumColumnElements operation should have one input."); + + if (Inputs(0)->FunctionValues().GetNumElements() == 0) + throw std::logic_error("SumColumnElements operation: the input node has 0 element."); + + FunctionValues().Resize(1, Inputs(0)->FunctionValues().GetNumCols()); + CopyImageSizeFromInputs(); + } + + virtual void CopyImageSizeFromInputs() + { + CopyImageSizeFromInput(0, false); + + m_outputWidth = 1; + m_outputHeight = 1; + m_outputChannels = 1; + } + + virtual void AttachInputs(const ComputationNodePtr singleInput) + { + m_children.resize(1); + m_children[0] = singleInput; + } + + virtual void CopyTo(const ComputationNodePtr nodeP, const std::wstring& newName, const CopyNodeFlags flags) const + { + ComputationNode::CopyTo(nodeP, newName, flags); + SumColumnElementsNode* node = (SumColumnElementsNode*) nodeP; + + if (flags & CopyNodeFlags::copyNodeValue) + { + node->m_sumValue = m_sumValue; + } + } + + private: + Matrix m_sumValue; + }; + + template class SumColumnElementsNode; + template class SumColumnElementsNode; + //this node is used to extract part of the input by rows as the output //it has to be continuous segments of rows since each column is treated as one sample template @@ -1136,7 +1266,18 @@ namespace Microsoft { namespace MSR { namespace CNTK { // inputIndex == 1 (right) - inputGradientValues[1], inputFunctionValues[0] static void WINAPI ComputeInputPartialS(Matrix& inputFunctionValues, Matrix& inputGradientValues, const Matrix& gradientValues) { - inputGradientValues.AddElementProductOf(gradientValues, inputFunctionValues); + size_t gradCol = gradientValues.GetNumCols(); + size_t inputCol = inputFunctionValues.GetNumCols(); + + if (gradCol != inputCol && inputCol == 1) + { + inputGradientValues.SetValue(gradientValues); + inputGradientValues.ColumnElementMultiplyWith(inputFunctionValues); + } + else + { + inputGradientValues.AddElementProductOf(gradientValues, inputFunctionValues); + } #if NANCHECK inputGradientValues.HasNan("ElementTimes"); #endif @@ -1159,7 +1300,31 @@ namespace Microsoft { namespace MSR { namespace CNTK { static void WINAPI EvaluateThisNodeS(Matrix& functionValues, const Matrix& input0, const Matrix& input1) { - functionValues.AssignElementProductOf(input0, input1); + size_t rows0 = input0.GetNumRows(), cols0 = input0.GetNumCols(); + size_t rows1 = input1.GetNumRows(), cols1 = input1.GetNumCols(); + if (rows0 == rows1 && cols0 == cols1) + { + functionValues.AssignElementProductOf(input0, input1); + } + else if ((cols0 == 1 || cols1 == 1) && rows1 == rows0) // col vec with matching rows + { + Matrix tmpMat; + if (cols0 == 1) + { + functionValues.SetValue(input1); + tmpMat = Matrix::RepMat(input0, 1, cols1); + } + else if (cols1 == 1) + { + functionValues.SetValue(input0); + tmpMat = Matrix::RepMat(input1, 1, cols0); + } + functionValues.ColumnElementMultiplyWith(tmpMat); + } + else + { + throw std::logic_error("The Matrix dimension in the ElementTimes operation does not match."); + } #if NANCHECK functionValues.HasNan("ElementTimes"); #endif @@ -1191,8 +1356,10 @@ namespace Microsoft { namespace MSR { namespace CNTK { if (Inputs(0)->FunctionValues().GetNumElements() == 0 || Inputs(1)->FunctionValues().GetNumElements() == 0) throw std::logic_error("ElementTimes operation: one of the operants has 0 element."); - if (Inputs(1)->FunctionValues().GetNumRows() != Inputs(0)->FunctionValues().GetNumRows() || - Inputs(1)->FunctionValues().GetNumCols() != Inputs(0)->FunctionValues().GetNumCols()) + size_t rows0 = Inputs(0)->FunctionValues().GetNumRows(), cols0 = Inputs(0)->FunctionValues().GetNumCols(); + size_t rows1 = Inputs(1)->FunctionValues().GetNumRows(), cols1 = Inputs(1)->FunctionValues().GetNumCols(); + + if (rows0 != rows1 || (cols0 != cols1 && cols0 != 1 && cols1 != 1)) throw std::logic_error("The Matrix dimension in the ElementTimes operation does not match."); FunctionValues().Resize(Inputs(0)->FunctionValues().GetNumRows(), Inputs(0)->FunctionValues().GetNumCols()); diff --git a/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp b/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp index 24b1d7b5bca5..be484700aee3 100644 --- a/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp +++ b/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp @@ -154,6 +154,8 @@ bool CheckFunction(std::string& p_nodeType, bool* allowUndeterminedVariable) ret = true; else if (EqualInsensitive(nodeType, SumElementsNode::TypeName())) ret = true; + else if (EqualInsensitive(nodeType, SumColumnElementsNode::TypeName())) + ret = true; else if (EqualInsensitive(nodeType, ScaleNode::TypeName())) ret = true; else if (EqualInsensitive(nodeType, TransposeNode::TypeName())) From de33117f1b458eb6a6cadbe07fe4487d01785c9e Mon Sep 17 00:00:00 2001 From: Nguyen Quy Hy Date: Tue, 14 Jul 2015 13:23:01 +0800 Subject: [PATCH 2/7] Fix input output mixed up in GPU VectorSum. --- Math/Math/GPUMatrix.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Math/Math/GPUMatrix.cu b/Math/Math/GPUMatrix.cu index ad61b0bc0bb0..22c13738928f 100644 --- a/Math/Math/GPUMatrix.cu +++ b/Math/Math/GPUMatrix.cu @@ -2533,7 +2533,7 @@ namespace Microsoft { namespace MSR { namespace CNTK { } if (do_sync) CUDA_CALL(cudaEventCreate(&done)); - _vectorSum << > >(a.m_pArray, c.m_pArray, n, m, isColWise); + _vectorSum << > >(c.m_pArray, a.m_pArray, n, m, isColWise); if (do_sync) CUDA_CALL(cudaEventRecord(done)); if (do_sync) CUDA_CALL(cudaEventSynchronize(done)); if (do_sync) CUDA_CALL(cudaEventDestroy(done)); From 646fcd7f38fd4646b3a63a8083cc973da791f649 Mon Sep 17 00:00:00 2001 From: Nguyen Quy Hy Date: Tue, 14 Jul 2015 13:36:10 +0800 Subject: [PATCH 3/7] Fix wrong indent type. --- MachineLearning/CNTK/LinearAlgebraNodes.h | 310 +++++++++--------- .../CNTK/NetworkDescriptionLanguage.cpp | 4 +- 2 files changed, 157 insertions(+), 157 deletions(-) diff --git a/MachineLearning/CNTK/LinearAlgebraNodes.h b/MachineLearning/CNTK/LinearAlgebraNodes.h index 9dbc4fa455d4..df11b40a73b6 100644 --- a/MachineLearning/CNTK/LinearAlgebraNodes.h +++ b/MachineLearning/CNTK/LinearAlgebraNodes.h @@ -241,135 +241,135 @@ namespace Microsoft { namespace MSR { namespace CNTK { template class SumElementsNode; template class SumElementsNode; - template - class SumColumnElementsNode : public ComputationNode - { - UsingComputationNodeMembers; - public: - SumColumnElementsNode(const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"") - : ComputationNode(deviceId), m_sumValue(deviceId) - { - m_nodeName = (name == L"" ? CreateUniqNodeName() : name); - m_deviceId = deviceId; - MoveMatricesToDevice(deviceId); - InitRecurrentNode(); - } - - SumColumnElementsNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"") - : ComputationNode(deviceId), m_sumValue(deviceId) - { - m_nodeName = (name == L"" ? CreateUniqNodeName() : name); - LoadFromFile(fstream, modelVersion, deviceId); - } - - // copy constructor - SumColumnElementsNode(const SumColumnElementsNode* node, const std::wstring& newName, const CopyNodeFlags flags) - : ComputationNode(node->m_deviceId), m_sumValue(node->m_deviceId) - { - node->CopyTo(this, newName, flags); - } - - virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const - { - const std::wstring& name = (newName == L"") ? NodeName() : newName; - - ComputationNodePtr node = new SumColumnElementsNode(this, name, flags); - return node; - } - - virtual const std::wstring OperationName() const { return TypeName(); } - static const std::wstring TypeName() { return L"SumColumnElements"; } - - virtual void ComputeInputPartial(const size_t inputIndex) - { - if (inputIndex != 0) - throw std::invalid_argument("SumColumnElements only has one input."); - ComputeInputPartialS(Inputs(0)->GradientValues(), GradientValues()); - } - - virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq) - { - if (inputIndex != 0) - throw std::invalid_argument("SumColumnElements only has one input."); - - Matrix sliceInputGrad = Inputs(0)->GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); - Matrix sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); - - ComputeInputPartialS(sliceInputGrad, sliceOutputGrad); - } - - static void WINAPI ComputeInputPartialS(Matrix& inputGradientValues, const Matrix& gradientValues) - { - inputGradientValues += gradientValues; //here the assumption is that gradientValues are 1x1 matrix - } - - virtual void EvaluateThisNode() - { - EvaluateThisNodeS(m_functionValues, Inputs(0)->FunctionValues()); - } - - virtual void EvaluateThisNode(const size_t timeIdxInSeq) - { - Matrix sliceInputValue = Inputs(0)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); - Matrix sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); - - EvaluateThisNodeS(sliceOutputValue, sliceInputValue); - } - - static void WINAPI EvaluateThisNodeS(Matrix& functionValues, const Matrix& inputFunctionValues) - { - Matrix::VectorSum(inputFunctionValues, functionValues, true); + template + class SumColumnElementsNode : public ComputationNode + { + UsingComputationNodeMembers; + public: + SumColumnElementsNode(const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"") + : ComputationNode(deviceId), m_sumValue(deviceId) + { + m_nodeName = (name == L"" ? CreateUniqNodeName() : name); + m_deviceId = deviceId; + MoveMatricesToDevice(deviceId); + InitRecurrentNode(); + } + + SumColumnElementsNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"") + : ComputationNode(deviceId), m_sumValue(deviceId) + { + m_nodeName = (name == L"" ? CreateUniqNodeName() : name); + LoadFromFile(fstream, modelVersion, deviceId); + } + + // copy constructor + SumColumnElementsNode(const SumColumnElementsNode* node, const std::wstring& newName, const CopyNodeFlags flags) + : ComputationNode(node->m_deviceId), m_sumValue(node->m_deviceId) + { + node->CopyTo(this, newName, flags); + } + + virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const + { + const std::wstring& name = (newName == L"") ? NodeName() : newName; + + ComputationNodePtr node = new SumColumnElementsNode(this, name, flags); + return node; + } + + virtual const std::wstring OperationName() const { return TypeName(); } + static const std::wstring TypeName() { return L"SumColumnElements"; } + + virtual void ComputeInputPartial(const size_t inputIndex) + { + if (inputIndex != 0) + throw std::invalid_argument("SumColumnElements only has one input."); + ComputeInputPartialS(Inputs(0)->GradientValues(), GradientValues()); + } + + virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq) + { + if (inputIndex != 0) + throw std::invalid_argument("SumColumnElements only has one input."); + + Matrix sliceInputGrad = Inputs(0)->GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + Matrix sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + + ComputeInputPartialS(sliceInputGrad, sliceOutputGrad); + } + + static void WINAPI ComputeInputPartialS(Matrix& inputGradientValues, const Matrix& gradientValues) + { + inputGradientValues += gradientValues; //here the assumption is that gradientValues are 1x1 matrix + } + + virtual void EvaluateThisNode() + { + EvaluateThisNodeS(m_functionValues, Inputs(0)->FunctionValues()); + } + + virtual void EvaluateThisNode(const size_t timeIdxInSeq) + { + Matrix sliceInputValue = Inputs(0)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + Matrix sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep); + + EvaluateThisNodeS(sliceOutputValue, sliceInputValue); + } + + static void WINAPI EvaluateThisNodeS(Matrix& functionValues, const Matrix& inputFunctionValues) + { + Matrix::VectorSum(inputFunctionValues, functionValues, true); #if NANCHECK - functionValues.HasNan("SumColumnElements"); + functionValues.HasNan("SumColumnElements"); #endif - } + } - virtual void Validate() - { - PrintSelfBeforeValidation(); + virtual void Validate() + { + PrintSelfBeforeValidation(); - if (m_children.size() != 1) - throw std::logic_error("SumColumnElements operation should have one input."); + if (m_children.size() != 1) + throw std::logic_error("SumColumnElements operation should have one input."); - if (Inputs(0)->FunctionValues().GetNumElements() == 0) - throw std::logic_error("SumColumnElements operation: the input node has 0 element."); + if (Inputs(0)->FunctionValues().GetNumElements() == 0) + throw std::logic_error("SumColumnElements operation: the input node has 0 element."); - FunctionValues().Resize(1, Inputs(0)->FunctionValues().GetNumCols()); - CopyImageSizeFromInputs(); - } + FunctionValues().Resize(1, Inputs(0)->FunctionValues().GetNumCols()); + CopyImageSizeFromInputs(); + } - virtual void CopyImageSizeFromInputs() - { - CopyImageSizeFromInput(0, false); + virtual void CopyImageSizeFromInputs() + { + CopyImageSizeFromInput(0, false); - m_outputWidth = 1; - m_outputHeight = 1; - m_outputChannels = 1; - } + m_outputWidth = 1; + m_outputHeight = 1; + m_outputChannels = 1; + } - virtual void AttachInputs(const ComputationNodePtr singleInput) - { - m_children.resize(1); - m_children[0] = singleInput; - } + virtual void AttachInputs(const ComputationNodePtr singleInput) + { + m_children.resize(1); + m_children[0] = singleInput; + } - virtual void CopyTo(const ComputationNodePtr nodeP, const std::wstring& newName, const CopyNodeFlags flags) const - { - ComputationNode::CopyTo(nodeP, newName, flags); - SumColumnElementsNode* node = (SumColumnElementsNode*) nodeP; + virtual void CopyTo(const ComputationNodePtr nodeP, const std::wstring& newName, const CopyNodeFlags flags) const + { + ComputationNode::CopyTo(nodeP, newName, flags); + SumColumnElementsNode* node = (SumColumnElementsNode*) nodeP; - if (flags & CopyNodeFlags::copyNodeValue) - { - node->m_sumValue = m_sumValue; - } - } + if (flags & CopyNodeFlags::copyNodeValue) + { + node->m_sumValue = m_sumValue; + } + } - private: - Matrix m_sumValue; - }; + private: + Matrix m_sumValue; + }; - template class SumColumnElementsNode; - template class SumColumnElementsNode; + template class SumColumnElementsNode; + template class SumColumnElementsNode; //this node is used to extract part of the input by rows as the output //it has to be continuous segments of rows since each column is treated as one sample @@ -1266,18 +1266,18 @@ namespace Microsoft { namespace MSR { namespace CNTK { // inputIndex == 1 (right) - inputGradientValues[1], inputFunctionValues[0] static void WINAPI ComputeInputPartialS(Matrix& inputFunctionValues, Matrix& inputGradientValues, const Matrix& gradientValues) { - size_t gradCol = gradientValues.GetNumCols(); - size_t inputCol = inputFunctionValues.GetNumCols(); - - if (gradCol != inputCol && inputCol == 1) - { - inputGradientValues.SetValue(gradientValues); - inputGradientValues.ColumnElementMultiplyWith(inputFunctionValues); - } - else - { - inputGradientValues.AddElementProductOf(gradientValues, inputFunctionValues); - } + size_t gradCol = gradientValues.GetNumCols(); + size_t inputCol = inputFunctionValues.GetNumCols(); + + if (gradCol != inputCol && inputCol == 1) + { + inputGradientValues.SetValue(gradientValues); + inputGradientValues.ColumnElementMultiplyWith(inputFunctionValues); + } + else + { + inputGradientValues.AddElementProductOf(gradientValues, inputFunctionValues); + } #if NANCHECK inputGradientValues.HasNan("ElementTimes"); #endif @@ -1300,31 +1300,31 @@ namespace Microsoft { namespace MSR { namespace CNTK { static void WINAPI EvaluateThisNodeS(Matrix& functionValues, const Matrix& input0, const Matrix& input1) { - size_t rows0 = input0.GetNumRows(), cols0 = input0.GetNumCols(); - size_t rows1 = input1.GetNumRows(), cols1 = input1.GetNumCols(); - if (rows0 == rows1 && cols0 == cols1) - { - functionValues.AssignElementProductOf(input0, input1); - } - else if ((cols0 == 1 || cols1 == 1) && rows1 == rows0) // col vec with matching rows - { - Matrix tmpMat; - if (cols0 == 1) - { - functionValues.SetValue(input1); - tmpMat = Matrix::RepMat(input0, 1, cols1); - } - else if (cols1 == 1) - { - functionValues.SetValue(input0); - tmpMat = Matrix::RepMat(input1, 1, cols0); - } - functionValues.ColumnElementMultiplyWith(tmpMat); - } - else - { - throw std::logic_error("The Matrix dimension in the ElementTimes operation does not match."); - } + size_t rows0 = input0.GetNumRows(), cols0 = input0.GetNumCols(); + size_t rows1 = input1.GetNumRows(), cols1 = input1.GetNumCols(); + if (rows0 == rows1 && cols0 == cols1) + { + functionValues.AssignElementProductOf(input0, input1); + } + else if ((cols0 == 1 || cols1 == 1) && rows1 == rows0) // col vec with matching rows + { + Matrix tmpMat; + if (cols0 == 1) + { + functionValues.SetValue(input1); + tmpMat = Matrix::RepMat(input0, 1, cols1); + } + else if (cols1 == 1) + { + functionValues.SetValue(input0); + tmpMat = Matrix::RepMat(input1, 1, cols0); + } + functionValues.ColumnElementMultiplyWith(tmpMat); + } + else + { + throw std::logic_error("The Matrix dimension in the ElementTimes operation does not match."); + } #if NANCHECK functionValues.HasNan("ElementTimes"); #endif @@ -1356,8 +1356,8 @@ namespace Microsoft { namespace MSR { namespace CNTK { if (Inputs(0)->FunctionValues().GetNumElements() == 0 || Inputs(1)->FunctionValues().GetNumElements() == 0) throw std::logic_error("ElementTimes operation: one of the operants has 0 element."); - size_t rows0 = Inputs(0)->FunctionValues().GetNumRows(), cols0 = Inputs(0)->FunctionValues().GetNumCols(); - size_t rows1 = Inputs(1)->FunctionValues().GetNumRows(), cols1 = Inputs(1)->FunctionValues().GetNumCols(); + size_t rows0 = Inputs(0)->FunctionValues().GetNumRows(), cols0 = Inputs(0)->FunctionValues().GetNumCols(); + size_t rows1 = Inputs(1)->FunctionValues().GetNumRows(), cols1 = Inputs(1)->FunctionValues().GetNumCols(); if (rows0 != rows1 || (cols0 != cols1 && cols0 != 1 && cols1 != 1)) throw std::logic_error("The Matrix dimension in the ElementTimes operation does not match."); diff --git a/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp b/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp index be484700aee3..c6d8d092ea7b 100644 --- a/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp +++ b/MachineLearning/CNTK/NetworkDescriptionLanguage.cpp @@ -154,8 +154,8 @@ bool CheckFunction(std::string& p_nodeType, bool* allowUndeterminedVariable) ret = true; else if (EqualInsensitive(nodeType, SumElementsNode::TypeName())) ret = true; - else if (EqualInsensitive(nodeType, SumColumnElementsNode::TypeName())) - ret = true; + else if (EqualInsensitive(nodeType, SumColumnElementsNode::TypeName())) + ret = true; else if (EqualInsensitive(nodeType, ScaleNode::TypeName())) ret = true; else if (EqualInsensitive(nodeType, TransposeNode::TypeName())) From 0b27ed6cf1e50ab48937f9d55a0d926c15e391fb Mon Sep 17 00:00:00 2001 From: Nguyen Quy Hy Date: Tue, 14 Jul 2015 14:13:00 +0800 Subject: [PATCH 4/7] Add SumColumnElementsNode in to ComputationNetwork. --- MachineLearning/CNTK/ComputationNetwork.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/MachineLearning/CNTK/ComputationNetwork.h b/MachineLearning/CNTK/ComputationNetwork.h index 4763c5a80422..a6ecaabeee15 100644 --- a/MachineLearning/CNTK/ComputationNetwork.h +++ b/MachineLearning/CNTK/ComputationNetwork.h @@ -1168,7 +1168,11 @@ class ComputationNetwork else if (nodeType == SumElementsNode::TypeName()) { newNode = new SumElementsNode(fstream, modelVersion, m_deviceId, nodeName); - } + } + else if (nodeType == SumColumnElementsNode::TypeName()) + { + newNode = new SumColumnElementsNode(fstream, modelVersion, m_deviceId, nodeName); + } else if (nodeType == ScaleNode::TypeName()) { newNode = new ScaleNode(fstream, modelVersion, m_deviceId, nodeName); @@ -1472,7 +1476,11 @@ class ComputationNetwork else if (nodeType == SumElementsNode::TypeName()) { newNode = new SumElementsNode(m_deviceId, nodeName); - } + } + else if (nodeType == SumColumnElementsNode::TypeName()) + { + newNode = new SumColumnElementsNode(m_deviceId, nodeName); + } else if (nodeType == ScaleNode::TypeName()) { newNode = new ScaleNode(m_deviceId, nodeName); From f12d3e20f3c34f8a733bd3666f63fc31be873e54 Mon Sep 17 00:00:00 2001 From: Nguyen Quy Hy Date: Tue, 14 Jul 2015 16:43:27 +0800 Subject: [PATCH 5/7] Use ColumnElementMultiplyWith for multiplying with col vector instead of Repmat. --- MachineLearning/CNTK/LinearAlgebraNodes.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/MachineLearning/CNTK/LinearAlgebraNodes.h b/MachineLearning/CNTK/LinearAlgebraNodes.h index df11b40a73b6..44591d3f18af 100644 --- a/MachineLearning/CNTK/LinearAlgebraNodes.h +++ b/MachineLearning/CNTK/LinearAlgebraNodes.h @@ -300,7 +300,7 @@ namespace Microsoft { namespace MSR { namespace CNTK { static void WINAPI ComputeInputPartialS(Matrix& inputGradientValues, const Matrix& gradientValues) { - inputGradientValues += gradientValues; //here the assumption is that gradientValues are 1x1 matrix + inputGradientValues += gradientValues; //here the assumption is that gradientValues are a row vector } virtual void EvaluateThisNode() @@ -1312,14 +1312,13 @@ namespace Microsoft { namespace MSR { namespace CNTK { if (cols0 == 1) { functionValues.SetValue(input1); - tmpMat = Matrix::RepMat(input0, 1, cols1); + functionValues.ColumnElementMultiplyWith(input0); } else if (cols1 == 1) { functionValues.SetValue(input0); - tmpMat = Matrix::RepMat(input1, 1, cols0); + functionValues.ColumnElementMultiplyWith(input1); } - functionValues.ColumnElementMultiplyWith(tmpMat); } else { From 5699f5d318d3aa559f85df5aef9ae3321c6b02a9 Mon Sep 17 00:00:00 2001 From: Nguyen Quy Hy Date: Tue, 14 Jul 2015 16:53:58 +0800 Subject: [PATCH 6/7] Fix tab & space inconsistency. --- MachineLearning/CNTK/ComputationNetwork.h | 20 ++++++++++---------- MachineLearning/CNTK/LinearAlgebraNodes.h | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/MachineLearning/CNTK/ComputationNetwork.h b/MachineLearning/CNTK/ComputationNetwork.h index a6ecaabeee15..1d4fcdc866cb 100644 --- a/MachineLearning/CNTK/ComputationNetwork.h +++ b/MachineLearning/CNTK/ComputationNetwork.h @@ -1168,11 +1168,11 @@ class ComputationNetwork else if (nodeType == SumElementsNode::TypeName()) { newNode = new SumElementsNode(fstream, modelVersion, m_deviceId, nodeName); - } - else if (nodeType == SumColumnElementsNode::TypeName()) - { - newNode = new SumColumnElementsNode(fstream, modelVersion, m_deviceId, nodeName); - } + } + else if (nodeType == SumColumnElementsNode::TypeName()) + { + newNode = new SumColumnElementsNode(fstream, modelVersion, m_deviceId, nodeName); + } else if (nodeType == ScaleNode::TypeName()) { newNode = new ScaleNode(fstream, modelVersion, m_deviceId, nodeName); @@ -1476,11 +1476,11 @@ class ComputationNetwork else if (nodeType == SumElementsNode::TypeName()) { newNode = new SumElementsNode(m_deviceId, nodeName); - } - else if (nodeType == SumColumnElementsNode::TypeName()) - { - newNode = new SumColumnElementsNode(m_deviceId, nodeName); - } + } + else if (nodeType == SumColumnElementsNode::TypeName()) + { + newNode = new SumColumnElementsNode(m_deviceId, nodeName); + } else if (nodeType == ScaleNode::TypeName()) { newNode = new ScaleNode(m_deviceId, nodeName); diff --git a/MachineLearning/CNTK/LinearAlgebraNodes.h b/MachineLearning/CNTK/LinearAlgebraNodes.h index 44591d3f18af..94b5c79a8dd8 100644 --- a/MachineLearning/CNTK/LinearAlgebraNodes.h +++ b/MachineLearning/CNTK/LinearAlgebraNodes.h @@ -300,7 +300,7 @@ namespace Microsoft { namespace MSR { namespace CNTK { static void WINAPI ComputeInputPartialS(Matrix& inputGradientValues, const Matrix& gradientValues) { - inputGradientValues += gradientValues; //here the assumption is that gradientValues are a row vector + inputGradientValues += gradientValues; //here the assumption is that gradientValues is a row vector } virtual void EvaluateThisNode() @@ -1312,12 +1312,12 @@ namespace Microsoft { namespace MSR { namespace CNTK { if (cols0 == 1) { functionValues.SetValue(input1); - functionValues.ColumnElementMultiplyWith(input0); + functionValues.ColumnElementMultiplyWith(input0); } else if (cols1 == 1) { functionValues.SetValue(input0); - functionValues.ColumnElementMultiplyWith(input1); + functionValues.ColumnElementMultiplyWith(input1); } } else From 05d2db29af8a53ddc767a56f2abdc1955e957446 Mon Sep 17 00:00:00 2001 From: Yu Date: Wed, 15 Jul 2015 05:14:10 -0400 Subject: [PATCH 7/7] Fix a bug when forming the recurrent loop. --- MachineLearning/CNTK/ComputationNetwork.h | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/MachineLearning/CNTK/ComputationNetwork.h b/MachineLearning/CNTK/ComputationNetwork.h index 1d4fcdc866cb..25ac34b4d601 100644 --- a/MachineLearning/CNTK/ComputationNetwork.h +++ b/MachineLearning/CNTK/ComputationNetwork.h @@ -2894,7 +2894,7 @@ class ComputationNetwork for (ComputationNodePtr node : FinalCriterionNodes()) { if (!allowFragment) { - FormRecurentLoops(node); + FormRecurentLoops(node, true); } PrintComputationTree(node, false); size_t actualMBSize = this->GetActualMBSize(); @@ -3312,7 +3312,7 @@ class ComputationNetwork } // get the strong connected component from the graph - void getStrongSCC(const ComputationNodePtr rootNode) + void getStrongSCC(const ComputationNodePtr rootNode, bool isCriterion) { std::unordered_set visited; std::list sccStack; @@ -3320,13 +3320,13 @@ class ComputationNetwork size_t loopId = 0; if (rootNode->isVisisted() == false) { - strongSCC(rootNode, sccStack, index, loopId); + strongSCC(rootNode, sccStack, index, loopId, isCriterion); } } void strongSCC(ComputationNodePtr cur, std::list& sccStack, - size_t& index, size_t& loopId) + size_t& index, size_t& loopId, bool isCriterion) { cur->SetIndex(index); cur->Setlowlink(index); @@ -3340,7 +3340,7 @@ class ComputationNetwork { if (cur->Inputs(i)->isVisisted() == false) { - strongSCC(cur->Inputs(i), sccStack, index, loopId); + strongSCC(cur->Inputs(i), sccStack, index, loopId, isCriterion); cur->Setlowlink(min(cur->Getlowlink(), cur->Inputs(i)->Getlowlink())); } else if (cur->Inputs(i)->isInStack()) @@ -3368,7 +3368,7 @@ class ComputationNetwork } } rInfo.Reset(); - if (sccSize > 1) + if (sccSize > 1 && isCriterion) { loopId++; m_recurrentInfo.push_back(rInfo); @@ -3409,11 +3409,11 @@ class ComputationNetwork } } //must be called before ValidateNetwork - void FormRecurentLoops(const ComputationNodePtr rootNode) + void FormRecurentLoops(const ComputationNodePtr rootNode, bool isCriterion = false) { std::vector sourceLoopNodes; - getStrongSCC(rootNode); + getStrongSCC(rootNode, isCriterion); std::list& nodes = GetEvalOrder(rootNode, sourceLoopNodes); std::list nodesForGrad; @@ -3519,6 +3519,11 @@ class ComputationNetwork } #endif } + + for (auto iter = nodes.begin(); iter != nodes.end(); iter++) + { + (*iter)->clearCache(); + } } void ReorderLoops(std::list& nodes,