Skip to content

Commit

Permalink
changed feature packing flag names from utteranceXYZ and sentence_XYZ…
Browse files Browse the repository at this point in the history 
… to SequenceXYZ to make them consistent and be understood for both speech and text people.

Added flag to indicate the condition of no feature (with assumption that there is also no label)  to support efficient bi-directional model training.
Updated the windows version of HTKMLFReader, LM and LU Sequence readers.
Caution: this change will not allow for using past and future label information in the recurrent networks. The ultimate solution is to have a flag for each feature read-in. However, this will require huge change and we will do this after reader refactorization to reduce effort.
  • Loading branch information
Dong Yu committed Jul 19, 2015
1 parent 8830b5a commit d1e1478
Show file tree
Hide file tree
Showing 14 changed files with 979 additions and 973 deletions.
25 changes: 14 additions & 11 deletions Common/Include/basetypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1058,21 +1058,24 @@ class RegisterModule
/**
These macros are used for sentence segmentation information.
*/
#define SENTENCE_BEGIN ((int) MinibatchPackingFlag::UtteranceStart)
#define SENTENCE_MIDDLE ((int) MinibatchPackingFlag::None)
#define SENTENCE_END ((int) MinibatchPackingFlag::UtteranceEnd)
#define NO_LABELS ((int) MinibatchPackingFlag::NoLabel)
#define SEQUENCE_START ((int) MinibatchPackingFlag::SequenceStart)
#define SEQUENCE_MIDDLE ((int) MinibatchPackingFlag::None)
#define SEQUENCE_END ((int) MinibatchPackingFlag::SequenceEnd)
#define NO_INPUT ((int) MinibatchPackingFlag::NoInput)
#define NO_LABEL ((int) MinibatchPackingFlag::NoLabel)

enum class MinibatchPackingFlag : unsigned char
{
None = 0,
UtteranceStart = 1 << 0, //binary 0001
UtteranceEnd = 1 << 1, //binary 0010
NoLabel = 1 << 2, //binary 0100

UtteranceStartOrNoLabel = UtteranceStart | NoLabel,
UtteranceEndOrNoLabel = UtteranceEnd | NoLabel,
UtteranceStartOrEndOrNoLabel = UtteranceStart | UtteranceEnd | NoLabel,
SequenceStart = 1 << 0, //binary 0001
SequenceEnd = 1 << 1, //binary 0010
NoFeature = 1 << 2, //binary 0100
NoLabel = 1 << 3, //binary 1000

NoInput = NoFeature | NoLabel, //when we refactorize reader, NoInput will no longer needed
SequenceStartOrNoInput = SequenceStart | NoInput,
SequenceEndOrNoInput = SequenceEnd | NoInput,
SequenceStartOrEndOrNoInput = SequenceStart | SequenceEnd | NoInput,
};

inline MinibatchPackingFlag operator| (MinibatchPackingFlag a, MinibatchPackingFlag b)
Expand Down
22 changes: 11 additions & 11 deletions DataReader/HTKMLFReader/HTKMLFReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -790,11 +790,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_sentenceBegin.Resize((size_t)1, (size_t)feat.cols());
m_minibatchPackingFlag.resize(feat.cols());

m_sentenceBegin.SetValue((ElemType) SENTENCE_MIDDLE);
m_sentenceBegin.SetValue(0, 0, (ElemType) SENTENCE_BEGIN);
m_sentenceBegin.SetValue((ElemType) SEQUENCE_MIDDLE);
m_sentenceBegin.SetValue(0, 0, (ElemType) SEQUENCE_START);

std::fill(m_minibatchPackingFlag.begin(), m_minibatchPackingFlag.end(), MinibatchPackingFlag::None);
m_minibatchPackingFlag[0] = MinibatchPackingFlag::UtteranceStart;
m_minibatchPackingFlag[0] = MinibatchPackingFlag::SequenceStart;
first = false;
}

Expand Down Expand Up @@ -946,7 +946,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
for (size_t j = 0; j < m_mbSize; j++)
{
m_sentenceBegin.SetValue(i,j,(ElemType) SENTENCE_MIDDLE);
m_sentenceBegin.SetValue(i,j,(ElemType) SEQUENCE_MIDDLE);
}
}
std::fill(m_minibatchPackingFlag.begin(), m_minibatchPackingFlag.end(), MinibatchPackingFlag::None);
Expand All @@ -969,8 +969,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
m_switchFrame[i] = 0;
m_sentenceEnd[i] = true;
m_sentenceBegin.SetValue(i, 0, (ElemType)SENTENCE_BEGIN);
m_minibatchPackingFlag[0] = MinibatchPackingFlag::UtteranceStart;
m_sentenceBegin.SetValue(i, 0, (ElemType)SEQUENCE_START);
m_minibatchPackingFlag[0] = MinibatchPackingFlag::SequenceStart;
}
actualmbsize[i] = m_mbSize;
endFr = startFr + actualmbsize[i];
Expand Down Expand Up @@ -1121,8 +1121,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_switchFrame[i] = actualmbsize[i];
if (actualmbsize[i] < m_mbSize)
{
m_sentenceBegin.SetValue(i, actualmbsize[i], (ElemType)SENTENCE_BEGIN);
m_minibatchPackingFlag[actualmbsize[i]] |= MinibatchPackingFlag::UtteranceStart;
m_sentenceBegin.SetValue(i, actualmbsize[i], (ElemType)SEQUENCE_START);
m_minibatchPackingFlag[actualmbsize[i]] |= MinibatchPackingFlag::SequenceStart;
}
startFr = m_switchFrame[i];
endFr = m_mbSize;
Expand Down Expand Up @@ -1270,11 +1270,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
m_sentenceBegin.Resize((size_t)1, (size_t)feat.cols());
m_minibatchPackingFlag.resize((size_t)feat.cols());
m_sentenceBegin.SetValue((ElemType)SENTENCE_MIDDLE);
m_sentenceBegin.SetValue(0, 0, (ElemType)SENTENCE_BEGIN);
m_sentenceBegin.SetValue((ElemType)SEQUENCE_MIDDLE);
m_sentenceBegin.SetValue(0, 0, (ElemType)SEQUENCE_START);

std::fill(m_minibatchPackingFlag.begin(), m_minibatchPackingFlag.end(), MinibatchPackingFlag::None);
m_minibatchPackingFlag[0] = MinibatchPackingFlag::UtteranceStart;
m_minibatchPackingFlag[0] = MinibatchPackingFlag::SequenceStart;
first = false;
}

Expand Down
2 changes: 1 addition & 1 deletion DataReader/HTKMLFReader/HTKMLFReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ class HTKMLFReader : public IDataReader<ElemType>
virtual bool GetData(const std::wstring& sectionName, size_t numRecords, void* data, size_t& dataBufferSize, size_t recordStart=0);

virtual bool DataEnd(EndDataType endDataType);
void SetSentenceSegBatch(Matrix<ElemType> &sentenceBegin, vector<MinibatchPackingFlag>& sentenceExistsBeginOrNoLabels);
void SetSentenceSegBatch(Matrix<ElemType> &sentenceBegin, vector<MinibatchPackingFlag>& sentenceExistsBeginOrNoInputs);
void SetSentenceEnd(int /*actualMbSize*/){};
void SetRandomSeed(int){ NOT_IMPLEMENTED };
};
Expand Down
25 changes: 14 additions & 11 deletions DataReader/HTKMLFReader/basetypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,21 +145,24 @@ extern void _CHECKED_ASSERT_error(const char * file, int line, const char * exp)
/**
These macros are used for sentence segmentation information.
*/
#define SENTENCE_BEGIN ((int) MinibatchPackingFlag::UtteranceStart)
#define SENTENCE_MIDDLE ((int) MinibatchPackingFlag::None)
#define SENTENCE_END ((int) MinibatchPackingFlag::UtteranceEnd)
#define NO_LABELS ((int) MinibatchPackingFlag::NoLabel)
#define SEQUENCE_START ((int) MinibatchPackingFlag::SequenceStart)
#define SEQUENCE_MIDDLE ((int) MinibatchPackingFlag::None)
#define SEQUENCE_END ((int) MinibatchPackingFlag::SequenceEnd)
#define NO_INPUT ((int) MinibatchPackingFlag::NoInput)
#define NO_LABEL = ((int) MinibatchPackingFlag::NoLabel)

enum class MinibatchPackingFlag : unsigned char
{
None = 0,
UtteranceStart = 1 << 0, //binary 0001
UtteranceEnd = 1 << 1, //binary 0010
NoLabel = 1 << 2, //binary 0100

UtteranceStartOrNoLabel = UtteranceStart | NoLabel,
UtteranceEndOrNoLabel = UtteranceEnd | NoLabel,
UtteranceStartOrEndOrNoLabel = UtteranceStart | UtteranceEnd | NoLabel,
SequenceStart = 1 << 0, //binary 0001
SequenceEnd = 1 << 1, //binary 0010
NoFeature = 1 << 2, //binary 0100
NoLabel = 1 << 3, //binary 1000

NoInput = NoFeature | NoLabel, //when we refactorize reader, NoInput will no longer needed
SequenceStartOrNoInput = SequenceStart | NoInput,
SequenceEndOrNoInput = SequenceEnd | NoInput,
SequenceStartOrEndOrNoInput = SequenceStart | SequenceEnd | NoInput,
};

inline MinibatchPackingFlag operator| (MinibatchPackingFlag a, MinibatchPackingFlag b)
Expand Down
6 changes: 3 additions & 3 deletions DataReader/LMSequenceReader/SequenceReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1858,7 +1858,7 @@ bool BatchSequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix<E
size_t nT = actualmbsize / mToProcess.size();
mtSentenceBegin.TransferFromDeviceToDevice(mtSentenceBegin.GetDeviceId(), CPUDEVICE);
mtSentenceBegin.Resize(mToProcess.size(), nT);
mtSentenceBegin.SetValue((ElemType)SENTENCE_MIDDLE);
mtSentenceBegin.SetValue((ElemType)SEQUENCE_MIDDLE);
m_minibatchPackingFlag.resize(nT);
std::fill(m_minibatchPackingFlag.begin(), m_minibatchPackingFlag.end(), MinibatchPackingFlag::None);

Expand Down Expand Up @@ -1964,8 +1964,8 @@ void BatchSequenceReader<ElemType>::SetSentenceBegin(int wrd, int uttPos, int ti
if (wrd == (int)index)
{
mSentenceBegin = true;
mtSentenceBegin.SetValue(uttPos, timePos, (ElemType)SENTENCE_BEGIN);
m_minibatchPackingFlag[timePos] = MinibatchPackingFlag::UtteranceStart;
mtSentenceBegin.SetValue(uttPos, timePos, (ElemType)SEQUENCE_START);
m_minibatchPackingFlag[timePos] = MinibatchPackingFlag::SequenceStart;
}
}
}
Expand Down
18 changes: 9 additions & 9 deletions DataReader/LUSequenceReader/LUSequenceReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -705,8 +705,8 @@ bool BatchLUSequenceReader<ElemType>::EnsureDataAvailable(size_t /*mbStartSample
if (mMaxSentenceLength > m_mbSize)
throw std::runtime_error("LUSequenceReader : minibatch size needs to be large enough to accomodate the longest sentence");

/// reset sentenceending index to NO_LABELS, which is negative
mSentenceEndAt.assign(mSentenceEndAt.size(), NO_LABELS);
/// reset sentenceending index to NO_INPUT, which is negative
mSentenceEndAt.assign(mSentenceEndAt.size(), NO_INPUT);

/**
mtSentenceBegin : a matrix with [Ns x T]
Expand All @@ -715,7 +715,7 @@ bool BatchLUSequenceReader<ElemType>::EnsureDataAvailable(size_t /*mbStartSample
1 : case exists
*/
mtSentenceBegin.Resize(mToProcess.size(), mMaxSentenceLength);
mtSentenceBegin.SetValue((ElemType) SENTENCE_MIDDLE);
mtSentenceBegin.SetValue((ElemType) SEQUENCE_MIDDLE);
DEVICEID_TYPE sentenceSegDeviceId = mtSentenceBegin.GetDeviceId();
mtSentenceBegin.TransferFromDeviceToDevice(sentenceSegDeviceId, CPUDEVICE, true, false, false);

Expand All @@ -735,8 +735,8 @@ bool BatchLUSequenceReader<ElemType>::EnsureDataAvailable(size_t /*mbStartSample
mSentenceBeginAt[k] = i;
if (mIgnoreSentenceBeginTag == false) /// ignore sentence begin, this is used for decoder network reader, which carries activities from the encoder networks
{
mtSentenceBegin.SetValue(k, j, (ElemType)SENTENCE_BEGIN);
m_minibatchPackingFlag[j] |= MinibatchPackingFlag::UtteranceStart;
mtSentenceBegin.SetValue(k, j, (ElemType)SEQUENCE_START);
m_minibatchPackingFlag[j] |= MinibatchPackingFlag::SequenceStart;
}
}

Expand Down Expand Up @@ -798,8 +798,8 @@ bool BatchLUSequenceReader<ElemType>::EnsureDataAvailable(size_t /*mbStartSample
m_featureWordContext.push_back(tmpCxt);

m_labelIdData.push_back((LabelIdType)NULLLABEL);
mtSentenceBegin.SetValue(k, j, (ElemType) NO_LABELS);
m_minibatchPackingFlag[j] |= MinibatchPackingFlag::NoLabel;
mtSentenceBegin.SetValue(k, j, (ElemType) NO_INPUT);
m_minibatchPackingFlag[j] |= MinibatchPackingFlag::NoInput;
}

}
Expand Down Expand Up @@ -899,7 +899,7 @@ bool BatchLUSequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix

if (idx >= featInfo.dim)
{
if (mtSentenceBegin(utt_id, utt_t) != NO_LABELS) /// for those obs that are for no observations
if (mtSentenceBegin(utt_id, utt_t) != NO_INPUT) /// for those obs that are for no observations
{
LogicError("BatchLUSequenceReader::GetMinibatch observation is larger than its dimension but no_labels sign is not used to indicate that this observation has no labels. Possible reason is a bug in EnsureDataAvailable or a bug here. ");
}
Expand Down Expand Up @@ -1046,7 +1046,7 @@ bool BatchLUSequenceReader<ElemType>::DataEnd(EndDataType endDataType)
ret = true;
for (size_t i = 0; i < mToProcess.size(); i++)
{
if (mSentenceEndAt[i] == NO_LABELS)
if (mSentenceEndAt[i] == NO_INPUT)
{
LogicError("BatchLUSequenceReader: minibatch should be large enough to accomodate the longest sentence");
}
Expand Down
4 changes: 2 additions & 2 deletions DataReader/LUSequenceReader/LUSequenceReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,7 @@ class BatchLUSequenceReader : public LUSequenceReader<ElemType>
/// n_length is the maximum lenght of each stream
/// for example, two sentences used in parallel in one minibatch would be
/// [2 x 5] if the max length of one of the sentences is 5
/// the elements of the matrix is 0, 1, or -1, defined as SENTENCE_BEGIN, SENTENCE_MIDDLE, NO_LABELS in cbasetype.h
/// the elements of the matrix is 0, 1, or -1, defined as SEQUENCE_START, SEQUENCE_MIDDLE, NO_INPUT in cbasetype.h
/// 0 1 1 0 1
/// 1 0 1 0 0
/// for two parallel data streams. The first has two sentences, with 0 indicating begining of a sentence
Expand All @@ -367,7 +367,7 @@ class BatchLUSequenceReader : public LUSequenceReader<ElemType>
vector<MinibatchPackingFlag> m_minibatchPackingFlag;

/// by default it is false
/// if true, reader will set to SENTENCE_MIDDLE for time positions that are orignally correspond to SENTENCE_BEGIN
/// if true, reader will set to SEQUENCE_MIDDLE for time positions that are orignally correspond to SEQUENCE_START
/// set to true so that a current minibatch can uses state activities from the previous minibatch.
/// default will have truncated BPTT, which only does BPTT inside a minibatch
bool mIgnoreSentenceBeginTag;
Expand Down
6 changes: 3 additions & 3 deletions MachineLearning/CNTK/ComputationNode.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,9 +247,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {

Matrix<ElemType> colPos(sentenceBegin.GetDeviceId());
colPos.SetValue(sentenceBegin); /// -1 0 1
colPos.InplaceTruncateBottom(SENTENCE_BEGIN);
colPos.InplaceTruncateBottom(SEQUENCE_START);
Matrix<ElemType>::Scale((ElemType)-1.0, colPos);
colPos += SENTENCE_MIDDLE;
colPos += SEQUENCE_MIDDLE;
colSeg.SetDiagonalValue(colPos);
Matrix<ElemType> ones(sentenceBegin.GetDeviceId());
ones.Resize(nStateRow, nStream);
Expand Down Expand Up @@ -291,7 +291,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
colSeg = m_sentenceSeg->ColumnSlice(j,1);
for (int i = 0; i < nS; i++)
{
if (colSeg(i,0) == NO_LABELS)
if ((int)colSeg(i,0) & NO_LABEL)
{
matrixToBeMasked.ColumnSlice(utt_t+i, 1).SetValue(0);
}
Expand Down
Loading

0 comments on commit d1e1478

Please sign in to comment.