Introducing chunks

Source/Readers/ImageReader/ImageDataDeserializer.cpp

@@ -11,6 +11,7 @@
 #ifndef UNREFERENCED_PARAMETER
 #define UNREFERENCED_PARAMETER(P) (P)
 #endif
+#include "ImageSequence.h"
 
 namespace Microsoft { namespace MSR { namespace CNTK {
 
@@ -79,6 +80,68 @@ ImageDataDeserializer::ImageDataDeserializer(const ConfigParameters& config)
     CreateSequenceDescriptions(configHelper.GetMapPath(), labelDimension);
 }
 
+class ImageDataDeserializer::ImageChunk : public Chunk, public std::enable_shared_from_this<ImageChunk>
+{
+    ImageSequenceDescription m_description;
+    ImageDataDeserializer& m_parent;
+
+public:
+    ImageChunk(ImageSequenceDescription& description, ImageDataDeserializer& parent)
+        : m_description(description), m_parent(parent)
+    {
+    }
+
+    virtual std::vector<SequenceDataPtr> GetSequence(const size_t& sequenceId) override
+    {
+        assert(sequenceId == m_description.m_id);
+        UNREFERENCED_PARAMETER(sequenceId);
+        const auto& imageSequence = m_description;
+
+        auto image = std::make_shared<ImageSequenceData>();
+        image->m_image = std::move(cv::imread(imageSequence.m_path, cv::IMREAD_COLOR));
+        auto& cvImage = image->m_image;
+
+        if (!cvImage.data)
+        {
+            RuntimeError("Cannot open file '%s'", imageSequence.m_path.c_str());
+        }
+
+        // Convert element type.
+        int dataType = m_parent.m_featureElementType == ElementType::tfloat ? CV_32F : CV_64F;
+        if (cvImage.type() != CV_MAKETYPE(dataType, cvImage.channels()))
+        {
+            cvImage.convertTo(cvImage, dataType);
+        }
+
+        if (!cvImage.isContinuous())
+        {
+            cvImage = cvImage.clone();
+        }
+        assert(cvImage.isContinuous());
+
+        image->m_data = image->m_image.data;
+        ImageDimensions dimensions(cvImage.cols, cvImage.rows, cvImage.channels());
+        image->m_sampleLayout = std::make_shared<TensorShape>(dimensions.AsTensorShape(HWC));
+        image->m_numberOfSamples = 1;
+        image->m_chunk = shared_from_this();
+
+        SparseSequenceDataPtr label = std::make_shared<SparseSequenceData>();
+        label->m_chunk = shared_from_this();
+        m_parent.m_labelGenerator->CreateLabelFor(imageSequence.m_classId, *label);
+        return std::vector<SequenceDataPtr> { image, label };
+    }
+
+    ~ImageChunk()
+    {
+    }
+};
+
+ChunkPtr ImageDataDeserializer::GetChunk(size_t chunkId)
+{
+    auto sequenceDescription = m_imageSequences[chunkId];
+    return std::make_shared<ImageChunk>(sequenceDescription, *this);
+}
+
 void ImageDataDeserializer::CreateSequenceDescriptions(std::string mapPath, size_t labelDimension)
 {
     UNREFERENCED_PARAMETER(labelDimension);
@@ -127,72 +190,6 @@ std::vector<StreamDescriptionPtr> ImageDataDeserializer::GetStreamDescriptions()
     return m_streams;
 }
 
-std::vector<std::vector<SequenceDataPtr>> ImageDataDeserializer::GetSequencesById(const std::vector<size_t>& ids)
-{
-    if (ids.empty())
-    {
-        RuntimeError("Number of requested sequences cannot be zero.");
-    }
-
-    m_currentImages.resize(ids.size());
-    m_labels.resize(ids.size());
-
-    std::vector<std::vector<SequenceDataPtr>> result;
-    result.resize(ids.size());
-
-#pragma omp parallel for ordered schedule(dynamic)
-    for (int i = 0; i < ids.size(); ++i)
-    {
-        if (ids[i] >= m_imageSequences.size())
-        {
-            RuntimeError("Invalid sequence id is provided '%d', expected range [0..%d].",
-                         static_cast<int>(ids[i]),
-                         static_cast<int>(m_imageSequences.size()) - 1);
-        }
-
-        const auto& imageSequence = m_imageSequences[ids[i]];
-
-        // Construct image
-        m_currentImages[i] = std::move(cv::imread(imageSequence.m_path, cv::IMREAD_COLOR));
-        cv::Mat& cvImage = m_currentImages[i];
-
-        if (!cvImage.data)
-        {
-            RuntimeError("Cannot open file '%s'", imageSequence.m_path.c_str());
-        }
-
-        // Convert element type.
-        // TODO We should all native CV element types to be able to match the behavior of the old reader.
-        int dataType = m_featureElementType == ElementType::tfloat ? CV_32F : CV_64F;
-        if (cvImage.type() != CV_MAKETYPE(dataType, cvImage.channels()))
-        {
-            cvImage.convertTo(cvImage, dataType);
-        }
-
-        if (!cvImage.isContinuous())
-        {
-            cvImage = cvImage.clone();
-        }
-        assert(cvImage.isContinuous());
-
-        ImageDimensions dimensions(cvImage.cols, cvImage.rows, cvImage.channels());
-        auto image = std::make_shared<DenseSequenceData>();
-        image->m_data = cvImage.data;
-        image->m_sampleLayout = std::make_shared<TensorShape>(dimensions.AsTensorShape(HWC));
-        image->m_numberOfSamples = 1;
-
-        if (m_labels[i] == nullptr)
-        {
-            m_labels[i] = std::make_shared<SparseSequenceData>();
-        }
-
-        m_labelGenerator->CreateLabelFor(imageSequence.m_classId, *m_labels[i]);
-        result[i] = std::move(std::vector<SequenceDataPtr>{image, m_labels[i]});
-    }
-
-    return result;
-}
-
 void ImageDataDeserializer::FillSequenceDescriptions(SequenceDescriptions& timeline) const
 {
     timeline.resize(m_imageSequences.size());

Source/Readers/ImageReader/ImageDataDeserializer.h

@@ -24,7 +24,7 @@ class ImageDataDeserializer : public DataDeserializerBase
     std::vector<StreamDescriptionPtr> GetStreamDescriptions() const override;
 
     // Get sequences by specified ids. Order of returned sequences corresponds to the order of provided ids.
-    std::vector<std::vector<SequenceDataPtr>> GetSequencesById(const std::vector<size_t>& ids) override;
+    virtual ChunkPtr GetChunk(size_t chunkId) override;
 
 protected:
     void FillSequenceDescriptions(SequenceDescriptions& timeline) const override;
@@ -40,6 +40,8 @@ class ImageDataDeserializer : public DataDeserializerBase
         size_t m_classId;
     };
 
+    class ImageChunk;
+
     // A helper class for generation of type specific labels (currently float/double only).
     class LabelGenerator;
     typedef std::shared_ptr<LabelGenerator> LabelGeneratorPtr;
@@ -48,12 +50,6 @@ class ImageDataDeserializer : public DataDeserializerBase
     // Sequence descriptions for all input data.
     std::vector<ImageSequenceDescription> m_imageSequences;
 
-    // Buffer to store label data.
-    std::vector<SparseSequenceDataPtr> m_labels;
-
-    // Buffer to store feature data.
-    std::vector<cv::Mat> m_currentImages;
-
     // Element type of the feature/label stream (currently float/double only).
     ElementType m_featureElementType;
 };

Source/Readers/ImageReader/ImageReader.vcxproj

@@ -113,6 +113,7 @@
     <ClInclude Include="ImageConfigHelper.h" />
     <ClInclude Include="ImageDataDeserializer.h" />
     <ClInclude Include="ImageReader.h" />
+    <ClInclude Include="ImageSequence.h" />
     <ClInclude Include="ImageTransformers.h" />
     <ClInclude Include="stdafx.h" />
     <ClInclude Include="targetver.h" />
@@ -150,4 +151,4 @@
   <Target Name="CheckDependencies">
     <Warning Condition="!$(HasOpenCV)" Text="ImageReader requires the OpenCV library to build. Please see https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows#opencv for installation instructions." />
   </Target>
-</Project>
+</Project>

Source/Readers/ImageReader/ImageReader.vcxproj.filters

@@ -43,6 +43,7 @@
     <ClInclude Include="ImageDataDeserializer.h" />
     <ClInclude Include="ImageReader.h" />
     <ClInclude Include="ImageConfigHelper.h" />
+    <ClInclude Include="ImageSequence.h" />
   </ItemGroup>
   <ItemGroup>
     <Filter Include="Common">

Source/Readers/ImageReader/ImageSequence.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <opencv2/core/mat.hpp>
+#include "DataDeserializer.h"
+
+namespace Microsoft { namespace MSR { namespace CNTK {
+
+    // Used to keep track of the image. Used in the implementations of open cv transformers 
+    // and deserializer. Accessed only using DenseSequenceData interface.
+    struct ImageSequenceData : DenseSequenceData
+    {
+        cv::Mat m_image;
+
+        // In case we do not copy data - we preserve original sequence.
+        SequenceDataPtr m_original;
+    };
+}}}

Source/Readers/ImageReader/ImageTransformers.cpp

@@ -14,6 +14,7 @@
 #include "ImageConfigHelper.h"
 #include "StringUtil.h"
 #include "ElementTypeUtils.h"
+#include "ImageSequence.h"
 
 namespace Microsoft { namespace MSR { namespace CNTK
 {
@@ -34,10 +35,11 @@ void ImageTransformerBase::Initialize(TransformerPtr next,
 }
 
 SequenceDataPtr
-ImageTransformerBase::Apply(const DenseSequenceData &inputSequence,
-                            const StreamDescription &inputStream, cv::Mat &buffer,
+ImageTransformerBase::Apply(SequenceDataPtr sequence,
+                            const StreamDescription &inputStream,
                             const StreamDescription & /*outputStream*/)
 {
+    auto inputSequence = reinterpret_cast<const DenseSequenceData&>(*sequence.get());
     ImageDimensions dimensions(*inputSequence.m_sampleLayout, HWC);
     int columns = static_cast<int>(dimensions.m_width);
     int rows = static_cast<int>(dimensions.m_height);
@@ -57,20 +59,25 @@ ImageTransformerBase::Apply(const DenseSequenceData &inputSequence,
         RuntimeError("Unsupported type");
     }
 
+    auto result = std::make_shared<ImageSequenceData>();
     int type = CV_MAKETYPE(typeId, channels);
-    buffer = cv::Mat(rows, columns, type, inputSequence.m_data);
+    cv::Mat buffer = cv::Mat(rows, columns, type, inputSequence.m_data);
     this->Apply(buffer);
     if (!buffer.isContinuous())
     {
         buffer = buffer.clone();
     }
+    else
+    {
+        result->m_original = sequence;
+    }
     assert(buffer.isContinuous());
+    result->m_image = buffer;
+    result->m_data = buffer.ptr();
+    result->m_numberOfSamples = inputSequence.m_numberOfSamples;
 
-    auto result = std::make_shared<DenseSequenceData>();
     ImageDimensions outputDimensions(buffer.cols, buffer.rows, buffer.channels());
     result->m_sampleLayout = std::make_shared<TensorShape>(outputDimensions.AsTensorShape(HWC));
-    result->m_numberOfSamples = inputSequence.m_numberOfSamples;
-    result->m_data = buffer.ptr();
     return result;
 }
 
@@ -367,7 +374,7 @@ void MeanTransformer::Apply(cv::Mat &mat)
 void TransposeTransformer::Initialize(TransformerPtr next,
                                       const ConfigParameters &readerConfig)
 {
-    Base::Initialize(next, readerConfig);
+    TransformerBase::Initialize(next, readerConfig);
 
     // Currently we only support a single stream.
     ImageConfigHelper config(readerConfig);
@@ -392,39 +399,46 @@ void TransposeTransformer::Initialize(TransformerPtr next,
 }
 
 SequenceDataPtr
-TransposeTransformer::Apply(const DenseSequenceData &inputSequence,
+TransposeTransformer::Apply(SequenceDataPtr inputSequence,
                             const StreamDescription &inputStream,
-                            vector<char> &buffer,
                             const StreamDescription &outputStream)
 {
     if (inputStream.m_elementType == ElementType::tdouble)
     {
-        return TypedApply<double>(inputSequence, inputStream, buffer, outputStream);
+        return TypedApply<double>(inputSequence, inputStream, outputStream);
     }
 
     if (inputStream.m_elementType == ElementType::tfloat)
     {
-        return TypedApply<float>(inputSequence, inputStream, buffer, outputStream);
+        return TypedApply<float>(inputSequence, inputStream, outputStream);
     }
 
     RuntimeError("Unsupported type");
 }
 
+struct OwnedDenseSequence : DenseSequenceData
+{
+    std::vector<char> m_buffer;
+};
+
 template <class TElement>
 SequenceDataPtr
-TransposeTransformer::TypedApply(const DenseSequenceData &inputSequence,
+TransposeTransformer::TypedApply(SequenceDataPtr sequence,
                                  const StreamDescription &inputStream,
-                                 vector<char> &buffer,
                                  const StreamDescription &outputStream)
 {
+    assert(inputStream.m_storageType == StorageType::dense);
+    auto inputSequence = reinterpret_cast<DenseSequenceData&>(*sequence.get());
     assert(inputSequence.m_numberOfSamples == 1);
     assert(inputStream.m_sampleLayout->GetNumElements() ==
         outputStream.m_sampleLayout->GetNumElements());
 
     size_t count = inputStream.m_sampleLayout->GetNumElements() * GetSizeByType(inputStream.m_elementType);
-    buffer.resize(count);
 
-    TElement* typedBuffer = reinterpret_cast<TElement*>(&buffer[0]);
+    auto result = std::make_shared<OwnedDenseSequence>();
+    result->m_buffer.resize(count);
+
+    TElement* typedBuffer = reinterpret_cast<TElement*>(&result->m_buffer[0]);
     ImageDimensions dimensions(*inputStream.m_sampleLayout, ImageLayoutKind::HWC);
 
     size_t rowCount = dimensions.m_height * dimensions.m_width;
@@ -441,9 +455,8 @@ TransposeTransformer::TypedApply(const DenseSequenceData &inputSequence,
         }
     }
 
-    auto result = std::make_shared<DenseSequenceData>();
     result->m_sampleLayout = outputStream.m_sampleLayout;
-    result->m_data = &buffer[0];
+    result->m_data = &result->m_buffer[0];
     result->m_numberOfSamples = inputSequence.m_numberOfSamples;
     return result;
 }