Adding dependencies on the composite reader

nikoma · May 2, 2016 · 2256785 · 2256785
1 parent cae7a76
commit 2256785
Show file tree

Hide file tree

Showing 17 changed files with 84 additions and 62 deletions.
diff --git a/Source/CNTK/CNTK.vcxproj b/Source/CNTK/CNTK.vcxproj
@@ -148,6 +148,7 @@
     <ClInclude Include="..\Common\Include\Basics.h" />
     <ClInclude Include="..\Common\Include\BestGpu.h" />
     <ClInclude Include="..\Common\Include\DataReader.h" />
+    <ClInclude Include="..\Common\Include\CompositeDataReader.h" />
     <ClInclude Include="..\Common\Include\ExceptionWithCallStack.h" />
     <ClInclude Include="..\Common\Include\StringUtil.h" />
     <ClInclude Include="..\Common\Include\TensorShape.h" />
@@ -192,7 +193,6 @@
     <ClInclude Include="BrainScript\BrainScriptParser.h" />
     <ClInclude Include="..\ComputationNetworkLib\InputAndParamNodes.h" />
     <ClInclude Include="..\ComputationNetworkLib\LinearAlgebraNodes.h" />
-    <ClInclude Include="CompositeDataReader.h" />
     <ClInclude Include="ModelEditLanguage.h" />
     <ClInclude Include="..\ComputationNetworkLib\NonlinearityNodes.h" />
     <ClInclude Include="..\ComputationNetworkLib\RecurrentNodes.h" />
@@ -204,7 +204,6 @@
     <ClCompile Include="BrainScript\BrainScriptParser.cpp" />
     <ClCompile Include="BrainScript\BrainScriptTest.cpp" />
     <ClCompile Include="CNTK.cpp" />
-    <ClCompile Include="CompositeDataReader.cpp" />
     <ClCompile Include="ModelEditLanguage.cpp" />
     <ClCompile Include="stdafx.cpp" />
     <ClCompile Include="tests.cpp" />

diff --git a/Source/CNTK/CNTK.vcxproj.filters b/Source/CNTK/CNTK.vcxproj.filters
@@ -47,7 +47,7 @@
     <ClCompile Include="..\Common\ExceptionWithCallStack.cpp">
       <Filter>Common</Filter>
     </ClCompile>
-    <ClCompile Include="CompositeDataReader.cpp">
+    <ClCompile Include="..\Common\CompositeDataReader.cpp">
       <Filter>Common</Filter>
     </ClCompile>
   </ItemGroup>
@@ -169,9 +169,6 @@
     <ClInclude Include="..\Common\Include\ExceptionWithCallStack.h">
       <Filter>Common\Include</Filter>
     </ClInclude>
-    <ClInclude Include="CompositeDataReader.h">
-      <Filter>Common\Include</Filter>
-    </ClInclude>
     <ClInclude Include="..\Readers\ReaderLib\BlockRandomizer.h">
       <Filter>from ReaderLib</Filter>
     </ClInclude>
@@ -214,6 +211,9 @@
     <ClInclude Include="..\Readers\ReaderLib\TransformerBase.h">
       <Filter>from ReaderLib</Filter>
     </ClInclude>
+    <ClInclude Include="..\Common\Include\CompositeDataReader.h">
+      <Filter>Common\Include</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Text Include="modelEditor.txt">

diff --git a/Source/CNTK/CompositeDataReader.cpp → Source/Common/CompositeDataReader.cpp b/Source/CNTK/CompositeDataReader.cpp → Source/Common/CompositeDataReader.cpp
@@ -20,16 +20,19 @@
 #include "SampleModePacker.h"
 #include "SequencePacker.h"
 #include "HeapMemoryProvider.h"
+#include "DataDeserializer.h"
+#include "Transformer.h"
+#include "Reader.h"
+#include "Packer.h"
 
 namespace Microsoft { namespace MSR { namespace CNTK {
 
-template <class ElemType>
-CompositeDataReader<ElemType>::CompositeDataReader() : m_layout(make_shared<MBLayout>())
+CompositeDataReader::CompositeDataReader(const std::string& precision) : m_layout(make_shared<MBLayout>()),
+    m_precision(precision)
 {
 }
 
-template <class ElemType>
-void CompositeDataReader<ElemType>::Init(const ConfigParameters& config)
+void CompositeDataReader::Init(const ConfigParameters& config)
 {
     intargvector numberOfuttsPerMinibatchForAllEpochs =
         config(L"nbruttsineachrecurrentiter", ConfigParameters::Array(intargvector(vector<int> { 1 })));
@@ -81,14 +84,12 @@ void CompositeDataReader<ElemType>::Init(const ConfigParameters& config)
     }
 }
 
-template <class ElemType>
-void CompositeDataReader<ElemType>::StartMinibatchLoop(size_t mbSize, size_t epoch, size_t requestedEpochSamples)
+void CompositeDataReader::StartMinibatchLoop(size_t mbSize, size_t epoch, size_t requestedEpochSamples)
 {
     return StartDistributedMinibatchLoop(mbSize, epoch, 0, 1, requestedEpochSamples);
 }
 
-template <class ElemType>
-void CompositeDataReader<ElemType>::StartDistributedMinibatchLoop(
+void CompositeDataReader::StartDistributedMinibatchLoop(
     size_t requestedMBSize,
     size_t epoch,
     size_t subsetNum,
@@ -120,8 +121,7 @@ void CompositeDataReader<ElemType>::StartDistributedMinibatchLoop(
     });
 }
 
-template <class ElemType>
-bool CompositeDataReader<ElemType>::GetMinibatch(StreamMinibatchInputs& matrices)
+bool CompositeDataReader::GetMinibatch(StreamMinibatchInputs& matrices)
 {
     if (m_endOfEpoch)
     {
@@ -166,8 +166,17 @@ bool CompositeDataReader<ElemType>::GetMinibatch(StreamMinibatchInputs& matrices
             size_t columnNumber = m_layout->GetNumCols();
             size_t rowNumber = m_streams[streamId]->m_sampleLayout->GetNumElements();
 
-            auto* data = reinterpret_cast<const ElemType*>(stream->m_data);
-            matrices.GetInputMatrix<ElemType>(mx.first).SetValue(rowNumber, columnNumber, mx.second->GetDeviceId(), const_cast<ElemType*>(data), matrixFlagNormal);
+            if (m_precision == "float")
+            {
+                auto* data = reinterpret_cast<const float*>(stream->m_data);
+                matrices.GetInputMatrix<float>(mx.first).SetValue(rowNumber, columnNumber, mx.second->GetDeviceId(), const_cast<float*>(data), matrixFlagNormal);
+            }
+            else
+            {
+                assert(m_precision == "double");
+                auto* data = reinterpret_cast<const double*>(stream->m_data);
+                matrices.GetInputMatrix<double>(mx.first).SetValue(rowNumber, columnNumber, mx.second->GetDeviceId(), const_cast<double*>(data), matrixFlagNormal);
+            }
         }
     }
 
@@ -179,27 +188,23 @@ bool CompositeDataReader<ElemType>::GetMinibatch(StreamMinibatchInputs& matrices
     return !minibatch.m_data.empty();
 }
 
-template <class ElemType>
-bool CompositeDataReader<ElemType>::DataEnd()
+bool CompositeDataReader::DataEnd()
 {
     // Note: Return value never used.
     return false;
 }
 
-template <class ElemType>
-void CompositeDataReader<ElemType>::CopyMBLayoutTo(MBLayoutPtr layout)
+void CompositeDataReader::CopyMBLayoutTo(MBLayoutPtr layout)
 {
     layout->CopyFrom(m_layout);
 }
 
-template <class ElemType>
-size_t CompositeDataReader<ElemType>::GetNumParallelSequences()
+size_t CompositeDataReader::GetNumParallelSequences()
 {
     return m_layout->GetNumParallelSequences();
 }
 
-template <class ElemType>
-void CompositeDataReader<ElemType>::CreateDeserializers(const ConfigParameters& readerConfig)
+void CompositeDataReader::CreateDeserializers(const ConfigParameters& readerConfig)
 {
     argvector<ConfigValue> deserializerConfigs =
         readerConfig(L"deserializers", ConfigParameters::Array(argvector<ConfigValue>(vector<ConfigValue> {})));
@@ -212,15 +217,13 @@ void CompositeDataReader<ElemType>::CreateDeserializers(const ConfigParameters&
     }
 }
 
-template <class ElemType>
-IDataDeserializerPtr CompositeDataReader<ElemType>::CreateDeserializer(const ConfigParameters& deserializerConfig)
+IDataDeserializerPtr CompositeDataReader::CreateDeserializer(const ConfigParameters& deserializerConfig)
 {
     UNUSED(deserializerConfig);
     return nullptr;
 }
 
-template <class ElemType>
-void CompositeDataReader<ElemType>::StartEpoch(const EpochConfiguration& config)
+void CompositeDataReader::StartEpoch(const EpochConfiguration& config)
 {
     if (config.m_totalEpochSizeInSamples <= 0)
     {
@@ -251,8 +254,5 @@ void CompositeDataReader<ElemType>::StartEpoch(const EpochConfiguration& config)
     }
 }
 
-template class CompositeDataReader<float>;
-template class CompositeDataReader<double>;
-
 }}}
 
diff --git a/Source/Common/DataReader.cpp b/Source/Common/DataReader.cpp
@@ -16,6 +16,7 @@
 #include "Config.h"
 #include "ScriptableObjects.h"
 #include <string>
+#include "CompositeDataReader.h"
 
 using namespace std;
 
@@ -89,6 +90,7 @@ DataReader::DataReader(const ConfigRecordType& config)
     string precision = config(L"precision", "float");
 
     bool hasMultipleReaders = config.Exists(L"readers");
+    bool hasDeserializers = config.Exists(L"deserializers");
     if (hasMultipleReaders)
     {
         vector<wstring> ioNames = config(L"readers", ConfigRecordType::Array(stringargvector()));
@@ -103,6 +105,12 @@ DataReader::DataReader(const ConfigRecordType& config)
             getReaderProc(&m_dataReaders[ioName]); // instantiates the reader with the default constructor (no config processed at this point)
         }
     }
+    else if (hasDeserializers)
+    {
+        wstring ioName = L"ioName";
+        m_ioNames.push_back(ioName);
+        m_dataReaders[ioName] = new CompositeDataReader(precision);
+    }
     else // legacy
     {
         wstring ioName = L"ioName";

diff --git a/Source/CNTK/CompositeDataReader.h → Source/Common/Include/CompositeDataReader.h b/Source/CNTK/CompositeDataReader.h → Source/Common/Include/CompositeDataReader.h
@@ -9,21 +9,34 @@
 #include <string>
 #include <future>
 #include "DataReader.h"
-#include "DataDeserializer.h"
-#include "Transformer.h"
-#include "Reader.h"
-#include "Packer.h"
 
 namespace Microsoft { namespace MSR { namespace CNTK {
 
+class IDataDeserializer;
+typedef std::shared_ptr<IDataDeserializer> IDataDeserializerPtr;
+
+class Transformer;
+typedef std::shared_ptr<Transformer> TransformerPtr;
+
+class Packer;
+typedef std::shared_ptr<Packer> PackerPtr;
+
+class MemoryProvider;
+typedef std::shared_ptr<MemoryProvider> MemoryProviderPtr;
+
+struct StreamDescription;
+typedef std::shared_ptr<StreamDescription> StreamDescriptionPtr;
+
+struct EpochConfiguration;
+struct Minibatch;
+
 // TODO: Temporary shim for the new readers, will be removed and responsibilities will be moved to different parts of CNTK.
 // TODO: Currently binds together several deserializers, packer and randomizer. So that the actual reader developer has to provide deserializer only.
 // TODO: Same code as in ReaderLib shim, the one in the ReaderLib will be deleted as the next step.
-template <class ElemType>
 class CompositeDataReader : public IDataReader, protected Plugin, public ScriptableObjects::Object
 {
 public:
-    CompositeDataReader();
+    CompositeDataReader(const std::string& precision);
 
     // Currently we do not support BS configuration.
     virtual void Init(const ScriptableObjects::IConfigRecord& /*config*/) override
@@ -76,6 +89,8 @@ class CompositeDataReader : public IDataReader, protected Plugin, public Scripta
 
     PackerPtr m_packer;
 
+    std::string m_precision;
+
     bool m_frameMode;
     bool m_truncated;
 };

diff --git a/Source/Readers/BinaryReader/BinaryReader.vcxproj.filters b/Source/Readers/BinaryReader/BinaryReader.vcxproj.filters
@@ -10,9 +10,6 @@
     <ClCompile Include="..\..\Common\DataWriter.cpp">
       <Filter>Common</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\Common\DataReader.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Common\fileutil.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj.filters b/Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj.filters
@@ -3,9 +3,6 @@
   <ItemGroup>
     <ClCompile Include="Exports.cpp" />
     <ClCompile Include="stdafx.cpp" />
-    <ClCompile Include="..\..\Common\DataReader.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Common\ExceptionWithCallStack.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Source/Readers/DSSMReader/DSSMReader.vcxproj b/Source/Readers/DSSMReader/DSSMReader.vcxproj
@@ -111,7 +111,6 @@
     <ClInclude Include="targetver.h" />
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\Common\DataReader.cpp" />
     <ClCompile Include="..\..\Common\DataWriter.cpp">
       <PrecompiledHeader Condition="$(DebugBuild)">NotUsing</PrecompiledHeader>
     </ClCompile>

diff --git a/Source/Readers/DSSMReader/DSSMReader.vcxproj.filters b/Source/Readers/DSSMReader/DSSMReader.vcxproj.filters
@@ -17,9 +17,6 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\Common\DataReader.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Common\DataWriter.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Source/Readers/ImageReader/ImageReader.vcxproj b/Source/Readers/ImageReader/ImageReader.vcxproj
@@ -146,4 +146,4 @@ if "$(UseZip)" == "true" if exist "$(ZLIB_PATH)\bin\zlib1.dll" (xcopy /I /D /Y "
     <Warning Condition="!$(HasOpenCV)" Text="ImageReader requires the OpenCV library to build. Please see https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows#opencv for installation instructions." />
     <Warning Condition="!$(UseZip)" Text="zlib and libzip libraries were not found, ImageReader will be built without zip container support. Please see https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows#libzip for installation instructions." />
   </Target>
-</Project>
+</Project>
diff --git a/Source/Readers/ImageReader/ImageReader.vcxproj.filters b/Source/Readers/ImageReader/ImageReader.vcxproj.filters
@@ -3,9 +3,6 @@
   <ItemGroup>
     <ClCompile Include="Exports.cpp" />
     <ClCompile Include="stdafx.cpp" />
-    <ClCompile Include="..\..\Common\DataReader.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Common\fileutil.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj.filters b/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj.filters
@@ -7,9 +7,6 @@
     <ClCompile Include="stdafx.cpp" />
     <ClCompile Include="LUSequenceReader.cpp" />
     <ClCompile Include="LUSequenceParser.cpp" />
-    <ClCompile Include="..\..\Common\DataReader.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Common\fileutil.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj.filters b/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj.filters
@@ -13,9 +13,6 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\Common\DataReader.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Common\DataWriter.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Source/Readers/SparsePCReader/SparsePCReader.vcxproj.filters b/Source/Readers/SparsePCReader/SparsePCReader.vcxproj.filters
@@ -21,9 +21,6 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\Common\DataReader.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Common\DataWriter.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Source/Readers/UCIFastReader/UCIFastReader.vcxproj.filters b/Source/Readers/UCIFastReader/UCIFastReader.vcxproj.filters
@@ -9,6 +9,9 @@
     <ClCompile Include="..\..\Common\DataReader.cpp">
       <Filter>Common</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\Common\CompositeDataReader.cpp">
+      <Filter>Common</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\Common\DataWriter.cpp">
       <Filter>Common</Filter>
     </ClCompile>

diff --git a/Tests/UnitTests/EvalTest/EvalTest.vcxproj b/Tests/UnitTests/EvalTest/EvalTest.vcxproj
@@ -100,6 +100,22 @@
     <ClInclude Include="targetver.h" />
   </ItemGroup>
   <ItemGroup>
+<<<<<<< HEAD
+=======
+    <ClCompile Include="..\..\..\Source\Common\Config.cpp">
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="..\..\..\Source\Common\DataReader.cpp" />
+    <ClCompile Include="..\..\..\Source\Common\CompositeDataReader.cpp" />
+    <ClCompile Include="..\..\..\Source\Common\Eval.cpp" />
+    <ClCompile Include="..\..\..\Source\Common\ExceptionWithCallStack.cpp" />
+    <ClCompile Include="..\..\..\Source\Common\File.cpp">
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="..\..\..\Source\Common\fileutil.cpp">
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+    </ClCompile>
+>>>>>>> Adding dependencies on the composite reader
     <ClCompile Include="CNTKEvalTest.cpp" />
     <ClCompile Include="stdafx.cpp">
       <PrecompiledHeader>Create</PrecompiledHeader>

diff --git a/Tests/UnitTests/NetworkTests/NetworkTests.vcxproj.filters b/Tests/UnitTests/NetworkTests/NetworkTests.vcxproj.filters
@@ -19,6 +19,9 @@
     <ClCompile Include="..\..\..\Source\Common\DataReader.cpp">
       <Filter>Common</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\Source\Common\CompositeDataReader.cpp">
+      <Filter>Common</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\Source\Common\DataWriter.cpp">
       <Filter>Common</Filter>
     </ClCompile>