Added more unit tests, merged changes from master.

Source/Readers/ImageReader/ByteReader.h

@@ -0,0 +1,56 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+//
+
+#pragma once
+#include <opencv2/core/mat.hpp>
+#include "Config.h"
+#ifdef USE_ZIP
+#include <zip.h>
+#include <unordered_map>
+#include "ConcStack.h"
+#endif
+
+namespace Microsoft { namespace MSR { namespace CNTK {
+
+class ByteReader
+{
+public:
+    ByteReader() = default;
+    virtual ~ByteReader() = default;
+
+    virtual void Register(size_t seqId, const std::string& path) = 0;
+    virtual cv::Mat Read(size_t seqId, const std::string& path) = 0;
+
+    ByteReader(const ByteReader&) = delete;
+    ByteReader& operator=(const ByteReader&) = delete;
+    ByteReader(ByteReader&&) = delete;
+    ByteReader& operator=(ByteReader&&) = delete;
+};
+
+class FileByteReader : public ByteReader
+{
+public:
+    void Register(size_t, const std::string&) override {}
+    cv::Mat Read(size_t seqId, const std::string& path) override;
+};
+
+#ifdef USE_ZIP
+class ZipByteReader : public ByteReader
+{
+public:
+    ZipByteReader(const std::string& zipPath);
+    ~ZipByteReader();
+
+    void Register(size_t seqId, const std::string& path) override;
+    cv::Mat Read(size_t seqId, const std::string& path) override;
+
+private:
+    zip* m_zip;
+    std::unordered_map<size_t, std::pair<zip_uint64_t, zip_uint64_t>> m_seqIdToIndex;
+    conc_stack<std::vector<unsigned char>> m_workspace;
+};
+#endif
+
+}}}

Source/Readers/ImageReader/ImageDataDeserializer.cpp

@@ -66,7 +66,7 @@ class ImageDataDeserializer::ImageChunk : public Chunk, public std::enable_share
         const auto& imageSequence = m_description;
 
         auto image = std::make_shared<DeserializedImage>();
-        image->m_image = std::move(ReadImage(m_description.m_id, imageSequence.m_path));
+        image->m_image = std::move(m_parent.ReadImage(m_description.m_id, imageSequence.m_path));
         auto& cvImage = image->m_image;
 
         if (!cvImage.data)

Source/Readers/ImageReader/ZipByteReader.cpp

@@ -0,0 +1,85 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+//
+
+#include "stdafx.h"
+#include <opencv2/opencv.hpp>
+#include "ByteReader.h"
+
+#ifdef USE_ZIP
+
+namespace Microsoft { namespace MSR { namespace CNTK {
+
+std::string GetZipError(int err)
+{
+    zip_error_t error;
+    zip_error_init_with_code(&error, err);
+    std::string errS(zip_error_strerror(&error));
+    zip_error_fini(&error);
+    return errS;
+}
+
+ZipByteReader::ZipByteReader(const std::string& zipPath)
+    : m_zip(nullptr)
+{
+    assert(!zipPath.empty());
+    int err = 0;
+    m_zip = zip_open(zipPath.c_str(), 0, &err);
+    if (ZIP_ER_OK != err)
+        RuntimeError("Failed to open %s, zip library error: %s", zipPath.c_str(), GetZipError(err).c_str());
+}
+
+ZipByteReader::~ZipByteReader()
+{
+    if (m_zip != nullptr)
+        zip_close(m_zip);
+}
+
+void ZipByteReader::Register(size_t seqId, const std::string& path)
+{
+    struct zip_stat stat;
+    zip_stat_init(&stat);
+    int err = zip_stat(m_zip, path.c_str(), 0, &stat);
+    if (ZIP_ER_OK != err)
+        RuntimeError("Failed to get file info of %s, zip library error: %s", path.c_str(), GetZipError(err).c_str());
+    m_seqIdToIndex[seqId] = std::make_pair(stat.index, stat.size);
+}
+
+cv::Mat ZipByteReader::Read(size_t seqId, const std::string& path)
+{
+    // Find index of the file in .zip file.
+    auto r = m_seqIdToIndex.find(seqId);
+    if (r == m_seqIdToIndex.end())
+        RuntimeError("Could not find file %s in the zip file, sequence id = %lu", path.c_str(), (long)seqId);
+
+    zip_uint64_t index = std::get<0>((*r).second);
+    zip_uint64_t size = std::get<1>((*r).second);
+
+    auto contents = m_workspace.pop_or_create([size]() { return vector<unsigned char>(size); });
+    if (contents.size() < size)
+        contents.resize(size);
+    zip_file *file = zip_fopen_index(m_zip, index, 0);
+    assert(nullptr != file);
+    if (nullptr == file)
+    {
+        RuntimeError("Could not open file %s in the zip file, sequence id = %lu, zip library error: %s",
+                     path.c_str(), (long)seqId, GetZipError(zip_error_code_zip(zip_get_error(m_zip))).c_str());
+    }
+    assert(contents.size() >= size);
+    zip_uint64_t bytesRead = zip_fread(file, contents.data(), size);
+    assert(bytesRead == size);
+    if (bytesRead != size)
+    {
+        zip_fclose(file);
+        RuntimeError("Bytes read %lu != expected %lu while reading file %s",
+                     (long)bytesRead, (long)size, path.c_str());
+    }
+    zip_fclose(file);
+    cv::Mat img = cv::imdecode(cv::Mat(1, (int)size, CV_8UC1, contents.data()), cv::IMREAD_COLOR);
+    m_workspace.push(std::move(contents));
+    return img;
+}
+}}}
+
+#endif

Tests/UnitTests/ReaderTests/Config/ImageReaderZipMissing_Config.cntk

@@ -0,0 +1,39 @@
+RootDir = .
+ModelDir = "models"
+command = "ZipMissing_Test"
+
+precision = "float"
+
+modelPath = "$ModelDir$/ImageReaderZipMissing_Model.dnn"
+
+# deviceId = -1 for CPU, >= 0 for GPU devices
+deviceId = -1
+
+outputNodeNames = "Dummy"
+traceLevel = 1
+
+ZipMissing_Test = [
+    # Parameter values for the reader
+    reader = [
+        # reader to use
+        readerType = "ImageReader"
+        file = "$RootDir$/ImageReaderZipMissing_map.txt"
+
+        randomize = "auto"
+        verbosity = 1
+
+		numCPUThreads = 1
+        features=[
+            width=4
+            height=8
+            channels=3
+            cropType=Center
+            cropRatio=1.0
+            jitterType=UniRatio
+            interpolations=Linear
+        ]
+        labels=[
+            labelDim=4
+        ]
+    ]
+]

Tests/UnitTests/ReaderTests/Config/ImageReaderZip_Config.cntk

@@ -0,0 +1,40 @@
+RootDir = .
+ModelDir = "models"
+command = "Zip_Test"
+
+precision = "float"
+
+modelPath = "$ModelDir$/ImageReaderZip_Model.dnn"
+
+# deviceId = -1 for CPU, >= 0 for GPU devices
+deviceId = -1
+
+outputNodeNames = "Dummy"
+traceLevel = 1
+
+Zip_Test = [
+    # Parameter values for the reader
+    reader = [
+        # reader to use
+        readerType = "ImageReader"
+        file = "$RootDir$/ImageReaderZip_map.txt"
+
+        randomize = "auto"
+        verbosity = 1
+
+		numCPUThreads = 1
+        features=[
+            width=4
+            height=8
+            channels=3
+            cropType=Center
+            cropRatio=1.0
+            jitterType=UniRatio
+            interpolations=Linear
+            #meanFile=$RootDir$/ImageReaderZip_mean.xml
+        ]
+        labels=[
+            labelDim=4
+        ]
+    ]
+]