Skip to content

Commit

Permalink
Adds helper methods to the eval wrapper to enable exposing hidden lay…
Browse files Browse the repository at this point in the history 
…er outputs for evaluation
  • Loading branch information
@gaizkan
gaizkan committed Jun 1, 2016
1 parent 1e01cbe commit f36527f
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 28 deletions.
11 changes: 9 additions & 2 deletions Source/Extensibility/CPPEvalClient/CPPEvalClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,14 @@ int _tmain(int argc, _TCHAR* argv[])
const std::string modelWorkingDirectory = path + "\\..\\..\\Examples\\Image\\MNIST\\Data\\";
const std::string modelFilePath = modelWorkingDirectory + "..\\Output\\Models\\01_OneHidden";

// Load model
model->CreateNetwork("modelPath=\"" + modelFilePath + "\"");
// Load model with desired outputs
std::string networkConfiguration;
// Uncomment the following line to re-define the outputs (include h1.z AND the output ol.z)
// When specifying outputNodeNames in the configuration, it will REPLACE the list of output nodes
// with the ones specified.
//networkConfiguration += "outputNodeNames=\"h1.z:ol.z\"\n";
networkConfiguration += "modelPath=\"" + modelFilePath + "\"";
model->CreateNetwork(networkConfiguration);

// get the model's layers dimensions
std::map<std::wstring, size_t> inDims;
Expand Down Expand Up @@ -90,6 +96,7 @@ int _tmain(int argc, _TCHAR* argv[])
model->Evaluate(inputLayer, outputLayer);

// Output the results
fprintf(stderr, "Layer '%ls' output:\n", outputLayerName.c_str());
for each (auto& value in outputs)
{
fprintf(stderr, "%f\n", value);
Expand Down
64 changes: 41 additions & 23 deletions Source/Extensibility/CSEvalClient/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,27 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;

namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
{
/// <summary>
/// Program for demonstrating how to run model evaluations using the CLIWrapper
/// </summary>
/// <description>
/// This program is a managed client using the CLIWrapper to run the model evaluator in CTNK.
/// There are four cases shown in this program related to model loading/network creation and evaluation.
/// The first two use the trained model from one of the examples provided in the CNTK source code.
/// In order to run this program the model must already exist in the example. To create the model,
/// first run the example in <CNTK>/Examples/Image/MNIST. Once the model file 01_OneHidden is created,
/// you can run this client.
/// The last two cases show how to evaluate a network without first training the model. This is accomplished
/// by building the network and evaluating a single forward pass.
/// This program also shows how to obtaining the output results from the evaluation, either as the default output layer,
/// or by specifying one or more layers as outputs.
/// This program is a managed client using the CLIWrapper to run the model evaluator in CNTK.
/// There are four cases shown in this program related to model loading, network creation and evaluation.
///
/// EvaluateModelSingleLayer and EvaluateModelMultipleLayers
/// --------------------------------------------------------
/// These two cases require the 01_OneHidden model which is part of the <CNTK>/Examples/Image/MNIST example.
/// Refer to <see cref="https://github.com/Microsoft/CNTK/blob/master/Examples/Image/MNIST/README.md"/> for how to train
/// the model used in these examples.
///
/// EvaluateNetworkSingleLayer and EvaluateNetworkSingleLayerNoInput
/// ----------------------------------------------------------------
/// These two cases do not required a trained model (just the network description). These cases show how to extract values from a single forward-pass
/// without any input to the model.
/// </description>
class Program
{
Expand All @@ -44,7 +48,7 @@ private static void Main(string[] args)

Console.WriteLine("\n====== EvaluateModelMultipleLayers ========");
EvaluateModelMultipleLayers();

Console.WriteLine("\n====== EvaluateNetworkSingleLayer ========");
EvaluateNetworkSingleLayer();

Expand All @@ -58,6 +62,9 @@ private static void Main(string[] args)
/// <summary>
/// Evaluates a trained model and obtains a single layer output
/// </summary>
/// <remarks>
/// This example requires the 01_OneHidden trained model
/// </remarks>
private static void EvaluateModelSingleLayer()
{
try
Expand All @@ -73,12 +80,12 @@ private static void EvaluateModelSingleLayer()
{
// Load model
string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\01_OneHidden");
model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId:-1);
model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId: -1);

// Generate random input values in the appropriate structure and size
var inDims = model.GetNodeDimensions(NodeGroup.nodeInput);
var inputs = GetDictionary(inDims.First().Key, inDims.First().Value, 255);

// We request the output layer names(s) and dimension, we'll use the first one.
var outDims = model.GetNodeDimensions(NodeGroup.nodeOutput);
outputLayerName = outDims.First().Key;
Expand All @@ -99,8 +106,11 @@ private static void EvaluateModelSingleLayer()
}

/// <summary>
/// Evaluates a trained model and obtains multiple layers output
/// Evaluates a trained model and obtains multiple layers output (including hidden layer)
/// </summary>
/// <remarks>
/// This example requires the 01_OneHidden trained model
/// </remarks>
private static void EvaluateModelMultipleLayers()
{
try
Expand All @@ -113,20 +123,28 @@ private static void EvaluateModelMultipleLayers()

using (var model = new IEvaluateModelManagedF())
{
// Desired output layers
string hiddenLayerName = "h1.z";
string outputLayerName = "ol.z";

// Load model
string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\01_OneHidden");
model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId:-1);
List<string> desiredOutputLayers = new List<string>() { hiddenLayerName, outputLayerName };
model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId: -1, outputNodeNames: desiredOutputLayers);

// Generate random input values in the appropriate structure and size
var inDims = model.GetNodeDimensions(NodeGroup.nodeInput);
var inputs = GetDictionary(inDims.First().Key, inDims.First().Value, 255);

// We request the output layer names(s) and dimension, we'll use the first one.
// We request the output layer names(s) and dimension, we'll get both the hidden layer and the output layer
var outDims = model.GetNodeDimensions(NodeGroup.nodeOutput);
string outputLayerName = outDims.First().Key;

// We can preallocate the output structure and pass it in (multiple output layers)
outputs = GetDictionary(outputLayerName, outDims[outputLayerName], 1);
outputs = new Dictionary<string, List<float>>()
{
{ hiddenLayerName, GetFloatArray(outDims[hiddenLayerName], 1) },
{ outputLayerName, GetFloatArray(outDims[outputLayerName], 1) }
};
model.Evaluate(inputs, outputs);
}

Expand All @@ -143,7 +161,7 @@ private static void EvaluateModelMultipleLayers()
}

/// <summary>
/// Evaluates a network (without a model) and obtains a single layer output
/// Evaluates a network (without a model, but requiring input) and obtains a single layer output
/// </summary>
private static void EvaluateNetworkSingleLayer()
{
Expand All @@ -163,12 +181,12 @@ private static void EvaluateNetworkSingleLayer()
// This network (AddOperatorConstant.cntk) is a simple network consisting of a single binary operator (Plus)
// operating over a single input and a constant
string networkDescription = File.ReadAllText(Path.Combine(workingDirectory, @"AddOperatorConstant.cntk"));
model.CreateNetwork(networkDescription, deviceId:-1);
model.CreateNetwork(networkDescription, deviceId: -1);

// Generate random input value in the appropriate structure and size
// Prepare input value in the appropriate structure and size
var inputs = new Dictionary<string, List<float>>() { { "features", new List<float>() { 1.0f } } };

// We can call the evaluate method and get back the results (single layer)...
// We can call the evaluate method and get back the results (single layer output)...
var outDims = model.GetNodeDimensions(NodeGroup.nodeOutput);
outputLayerName = outDims.First().Key;
outputs = model.Evaluate(inputs, outputLayerName);
Expand Down Expand Up @@ -206,7 +224,7 @@ private static void EvaluateNetworkSingleLayerNoInput()
// This network (AddOperatorConstantNoInput.cntk) is a simple network consisting of a single binary operator (Plus)
// operating over a two constants, therefore no input is necessary.
string networkDescription = File.ReadAllText(Path.Combine(workingDirectory, @"AddOperatorConstantNoInput.cntk"));
model.CreateNetwork(networkDescription, deviceId:-1);
model.CreateNetwork(networkDescription, deviceId: -1);

// We can call the evaluate method and get back the results (single layer)...
outputs = model.Evaluate("ol", 1);
Expand Down
41 changes: 38 additions & 3 deletions Source/Extensibility/EvalWrapper/EvalWrapper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public ref class IEvaluateModelManaged : IDisposable
}
}

/// <summary>Creates a network based from the network description in the configuration</summary>
/// <summary>Creates a network based on the network description in the configuration</summary>
/// <param name="networkDescription">The configuration file containing the network description</param>
void CreateNetwork(String^ networkDescription)
{
Expand All @@ -118,7 +118,22 @@ public ref class IEvaluateModelManaged : IDisposable
}
}

/// <summary>Creates a network based from the network description in the configuration</summary>
/// <summary>Creates a network based on the network description in the configuration</summary>
/// <param name="networkDescription">The configuration file containing the network description</param>
/// <param name="outputNodeNames">The output list of nodes (replaces the model's list of output nodes)</param>
void CreateNetwork(String^ networkDescription, List<String^>^ outputNodeNames)
{
if (m_eval == nullptr)
{
throw gcnew ObjectDisposedException("Object has been disposed.");
}

String^ outputNodeNamesProperty = outputNodeNames != nullptr ? String::Concat("outputNodeNames=", String::Join(":", outputNodeNames)) : "";
String^ newNetworkConfig = String::Format("{0}\n{1}", outputNodeNamesProperty, networkDescription);
this->CreateNetwork(newNetworkConfig);
}

/// <summary>Creates a network based on the network description in the configuration</summary>
/// <param name="networkDescription">The configuration file containing the network description</param>
/// <param name="deviceId">The device ID to specify for the network</param>
void CreateNetwork(String^ networkDescription, int deviceId)
Expand All @@ -128,7 +143,23 @@ public ref class IEvaluateModelManaged : IDisposable
throw gcnew ObjectDisposedException("Object has been disposed.");
}

this->CreateNetwork(String::Format("deviceId={0}\n{1}", deviceId, networkDescription));
this->CreateNetwork(networkDescription, deviceId, nullptr);
}

/// <summary>Creates a network based on the network description in the configuration</summary>
/// <param name="networkDescription">The configuration file containing the network description</param>
/// <param name="deviceId">The device ID to specify for the network</param>
/// <param name="outputNodeNames">The output list of nodes (replaces the model's list of output nodes)</param>
void CreateNetwork(String^ networkDescription, int deviceId, List<String^>^ outputNodeNames)
{
if (m_eval == nullptr)
{
throw gcnew ObjectDisposedException("Object has been disposed.");
}

String^ outputNodeNamesProperty = outputNodeNames != nullptr ? String::Concat("outputNodeNames=", String::Join(":", outputNodeNames)) : "";
String^ newNetworkConfig = String::Format("deviceId={0}\n{1}\n{2}", deviceId, outputNodeNamesProperty, networkDescription);
this->CreateNetwork(newNetworkConfig);
}

/// <summary>Evaluates the model using a single forward feed pass and retrieves the output layer data</summary>
Expand Down Expand Up @@ -594,6 +625,8 @@ void emit()
f.Evaluate("");
f.CreateNetwork("");
f.CreateNetwork("", 0);
f.CreateNetwork("", nullptr);
f.CreateNetwork("", 0, nullptr);
f.GetNodeDimensions(NodeGroup::nodeSpecified);

IEvaluateModelManagedD d;
Expand All @@ -603,6 +636,8 @@ void emit()
d.Evaluate("");
d.CreateNetwork("");
d.CreateNetwork("", 0);
d.CreateNetwork("", nullptr);
d.CreateNetwork("", 0,nullptr);
d.GetNodeDimensions(NodeGroup::nodeSpecified);

// Deprecated code, hush warnings locally only
Expand Down

0 comments on commit f36527f

Please sign in to comment.