Merge remote-tracking branch 'origin/master' into nikosk/keras_support2

Examples/Evaluation/CNTKLibraryCSEvalCPUOnlyExamples/CNTKLibraryCSEvalCPUOnlyExamples.csproj

@@ -40,7 +40,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="CNTKLibraryManaged-2.0, Version=1.0.0.0, Culture=neutral, processorArchitecture=AMD64">
-      <HintPath>..\packages\CNTK.CPUOnly.2.0-beta9\lib\net45\x64\CNTKLibraryManaged-2.0.dll</HintPath>
+      <HintPath>..\packages\CNTK.CPUOnly.2.0-beta10\lib\net45\x64\CNTKLibraryManaged-2.0.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System" />
@@ -58,12 +58,12 @@
     <None Include="packages.config" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-  <Import Project="..\packages\CNTK.CPUOnly.2.0-beta9\build\net45\CNTK.CPUOnly.targets" Condition="Exists('..\packages\CNTK.CPUOnly.2.0-beta9\build\net45\CNTK.CPUOnly.targets')" />
+  <Import Project="..\packages\CNTK.CPUOnly.2.0-beta10\build\net45\CNTK.CPUOnly.targets" Condition="Exists('..\packages\CNTK.CPUOnly.2.0-beta10\build\net45\CNTK.CPUOnly.targets')" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
-    <Error Condition="!Exists('..\packages\CNTK.CPUOnly.2.0-beta9\build\net45\CNTK.CPUOnly.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\CNTK.CPUOnly.2.0-beta9\build\net45\CNTK.CPUOnly.targets'))" />
+    <Error Condition="!Exists('..\packages\CNTK.CPUOnly.2.0-beta10\build\net45\CNTK.CPUOnly.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\CNTK.CPUOnly.2.0-beta10\build\net45\CNTK.CPUOnly.targets'))" />
   </Target>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

Examples/Evaluation/CNTKLibraryCSEvalCPUOnlyExamples/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="CNTK.CPUOnly" version="2.0-beta9" targetFramework="net45" />
+  <package id="CNTK.CPUOnly" version="2.0-beta10" targetFramework="net45" />
 </packages>

Examples/Evaluation/CNTKLibraryCSEvalGPUExamples/CNTKLibraryCSEvalGPUExamples.csproj

@@ -40,7 +40,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="CNTKLibraryManaged-2.0, Version=1.0.0.0, Culture=neutral, processorArchitecture=AMD64">
-      <HintPath>..\packages\CNTK.GPU.2.0-beta9\lib\net45\x64\CNTKLibraryManaged-2.0.dll</HintPath>
+      <HintPath>..\packages\CNTK.GPU.2.0-beta10\lib\net45\x64\CNTKLibraryManaged-2.0.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System" />
@@ -62,12 +62,12 @@
     <None Include="packages.config" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-  <Import Project="..\packages\CNTK.GPU.2.0-beta9\build\net45\CNTK.GPU.targets" Condition="Exists('..\packages\CNTK.GPU.2.0-beta9\build\net45\CNTK.GPU.targets')" />
+  <Import Project="..\packages\CNTK.GPU.2.0-beta10\build\net45\CNTK.GPU.targets" Condition="Exists('..\packages\CNTK.GPU.2.0-beta10\build\net45\CNTK.GPU.targets')" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
-    <Error Condition="!Exists('..\packages\CNTK.GPU.2.0-beta9\build\net45\CNTK.GPU.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\CNTK.GPU.2.0-beta9\build\net45\CNTK.GPU.targets'))" />
+    <Error Condition="!Exists('..\packages\CNTK.GPU.2.0-beta10\build\net45\CNTK.GPU.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\CNTK.GPU.2.0-beta10\build\net45\CNTK.GPU.targets'))" />
   </Target>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

Examples/Evaluation/CNTKLibraryCSEvalGPUExamples/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="CNTK.GPU" version="2.0-beta9" targetFramework="net45" />
+  <package id="CNTK.GPU" version="2.0-beta10" targetFramework="net45" />
 </packages>

Examples/Evaluation/CSEvalClient/CSEvalClient.csproj

@@ -50,7 +50,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="EvalWrapper, Version=0.0.0.0, Culture=neutral, PublicKeyToken=52681d72504348ec, processorArchitecture=AMD64">
-      <HintPath>..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta9\lib\net45\x64\EvalWrapper.dll</HintPath>
+      <HintPath>..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta10\lib\net45\x64\EvalWrapper.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System" />
@@ -88,11 +88,11 @@
     </BootstrapperPackage>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-  <Import Project="..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta9\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets" Condition="Exists('..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta9\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets')" />
+  <Import Project="..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta10\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets" Condition="Exists('..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta10\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets')" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
-    <Error Condition="!Exists('..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta9\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta9\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets'))" />
+    <Error Condition="!Exists('..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta10\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta10\build\net45\Microsoft.Research.CNTK.CpuEval-mkl.targets'))" />
   </Target>
 </Project>

Examples/Evaluation/CSEvalClient/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Research.CNTK.CpuEval-mkl" version="2.0-beta9" targetFramework="net45" />
+  <package id="Microsoft.Research.CNTK.CpuEval-mkl" version="2.0-beta10" targetFramework="net45" />
 </packages>

Examples/Image/Classification/ConvNet/Python/ConvNet_CIFAR10_DataAug_Distributed.py

@@ -12,10 +12,6 @@
 import cntk
 import _cntk_py
 
-from cntk.utils import *
-from cntk.device import set_default_device, gpu
-from cntk.distributed import data_parallel_distributed_learner, block_momentum_distributed_learner, Communicator
-
 # default Paths relative to current python file.
 abs_path   = os.path.dirname(os.path.abspath(__file__))
 model_path = os.path.join(abs_path, "Models")
@@ -109,9 +105,9 @@ def create_trainer(network, epoch_size, num_quantization_bits, block_size, warm_
                                               l2_regularization_weight=l2_reg_weight)
 
     if block_size != None:
-        learner = block_momentum_distributed_learner(local_learner, block_size=block_size)
+        learner = cntk.distributed.block_momentum_distributed_learner(local_learner, block_size=block_size)
     else:
-        learner = data_parallel_distributed_learner(local_learner, num_quantization_bits=num_quantization_bits, distributed_after=warm_up)
+        learner = cntk.distributed.data_parallel_distributed_learner(local_learner, num_quantization_bits=num_quantization_bits, distributed_after=warm_up)
 
     # Create trainer
     return cntk.Trainer(network['output'], network['ce'], network['pe'], learner)
@@ -126,7 +122,11 @@ def train_and_test(network, trainer, train_source, test_source, progress_printer
     }
 
     training_session = cntk.training_session(train_source, trainer,
-        cntk.minibatch_size_schedule(64), progress_printer, input_map, os.path.join(model_path, "ConvNet_CIFAR10_DataAug_"), epoch_size)
+                                             cntk.minibatch_size_schedule(64),
+                                             progress_printer, input_map,
+                                             os.path.join(model_path, "ConvNet_CIFAR10_DataAug_"),
+                                             epoch_size)
+    # Train all minibatches 
     training_session.train()
 
     ### TODO: Stay tuned for an upcoming simpler EvalSession API for test/validation.    
@@ -139,6 +139,7 @@ def train_and_test(network, trainer, train_source, test_source, progress_printer
     metric_denom    = 0
     minibatch_index = 0
 
+    # Test minibatch loop
     while True:
         data = test_source.next_minibatch(minibatch_size, input_map=input_map)
         if not data: break
@@ -147,7 +148,6 @@ def train_and_test(network, trainer, train_source, test_source, progress_printer
         metric_denom += local_mb_samples
         minibatch_index += 1
 
-
     fin_msg = "Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom)
     progress_printer.end_progress_print(fin_msg)
 
@@ -159,14 +159,16 @@ def train_and_test(network, trainer, train_source, test_source, progress_printer
 
 
 # Train and evaluate the network.
-def convnet_cifar10_dataaug(train_data, test_data, mean_data, epoch_size=50000, num_quantization_bits=32, block_size=3200, warm_up=0, max_epochs=2, log_to_file=None, num_mbs_per_log=None, gen_heartbeat=False):
+def convnet_cifar10_dataaug(train_data, test_data, mean_data, epoch_size=50000, num_quantization_bits=32, 
+                            block_size=3200, warm_up=0, max_epochs=2, log_to_file=None, 
+                            num_mbs_per_log=None, gen_heartbeat=False):
     _cntk_py.set_computation_network_trace_level(0)
 
-    progress_printer = ProgressPrinter(
+    progress_printer = cntk.utils.ProgressPrinter(
         freq=num_mbs_per_log,
         tag='Training',
         log_to_file=log_to_file,
-        rank=Communicator.rank(),
+        rank=cntk.distributed.Communicator.rank(),
         gen_heartbeat=gen_heartbeat,
         num_epochs=max_epochs)
 
@@ -196,7 +198,7 @@ def convnet_cifar10_dataaug(train_data, test_data, mean_data, epoch_size=50000,
     if args['outputdir'] is not None:
         model_path = args['o'] + "/models"
     if args['device'] is not None:
-        set_default_device(gpu(args['device']))
+        cntk.device.set_default_device(cntk.device.gpu(args['device']))
     if args['datadir'] is not None:
         data_path = args['datadir']
 
@@ -208,10 +210,12 @@ def convnet_cifar10_dataaug(train_data, test_data, mean_data, epoch_size=50000,
         convnet_cifar10_dataaug(train_data, test_data, mean_data, 
                                 epoch_size=50000,
                                 num_quantization_bits=args['quantized_bits'],
-                                block_size=args['block_samples'], 
+                                block_size=args['block_samples'],
                                 warm_up=args['distributed_after'],
                                 max_epochs=args['epochs'],
-                                log_to_file=args['log'], num_mbs_per_log=10)
+                                log_to_file=args['log'],
+                                num_mbs_per_log=10,
+                                gen_heartbeat=True)
     finally:
-        Communicator.finalize()
+        cntk.distributed.Communicator.finalize()
 

Examples/LanguageUnderstanding/ATIS/Python/LanguageUnderstanding.py

@@ -7,14 +7,7 @@
 from __future__ import print_function
 import os
 import math
-from cntk.blocks import *  # non-layer like building blocks such as LSTM()
-from cntk.layers import *  # layer-like stuff such as Linear()
-from cntk.models import *  # higher abstraction level, e.g. entire standard models and also operators like Sequential()
-from cntk.utils import *
-from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs
-from cntk import Trainer
-from cntk.learner import adam_sgd, learning_rate_schedule, UnitType, momentum_as_time_constant_schedule
-from cntk.ops import cross_entropy_with_softmax, classification_error
+import cntk
 
 ########################
 # variables and stuff  #
@@ -37,22 +30,22 @@
 ########################
 
 def create_reader(path):
-    return MinibatchSource(CTFDeserializer(path, StreamDefs(
-        query         = StreamDef(field='S0', shape=input_dim,   is_sparse=True),
-        intent_unused = StreamDef(field='S1', shape=num_intents, is_sparse=True),  # BUGBUG: unused, and should infer dim
-        slot_labels   = StreamDef(field='S2', shape=label_dim,   is_sparse=True)
+    return cntk.io.MinibatchSource(cntk.io.CTFDeserializer(path, cntk.io.StreamDefs(
+        query = cntk.io.StreamDef(field='S0', shape=input_dim,   is_sparse=True),
+        intent_unused = cntk.io.StreamDef(field='S1', shape=num_intents, is_sparse=True),  # BUGBUG: unused, and should infer dim
+        slot_labels = cntk.io.StreamDef(field='S2', shape=label_dim,   is_sparse=True)
     )))
 
 ########################
 # define the model     #
 ########################
 
 def create_model():
-  with default_options(initial_state=0.1):  # inject an option to mimic the BS version identically; remove some day
-    return Sequential([
-        Embedding(emb_dim),
-        Recurrence(LSTM(hidden_dim), go_backwards=False),
-        Dense(label_dim)
+  with cntk.layers.default_options(initial_state=0.1):  # inject an option to mimic the BS version identically; remove some day
+    return cntk.models.Sequential([
+        cntk.layers.Embedding(emb_dim),
+        cntk.layers.Recurrence(cntk.blocks.LSTM(hidden_dim), go_backwards=False),
+        cntk.layers.Dense(label_dim)
     ])
 
 ########################
@@ -61,32 +54,35 @@ def create_model():
 
 def train(reader, model, max_epochs):
     # Input variables denoting the features and label data
-    query       = Input(input_dim,  is_sparse=False)
-    slot_labels = Input(num_labels, is_sparse=True)  # TODO: make sparse once it works
+    query = cntk.blocks.Input(input_dim,  is_sparse=False)
+    slot_labels = cntk.blocks.Input(num_labels, is_sparse=True)  # TODO: make sparse once it works
 
     # apply model to input
     z = model(query)
 
     # loss and metric
-    ce = cross_entropy_with_softmax(z, slot_labels)
-    pe = classification_error      (z, slot_labels)
+    ce = cntk.ops.cross_entropy_with_softmax(z, slot_labels)
+    pe = cntk.ops.classification_error(z, slot_labels)
 
     # training config
     epoch_size = 36000
     minibatch_size = 70
     num_mbs_to_show_result = 100
-    momentum_time_constant = momentum_as_time_constant_schedule(minibatch_size / -math.log(0.9))  # TODO: Change to round number. This is 664.39. 700?
 
-    lr_schedule = [0.003]*2+[0.0015]*12+[0.0003] # LR schedule over epochs (we don't run that many epochs, but if we did, these are good values)
+    # TODO: Change to round number. This is 664.39. 700?
+    momentum_time_constant = cntk.learner.momentum_as_time_constant_schedule(minibatch_size / -math.log(0.9))  
+
+    # LR schedule over epochs (we don't run that many epochs, but if we did, these are good values)
+    lr_schedule = [0.003]*2+[0.0015]*12+[0.0003] 
 
     # trainer object
-    lr_per_sample = learning_rate_schedule(lr_schedule, UnitType.sample, epoch_size)
-    learner = adam_sgd(z.parameters,
-                       lr=lr_per_sample, momentum=momentum_time_constant,
-                       low_memory=True,
-                       gradient_clipping_threshold_per_sample=15, gradient_clipping_with_truncation=True)
+    lr_per_sample = cntk.learner.learning_rate_schedule(lr_schedule, cntk.learner.UnitType.sample, epoch_size)
+    learner = cntk.learner.adam_sgd(z.parameters,
+                                    lr=lr_per_sample, momentum=momentum_time_constant,
+                                    low_memory=True,
+                                    gradient_clipping_threshold_per_sample=15, gradient_clipping_with_truncation=True)
 
-    trainer = Trainer(z, ce, pe, [learner])
+    trainer = cntk.Trainer(z, ce, pe, [learner])
 
     # define mapping from reader streams to network inputs
     input_map = {
@@ -95,25 +91,31 @@ def train(reader, model, max_epochs):
     }
 
     # process minibatches and perform model training
-    log_number_of_parameters(z) ; print()
-    progress_printer = ProgressPrinter(freq=100, first=10, tag='Training') # more detailed logging
+    cntk.utils.log_number_of_parameters(z) ; print()
+    progress_printer = cntk.ProgressPrinter(freq=100, first=10, tag='Training') # more detailed logging
     #progress_printer = ProgressPrinter(tag='Training')
 
     t = 0
-    for epoch in range(max_epochs):         # loop over epochs
+
+    # loop over epochs
+    for epoch in range(max_epochs):
         epoch_end = (epoch+1) * epoch_size
-        while t < epoch_end:               # loop over minibatches on the epoch
+
+        # loop over minibatches on the epoch
+        while t < epoch_end:
             # BUGBUG? The change of minibatch_size parameter vv has no effect.
             data = reader.next_minibatch(min(minibatch_size, epoch_end-t), input_map=input_map) # fetch minibatch
             trainer.train_minibatch(data)                                   # update model with it
             t += trainer.previous_minibatch_sample_count                    # count samples processed so far
             progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
+
             #def trace_node(name):
             #    nl = [n for n in z.parameters if n.name() == name]
             #    if len(nl) > 0:
             #        print (name, np.asarray(nl[0].value))
             #trace_node('W')
             #trace_node('stabilizer_param')
+
         loss, metric, actual_samples = progress_printer.epoch_summary(with_metric=True)
 
     return loss, metric
@@ -125,14 +127,18 @@ def train(reader, model, max_epochs):
 if __name__=='__main__':
     # TODO: leave these in for now as debugging aids; remove for beta
     from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms
+
     #set_computation_network_trace_level(1)  # TODO: remove debugging facilities once this all works
     set_fixed_random_seed(1)  # BUGBUG: has no effect at present  # TODO: remove debugging facilities once this all works
     force_deterministic_algorithms()
 
     reader = create_reader(data_dir + "/atis.train.ctf")
     model = create_model()
+
     # train
     train(reader, model, max_epochs=8)
+
     # test (TODO)
     reader = create_reader(data_dir + "/atis.test.ctf")
+
     #test(reader, model_dir + "/slu.cmf")  # TODO: what is the correct pattern here?