Enabling evaluation tests

bindings/python/cntk/ops/__init__.py

@@ -40,10 +40,10 @@ def combine(operands, name=''):
 def alias(x, name=''):
     '''
      Create a new Function instance which just aliases the specified 'x' Function/Variable
-     such that the 'Output' of the new 'Function' is same as the 'Output' of the specified
+     such that the 'Output' of the new 'Function' is same as the 'Output' of the specified 
      'x' Function/Variable, and has the newly specified name.
      The purpose of this operator is to create a new distinct reference to a symbolic
-     computation which is different from the original Function/Variable that it aliases and can
+     computation which is different from the original Function/Variable that it aliases and can 
      be used for e.g. to substitute a specific instance of the aliased Function/Variable in the
      computation graph instead of substituting all usages of the aliased Function/Variable.
 
@@ -571,7 +571,7 @@ def minus(left, right, name=''):
 def element_times(left, right, name=''):
     '''
     The output of this operation is the element-wise product of the two input
-    tensors. It supports broadcasting.
+    tensors. It supports broadcasting. 
 
     Example:
         >>> C.element_times([1., 1., 1., 1.], [0.5, 0.25, 0.125, 0.]).eval()
@@ -598,7 +598,7 @@ def element_times(left, right, name=''):
 def element_divide(left, right, name=''):
     '''
     The output of this operation is the element-wise division of the two input
-    tensors. It supports broadcasting.
+    tensors. It supports broadcasting. 
 
     Example:
         >>> C.element_divide([1., 1., 1., 1.], [0.5, 0.25, 0.125, 0.]).eval()
@@ -1860,7 +1860,7 @@ def placeholder_variable(shape=None, dynamic_axes=None, name=''):
     if shape is None:
         shape = NDShape.unknown.dimensions()
     else:
-        shape = sanitize_shape(shape)
+    shape = sanitize_shape(shape)
 
     if dynamic_axes is None:
         dynamic_axes = Axis.unknown_dynamic_axes()
@@ -1942,7 +1942,7 @@ def constant(value=None, shape=None, device=None, name=''):
     #if np.isscalar(value) and not shape:
     if (np.isscalar(value) or isinstance(value, np.ndarray)) and not shape:
         shape = ()
-    if isinstance(value, np.ndarray):
+        if isinstance(value, np.ndarray):
         dtype = value.dtype
     else:
         dtype = np.float32

bindings/python/cntk/ops/tests/evaluation_test.py

@@ -17,49 +17,35 @@
 
 TARGET_OUT_PAIRS = [
     ([[0., 0., 0., 1]], [[1., 2., 3., 4.]]),
-    #([[0., 0., 0.5, 0.5]], [[1., 2., 3., 4.]]),
-    #([[0., 0.4, 0.3, 0.3]], [[2., 1., 1., 4.]])
+    ([[0., 0., 0.5, 0.5]], [[1., 2., 3., 4.]]),
+    ([[0., 0.4, 0.3, 0.3]], [[2., 1., 1., 4.]])
 ]
 
-# TODO: Enable tests when 0d arrays are correctly handled for backward
-# propagation e. g. array(29.0)
-
-
 @pytest.mark.parametrize("target_vector, output_vector", TARGET_OUT_PAIRS)
-def _test_op_cross_entropy_with_soft_max(output_vector, target_vector, device_id, precision):
+def test_op_cross_entropy_with_soft_max(output_vector, target_vector, device_id, precision):
     dt = PRECISION_TO_TYPE[precision]
-
-    def numpy_softmax(x):
-        x = AA(x, dtype=PRECISION_TO_TYPE[precision])
-        ox = x - x.max()  # subtract max to avoid overflow
-
-        expX = np.exp(ox)
-        return expX / np.sum(expX)
-
-    def numpy_op(label, softmax):
-        return -np.sum(label * np.log(softmax, dtype=PRECISION_TO_TYPE[precision]), dtype=PRECISION_TO_TYPE[precision])
 
-    def numpy_grad(softmax, target):
-        s = np.sum(target, dtype=dt) #This should be 1.0
-        return np.subtract(softmax * s , target)
+    s_max = exp_x / np.sum(exp_x)
 
+    t = AA(target_vector, dtype=dt)
 
-    expected_forward = [[[[numpy_op(AA(target_vector, dtype=PRECISION_TO_TYPE[precision]), 
-                           numpy_softmax(output_vector))]]]]
+    expected_forward = np.asarray(-np.sum(t * np.log(s_max, dtype=dt),
+        dtype=dt))
+    expected_forward.shape = (1,1,1,1)+expected_forward.shape
 
-    backward = [[numpy_grad(numpy_softmax(output_vector), 
-                AA(target_vector, dtype=PRECISION_TO_TYPE[precision]))]]
+    s = np.sum(t, dtype=dt)
+    backward = np.subtract(s_max * s, t)
+    backward.shape = (1,1) + backward.shape
 
     expected_backward = {
         'left_arg':  backward,
-        'right_arg': backward
+        'right_arg': [[-1*o]]
     }
 
     from .. import cross_entropy_with_softmax
     _test_binary_op(precision, device_id, cross_entropy_with_softmax,
                     output_vector, target_vector,
-                    expected_forward, expected_backward, True)
-
+                    expected_forward, expected_backward)
 
 @pytest.mark.parametrize("target_vector, output_vector", TARGET_OUT_PAIRS)
 def test_op_squared_error(output_vector, target_vector, device_id, precision):
@@ -68,36 +54,47 @@ def test_op_squared_error(output_vector, target_vector, device_id, precision):
     o = AA(output_vector, dtype=dt)
     t = AA(target_vector, dtype=dt)
 
-    expected_forward = [[np.sum((t - o)**2)]]
+    expected_forward = AA([[np.sum((t - o)**2)]])
 
+    backward = 2 * np.subtract(o, t)
     expected_backward = {
-        'left_arg':  [[-2 * np.subtract(t, o)]],
-        'right_arg': [[-2 * np.subtract(o, t)]]
+        'left_arg':  [[backward]],
+        'right_arg': [[-1*backward]]
     }
 
     from .. import squared_error
     _test_binary_op(precision, device_id, squared_error,
                     output_vector, target_vector,
-                    expected_forward, expected_backward, True)
+                    expected_forward, expected_backward)
 
 TARGET_OUT_PAIRS_EP = [
     ([[1., 0., 0., 0]], [[1., 2., 3., 4.]]),
     ([[0., 0., 0., 1]], [[1., 2., 3., 4.]]),
 ]
 
-# -- ErrorPrediction with softmax operation tests --
-
 @pytest.mark.parametrize("target_vector, output_vector", TARGET_OUT_PAIRS_EP)
 def test_op_classification_error(output_vector, target_vector, device_id, precision):
     dt = PRECISION_TO_TYPE[precision]
 
     o = AA(output_vector, dtype=dt)
     t = AA(target_vector, dtype=dt)
 
-    expected_forward = [[int(np.argmax(t) != np.argmax(o))]]
+    different_position = np.argmax(t) != np.argmax(o)
+
+    expected_forward = [[AA([[int(different_position)]], dtype=dt)]]
+
+    zero_backward = np.zeros_like([[t]], dtype=dt)
+    left_backward = np.copy(zero_backward)
 
-    from .. import classification_error
-    op = classification_error(o, t)
+    zero_backward[..., np.argmax(o)] = -1.
+    right_backward = zero_backward
+
+    expected_backward = {
+        'left_arg':  left_backward,
+        'right_arg': right_backward
+    }
 
-    unittest_helper(op, {}, expected_forward, {},
-            device_id=device_id, precision=precision)
+    from .. import classification_error
+    _test_binary_op(precision, device_id, classification_error,
+                    output_vector, target_vector,
+                    expected_forward, expected_backward)

bindings/python/cntk/ops/tests/ops_test_utils.py

@@ -59,8 +59,7 @@ def _test_unary_op(precision, device_id, op_func,
 
 
 def _test_binary_op(precision, device_id, op_func, left_operand, right_operand,
-                    expected_forward, expected_backward_all,
-                    only_input_variables=False, wrap_batch_seq=True):
+                    expected_forward, expected_backward_all, wrap_batch_seq=True):
 
     left_value = AA(left_operand, dtype=PRECISION_TO_TYPE[precision])
     right_value = AA(right_operand, dtype=PRECISION_TO_TYPE[precision])
@@ -97,18 +96,17 @@ def _test_binary_op(precision, device_id, op_func, left_operand, right_operand,
                     forward_input, expected_forward, expected_backward,
                     device_id=device_id, precision=precision)
 
-    if not only_input_variables:
-        forward_input = {a: left_value}
-        expected_backward = {a: expected_backward_all['left_arg'], }
-        unittest_helper(input_op_constant,
-                        forward_input, expected_forward, expected_backward,
-                        device_id=device_id, precision=precision)
-
-        forward_input = {b: right_value}
-        expected_backward = {b: expected_backward_all['right_arg'], }
-        unittest_helper(constant_op_input,
-                        forward_input, expected_forward, expected_backward,
-                        device_id=device_id, precision=precision)
+    forward_input = {a: left_value}
+    expected_backward = {a: expected_backward_all['left_arg'], }
+    unittest_helper(input_op_constant,
+                    forward_input, expected_forward, expected_backward,
+                    device_id=device_id, precision=precision)
+
+    forward_input = {b: right_value}
+    expected_backward = {b: expected_backward_all['right_arg'], }
+    unittest_helper(constant_op_input,
+                    forward_input, expected_forward, expected_backward,
+                    device_id=device_id, precision=precision)
 
 
 def unittest_helper(root_node,