Add top-k classification accuracy metrics (keras-team#3987)

* add categorical accuracy metric which tracks over top-k predictions

* remove top_k_categorical_accuracy from being tested together with other all_metrics

* fix in_top_k to work with batches. correct metrics.py and test_metrics.py appropriately

* style fixes for documentation on in_top_k function

* default to k=5 for top_k_categorical_accuracy metric

Author: ramananbalakrishnan

keras/backend/tensorflow_backend.py

@@ -1485,6 +1485,20 @@ def l2_normalize(x, axis):
         axis = axis % len(x.get_shape())
     return tf.nn.l2_normalize(x, dim=axis)
 
+def in_top_k(predictions, targets, k):
+    '''Says whether the `targets` are in the top `k` `predictions`
+
+    # Arguments
+        predictions: A tensor of shape batch_size x classess and type float32.
+        targets: A tensor of shape batch_size and type int32 or int64.
+        k: An int, number of top elements to consider.
+
+    # Returns
+        A tensor of shape batch_size and type bool. output_i is True if
+        targets_i is within top-k values of predictions_i
+    '''
+    return tf.nn.in_top_k(predictions, targets, k)
+
 
 # CONVOLUTIONS
 

keras/backend/theano_backend.py

@@ -1043,6 +1043,23 @@ def l2_normalize(x, axis):
     return x / norm
 
 
+def in_top_k(predictions, targets, k):
+    '''Says whether the `targets` are in the top `k` `predictions`
+
+    # Arguments
+        predictions: A tensor of shape batch_size x classess and type float32.
+        targets: A tensor of shape batch_size and type int32 or int64.
+        k: An int, number of top elements to consider.
+
+    # Returns
+        A tensor of shape batch_size and type int. output_i is 1 if
+        targets_i is within top-k values of predictions_i
+    '''
+    predictions_top_k = T.argsort(predictions)[:, -k:]
+    result, _ = theano.map(lambda prediction, target: any(equal(prediction, target)), sequences=[predictions_top_k, targets])
+    return result
+
+
 # CONVOLUTIONS
 
 def _preprocess_conv2d_input(x, dim_ordering):

keras/metrics.py

@@ -17,6 +17,10 @@ def sparse_categorical_accuracy(y_true, y_pred):
                           K.cast(K.argmax(y_pred, axis=-1), K.floatx())))
 
 
+def top_k_categorical_accuracy(y_true, y_pred, k=5):
+    return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
+
+
 def mean_squared_error(y_true, y_pred):
     return K.mean(K.square(y_pred - y_true))
 

tests/keras/test_metrics.py

@@ -65,5 +65,19 @@ def test_sparse_metrics():
         assert K.eval(metric(y_a, y_b)).shape == ()
 
 
+def test_top_k_categorical_accuracy():
+    y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]))
+    y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]]))
+    success_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred,
+                            k=3))
+    assert success_result == 1
+    partial_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred,
+                            k=2))
+    assert partial_result == 0.5
+    failure_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred,
+                            k=1))
+    assert failure_result == 0
+
+
 if __name__ == "__main__":
     pytest.main([__file__])