Merge pull request tensorflow#11824 from tiagofrepereira2012/master

Add maxout op to tf.contrib.layers

tensorflow/contrib/layers/__init__.py

@@ -51,6 +51,7 @@
 @@unit_norm
 @@bow_encoder
 @@embed_sequence
+@@maxout
 
 @@apply_regularization
 @@l1_l2_regularizer

tensorflow/contrib/layers/python/layers/layers.py

@@ -50,6 +50,7 @@
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.training import moving_averages
+from tensorflow.python.layers.maxout import maxout
 
 # TODO(b/28426988): Replace legacy_* fns migrated from slim.
 # TODO(b/28426988): Remove legacy_* when all uses have migrated to new API.
@@ -92,7 +93,8 @@
            'unit_norm',
            'legacy_fully_connected',
            'legacy_linear',
-           'legacy_relu']
+           'legacy_relu',
+           'maxout']
 
 DATA_FORMAT_NCHW = 'NCHW'
 DATA_FORMAT_NHWC = 'NHWC'

tensorflow/python/BUILD

@@ -3745,6 +3745,7 @@ py_library(
         "layers/convolutional.py",
         "layers/core.py",
         "layers/layers.py",
+        "layers/maxout.py",
         "layers/normalization.py",
         "layers/pooling.py",
     ],
@@ -3825,6 +3826,22 @@ py_test(
     ],
 )
 
+py_test(
+    name = "layers_maxout_test",
+    size = "small",
+    srcs = ["layers/maxout_test.py"],
+    main = "layers/maxout_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":client_testlib",
+        ":framework_for_generated_wrappers",
+        ":layers",
+        ":math_ops",
+        ":nn_ops",
+        ":random_ops",
+    ],
+)
+
 py_test(
     name = "layers_utils_test",
     size = "small",

tensorflow/python/layers/maxout.py

@@ -0,0 +1,108 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+# pylint: disable=unused-import,g-bad-import-order
+"""Contains the maxout layer
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import gen_array_ops
+
+from tensorflow.python.layers import base
+
+
+def maxout(inputs, num_units, axis=-1, name=None):
+  """Adds a maxout op from https://arxiv.org/abs/1302.4389
+
+  "Maxout Networks" Ian J. Goodfellow, David Warde-Farley, Mehdi Mirza, Aaron Courville,
+   Yoshua Bengio
+
+   Usually the operation is performed in the filter/channel dimension. This can also be
+   used after fully-connected layers to reduce number of features.
+
+   Arguments:
+   inputs: Tensor input
+   num_units: Specifies how many features will remain after maxout in the `axis` dimension
+         (usually channel). This must be multiple of number of `axis`.
+   axis: The dimension where max pooling will be performed. Default is the
+   last dimension.
+   name: Optional scope for name_scope.
+
+   Returns:
+    A `Tensor` representing the results of the pooling operation.
+
+   Raises:
+    ValueError: if num_units is not multiple of number of features.
+  """
+  return MaxOut(num_units=num_units, axis=axis, name=name)(inputs)
+
+
+class MaxOut(base.Layer):
+  """Adds a maxout op from https://arxiv.org/abs/1302.4389
+
+  "Maxout Networks" Ian J. Goodfellow, David Warde-Farley, Mehdi Mirza, Aaron Courville, Yoshua
+  Bengio
+
+  Usually the operation is performed in the filter/channel dimension. This can also be
+  used after fully-connected layers to reduce number of features.
+
+  Arguments:
+    inputs: Tensor input
+    num_units: Specifies how many features will remain after maxout in the `axis` dimension
+         (usually channel).
+    This must be multiple of number of `axis`.
+    axis: The dimension where max pooling will be performed. Default is the
+    last dimension.
+    name: Optional scope for name_scope.
+
+  Returns:
+    A `Tensor` representing the results of the pooling operation.
+
+  Raises:
+    ValueError: if num_units is not multiple of number of features.
+  """
+
+  def __init__(self,
+         num_units,
+         axis=-1,
+         name=None,
+         **kwargs):
+    super(MaxOut, self).__init__(
+      name=name, trainable=False, **kwargs)
+    self.axis = axis
+    self.num_units = num_units
+
+  def call(self, inputs):
+    inputs = ops.convert_to_tensor(inputs)
+    shape = inputs.get_shape().as_list()
+    num_channels = shape[self.axis]
+    if num_channels % self.num_units:
+      raise ValueError('number of features({}) is not '
+               'a multiple of num_units({})'
+               .format(num_channels, self.num_units))
+    shape[self.axis] = -1
+    shape += [num_channels // self.num_units]
+
+    # Dealing with batches with arbitrary sizes
+    for i in range(len(shape)):
+      if shape[i] is None:
+        shape[i] = gen_array_ops.shape(inputs)[i]
+    outputs = math_ops.reduce_max(gen_array_ops.reshape(inputs, shape), -1, keep_dims=False)
+
+    return outputs

tensorflow/python/layers/maxout_test.py

@@ -0,0 +1,61 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+# pylint: disable=unused-import,g-bad-import-order
+
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.layers import maxout
+from tensorflow.python.layers import convolutional as conv_layers
+from tensorflow.python.layers import core as core_layers
+
+from tensorflow.python.ops import random_ops
+from tensorflow.python.platform import test
+import numpy as np
+
+"""
+Contains the maxout layer tests
+"""
+
+
+class MaxOutTest(test.TestCase):
+  def test_simple(self):
+    inputs = random_ops.random_uniform((64, 10, 36), seed=1)
+    graph = maxout.maxout(inputs, num_units=3)
+    self.assertEqual(graph.get_shape().as_list(), [64, 10, 3])
+
+  def test_fully_connected(self):
+    inputs = random_ops.random_uniform((64, 50), seed=1)
+    graph = core_layers.dense(inputs, 50)
+    graph = maxout.maxout(graph, num_units=10)
+    self.assertEqual(graph.get_shape().as_list(), [64, 10])
+
+  def test_nchw(self):
+    inputs = random_ops.random_uniform((10, 100, 100, 3), seed=1)
+    graph = conv_layers.conv2d(inputs, 10, 3, padding="SAME")
+    graph = maxout.maxout(graph, num_units=1)
+    self.assertEqual(graph.get_shape().as_list(), [10, 100, 100, 1])
+
+  def test_invalid_shape(self):
+    inputs = random_ops.random_uniform((10, 100, 100, 3), seed=1)
+    graph = conv_layers.conv2d(inputs, 3, 10, strides=(1, 1))
+    with self.assertRaisesRegexp(ValueError, 'number of features'):
+      graph = maxout.maxout(graph, num_units=2)
+
+if __name__ == '__main__':
+  test.main()