Add masking support to the activation layer

Author: wxs

keras/activations.py

@@ -11,9 +11,7 @@ def softmax(x):
 def time_distributed_softmax(x, mask_val=default_mask_val):
     xshape = x.shape
     X = x.reshape((xshape[0] * xshape[1], xshape[2]))
-    mask = get_mask(X, mask_val)
-    r =  mask * T.nnet.softmax(X) + (1 - mask) * mask_val
-    return r.reshape(xshape)
+    return T.nnet.softmax(X).reshape(xshape)
 
 def softplus(x):
     return T.nnet.softplus(x)

keras/layers/core.py

@@ -168,15 +168,17 @@ class Activation(Layer):
     '''
         Apply an activation function to an output.
     '''
-    def __init__(self, activation, target=0, beta=0.1):
+    def __init__(self, activation, target=0, beta=0.1, mask_val=default_mask_val):
         super(Activation,self).__init__()
         self.activation = activations.get(activation)
         self.target = target
         self.beta = beta
+        self.mask_val = shared_scalar(mask_val)
 
     def get_output(self, train):
         X = self.get_input(train)
-        return self.activation(X)
+        mask = get_mask(X, self.mask_val)
+        return mask * self.activation(X) + (1 - mask) * self.mask_val
 
     def get_config(self):
         return {"name":self.__class__.__name__,

tests/manual/check_masked_recurrent.py

@@ -117,11 +117,12 @@
 model2.add(TimeDistributedDense(4,4))
 model2.add(Activation('time_distributed_softmax'))
 model2.add(LSTM(4,4, return_sequences=True))
+model2.add(Activation('tanh'))
 model2.add(GRU(4,4, activation='softmax', return_sequences=True))
 model2.add(SimpleDeepRNN(4,4, depth=2, activation='relu', return_sequences=True)) 
 model2.add(SimpleRNN(4,4, activation='relu', return_sequences=False))
 model2.compile(loss='categorical_crossentropy',
-        optimizer='rmsprop', theano_mode=theano.compile.mode.FAST_RUN)
+        optimizer='rmsprop', theano_mode=theano.compile.mode.FAST_COMPILE)
 print("Compiled model2")
 
 X2 = np.random.random_integers(1, 4, size=(2,5))