further optimisation

oztc · Jul 29, 2015 · 9c7c52d · 9c7c52d
1 parent 10b767d
commit 9c7c52d
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/keras/optimizers.py b/keras/optimizers.py
@@ -180,14 +180,14 @@ def get_updates(self, params, constraints, loss):
         self.updates = [(self.iterations, self.iterations+1.)]
 
         t = self.iterations + 1
+        lr_t = self.lr * T.sqrt(1-self.beta_2**t)/(1-self.beta_1**t)
 
         for p, g, c in zip(params, grads, constraints):
             m = theano.shared(p.get_value() * 0.)  # zero init of moment
             v = theano.shared(p.get_value() * 0.)  # zero init of velocity
 
             m_t = (self.beta_1 * m) + (1 - self.beta_1) * g
             v_t = (self.beta_2 * v) + (1 - self.beta_2) * (g**2)
-            lr_t = self.lr * T.sqrt(1-self.beta_2**t)/(1-self.beta_1**t)
             p_t = p - lr_t * m_t / (T.sqrt(v_t) + self.epsilon)
 
             self.updates.append((m, m_t))