function approx with eligibility trace hardly works

Author: MorvanZhou

Reinforcement_learning_TUT/8_Actor_Critic_Advantage/AC_CartPole.py

@@ -1,7 +1,7 @@
 """
 Actor-Critic using TD-error as the Advantage, Reinforcement Learning.
 
-The cart pole example (based on https://github.com/dennybritz/reinforcement-learning/blob/master/PolicyGradient/CliffWalk%20Actor%20Critic%20Solution.ipynb)
+The cart pole example. Policy is oscillated.
 
 View more on [莫烦Python] : https://morvanzhou.github.io/tutorials/
 
@@ -19,86 +19,89 @@
 
 
 class Actor(object):
-    def __init__(self, n_features, n_actions, lr=0.001):
-        with tf.name_scope('inputs'):
-            self.state = tf.placeholder(tf.float32, [n_features, ], "state")
-            state = tf.expand_dims(self.state, axis=0)
-            self.act_index = tf.placeholder(tf.int32, name="act")
-            self.advantage = tf.placeholder(tf.float32, name="adv")  # TD_error
-
-        with tf.variable_scope('Actor'):
-            l1 = tf.layers.dense(
-                inputs=state,
-                units=20,    # number of hidden units
-                activation=tf.nn.tanh,
-                kernel_initializer=tf.random_normal_initializer(0., .1),    # weights
-                bias_initializer=tf.constant_initializer(0.1),  # biases
-                name='l1'
-            )
-
-            self.acts_prob = tf.layers.dense(
-                inputs=l1,
-                units=n_actions,    # output units
-                activation=tf.nn.softmax,   # get action probabilities
-                kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
-                bias_initializer=tf.constant_initializer(0.1),  # biases
-                name='l2'
-            )
-
-        with tf.name_scope('loss'):
-            neg_log_prob = -tf.log(self.acts_prob[0, self.act_index])   # loss without advantage
-            self.loss = tf.reduce_mean(neg_log_prob * self.advantage)  # advantage (TD_error) guided loss
-
-        with tf.name_scope('train'):
-            self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)
-
-    def update(self, s, a, adv):
-        feed_dict = {self.state: s, self.act_index: a, self.advantage: adv}
-        _, loss = self.sess.run([self.train_op, self.loss], feed_dict)
-        return loss
+    def __init__(self, sess, n_features, n_actions, lr=0.001):
+        self.sess = sess
+
+        self.state = tf.placeholder(tf.float32, [1, n_features], "state")
+        self.act_index = tf.placeholder(tf.int32, name="act")
+        self.td_error = tf.placeholder(tf.float32, name="td_error")  # TD_error
+
+        l1 = tf.layers.dense(
+            inputs=self.state,
+            units=20,    # number of hidden units
+            activation=tf.nn.relu,
+            kernel_initializer=tf.random_normal_initializer(0., .1),    # weights
+            bias_initializer=tf.constant_initializer(0.1),  # biases
+            name='l1'
+        )
+
+        self.acts_prob = tf.layers.dense(
+            inputs=l1,
+            units=n_actions,    # output units
+            activation=tf.nn.softmax,   # get action probabilities
+            kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
+            bias_initializer=tf.constant_initializer(0.1),  # biases
+            name='acts_prob'
+        )
+
+        with tf.variable_scope('exp_v'):
+            log_prob = tf.log(self.acts_prob[0, self.act_index])
+            self.exp_r = tf.reduce_mean(log_prob * self.td_error)  # advantage (TD_error) guided loss
+
+        with tf.variable_scope('train'):
+            self.train_op = tf.train.AdamOptimizer(lr).minimize(-self.exp_r)  # minimize(-exp_v) = maximize(exp_v)
+
+    def update(self, s, a, td):
+        s = s[np.newaxis, :]
+        feed_dict = {self.state: s, self.act_index: a, self.td_error: td}
+        _, exp_v = self.sess.run([self.train_op, self.exp_r], feed_dict)
+        return exp_v
 
     def choose_action(self, s):
+        s = s[np.newaxis, :]
         probs = self.sess.run(self.acts_prob, {self.state: s})   # get probabilities for all actions
         return np.random.choice(np.arange(probs.shape[1]), p=probs.ravel())   # return a int
 
 
 class Critic(object):
-    def __init__(self, n_features, lr=0.01):
-        with tf.name_scope('inputs'):
-            self.state = tf.placeholder(tf.float32, [n_features, ], "state")
-            state = tf.expand_dims(self.state, axis=0)
-            self.target = tf.placeholder(dtype=tf.float32, name="target")  # TD target=r+gamma*V_next
-
-        with tf.variable_scope('Critic'):
-            l1 = tf.layers.dense(
-                inputs=state,
-                units=20,  # number of hidden units
-                activation=tf.nn.relu,
-                kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
-                bias_initializer=tf.constant_initializer(0.1),  # biases
-                name='l1'
-            )
-
-            self.eval = tf.layers.dense(
-                inputs=l1,
-                units=1,  # output units
-                activation=None,
-                kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
-                bias_initializer=tf.constant_initializer(0.1),  # biases
-                name='l2'
-            )
-
-        with tf.name_scope('loss'):
-            self.loss = tf.reduce_mean(tf.squared_difference(self.target, self.eval))    # TD_error = (r+gamma*V_next) - V_eval
-        with tf.name_scope('train'):
+    def __init__(self, sess, n_features, lr=0.01):
+        self.sess = sess
+
+        self.state = tf.placeholder(tf.float32, [1, n_features], "state")
+        self.v_next = tf.placeholder(tf.float32, [1, 1], name="v_next")
+        self.r = tf.placeholder(tf.float32, name='r')
+
+        l1 = tf.layers.dense(
+            inputs=self.state,
+            units=20,  # number of hidden units
+            activation=tf.nn.relu,
+            kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
+            bias_initializer=tf.constant_initializer(0.1),  # biases
+            name='l1'
+        )
+
+        self.v = tf.layers.dense(
+            inputs=l1,
+            units=1,  # output units
+            activation=None,
+            kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
+            bias_initializer=tf.constant_initializer(0.1),  # biases
+            name='V'
+        )
+
+        with tf.variable_scope('squared_TD_error'):
+            self.td_error = tf.reduce_mean(self.r + GAMMA * self.v_next - self.v)
+            self.loss = tf.square(self.td_error)    # TD_error = (r+gamma*V_next) - V_eval
+        with tf.variable_scope('train'):
             self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)
 
-    def update(self, s, target):
-        _, loss = self.sess.run([self.train_op, self.loss], {self.state: s, self.target: target})
-        return loss
+    def update(self, s, r, s_):
+        s, s_ = s[np.newaxis, :], s_[np.newaxis, :]
 
-    def evaluate(self, s):
-        return self.sess.run(self.eval, {self.state: s})[0, 0]  # return a float
+        v_next = self.sess.run(self.v, {self.state: s_})
+        td_error, loss, _ = self.sess.run([self.td_error, self.loss, self.train_op],
+                                          {self.state: s, self.v_next: v_next, self.r: r})
+        return td_error, loss
 
 
 OUTPUT_GRAPH = False
@@ -110,53 +113,47 @@ def evaluate(self, s):
 env = gym.make('CartPole-v0')
 env.seed(1)  # reproducible
 
-actor = Actor(n_features=env.observation_space.shape[0], n_actions=env.action_space.n, lr=0.001)
-critic = Critic(n_features=env.observation_space.shape[0], lr=0.01)     # we need a good teacher, so the teacher should learn faster than the actor
+sess = tf.Session()
 
-with tf.Session() as sess:
-    if OUTPUT_GRAPH:
-        tf.summary.FileWriter("logs/", sess.graph)
+with tf.variable_scope('Actor'):
+    actor = Actor(sess, n_features=env.observation_space.shape[0], n_actions=env.action_space.n, lr=0.001)
+with tf.variable_scope('Critic'):
+    critic = Critic(sess, n_features=env.observation_space.shape[0], lr=0.01)     # we need a good teacher, so the teacher should learn faster than the actor
 
-    actor.sess, critic.sess = sess, sess    # define the tf session
-    tf.global_variables_initializer().run()
+sess.run(tf.global_variables_initializer())
 
-    for i_episode in range(3000):
-        observation = env.reset()
-        t = 0
-        track_r = []
-        while True:
-            if RENDER: env.render()
+if OUTPUT_GRAPH:
+    tf.summary.FileWriter("logs/", sess.graph)
 
-            action = actor.choose_action(observation)
+for i_episode in range(3000):
+    s = env.reset()
+    t = 0
+    track_r = []
+    while True:
+        if RENDER: env.render()
 
-            observation_, reward, done, info = env.step(action)
+        a = actor.choose_action(s)
 
-            x, x_dot, theta, theta_dot = observation_
-            # the smaller theta and closer to center, the better
-            r1 = (env.x_threshold - abs(x)) / env.x_threshold - 0.5
-            r2 = (env.theta_threshold_radians - abs(theta)) / env.theta_threshold_radians - 0.5
-            reward = r1 + r2
+        s_, r, done, info = env.step(a)
 
-            track_r.append(reward)
+        if done: r = -20
 
-            TD_target = reward + GAMMA * critic.evaluate(observation_)    # r + gamma * V_next
-            TD_eval = critic.evaluate(observation)    # V_now
-            TD_error = TD_target - TD_eval
+        track_r.append(r)
 
-            actor.update(s=observation, a=action, adv=TD_error)
-            critic.update(s=observation, target=TD_target)
+        td_error, loss = critic.update(s, r, s_)  # gradient = grad[r + gamma * V(s_) - V(s)]
+        actor.update(s, a, td_error)     # true_gradient = grad[logPi(s,a) * td_error]
 
-            observation = observation_
-            t += 1
+        s = s_
+        t += 1
 
-            if done or t >= EPISODE_TIME_THRESHOLD:
-                ep_rs_sum = sum(track_r)
+        if done or t >= EPISODE_TIME_THRESHOLD:
+            ep_rs_sum = sum(track_r)
 
-                if 'running_reward' not in globals():
-                    running_reward = ep_rs_sum
-                else:
-                    running_reward = running_reward * 0.95 + ep_rs_sum * 0.05
-                if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True  # rendering
-                print("episode:", i_episode, "  reward:", int(running_reward))
-                break
+            if 'running_reward' not in globals():
+                running_reward = ep_rs_sum
+            else:
+                running_reward = running_reward * 0.95 + ep_rs_sum * 0.05
+            if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True  # rendering
+            print("episode:", i_episode, "  reward:", int(running_reward))
+            break
 

Reinforcement_learning_TUT/8_Actor_Critic_Advantage/AC_continue_Pendulum.py

@@ -3,6 +3,8 @@
 
 The cart pole example (based on https://github.com/dennybritz/reinforcement-learning/blob/master/PolicyGradient/CliffWalk%20Actor%20Critic%20Solution.ipynb)
 
+Cannot converge!!!
+
 View more on [莫烦Python] : https://morvanzhou.github.io/tutorials/
 
 Using:
@@ -26,50 +28,43 @@ def __init__(self, n_features, action_range, lr=0.0001):
             self.act = tf.placeholder(tf.float32, name="act")
             self.advantage = tf.placeholder(tf.float32, name="adv")  # TD_error
 
-        mu_ = tf.layers.dense(
+        l1 = tf.layers.dense(
             inputs=state,
-            units=40,  # number of hidden units
-            activation=tf.nn.relu,
+            units=30,  # number of hidden units
+            activation=None,
             kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
             bias_initializer=tf.constant_initializer(0.1),  # biases
-            name='mu_'
+            name='l1'
         )
 
         mu = tf.layers.dense(
-            inputs=mu_,
+            inputs=l1,
             units=1,  # number of hidden units
-            activation=None,
+            activation=tf.nn.tanh,
             kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
             bias_initializer=tf.constant_initializer(0.1),  # biases
             name='mu'
         )
 
-        sigma_ = tf.layers.dense(
-            inputs=state,
-            units=40,  # output units
-            activation=None,  # get action probabilities
-            kernel_initializer=tf.random_normal_initializer(0., .3),  # weights
-            bias_initializer=tf.constant_initializer(0.3),  # biases
-            name='sigma_'
-        )
         sigma = tf.layers.dense(
-            inputs=sigma_,
+            inputs=l1,
             units=1,  # output units
-            activation=tf.nn.softplus,  # get action probabilities
-            kernel_initializer=tf.random_normal_initializer(0., .3),  # weights
-            bias_initializer=tf.constant_initializer(.5),  # biases
+            activation=tf.nn.relu,  # get action probabilities
+            kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
+            bias_initializer=tf.constant_initializer(1.),  # biases
             name='sigma'
         )
 
-        self.mu, self.sigma = tf.squeeze(mu), tf.squeeze(sigma+1e-1)
+        self.mu, self.sigma = tf.squeeze(mu*2), tf.squeeze(sigma+1e-2)
         self.normal_dist = tf.contrib.distributions.Normal(self.mu, self.sigma)
 
         self.action = tf.clip_by_value(self.normal_dist.sample(1), action_range[0], action_range[1])
 
         with tf.name_scope('loss'):
             neg_log_prob = -self.normal_dist.log_prob(self.act)  # loss without advantage
-            self.loss = tf.reduce_mean(neg_log_prob * self.advantage)  # advantage (TD_error) guided loss
-            self.loss -= 3e-1 * self.normal_dist.entropy()
+            self.loss = neg_log_prob * self.advantage  # advantage (TD_error) guided loss
+            # Add cross entropy cost to encourage exploration
+            self.loss -= 1e-1 * self.normal_dist.entropy()
 
         with tf.name_scope('train'):
             self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)
@@ -93,8 +88,8 @@ def __init__(self, n_features, lr=0.01):
         with tf.variable_scope('Critic'):
             l1 = tf.layers.dense(
                 inputs=state,
-                units=40,  # number of hidden units
-                activation=tf.nn.relu,  # open end
+                units=30,  # number of hidden units
+                activation=None,
                 kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
                 bias_initializer=tf.constant_initializer(0.1),  # biases
                 name='l1'
@@ -112,7 +107,7 @@ def __init__(self, n_features, lr=0.01):
         with tf.name_scope('loss'):
             self.loss = tf.reduce_mean(tf.squared_difference(self.target, self.eval))    # TD_error = (r+gamma*V_next) - V_eval
         with tf.name_scope('train'):
-            self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)
+            self.train_op = tf.train.RMSPropOptimizer(lr).minimize(self.loss)
 
     def update(self, s, target):
         _, loss = self.sess.run([self.train_op, self.loss], {self.state: s, self.target: target})
@@ -123,16 +118,16 @@ def evaluate(self, s):
 
 
 OUTPUT_GRAPH = False
-EPISODE_TIME_THRESHOLD = 100
+EPISODE_TIME_THRESHOLD = 300
 DISPLAY_REWARD_THRESHOLD = -550  # renders environment if total episode reward is greater then this threshold
 RENDER = False  # rendering wastes time
 GAMMA = 0.9
 
 env = gym.make('Pendulum-v0')
 # env.seed(1)  # reproducible
 
-actor = Actor(n_features=env.observation_space.shape[0], action_range=[env.action_space.low[0], env.action_space.high[0]], lr=0.0001)
-critic = Critic(n_features=env.observation_space.shape[0], lr=0.0001)
+actor = Actor(n_features=env.observation_space.shape[0], action_range=[env.action_space.low[0], env.action_space.high[0]], lr=0.001)
+critic = Critic(n_features=env.observation_space.shape[0], lr=0.002)
 
 with tf.Session() as sess:
     if OUTPUT_GRAPH:
@@ -146,12 +141,12 @@ def evaluate(self, s):
         t = 0
         ep_rs = []
         while True:
-            if RENDER: env.render()
+            # if RENDER:
+            env.render()
             action, mu, sigma = actor.choose_action(observation)
 
             observation_, reward, done, info = env.step(action)
-            # if reward > -2: reward = (reward+5)*2
-
+            reward /= 10
             TD_target = reward + GAMMA * critic.evaluate(observation_)    # r + gamma * V_next
             TD_eval = critic.evaluate(observation)    # V_now
             TD_error = TD_target - TD_eval
@@ -168,7 +163,7 @@ def evaluate(self, s):
                 if 'running_reward' not in globals():
                     running_reward = ep_rs_sum
                 else:
-                    running_reward = running_reward * 0.99 + ep_rs_sum * 0.01
+                    running_reward = running_reward * 0.9 + ep_rs_sum * 0.1
                 if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True  # rendering
                 print("episode:", i_episode, "  reward:", int(running_reward))
                 break