add new

Author: MorvanZhou

Reinforcement_learning_TUT/10_A3C/A3C_v1.py Reinforcement_learning_TUT/10_A3C/A3C_continuous_action.py

@@ -1,5 +1,5 @@
 """
-Asynchronous Advantage Actor Critic (A3C), Reinforcement Learning.
+Asynchronous Advantage Actor Critic (A3C) with continuous action space, Reinforcement Learning.
 
 The Pendulum example. Version 1: convergence promised
 
@@ -22,19 +22,19 @@
 tf.set_random_seed(2)  # reproducible
 
 GAME = 'Pendulum-v0'
-OUTPUT_GRAPH = False
+OUTPUT_GRAPH = True
 LOG_DIR = './log'
 N_WORKERS = multiprocessing.cpu_count()
 MAX_EP_STEP = 300
-MAX_GLOBAL_EP = 800
+MAX_GLOBAL_EP = 1000
 GLOBAL_NET_SCOPE = 'Global_Net'
 UPDATE_GLOBAL_ITER = 5
 GAMMA = 0.9
+ENTROPY_BETA = 0.01
 LR_A = 0.001    # learning rate for actor
-LR_C = 0.002    # learning rate for critic
+LR_C = 0.001    # learning rate for critic
 
 env = gym.make(GAME)
-env.seed(1)  # reproducible
 
 N_S = env.observation_space.shape[0]
 N_A = env.action_space.shape[0]
@@ -57,23 +57,29 @@ def __init__(self, scope, n_s, n_a,
             with tf.variable_scope(scope):
                 self.s = tf.placeholder(tf.float32, [None, n_s], 'S')
                 self.a_his = tf.placeholder(tf.float32, [None, n_a], 'A')
-                self.v_target = tf.placeholder(tf.float32, [None, 1], 'R')
+                self.v_target = tf.placeholder(tf.float32, [None, 1], 'Vtarget')
 
                 mu, sigma, self.v = self._build_net(n_a)
 
                 td = tf.subtract(self.v_target, self.v, name='TD_error')
                 with tf.name_scope('c_loss'):
-                    self.c_loss = tf.reduce_mean(tf.square(td))
-                self.mu, self.sigma = tf.squeeze(mu * a_bound[1]), tf.squeeze(sigma + 1e-2)
-                self.normal_dist = tf.contrib.distributions.Normal(self.mu, self.sigma)
+                    self.c_loss = tf.reduce_sum(tf.square(td))
+
+                with tf.name_scope('wrap_a_out'):
+                    mu, sigma = mu * a_bound[1], sigma*2 + 1e-2
+                    self.test = sigma[0]
+
+                normal_dist = tf.contrib.distributions.Normal(mu, sigma)
+
                 with tf.name_scope('a_loss'):
-                    log_prob = self.normal_dist.log_prob(self.a_his)
-                    self.exp_v = tf.reduce_mean(log_prob * td)
-                    self.exp_v += 0.01*self.normal_dist.entropy()  # encourage exploration
+                    log_prob = normal_dist.log_prob(self.a_his)
+                    exp_v = log_prob * td
+                    entropy = normal_dist.entropy()  # encourage exploration
+                    self.exp_v = tf.reduce_sum(ENTROPY_BETA * entropy + exp_v)
                     self.a_loss = -self.exp_v
 
                 with tf.name_scope('choose_a'):  # use local params to choose action
-                    self.A = tf.clip_by_value(self.normal_dist.sample([1]), a_bound[0], a_bound[1])
+                    self.A = tf.clip_by_value(tf.squeeze(normal_dist.sample(1), axis=0), a_bound[0], a_bound[1])
                 with tf.name_scope('local_grad'):
                     self.a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor')
                     self.c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic')
@@ -91,11 +97,11 @@ def __init__(self, scope, n_s, n_a,
     def _build_net(self, n_a):
         w_init = tf.random_normal_initializer(0., .1)
         with tf.variable_scope('actor'):
-            l_a = tf.layers.dense(self.s, 100, tf.nn.relu, kernel_initializer=w_init, name='la')
+            l_a = tf.layers.dense(self.s, 50, tf.nn.relu, kernel_initializer=w_init, name='la')
             mu = tf.layers.dense(l_a, n_a, tf.nn.tanh, kernel_initializer=w_init, name='mu')
-            sigma = tf.layers.dense(l_a, n_a, tf.nn.softplus, kernel_initializer=w_init, name='sigma')
+            sigma = tf.layers.dense(l_a, n_a, tf.nn.sigmoid, kernel_initializer=w_init, name='sigma')
         with tf.variable_scope('critic'):
-            l_c = tf.layers.dense(self.s, 60, tf.nn.relu, kernel_initializer=w_init, name='lc')
+            l_c = tf.layers.dense(self.s, 50, tf.nn.relu, kernel_initializer=w_init, name='lc')
             v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v')  # state value
         return mu, sigma, v
 
@@ -107,7 +113,7 @@ def pull_global(self):  # run by a local
 
     def choose_action(self, s):  # run by a local
         s = s[np.newaxis, :]
-        return self.sess.run(self.A, {self.s: s})
+        return self.sess.run(self.A, {self.s: s})[0]
 
 
 class Worker(object):
@@ -119,7 +125,7 @@ def __init__(self, env, name, n_s, n_a, a_bound, sess, opt_a, opt_c, g_a_params,
 
     def work(self, update_iter, max_ep_step, gamma, coord):
         total_step = 1
-        buffer_s, buffer_a, buffer_r, buffer_s_ = [], [], [], []
+        buffer_s, buffer_a, buffer_r = [], [], []
         while not coord.should_stop() and GLOBAL_EP.eval(self.sess) < MAX_GLOBAL_EP:
             s = self.env.reset()
             ep_r = 0
@@ -128,27 +134,31 @@ def work(self, update_iter, max_ep_step, gamma, coord):
                     self.env.render()
                 a = self.AC.choose_action(s)
                 s_, r, done, info = self.env.step(a)
-                r /= 10
+                r /= 10     # normalize reward
                 ep_r += r
                 buffer_s.append(s)
                 buffer_a.append(a)
                 buffer_r.append(r)
-                buffer_s_.append(s_)
 
                 if total_step % update_iter == 0 or done:   # update global and assign to local net
-                    buffer_s, buffer_a, buffer_r, buffer_s_ = np.vstack(buffer_s), np.vstack(buffer_a), np.vstack(buffer_r), np.vstack(buffer_s_)
-
-                    v_next = self.sess.run(self.AC.v, {self.AC.s: buffer_s_})
-                    if done: v_next[-1, 0] = 0
-                    v_target = buffer_r + gamma * v_next
-
+                    if done:
+                        v_s_ = 0   # terminal
+                    else:
+                        v_s_ = self.sess.run(self.AC.v, {self.AC.s: s_[np.newaxis, :]})[0, 0]
+                    buffer_v_target = []
+                    for r in buffer_r[::-1]:    # reverse buffer r
+                        v_s_ = r + gamma * v_s_
+                        buffer_v_target.append(v_s_)
+                    buffer_v_target.reverse()
+
+                    buffer_s, buffer_a, buffer_v_target = np.vstack(buffer_s), np.vstack(buffer_a), np.vstack(buffer_v_target)
                     feed_dict = {
                         self.AC.s: buffer_s,
                         self.AC.a_his: buffer_a,
-                        self.AC.v_target: v_target,
+                        self.AC.v_target: buffer_v_target,
                     }
                     self.AC.update_global(feed_dict)
-                    buffer_s, buffer_a, buffer_r, buffer_s_ = [], [], [], []
+                    buffer_s, buffer_a, buffer_r = [], [], []
                     self.AC.pull_global()
 
                 s = s_
@@ -168,8 +178,8 @@ def work(self, update_iter, max_ep_step, gamma, coord):
     with tf.device("/cpu:0"):
         GLOBAL_EP = tf.Variable(0, dtype=tf.int32, name='global_ep', trainable=False)
         COUNT_GLOBAL_EP = tf.assign(GLOBAL_EP, tf.add(GLOBAL_EP, tf.constant(1), name='step_ep'))
-        OPT_A = tf.train.RMSPropOptimizer(LR_A)
-        OPT_C = tf.train.RMSPropOptimizer(LR_C)
+        OPT_A = tf.train.RMSPropOptimizer(LR_A, name='RMSPropA')
+        OPT_C = tf.train.RMSPropOptimizer(LR_C, name='RMSPropC')
         globalAC = ACNet(GLOBAL_NET_SCOPE, N_S, N_A)  # we only need its params
         workers = []
         # Create worker