add new

Author: MorvanZhou

Reinforcement_learning_TUT/10_A3C/A3C_v1.py Reinforcement_learning_TUT/10_A3C/A3C_continuous_action.py

@@ -1,5 +1,5 @@
 """
-Asynchronous Advantage Actor Critic (A3C), Reinforcement Learning.
+Asynchronous Advantage Actor Critic (A3C) with continuous action space, Reinforcement Learning.
 
 The Pendulum example. Version 1: convergence promised
 
@@ -22,19 +22,19 @@
 tf.set_random_seed(2)  # reproducible
 
 GAME = 'Pendulum-v0'
-OUTPUT_GRAPH = False
+OUTPUT_GRAPH = True
 LOG_DIR = './log'
 N_WORKERS = multiprocessing.cpu_count()
 MAX_EP_STEP = 300
-MAX_GLOBAL_EP = 800
+MAX_GLOBAL_EP = 1000
 GLOBAL_NET_SCOPE = 'Global_Net'
 UPDATE_GLOBAL_ITER = 5
 GAMMA = 0.9
+ENTROPY_BETA = 0.01
 LR_A = 0.001    # learning rate for actor
-LR_C = 0.002    # learning rate for critic
+LR_C = 0.001    # learning rate for critic
 
 env = gym.make(GAME)
-env.seed(1)  # reproducible
 
 N_S = env.observation_space.shape[0]
 N_A = env.action_space.shape[0]
@@ -57,23 +57,29 @@ def __init__(self, scope, n_s, n_a,
             with tf.variable_scope(scope):
                 self.s = tf.placeholder(tf.float32, [None, n_s], 'S')
                 self.a_his = tf.placeholder(tf.float32, [None, n_a], 'A')
-                self.v_target = tf.placeholder(tf.float32, [None, 1], 'R')
+                self.v_target = tf.placeholder(tf.float32, [None, 1], 'Vtarget')
 
                 mu, sigma, self.v = self._build_net(n_a)
 
                 td = tf.subtract(self.v_target, self.v, name='TD_error')
                 with tf.name_scope('c_loss'):
-                    self.c_loss = tf.reduce_mean(tf.square(td))
-                self.mu, self.sigma = tf.squeeze(mu * a_bound[1]), tf.squeeze(sigma + 1e-2)
-                self.normal_dist = tf.contrib.distributions.Normal(self.mu, self.sigma)
+                    self.c_loss = tf.reduce_sum(tf.square(td))
+
+                with tf.name_scope('wrap_a_out'):
+                    mu, sigma = mu * a_bound[1], sigma*2 + 1e-2
+                    self.test = sigma[0]
+
+                normal_dist = tf.contrib.distributions.Normal(mu, sigma)
+
                 with tf.name_scope('a_loss'):
-                    log_prob = self.normal_dist.log_prob(self.a_his)
-                    self.exp_v = tf.reduce_mean(log_prob * td)
-                    self.exp_v += 0.01*self.normal_dist.entropy()  # encourage exploration
+                    log_prob = normal_dist.log_prob(self.a_his)
+                    exp_v = log_prob * td
+                    entropy = normal_dist.entropy()  # encourage exploration
+                    self.exp_v = tf.reduce_sum(ENTROPY_BETA * entropy + exp_v)
                     self.a_loss = -self.exp_v
 
                 with tf.name_scope('choose_a'):  # use local params to choose action
-                    self.A = tf.clip_by_value(self.normal_dist.sample([1]), a_bound[0], a_bound[1])
+                    self.A = tf.clip_by_value(tf.squeeze(normal_dist.sample(1), axis=0), a_bound[0], a_bound[1])
                 with tf.name_scope('local_grad'):
                     self.a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor')
                     self.c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic')
@@ -91,11 +97,11 @@ def __init__(self, scope, n_s, n_a,
     def _build_net(self, n_a):
         w_init = tf.random_normal_initializer(0., .1)
         with tf.variable_scope('actor'):
-            l_a = tf.layers.dense(self.s, 100, tf.nn.relu, kernel_initializer=w_init, name='la')
+            l_a = tf.layers.dense(self.s, 50, tf.nn.relu, kernel_initializer=w_init, name='la')
             mu = tf.layers.dense(l_a, n_a, tf.nn.tanh, kernel_initializer=w_init, name='mu')
-            sigma = tf.layers.dense(l_a, n_a, tf.nn.softplus, kernel_initializer=w_init, name='sigma')
+            sigma = tf.layers.dense(l_a, n_a, tf.nn.sigmoid, kernel_initializer=w_init, name='sigma')
         with tf.variable_scope('critic'):
-            l_c = tf.layers.dense(self.s, 60, tf.nn.relu, kernel_initializer=w_init, name='lc')
+            l_c = tf.layers.dense(self.s, 50, tf.nn.relu, kernel_initializer=w_init, name='lc')
             v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v')  # state value
         return mu, sigma, v
 
@@ -107,7 +113,7 @@ def pull_global(self):  # run by a local
 
     def choose_action(self, s):  # run by a local
         s = s[np.newaxis, :]
-        return self.sess.run(self.A, {self.s: s})
+        return self.sess.run(self.A, {self.s: s})[0]
 
 
 class Worker(object):
@@ -119,7 +125,7 @@ def __init__(self, env, name, n_s, n_a, a_bound, sess, opt_a, opt_c, g_a_params,
 
     def work(self, update_iter, max_ep_step, gamma, coord):
         total_step = 1
-        buffer_s, buffer_a, buffer_r, buffer_s_ = [], [], [], []
+        buffer_s, buffer_a, buffer_r = [], [], []
         while not coord.should_stop() and GLOBAL_EP.eval(self.sess) < MAX_GLOBAL_EP:
             s = self.env.reset()
             ep_r = 0
@@ -128,27 +134,31 @@ def work(self, update_iter, max_ep_step, gamma, coord):
                     self.env.render()
                 a = self.AC.choose_action(s)
                 s_, r, done, info = self.env.step(a)
-                r /= 10
+                r /= 10     # normalize reward
                 ep_r += r
                 buffer_s.append(s)
                 buffer_a.append(a)
                 buffer_r.append(r)
-                buffer_s_.append(s_)
 
                 if total_step % update_iter == 0 or done:   # update global and assign to local net
-                    buffer_s, buffer_a, buffer_r, buffer_s_ = np.vstack(buffer_s), np.vstack(buffer_a), np.vstack(buffer_r), np.vstack(buffer_s_)
-
-                    v_next = self.sess.run(self.AC.v, {self.AC.s: buffer_s_})
-                    if done: v_next[-1, 0] = 0
-                    v_target = buffer_r + gamma * v_next
-
+                    if done:
+                        v_s_ = 0   # terminal
+                    else:
+                        v_s_ = self.sess.run(self.AC.v, {self.AC.s: s_[np.newaxis, :]})[0, 0]
+                    buffer_v_target = []
+                    for r in buffer_r[::-1]:    # reverse buffer r
+                        v_s_ = r + gamma * v_s_
+                        buffer_v_target.append(v_s_)
+                    buffer_v_target.reverse()
+
+                    buffer_s, buffer_a, buffer_v_target = np.vstack(buffer_s), np.vstack(buffer_a), np.vstack(buffer_v_target)
                     feed_dict = {
                         self.AC.s: buffer_s,
                         self.AC.a_his: buffer_a,
-                        self.AC.v_target: v_target,
+                        self.AC.v_target: buffer_v_target,
                     }
                     self.AC.update_global(feed_dict)
-                    buffer_s, buffer_a, buffer_r, buffer_s_ = [], [], [], []
+                    buffer_s, buffer_a, buffer_r = [], [], []
                     self.AC.pull_global()
 
                 s = s_
@@ -168,8 +178,8 @@ def work(self, update_iter, max_ep_step, gamma, coord):
     with tf.device("/cpu:0"):
         GLOBAL_EP = tf.Variable(0, dtype=tf.int32, name='global_ep', trainable=False)
         COUNT_GLOBAL_EP = tf.assign(GLOBAL_EP, tf.add(GLOBAL_EP, tf.constant(1), name='step_ep'))
-        OPT_A = tf.train.RMSPropOptimizer(LR_A)
-        OPT_C = tf.train.RMSPropOptimizer(LR_C)
+        OPT_A = tf.train.RMSPropOptimizer(LR_A, name='RMSPropA')
+        OPT_C = tf.train.RMSPropOptimizer(LR_C, name='RMSPropC')
         globalAC = ACNet(GLOBAL_NET_SCOPE, N_S, N_A)  # we only need its params
         workers = []
         # Create worker

Reinforcement_learning_TUT/10_A3C/A3C_v2.py

@@ -0,0 +1,235 @@
+"""
+Asynchronous Advantage Actor Critic (A3C), Reinforcement Learning.
+
+The BipedalWalker example.
+
+View more on [莫烦Python] : https://morvanzhou.github.io/tutorials/
+
+Using:
+tensorflow 1.0
+gym 0.8.0
+"""
+
+import multiprocessing
+import threading
+import tensorflow as tf
+import numpy as np
+import gym
+import os
+import shutil
+
+np.random.seed(2)
+tf.set_random_seed(2)  # reproducible
+
+GAME = 'BipedalWalker-v2'
+# GAME = 'Pendulum-v0'
+OUTPUT_GRAPH = True
+LOG_DIR = './log'
+N_WORKERS = multiprocessing.cpu_count() + 2
+MAX_GLOBAL_EP = 10000
+GLOBAL_NET_SCOPE = 'Global_Net'
+SHARED_LAYER = True
+UPDATE_GLOBAL_ITER = 5
+GAMMA = 0.99
+ENTROPY_BETA = 0.01
+LR_A = 0.0001    # learning rate for actor
+LR_C = 0.0001    # learning rate for critic
+
+env = gym.make(GAME)
+env.seed(1)  # reproducible
+
+N_S = env.observation_space.shape[0]
+N_A = env.action_space.shape[0]
+A_BOUND = [env.action_space.low, env.action_space.high]
+
+
+class ACNet(object):
+    def __init__(self, scope, n_s, n_a,
+                 a_bound=None, sess=None,
+                 opt_a=None, opt_c=None, global_a_params=None, global_c_params=None):
+
+        if scope == GLOBAL_NET_SCOPE:   # get global network
+            with tf.variable_scope(scope):
+                self.s = tf.placeholder(tf.float32, [None, n_s], 'S')
+                self._build_net(n_a, SHARED_LAYER)
+                self.a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor')
+                self.c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic')
+        else:   # local net, calculate losses
+            self.sess = sess
+            with tf.variable_scope(scope):
+                self.s = tf.placeholder(tf.float32, [None, n_s], 'S')
+                self.a_his = tf.placeholder(tf.float32, [None, n_a], 'A')
+                # self.a_his = tf.squeeze(self.a_his)
+                self.v_target = tf.placeholder(tf.float32, [None, 1], 'Vtarget')
+
+                mu, sigma, self.v = self._build_net(n_a, SHARED_LAYER)
+
+                td = tf.subtract(self.v_target, self.v, name='TD_error')
+                with tf.name_scope('c_loss'):
+                    self.c_loss = tf.reduce_sum(tf.square(td))
+
+                with tf.name_scope('wrap_a_out'):
+                    mu, sigma = mu * a_bound[1], sigma + 1e-2
+                    self.test = sigma[0]
+
+                # the distribution result has been flatten
+                normal_dist = tf.contrib.distributions.Normal(mu, sigma)
+
+                with tf.name_scope('a_loss'):
+                    log_prob = normal_dist.log_prob(self.a_his)
+                    exp_v = log_prob * td
+                    entropy = normal_dist.entropy()  # encourage exploration
+                    self.exp_v = tf.reduce_sum(ENTROPY_BETA*entropy+exp_v)
+                    self.a_loss = -self.exp_v
+
+                with tf.name_scope('choose_a'):  # use local params to choose action
+                    self.A = tf.clip_by_value(tf.squeeze(normal_dist.sample(1), axis=0), a_bound[0], a_bound[1])
+
+                with tf.name_scope('local_grad'):
+                    self.a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor')
+                    self.c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic')
+                    self.a_grads = tf.gradients(self.a_loss, self.a_params)  # get local gradients
+                    self.c_grads = tf.gradients(self.c_loss, self.c_params)
+
+            with tf.name_scope('sync'):
+                with tf.name_scope('pull'):
+                    self.pull_a_params_op = [l_p.assign(g_p) for l_p, g_p in zip(self.a_params, global_a_params)]
+                    self.pull_c_params_op = [l_p.assign(g_p) for l_p, g_p in zip(self.c_params, global_c_params)]
+                with tf.name_scope('push'):
+                    self.update_a_op = opt_a.apply_gradients(zip(self.a_grads, global_a_params))
+                    self.update_c_op = opt_c.apply_gradients(zip(self.c_grads, global_c_params))
+
+    def _build_net(self, n_a, shared):
+        w_init = tf.random_normal_initializer(0., .01)
+        if shared:
+            with tf.variable_scope('actor'):
+                la = tf.layers.dense(self.s, 200, tf.nn.relu, kernel_initializer=w_init, name='share')
+                la = tf.layers.dense(la, 20, tf.nn.relu, kernel_initializer=w_init, name='la')
+                mu = tf.layers.dense(la, n_a, tf.nn.tanh, kernel_initializer=w_init, name='mu')
+                sigma = tf.layers.dense(la, n_a, tf.nn.sigmoid, kernel_initializer=w_init, name='sigma')
+
+                # shared layer for critic
+                lc = tf.layers.dense(self.s, 200, tf.nn.relu, name='share', reuse=True)
+
+            with tf.variable_scope('critic'):
+                lc = tf.layers.dense(lc, 20, tf.nn.relu, kernel_initializer=w_init, name='lc2')
+                v = tf.layers.dense(lc, 1, kernel_initializer=w_init, name='v')  # state value
+        else:
+            with tf.variable_scope('actor'):
+                l_a = tf.layers.dense(self.s, 200, tf.nn.relu, kernel_initializer=w_init, name='la')
+                l_a = tf.layers.dense(l_a, 20, tf.nn.tanh, kernel_initializer=w_init, name='la2')
+                mu = tf.layers.dense(l_a, n_a, tf.nn.tanh, kernel_initializer=w_init, name='mu')
+                sigma = tf.layers.dense(l_a, n_a, tf.nn.softplus, kernel_initializer=w_init,  name='sigma')
+            with tf.variable_scope('critic'):
+                l_c = tf.layers.dense(self.s, 200, tf.nn.relu, kernel_initializer=w_init, name='lc1')
+                l_c = tf.layers.dense(l_c, 20, tf.nn.relu, kernel_initializer=w_init, name='lc2')
+                v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v')  # state value
+        return mu, sigma, v
+
+    def update_global(self, feed_dict):  # run by a local
+        _, _, test = self.sess.run([self.update_a_op, self.update_c_op, self.test], feed_dict)  # local grads applies to global net
+        return test
+
+    def pull_global(self):  # run by a local
+        self.sess.run([self.pull_a_params_op, self.pull_c_params_op])
+
+    def choose_action(self, s):  # run by a local
+        s = s[np.newaxis, :]
+        a = self.sess.run(self.A, {self.s: s})
+        return a.flatten()
+
+
+class Worker(object):
+    def __init__(self, env, i, n_s, n_a, a_bound, sess, opt_a, opt_c, g_a_params, g_c_params):
+        self.env = env
+        self.id = i
+        self.sess = sess
+        self.name = 'W_%i' % i
+        self.AC = ACNet(self.name, n_s, n_a, a_bound, sess, opt_a, opt_c, g_a_params, g_c_params)
+
+    def work(self, update_iter, gamma, coord):
+        total_step = 1
+        buffer_s, buffer_a, buffer_r = [], [], []
+        while not coord.should_stop() and GLOBAL_EP.eval(self.sess) < MAX_GLOBAL_EP:
+            s = self.env.reset()
+            ep_r = 0
+            while True:
+                # if self.name == 'W_0' and total_step % 50 == 0:
+                #     self.env.render()
+                a = self.AC.choose_action(s)
+                s_, r, done, info = self.env.step(a)
+                if r == -100: r = -2
+                ep_r += r
+                buffer_s.append(s)
+                buffer_a.append(a)
+                buffer_r.append(r)
+
+                if total_step % update_iter == 0 or done:   # update global and assign to local net
+                    if done:
+                        v_s_ = 0   # terminal
+                    else:
+                        v_s_ = self.sess.run(self.AC.v, {self.AC.s: s_[np.newaxis, :]})[0, 0]
+                    buffer_v_target = []
+                    for r in buffer_r[::-1]:    # reverse buffer r
+                        v_s_ = r + gamma * v_s_
+                        buffer_v_target.append(v_s_)
+                    buffer_v_target.reverse()
+
+                    buffer_s, buffer_a, buffer_v_target = np.vstack(buffer_s), np.vstack(buffer_a), np.vstack(buffer_v_target)
+                    feed_dict = {
+                        self.AC.s: buffer_s,
+                        self.AC.a_his: buffer_a,
+                        self.AC.v_target: buffer_v_target,
+                    }
+                    test = self.AC.update_global(feed_dict)
+                    buffer_s, buffer_a, buffer_r = [], [], []
+                    self.AC.pull_global()
+
+                s = s_
+                total_step += 1
+
+                if done:
+                    print(
+                        self.name,
+                        "Ep:", GLOBAL_EP.eval(self.sess),
+                        '| pos: %i' % self.env.unwrapped.hull.position[0],
+                        "| r: %.2f" % np.sum(ep_r),
+                        '| stddev:', test,
+                          )
+                    sess.run(COUNT_GLOBAL_EP)
+                    break
+
+if __name__ == "__main__":
+    sess = tf.Session()
+
+    with tf.device("/cpu:0"):
+        GLOBAL_EP = tf.Variable(0, dtype=tf.int32, name='global_ep', trainable=False)
+        COUNT_GLOBAL_EP = tf.assign(GLOBAL_EP, tf.add(GLOBAL_EP, tf.constant(1), name='step_ep'))
+        OPT_A = tf.train.RMSPropOptimizer(LR_A, .9)
+        OPT_C = tf.train.RMSPropOptimizer(LR_C, .9)
+        globalAC = ACNet(GLOBAL_NET_SCOPE, N_S, N_A)  # we only need its params
+        workers = []
+        # Create worker
+        for i in range(N_WORKERS):
+            workers.append(
+                Worker(
+                    gym.make(GAME), i, N_S, N_A, A_BOUND, sess,
+                    OPT_A, OPT_C, globalAC.a_params, globalAC.c_params
+                ))
+
+    coord = tf.train.Coordinator()
+    sess.run(tf.global_variables_initializer())
+
+    if OUTPUT_GRAPH:
+        if os.path.exists(LOG_DIR):
+            shutil.rmtree(LOG_DIR)
+        tf.summary.FileWriter(LOG_DIR, sess.graph)
+
+    worker_threads = []
+    for worker in workers:
+        job = lambda: worker.work(UPDATE_GLOBAL_ITER, GAMMA, coord)
+        t = threading.Thread(target=job)
+        t.start()
+        worker_threads.append(t)
+    coord.join(worker_threads)
+

Reinforcement_learning_TUT/experiments/Solve_BipedalWalker/A3C.py

@@ -7,6 +7,44 @@
 np.random.seed(1)
 tf.set_random_seed(1)
 
+MAX_EPISODES = 2000
+LR_A = 0.0002  # learning rate for actor
+LR_C = 0.0002  # learning rate for critic
+GAMMA = 0.9999  # reward discount
+REPLACE_ITER_A = 1700
+REPLACE_ITER_C = 1500
+MEMORY_CAPACITY = 200000
+BATCH_SIZE = 32
+DISPLAY_THRESHOLD = 100  # display until the running reward > 100
+DATA_PATH = './data'
+LOAD_MODEL = False
+SAVE_MODEL_ITER = 50000
+RENDER = False
+OUTPUT_GRAPH = False
+ENV_NAME = 'BipedalWalker-v2'
+
+GLOBAL_STEP = tf.Variable(0, trainable=False)
+INCREASE_GS = GLOBAL_STEP.assign(tf.add(GLOBAL_STEP, 1))
+LR_A = tf.train.exponential_decay(LR_A, GLOBAL_STEP, 10000, .97, staircase=True)
+LR_C = tf.train.exponential_decay(LR_C, GLOBAL_STEP, 10000, .97, staircase=True)
+END_POINT = (200 - 10) * (14/30)    # from game
+
+env = gym.make(ENV_NAME)
+env.seed(1)
+
+STATE_DIM = env.observation_space.shape[0]  # 24
+ACTION_DIM = env.action_space.shape[0]  # 4
+ACTION_BOUND = env.action_space.high    # [1, 1, 1, 1]
+
+# all placeholder for tf
+with tf.name_scope('S'):
+    S = tf.placeholder(tf.float32, shape=[None, STATE_DIM], name='s')
+with tf.name_scope('A'):
+    A = tf.placeholder(tf.float32, shape=[None, ACTION_DIM], name='a')
+with tf.name_scope('R'):
+    R = tf.placeholder(tf.float32, [None, 1], name='r')
+with tf.name_scope('S_'):
+    S_ = tf.placeholder(tf.float32, shape=[None, STATE_DIM], name='s_')
 
 ###############################  Actor  ####################################
 
@@ -31,14 +69,13 @@ def __init__(self, sess, action_dim, action_bound, learning_rate, t_replace_iter
 
     def _build_net(self, s, scope, trainable):
         with tf.variable_scope(scope):
-            init_w = tf.random_normal_initializer(0., 0.1)
-            init_b = tf.constant_initializer(0.1)
-            net = tf.layers.dense(s, 400, activation=tf.nn.relu,
-                                  kernel_initializer=init_w, bias_initializer=init_b, name='l1',
-                                  trainable=trainable)
-            net = tf.layers.dense(net, 20, activation=tf.nn.relu,
-                                  kernel_initializer=init_w, bias_initializer=init_b, name='l2',
-                                  trainable=trainable)
+            init_w = tf.random_normal_initializer(0., 0.01)
+            init_b = tf.constant_initializer(0.01)
+            net = tf.layers.dense(s, 500, activation=tf.nn.relu,
+                                  kernel_initializer=init_w, bias_initializer=init_b, name='l1', trainable=trainable)
+            net = tf.layers.dense(net, 200, activation=tf.nn.relu,
+                                  kernel_initializer=init_w, bias_initializer=init_b, name='l2', trainable=trainable)
+
             with tf.variable_scope('a'):
                 actions = tf.layers.dense(net, self.a_dim, activation=tf.nn.tanh, kernel_initializer=init_w,
                                           bias_initializer=init_b, name='a', trainable=trainable)
@@ -107,19 +144,19 @@ def __init__(self, sess, state_dim, action_dim, learning_rate, gamma, t_replace_
 
     def _build_net(self, s, a, scope, trainable):
         with tf.variable_scope(scope):
-            init_w = tf.random_normal_initializer(0., 0.1)
-            init_b = tf.constant_initializer(0.1)
+            init_w = tf.random_normal_initializer(0., 0.01)
+            init_b = tf.constant_initializer(0.01)
 
             with tf.variable_scope('l1'):
-                n_l1 = 400
+                n_l1 = 700
+                # combine the action and states together in this way
                 w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], initializer=init_w, trainable=trainable)
                 w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], initializer=init_w, trainable=trainable)
                 b1 = tf.get_variable('b1', [1, n_l1], initializer=init_b, trainable=trainable)
                 net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)
             with tf.variable_scope('l2'):
                 net = tf.layers.dense(net, 20, activation=tf.nn.relu, kernel_initializer=init_w,
                                       bias_initializer=init_b, name='l2', trainable=trainable)
-
             with tf.variable_scope('q'):
                 q = tf.layers.dense(net, 1, kernel_initializer=init_w, bias_initializer=init_b, trainable=trainable)   # Q(s,a)
         return q
@@ -217,7 +254,7 @@ class Memory(object):  # stored as ( s, a, r, s_ ) in SumTree
     epsilon = 0.001  # small amount to avoid zero priority
     alpha = 0.6  # [0~1] convert the importance of TD error to priority
     beta = 0.4  # importance-sampling, from initial value increasing to 1
-    beta_increment_per_sampling = 1e-4  # annealing the bias
+    beta_increment_per_sampling = 1e-5  # annealing the bias
     abs_err_upper = 1   # for stability refer to paper
 
     def __init__(self, capacity):
@@ -268,48 +305,11 @@ def _get_priority(self, error):
         return np.power(clipped_error, self.alpha)
 
 
-MAX_EPISODES = 2000
-LR_A = 0.0001  # learning rate for actor
-LR_C = 0.0001  # learning rate for critic
-GAMMA = 0.999  # reward discount
-REPLACE_ITER_A = 1700
-REPLACE_ITER_C = 1500
-MEMORY_CAPACITY = 200000
-BATCH_SIZE = 32
-DISPLAY_THRESHOLD = 60
-DATA_PATH = './data'
-LOAD_MODEL = True
-SAVE_MODEL_ITER = 50000
-RENDER = False
-OUTPUT_GRAPH = False
-ENV_NAME = 'BipedalWalker-v2'
-
-GLOBAL_STEP = tf.Variable(0, trainable=False)
-INCREASE_GS = GLOBAL_STEP.assign(tf.add(GLOBAL_STEP, 1))
-END_POINT = (200 - 10) * (14/30)    # from game
-
-env = gym.make(ENV_NAME)
-env.seed(1)
-
-state_dim = env.observation_space.shape[0]  # 24
-action_dim = env.action_space.shape[0]  # 4
-action_bound = env.action_space.high    # [1, 1, 1, 1]
-
-# all placeholder for tf
-with tf.name_scope('S'):
-    S = tf.placeholder(tf.float32, shape=[None, state_dim], name='s')
-with tf.name_scope('A'):
-    A = tf.placeholder(tf.float32, shape=[None, action_dim], name='a')
-with tf.name_scope('R'):
-    R = tf.placeholder(tf.float32, [None, 1], name='r')
-with tf.name_scope('S_'):
-    S_ = tf.placeholder(tf.float32, shape=[None, state_dim], name='s_')
-
 sess = tf.Session()
 
 # Create actor and critic.
-actor = Actor(sess, action_dim, action_bound, LR_A, REPLACE_ITER_A)
-critic = Critic(sess, state_dim, action_dim, LR_C, GAMMA, REPLACE_ITER_C, actor.a_)
+actor = Actor(sess, ACTION_DIM, ACTION_BOUND, LR_A, REPLACE_ITER_A)
+critic = Critic(sess, STATE_DIM, ACTION_DIM, LR_C, GAMMA, REPLACE_ITER_C, actor.a_)
 actor.add_grad_to_graph(critic.a_grads)
 
 M = Memory(MEMORY_CAPACITY)
@@ -328,7 +328,7 @@ def _get_priority(self, error):
     tf.summary.FileWriter('logs', graph=sess.graph)
 
 var = 3  # control exploration
-var_min = 0.008
+var_min = 0.001
 
 for i_episode in range(MAX_EPISODES):
     # s = (hull angle speed, angular velocity, horizontal speed, vertical speed, position of joints and joints angular speed, legs contact with ground, and 10 lidar rangefinder measurements.)
@@ -342,17 +342,19 @@ def _get_priority(self, error):
         s_, r, done, _ = env.step(a)    # r = total 300+ points up to the far end. If the robot falls, it gets -100.
 
         if r == -100: r = -2
+        ep_r += r
+
         transition = np.hstack((s, a, [r], s_))
         max_p = np.max(M.tree.tree[-M.tree.capacity:])
         M.store(max_p, transition)
 
         if GLOBAL_STEP.eval(sess) > MEMORY_CAPACITY/20:
-            var = max([var*0.99995, var_min])  # decay the action randomness
+            var = max([var*0.9999, var_min])  # decay the action randomness
             tree_idx, b_M, ISWeights = M.prio_sample(BATCH_SIZE)    # for critic update
-            b_s = b_M[:, :state_dim]
-            b_a = b_M[:, state_dim: state_dim + action_dim]
-            b_r = b_M[:, -state_dim - 1: -state_dim]
-            b_s_ = b_M[:, -state_dim:]
+            b_s = b_M[:, :STATE_DIM]
+            b_a = b_M[:, STATE_DIM: STATE_DIM + ACTION_DIM]
+            b_r = b_M[:, -STATE_DIM - 1: -STATE_DIM]
+            b_s_ = b_M[:, -STATE_DIM:]
 
             abs_td = critic.learn(b_s, b_a, b_r, b_s_, ISWeights)
             actor.learn(b_s, b_a)
@@ -379,9 +381,10 @@ def _get_priority(self, error):
                   '| Epi_r: %.2f' % ep_r,
                   '| Exploration: %.3f' % var,
                   '| Pos: %.i' % int(env.unwrapped.hull.position[0]),
+                  '| LR_A: %.6f' % sess.run(LR_A),
+                  '| LR_C: %.6f' % sess.run(LR_C),
                   )
             break
 
         s = s_
-        ep_r += r
         sess.run(INCREASE_GS)

Reinforcement_learning_TUT/experiments/Solve_BipedalWalker/DDPG.py

@@ -149,10 +149,10 @@ def _bias_variable(self, shape, name='biases'):
             feed_dict=feed_dict)
 
         # plotting
-        # plt.plot(xs[0, :], res[0].flatten(), 'r', xs[0, :], pred.flatten()[:TIME_STEPS], 'b--')
-        # plt.ylim((-1.2, 1.2))
-        # plt.draw()
-        # plt.pause(0.3)
+        plt.plot(xs[0, :], res[0].flatten(), 'r', xs[0, :], pred.flatten()[:TIME_STEPS], 'b--')
+        plt.ylim((-1.2, 1.2))
+        plt.draw()
+        plt.pause(0.3)
 
         if i % 20 == 0:
             print('cost: ', round(cost, 4))