add rendering and update README

README.md

@@ -37,6 +37,11 @@ To get started, checkout the example scripts, linked above.
 ## What's New
 ### Version 0.2
 
+#### 04/05-15/2019
+ - Add rendering
+ - Fix SAC bug to account for future entropy (#41, #43)
+ - Add online algorithm mode (#42)
+
 #### 04/05/2019
 
 The initial release for 0.2 has the following major changes:

rlkit/samplers/data_collector/path_collector.py

@@ -12,11 +12,17 @@ def __init__(
             env,
             policy,
             max_num_epoch_paths_saved=None,
+            render=False,
+            render_kwargs=None,
     ):
+        if render_kwargs is None:
+            render_kwargs = {}
         self._env = env
         self._policy = policy
         self._max_num_epoch_paths_saved = max_num_epoch_paths_saved
         self._epoch_paths = deque(maxlen=self._max_num_epoch_paths_saved)
+        self._render = render
+        self._render_kwargs = render_kwargs
 
         self._num_steps_total = 0
         self._num_paths_total = 0
@@ -85,12 +91,18 @@ def __init__(
             env,
             policy,
             max_num_epoch_paths_saved=None,
+            render=False,
+            render_kwargs=None,
             observation_key='observation',
             desired_goal_key='desired_goal',
     ):
+        if render_kwargs is None:
+            render_kwargs = {}
         self._env = env
         self._policy = policy
         self._max_num_epoch_paths_saved = max_num_epoch_paths_saved
+        self._render = render
+        self._render_kwargs = render_kwargs
         self._epoch_paths = deque(maxlen=self._max_num_epoch_paths_saved)
         self._observation_key = observation_key
         self._desired_goal_key = desired_goal_key
@@ -115,6 +127,8 @@ def collect_new_paths(
                 self._env,
                 self._policy,
                 max_path_length=max_path_length_this_loop,
+                render=self._render,
+                render_kwargs=self._render_kwargs,
                 observation_key=self._observation_key,
                 desired_goal_key=self._desired_goal_key,
                 return_dict_obs=True,

rlkit/samplers/data_collector/step_collector.py

@@ -13,11 +13,17 @@ def __init__(
             env,
             policy,
             max_num_epoch_paths_saved=None,
+            render=False,
+            render_kwargs=None,
     ):
+        if render_kwargs is None:
+            render_kwargs = {}
         self._env = env
         self._policy = policy
         self._max_num_epoch_paths_saved = max_num_epoch_paths_saved
         self._epoch_paths = deque(maxlen=self._max_num_epoch_paths_saved)
+        self._render = render
+        self._render_kwargs = render_kwargs
 
         self._num_steps_total = 0
         self._num_paths_total = 0
@@ -70,6 +76,8 @@ def collect_one_step(
         next_ob, reward, terminal, env_info = (
             self._env.step(action)
         )
+        if self._render:
+            self._env.render(**self._render_kwargs)
         terminal = np.array([terminal])
         reward = np.array([reward])
         # store path obs
@@ -118,13 +126,19 @@ def __init__(
             env,
             policy,
             max_num_epoch_paths_saved=None,
+            render=False,
+            render_kwargs=None,
             observation_key='observation',
             desired_goal_key='desired_goal',
     ):
+        if render_kwargs is None:
+            render_kwargs = {}
         self._env = env
         self._policy = policy
         self._max_num_epoch_paths_saved = max_num_epoch_paths_saved
         self._epoch_paths = deque(maxlen=self._max_num_epoch_paths_saved)
+        self._render = render
+        self._render_kwargs = render_kwargs
         self._observation_key = observation_key
         self._desired_goal_key = desired_goal_key
 
@@ -192,6 +206,8 @@ def collect_one_step(
         next_ob, reward, terminal, env_info = (
             self._env.step(action)
         )
+        if self._render:
+            self._env.render(**self._render_kwargs)
         terminal = np.array([terminal])
         reward = np.array([reward])
         # store path obs

rlkit/samplers/rollout_functions.py

@@ -5,12 +5,15 @@ def multitask_rollout(
         env,
         agent,
         max_path_length=np.inf,
-        animated=False,
+        render=False,
+        render_kwargs=None,
         observation_key=None,
         desired_goal_key=None,
         get_action_kwargs=None,
         return_dict_obs=False,
 ):
+    if render_kwargs is None:
+        render_kwargs = {}
     if get_action_kwargs is None:
         get_action_kwargs = {}
     dict_obs = []
@@ -25,8 +28,8 @@ def multitask_rollout(
     path_length = 0
     agent.reset()
     o = env.reset()
-    if animated:
-        env.render()
+    if render:
+        env.render(**render_kwargs)
     goal = o[desired_goal_key]
     while path_length < max_path_length:
         dict_obs.append(o)
@@ -35,8 +38,8 @@ def multitask_rollout(
         new_obs = np.hstack((o, goal))
         a, agent_info = agent.get_action(new_obs, **get_action_kwargs)
         next_o, r, d, env_info = env.step(a)
-        if animated:
-            env.render()
+        if render:
+            env.render(**render_kwargs)
         observations.append(o)
         rewards.append(r)
         terminals.append(d)
@@ -70,7 +73,13 @@ def multitask_rollout(
     )
 
 
-def rollout(env, agent, max_path_length=np.inf, animated=False):
+def rollout(
+        env,
+        agent,
+        max_path_length=np.inf,
+        render=False,
+        render_kwargs=None,
+):
     """
     The following value for the following keys will be a 2D array, with the
     first dimension corresponding to the time dimension.
@@ -85,6 +94,8 @@ def rollout(env, agent, max_path_length=np.inf, animated=False):
      - agent_infos
      - env_infos
     """
+    if render_kwargs is None:
+        render_kwargs = {}
     observations = []
     actions = []
     rewards = []
@@ -95,8 +106,8 @@ def rollout(env, agent, max_path_length=np.inf, animated=False):
     agent.reset()
     next_o = None
     path_length = 0
-    if animated:
-        env.render()
+    if render:
+        env.render(**render_kwargs)
     while path_length < max_path_length:
         a, agent_info = agent.get_action(o)
         next_o, r, d, env_info = env.step(a)
@@ -110,8 +121,8 @@ def rollout(env, agent, max_path_length=np.inf, animated=False):
         if d:
             break
         o = next_o
-        if animated:
-            env.render()
+        if render:
+            env.render(**render_kwargs)
 
     actions = np.array(actions)
     if len(actions.shape) == 1:

rlkit/samplers/util.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 
-def rollout(env, agent, max_path_length=np.inf, animated=False):
+def rollout(env, agent, max_path_length=np.inf, render=False):
     """
     The following value for the following keys will be a 2D array, with the
     first dimension corresponding to the time dimension.
@@ -19,7 +19,7 @@ def rollout(env, agent, max_path_length=np.inf, animated=False):
     :param env:
     :param agent:
     :param max_path_length:
-    :param animated:
+    :param render:
     :return:
     """
     observations = []
@@ -31,7 +31,7 @@ def rollout(env, agent, max_path_length=np.inf, animated=False):
     o = env.reset()
     next_o = None
     path_length = 0
-    if animated:
+    if render:
         env.render()
     while path_length < max_path_length:
         a, agent_info = agent.get_action(o)
@@ -46,7 +46,7 @@ def rollout(env, agent, max_path_length=np.inf, animated=False):
         if d:
             break
         o = next_o
-        if animated:
+        if render:
             env.render()
 
     actions = np.array(actions)

rlkit/torch/ddpg/ddpg.py

@@ -7,7 +7,7 @@
 import torch
 from rlkit.core.eval_util import create_stats_ordered_dict
 from rlkit.policies.simple import RandomPolicy
-from rlkit.samplers.util import rollout
+from rlkit.samplers.rollout_functions import rollout
 from rlkit.torch.torch_rl_algorithm import TorchRLAlgorithm
 from rlkit.torch.data_management.normalizer import TorchFixedNormalizer
 from torch import nn as nn

rlkit/torch/skewfit/video_gen.py

@@ -60,7 +60,7 @@ def dump_video(
             env,
             policy,
             max_path_length=horizon,
-            animated=False,
+            render=False,
         )
         is_vae_env = isinstance(env, VAEWrappedEnv)
         l = []

rlkit/util/video.py

@@ -35,7 +35,7 @@ def dump_video(
             env,
             policy,
             max_path_length=horizon,
-            animated=False,
+            render=False,
         )
         is_vae_env = isinstance(env, VAEWrappedEnv)
         l = []

scripts/run_goal_conditioned_policy.py

@@ -26,7 +26,7 @@ def simulate_policy(args):
             env,
             policy,
             max_path_length=args.H,
-            animated=not args.hide,
+            render=not args.hide,
             observation_key='observation',
             desired_goal_key='desired_goal',
         ))

scripts/run_policy.py

@@ -1,4 +1,4 @@
-from rlkit.samplers.util import rollout
+from rlkit.samplers.rollout_functions import rollout
 from rlkit.torch.pytorch_util import set_gpu_mode
 import argparse
 import joblib
@@ -21,7 +21,7 @@ def simulate_policy(args):
             env,
             policy,
             max_path_length=args.H,
-            animated=True,
+            render=True,
         )
         if hasattr(env, "log_diagnostics"):
             env.log_diagnostics([path])