From 1bf8b0e2d72a84cbc923833311091134c22340e4 Mon Sep 17 00:00:00 2001 From: Alex Nikulkov Date: Tue, 1 Dec 2020 14:06:03 -0800 Subject: [PATCH] add example notebooks to reagent Summary: Adding a notebook (courtesy of Badri) with example of how to use REINFORCE along with a test to make sure the example is up-to-date Reviewed By: czxttkl Differential Revision: D25133358 fbshipit-source-id: 7e486ede4bcf0c47831ee89dd7696e095e25df71 --- .../REINFORCE_for_CartPole_Control.ipynb | 527 ++++++++++++++++++ reagent/test/notebooks/test_notebooks.py | 10 + 2 files changed, 537 insertions(+) create mode 100644 reagent/notebooks/REINFORCE_for_CartPole_Control.ipynb create mode 100644 reagent/test/notebooks/test_notebooks.py diff --git a/reagent/notebooks/REINFORCE_for_CartPole_Control.ipynb b/reagent/notebooks/REINFORCE_for_CartPole_Control.ipynb new file mode 100644 index 000000000..c367f1d3d --- /dev/null +++ b/reagent/notebooks/REINFORCE_for_CartPole_Control.ipynb @@ -0,0 +1,527 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use the [CartPole-v1](https://gym.openai.com/envs/CartPole-v0/) OpenAI Gym environment. For reproducibility, let is fix a random seed." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:57.506601Z", + "start_time": "2020-11-20T19:04:56.642944Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I1120 110456.710 dataclasses.py:49] USE_VANILLA_DATACLASS: True\n", + "I1120 110456.712 dataclasses.py:50] ARBITRARY_TYPES_ALLOWED: True\n", + "I1120 110456.736 io.py:19] Registered Manifold PathManager\n", + "I1120 110456.984 patch.py:95] Patched torch.load, torch.save, torch.jit.load and save to handle Manifold uri\n", + "I1120 110457.027 registry_meta.py:19] Adding REGISTRY to type TrainingReport\n", + "I1120 110457.028 registry_meta.py:40] Not Registering TrainingReport to TrainingReport. Abstract method [] are not implemented.\n", + "I1120 110457.029 registry_meta.py:19] Adding REGISTRY to type PublishingResult\n", + "I1120 110457.030 registry_meta.py:40] Not Registering PublishingResult to PublishingResult. Abstract method [] are not implemented.\n", + "I1120 110457.031 registry_meta.py:19] Adding REGISTRY to type ValidationResult\n", + "I1120 110457.032 registry_meta.py:40] Not Registering ValidationResult to ValidationResult. Abstract method [] are not implemented.\n", + "I1120 110457.033 registry_meta.py:31] Registering NoPublishingResults to PublishingResult\n", + "I1120 110457.033 registry_meta.py:34] Using no_publishing_results instead of NoPublishingResults\n", + "I1120 110457.034 registry_meta.py:31] Registering NoValidationResults to ValidationResult\n", + "I1120 110457.035 registry_meta.py:34] Using no_validation_results instead of NoValidationResults\n", + "I1120 110457.048 registry_meta.py:31] Registering SchedulingFrequencyValidationResults to ValidationResult\n", + "I1120 110457.049 registry_meta.py:34] Using scheduling_frequency_validation_results instead of SchedulingFrequencyValidationResults\n", + "I1120 110457.050 registry_meta.py:31] Registering PDIVFilterValidationResults to ValidationResult\n", + "I1120 110457.050 registry_meta.py:34] Using pdiv_filter_validation_results instead of PDIVFilterValidationResults\n", + "I1120 110457.051 registry_meta.py:31] Registering Seq2SlateValidationResults to ValidationResult\n", + "I1120 110457.053 registry_meta.py:34] Using seq2slate_validation_results instead of Seq2SlateValidationResults\n", + "I1120 110457.053 registry_meta.py:31] Registering SchedulingFrequencyPublishingResults to PublishingResult\n", + "I1120 110457.054 registry_meta.py:34] Using scheduling_frequency_publishing_results instead of SchedulingFrequencyPublishingResults\n", + "I1120 110457.055 registry_meta.py:31] Registering PDIVFilterPublishingResults to PublishingResult\n", + "I1120 110457.055 registry_meta.py:34] Using pdiv_filter_publishing_results instead of PDIVFilterPublishingResults\n", + "I1120 110457.057 registry_meta.py:31] Registering FeedPublishingResults to PublishingResult\n", + "I1120 110457.057 registry_meta.py:34] Using feed_publishing_results instead of FeedPublishingResults\n", + "I1120 110457.058 registry_meta.py:31] Registering ScoreFblearnerPredictorPublishingResult to PublishingResult\n", + "I1120 110457.059 registry_meta.py:34] Using score_offline_results instead of ScoreFblearnerPredictorPublishingResult\n", + "I1120 110457.060 registry_meta.py:31] Registering ScoreSeq2SlateOutput to PublishingResult\n", + "I1120 110457.060 registry_meta.py:34] Using score_seq2slate_offline instead of ScoreSeq2SlateOutput\n", + "I1120 110457.062 registry_meta.py:31] Registering SlateRewardFeatureImportanceOutput to PublishingResult\n", + "I1120 110457.062 registry_meta.py:34] Using slate_reward_feature_importance instead of SlateRewardFeatureImportanceOutput\n", + "I1120 110457.065 dataclasses.py:74] Setting IdMapping.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.066 dataclasses.py:74] Setting ModelFeatureConfig.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.100 registry_meta.py:19] Adding REGISTRY to type LearningRateSchedulerConfig\n", + "I1120 110457.100 registry_meta.py:40] Not Registering LearningRateSchedulerConfig to LearningRateSchedulerConfig. Abstract method [] are not implemented.\n", + "I1120 110457.101 registry_meta.py:31] Registering LambdaLR to LearningRateSchedulerConfig\n", + "I1120 110457.102 registry_meta.py:31] Registering MultiplicativeLR to LearningRateSchedulerConfig\n", + "I1120 110457.103 registry_meta.py:31] Registering StepLR to LearningRateSchedulerConfig\n", + "I1120 110457.105 registry_meta.py:31] Registering MultiStepLR to LearningRateSchedulerConfig\n", + "I1120 110457.106 registry_meta.py:31] Registering ExponentialLR to LearningRateSchedulerConfig\n", + "I1120 110457.107 registry_meta.py:31] Registering CosineAnnealingLR to LearningRateSchedulerConfig\n", + "I1120 110457.108 registry_meta.py:31] Registering CyclicLR to LearningRateSchedulerConfig\n", + "I1120 110457.109 registry_meta.py:31] Registering OneCycleLR to LearningRateSchedulerConfig\n", + "I1120 110457.110 registry_meta.py:31] Registering CosineAnnealingWarmRestarts to LearningRateSchedulerConfig\n", + "I1120 110457.113 registry_meta.py:19] Adding REGISTRY to type OptimizerConfig\n", + "I1120 110457.113 registry_meta.py:40] Not Registering OptimizerConfig to OptimizerConfig. Abstract method [] are not implemented.\n", + "I1120 110457.114 registry_meta.py:31] Registering Adam to OptimizerConfig\n", + "I1120 110457.115 registry_meta.py:31] Registering SGD to OptimizerConfig\n", + "I1120 110457.117 registry_meta.py:31] Registering AdamW to OptimizerConfig\n", + "I1120 110457.118 registry_meta.py:31] Registering SparseAdam to OptimizerConfig\n", + "I1120 110457.119 registry_meta.py:31] Registering Adamax to OptimizerConfig\n", + "I1120 110457.121 registry_meta.py:31] Registering LBFGS to OptimizerConfig\n", + "I1120 110457.122 registry_meta.py:31] Registering Rprop to OptimizerConfig\n", + "I1120 110457.123 registry_meta.py:31] Registering ASGD to OptimizerConfig\n", + "I1120 110457.125 registry_meta.py:31] Registering Adadelta to OptimizerConfig\n", + "I1120 110457.126 registry_meta.py:31] Registering Adagrad to OptimizerConfig\n", + "I1120 110457.127 registry_meta.py:31] Registering RMSprop to OptimizerConfig\n", + "I1120 110457.374 dataclasses.py:74] Setting Seq2SlateNet.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.386 registry_meta.py:19] Adding REGISTRY to type EnvWrapper\n", + "I1120 110457.386 registry_meta.py:40] Not Registering EnvWrapper to EnvWrapper. Abstract method ['obs_preprocessor', 'serving_obs_preprocessor', 'make'] are not implemented.\n", + "I1120 110457.387 dataclasses.py:74] Setting EnvWrapper.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.391 registry_meta.py:31] Registering ChangingArms to EnvWrapper\n", + "I1120 110457.409 registry_meta.py:31] Registering Gym to EnvWrapper\n", + "I1120 110457.414 utils.py:19] Registering id=Pocman-v0, entry_point=reagent.gym.envs.pomdp.pocman:PocManEnv.\n", + "I1120 110457.415 utils.py:19] Registering id=StringGame-v0, entry_point=reagent.gym.envs.pomdp.string_game:StringGameEnv.\n", + "I1120 110457.415 utils.py:19] Registering id=LinearDynamics-v0, entry_point=reagent.gym.envs.dynamics.linear_dynamics:LinDynaEnv.\n", + "I1120 110457.416 utils.py:19] Registering id=PossibleActionsMaskTester-v0, entry_point=reagent.gym.envs.functionality.possible_actions_mask_tester:PossibleActionsMaskTester.\n", + "I1120 110457.447 registry_meta.py:31] Registering RecSim to EnvWrapper\n", + "I1120 110457.448 dataclasses.py:74] Setting RecSim.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.449 registry_meta.py:31] Registering OraclePVM to EnvWrapper\n", + "I1120 110457.450 dataclasses.py:74] Setting OraclePVM.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.464 env_wrapper.py:40] Env: >>;\n", + "observation_space: Box(4,);\n", + "action_space: Discrete(2);\n" + ] + } + ], + "source": [ + "from reagent.gym.envs.gym import Gym\n", + "\n", + "env = Gym('CartPole-v0')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:57.547338Z", + "start_time": "2020-11-20T19:04:57.508500Z" + } + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "\n", + "def reset_env(env, seed):\n", + " np.random.seed(seed)\n", + " env.seed(seed)\n", + " env.action_space.seed(seed)\n", + " torch.manual_seed(seed)\n", + " env.reset()\n", + "\n", + "reset_env(env, seed=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `policy` is composed of a simple scorer (a MLP) and a softmax sampler. Our `agent` simply executes this policy in the CartPole Environment." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:57.640570Z", + "start_time": "2020-11-20T19:04:57.549258Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I1120 110457.591 registry_meta.py:19] Adding REGISTRY to type DiscreteDQNNetBuilder\n", + "I1120 110457.592 registry_meta.py:40] Not Registering DiscreteDQNNetBuilder to DiscreteDQNNetBuilder. Abstract method ['build_q_network'] are not implemented.\n", + "I1120 110457.592 registry_meta.py:31] Registering Dueling to DiscreteDQNNetBuilder\n", + "I1120 110457.593 dataclasses.py:74] Setting Dueling.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.595 registry_meta.py:31] Registering FullyConnected to DiscreteDQNNetBuilder\n", + "I1120 110457.596 dataclasses.py:74] Setting FullyConnected.__post_init__ to its __post_init_post_parse__\n", + "I1120 110457.597 registry_meta.py:31] Registering FullyConnectedWithEmbedding to DiscreteDQNNetBuilder\n", + "I1120 110457.597 dataclasses.py:74] Setting FullyConnectedWithEmbedding.__post_init__ to its __post_init_post_parse__\n" + ] + } + ], + "source": [ + "from reagent.net_builder.discrete_dqn.fully_connected import FullyConnected\n", + "from reagent.gym.utils import build_normalizer\n", + "\n", + "norm = build_normalizer(env)\n", + "net_builder = FullyConnected(sizes=[8], activations=[\"linear\"])\n", + "cartpole_scorer = net_builder.build_q_network(\n", + " state_feature_config=None, \n", + " state_normalization_data=norm['state'],\n", + " output_dim=len(norm['action'].dense_normalization_parameters))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:57.681315Z", + "start_time": "2020-11-20T19:04:57.642496Z" + } + }, + "outputs": [], + "source": [ + "from reagent.gym.policies.policy import Policy\n", + "from reagent.gym.policies.samplers.discrete_sampler import SoftmaxActionSampler\n", + "from reagent.gym.agents.agent import Agent\n", + "\n", + "\n", + "policy = Policy(scorer=cartpole_scorer, sampler=SoftmaxActionSampler())\n", + "agent = Agent.create_for_env(env, policy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a trainer that uses the REINFORCE Algorithm to train." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:57.745840Z", + "start_time": "2020-11-20T19:04:57.682931Z" + } + }, + "outputs": [], + "source": [ + "from reagent.training.reinforce import (\n", + " Reinforce, ReinforceParams\n", + ")\n", + "from reagent.optimizer.union import classes\n", + "\n", + "\n", + "trainer = Reinforce(policy, ReinforceParams(\n", + " gamma=0.99,\n", + " optimizer=classes['Adam'](lr=5e-3, weight_decay=1e-3)\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Transform the trajectory of observed transitions into a training batch" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:57.785002Z", + "start_time": "2020-11-20T19:04:57.747286Z" + } + }, + "outputs": [], + "source": [ + "import torch.nn.functional as F\n", + "import reagent.types as rlt\n", + "\n", + "\n", + "def to_train_batch(trajectory):\n", + " return rlt.PolicyGradientInput(\n", + " state=rlt.FeatureData(torch.from_numpy(np.stack(trajectory.observation)).float()),\n", + " action=F.one_hot(torch.from_numpy(np.stack(trajectory.action)), 2),\n", + " reward=torch.tensor(trajectory.reward),\n", + " log_prob=torch.tensor(trajectory.log_prob)\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "RL Interaction Loop" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:57.822558Z", + "start_time": "2020-11-20T19:04:57.786562Z" + } + }, + "outputs": [], + "source": [ + "from reagent.gym.runners.gymrunner import evaluate_for_n_episodes" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:04:58.478743Z", + "start_time": "2020-11-20T19:04:57.824212Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I1120 110458.392 gymrunner.py:134] For gamma=1.0, average reward is 17.7\n", + "Rewards list: [14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23.\n", + " 14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23. 14. 23.\n", + " 14. 23. 14. 23. 25. 13. 25. 13. 25. 13. 25. 13. 25. 13. 25. 13. 25. 13.\n", + " 25. 13. 25. 13. 25. 13. 25. 13. 25. 13. 25. 13. 25. 13. 25. 13. 25. 13.\n", + " 25. 13. 25. 13. 25. 13. 25. 13. 13. 14. 13. 14. 13. 14. 13. 14. 13. 14.\n", + " 13. 14. 13. 14. 13. 14. 13. 14. 13. 14.]\n" + ] + } + ], + "source": [ + "eval_rewards = evaluate_for_n_episodes(100, env, agent, 500, num_processes=20)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:05:33.327901Z", + "start_time": "2020-11-20T19:04:58.481482Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 500/500 [00:34<00:00, 14.37 epoch/s, reward=200] \n" + ] + } + ], + "source": [ + "num_episodes = 500\n", + "reward_min = 20\n", + "max_steps = 500\n", + "reward_decay = 0.8\n", + "\n", + "train_rewards = []\n", + "running_reward = reward_min\n", + "\n", + "\n", + "import tqdm.autonotebook as tqdm\n", + "from reagent.gym.runners.gymrunner import run_episode\n", + "\n", + "\n", + "with tqdm.trange(num_episodes, unit=\" epoch\") as t:\n", + " for i in t:\n", + " trajectory = run_episode(env, agent, max_steps=max_steps, mdp_id=i)\n", + " batch = to_train_batch(trajectory)\n", + " trainer.train(batch)\n", + " ep_reward = trajectory.calculate_cumulative_reward(1.0)\n", + " running_reward *= reward_decay\n", + " running_reward += (1 - reward_decay) * ep_reward\n", + " train_rewards.append(ep_reward)\n", + " t.set_postfix(reward=running_reward)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Print the mean reward." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:05:34.634251Z", + "start_time": "2020-11-20T19:05:33.329881Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I1120 110534.523 gymrunner.py:134] For gamma=1.0, average reward is 200.0\n", + "Rewards list: [200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200.\n", + " 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200.\n", + " 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200.\n", + " 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200.\n", + " 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200.\n", + " 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200.\n", + " 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200. 200.\n", + " 200. 200.]\n" + ] + } + ], + "source": [ + "eval_episodes = 200\n", + "eval_rewards = evaluate_for_n_episodes(100, env, agent, 500, num_processes=20).T[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:05:34.689980Z", + "start_time": "2020-11-20T19:05:34.636213Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean reward: 200.00\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "mean_reward = pd.Series(eval_rewards).mean()\n", + "print(f'Mean reward: {mean_reward:.2f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the rewards over training episodes." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:05:35.227775Z", + "start_time": "2020-11-20T19:05:34.692199Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Bad key \"axes.color_cycle\" on line 214 in\n", + "/home/alexnik/.matplotlib/matplotlibrc.\n", + "You probably need to get an updated matplotlibrc file from\n", + "https://github.com/matplotlib/matplotlib/blob/v3.1.2/matplotlibrc.template\n", + "or from the matplotlib source distribution\n" + ] + } + ], + "source": [ + "from matplotlib import pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "def plot_rewards(rewards):\n", + " fig, ax = plt.subplots(1, 1, figsize=(12, 10));\n", + " pd.Series(rewards).rolling(50).mean().plot(ax=ax);\n", + " pd.Series(rewards).plot(ax=ax,alpha=0.5,color='lightblue');\n", + " ax.set_xlabel('Episodes');\n", + " ax.set_ylabel('Reward');\n", + " plt.title('REINFORCE on CartPole');\n", + " plt.legend(['Moving Average Reward', 'Instantaneous Episode Reward'])\n", + " return fig, ax\n", + "\n", + "sns.set_style('darkgrid')\n", + "sns.set()\n", + "\n", + "\n", + "plot_rewards(train_rewards);" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-20T19:05:35.655795Z", + "start_time": "2020-11-20T19:05:35.229537Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt0AAAJlCAYAAAAGrk7qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeZgdVYH/4W93OixJWLIxLLIokOMCAiEgmzuMgODKqCA7joCCzDjuGyqIjKKCCMggIAzKiDqAgCCi4ygj/oCIMjJ6JCI7SAhrwCSku39/5CaGkBU43ST9vs+Th+6qe6vOvScJn65U1e3q7+8PAADQTvdgDwAAAFZ0ohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AHaWUW0spOw/2OIAVT89gDwBgsJVSbk3yd0l6k0xPckWSI2qt0zvrv5lknySz5nvan2qtW5RSNkry5yTDa62zO489IMnLaq3Xdp6/SZKba61dne9/lmS7JLPn294utdZrSildST6Q5N1JnpdkapJvJzm61jpzIeOZlWRykiNrrX+Y7zWtk+TYJLsnGZXkriTfSfKFWutjpZT+JI8nmf/DGj5ba/3CALzf+yR5f5IXJnk0yW+SfK7WevXT3F5/kk1rrVM6378qyU/ne313Jzm+1nr2s/5iAJaSI90Ac+xZax2VZMskWyX56ALrv1BrHTXfry0Ws60HOsG7OEcssL1rOsu/2gnu/ZOslmS3JK9JcsHCxpNkvU5Qnzl3RSllTJJrkqyaZPta62pJdkmyZpKN59vGFguMYSCC+/1JTkxyXOcHnQ2SnJrkjU9jW4s7cHR35/1ZPcmHk5xRSnnxMxs9wNPnSDfAfGqt95ZSftSJ76frnCT7lFJeWWv976V9Uill0yTv6YTytZ3FN5VS3ppkSinlNbXWny4w3r+WUi5I8t35Fr+/cwR531prX+dxdyQ56um8mFLKGklO7vwA8HiSM5IcV2vtK6UcmORdSX6V5JAkDyV5T6318kVs57NJDqq1/ud8qy7p/EopZdskJyV5UZK/Jvl+kvfXWmflb0e1j0jyT0l6Sil3dLbx2866Q5L8Zb73pz/JRaWUB5O8OMn/lVLekOTznR9YfpPk8Frr7xcy3u4kH0ryj50fWH6S5LBa6wNP530EhjZHugHmU0p5XicupzyDzTzeOZL7uWV83muT3DlfcCd/C+ZfdY5WLzjekUn2XmC8Oyf5z7nB/Sw4OckaSV6Q5JWdo/AHzbf+ZUlqknFJvpDkzM5pMgvaPskqSS5czL56k/xzZ1vbd96T9yzwmDd19vniWusrOsvmHrX/zvwPLKV0l1Le3Inm/y2lTEhyfifaxyf5YZJLSikrLWQs7+vs65VJ1k3yYJJTlv5tA/gb0Q0wx0WllEeT3JHkviRHL7D+A6WUh+b7dc4Stnd6kg1KKbstYv1X59vWrzvLxiW5ZxGPv6ez/knj6RzR3inJfvOtG7uY7czv1wu8ptct+IBSyrAkb0/y0Vrro7XWW5N8aYH93VZrPaPW2ts5yr9O59SRBY1Ncn+tdfZC1iVzfsCYXGv9Va11dmdfp3eid36fr7U+UGv962Je27qd9+f+zlzuV2utnddyWa31x7XWJ5Kc0DkNZ4eFbOPQJB+vtd7ZOZ/+00n2WsJpLQAL5S8OgDneVGu9qpTyys6Fi+M6p0rMdUKt9RNLu7Fa68xSyjFJjukciV7Q+2qt31hg2f2dYF2YdToXbD5pPKWUDToXfpYkN3bWTVvMduY3ce7Fh4sxLslKSW6bb9ltnVMz5rp37he11sdLKelcvLmgaUnGlVJ6FhXenSPRX04yKcmIzv+nJi/wsDsW9twF3F1rfd5Clq87/2vpnCJzxwKvZ64Nk1xYSpn/Xwx6Oz9Q3LUUYwCYx5FugPl0zsH+ZucI6DN1due0jDcv5eN/mmT9znnN85RS1u/c7eQnCxnv7Z1ztU8qpazaWXxVkjd3zkl+pu5P8kQnQOfa4GlG5zVJZnRO2ViU05L8oXM3ktWTfCzJgqeq9C/iuUvj7vlfS+c0mPUX8XruSLJbrXXN+X6tUmsV3MAyE90AT3Vikl1KKc/kYsp0juZ+unP3jKV5/B+TfD3Jt0op25VShpVSXtK5mPCqWutVi3jejzsx+e7Ooi937tpxTillw8yJy/VKKV8upbx0GV9Db+fOKZ8rpazW2d77k5y3LNvpbOvhJJ9Kckop5U2llBGllOGllN1KKXPvnLJakkeSTC+lvDDJ4Uux6b90zjdfGhckeX0p5bWllOFJ/iXJzCS/XMhjv9553XPfw/GllGW+ywpARDfAU9VapyY5N8kn51v8oVLK9Pl+3b+Umzt/Kc+vnuuIJN/oRO3ce4b/LMlbl/C8L3bGuHLn7ho7dI5Q/7/Oueo/SfLwAhdc/naB13TiIrZ9ZJLHktyS5OrO6TdnLcNrmqfW+uVOtH+icw/yOzqv+aLOQz7QuQf5o527pHxnCZtM5webczrnpb9tCfuvSfbtXBx6f5I9O7eLnLWQh5+U5AdJruy8h7/qXMAJsMy6+vufyb/SAQAAS+JINwAANCa6AQCgMdENAACNiW4AAGhsKHw4zspJtuncPaB3sAcDAMAKa1jnw8mu69yOdJ6hEN3bJPnFYA8CAIAh4+WdW6zOMxSi+54kefDBx9LXN/C3Rxw7dlSmTZs+4Ptl4JnrocNcDx3meugw10NHy7nu7u7K6NEjs7DPZxgK0d2bJH19/YMS3XP3zdBgrocOcz10mOuhw1wPHQMw1085pdmFlAAA0JjoBgCAxkQ3AAA0NhTO6QYA5tPbOzsPPjg1s2fPGuyhPGfcd193+vr6BnsYDIBnY657elbK6NHjM2zY0qf0gEV3KeXWJDM6v5Lkw7XWH5VStktyepJVk9yaZN9a632d5yxyHQDw9Dz44NSsssqIjBy5drq6ugZ7OM8JPT3dmT1bdA8Fz3Su+/v789hjj+TBB6dm3Lh1lvp5A316yV611i07v35USulKcl6S99ZaJyT5eZLjMye4F7kOAHj6Zs+elZEjVxfc8DR0dXVl5MjVl/lfigb7nO5JSWbUWufePPzrSd62FOsAgGdAcMPT93T+/Ax0dH+rlHJjKeXUUsqaSTZIctvclbXW+5N0l1LGLGEdALCC2GuvPfPGN74uvb1/u7XxZZf9IDvtNCnf//53nvZ2//CH/8tnPvOJZ2mUT/bJT34ke+yxS2bPnt1k+63ttdee2Weft+aAA/bOO9+5Vy655KLBHlKS5J577s7rX//awR5GEwN5IeXLa613lFJWTnJikq8luXCgdj527KiB2tVTjB+/2qDtm4FlrocOcz10rIhzfd993enpGex/7H6ysWPHZ/Lk/5cddtgpSXLFFZflhS98Ubq7u572WDfbbLNsttlxS/34pd3Pww8/nMmTr80GG2yYa675RV796mcvEmfPnp2enoHJs89//ovZeONN8qc/TckBB+yTnXZ6ecaPHz8g+84iXuuwYd1Jnv6cL61nY/vd3d3L9PfDgEV3rfWOzn9nllJOTfKDJCcl2XDuY0op45L011ofKKXcvqh1T2f/06ZNH5RPmho/frVMnfrogO+XgWeuhw5zPXSsqHPd19f3nLtocLfd9sgll/wg2267Q+6++67MmPHXPP/5G6evrz+zZ/fl8ccfz4knfjG///1NSZLXvW737Lvvgfntb2/IiSd+MWef/e152zr44H1z5JH/nP7+/pxyykk588x/zz333J13vWu/vOENb8mvfvU/mTFjRj7ykU9liy22TJJceOEF+c53zs+oUatl++13zH/+5wW57LKfLHSsP/zhZdl++x2z7bbb55JLLs7LX/7qJMnnP//ZbLzxpnnb2/ZOktxyy5R8+MP/kgsuuCiPP/5YTj75K/nTn27OrFmzstVWk3Lkkf+cYcOG5Ygj3p1NNy256ab/zeqrr57jj/9yPvShf8rDDz+cmTNn5sUvfkk++MGPZfjw4XniiSfy5S9/ITfcMDmjR4/OpptOyAMPTMuxx34hSfKtb52Tn/3sJ+nt7c24cWvlwx/+eMaOHbfQ19HbO+f3wYYbviCrrbZ67r333owePXax23nTm3bL2Wd/K6NHj8kHPvC+dHV15YtfPCkPPvhADjronbnoostz/fXX5owzTsusWTPT29ub/fc/ODvv/LokecprPeGEr+b7378gF1zw7YwcOTLbb79Tkv6mvz+frYtm+/r6nvL3Q3d31yIP9A5IdJdSRibpqbU+3LlA8h1JfpNkcpJVSyk7dc7dPizJBZ2nLW4dAPAs+J//vSdX33hPk23v9NJ1suPmS3d3h4kTJ+XCC7+bRx55JJdffml23fX1+cMffj9v/Te/+Y309fXl3HO/k8cffyyHHnpwNt5402y//Y7561//milTbs4mm2yaW26ZkunTH82WW07MDTdMftI+Hn744Wy22Utz6KHvzZVXXp6vf/2rOe20szJlys0599yzc9ZZ387o0aNz0klfWuxYf/jDH+SII/45m222eU466Uu5//6pGTdufHbffc+cdNIJ86L7sssuye6775Gurq6cfPJXsuWWE/ORj3wyfX19+cxnPpHLLvtB3vCGNydJ7r77zpx66jfS09OT/v7+HH30sVljjTXT39+fY489OpdddnHe9Ka9cvHF389f/nJvzjvvgvT29ubIIw/NWmutlST50Y9+mDvvvDOnn/7NdHd358ILv5evfe3EHH30sYt9PTfe+Jusscaa2WSTCUvczsSJkzJ58nV51atem3vvvSf9/f2ZPXt2rr/+2my99aQkyYQJL8ypp34jw4YNywMPTMshh+yXbbfdPquvvvpTXuuc9/6snH32tzJmzNiccMKKe8+MgTrS/XdJvl9KGZZkWJL/S/KeWmtfKWW/JKeXUlaZe1vAzDkivsh1AMCKpasrec1rdslPfnJlfvKTK3PaaWc+Kbqvv/7aHHXUBzp3jhiVnXf++1x//bXZfvsds+uur8/ll1+SI498fyd091zohW6rrjoiO+748iTJS16yeb72tROTJDfcMDnbb79TRo8enSTZffc9c+WVP1zoOP/4xz/k0UcfzcSJk9LV1ZVXvvLVufzyy7Lffgdmiy22yuOPP54pU27ORhs9P1dd9aOcfvrZSZKrr/55fv/7m/If//GtJMmMGTOy1lp/N2+7u+yy67xTLfr6+nL++eflV7/6Zfr6evPoo49mlVVWSZL8+teTs+uuu6enpyc9PT3ZeefX5cYbb5i3jz/84fc5+OA5udTbOzujRi369NpPfOLD6e/vz1133Zljjjk+w4cPX+J2Jk6clOuvvzbjx6+VF794s/T39+emm37Xie5tkyQPPfRgPv/5z+bOO2/PsGE9eeSRh3P77bdls802f8prveGGydlhh50yZsycI+xvfOOb81//9eMl/n5ZHg1IdNdab0my1SLW/TLJ5su6DgB45nbcfOmPRre222575NBDD8yWW07MGmusucDa/izY0XPDetdd98ihhx6Qd7/7vU8K3QWttNLweV93d3ent3fORZD9/U/d9qJceunFmT790fzDP7whSfLEE7MyYsTI7LffgZ2xvD6XX35pttpq62y00fOz9tpz39v+HHfcCVlvvectdLurrjpi3tc//vEVufHG3+TUU8/IiBEjc+65Z+WOO26fN9Zk4YPt7+/PAQccnD32eONSvZZjj/3XvOAFm+SnP70qxx33mWy++RYZM2bsYrczadK2OeecMzN+/FrZeutt0t/fn8mTr83kydfloIPenST50peOz447viLHHffFdHV15R3veEtmzZq50Nc65/UMDc+tqygAgCFrvfWel3/8x/fkgAPe9ZR1kya9LJdeenH6+/vz+OOP5Sc/uTKTJs05srr22mtno41ekBNPPCEbbfSC+UJ36Wy11db55S//Jw899FCS5IorLl3o42bNmpWrrroyZ5xxbr73vUvyve9dkosv/lG6urry29/+Jun8AHDVVT/KpZdelN1333Pec3fc8RU577xz5t2h5aGHHsrdd9+10P1Mn/5o1lhjzYwYMTLTp0/Pj398xbx1EydOypVX/jCzZ8/OzJkz89Of/u2o8E47vSIXXvi9PPLII/PGe/PNf1zi63/Na3bONttsl/PO++YSt7P22uuku7s7V1xxWbbeettMmvSyXH75penp6cnaa6+dJHn00UezzjrrpKurK9dd96vcddcdi9z3xImTcs01/5MHH5xzyd6ll168xPEur3wMPADwnPHGN75locsPPPBd+cpXvpD993970rmQcrvtdpi3fvfd98wxx3wqn/zkZ5d5n5tuOiH77ntADjvsoIwYMTKTJm2TkSOfelrGL37xs6y33vOy/vobPGn5LrvsmssuuzhbbLHlvB8Abrhhcj796b/dOeWoo/4lp5761Rx44N7p6urK8OEr5X3v+5esu+56T9nPrrvukV/84ufZZ5+3ZvToMdlii60yc+acI8VvetNbM2XKH7Pvvm/LmmuumQ033Gi+570+Dz/8UI48cs4R576+vrz5zf+QTTedsMT34LDDjsghh+ybd77zgCVuZ+utt8mNN/4248bNuUBz5ZVXzktfuuW8bR1++BH50pf+NWee+W950YtenI033nSR+91kk02z334H5fDDD8mIESOz/fY7LnGsy6uuIXBYf6Mkf3b3Eloz10OHuR46VtS5vvfe27L22hsuxSOHjpkz/5qVV141SXLmmafnrrvuzKc+dcxgD2uhHn/8sYwYMTKzZs3KRz7y/rz61Ttnzz3fNNjDWm48W3cvWdifo/nuXvL8zvWIf9vvM94jAMBy7tRTv5rf/va3mT37iay77nr50Ic+PthDWqSjjnpPnnjiicyaNTOTJm2b3XbbY7CHxFIQ3QDAkPfBD370OXfv8kU544xzBnsIPA0upAQAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAg2qvvfbMLbdMedrPP/PM0/PEE08843FccMG3530y4vJor732zD77vDUHHrjPvF/33HP3Ep934IH7ZObMGc/KGH74w0vyiU98aJmfN3fsBxywd975zr1yySUXPSvjeabuuefuvP71r31WtuWWgQDAcu3ss8/I3nvvl+HDhz+j7VxwwfmZNGnbjB495lkb20A79th/zQtesMkyPeeb3/x2s/Esi7ljv+WWKTn44H2z/fY7Zty48QO2/9mzZ6enp10ai24A4DnjiCPenRe96CX53e9uzP3335/XvGbnHH74kUmSs876t1x11Y+y0korp6sr+epXT8+//dupSZLDDz84XV3dOfnk03PNNf+T7373/MyePefo93vf+0+ZNGnbpHNEddddX5/rrvt/mTbt/uy9975561vfnrPP/kbuv39qPvGJD2ellVbO0Ucfm2nT7s8ZZ5yWWbNmpre3N/vvf3B23vl1Sxzn/fffnxNP/EL+8pd7M3PmzOy88+uy//4HJ0l+//ubcuKJJ2TGjL9mlVVWzT/90wfyohe9JL/+9fU55ZSTcuaZ/54kT/r+9ttvzec+95nMmDEjfX292W23PbPPPvst0/u6006TctBB/5hf/OK/M3PmjBx66Hvzqle9dt66K6/8eVZZZZV8+ctfyK9/fV2GD18pI0asmtNOOytJcvnll+b88/89XV1dWXfd5+VDH/pYRo8ekyeeeCJf+coX8utfX5811lgzm25anrTfb33rnPzsZz9Jb29vxo1bKx/+8Mczduy4xY71BS/YJKuttnqmTr1vXnQvajtvetNuOfvsb2X06DH5wAfel66urnzxiyflwQcfyEEHvTMXXXR5rr/+2ifN40EHvSuvfvUu8+Zx001Lbrrpf7P66qvnhBO+mu9//4JccMG3M3LkyGy//U7L9D4vjugGgCFs+uzePNbb5kNhRg7rzqieYcv8vL/85d6ccsoZefzxx/P2t78xe+zxxqyxxpq54IJv5+KLr8jKK6+Sxx9/LCuttHL+5V8+nAsv/G5OO+2sjBgxIknyspdtl112eV26urpy++235qij3pMLL/zhvO3PmDEjp59+du655+7sv//bs9tue+agg96Viy++8ElHiseOHZdTT/1Ghg0blgcemJZDDtkv2267fVZfffVFjnP99TfIscd+Kgce+K5sueXEPPHEEznqqMPzohe9OFtuuXU+/vEP5aMf/VS22eZluf76a/Pxj38o3/nO4k+l+M///F522ukV2W+/g5IkjzzyyCIfO/eHhiQZNmzYvIhPku7u7nzzm9/O7bffmsMOOyRbbLHVk47qT5nyx9xww/U577zvpru7e95+brllSr7+9a/lzDPPy7hx43LGGaflK1/5Yj772c/n4ou/n3vuuTvnnffdzJ49O+997z9mnXXWSZL86Ec/zJ133pnTT/9muru7c+GF38vXvnZijj762MW+3htv/E3WWGPNbLLJhCVuZ+LESZk8+bq86lWvzb333pP+/v7Mnj07119/bbbeelKSZMKEFz5lHrfe+mXz5vHuu+/Mqad+Iz09PZky5eace+5ZOfvsb2XMmLE54YTjFzvWZSG6AYDnlFe/+rXp7u7OqFGjsuGGz89dd92ZddddL+utt36OOebobLvtdtlhh5dnxIiRC33+XXfdmU9/+uOZOnVqenp68sAD0zJt2v3zjrDuvPPfJ0nWWWfdeUdUV1991FO289BDD+bzn/9s7rzz9gwb1pNHHnk4t99+WzbbbPNFjnPcuPG54YbJeeihh+Zt5/HHH8utt96aMWPGZfjw4dlmm5clSSZN2jbDhw/P7bffttj3Y8stt8qpp341M2bMyMSJkzJx4qRFPnZxp5fssccbkyQbbLBRJkyYc3R3p51eOW/9uus+L7Nnz87xxx+TiRMnZYcdXp50jrrPOdVjzvv3xje+JQceuE9n3eTsttse6enpSU9PT173ut1y442/SZJcffXP84c//D4HH7xvkqS3d3ZGjXrq+zzXJz7x4fT39+euu+7MMcccP+90ocVtZ+LESbn++mszfvxaefGLN0t/f39uuul3nejedhHz+MiT5nGXXXadd1rJDTdMzg477JQxY8Z2Xuub81//9ePFzs/SEt0AMISN6hn2tI5GtzT3SG06R2d7e3szbNiwnH762fnf//1tfv3r63PIIfvmS186OZtssulTnv/pT388Rxzxz3nFK16Vvr6+7LzzTpk1a9Z8219pge3PXug4vvSl47Pjjq/Iccd9MV1dXXnHO96SWbNmLnac/f196erqyje+ce5Tzg+eMuXmdHV1PWU/XV3JsGE96e//2784zD/eV73qtdlss5fm2mt/lfPO+2Yuu+wH+dSnjlni+7g4/f1J8uSxjBo1Kuee+53ccMPkTJ58XU477eScddZ56e/PU8Y999v+ORtaxD76c8ABB8+L/SWZ+wPDT396VY477jPZfPMtMmbM2MVuZ9KkbXPOOWdm/Pi1svXW26S/vz+TJ1+byZOvy0EHvTtZyDzuvfeT53HVVUc8acytuHsJAPCc9/jjj+Whhx7KVlttnUMOOTQveMHGueWWPyVJRowYmccemz7vsdOnT88666ybJLn00oufFLCLM3LkyEyf/rftPProo1lnnXXS1dWV6677Ve66644lbmPEiJHZYoutct5535y37C9/uTfTpt2fDTfcKLNmzcqvf3190jmCPHv27Ky//oZZd911c/fdd+WRRx5Jf39/rrrqR/Oef+edd2TMmLHZffc9c9BB/5j/+7+blur1LOiyy36QJLnjjtszZUrNS16y2ZPWP/jgg5k5c2a2226HHHbYERk1alTuvvuubL31Nrnmmv/JtGn3J0kuueSieefIT5q0Ta644oeZPXt2Zs6ckR//+Ip529tpp1fkwgu/N+80lVmzZuXmm/+4xHG+5jU7Z5tttpv3Hi5uO2uvvU66u7tzxRWXZeutt82kSS/L5Zdfmp6enqy99trJQubxzjsXPY8TJ07KNdf8z7y72Fx66cXL9B4vjiPdAMBz3vTp0/Pxj38os2bNTF9fXyZMeGFe+cpXJ0ne8Y535n3vOywrr7xKTj759Lzvfe/Pxz72gay22mp52ct2yBprrLFU+9hrr3fkuOM+m1VWWSVHH31sDj/8iHzpS/+aM8/8t7zoRS/Oxhs/9aj6wnzqU8fkq1/9cvbf/+1JJ8Q/+tFPZezYcfnc577wpAspjz32XzN8+PCMH79W3vGOfXPIIftlzJgx2XLLifnzn29Jkvz0pz/OlVdekeHDe9LV1ZWjjvqXRe57/nO6k+QjH/lEXvjCFydJ5yLCfTJjxox88IMfe8pdWu677y/51389Nr29vent7c122+2Ql7xk83R3d+fQQ9+bf/7n93YupFwvH/zgx5Ikb3jDWzJlypTsu+8/ZI011swLX/iSPPjgtCTJrru+Pg8//FCOPHLOEee+vr68+c3/kE03nbDE9/Cww47IIYfsm3e+84AlbmfrrbfJjTf+dt7pLyuvvHJe+tIt521rwXlc2L+OzLXJJptmv/0OyuGHH5IRI0Zm++13XOJYl1ZXy8PozxEbJfnztGnT09c38K91/PjVMnXqowO+XwaeuR46zPXQsaLO9b333pa1195wsIfxnNLT053Zs9tcUPpcMPcOJXMvNh3Knq25Xtifo+7urowdOypJnp/k1iete8Z7BAAAFsvpJQAAK7irr75+sIcw5DnSDQAAjYluABiChsA1XdDM0/nzI7oBYIjp6Vkpjz32iPCGp6G/vz+PPfZIenpWWopH/41zugFgiBk9enwefHBqpk9/aCkePTR0d3enr2/FvXsJf/NszHVPz0oZPXr8sj3nGe0RAFjuDBvWk3Hj1hnsYTynrKi3h+SpBmuunV4CAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjokVR6BcAABr7SURBVBsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANNYz0DsspRyd5NNJNq+1/q6Usl2S05OsmuTWJPvWWu/rPHaR6wAAYHkxoEe6SykTk2yX5PbO911Jzkvy3lrrhCQ/T3L8ktYBAMDyZMCiu5SycpJTkrwnSX9n8aQkM2qtV3e+/3qSty3FOgAAWG4M5JHuzyY5r9b65/mWbZDktrnf1FrvT9JdShmzhHUAALDcGJBzuksp2yfZJslHBmJ/CzN27KjB2nXGj19t0PbNwDLXQ4e5HjrM9dBhroeOwZjrgbqQ8pVJXpjkz6WUJHlekh8l+WqSDec+qJQyLkl/rfWBUsrti1r3dAYwbdr09PX1L8Ujn13jx6+WqVMfHfD9MvDM9dBhrocOcz10mOuho+Vcd3d3LfJA74CcXlJrPb7Wum6tdaNa60ZJ7kzyuiRfTLJqKWWnzkMPS3JB5+vJi1kHAADLjUG9T3ettS/JfklOK6Xc3Dki/pElrQMAgOXJgN+nO3OCeqP5vv5lks0X8bhFrgMAgOWFT6QEAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADTWM1A7KqVclOT5SfqSTE9yZK31N6WUCUnOSTI2ybQk+9dab+48Z5HrAABgeTGQR7oPqLVuUWvdKskJSc7qLP96klNqrROSnJLk9Pmes7h1AACwXBiw6K61Pjzft2sk6SulrJVkYpLzO8vPTzKxlDJ+cesGaswAAPBsGNBzuksp3yil3J7kc0kOSLJ+krtqrb2ZE+a9Se7uLF/cOgAAWG4M2DndmRPO78qc+N4vyReTfHKg9j127KiB2tVTjB+/2qDtm4FlrocOcz10mOuhw1wPHYMx1139/f0DvtPMCe+/JtkoSU0yttbaW0oZ1rlgctMkXUn+uLB1tdapy7CrjZL8edq06enrG/jXOn78apk69dEB3y8Dz1wPHeZ66DDXQ4e5HjpaznV3d9fcA73PT3Lrk9Y12eMCSimjSinrz/f9nkkeSHJfkt8k2buzau8kN9Rap9ZaF7luIMYMAADPloE6vWRkku+WUkYm6e0E95611v5SymFJzimlfCrJg0n2n+95i1sHAADLhQGJ7lrrX5Jst4h1f0jysmVdBwAAywufSAkAAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKCxnsWtLKV8dmk2Umv91LM2IgAAWMEsNrqTrD/f16skeWuS65LclmSDJNsm+X7jMQIAwHJtsdFdaz1o7tellP9Isnet9fvzLXtLkn9oPUgAAFieLcs53bsluWiBZRcn2f1ZHhMAAKxQliW6pyR57wLL3pPkT8/ymAAAYIWypHO65/euJBeWUj6U5K4k6yWZneQtDccHAADLvWWJ7t8m2TTJdknWTXJPkmtqrU80HB8AACz3liq6SynDkkxPsmat9RfthwUAACuOpTqnu9bam+SPSca2HxIAAKxYluX0km8lubSUclKSO5P0z11Ra/1pm+EBAMDyb1mi+/DOfz+9wPL+JC94FscEAAArlKWO7lrr89sOBQAAVkzLcp9uAADgaVjqI92llNU7p5a8Msm4JF1z19VaN2g2QgAAWM4ty5HuU5NMTPLZJGOSHJnk9iRfaTg+AABY7i1LdP99krfWWi9O0tv579uT7NdwfAAAsNxblujuTvJw5+vppZQ1O59KuUmjsQEAwAphWT8G/pVJfpLkF0lO6XxK5R8bjg8AAJZ7y3Kk+x+T3Nr5+n1J/ppkzST7NxobAACsEJblPt23zPf11CTvajYqAABYgSzLLQNvSPKzJP+d5Oe11gfaDg0AAFYMy3J6yQeSPJLkn5LcWUq5sZRycillr4bjAwCA5d6ynF7yk85FlCmljE3y/iRHJHlPkmFNRwkAAMuxZTm9ZNfO3UtemWT9JNck+WjndBMAAGARluWWgT9M8qckn09ybq11dsNxAQDACmNZovsVSV6e5B+SHFtK+d18F1X+ouEYAQBgubYs53RfneTqJJ8vpayV5KgkH0ryWed0AwDAoi3LOd1vTvKqzjndE5JMTvI153QDAMDiLcvpJUd1Avv9Sa6ptf614bgAAGCFsSynl7yq7VAAAGDFtCynl6yc5FNJ9k4ytta6Rinl75NMqLV+re0wAQBg+bUsn0h5YpLNkrwzSX9n2U1JDm80NgAAWCEsS3S/Kck+tdZrkvRlzikndyVZr93wAABg+bcs0T1rwdNRSinjk0x79ocFAAArjmWJ7u8mOaeU8vzMCe51OrcM/I92wwMAgOXfskT3x5LcmuR/k6yZ5OYkdyf5TMPxAQDAcm+po7vWOqvW+k+11lFJ/i7Jakm+meRbbYcIAADLtyXeMrCUMiLJR5Ns2Tm6/elOcJ+eZJck5w7MUAEAYPm0NPfpPiXJVkl+lGS3JJsneWGSc5K8u9Z6/wCMEwAAlltLE92vS7JlrfW+UsrJSW5P8qpa688HYHwAALDcW5pzukfVWu/LnPO670wyXXADAMDSW5oj3T2llFcn6Zq7YMHva60/bTZCAABYzi1NdN+X5Kz5vp+2wPf9SV7QYGwAALBCWGJ011o3GpihAADAimlZPhwHAAB4GkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAYz0DsZNSytgk/55k4yQzk0xJcmitdWopZbskpydZNcmtSfattd7Xed4i1wEAwPJioI509yf5Qq211FpfmuRPSY4vpXQlOS/Je2utE5L8PMnxmRPci1wHAADLkwGJ7lrrA7XWn8236FdJNkwyKcmMWuvVneVfT/K2zteLWwcAAMuNATm9ZH6llO4khyf5QZINktw2d12t9f5SSncpZczi1tVaHxjocT8d02f35vGHH8+jM58Y7KEwAMz10LGizvXNdz6UP97x8GAP4zllWE93emf3DfYwGADmesUy4e9Wy04v+rvBHsaTDHh0Jzk5yfQkX0vy5oHa6dixowZqV/MMn/FEHp75RFZbbZUB3zeDw1wPHSviXK+yyvAM63F9/YK8J0OHuV5xjBq1UsaPX22R6xe3rpUBje5SyglJNk2yZ621r5Rye+c0k7nrxyXpr7U+sLh1T2ff06ZNT19f/7P1UpbahuNXy9Spjw74fhl44831kLGizvWk9dbMpPXWHOxhPKesqHPNU5nrFc+i5rPlXHd3dy3yQO+A/UhXSvlckq2TvKnWOrOzeHKSVUspO3W+PyzJBUuxDgAAlhsDdcvAlyT5WJI/JvllKSVJ/lxrfXMpZb8kp5dSVpl7W8DMOYe7b1HrAABgeTIg0V1rvSlJ1yLW/TLJ5su6DgAAlheuGAAAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZ6BmInpZQTkrw1yUZJNq+1/q6zfEKSc5KMTTItyf611puXtA4AAJYnA3Wk+6Ikr0hy2wLLv57klFrrhCSnJDl9KdcBAMByY0Ciu9Z6da31jvmXlVLWSjIxyfmdRecnmVhKGb+4dQMxXgAAeDYN5jnd6ye5q9bamzlh3pvk7s7yxa0DAIDlyoCc0/1cMHbsqEHb9/jxqw3avhlY5nroMNdDh7keOsz10DEYcz2Y0X1HkvVKKcNqrb2llGFJ1u0s71rMuqdl2rTp6evrf3ZfwVIYP361TJ366IDvl4FnrocOcz10mOuhw1wPHS3nuru7a5EHegft9JJa631JfpNk786ivZPcUGudurh1gzVeAAB4ugYkukspXy2l3JnkeUmuKqXc1Fl1WJIjSyl/THJk5/ssxToAAFhuDMjpJbXW9yV530KW/yHJyxbxnEWuAwCA5YlPpAQAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANCa6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQmOgGAIDGRDcAADQmugEAoDHRDQAAjYluAABoTHQDAEBjohsAABoT3QAA0JjoBgCAxkQ3AAA0JroBAKAx0Q0AAI2JbgAAaEx0AwBAY6IbAAAaE90AANCY6AYAgMZENwAANNYz2ANYklLKhCTnJBmbZFqS/WutNw/2uAAAYGktD0e6v57klFrrhCSnJDl9sAcEAADL4jkd3aWUtZJMTHJ+Z9H5SSaWUsYP8tAAAGCpPddPL1k/yV211t4kqbX2llLu7iyfupTbGJYk3d1dTQe6OIO5bwaWuR46zPXQYa6HDnM9dLSa6/m2O2zBdc/16H42rJMko0ePHLQBjB07atD2zcAy10OHuR46zPXQYa6HjgGY63WS/Gn+Bc/16L4jyXqllGGdo9zDkqzbWb60rkvy8iT3JOltOFYAAIa2YZ3gvm7BFc/p6K613ldK+U2SvZOc1/nvDbXWpT21JElmJrm64TABAGCuPy1sYVd/f//AD2UZlFJe2Lll4OgkD3ZuGVgHe1wAALC0nvPRDQAAy7vn9C0DAQBgRSC6AQCgMdENAACNiW4AAGhMdAMAQGOiGwAAGhPdAADQ2HP6EymXZ6WUCZ0P9RmbZFrnQ31uHuxx8cyUUsYm+fckG3c+7XRKkkNrrVNLKdslOT3JqkluTbJvrfW+wR4zz1wp5egkn06yea31d+Z6xVNKWSXJV5LsnGRGkmtqre/2d/mKp5SyR5JjknR1Dj5+utb6n+Z6+VdKOSHJW5NsNPfv6yyhyQZy3h3pbufrSU6ptU5Ickrnf9As//qTfKHWWmqtL+181OvxpZSuJOcleW9nzn+e5PjBHizPXCllYpLtktze+d5cr5i+0IntCbXWzZN8srPc3+UrkM6f339Psl+tdcsk+yY5p5TSba5XCBcleUWS2xZYvri5HbB5F90NlFLWSjIxyfmdRecnmVhKGT/IQ+MZqrU+UGv92XyLfpVkwySTksyotV7dWf71JG8bpGHyLCmlrNz5S/g9nR+4Yq5XPKWUUUn2T/LJWmt/5vxZ/4u/y1dYfUnW6Hy9ZpJ7kowz18u/WuvVtdY75l+2uD/HA/1nXHS3sX6Su2qtvZnzm6A3yd2d5awgOkdGDk/ygyQbzP+Tda31/iTdpZQxgztKnqHPJjmv1vrn+ZaZ6xXPxp1/Vj66lHJ9KeVnpZSd/F2+4un8UPW2JBeXUm7rHBk9wFyv0BY3twM676Ibnr6T8//bu7cQu6o7juPfYAqKlWqqtk5CYkX7i7UPUo3aEOlLRdQE06oEtdgiFbSKPthqb9SKFaQF8YY+qMVLVJQqWk00mlSRqtV6e6gP/yqmMWnjJZo+KEG8nD7MGnschhDJ7DmZ4/cD52HWXmvPmr2YM79ZZ+214V3gmkF3RJMvybeBBcC1g+6LOjcT2A94oaoOBS4E7gG+OOiOaXIlmQn8Aji+quYBS4A7HWtNBUN3N9YDs5PsxOgv+U7ASCvXEGg3axwALKuqj9t633l9x/cEelX1zmB7qu3wHWA+sDbJv4A5wCpgf8d66KwDPhz7iLmqngY2AVt8Lx86BwMjVfUEo2P9BPBeW8/vWA+nrWWyKc1rhu4OtF0MXgRObkUntxmUtwbcNU2CJJcChwBLq+r9VvwcsEv7SBrgTOCuAXZT26mqLquqkarat6r2BTYARwN/cKyHS1si9ChwFP/fzWBv4J++lw+dDcCcJGF0rA8Evgq87FgPp61lsqnOazN6vd42VNNnlWR+24JmD2Bz24KmBt0vbZ8kBwH/aH+Mt7TitVX1vSQL213PO/dtI/fGgLusSdJmuxe3LQMd6yGTZD/gj23bsA+AX1XVg76XD58kpwI/bzdUAlxUVfc61tNfkquA77d/pDYBb1fVQVsb26kcd0O3JEmS1DGXl0iSJEkdM3RLkiRJHTN0S5IkSR0zdEuSJEkdM3RLkiRJHTN0S9LnRJIHk/xwks/52yTLJ/OckjSMZg66A5Kkz6btGf4V4KO+4puq6pyttauqY7rvnSRpIoZuSZqellTV6kF3QpK0bQzdkjQkkvwIOAN4HjgN2AicXVVr2vHHgOVVdUOS/YEbgYPbExjXVNWyVm8hcCXw9fb01fOq6sl27GvATcC3gL8BNa4PRwCXA98A1rW2j/X17zfAXu1pcb+uqtsGc7UkaWq5pluShsvhwKvAnsBFwD1JZk1Q7xLg4fbo4znA1YwG41nACuCq9kj0y4EVSb7c2t0OPNfOfwnwyRrxJLNb298Bs4CfAncn2SvJru2cx1TVbsBC4MUpuyqSNGDOdEvS9HRvkg/7vv5Zm7F+E7iiqnrAnUnOB44Dbh3X/gNgHjBSVRuAv7by44CXq2qs/h1JzgWWJPkLsAD4blW9Dzye5P6+c/4AWFlVK9vXjyR5FjgW+BPwMfDNJK9V1cY2Ey9JnwvOdEvS9LS0qnbve13fyv/dAveYdcDIBO0vAGYAzyR5KcnprXyktWHcOWa3Y5ur6r1xx8bMA05K8t+xF7AI2Ke1WQacCWxMsiLJ/Em6FpK0w3OmW5KGy+wkM/qC91zgz+MrVdXrbf03SRYBq5M8Dvynhed+c4GH2sz0Hkl27Qvec4Gx77UeuLWqzpioY1W1CliVZJe2BOV64MjJ/OElaUdl6Jak4bI3cG6Sa4GlwIHAyvGVkpwEPNWWlmxuwfmjVvfqJKcAdwEntJsiH6iqTW25yMVJfgkcBizpC/XLgb8nORpYDXwBOAJ4pS1nORxYA2wB3h235aEkDTVDtyRNT/cn6Q+tjwD3AU8DB7TdQd4ATqyqtydovwC4IsmXWr3zqmoto4F8cdu95LoWmBdX1abW7hTgZuAd4CngFmB3Rmey1yc5Hvg9cEcL1c8AZ7XljOe3teW9dhPlT6bkSknSDmBGr9fbhmqSpB1d25Lvx1W1aNB9kSR9mjdSSpIkSR0zdEuSJEkdc3mJJEmS1DFnuiVJkqSOGbolSZKkjhm6JUmSpI4ZuiVJkqSOGbolSZKkjhm6JUmSpI79D6rLs3O/9LXaAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "bento_obj_id": "139854087711184", + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_rewards(eval_rewards);\n", + "plt.ylim([0, 510]);" + ] + } + ], + "metadata": { + "anp_cloned_from": { + "revision_id": "351369499371280" + }, + "bento_stylesheets": { + "bento/extensions/flow/main.css": true, + "bento/extensions/kernel_selector/main.css": true, + "bento/extensions/kernel_ui/main.css": true, + "bento/extensions/new_kernel/main.css": true, + "bento/extensions/system_usage/main.css": true, + "bento/extensions/theme/main.css": true + }, + "kernelspec": { + "display_name": "reagent", + "language": "python", + "name": "reinforcement_learning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5+" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/reagent/test/notebooks/test_notebooks.py b/reagent/test/notebooks/test_notebooks.py new file mode 100644 index 000000000..caf5a4865 --- /dev/null +++ b/reagent/test/notebooks/test_notebooks.py @@ -0,0 +1,10 @@ +import unittest + +from bento.testutil import run_notebook + + +class NotebookTests(unittest.TestCase): + def test_reinforce(self): + path = "reagent/notebooks/REINFORCE_for_CartPole_Control.ipynb" + variables = run_notebook(path) + self.assertGreater(variables["mean_reward"], 180)