PPO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration 0: 100%|██████████| 60/60 [00:01<00:00, 46.94it/s, episode=60, return=0.037]\n",
      "Iteration 1: 100%|██████████| 60/60 [00:01<00:00, 48.46it/s, episode=120, return=0.037]\n",
      "Iteration 2: 100%|██████████| 60/60 [00:01<00:00, 40.94it/s, episode=180, return=0.037]\n",
      "Iteration 3: 100%|██████████| 60/60 [00:01<00:00, 31.50it/s, episode=240, return=0.037]\n",
      "Iteration 4: 100%|██████████| 60/60 [00:01<00:00, 30.01it/s, episode=300, return=0.037]\n",
      "Iteration 5: 100%|██████████| 60/60 [00:02<00:00, 27.92it/s, episode=360, return=0.037]\n",
      "Iteration 6: 100%|██████████| 60/60 [00:01<00:00, 30.00it/s, episode=420, return=0.037]\n",
      "Iteration 7: 100%|██████████| 60/60 [00:01<00:00, 31.11it/s, episode=480, return=0.037]\n",
      "Iteration 8: 100%|██████████| 60/60 [00:01<00:00, 31.61it/s, episode=540, return=0.037]\n",
      "Iteration 9: 100%|██████████| 60/60 [00:01<00:00, 33.51it/s, episode=600, return=0.037]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqPklEQVR4nO3de7wdVX338c/3nFwIIIRIsCGJBjBFESHEyKVaRQElqI13wQvUx6fII7Tay6sGr9A+bdVW7YPygqKiYFVKFTXVWKSIUvoSJCjERG4RQQIRAgJyD8n5PX/M7Jw5s/fZe+acPdl7z/m+X6+ds2dmzZy1ss/Mb6+1Zq1RRGBmZlbUUK8zYGZmg8WBw8zMSnHgMDOzUhw4zMysFAcOMzMrxYHDzMxKceAwM7NSHDhs4Ei6XdLjkh6RdI+kL0raNd32Q0lPpNvuk3SJpHmZff9A0g8kPSzpIUn/IemAHZTvL0kKSX+UW//P6fo/LnCMIyWNpOXLvo6oLOOt83GUpJskPSbpCknP2pG/33rLgcMG1WsiYldgKfBC4EOZbael234fmA18GiC9uH4f+DawN7APcAPwP5L23UH5vgU4qbEgaRrwJuCXJY5xd0Tsmnv9uNsZHY+kPYFLgA8Dc4A1wL/tqN9vvefAYQMtIu4Cvgcc2GLbb4FvZLZ9ArgwIv5fRDwcEb+NiA8BVwNntDq+pCFJH5J0h6R7JV0oafd026K0pnCSpF+nNZwPdsjyfwAvkrRHunwssBb4TXrMmZJ+K+n5mTzsldaw5nb6/0hrXH8r6X/SWtX30ws9kv5T0mm59DdIen1u3UxJD0o6MLNubpqHvYDXA+sj4t8j4on0/+5gSc/plD+rBwcOG2iSFgLHAT9rsW1P4A3AzyTtDPwB8O8tDnMxcMw4v+KP09fLgH2BXYHP5tK8GNgfOAr4iKTntsnyE8Aq4Ph0+UTgwsbGiHgSuAh4e2afE4D/iojNbY6b9VbgncBewAzgr9L1X02PBUDaRPcs4LvZndM8XJJNC7wZ+FFE3As8j6Sm1kj/KEmN6XkF82cDzoHDBtW3JD0IXAX8CPj7zLaz0m03AJuAvyBpUhlKl/M2AXuO83veBnwqIm6LiEeA04Hj0yamhjMj4vGIuCH9nQd3yPuFwIlpzeWlwLdy2y8A3iqpcX6+A/hyZvveaY0g+9ols/2LEXFLRDxOEhSXpOu/CSzJ9Ee8DbgkDRR5Y4IMSTD6avp+V+ChXPqHgKeNW2KrlWmdk5j1pddGxH+Ns+3PIuLz2RXphXUEmAfclEs/D7hvnGPtDdyRWb6D5Lx5RmbdbzLvHyO5sI4rIq5Km50+BHwnIh6XlN1+jaRHgZdK2gQ8m6SW0nB3RCxo8yta5iciHpb0XZLazsfTnycDSFpPUvsAWA78AJgl6bD0eEtIAg/AI8Buud+5G/Bwu3JbfThw2JQQEY9K+jFJR/QVuc1vBi4fZ9e7Gb2gAjwT2ArcA7S7eHfyr8BHSJrAWrmApLnqN8DX076Ebvga8FFJVwKzSP8vIqKpmUnSxSS1jntIAlwjMKxnbAf/LsB+6XqbAhw4bCpZCVwq6SbgiyR//38JHEFyZ1YrXwPeL+l7wGaSJrF/i4it2VrCBJwF/Ddw5Tjbv0zSaf4wSVNVt6wGzgf+hqQcI23SfpWkGe1+INvp/03gHyW9gaR/5CPA2ojI1+SsptzHYVNGRFwFvJLkrqBNJM1OhwAvjohbx9ntfJKL+JXAr0g6t/+0C3n5bURcHuM8ECciNgI/BYIkwGTt3WIcxxsK/t5Gx/fRjPZZjJf2GuBRkua672XWbya56eDvgAeAwxjt7LcpQH6Qk1l/knQ+SX/GhzomNtuB3FRl1ockLSKpGR3S46yYNXFTlVmfkfS3wDrgHyPiV73Oj1mem6rMzKwU1zjMzKyUKdHHseeee8aiRYt6nQ0zs4Fy3XXX3RcRTXOkTYnAsWjRItasWdPrbJiZDRRJd7Ra76YqMzMrxYHDzMxKceAwM7NSHDjMzKwUBw4zMyvFgcPMzEpx4DAzs1KmxDiOHe2Km+5l/d0PIYkhiZ1nDPPolq3sMiP5777/keRJnXN2mcHQkLjv4VZP7oRdZk5jy9YRhofFE1u2MWvGNJ7cuo2RkfGnidl1p2k88sTWCeV7xrTke8SWrc2PaNht1nRmTh/m/keebPv7bQeQ2GXGMI8+ObHPud9MHx5iaEg8+dS2Xmelll63dAH77LlL54QlOHBU4J1furb0PvlnArWbQmy85wdl9yn7jKH878vu326b7XiT+Zz7jf+2qrf0WXs4cNTBEfs+nRnThvjRLZsBOGDebqx+7x+OSfPhb63jy1ePDtrcY+fpPPDYU8yYNsQt/3d5y+O+6GM/4K4HH+cNSxfwyTcfXCpPmx9+khf+XfII78+ccAivOXjv7dv+8dKbOPuKX25f/tU/vKrUsa27Dj7z+zz0+FOcdMSzOHPFgb3OzqTc87snOOzvk6f2/t3rDuRthz2rwx7WD9zH0QNDQ52/WQ3lts+aPtzxuI1j5vctYub00T+FmdPG/lkMZTLrb4S91/h8J/no2r6QLcJQDcozVThw9IAQ2VOk1fmSvyjMmtE5cDROvImcgDtNGz3+Trkglc2LT+3em8zn3G+yZZjIFx7rDQeOHpByF+OWgWPs8s4zOrcqSmN/ljF9eHSnpsCRee9u8d6bzOfcb8Z8gfLXkoHhwNEDkjqeMPlvk2VqHBNpwsjus9P08ZuqrPe0vcbR44x0gZtBB5MDRw8MFahx5C8KO6eBo925NZk+jqx8jSN7PJ/bvTe0/XMe/E9jbFPV4JdnqnDg6AExNli0Ol3ytYZG4GjXVNTYY7LnX7a/I388N1X13vYaah2us9nzoA7lmSIcOHpgKNdU1eqMya+aNb1zH0e3Ok1n5pqq6nD3Tp3Uq8aRfT/45ZkqHDh6IOkczyy3SJM/iXau+K6qrHyNwyd0f3Efh/VapYFD0rGSbpa0QdLKFtsl6ax0+1pJS9P1O0n6iaQbJK2XdGZmnzMk3SXp+vR1XJVlqELSOT6xPo72xx3/eGU01zgmdzzrrqH046lDQHcfx2CqbOS4pGHgbOAYYCNwraRVEfGLTLLlwOL0dRhwTvrzSeDlEfGIpOnAVZK+FxFXp/t9OiL+qaq8V02MnvyN5eY05e+qanwTnextjc0DACd1OOuyxudbh49lTM27DgWaIqqscRwKbIiI2yJiC3ARsCKXZgVwYSSuBmZLmpcuP5KmmZ6+atMvO9RU42h1O+7Y5SJ3VY22fU8uf/n8+JtgfxmtWQ7+5+KR44OpysAxH7gzs7wxXVcojaRhSdcD9wKXRcQ1mXSnpU1b50vao9Uvl3SypDWS1mzevHmSRekuiY5fF5tHjpfoHK+wiuBTu/can0EdLrQeOT6Yqgwcrf4M8rWGcdNExLaIWAIsAA6V1JjN7RxgP2AJsAn4ZKtfHhHnRcSyiFg2d+7c8rmvUP6uqta3445dbsxVNa3N2bX9m+jkstcke3LXpto3wBqfQQ3iRu5vtQYFmiKqnB13I7Aws7wAuLtsmoh4UNIPgWOBdRFxT2ObpM8B3+linneMQgMAczWO6cP8nyP341XPnzf+YScxchzgS+98Ib/c/GiLvEzocFaxOnwurnEMpiprHNcCiyXtI2kGcDywKpdmFXBienfV4cBDEbFJ0lxJswEkzQKOBm5Kl7NXztcB6yosQyWaaxyd+ziGh+D9xz6HA+fv3ua4rfct6sj99+JdL96n+bg+o/vK6EDPwf9c3McxmCqrcUTEVkmnAZcCw8D5EbFe0inp9nOB1cBxwAbgMeCd6e7zgAvSO7OGgIsjolGz+ISkJSQ19tuBd1dVhqqI3MW9xfkykQ7qqmZN9encX2rVVOVxHAOp0gc5RcRqkuCQXXdu5n0Ap7bYby1wyDjHfEeXs7nDNc1V1SJN/iQqEgy6NeVIc158Rvejun1Dr1t56swjx3ugaXbcAn0cQwU+qcnMjlvkuNYfRu+q6mk2us5/ZoPDj47tgfwJUqSPo1Aw6NI4jqbD+oTuS3UL6HUrT505cPRAPlC0fJBTLk2hkypa7ztZdftma/3JcWNwOHD0QHNtojlNcx/HxI8/We7j6E91+4Zet/LUmQNHlyX9/e0Va6qaQI2j0VTV5cjh07k/1a0m6LgxONw53mUF4kaLuaqa0zQFlxInVbdPQH8T7C+jt+PW63PxM8cHh2scXTYygRpHK/mL9XCJi0TX+zj89aIv1a3GUbfy1JkDR5cVmcsp+abYPmX+JCrT/NT1Pg5/E+wrdRo5nuUZCgaHv0t2WaEaR+ZfGOcC0NTHUTwPXR857vO5r9Rp5HhWzYpTa65xdFnRPo5sjaPVCTOhcRyN23HdxzEl1O1zqVsNqs4cOLqsSOCQKD9yvEwfh0eO11pdR47XrTx15qaqLosCvRxNd1W1TNN+uSWPHJ9S6vYN3V9QBocDR5eNZOLG/Nmzxk039lnLzSfMhEaOTyBtseN19XDWJXX7WBw3BoebqrqsMQDwT1/+bN5z5LN5/KltLP3by8akGZLGDBQsMjtuL8dx1O2bbV3U7Rt63cpTZw4cXdaocew+azqzZgwzc1pzpU75adUL9HEMl/ja3+0LffZoRfpwbMeo2/gax43BUbM/vT6QXlhHpzhvTpI8jyO7pkVTVVMfR+/GcfibYH/ZfjtuzRqr6laeOnPg6IInntrGurseAkbHcTSutS37LwpMOdJ8V1WBjFQ1O27mr8QxpH/U7bOoWw2qzvxRdcEHv7mOV3/mKu753RPbvw22+5auXI2jWB+HR45bYvR23Hp9LnUrT505cHTBT3/9AACPPLm1qcbRigpciic1O65Hjk8JdbvQ+u69weHA0QWNYJHcLZWsa1dDGNLYeXm69jyO0cbvrqrbBaou6vex1K5AteXA0QWxvUN89HbcdqdA08jxbj2PYwJpi6jfBaoe6vYNvW7lqTMHji4YU+NI17W7eA/lIsd4d15lFbp4VzRyPFsW347bP+o2vsY128FRaeCQdKykmyVtkLSyxXZJOivdvlbS0nT9TpJ+IukGSeslnZnZZ46kyyTdmv7co8oyFDEyMtqvUayPg453VeXrLGXGcbjGMTXU7WPx39ngqCxwSBoGzgaWAwcAJ0g6IJdsObA4fZ0MnJOufxJ4eUQcDCwBjpV0eLptJXB5RCwGLk+Xe2okM3ZjJNNsNR5JubuqWjVV5ZfLDAAsnLTY8ToGOeuFun1Dr1t56qzKGsehwIaIuC0itgAXAStyaVYAF0biamC2pHnp8iNpmunpKzL7XJC+vwB4bYVlKKQxsWGQ6ePodDvumBXNafInUbkpR6qbq8pNVf2jbuMeHDcGR5V/evOBOzPLG9N1hdJIGpZ0PXAvcFlEXJOmeUZEbAJIf+7V6pdLOlnSGklrNm/ePNmyjHH/I0/yo1tGj9moZUTE6F1VbfYfksYEhiLjOHo6cty9ln2ltiPHHTkGRpWBo9VfQf776rhpImJbRCwBFgCHSjqwzC+PiPMiYllELJs7d26ZXTt62+ev4aTzf8JT20Yavyv9mb3Dqk2Ng86z407orqqqRo536Mi33qjbZ+HvJ4OjysCxEViYWV4A3F02TUQ8CPwQODZddY+keQDpz3u7luOCbrnnYQC+es2vgbFTqRfpHIf87bgttk9kHMcE0hbjM7qfeOS49VqVgeNaYLGkfSTNAI4HVuXSrAJOTO+uOhx4KCI2SZoraTaApFnA0cBNmX1OSt+fBHy7wjK01AgUH121noefeGp7sBiJKHQ7LjAmMhSZq6pQNV4l0pbgPo7+VLcLbc2KU2uVTaseEVslnQZcCgwD50fEekmnpNvPBVYDxwEbgMeAd6a7zwMuSO/MGgIujojvpNs+Blws6V3Ar4E3VVWGIkZi9HbciInVOFpuzyUodztu4aQFj+czuh/V7WOpW59NnVX6PI6IWE0SHLLrzs28D+DUFvutBQ4Z55j3A0d1N6eTkOnXSO6qSt53uquqU+f4hGbH3X786sZx1O1iNcjq9lm4j2Nw1OyGvh0vCLZt7xyPQlOOQIFHx/bTXVUeOd6X6lYTrFt56syBowsazVMBhfs4OnaO59aWOac8cnxqqNvH4r+zweHAMUnZW3AjongfR4fIMaGR4xXNjuuR4/2pbuNrPI5jcDhwTFK2X2PsOI72+4155nirKUdyB+jl7LjZEcpuquofNYsbtStPnTlwTNJIppYRjDZblfna3/J5HLnlQieVSqQtwW3P/WU0dtfrc3GNY3A4cEzSmMBRosbRecqRCYzjaHHsbvDp3J/q9g29buWpMweOgiKCD39rHWs3PphbPzogcCQ7V1WnzvEOt7hO5iTq9vk3plnNJ3fPeeS49ZoDR0G/e3wrX776Dt72+WvGrB/JNPpHjM6Um73w//dfv4xXHPCMMft1OkUmcxJ55PjUULcLbc2KU2sOHAVtSSc0nDlt7H9Zdp6qILYvZ0+ChXN25vd232nMfp2exzGZk6jrz+PwGd2X6vaxeOT44Kh05Hhd/Nu1v+YzP9gAwIzhXOAYydU4xnkeR/6UyJ4k0TRpcH99m3Tbc3/qoz+RrvDf2eBw4Cjg/d/4+fb303M1jmzTTba/oylQlHwwUz9dFPopiNmoun0udStPnbmpqqSmGke2jyMzdrzjyPGOI8v75yTy+dyf6va51K08debAUdKMpj6OsU1Vrfo4Wi53+D2NQXd77jqD/3zfH04gp93jPo7+VLdv6P47GxwOHCVNb6pxjL7PjiJvep5GybmnGvvvPms6z/m93SaU125x23N/8udiveLAUVK+xhFjahyZuapy+5WtcfTTNaFu32wHXV1HjtvgcOAoKd/HsS0TOEYyI8c73lU1QBfjwcnp1OIah/WKA0dJTX0cI9mlzPM4JnlS91NcGaQgNxU0Pg1/LtYrDhwltb2rKko8j2OAznl/szWzLAeOktqO46D4M8cHib/ZmlmWA0dJ04fHXkSzNY6RkSg8O24lcz5lngvSTa5xmFmWA0dJ+Yvy2AGAE3weR593P7vGYWZZDhwlbRsZGznGjOMY08dR/Jit5qqakPR3dn+Sw+4ez8wGW6WBQ9Kxkm6WtEHSyhbbJemsdPtaSUvT9QslXSHpRknrJb03s88Zku6SdH36Oq7KMuRty1U5IjflyHiTHOYN0sXY4zjMLKuySQ4lDQNnA8cAG4FrJa2KiF9kki0HFqevw4Bz0p9bgb+MiJ9KehpwnaTLMvt+OiL+qaq8tzPSqcZRsI9jkNSoKGbWBVXWOA4FNkTEbRGxBbgIWJFLswK4MBJXA7MlzYuITRHxU4CIeBi4EZhfYV4La26qGmeuqhpdbl3hMLOsKgPHfODOzPJGmi/+HdNIWgQcAmQfvXda2rR1vqQ9Wv1ySSdLWiNpzebNmydYhGZb2wWOEgMA/SQ9MxtUVQaOVpfO/OWybRpJuwLfAN4XEb9LV58D7AcsATYBn2z1yyPivIhYFhHL5s6dWzLr48sHjmwA2PjA43x01fo075P7PTOnDQOw9+xZhfc56jl7ATC/xD5FZPs4Xn9IX1T8prT5eySfb34WA7MdpcoHOW0EFmaWFwB3F00jaTpJ0PhKRFzSSBAR9zTeS/oc8J3uZru9rdvGzDEypsZx+iWjD3ya7MjxhXN25rNvPYQXP3vPwnk7+SX78qZlC5mzy4zC+xSx0/Rh/uUdL2C/ubuy6Ok7d/XYVt7Zb13K/2y4v+tfEHrlh391JHc9+Hivs2ElVBk4rgUWS9oHuAs4HnhrLs0qkmani0g6xR+KiE1Kbkn6AnBjRHwqu0OjDyRdfB2wrsIy8MiTW8csb902fud4Vjf6BV590N6l0kvqetBoeOXzfq+S41p5s3eewasOmtfrbHTNoj13YdGeu/Q6G1ZCZYEjIrZKOg24FBgGzo+I9ZJOSbefC6wGjgM2AI8B70x3fxHwDuDnkq5P130gIlYDn5C0hKRJ63bg3VWV4cHHtrDkby4bs27ryPg1jizfwmpmdVXpM8fTC/3q3LpzM+8DOLXFflcxzl2gEfGOLmdzXPc9sqVpXXMfR+vA0SlsuHPczAaVe9dKeirfVDXSOp2n6TCzunLgaKu5WrCtYFOV44aZ1VWhwCHpvZJ2S6cI+YKkn0p6RdWZ60dFO8fr9DwOM7OsojWO/5WOo3gFMJekE/tjleWqT7SqTGzJ3Y470T4OM7NBVTRwNK6DxwFfjIgbmALXxlYhod3suFm+q8rM6qpo4LhO0vdJAsel6cSD43QL11tT5/gE+jiWLZrju6rMbGAVDRzvAlYCL4yIx4AZjI65qK1WF/ei4zjG8+dH/z6H7/v0yWTLzKynCo3jiIgRSfcAB0iqdOxHv8t3jueXG4bGmVd95xnJHFRuyTKzQVUoCEj6OPAW4BfAtnR1AFdWlK++la9x5Ps8GhwXzKyuitYeXgvsHxFPVpiXvtOqGSpfw8g/EbDBNQozq6uifRy3AdOrzEg/ahk4cjWM/HJDnR7kZGaWVbTG8RhwvaTLge21joj4s0py1SeK9Htv29b65jI/yMnM6qpo4FiVvqaU8fovssavcZiZ1VPHwCFpGHhHRBy9A/LTV4rcajtu57inHDGzmurYxxER24DHJO2+A/LTV4oEjnFrHA4MZlZTRZuqniB5qNJlwKONlXXv4yjQUsUVN93bcr3jhpnVVdHA8d30NaWMFIgca+54oOX6Tk1V7hw3s0FVdOT4BVVnpB+NN0ajCNc4zKyuio4c/xUtJouNiH27nqM+MplaQac+DveBmNmgKtpUtSzzfifgTcCc7menv5SdwDDLj441s7oqNHI8Iu7PvO6KiH8GXl5t1nqvyDiO8ThumFldFW2qWppZHCKpgTytkhz1kUk1VXUvG2ZmfaVoU9UnM++3Ar8C3tz97PSXydU4fFeVmdVT4Qc5RcTL0tcxEXEysKXTTpKOlXSzpA2SVrbYLklnpdvXNmo2khZKukLSjZLWS3pvZp85ki6TdGv6c4+ihS1rUn0cXcyHmVk/KRo4vl5w3XbpVCVnA8uBA4ATJB2QS7YcWJy+TgbOSddvBf4yIp4LHA6cmtl3JXB5RCwGLk+XKzGJCofvqjKz2mrbVCXpOcDzgN0lvT6zaTeSu6vaORTYEBG3pce6CFhB8jCohhXAhRERwNWSZkuaFxGbgE0AEfGwpBuB+em+K4Aj0/0vAH4IvL9DXiZkMjWOIUcGM6upTn0c+wOvBmYDr8msfxj4kw77zgfuzCxvBA4rkGY+adAAkLQIOAS4Jl31jDSwEBGbJO3V6pdLOpmkFsMzn/nMDlltbTKBw8ysrtoGjoj4NvBtSUdExI9LHrvVV+78lbhtGkm7At8A3hcRvyvzyyPiPOA8gGXLlk0oAlTZVGVmNqiK9nHcL+lySesAJB0k6UMd9tkILMwsLwDuLppG0nSSoPGViLgkk+YeSfPSNPOA1rMMdkGRuarG4ycAmlldFQ0cnwNOB54CiIi1wPEd9rkWWCxpH0kz0vT5h0GtAk5M7646HHgobX4S8AXgxoj4VIt9TkrfnwR8u2AZSpvcyPH223efNeWexGtmNVE0cOwcET/JrdvaboeI2AqcBlwK3AhcHBHrJZ0i6ZQ02WqS55lvIAlO70nXvwh4B/BySdenr+PSbR8DjpF0K3BMulyJSY3j6LD9jw7emxcuquxOYjOzyhQdAHifpP1I+x8kvZFMB/Z4ImI1SXDIrjs38z6AU1vsdxXjXHsj4n7gqIL5npTJ9I13uqtKEm98wQKuvb31tOxmZv2qaOA4laSj+TmS7iIZOf62ynLVJ/JNVTOmDbFl60ihfd05bmZ1VfR5HLcBR0vahaR563HgLcAdFeat5/LP45g5XCZwOHKYWT217eOQtJuk0yV9VtIxwGMkHdIbmAJzVTV1cTgWmJl1rHF8GXgA+DHJgL+/BmYAr42I66vNWu/lb8d13DAz6xw49o2I5wNI+jxwH/DMiHi48pz1AY8cNzNr1ul23KcabyJiG/CrqRI0YHIjx83M6qpTjeNgSY2pPgTMSpdFcjftbpXmrscmM3LczKyuOs1VNbyjMtKP3FRlZtas6MjxKSlf4fAttmZmDhxtucZhZtbMgaMN93GYmTVz4GgjP3LczMwcONpyhcPMrJkDRxuRq3G4b9zMzIGjrck8j8PMrK4cONpouh23N9kwM+srDhxt5JuqzMzMgaMtN1WZmTVz4GjDccPMrJkDRxseOW5m1syBo4184PBcVWZmDhxtucZhZtas0sAh6VhJN0vaIGlli+2SdFa6fa2kpZlt50u6V9K63D5nSLpL0vXp67iq8v+B455b1aHNzAZWZYFD0jBwNrAcOAA4QdIBuWTLgcXp62TgnMy2LwHHjnP4T0fEkvS1uqsZz9h5RqfnXJmZTT1V1jgOBTZExG0RsQW4CFiRS7MCuDASVwOzJc0DiIgrgd9WmL/S3MNhZlZt4JgP3JlZ3piuK5umldPSpq3zJe3RKoGkkyWtkbRm8+bNZfJtZmZtVBk4Wn1Bz/c2F0mTdw6wH7AE2AR8slWiiDgvIpZFxLK5c+d2OKSZmRVVZeDYCCzMLC8A7p5AmjEi4p6I2BYRI8DnSJrEdgjfjWtmVm3guBZYLGkfSTOA44FVuTSrgBPTu6sOBx6KiE3tDtroA0m9Dlg3XlozM+u+ygJHRGwFTgMuBW4ELo6I9ZJOkXRKmmw1cBuwgaT28J7G/pK+BvwY2F/SRknvSjd9QtLPJa0FXgb8eVVlyDpi36dzzttfMO72o5/7DD7Y5vbdP3nJvhy2zxze+IIF29ctf/48Dt93Dn/68sVdzauZWZU0FWaAXbZsWaxZs2ZC+y5a+V0A1p/5SnaZOW37ct41HziKkQiO+IcfAHD7x141scyamfUJSddFxLL8eo8cL2jacPsODgnkG3bNbApw4Cho2lDr/6ohNX7KnedmNiU4cBQ0NE5QmDltON3u+oaZTQ0OHAWNNzPuzOmZ/0JHDjObAhw4JmnGcPJfuGXriPs4zGxKcOCYpI++5nk8beY09thluvs4zGxKcODo4PTlz2HxXrtuX/7Y65/PXk+bCcBz5+3Gqw6ax8/PfCUzpw27vmFmU4LnDe/g3S/dj3e/dL/ty8cf+kyOP/SZrLvrIRbusfOYtH5CoJlNBQ4cE3Tg/N2b1jlsmNlU4KaqLnKFw8ymAgeOLvJdVWY2FThwdJPjhplNAQ4cXeSmKjObChw4umjIkcPMpgAHji5y2DCzqcCBo4tc4TCzqcCBo4t8V5WZTQUOHF3kGoeZTQUOHGZmVooDRxe5xmFmU4EDRxe5j8PMpgIHji5yjcPMpoJKA4ekYyXdLGmDpJUttkvSWen2tZKWZradL+leSety+8yRdJmkW9Ofe1RZhjIcN8xsKqgscEgaBs4GlgMHACdIOiCXbDmwOH2dDJyT2fYl4NgWh14JXB4Ri4HL0+W+4OdxmNlUUGWN41BgQ0TcFhFbgIuAFbk0K4ALI3E1MFvSPICIuBL4bYvjrgAuSN9fALy2isxPhMOGmU0FVQaO+cCdmeWN6bqyafKeERGbANKfe7VKJOlkSWskrdm8eXOpjE+UKxxmNhVUGThaXUZjAmkmJCLOi4hlEbFs7ty53ThkR26qMrOpoMrAsRFYmFleANw9gTR59zSas9Kf904yn2ZmVkKVgeNaYLGkfSTNAI4HVuXSrAJOTO+uOhx4qNEM1cYq4KT0/UnAt7uZaTMza6+ywBERW4HTgEuBG4GLI2K9pFMknZImWw3cBmwAPge8p7G/pK8BPwb2l7RR0rvSTR8DjpF0K3BMumxmZjvItCoPHhGrSYJDdt25mfcBnDrOvieMs/5+4KguZtPMzErwyHEzMyvFgcPMzEpx4DAzs1IcOMzMrBQHDjMzK8WBw8zMSnHgMDOzUhw4zMysFAcOMzMrxYHDzMxKceAwM7NSHDjMzKwUBw4zMyvFgcPMzEpx4DAzs1IcOMzMrBQHDjMzK8WBw8zMSnHgMDOzUhw4zMysFAcOMzMrxYHDzMxKceAwM7NSKg0cko6VdLOkDZJWttguSWel29dKWtppX0lnSLpL0vXp67gqy1DWJ954EP9+yhG9zoaZWWWmVXVgScPA2cAxwEbgWkmrIuIXmWTLgcXp6zDgHOCwAvt+OiL+qaq8T8ably3sdRbMzCpVZY3jUGBDRNwWEVuAi4AVuTQrgAsjcTUwW9K8gvuamVkPVBk45gN3ZpY3puuKpOm072lp09b5kvZo9cslnSxpjaQ1mzdvnmgZzMwsp8rAoRbromCadvueA+wHLAE2AZ9s9csj4ryIWBYRy+bOnVsow2Zm1lllfRwktYRsg/8C4O6CaWaMt29E3NNYKelzwHe6l2UzM+ukyhrHtcBiSftImgEcD6zKpVkFnJjeXXU48FBEbGq3b9oH0vA6YF2FZTAzs5zKahwRsVXSacClwDBwfkSsl3RKuv1cYDVwHLABeAx4Z7t900N/QtISkqar24F3V1UGMzNrpoh8t0P9LFu2LNasWdPrbJiZDRRJ10XEsvx6jxw3M7NSpkSNQ9Jm4I4J7r4ncF8Xs9NLLkt/qktZ6lIOcFkanhURTbelTonAMRmS1rSqqg0il6U/1aUsdSkHuCyduKnKzMxKceAwM7NSHDg6O6/XGegil6U/1aUsdSkHuCxtuY/DzMxKcY3DzMxKceAwM7NSHDja6PQEw36TTjN/r6R1mXVzJF0m6db05x6ZbaenZbtZ0it7k+tmkhZKukLSjZLWS3pvun4Qy7KTpJ9IuiEty5np+oErCyQPaJP0M0nfSZcHtRy3S/p5+hTRNem6QS3LbElfl3RTes4cUXlZIsKvFi+SObJ+CexLMlvvDcABvc5Xhzy/BFgKrMus+wSwMn2/Evh4+v6AtEwzgX3Ssg73ugxp3uYBS9P3TwNuSfM7iGURsGv6fjpwDXD4IJYlzd9fAF8FvjOof19p/m4H9sytG9SyXAD87/T9DGB21WVxjWN8A/cUwoi4EvhtbvUKkj8s0p+vzay/KCKejIhfkUw0eeiOyGcnEbEpIn6avn8YuJHkQV6DWJaIiEfSxenpKxjAskhaALwK+Hxm9cCVo42BK4uk3Ui+MH4BICK2RMSDVFwWB47xFXmC4SB4RiRT1ZP+3CtdPxDlk7QIOITkm/pAliVt3rkeuBe4LCIGtSz/DPw1MJJZN4jlgCR4f1/SdZJOTtcNYln2BTYDX0ybED8vaRcqLosDx/iKPMFwkPV9+STtCnwDeF9E/K5d0hbr+qYsEbEtIpaQPJDsUEkHtknel2WR9Grg3oi4ruguLdb1vBwZL4qIpcBy4FRJL2mTtp/LMo2kefqciDgEeJSkaWo8XSmLA8f4ijzBcBDc03j4Vfrz3nR9X5dP0nSSoPGViLgkXT2QZWlImxB+CBzL4JXlRcAfSbqdpNn25ZL+lcErBwAR0Xii6L3AN0maawaxLBuBjWktFuDrJIGk0rI4cIyvyBMMB8Eq4KT0/UnAtzPrj5c0U9I+wGLgJz3IXxNJImmzvTEiPpXZNIhlmStpdvp+FnA0cBMDVpaIOD0iFkTEIpJz4QcR8XYGrBwAknaR9LTGe+AVJE8SHbiyRMRvgDsl7Z+uOgr4BVWXpdd3BPTzi+TphLeQ3HnwwV7np0B+vwZsAp4i+WbxLuDpwOXArenPOZn0H0zLdjOwvNf5z+TrxSTV57XA9enruAEty0HAz9KyrAM+kq4fuLJk8ncko3dVDVw5SPoFbkhf6xvn9iCWJc3bEmBN+jf2LWCPqsviKUfMzKwUN1WZmVkpDhxmZlaKA4eZmZXiwGFmZqU4cJiZWSkOHGYFSdqWzqbaeLWdMVnSKZJO7MLvvV3SnpM9jlm3+HZcs4IkPRIRu/bg994OLIuI+3b07zZrxTUOs0lKawQfT5+78RNJz07XnyHpr9L3fybpF5LWSrooXTdH0rfSdVdLOihd/3RJ308nrfsXMvMLSXp7+juul/Qv6QSKw5K+JGld+oyJP+/Bf4NNIQ4cZsXNyjVVvSWz7XcRcSjwWZJZZPNWAodExEHAKem6M4Gfpes+AFyYrv8ocFUkk9atAp4JIOm5wFtIJuhbAmwD3kYycnh+RBwYEc8HvtitApu1Mq3XGTAbII+nF+xWvpb5+ekW29cCX5H0LZJpISCZWuUNABHxg7SmsTvJ8xVen67/rqQH0vRHAS8Ark2m82IWyeR1/wHsK+kzwHeB70+wfGaFuMZh1h0xzvuGVwFnk1z4r5M0jfZTXLc6hoALImJJ+to/Is6IiAeAg0lm3j2VsQ9aMus6Bw6z7nhL5uePsxskDQELI+IKkgchzQZ2Ba4kaWpC0pHAfZE8dyS7fjnJpHWQTFb3Rkl7pdvmSHpWesfVUER8A/gwybTaZpVxU5VZcbPSJ/k1/GdENG7JnSnpGpIvYyfk9hsG/jVthhLw6Yh4UNIZJE9uWws8xug02GcCX5P0U+BHwK8BIuIXkj5E8uS6IZJZkE8FHk+P0/gieHrXSmzWgm/HNZsk3y5rU42bqszMrBTXOMzMrBTXOMzMrBQHDjMzK8WBw8zMSnHgMDOzUhw4zMyslP8PRQIAisOKqVMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAou0lEQVR4nO3debhdVX3/8ff3njsnZIAEyAQBDEMECSGNUKx1QCVYBWdQgVIr5fnBr7W1T4tKFar2QVrF+isPFBUKVkWrKFFRZFJKK0iYImEMgxAIGQhkusOZvr8/9jrn7nvuuWdI7s69Z9/P63nOc/bZe+1z1rrD+p417LXN3REREWlU23hnQEREWosCh4iINEWBQ0REmqLAISIiTVHgEBGRpihwiIhIUxQ4RESkKQoc0nLM7Fkz6zezHWa2wcyuMbOp4divzGwgHNtsZjeY2ZzYuX9oZreb2XYz22pmPzGzxXso3/9hZm5m767Y/9Ww/08beI83mVkxlC/+OD6xjFfPx1vN7DEz6zOzO8zswD35+TK+FDikVb3L3acCS4E/AC6MHTs/HDsUmAFcBhAq118CNwJzgYOAh4D/MbOD91C+nwDOKr0ws3bgA8BTTbzHi+4+teLxm7HO6GjMbBZwA/APwN7AKuB7e+rzZfwpcEhLc/cXgJ8DR1Y5tgX4YezYpcB17v6v7r7d3be4+4XA3cBF1d7fzNrM7EIz+72ZbTSz68xseji2MLQUzjKz50IL5zN1svwT4AQzmxlenwSsBl4K79llZlvM7KhYHvYNLazZ9X4eocX1eTP7n9Cq+mWo6DGzX5jZ+RXpHzKz91bs6zKzV83syNi+2SEP+wLvBda4+3+5+0D42R1tZofXy5+kgwKHtDQzWwCcDDxQ5dgs4H3AA2bWC/wh8F9V3ub7wNtG+Yg/DY83AwcDU4F/q0jzBuAw4K3AZ83siBpZHgBWAqeF12cC15UOuvsgcD3w0dg5pwO3uvumGu8b92HgbGBfoBP427D/O+G9AAhddAcCP4ufHPJwQzwt8EHg1+6+EXgtUUutlH4nUYvptQ3mT1qcAoe0qh+b2avAXcCvgX+KHftaOPYQsB74G6IulbbwutJ6YNYon/MR4Cvu/rS77wA+BZwWuphKLnb3fnd/KHzm0XXyfh1wZmi5/DHw44rj1wIfNrPS/+cZwLdix+eGFkH8MSV2/Bp3f8Ld+4mC4pKw/0fAkth4xEeAG0KgqDQsyBAFo++E7anA1or0W4G9Ri2xpEp7/SQiE9Kp7n7rKMf+0t2/Ed8RKtYiMAd4rCL9HGDzKO81F/h97PXvif5v9ovteym23UdUsY7K3e8K3U4XAj91934zix+/x8x2An9sZuuB1xC1UkpedPf5NT6ian7cfbuZ/YyotfOl8HwOgJmtIWp9AKwAbgd6zOz14f2WEAUegB3AtIrPnAZsr1VuSQ8FDpkU3H2nmf2GaCD6jorDHwRuG+XUFxmqUAEOAPLABqBW5V3PfwKfJeoCq+Zaou6ql4AfhLGEsfBd4HNmdifQQ/hZuPuIbiYz+z5Rq2MDUYArBYY1DB/gnwIcEvbLJKDAIZPJBcDNZvYYcA3R3/8ngeOJZmZV813g783s58Amoi6x77l7Pt5K2AVfA/4buHOU498iGjTfTtRVNVZuAq4G/pGoHMUaab9D1I32MhAf9P8R8M9m9j6i8ZHPAqvdvbIlJymlMQ6ZNNz9LuAdRLOC1hN1Ox0DvMHdnxzltKuJKvE7gWeIBrf/7xjkZYu73+aj3BDH3dcB9wNOFGDi5la5juN9DX5uaeD7RIbGLEZLew+wk6i77uex/ZuIJh18EXgFeD1Dg/0yCZhu5CQyMZnZ1UTjGRfWTSyyB6mrSmQCMrOFRC2jY8Y5KyIjqKtKZIIxs88DDwP/7O7PjHd+RCqpq0pERJqiFoeIiDRlUoxxzJo1yxcuXDje2RARaSn33XffZncfsUbapAgcCxcuZNWqVeOdDRGRlmJmv6+2X11VIiLSFAUOERFpigKHiIg0RYFDRESaosAhIiJNUeAQEZGmKHCIiEhTJsV1HK3gkRe38YuHo7uaHr1gBm89Yr9R067f2s/37n2eYjEsF2PGKUvmcsjsoRvP3fnEJlY9u6Whz+5sb+PMP1zItO4OANydb9/zHBu3DXD4nGmcfNScYek3bBvg+t8+T6FY61YOMtbMjPcfO58Fe/eW9936yAZWr3t1/DK1Gzrb2zjj+IVM7+ko7/vevc/xwiv945ir9HnP0vkcNGtK/YRNUOCYIK749VP85KEXAZg7vbtm4Lj6rmf4+n8/Q+k+Qu6wZecgXzj1qHKai3+yhqc27aTevYZKS5XN6O3ko8dFN7p7atNOLvzxwwBM7WofEThuuP8FLrv1CYC67y9jxx0G8gU+teKI8r4Lf/wwL20baLnfQ+nvbu6MHt67NLqR4ta+HH//w98B+rsaS0sPnKnAkVaDuQKH778Xxx28Dz964IWqafqzBbYP5rjv969wzAEz+NH/OQGAP7r0dvoGC8PS7hws8MFl87n0/UfX/Fx359gv3Mq9z27h7a+NgtX/PhXdfvuk1+7PzY+8hLsTv9vdzsE8mTZj7RdXsJt3wZMmLP38LVV+z3nOPmEhn3vXiDu/Tmibtg/yB1+8lZ3ZofLszOYBuOS9R3Ha8gPGK2vSAAWOCSJfdDoybXR3ZOjPFUYcLxSdP7r0djbvyAJw9gkLy8d6OjL0ZYef058r0NORqfu5ZsaSBTO48cEXufHBF8v7p3a1c+S8afxizUsM5ot0x96rLxu9t4LGnrU7v+eJpqczynN/CBZA+e++dEwmLgWOCSJXKNKeMXo6MmTzRQpFJ9MW+5afzbN5R5Z3HjWHE14zi7ctHurK6qkSbPqzBbob/Af83LsW85bD9x2279D99mLNi1uH3itWOfXnhr+WPaO7o42B2O85my+SL3prBo6OUuAYGifrD0FRf1sTnwLHBJHNF+nItNFb+iaWKzC1a+jXs3Mw+mb2hkWzOL2iGd/TmSn/0wHkC0WyhSK9HY39eg/cZwoH7jOyD/SZzTvKeZkZ2z+QK5TzKXtOb2f7sC8IrfwNPdNmdLa3VS2P/rYmPk3HnSDyRacz01ZuJfTFmvAwFDimdI0MBpUtjqEKZfd+vaVvfpXdI33ZfEt+y211UVfV0N/FQAsHDgh/t/GuqvB3pr+tiU+BY4IodVX1hn+agezwqa47wqDo1K6R/1SjfxPdvQZl6R94oLIbLFdsuBtMxk5PZ4b+3NDfRSmgt+o39N7O4V94SuVp1UA4mShwTBC5QjQ43hPrqoortziqBIPujuFdVaWgs7vf3HrDZ40cP8mXA5zsOWn7hl452F9uQbVoeSYTBY4JIlco0hEGx2FkV9WOGl1VI7655fLl/buj1NVVdSaPvhXucT2d1bskW3UwubsjM6w128pjNpONAscEEQWO+i2OqdXGODqH932P1TfR7vLMl5EztvTPvedFkyBGzkLq3c0uyfEyWldVo5M6ZPwocEwQ+VJX1SjjCvUGxwdyxfISJGP1za1UIY0Y48i25rUDrW5EV1WLd+1EX3hGdlV17+akDkmefkMTRDZ0VfV2Vp/J9PLO6MK/0VocANsH8mTzRbYPRJXL7lYoPaPMqto+qFlV46H0Dd19bL8gjJeeirG5/mwhmqabUbU00alNOEHkQ1dVbwgM53/nAWb2dnLCa2bxrd88y1dvfRKz6CKwSqVgcvQ//nLY/mqtk2b0hhlcn/7R7/jw66NrR2588AW2D+TLx2TP6e1sp+hw6IU/xzAKIYC06qyqKV3tPPbSdg79zM8ByBeLTOls14oELUCBY4LIFZz2tjbmTu/mr088lMtufYJH12/jhNfMYu3G6EK8q85YVvWf6k9eN4e+bJ5cwcv7ZvR2cMjs3VvYbFp3Bwfs3ctzW/rK61WV8nLm8Qt3672leaceM5e+bJ58cej3vO9eXcyZ3j2Oudp1H/+jg9m/Iu+L50wbp9xIMxQ4JohsoUhHu2FmnPfmQ7js1ifKM6n6sgXmTO8etsxI3IzeTs554yGJ5OtDf7CAf775cbKFIl3tUddCb2eGeTN6Evk8Gd2c6T188u2HjXc2xsziudNYPFeBohUl2ploZieZ2eNmttbMLqhy3Mzsa+H4ajNbGvZ3m9lvzewhM1tjZhfHzrnIzF4wswfD4+Qky7Cn5AvFct9ue6aN7o628tjCeC5kVzmzqk/LjYhMeom1OMwsA1wOvA1YB9xrZivd/ZFYshXAovB4PXBFeB4E3uLuO8ysA7jLzH7u7neH8y5z939JKu97WqHoFB3a24bi+NSu9nKLY2AcFxWMr501AxjIaoFDkckuyRbHcmCtuz/t7lngeuCUijSnANd55G5ghpnNCa93hDQd4eGkVK4Qzc3vaB8av5jS1V6egtuXHb9v+T2VLY5xzIuITAxJBo55wPOx1+vCvobSmFnGzB4ENgK3uPs9sXTnh66tq80svnDrhFIsOn3ZfHn65GjKgSPW4pjSORQ4xvNK7Z6K6cGtev8HERk7SQaOanPqKmvQUdO4e8HdlwDzgeVmdmQ4fgVwCLAEWA98ueqHm51jZqvMbNWmTZuaz/0Y+LNr72XxZ2/m8z99tGa60myojky8xZEpd1VV3g9jT6q8IHE88yIiE0OSgWMdsCD2ej7wYrNp3P1V4FfASeH1hhBUisDXibrERnD3q9x9mbsvmz179m4UY9etXhfdCOnpzTtqpsuXu6piLY6udnYODn3LH7euqiotDnVViUxuSQaOe4FFZnaQmXUCpwErK9KsBM4Ms6uOA7a6+3ozm21mMwDMrAc4EXgsvJ4TO/89wMMJlmGXDeYLbAlXe2fzxZpps9W6qrray/dgHs8lPspjHLlYV5UCh8ikltisKnfPm9n5wM1ABrja3deY2bnh+JXATcDJwFqgDzg7nD4HuDbMzGoDvu/uPw3HLjWzJURdWs8Cf5FUGXbHxm2D5e16gaN0H/H44PheXe08s3knSz9/C1t2Zsd9jKM0OK6uKhFJ9AJAd7+JKDjE910Z23bgvCrnrQaOGeU9zxjjbCZiw7YBILpFZqlFMZrL71gLwIyezvK+jx53IJ3tbazduIP/fepltvbnkstsDb2dI1sc6qoSmdx05XhCNm2PWhzzZvTUbXEUi86UzgxvOmxoLObIedM5ct507nn6Zf73qZd5ckPtcZKklLqqbl7zEhu3DbJ9IKdZVSKTnAJHQkqtjOk9HeXZUaMZzBc5dP+9qq5DddT86WTajI+/8eBE8lnP1K525s/s4VePb+JXj2/CDA7db69xyYuITAwKHAnJhym2PZ2Z8iD5aLL5Il3t1ecp9Ha289Q/jd+qKu2ZNv77795M/FKUtjatXioymWnh+4QUikNLXg/W6aoaLBTpbJ+43T9mRlvb0ENEJjcFjoTkilGw6O3MkM0XaqbN5ou6eY2ItAzVVgkptTh6OtrrzqrK5gujdlWJiEw0qq0SUhrjKHVV1VqvajBfpFOBQ0RahGqrhJTHOLoyuDPsrm0v7xgsT9eF2oPjIiITjWZVJaQUKHo7oh9xNh/dUxzg2C/cCsCzl7wzOlZQi0NEWodqq4QUYoPjUHvZEQ2Oi0grUW2VkFKLo7TWU60B8qzGOESkhai2Skih6LQZ5bGL0VochaKTL7oCh4i0DNVWCckXnfa2tnJAGAzXcrzwav+wdKWAosAhIq1CtVVCCkUn02blJciv/PXTPPzCVk645PZh6UqBo2sCXzkuIhKnwJGQfMFpbzOOP2QfIFpmvbK1ATBYiFoianGISKtQbZWQQrFIJmNM6+7g+IP3oT9bYMfAyFVyyy0OzaoSkRah6zgSEo1xRAsC9nRm2L49N2J59cdf2s66V/oAtThEpHUocCSkNMYB0N3RxkCuOCJwvOOrd5a3p/XoVyEirUG1VUJKs6oAutszDOQKbK/oqjp0v6l84sRD6e5o442LZld7GxGRCUeBIyHxFkdXRya0OIbfN3y/ad2cfNSc8cieiMguU8d6QuJjHFFX1cjB8XbdFElEWpACR0IKxWK5xdHTUb2rKtOmH7+ItB7VXAnJF+KD4xnyRee2xzYOS6MWh4i0IgWOhBSKTntmqKuqJL6dUeAQkRakwJGQfNHLXVGlZUcA3nPM/PK2AoeItCIFjoQU4oPjsXWopvd0lLfVVSUirUiBIyG5wtDgeHfnUOCIxwq1OESkFSlwJGR4iyP6Me89pZM2GwoWChwi0op0AWBC8kWnJwSGYw+cyalL5nLi4v14/KXt5TRtChwi0oIUOBISb3HsM7WLr552DABPbHiinEZjHCLSitRVlZD4rKq4eKhQV5WItCIFjoQUisWqLYq5M7rL2/Nn9u7JLImIjAl1VSUkX3QymZGB4wPHLmB6TyfgvG3x/ns+YyIiu0mBIyF9gwV6O0beR7ytzTjpSAUMEWld6qpKyNb+3LCL/URE0kKBIwGD+QL9uQIzehU4RCR9FDgSsLU/umGTWhwikkYKHAnYVgocvZ3jnBMRkbGXaOAws5PM7HEzW2tmF1Q5bmb2tXB8tZktDfu7zey3ZvaQma0xs4tj5+xtZreY2ZPheWaSZdgVr/apxSEi6ZVY4DCzDHA5sAJYDJxuZosrkq0AFoXHOcAVYf8g8BZ3PxpYApxkZseFYxcAt7n7IuC28HpCUVeViKRZki2O5cBad3/a3bPA9cApFWlOAa7zyN3ADDObE17vCGk6wsNj51wbtq8FTk2wDLtk20AUOKZ1a7aziKRPkoFjHvB87PW6sK+hNGaWMbMHgY3ALe5+T0izn7uvBwjP+1b7cDM7x8xWmdmqTZs27W5ZmjKYKwLQ0znyOg4RkVaXZOCothCTN5rG3QvuvgSYDyw3syOb+XB3v8rdl7n7stmzZzdz6m4bzEeBozOjuQcikj5J1mzrgAWx1/OBF5tN4+6vAr8CTgq7NpjZHIDwvHHMcjxGBvMFALqqXDkuItLqkgwc9wKLzOwgM+sETgNWVqRZCZwZZlcdB2x19/VmNtvMZgCYWQ9wIvBY7JyzwvZZwI0JlmGXlLqqutrV4hCR9Els9Nbd82Z2PnAzkAGudvc1ZnZuOH4lcBNwMrAW6APODqfPAa4NM7PagO+7+0/DsUuA75vZx4DngA8kVYZdNZgv0ma634aIpFOi037c/Sai4BDfd2Vs24Hzqpy3GjhmlPd8GXjr2OZ0bA3mC3S1ZzBT4BCR9FFfSgKy+SJdHfrRikg6qXZLwGC+qPENEUkt1W4JiAKHZlSJSDopcCRgMF+gUy0OEUkp1W4JGMypq0pE0ku1WwI0xiEiaabaLQGl6bgiImmkwJEATccVkTRT7ZYAdVWJSJqpdkuApuOKSJopcCRgMKfpuCKSXqrdEqCuKhFJs4ZqNzP7KzObFpY//6aZ3W9mb086c61KXVUikmaNfi3+M3ffBrwdmE20/PklieWqxQ3mC5pVJSKp1WjtVlof/GTgGnd/iOq3fZ30ikUnV3B1VYlIajVau91nZr8kChw3m9leQDG5bLWubKF09z91VYlIOjV6I6ePAUuAp929z8z2YehufRKj28aKSNo1FDjcvWhmG4DFZpboXQNb3WC+AKDpuCKSWg0FATP7EvAh4BGgEHY7cGdC+WpZg3m1OEQk3RptPZwKHObugwnmJRVKLY6uDo1xiEg6Nfq1+GmgI8mMpMWAxjhEJOUabXH0AQ+a2W1AudXh7n+ZSK5a2NCsKgUOEUmnRgPHyvCQOoZmVamrSkTSqW7gMLMMcIa7n7gH8tPyhsY41OIQkXSqW7u5ewHoM7PpeyA/LU+zqkQk7RrtqhoAfmdmtwA7Szs1xjGSAoeIpF2jgeNn4SF1DObCBYAZjXGISDo1euX4tUlnJC3yRQd05biIpFejV44/Q3Sl+DDufvCY56jF5cJ03PaMFg8WkXRqtKtqWWy7G/gAsPfYZ6f1ZcMYR0dGLQ4RSaeGajd3fzn2eMHdvwq8JdmstaZyV5UCh4ikVKNdVUtjL9uIWiB7JZKjFpfLq6tKRNKt0a6qL8e288AzwAfHPjutLxdaHO1tChwikk4N38jJ3Z+O7zCzgxLIT8vLFYp0ZAwzBQ4RSadGO+J/0OC+SS+XL2pgXERSrWaLw8wOB14LTDez98YOTSOaXSUV8kVX4BCRVKvXVXUY8CfADOBdsf3bgY8nlKeWlg1dVSIiaVUzcLj7jcCNZna8u/+m2Tc3s5OAfwUywDfc/ZKK4xaOn0x0z48/dff7zWwBcB2wP1AErnL3fw3nXEQUtDaFt/m0u9/UbN6Ski+oq0pE0q3RGu5lM7vNzB4GMLPXmdmFtU4Iy7FfDqwAFgOnm9niimQrgEXhcQ5wRdifBz7p7kcAxwHnVZx7mbsvCY8JEzQAcgXXVFwRSbVGA8fXgU8BOQB3Xw2cVuec5cBad3/a3bPA9cApFWlOAa7zyN3ADDOb4+7r3f3+8FnbgUeBeQ3mdVxl1eIQkZRrtIbrdfffVuzL1zlnHvB87PU6Rlb+ddOY2ULgGOCe2O7zzWy1mV1tZjOrfbiZnWNmq8xs1aZNm6olSUS+UNRV4yKSao3WcJvN7BDCQodm9n5gfZ1zqvXXVC6UWDONmU0Ffgh8wt23hd1XAIcAS0Ievlz5BgDufpW7L3P3ZbNnz66T1bGjrioRSbtGLwA8D7gKONzMXiC6cvwjdc5ZByyIvZ4PvNhoGjPrIAoa33b3G0oJ3H1DadvMvg78tMEy7BE5dVWJSMo1usjh0+Ge47OBw4E3AW+oc9q9wCIzO8jMOonGRFZWpFkJnGmR44Ct7r4+zLb6JvCou38lfoKZzYm9fA/wcCNl2FNyhSIdbQocIpJe9S4AnEbU2pgH3AjcGl7/LfAQ8O3RznX3vJmdD9xMNB33andfY2bnhuNXAjcRTcVdSzQd9+xw+gnAGUS3q30w7CtNu73UzJYQdWk9C/xFUyVOWK7gdHcocIhIetXrqvoW8ArwG6JrJ/4O6AROdfcH6715qOhvqth3ZWzbiQJR5Xl3UX38A3c/o97njqd8oUhHd6M9gCIiradeDXewux8FYGbfADYDB4QpslJFtuC0q6tKRFKsXg2XK224ewF4RkGjtnyhSGe7ZlWJSHrVa3EcbWalabAG9ITXRtTTNC3R3LWgXKGoFoeIpFq9taoyeyojaTCQK/Bqf46udgUOEUkv1XBj6OY1L/FqX46Tj5pTP7GISItS4BhDW3ZmAViyYMb4ZkREJEEKHGOoP1cAoKdTPXwikl4KHGOoP1vADI1xiEiqqYYbQ/3ZAj0dGaIVU0RE0kmBYwz156LAISKSZgocY6g/W9D4hoikngLHGFKLQ0QmAwWOMdSXLdCrFoeIpJwCxxjqzxXoVotDRFJOgaNJG7cNcPtjG6oeG8hpjENE0k+Bowl3PbmZ5f90G39+7Sqy+eKI4+qqEpHJQIGjCR/95j0AFB1e7cuOON6fVVeViKSfAscu2lIlcAzmi3S1K3CISLopcOyiLTtGBo5coUhnRleNi0i6KXDsomotjnyhSHtGP1IRSTfVcruotIR6XK7otKvFISIpp8Cxi7bszLJ24w7+5ebHcXeg1FWlH6mIpJtquV30ys4sH/763fzbHWt5tS9Hoei4o/uNi0jqqZbbRS/vzLJ5xyAAuWKRXCG6rkNdVSKSdgocu+iVvizFqIeKwdxQ4FBXlYiknWq5XbRlZ668PZgvki9EUUQtDhFJOwWOXbRl52B5ezBfIFcsdVXpRyoi6aZarkGlmVMlr1S0OHKhxaELAEUk7RQ4GvTNu54Z9jpbGFrkcCBXIF8aHNesKhFJOdVyDVj3Sh9f+Nmjox6Ptzg0xiEiaafA0YDnXu6refzsa+5l20DUdaVZVSKSdqrlGvDU5p3l7bccvi+Xf3jpiDQvbR0ANDguIunXPt4ZaAXPbxlqcZy+/AAO22+vEWnO/879gLqqRCT99PW4Adv6h2ZQ9XZmmDmlY0Sa0sWA6qoSkbRTLdeAHYP58nZ3R4apXaM31Nrb1OIQkXRT4GhAPHD0dmYwGz04aIxDRNJOtVwDdsYCR0+de4qrq0pE0i7RWs7MTjKzx81srZldUOW4mdnXwvHVZrY07F9gZneY2aNmtsbM/ip2zt5mdouZPRmeZyZZBoAdg4Xydm9n7cChwXERSbvEAoeZZYDLgRXAYuB0M1tckWwFsCg8zgGuCPvzwCfd/QjgOOC82LkXALe5+yLgtvA6UTsGhwbHuysCx/lvfs2w1x0KHCKSckm2OJYDa939aXfPAtcDp1SkOQW4ziN3AzPMbI67r3f3+wHcfTvwKDAvds61Yfta4NQEywDAzniLI3RVlQLEx994MBe/+7Xl41pyRETSLslabh7wfOz1OoYq/4bTmNlC4BjgnrBrP3dfDxCe96324WZ2jpmtMrNVmzZt2tUyALBjYGiMozT4/eUPLmHB3j1M7Wrn2AOHess62hU4RCTdkrwAsFqfjTeTxsymAj8EPuHu25r5cHe/CrgKYNmyZZWf27Bsvki2UOSM4w7kHa/dv7z/3UfP5d1HzwWgu2MoWKirSkTSLsmvx+uABbHX84EXG01jZh1EQePb7n5DLM0GM5sT0swBNo5xvof5zdMvA7Bs4UzesGhW1TRd7UPjHtO6R14cKCKSJkkGjnuBRWZ2kJl1AqcBKyvSrATODLOrjgO2uvt6iy6U+CbwqLt/pco5Z4Xts4AbkysC3PHYRno7M6w4cs6oabpi3VPddabrioi0usS6qtw9b2bnAzcDGeBqd19jZueG41cCNwEnA2uBPuDscPoJwBnA78zswbDv0+5+E3AJ8H0z+xjwHPCBpMoA0d39pnS101lj7CLe4hARSbtEFzkMFf1NFfuujG07cF6V8+6i+vgH7v4y8NaxzenoBvPFuhf1dXVoQFxEJg/VeHVk88VhXVHV6GpxEZlMVOPVkc0Xa3ZTAbRpYUMRmUQUOOrIFuoHDhGRyUQ1Yh3ZBsY4So45YEaymRERmQB0B8A6GumqAnjgH95GT50FEEVE0kCBo45socjU7vo/pplTOvdAbkRExp+6qurI5ot0aNaUiEiZasQ6Gu2qEhGZLFQj1jGYL9KlFoeISJlqxDpymo4rIjKMasQ6dB2HiMhwqhHraOY6DhGRyUA1Yh0aHBcRGU41Yg3FopMvugKHiEiMasQasoUigAKHiEiMasQaBvMhcGiMQ0SkTDViDdkQOOrdj0NEZDJRjVhDqatKS46IiAxRjVhDqcWhMQ4RkSGqEWvIaXBcRGQE1Yg1ZDU4LiIygmrEGgbVVSUiMoJqxBo0xiEiMpJqxBpKs6o0HVdEZIhqxBqGxjh0L3ERkRIFjhrUVSUiMpJqxBqyhQKgwCEiEqcasQa1OERERlKNWEMpcHRkbJxzIiIycShw1FC6jqNLg+MiImUKHDXkCg6oq0pEJE41Yg0a4xARGUk1Yg3ZQoFMm5Fp0xiHiEiJAkcN2XxRCxyKiFRQrVhDNl9UN5WISIX28c7ARHbEnGn05wrjnQ0RkQlFgaOG05YfwGnLDxjvbIiITCiJ9sOY2Ulm9riZrTWzC6ocNzP7Wji+2syWxo5dbWYbzezhinMuMrMXzOzB8Dg5yTKIiMhwiQUOM8sAlwMrgMXA6Wa2uCLZCmBReJwDXBE79h/ASaO8/WXuviQ8bhrTjIuISE1JtjiWA2vd/Wl3zwLXA6dUpDkFuM4jdwMzzGwOgLvfCWxJMH8iIrILkgwc84DnY6/XhX3Npqnm/NC1dbWZzayWwMzOMbNVZrZq06ZNzeRbRERqSDJwVLtqznchTaUrgEOAJcB64MvVErn7Ve6+zN2XzZ49u85biohIo5IMHOuABbHX84EXdyHNMO6+wd0L7l4Evk7UJSYiIntIkoHjXmCRmR1kZp3AacDKijQrgTPD7KrjgK3uvr7Wm5bGQIL3AA+PllZERMZeYtdxuHvezM4HbgYywNXuvsbMzg3HrwRuAk4G1gJ9wNml883su8CbgFlmtg74nLt/E7jUzJYQdWk9C/xFUmUQEZGRzL3ekELrM7NNwO938fRZwOYxzM54UlkmprSUJS3lAJWl5EB3HzFIPCkCx+4ws1Xuvmy88zEWVJaJKS1lSUs5QGWpRyv4iYhIUxQ4RESkKQoc9V013hkYQyrLxJSWsqSlHKCy1KQxDhERaYpaHCIi0hQFDhERaYoCRw317icy0VS7h4mZ7W1mt5jZk+F5ZuzYp0LZHjezd4xPrkcyswVmdoeZPWpma8zsr8L+VixLt5n91sweCmW5OOxvubJAdLsEM3vAzH4aXrdqOZ41s9+Fe/qsCvtatSwzzOwHZvZY+J85PvGyuLseVR5EV7s/BRwMdAIPAYvHO1918vxGYCnwcGzfpcAFYfsC4Ethe3EoUxdwUChrZrzLEPI2B1gatvcCngj5bcWyGDA1bHcA9wDHtWJZQv7+BvgO8NNW/fsK+XsWmFWxr1XLci3w52G7E5iRdFnU4hhdI/cTmVC8+j1MTiH6wyI8nxrbf727D7r7M0TLvkyIBSPdfb273x+2twOPEi2334plcXffEV52hIfTgmUxs/nAO4FvxHa3XDlqaLmymNk0oi+M3wRw96y7v0rCZVHgGN2u3itkotnPw8KR4XnfsL8lymdmC4FjiL6pt2RZQvfOg8BG4BZ3b9WyfBX4O6AY29eK5YAoeP/SzO4zs3PCvlYsy8HAJuCa0IX4DTObQsJlUeAY3a7cK6SVTPjymdlU4IfAJ9x9W62kVfZNmLJ4dBuAJUS3DVhuZkfWSD4hy2JmfwJsdPf7Gj2lyr5xL0fMCe6+lOj21eeZ2RtrpJ3IZWkn6p6+wt2PAXYSdU2NZkzKosAxuqbvFTJBbSgtRR+eN4b9E7p8ZtZBFDS+7e43hN0tWZaS0IXwK+AkWq8sJwDvNrNnibpt32Jm/0nrlQMAd38xPG8EfkTUXdOKZVkHrAutWIAfEAWSRMuiwDG6Ru4n0gpWAmeF7bOAG2P7TzOzLjM7CFgE/HYc8jeCmRlRn+2j7v6V2KFWLMtsM5sRtnuAE4HHaLGyuPun3H2+uy8k+l+43d0/SouVA8DMppjZXqVt4O1E9/VpubK4+0vA82Z2WNj1VuARki7LeM8ImMgPonuFPEE08+Az452fBvL7XaLb6eaIvll8DNgHuA14MjzvHUv/mVC2x4EV453/WL7eQNR8Xg08GB4nt2hZXgc8EMryMPDZsL/lyhLL35sYmlXVcuUgGhd4KDzWlP63W7EsIW9LgFXhb+zHwMyky6IlR0REpCnqqhIRkaYocIiISFMUOEREpCkKHCIi0hQFDhERaYoCh0iDzKwQVlMtPWqumGxm55rZmWPwuc+a2azdfR+RsaLpuCINMrMd7j51HD73WWCZu2/e058tUo1aHCK7KbQIvhTuu/FbM3tN2H+Rmf1t2P5LM3vEzFab2fVh395m9uOw724ze13Yv4+Z/TIsWvfvxNYXMrOPhs940Mz+PSygmDGz/zCzh8M9Jv56HH4MMokocIg0rqeiq+pDsWPb3H058G9Eq8hWugA4xt1fB5wb9l0MPBD2fRq4Luz/HHCXR4vWrQQOADCzI4APES3QtwQoAB8hunJ4nrsf6e5HAdeMVYFFqmkf7wyItJD+UGFX893Y82VVjq8Gvm1mPyZaFgKipVXeB+Dut4eWxnSi+yu8N+z/mZm9EtK/FTgWuDdazoseosXrfgIcbGb/D/gZ8MtdLJ9IQ9TiEBkbPsp2yTuBy4kq/vvMrJ3aS1xXew8DrnX3JeFxmLtf5O6vAEcTrbx7HsNvtCQy5hQ4RMbGh2LPv4kfMLM2YIG730F0I6QZwFTgTqKuJszsTcBmj+47Et+/gmjROogWq3u/me0bju1tZgeGGVdt7v5D4B+IltUWSYy6qkQa1xPu5FfyC3cvTcntMrN7iL6MnV5xXgb4z9ANZcBl7v6qmV1EdOe21UAfQ8tgXwx818zuB34NPAfg7o+Y2YVEd65rI1oF+TygP7xP6Yvgp8asxCJVaDquyG7SdFmZbNRVJSIiTVGLQ0REmqIWh4iINEWBQ0REmqLAISIiTVHgEBGRpihwiIhIU/4/9I4x1ODo3nAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import rl_utils\n",
    "\n",
    "\n",
    "class PolicyNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim):\n",
    "        super(PolicyNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, action_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        return F.softmax(self.fc2(x), dim=1)\n",
    "\n",
    "\n",
    "class ValueNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim):\n",
    "        super(ValueNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        return self.fc2(x)\n",
    "\n",
    "\n",
    "class PPO:\n",
    "    ''' PPO算法,采用截断方式 '''\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim, actor_lr, critic_lr,lmbda, epochs, eps, gamma, device):\n",
    "        # 创建并初始化策略网络（Actor）和价值网络（Critic）\n",
    "        self.actor = PolicyNet(state_dim, hidden_dim, action_dim).to(device)\n",
    "        self.critic = ValueNet(state_dim, hidden_dim).to(device)\n",
    "\n",
    "        # 定义策略网络和价值网络的优化器，使用Adam优化器\n",
    "        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),lr=actor_lr)\n",
    "        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),lr=critic_lr)\n",
    "\n",
    "        self.gamma = gamma\n",
    "        self.lmbda = lmbda   # Generalized Advantage Estimation (GAE)的参数\n",
    "        self.epochs = epochs  # 一条序列的数据用来训练轮数\n",
    "        self.eps = eps  # PPO中截断范围的参数\n",
    "        self.device = device\n",
    "\n",
    "    def take_action(self, state):\n",
    "        # 将输入状态转换为PyTorch张量，并移动到指定的设备（CPU或GPU）\n",
    "        state = torch.tensor([state], dtype=torch.float).to(self.device)\n",
    "\n",
    "        # 通过策略网络（Actor）获取在当前状态下采取每个动作的概率分布\n",
    "        probs = self.actor(state)\n",
    "\n",
    "        # 使用Categorical分布对象根据概率分布进行采样，得到选定的动作\n",
    "        action_dist = torch.distributions.Categorical(probs)\n",
    "        action = action_dist.sample()\n",
    "\n",
    "        return action.item()\n",
    "\n",
    "    def update(self, transition_dict):\n",
    "        # 从传入的transition_dict中获取训练所需的数据\n",
    "        states = torch.tensor(transition_dict['states'],dtype=torch.float).to(self.device)\n",
    "        actions = torch.tensor(transition_dict['actions']).view(-1, 1).to(self.device)\n",
    "        rewards = torch.tensor(transition_dict['rewards'],dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        next_states = torch.tensor(transition_dict['next_states'],dtype=torch.float).to(self.device)\n",
    "        dones = torch.tensor(transition_dict['dones'],dtype=torch.float).view(-1, 1).to(self.device)\n",
    "\n",
    "         # 计算 TD 目标（Temporal Difference Target）\n",
    "        td_target = rewards + self.gamma * self.critic(next_states) * (1 -dones)\n",
    "\n",
    "        # 计算 TD 误差（Temporal Difference Delta）\n",
    "        td_delta = td_target - self.critic(states)\n",
    "\n",
    "        # 计算 Advantage（优势函数）\n",
    "        advantage = rl_utils.compute_advantage(self.gamma, self.lmbda,td_delta.cpu()).to(self.device)\n",
    "\n",
    "        # 计算旧的动作概率的对数值\n",
    "        old_log_probs = torch.log(self.actor(states).gather(1,actions)).detach()\n",
    "\n",
    "        for _ in range(self.epochs):\n",
    "            # 计算新的动作概率的对数值\n",
    "            log_probs = torch.log(self.actor(states).gather(1, actions))\n",
    "\n",
    "            # 计算 PPO 损失函数的比率（Clipped Surrogate Objective）\n",
    "            ratio = torch.exp(log_probs - old_log_probs)\n",
    "\n",
    "            # 计算两个截断项（clipping terms）\n",
    "            surr1 = ratio * advantage\n",
    "            surr2 = torch.clamp(ratio, 1 - self.eps,1 + self.eps) * advantage  # 截断\n",
    "\n",
    "            # 计算 PPO 损失函数\n",
    "            actor_loss = torch.mean(-torch.min(surr1, surr2))  # PPO损失函数\n",
    "\n",
    "            # 计算 Critic 损失函数\n",
    "            critic_loss = torch.mean(F.mse_loss(self.critic(states), td_target.detach()))\n",
    "\n",
    "            # 梯度清零\n",
    "            self.actor_optimizer.zero_grad()\n",
    "            self.critic_optimizer.zero_grad()\n",
    "\n",
    "            # 反向传播和梯度更新\n",
    "            actor_loss.backward()\n",
    "            critic_loss.backward()\n",
    "            self.actor_optimizer.step()\n",
    "            self.critic_optimizer.step()\n",
    "\n",
    "\n",
    "\n",
    "actor_lr = 1e-3\n",
    "critic_lr = 1e-2\n",
    "num_episodes = 600\n",
    "hidden_dim = 128\n",
    "gamma = 0.98\n",
    "lmbda = 0.95\n",
    "epochs = 10\n",
    "eps = 0.2\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "\n",
    "env_name = 'MyEnv-v0'\n",
    "env = gym.make(env_name)\n",
    "env.seed(0)\n",
    "torch.manual_seed(0)\n",
    "state_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.n\n",
    "agent = PPO(state_dim, hidden_dim, action_dim, actor_lr, critic_lr, lmbda,epochs, eps, gamma, device)\n",
    "\n",
    "return_list = rl_utils.train_on_policy_agent(env, agent, num_episodes)\n",
    "  \n",
    "episodes_list = list(range(len(return_list)))\n",
    "plt.plot(episodes_list, return_list)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('PPO on {}'.format(env_name))\n",
    "plt.show()\n",
    "\n",
    "mv_return = rl_utils.moving_average(return_list, 9)\n",
    "plt.plot(episodes_list, mv_return)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('PPO on {}'.format(env_name))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cyj",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}