A2C.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Actor(nn.Module):\n",
    "    def __init__(self, n_actions, space_dims, hidden_dims):\n",
    "        super(Actor, self).__init__()\n",
    "        self.feature_extractor = nn.Sequential(\n",
    "            nn.Linear(space_dims, hidden_dims),\n",
    "            nn.ReLU(True),\n",
    "        )\n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(hidden_dims, n_actions),\n",
    "            nn.Softmax(dim=-1),\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        features = self.feature_extractor(x)\n",
    "        policy = self.actor(features)\n",
    "        return policy\n",
    "    \n",
    "class Critic(nn.Module):\n",
    "    def __init__(self, space_dims, hidden_dims):\n",
    "        super(Critic, self).__init__()\n",
    "        self.feature_extractor = nn.Sequential(\n",
    "            nn.Linear(space_dims, hidden_dims),\n",
    "            nn.ReLU(True),\n",
    "        )\n",
    "        self.critic = nn.Linear(hidden_dims, 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        features = self.feature_extractor(x)\n",
    "        est_reward = self.critic(features)\n",
    "        return est_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class PGLoss(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(PGLoss, self).__init__()\n",
    "    \n",
    "    def forward(self, action_prob, reward):\n",
    "        loss = -torch.mean(torch.log(action_prob+1e-6)*reward)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def select_action(policy):\n",
    "    return np.random.choice(len(policy), 1, p=policy)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def to_tensor(x, dtype=None):\n",
    "    return torch.tensor(x, dtype=dtype).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ConfigArgs:\n",
    "    discounted_factor = 0.99\n",
    "    lr_critic = 0.005\n",
    "    lr_actor = 0.001\n",
    "    max_eps = 1000\n",
    "    sparse_mode = True\n",
    "\n",
    "args = ConfigArgs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pg_loss = PGLoss()\n",
    "mse_loss = nn.MSELoss()\n",
    "nll_loss = nn.NLLLoss()\n",
    "xe_loss = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "actor = Actor(n_actions=env.action_space.n, space_dims=4, hidden_dims=32)\n",
    "critic = Critic(space_dims=4, hidden_dims=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a_optim = torch.optim.Adam(actor.parameters(), lr=args.lr_actor)\n",
    "c_optim = torch.optim.Adam(critic.parameters(), lr=args.lr_critic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 1, AVG Score: 14.000, Score: 14.0\n",
      "Episodes: 2, AVG Score: 11.500, Score: 9.0\n",
      "Episodes: 3, AVG Score: 14.667, Score: 21.0\n",
      "Episodes: 4, AVG Score: 15.500, Score: 18.0\n",
      "Episodes: 5, AVG Score: 15.800, Score: 17.0\n",
      "Episodes: 6, AVG Score: 17.667, Score: 27.0\n",
      "Episodes: 7, AVG Score: 16.571, Score: 10.0\n",
      "Episodes: 8, AVG Score: 17.625, Score: 25.0\n",
      "Episodes: 9, AVG Score: 17.889, Score: 20.0\n",
      "Episodes: 10, AVG Score: 18.700, Score: 26.0\n",
      "Episodes: 11, AVG Score: 18.182, Score: 13.0\n",
      "Episodes: 12, AVG Score: 18.333, Score: 20.0\n",
      "Episodes: 13, AVG Score: 18.692, Score: 23.0\n",
      "Episodes: 14, AVG Score: 19.786, Score: 34.0\n",
      "Episodes: 15, AVG Score: 19.267, Score: 12.0\n",
      "Episodes: 16, AVG Score: 18.750, Score: 11.0\n",
      "Episodes: 17, AVG Score: 19.941, Score: 39.0\n",
      "Episodes: 18, AVG Score: 19.389, Score: 10.0\n",
      "Episodes: 19, AVG Score: 19.263, Score: 17.0\n",
      "Episodes: 20, AVG Score: 21.300, Score: 60.0\n",
      "Episodes: 21, AVG Score: 21.300, Score: 14.0\n",
      "Episodes: 22, AVG Score: 22.300, Score: 29.0\n",
      "Episodes: 23, AVG Score: 22.400, Score: 23.0\n",
      "Episodes: 24, AVG Score: 22.350, Score: 17.0\n",
      "Episodes: 25, AVG Score: 22.150, Score: 13.0\n",
      "Episodes: 26, AVG Score: 22.600, Score: 36.0\n",
      "Episodes: 27, AVG Score: 23.250, Score: 23.0\n",
      "Episodes: 28, AVG Score: 22.650, Score: 13.0\n",
      "Episodes: 29, AVG Score: 22.700, Score: 21.0\n",
      "Episodes: 30, AVG Score: 22.500, Score: 22.0\n",
      "Episodes: 31, AVG Score: 22.800, Score: 19.0\n",
      "Episodes: 32, AVG Score: 22.900, Score: 22.0\n",
      "Episodes: 33, AVG Score: 22.200, Score: 9.0\n",
      "Episodes: 34, AVG Score: 22.550, Score: 41.0\n",
      "Episodes: 35, AVG Score: 22.450, Score: 10.0\n",
      "Episodes: 36, AVG Score: 23.150, Score: 25.0\n",
      "Episodes: 37, AVG Score: 22.350, Score: 23.0\n",
      "Episodes: 38, AVG Score: 23.750, Score: 38.0\n",
      "Episodes: 39, AVG Score: 25.500, Score: 52.0\n",
      "Episodes: 40, AVG Score: 26.550, Score: 81.0\n",
      "Episodes: 41, AVG Score: 27.250, Score: 28.0\n",
      "Episodes: 42, AVG Score: 27.000, Score: 24.0\n",
      "Episodes: 43, AVG Score: 26.900, Score: 21.0\n",
      "Episodes: 44, AVG Score: 26.850, Score: 16.0\n",
      "Episodes: 45, AVG Score: 26.750, Score: 11.0\n",
      "Episodes: 46, AVG Score: 26.250, Score: 26.0\n",
      "Episodes: 47, AVG Score: 25.850, Score: 15.0\n",
      "Episodes: 48, AVG Score: 26.300, Score: 22.0\n",
      "Episodes: 49, AVG Score: 26.250, Score: 20.0\n",
      "Episodes: 50, AVG Score: 25.650, Score: 10.0\n",
      "Episodes: 51, AVG Score: 25.900, Score: 24.0\n",
      "Episodes: 52, AVG Score: 25.800, Score: 20.0\n",
      "Episodes: 53, AVG Score: 26.000, Score: 13.0\n",
      "Episodes: 54, AVG Score: 24.450, Score: 10.0\n",
      "Episodes: 55, AVG Score: 24.450, Score: 10.0\n",
      "Episodes: 56, AVG Score: 24.400, Score: 24.0\n",
      "Episodes: 57, AVG Score: 23.900, Score: 13.0\n",
      "Episodes: 58, AVG Score: 24.800, Score: 56.0\n",
      "Episodes: 59, AVG Score: 23.850, Score: 33.0\n",
      "Episodes: 60, AVG Score: 20.450, Score: 13.0\n",
      "Episodes: 61, AVG Score: 20.250, Score: 24.0\n",
      "Episodes: 62, AVG Score: 19.950, Score: 18.0\n",
      "Episodes: 63, AVG Score: 20.300, Score: 28.0\n",
      "Episodes: 64, AVG Score: 20.350, Score: 17.0\n",
      "Episodes: 65, AVG Score: 20.500, Score: 14.0\n",
      "Episodes: 66, AVG Score: 19.950, Score: 15.0\n",
      "Episodes: 67, AVG Score: 21.500, Score: 46.0\n",
      "Episodes: 68, AVG Score: 22.300, Score: 38.0\n",
      "Episodes: 69, AVG Score: 22.100, Score: 16.0\n",
      "Episodes: 70, AVG Score: 23.600, Score: 40.0\n",
      "Episodes: 71, AVG Score: 23.500, Score: 22.0\n",
      "Episodes: 72, AVG Score: 23.400, Score: 18.0\n",
      "Episodes: 73, AVG Score: 23.700, Score: 19.0\n",
      "Episodes: 74, AVG Score: 24.150, Score: 19.0\n",
      "Episodes: 75, AVG Score: 24.950, Score: 26.0\n",
      "Episodes: 76, AVG Score: 24.750, Score: 20.0\n",
      "Episodes: 77, AVG Score: 29.000, Score: 98.0\n",
      "Episodes: 78, AVG Score: 26.800, Score: 12.0\n",
      "Episodes: 79, AVG Score: 26.900, Score: 35.0\n",
      "Episodes: 80, AVG Score: 28.700, Score: 49.0\n",
      "Episodes: 81, AVG Score: 28.850, Score: 27.0\n",
      "Episodes: 82, AVG Score: 29.050, Score: 22.0\n",
      "Episodes: 83, AVG Score: 28.500, Score: 17.0\n",
      "Episodes: 84, AVG Score: 30.100, Score: 49.0\n",
      "Episodes: 85, AVG Score: 31.700, Score: 46.0\n",
      "Episodes: 86, AVG Score: 32.600, Score: 33.0\n",
      "Episodes: 87, AVG Score: 31.950, Score: 33.0\n",
      "Episodes: 88, AVG Score: 32.350, Score: 46.0\n",
      "Episodes: 89, AVG Score: 36.850, Score: 106.0\n",
      "Episodes: 90, AVG Score: 37.300, Score: 49.0\n",
      "Episodes: 91, AVG Score: 36.900, Score: 14.0\n",
      "Episodes: 92, AVG Score: 37.750, Score: 35.0\n",
      "Episodes: 93, AVG Score: 41.300, Score: 90.0\n",
      "Episodes: 94, AVG Score: 41.250, Score: 18.0\n",
      "Episodes: 95, AVG Score: 41.700, Score: 35.0\n",
      "Episodes: 96, AVG Score: 44.800, Score: 82.0\n",
      "Episodes: 97, AVG Score: 42.050, Score: 43.0\n",
      "Episodes: 98, AVG Score: 43.200, Score: 35.0\n",
      "Episodes: 99, AVG Score: 47.750, Score: 126.0\n",
      "Episodes: 100, AVG Score: 46.450, Score: 23.0\n",
      "Episodes: 101, AVG Score: 48.100, Score: 60.0\n",
      "Episodes: 102, AVG Score: 50.250, Score: 65.0\n",
      "Episodes: 103, AVG Score: 51.250, Score: 37.0\n",
      "Episodes: 104, AVG Score: 49.800, Score: 20.0\n",
      "Episodes: 105, AVG Score: 49.400, Score: 38.0\n",
      "Episodes: 106, AVG Score: 51.150, Score: 68.0\n",
      "Episodes: 107, AVG Score: 50.900, Score: 28.0\n",
      "Episodes: 108, AVG Score: 51.300, Score: 54.0\n",
      "Episodes: 109, AVG Score: 47.650, Score: 33.0\n",
      "Episodes: 110, AVG Score: 48.050, Score: 57.0\n",
      "Episodes: 111, AVG Score: 49.050, Score: 34.0\n",
      "Episodes: 112, AVG Score: 49.500, Score: 44.0\n",
      "Episodes: 113, AVG Score: 48.350, Score: 67.0\n",
      "Episodes: 114, AVG Score: 49.900, Score: 49.0\n",
      "Episodes: 115, AVG Score: 50.500, Score: 47.0\n",
      "Episodes: 116, AVG Score: 49.000, Score: 52.0\n",
      "Episodes: 117, AVG Score: 48.400, Score: 31.0\n",
      "Episodes: 118, AVG Score: 48.700, Score: 41.0\n",
      "Episodes: 119, AVG Score: 45.300, Score: 58.0\n",
      "Episodes: 120, AVG Score: 46.800, Score: 53.0\n",
      "Episodes: 121, AVG Score: 47.950, Score: 83.0\n",
      "Episodes: 122, AVG Score: 49.500, Score: 96.0\n",
      "Episodes: 123, AVG Score: 52.450, Score: 96.0\n",
      "Episodes: 124, AVG Score: 55.900, Score: 89.0\n",
      "Episodes: 125, AVG Score: 58.150, Score: 83.0\n",
      "Episodes: 126, AVG Score: 57.450, Score: 54.0\n",
      "Episodes: 127, AVG Score: 61.350, Score: 106.0\n",
      "Episodes: 128, AVG Score: 61.050, Score: 48.0\n",
      "Episodes: 129, AVG Score: 63.950, Score: 91.0\n",
      "Episodes: 130, AVG Score: 65.250, Score: 83.0\n",
      "Episodes: 131, AVG Score: 67.700, Score: 83.0\n",
      "Episodes: 132, AVG Score: 67.550, Score: 41.0\n",
      "Episodes: 133, AVG Score: 69.000, Score: 96.0\n",
      "Episodes: 134, AVG Score: 71.550, Score: 100.0\n",
      "Episodes: 135, AVG Score: 71.450, Score: 45.0\n",
      "Episodes: 136, AVG Score: 71.650, Score: 56.0\n",
      "Episodes: 137, AVG Score: 72.400, Score: 46.0\n",
      "Episodes: 138, AVG Score: 74.200, Score: 77.0\n",
      "Episodes: 139, AVG Score: 73.700, Score: 48.0\n",
      "Episodes: 140, AVG Score: 72.800, Score: 35.0\n",
      "Episodes: 141, AVG Score: 72.450, Score: 76.0\n",
      "Episodes: 142, AVG Score: 70.700, Score: 61.0\n",
      "Episodes: 143, AVG Score: 72.550, Score: 133.0\n",
      "Episodes: 144, AVG Score: 70.850, Score: 55.0\n",
      "Episodes: 145, AVG Score: 69.900, Score: 64.0\n",
      "Episodes: 146, AVG Score: 71.400, Score: 84.0\n",
      "Episodes: 147, AVG Score: 70.100, Score: 80.0\n",
      "Episodes: 148, AVG Score: 71.650, Score: 79.0\n",
      "Episodes: 149, AVG Score: 69.550, Score: 49.0\n",
      "Episodes: 150, AVG Score: 67.600, Score: 44.0\n",
      "Episodes: 151, AVG Score: 66.750, Score: 66.0\n",
      "Episodes: 152, AVG Score: 68.950, Score: 85.0\n",
      "Episodes: 153, AVG Score: 67.650, Score: 70.0\n",
      "Episodes: 154, AVG Score: 65.400, Score: 55.0\n",
      "Episodes: 155, AVG Score: 65.900, Score: 55.0\n",
      "Episodes: 156, AVG Score: 69.050, Score: 119.0\n",
      "Episodes: 157, AVG Score: 71.250, Score: 90.0\n",
      "Episodes: 158, AVG Score: 69.800, Score: 48.0\n",
      "Episodes: 159, AVG Score: 70.400, Score: 60.0\n",
      "Episodes: 160, AVG Score: 72.000, Score: 67.0\n",
      "Episodes: 161, AVG Score: 71.950, Score: 75.0\n",
      "Episodes: 162, AVG Score: 71.450, Score: 51.0\n",
      "Episodes: 163, AVG Score: 68.650, Score: 77.0\n",
      "Episodes: 164, AVG Score: 69.100, Score: 64.0\n",
      "Episodes: 165, AVG Score: 72.000, Score: 122.0\n",
      "Episodes: 166, AVG Score: 71.850, Score: 81.0\n",
      "Episodes: 167, AVG Score: 71.250, Score: 68.0\n",
      "Episodes: 168, AVG Score: 70.800, Score: 70.0\n",
      "Episodes: 169, AVG Score: 71.600, Score: 65.0\n",
      "Episodes: 170, AVG Score: 73.600, Score: 84.0\n",
      "Episodes: 171, AVG Score: 72.750, Score: 49.0\n",
      "Episodes: 172, AVG Score: 73.450, Score: 99.0\n",
      "Episodes: 173, AVG Score: 75.200, Score: 105.0\n",
      "Episodes: 174, AVG Score: 77.200, Score: 95.0\n",
      "Episodes: 175, AVG Score: 78.650, Score: 84.0\n",
      "Episodes: 176, AVG Score: 78.950, Score: 125.0\n",
      "Episodes: 177, AVG Score: 79.250, Score: 96.0\n",
      "Episodes: 178, AVG Score: 79.850, Score: 60.0\n",
      "Episodes: 179, AVG Score: 87.200, Score: 207.0\n",
      "Episodes: 180, AVG Score: 87.150, Score: 66.0\n",
      "Episodes: 181, AVG Score: 91.200, Score: 156.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 182, AVG Score: 92.300, Score: 73.0\n",
      "Episodes: 183, AVG Score: 91.500, Score: 61.0\n",
      "Episodes: 184, AVG Score: 94.350, Score: 121.0\n",
      "Episodes: 185, AVG Score: 91.150, Score: 58.0\n",
      "Episodes: 186, AVG Score: 93.350, Score: 125.0\n",
      "Episodes: 187, AVG Score: 93.500, Score: 71.0\n",
      "Episodes: 188, AVG Score: 93.950, Score: 79.0\n",
      "Episodes: 189, AVG Score: 93.800, Score: 62.0\n",
      "Episodes: 190, AVG Score: 95.650, Score: 121.0\n",
      "Episodes: 191, AVG Score: 97.600, Score: 88.0\n",
      "Episodes: 192, AVG Score: 97.000, Score: 87.0\n",
      "Episodes: 193, AVG Score: 94.300, Score: 51.0\n",
      "Episodes: 194, AVG Score: 92.500, Score: 59.0\n",
      "Episodes: 195, AVG Score: 93.300, Score: 100.0\n",
      "Episodes: 196, AVG Score: 90.800, Score: 75.0\n",
      "Episodes: 197, AVG Score: 92.600, Score: 132.0\n",
      "Episodes: 198, AVG Score: 92.450, Score: 57.0\n",
      "Episodes: 199, AVG Score: 85.350, Score: 65.0\n",
      "Episodes: 200, AVG Score: 85.050, Score: 60.0\n",
      "Episodes: 201, AVG Score: 83.250, Score: 120.0\n",
      "Episodes: 202, AVG Score: 83.950, Score: 87.0\n",
      "Episodes: 203, AVG Score: 88.150, Score: 145.0\n",
      "Episodes: 204, AVG Score: 86.050, Score: 79.0\n",
      "Episodes: 205, AVG Score: 85.650, Score: 50.0\n",
      "Episodes: 206, AVG Score: 84.350, Score: 99.0\n",
      "Episodes: 207, AVG Score: 87.900, Score: 142.0\n",
      "Episodes: 208, AVG Score: 88.750, Score: 96.0\n",
      "Episodes: 209, AVG Score: 89.650, Score: 80.0\n",
      "Episodes: 210, AVG Score: 87.250, Score: 73.0\n",
      "Episodes: 211, AVG Score: 89.900, Score: 141.0\n",
      "Episodes: 212, AVG Score: 89.650, Score: 82.0\n",
      "Episodes: 213, AVG Score: 92.500, Score: 108.0\n",
      "Episodes: 214, AVG Score: 93.600, Score: 81.0\n",
      "Episodes: 215, AVG Score: 93.650, Score: 101.0\n",
      "Episodes: 216, AVG Score: 93.400, Score: 70.0\n",
      "Episodes: 217, AVG Score: 90.200, Score: 68.0\n",
      "Episodes: 218, AVG Score: 92.350, Score: 100.0\n",
      "Episodes: 219, AVG Score: 94.050, Score: 99.0\n",
      "Episodes: 220, AVG Score: 93.400, Score: 47.0\n",
      "Episodes: 221, AVG Score: 92.850, Score: 109.0\n",
      "Episodes: 222, AVG Score: 95.300, Score: 136.0\n",
      "Episodes: 223, AVG Score: 94.300, Score: 125.0\n",
      "Episodes: 224, AVG Score: 93.950, Score: 72.0\n",
      "Episodes: 225, AVG Score: 95.450, Score: 80.0\n",
      "Episodes: 226, AVG Score: 92.850, Score: 47.0\n",
      "Episodes: 227, AVG Score: 93.100, Score: 147.0\n",
      "Episodes: 228, AVG Score: 93.550, Score: 105.0\n",
      "Episodes: 229, AVG Score: 92.600, Score: 61.0\n",
      "Episodes: 230, AVG Score: 92.700, Score: 75.0\n",
      "Episodes: 231, AVG Score: 89.200, Score: 71.0\n",
      "Episodes: 232, AVG Score: 88.700, Score: 72.0\n",
      "Episodes: 233, AVG Score: 86.600, Score: 66.0\n",
      "Episodes: 234, AVG Score: 86.750, Score: 84.0\n",
      "Episodes: 235, AVG Score: 84.700, Score: 60.0\n",
      "Episodes: 236, AVG Score: 86.800, Score: 112.0\n",
      "Episodes: 237, AVG Score: 86.500, Score: 62.0\n",
      "Episodes: 238, AVG Score: 84.650, Score: 63.0\n",
      "Episodes: 239, AVG Score: 84.950, Score: 105.0\n",
      "Episodes: 240, AVG Score: 85.950, Score: 67.0\n",
      "Episodes: 241, AVG Score: 85.450, Score: 99.0\n",
      "Episodes: 242, AVG Score: 85.500, Score: 137.0\n",
      "Episodes: 243, AVG Score: 82.450, Score: 64.0\n",
      "Episodes: 244, AVG Score: 82.600, Score: 75.0\n",
      "Episodes: 245, AVG Score: 81.800, Score: 64.0\n",
      "Episodes: 246, AVG Score: 83.150, Score: 74.0\n",
      "Episodes: 247, AVG Score: 78.700, Score: 58.0\n",
      "Episodes: 248, AVG Score: 79.800, Score: 127.0\n",
      "Episodes: 249, AVG Score: 79.900, Score: 63.0\n",
      "Episodes: 250, AVG Score: 80.450, Score: 86.0\n",
      "Episodes: 251, AVG Score: 81.800, Score: 98.0\n",
      "Episodes: 252, AVG Score: 83.300, Score: 102.0\n",
      "Episodes: 253, AVG Score: 83.850, Score: 77.0\n",
      "Episodes: 254, AVG Score: 84.600, Score: 99.0\n",
      "Episodes: 255, AVG Score: 92.000, Score: 208.0\n",
      "Episodes: 256, AVG Score: 90.350, Score: 79.0\n",
      "Episodes: 257, AVG Score: 92.100, Score: 97.0\n",
      "Episodes: 258, AVG Score: 93.600, Score: 93.0\n",
      "Episodes: 259, AVG Score: 91.850, Score: 70.0\n",
      "Episodes: 260, AVG Score: 91.400, Score: 58.0\n",
      "Episodes: 261, AVG Score: 91.900, Score: 109.0\n",
      "Episodes: 262, AVG Score: 87.950, Score: 58.0\n",
      "Episodes: 263, AVG Score: 86.900, Score: 43.0\n",
      "Episodes: 264, AVG Score: 87.050, Score: 78.0\n",
      "Episodes: 265, AVG Score: 86.600, Score: 55.0\n",
      "Episodes: 266, AVG Score: 85.550, Score: 53.0\n",
      "Episodes: 267, AVG Score: 87.350, Score: 94.0\n",
      "Episodes: 268, AVG Score: 82.850, Score: 37.0\n",
      "Episodes: 269, AVG Score: 81.850, Score: 43.0\n",
      "Episodes: 270, AVG Score: 79.500, Score: 39.0\n",
      "Episodes: 271, AVG Score: 77.700, Score: 62.0\n",
      "Episodes: 272, AVG Score: 77.300, Score: 94.0\n",
      "Episodes: 273, AVG Score: 76.150, Score: 54.0\n",
      "Episodes: 274, AVG Score: 73.950, Score: 55.0\n",
      "Episodes: 275, AVG Score: 67.950, Score: 88.0\n",
      "Episodes: 276, AVG Score: 69.700, Score: 114.0\n",
      "Episodes: 277, AVG Score: 69.500, Score: 93.0\n",
      "Episodes: 278, AVG Score: 69.300, Score: 89.0\n",
      "Episodes: 279, AVG Score: 68.750, Score: 59.0\n",
      "Episodes: 280, AVG Score: 69.750, Score: 78.0\n",
      "Episodes: 281, AVG Score: 67.450, Score: 63.0\n",
      "Episodes: 282, AVG Score: 67.600, Score: 61.0\n",
      "Episodes: 283, AVG Score: 69.400, Score: 79.0\n",
      "Episodes: 284, AVG Score: 68.250, Score: 55.0\n",
      "Episodes: 285, AVG Score: 71.700, Score: 124.0\n",
      "Episodes: 286, AVG Score: 71.500, Score: 49.0\n",
      "Episodes: 287, AVG Score: 69.400, Score: 52.0\n",
      "Episodes: 288, AVG Score: 70.650, Score: 62.0\n",
      "Episodes: 289, AVG Score: 73.700, Score: 104.0\n",
      "Episodes: 290, AVG Score: 73.950, Score: 44.0\n",
      "Episodes: 291, AVG Score: 73.650, Score: 56.0\n",
      "Episodes: 292, AVG Score: 71.550, Score: 52.0\n",
      "Episodes: 293, AVG Score: 73.100, Score: 85.0\n",
      "Episodes: 294, AVG Score: 72.800, Score: 49.0\n",
      "Episodes: 295, AVG Score: 72.200, Score: 76.0\n",
      "Episodes: 296, AVG Score: 69.750, Score: 65.0\n",
      "Episodes: 297, AVG Score: 67.400, Score: 46.0\n",
      "Episodes: 298, AVG Score: 65.050, Score: 42.0\n",
      "Episodes: 299, AVG Score: 64.400, Score: 46.0\n",
      "Episodes: 300, AVG Score: 62.550, Score: 41.0\n",
      "Episodes: 301, AVG Score: 61.100, Score: 34.0\n",
      "Episodes: 302, AVG Score: 60.350, Score: 46.0\n",
      "Episodes: 303, AVG Score: 58.300, Score: 38.0\n",
      "Episodes: 304, AVG Score: 57.300, Score: 35.0\n",
      "Episodes: 305, AVG Score: 56.200, Score: 102.0\n",
      "Episodes: 306, AVG Score: 55.750, Score: 40.0\n",
      "Episodes: 307, AVG Score: 55.750, Score: 52.0\n",
      "Episodes: 308, AVG Score: 56.100, Score: 69.0\n",
      "Episodes: 309, AVG Score: 53.450, Score: 51.0\n",
      "Episodes: 310, AVG Score: 53.600, Score: 47.0\n",
      "Episodes: 311, AVG Score: 54.450, Score: 73.0\n",
      "Episodes: 312, AVG Score: 54.750, Score: 58.0\n",
      "Episodes: 313, AVG Score: 53.200, Score: 54.0\n",
      "Episodes: 314, AVG Score: 52.550, Score: 36.0\n",
      "Episodes: 315, AVG Score: 51.650, Score: 58.0\n",
      "Episodes: 316, AVG Score: 50.750, Score: 47.0\n",
      "Episodes: 317, AVG Score: 50.500, Score: 41.0\n",
      "Episodes: 318, AVG Score: 50.500, Score: 42.0\n",
      "Episodes: 319, AVG Score: 49.700, Score: 30.0\n",
      "Episodes: 320, AVG Score: 49.250, Score: 32.0\n",
      "Episodes: 321, AVG Score: 50.000, Score: 49.0\n",
      "Episodes: 322, AVG Score: 49.900, Score: 44.0\n",
      "Episodes: 323, AVG Score: 50.100, Score: 42.0\n",
      "Episodes: 324, AVG Score: 49.950, Score: 32.0\n",
      "Episodes: 325, AVG Score: 47.450, Score: 52.0\n",
      "Episodes: 326, AVG Score: 47.450, Score: 40.0\n",
      "Episodes: 327, AVG Score: 47.500, Score: 53.0\n",
      "Episodes: 328, AVG Score: 47.550, Score: 70.0\n",
      "Episodes: 329, AVG Score: 46.850, Score: 37.0\n",
      "Episodes: 330, AVG Score: 47.150, Score: 53.0\n",
      "Episodes: 331, AVG Score: 45.650, Score: 43.0\n",
      "Episodes: 332, AVG Score: 45.000, Score: 45.0\n",
      "Episodes: 333, AVG Score: 45.050, Score: 55.0\n",
      "Episodes: 334, AVG Score: 45.900, Score: 53.0\n",
      "Episodes: 335, AVG Score: 47.100, Score: 82.0\n",
      "Episodes: 336, AVG Score: 49.200, Score: 89.0\n",
      "Episodes: 337, AVG Score: 49.350, Score: 44.0\n",
      "Episodes: 338, AVG Score: 49.150, Score: 38.0\n",
      "Episodes: 339, AVG Score: 49.600, Score: 39.0\n",
      "Episodes: 340, AVG Score: 50.700, Score: 54.0\n",
      "Episodes: 341, AVG Score: 50.750, Score: 50.0\n",
      "Episodes: 342, AVG Score: 51.200, Score: 53.0\n",
      "Episodes: 343, AVG Score: 53.400, Score: 86.0\n",
      "Episodes: 344, AVG Score: 54.500, Score: 54.0\n",
      "Episodes: 345, AVG Score: 54.250, Score: 47.0\n",
      "Episodes: 346, AVG Score: 57.650, Score: 108.0\n",
      "Episodes: 347, AVG Score: 57.050, Score: 41.0\n",
      "Episodes: 348, AVG Score: 55.700, Score: 43.0\n",
      "Episodes: 349, AVG Score: 56.100, Score: 45.0\n",
      "Episodes: 350, AVG Score: 55.400, Score: 39.0\n",
      "Episodes: 351, AVG Score: 57.700, Score: 89.0\n",
      "Episodes: 352, AVG Score: 57.450, Score: 40.0\n",
      "Episodes: 353, AVG Score: 57.950, Score: 65.0\n",
      "Episodes: 354, AVG Score: 57.450, Score: 43.0\n",
      "Episodes: 355, AVG Score: 55.100, Score: 35.0\n",
      "Episodes: 356, AVG Score: 53.700, Score: 61.0\n",
      "Episodes: 357, AVG Score: 53.150, Score: 33.0\n",
      "Episodes: 358, AVG Score: 53.950, Score: 54.0\n",
      "Episodes: 359, AVG Score: 55.100, Score: 62.0\n",
      "Episodes: 360, AVG Score: 56.300, Score: 78.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 361, AVG Score: 56.250, Score: 49.0\n",
      "Episodes: 362, AVG Score: 55.800, Score: 44.0\n",
      "Episodes: 363, AVG Score: 53.550, Score: 41.0\n",
      "Episodes: 364, AVG Score: 55.600, Score: 95.0\n",
      "Episodes: 365, AVG Score: 55.700, Score: 49.0\n",
      "Episodes: 366, AVG Score: 53.600, Score: 66.0\n",
      "Episodes: 367, AVG Score: 54.800, Score: 65.0\n",
      "Episodes: 368, AVG Score: 56.650, Score: 80.0\n",
      "Episodes: 369, AVG Score: 58.150, Score: 75.0\n",
      "Episodes: 370, AVG Score: 58.850, Score: 53.0\n",
      "Episodes: 371, AVG Score: 60.800, Score: 128.0\n",
      "Episodes: 372, AVG Score: 62.900, Score: 82.0\n",
      "Episodes: 373, AVG Score: 70.200, Score: 211.0\n",
      "Episodes: 374, AVG Score: 73.950, Score: 118.0\n",
      "Episodes: 375, AVG Score: 76.500, Score: 86.0\n",
      "Episodes: 376, AVG Score: 75.750, Score: 46.0\n",
      "Episodes: 377, AVG Score: 76.600, Score: 50.0\n",
      "Episodes: 378, AVG Score: 76.900, Score: 60.0\n",
      "Episodes: 379, AVG Score: 76.300, Score: 50.0\n",
      "Episodes: 380, AVG Score: 74.850, Score: 49.0\n",
      "Episodes: 381, AVG Score: 75.600, Score: 64.0\n",
      "Episodes: 382, AVG Score: 76.550, Score: 63.0\n",
      "Episodes: 383, AVG Score: 77.250, Score: 55.0\n",
      "Episodes: 384, AVG Score: 76.000, Score: 70.0\n",
      "Episodes: 385, AVG Score: 76.300, Score: 55.0\n",
      "Episodes: 386, AVG Score: 78.000, Score: 100.0\n",
      "Episodes: 387, AVG Score: 79.300, Score: 91.0\n",
      "Episodes: 388, AVG Score: 82.950, Score: 153.0\n",
      "Episodes: 389, AVG Score: 82.100, Score: 58.0\n",
      "Episodes: 390, AVG Score: 83.150, Score: 74.0\n",
      "Episodes: 391, AVG Score: 80.800, Score: 81.0\n",
      "Episodes: 392, AVG Score: 80.950, Score: 85.0\n",
      "Episodes: 393, AVG Score: 73.450, Score: 61.0\n",
      "Episodes: 394, AVG Score: 73.750, Score: 124.0\n",
      "Episodes: 395, AVG Score: 73.800, Score: 87.0\n",
      "Episodes: 396, AVG Score: 75.700, Score: 84.0\n",
      "Episodes: 397, AVG Score: 79.800, Score: 132.0\n",
      "Episodes: 398, AVG Score: 84.500, Score: 154.0\n",
      "Episodes: 399, AVG Score: 87.050, Score: 101.0\n",
      "Episodes: 400, AVG Score: 89.000, Score: 88.0\n",
      "Episodes: 401, AVG Score: 90.800, Score: 100.0\n",
      "Episodes: 402, AVG Score: 91.250, Score: 72.0\n",
      "Episodes: 403, AVG Score: 91.500, Score: 60.0\n",
      "Episodes: 404, AVG Score: 92.250, Score: 85.0\n",
      "Episodes: 405, AVG Score: 92.600, Score: 62.0\n",
      "Episodes: 406, AVG Score: 91.200, Score: 72.0\n",
      "Episodes: 407, AVG Score: 92.000, Score: 107.0\n",
      "Episodes: 408, AVG Score: 88.150, Score: 76.0\n",
      "Episodes: 409, AVG Score: 88.350, Score: 62.0\n",
      "Episodes: 410, AVG Score: 87.200, Score: 51.0\n",
      "Episodes: 411, AVG Score: 85.050, Score: 38.0\n",
      "Episodes: 412, AVG Score: 86.850, Score: 121.0\n",
      "Episodes: 413, AVG Score: 87.350, Score: 71.0\n",
      "Episodes: 414, AVG Score: 85.350, Score: 84.0\n",
      "Episodes: 415, AVG Score: 83.600, Score: 52.0\n",
      "Episodes: 416, AVG Score: 83.800, Score: 88.0\n",
      "Episodes: 417, AVG Score: 81.250, Score: 81.0\n",
      "Episodes: 418, AVG Score: 78.700, Score: 103.0\n",
      "Episodes: 419, AVG Score: 76.300, Score: 53.0\n",
      "Episodes: 420, AVG Score: 74.350, Score: 49.0\n",
      "Episodes: 421, AVG Score: 73.800, Score: 89.0\n",
      "Episodes: 422, AVG Score: 73.350, Score: 63.0\n",
      "Episodes: 423, AVG Score: 74.000, Score: 73.0\n",
      "Episodes: 424, AVG Score: 71.850, Score: 42.0\n",
      "Episodes: 425, AVG Score: 71.300, Score: 51.0\n",
      "Episodes: 426, AVG Score: 70.800, Score: 62.0\n",
      "Episodes: 427, AVG Score: 68.150, Score: 54.0\n",
      "Episodes: 428, AVG Score: 66.800, Score: 49.0\n",
      "Episodes: 429, AVG Score: 66.750, Score: 61.0\n",
      "Episodes: 430, AVG Score: 65.900, Score: 34.0\n",
      "Episodes: 431, AVG Score: 67.550, Score: 71.0\n",
      "Episodes: 432, AVG Score: 64.450, Score: 59.0\n",
      "Episodes: 433, AVG Score: 62.650, Score: 35.0\n",
      "Episodes: 434, AVG Score: 60.600, Score: 43.0\n",
      "Episodes: 435, AVG Score: 60.350, Score: 47.0\n",
      "Episodes: 436, AVG Score: 59.050, Score: 62.0\n",
      "Episodes: 437, AVG Score: 57.150, Score: 43.0\n",
      "Episodes: 438, AVG Score: 56.500, Score: 90.0\n",
      "Episodes: 439, AVG Score: 60.900, Score: 141.0\n",
      "Episodes: 440, AVG Score: 61.200, Score: 55.0\n",
      "Episodes: 441, AVG Score: 58.650, Score: 38.0\n",
      "Episodes: 442, AVG Score: 57.700, Score: 44.0\n",
      "Episodes: 443, AVG Score: 56.150, Score: 42.0\n",
      "Episodes: 444, AVG Score: 56.100, Score: 41.0\n",
      "Episodes: 445, AVG Score: 56.250, Score: 54.0\n",
      "Episodes: 446, AVG Score: 55.450, Score: 46.0\n",
      "Episodes: 447, AVG Score: 54.650, Score: 38.0\n",
      "Episodes: 448, AVG Score: 55.150, Score: 59.0\n",
      "Episodes: 449, AVG Score: 55.350, Score: 65.0\n",
      "Episodes: 450, AVG Score: 60.100, Score: 129.0\n",
      "Episodes: 451, AVG Score: 58.550, Score: 40.0\n",
      "Episodes: 452, AVG Score: 58.500, Score: 58.0\n",
      "Episodes: 453, AVG Score: 58.500, Score: 35.0\n",
      "Episodes: 454, AVG Score: 59.100, Score: 55.0\n",
      "Episodes: 455, AVG Score: 58.850, Score: 42.0\n",
      "Episodes: 456, AVG Score: 60.950, Score: 104.0\n",
      "Episodes: 457, AVG Score: 63.100, Score: 86.0\n",
      "Episodes: 458, AVG Score: 60.950, Score: 47.0\n",
      "Episodes: 459, AVG Score: 56.750, Score: 57.0\n",
      "Episodes: 460, AVG Score: 57.450, Score: 69.0\n",
      "Episodes: 461, AVG Score: 58.850, Score: 66.0\n",
      "Episodes: 462, AVG Score: 60.200, Score: 71.0\n",
      "Episodes: 463, AVG Score: 60.500, Score: 48.0\n",
      "Episodes: 464, AVG Score: 62.100, Score: 73.0\n",
      "Episodes: 465, AVG Score: 66.050, Score: 133.0\n",
      "Episodes: 466, AVG Score: 69.400, Score: 113.0\n",
      "Episodes: 467, AVG Score: 72.050, Score: 91.0\n",
      "Episodes: 468, AVG Score: 73.450, Score: 87.0\n",
      "Episodes: 469, AVG Score: 74.800, Score: 92.0\n",
      "Episodes: 470, AVG Score: 77.600, Score: 185.0\n",
      "Episodes: 471, AVG Score: 79.900, Score: 86.0\n",
      "Episodes: 472, AVG Score: 80.550, Score: 71.0\n",
      "Episodes: 473, AVG Score: 81.500, Score: 54.0\n",
      "Episodes: 474, AVG Score: 82.100, Score: 67.0\n",
      "Episodes: 475, AVG Score: 85.400, Score: 108.0\n",
      "Episodes: 476, AVG Score: 89.100, Score: 178.0\n",
      "Episodes: 477, AVG Score: 89.100, Score: 86.0\n",
      "Episodes: 478, AVG Score: 96.700, Score: 199.0\n",
      "Episodes: 479, AVG Score: 97.100, Score: 65.0\n",
      "Episodes: 480, AVG Score: 97.050, Score: 68.0\n",
      "Episodes: 481, AVG Score: 99.150, Score: 108.0\n",
      "Episodes: 482, AVG Score: 97.950, Score: 47.0\n",
      "Episodes: 483, AVG Score: 98.750, Score: 64.0\n",
      "Episodes: 484, AVG Score: 98.850, Score: 75.0\n",
      "Episodes: 485, AVG Score: 95.200, Score: 60.0\n",
      "Episodes: 486, AVG Score: 93.650, Score: 82.0\n",
      "Episodes: 487, AVG Score: 91.450, Score: 47.0\n",
      "Episodes: 488, AVG Score: 89.950, Score: 57.0\n",
      "Episodes: 489, AVG Score: 87.600, Score: 45.0\n",
      "Episodes: 490, AVG Score: 82.650, Score: 86.0\n",
      "Episodes: 491, AVG Score: 82.000, Score: 73.0\n",
      "Episodes: 492, AVG Score: 82.000, Score: 71.0\n",
      "Episodes: 493, AVG Score: 83.400, Score: 82.0\n",
      "Episodes: 494, AVG Score: 84.200, Score: 83.0\n",
      "Episodes: 495, AVG Score: 82.450, Score: 73.0\n",
      "Episodes: 496, AVG Score: 76.050, Score: 50.0\n",
      "Episodes: 497, AVG Score: 74.650, Score: 58.0\n",
      "Episodes: 498, AVG Score: 68.300, Score: 72.0\n",
      "Episodes: 499, AVG Score: 71.450, Score: 128.0\n",
      "Episodes: 500, AVG Score: 74.550, Score: 130.0\n",
      "Episodes: 501, AVG Score: 74.800, Score: 113.0\n",
      "Episodes: 502, AVG Score: 75.150, Score: 54.0\n",
      "Episodes: 503, AVG Score: 74.750, Score: 56.0\n",
      "Episodes: 504, AVG Score: 74.200, Score: 64.0\n",
      "Episodes: 505, AVG Score: 74.500, Score: 66.0\n",
      "Episodes: 506, AVG Score: 86.150, Score: 315.0\n",
      "Episodes: 507, AVG Score: 88.550, Score: 95.0\n",
      "Episodes: 508, AVG Score: 100.100, Score: 288.0\n",
      "Episodes: 509, AVG Score: 100.500, Score: 53.0\n",
      "Episodes: 510, AVG Score: 98.500, Score: 46.0\n",
      "Episodes: 511, AVG Score: 102.950, Score: 162.0\n",
      "Episodes: 512, AVG Score: 102.000, Score: 52.0\n",
      "Episodes: 513, AVG Score: 100.850, Score: 59.0\n",
      "Episodes: 514, AVG Score: 101.850, Score: 103.0\n",
      "Episodes: 515, AVG Score: 101.900, Score: 74.0\n",
      "Episodes: 516, AVG Score: 104.250, Score: 97.0\n",
      "Episodes: 517, AVG Score: 104.050, Score: 54.0\n",
      "Episodes: 518, AVG Score: 103.400, Score: 59.0\n",
      "Episodes: 519, AVG Score: 99.250, Score: 45.0\n",
      "Episodes: 520, AVG Score: 95.300, Score: 51.0\n",
      "Episodes: 521, AVG Score: 92.550, Score: 58.0\n",
      "Episodes: 522, AVG Score: 92.750, Score: 58.0\n",
      "Episodes: 523, AVG Score: 92.500, Score: 51.0\n",
      "Episodes: 524, AVG Score: 91.550, Score: 45.0\n",
      "Episodes: 525, AVG Score: 92.300, Score: 81.0\n",
      "Episodes: 526, AVG Score: 79.450, Score: 58.0\n",
      "Episodes: 527, AVG Score: 76.850, Score: 43.0\n",
      "Episodes: 528, AVG Score: 64.800, Score: 47.0\n",
      "Episodes: 529, AVG Score: 64.400, Score: 45.0\n",
      "Episodes: 530, AVG Score: 66.400, Score: 86.0\n",
      "Episodes: 531, AVG Score: 63.100, Score: 96.0\n",
      "Episodes: 532, AVG Score: 65.500, Score: 100.0\n",
      "Episodes: 533, AVG Score: 66.350, Score: 76.0\n",
      "Episodes: 534, AVG Score: 64.450, Score: 65.0\n",
      "Episodes: 535, AVG Score: 63.100, Score: 47.0\n",
      "Episodes: 536, AVG Score: 61.050, Score: 56.0\n",
      "Episodes: 537, AVG Score: 60.350, Score: 40.0\n",
      "Episodes: 538, AVG Score: 59.500, Score: 42.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 539, AVG Score: 63.150, Score: 118.0\n",
      "Episodes: 540, AVG Score: 62.600, Score: 40.0\n",
      "Episodes: 541, AVG Score: 62.700, Score: 60.0\n",
      "Episodes: 542, AVG Score: 62.000, Score: 44.0\n",
      "Episodes: 543, AVG Score: 66.250, Score: 136.0\n",
      "Episodes: 544, AVG Score: 66.350, Score: 47.0\n",
      "Episodes: 545, AVG Score: 65.100, Score: 56.0\n",
      "Episodes: 546, AVG Score: 67.050, Score: 97.0\n",
      "Episodes: 547, AVG Score: 67.850, Score: 59.0\n",
      "Episodes: 548, AVG Score: 67.600, Score: 42.0\n",
      "Episodes: 549, AVG Score: 68.000, Score: 53.0\n",
      "Episodes: 550, AVG Score: 66.800, Score: 62.0\n",
      "Episodes: 551, AVG Score: 64.900, Score: 58.0\n",
      "Episodes: 552, AVG Score: 65.350, Score: 109.0\n",
      "Episodes: 553, AVG Score: 65.850, Score: 86.0\n",
      "Episodes: 554, AVG Score: 73.050, Score: 209.0\n",
      "Episodes: 555, AVG Score: 74.000, Score: 66.0\n",
      "Episodes: 556, AVG Score: 73.950, Score: 55.0\n",
      "Episodes: 557, AVG Score: 76.150, Score: 84.0\n",
      "Episodes: 558, AVG Score: 76.950, Score: 58.0\n",
      "Episodes: 559, AVG Score: 74.400, Score: 67.0\n",
      "Episodes: 560, AVG Score: 75.000, Score: 52.0\n",
      "Episodes: 561, AVG Score: 76.000, Score: 80.0\n",
      "Episodes: 562, AVG Score: 76.700, Score: 58.0\n",
      "Episodes: 563, AVG Score: 79.300, Score: 188.0\n",
      "Episodes: 564, AVG Score: 81.450, Score: 90.0\n",
      "Episodes: 565, AVG Score: 82.000, Score: 67.0\n",
      "Episodes: 566, AVG Score: 80.450, Score: 66.0\n",
      "Episodes: 567, AVG Score: 80.700, Score: 64.0\n",
      "Episodes: 568, AVG Score: 81.950, Score: 67.0\n",
      "Episodes: 569, AVG Score: 85.750, Score: 129.0\n",
      "Episodes: 570, AVG Score: 87.950, Score: 106.0\n",
      "Episodes: 571, AVG Score: 90.000, Score: 99.0\n",
      "Episodes: 572, AVG Score: 93.300, Score: 175.0\n",
      "Episodes: 573, AVG Score: 99.800, Score: 216.0\n",
      "Episodes: 574, AVG Score: 94.650, Score: 106.0\n",
      "Episodes: 575, AVG Score: 110.000, Score: 373.0\n",
      "Episodes: 576, AVG Score: 113.450, Score: 124.0\n",
      "Episodes: 577, AVG Score: 114.450, Score: 104.0\n",
      "Episodes: 578, AVG Score: 120.050, Score: 170.0\n",
      "Episodes: 579, AVG Score: 120.600, Score: 78.0\n",
      "Episodes: 580, AVG Score: 122.650, Score: 93.0\n",
      "Episodes: 581, AVG Score: 122.300, Score: 73.0\n",
      "Episodes: 582, AVG Score: 123.500, Score: 82.0\n",
      "Episodes: 583, AVG Score: 117.850, Score: 75.0\n",
      "Episodes: 584, AVG Score: 118.450, Score: 102.0\n",
      "Episodes: 585, AVG Score: 118.000, Score: 58.0\n",
      "Episodes: 586, AVG Score: 120.600, Score: 118.0\n",
      "Episodes: 587, AVG Score: 120.450, Score: 61.0\n",
      "Episodes: 588, AVG Score: 120.050, Score: 59.0\n",
      "Episodes: 589, AVG Score: 119.400, Score: 116.0\n",
      "Episodes: 590, AVG Score: 118.300, Score: 84.0\n",
      "Episodes: 591, AVG Score: 117.050, Score: 74.0\n",
      "Episodes: 592, AVG Score: 110.850, Score: 51.0\n",
      "Episodes: 593, AVG Score: 103.300, Score: 65.0\n",
      "Episodes: 594, AVG Score: 100.550, Score: 51.0\n",
      "Episodes: 595, AVG Score: 84.300, Score: 48.0\n",
      "Episodes: 596, AVG Score: 80.850, Score: 55.0\n",
      "Episodes: 597, AVG Score: 80.500, Score: 97.0\n",
      "Episodes: 598, AVG Score: 75.800, Score: 76.0\n",
      "Episodes: 599, AVG Score: 74.300, Score: 48.0\n",
      "Episodes: 600, AVG Score: 74.200, Score: 91.0\n",
      "Episodes: 601, AVG Score: 73.600, Score: 61.0\n",
      "Episodes: 602, AVG Score: 72.300, Score: 56.0\n",
      "Episodes: 603, AVG Score: 71.700, Score: 63.0\n",
      "Episodes: 604, AVG Score: 69.900, Score: 66.0\n",
      "Episodes: 605, AVG Score: 74.550, Score: 151.0\n",
      "Episodes: 606, AVG Score: 72.000, Score: 67.0\n",
      "Episodes: 607, AVG Score: 73.200, Score: 85.0\n",
      "Episodes: 608, AVG Score: 74.800, Score: 91.0\n",
      "Episodes: 609, AVG Score: 72.200, Score: 64.0\n",
      "Episodes: 610, AVG Score: 71.450, Score: 69.0\n",
      "Episodes: 611, AVG Score: 70.150, Score: 48.0\n",
      "Episodes: 612, AVG Score: 76.650, Score: 181.0\n",
      "Episodes: 613, AVG Score: 77.100, Score: 74.0\n",
      "Episodes: 614, AVG Score: 77.950, Score: 68.0\n",
      "Episodes: 615, AVG Score: 80.000, Score: 89.0\n",
      "Episodes: 616, AVG Score: 82.650, Score: 108.0\n",
      "Episodes: 617, AVG Score: 83.750, Score: 119.0\n",
      "Episodes: 618, AVG Score: 84.400, Score: 89.0\n",
      "Episodes: 619, AVG Score: 84.800, Score: 56.0\n",
      "Episodes: 620, AVG Score: 82.850, Score: 52.0\n",
      "Episodes: 621, AVG Score: 83.350, Score: 71.0\n",
      "Episodes: 622, AVG Score: 83.450, Score: 58.0\n",
      "Episodes: 623, AVG Score: 83.450, Score: 63.0\n",
      "Episodes: 624, AVG Score: 83.000, Score: 57.0\n",
      "Episodes: 625, AVG Score: 79.150, Score: 74.0\n",
      "Episodes: 626, AVG Score: 81.550, Score: 115.0\n",
      "Episodes: 627, AVG Score: 86.050, Score: 175.0\n",
      "Episodes: 628, AVG Score: 88.300, Score: 136.0\n",
      "Episodes: 629, AVG Score: 89.600, Score: 90.0\n",
      "Episodes: 630, AVG Score: 94.700, Score: 171.0\n",
      "Episodes: 631, AVG Score: 95.950, Score: 73.0\n",
      "Episodes: 632, AVG Score: 90.550, Score: 73.0\n",
      "Episodes: 633, AVG Score: 94.450, Score: 152.0\n",
      "Episodes: 634, AVG Score: 96.950, Score: 118.0\n",
      "Episodes: 635, AVG Score: 99.850, Score: 147.0\n",
      "Episodes: 636, AVG Score: 98.700, Score: 85.0\n",
      "Episodes: 637, AVG Score: 96.500, Score: 75.0\n",
      "Episodes: 638, AVG Score: 94.850, Score: 56.0\n",
      "Episodes: 639, AVG Score: 94.750, Score: 54.0\n",
      "Episodes: 640, AVG Score: 97.050, Score: 98.0\n",
      "Episodes: 641, AVG Score: 97.750, Score: 85.0\n",
      "Episodes: 642, AVG Score: 98.950, Score: 82.0\n",
      "Episodes: 643, AVG Score: 108.250, Score: 249.0\n",
      "Episodes: 644, AVG Score: 109.350, Score: 79.0\n",
      "Episodes: 645, AVG Score: 119.750, Score: 282.0\n",
      "Episodes: 646, AVG Score: 116.850, Score: 57.0\n",
      "Episodes: 647, AVG Score: 112.900, Score: 96.0\n",
      "Episodes: 648, AVG Score: 109.750, Score: 73.0\n",
      "Episodes: 649, AVG Score: 113.300, Score: 161.0\n",
      "Episodes: 650, AVG Score: 116.900, Score: 243.0\n",
      "Episodes: 651, AVG Score: 118.000, Score: 95.0\n",
      "Episodes: 652, AVG Score: 119.600, Score: 105.0\n",
      "Episodes: 653, AVG Score: 120.050, Score: 161.0\n",
      "Episodes: 654, AVG Score: 121.750, Score: 152.0\n",
      "Episodes: 655, AVG Score: 121.300, Score: 138.0\n",
      "Episodes: 656, AVG Score: 126.700, Score: 193.0\n",
      "Episodes: 657, AVG Score: 138.300, Score: 307.0\n",
      "Episodes: 658, AVG Score: 143.200, Score: 154.0\n",
      "Episodes: 659, AVG Score: 147.800, Score: 146.0\n",
      "Episodes: 660, AVG Score: 148.950, Score: 121.0\n",
      "Episodes: 661, AVG Score: 155.150, Score: 209.0\n",
      "Episodes: 662, AVG Score: 163.000, Score: 239.0\n",
      "Episodes: 663, AVG Score: 159.850, Score: 186.0\n",
      "Episodes: 664, AVG Score: 163.900, Score: 160.0\n",
      "Episodes: 665, AVG Score: 174.750, Score: 499.0\n",
      "Episodes: 666, AVG Score: 196.850, Score: 499.0\n",
      "Episodes: 667, AVG Score: 217.000, Score: 499.0\n",
      "Episodes: 668, AVG Score: 238.300, Score: 499.0\n",
      "Episodes: 669, AVG Score: 255.200, Score: 499.0\n",
      "Episodes: 670, AVG Score: 268.000, Score: 499.0\n",
      "Episodes: 671, AVG Score: 288.200, Score: 499.0\n",
      "Episodes: 672, AVG Score: 307.900, Score: 499.0\n",
      "Episodes: 673, AVG Score: 324.800, Score: 499.0\n",
      "Episodes: 674, AVG Score: 342.150, Score: 499.0\n",
      "Episodes: 675, AVG Score: 360.200, Score: 499.0\n",
      "Episodes: 676, AVG Score: 375.500, Score: 499.0\n",
      "Episodes: 677, AVG Score: 385.100, Score: 499.0\n",
      "Episodes: 678, AVG Score: 400.200, Score: 456.0\n",
      "Episodes: 679, AVG Score: 408.150, Score: 305.0\n",
      "Episodes: 680, AVG Score: 415.800, Score: 274.0\n",
      "Episodes: 681, AVG Score: 414.100, Score: 175.0\n",
      "Episodes: 682, AVG Score: 411.450, Score: 186.0\n",
      "Episodes: 683, AVG Score: 410.300, Score: 163.0\n",
      "Episodes: 684, AVG Score: 411.050, Score: 175.0\n",
      "Episodes: 685, AVG Score: 394.900, Score: 176.0\n",
      "Episodes: 686, AVG Score: 380.150, Score: 204.0\n",
      "Episodes: 687, AVG Score: 367.550, Score: 247.0\n",
      "Episodes: 688, AVG Score: 358.500, Score: 318.0\n",
      "Episodes: 689, AVG Score: 347.350, Score: 276.0\n",
      "Episodes: 690, AVG Score: 334.300, Score: 238.0\n",
      "Episodes: 691, AVG Score: 322.700, Score: 267.0\n",
      "Episodes: 692, AVG Score: 314.400, Score: 333.0\n",
      "Episodes: 693, AVG Score: 302.900, Score: 269.0\n",
      "Episodes: 694, AVG Score: 291.850, Score: 278.0\n",
      "Episodes: 695, AVG Score: 280.900, Score: 280.0\n",
      "Episodes: 696, AVG Score: 269.250, Score: 266.0\n",
      "Episodes: 697, AVG Score: 255.900, Score: 232.0\n",
      "Episodes: 698, AVG Score: 244.350, Score: 225.0\n",
      "Episodes: 699, AVG Score: 241.500, Score: 248.0\n",
      "Episodes: 700, AVG Score: 240.400, Score: 252.0\n",
      "Episodes: 701, AVG Score: 245.800, Score: 283.0\n",
      "Episodes: 702, AVG Score: 254.200, Score: 354.0\n",
      "Episodes: 703, AVG Score: 268.100, Score: 441.0\n",
      "Episodes: 704, AVG Score: 284.300, Score: 499.0\n",
      "Episodes: 705, AVG Score: 297.750, Score: 445.0\n",
      "Episodes: 706, AVG Score: 303.500, Score: 319.0\n",
      "Episodes: 707, AVG Score: 302.700, Score: 231.0\n",
      "Episodes: 708, AVG Score: 298.650, Score: 237.0\n",
      "Episodes: 709, AVG Score: 296.600, Score: 235.0\n",
      "Episodes: 710, AVG Score: 304.300, Score: 392.0\n",
      "Episodes: 711, AVG Score: 313.000, Score: 441.0\n",
      "Episodes: 712, AVG Score: 318.100, Score: 435.0\n",
      "Episodes: 713, AVG Score: 328.850, Score: 484.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 714, AVG Score: 339.900, Score: 499.0\n",
      "Episodes: 715, AVG Score: 350.850, Score: 499.0\n",
      "Episodes: 716, AVG Score: 362.500, Score: 499.0\n",
      "Episodes: 717, AVG Score: 375.850, Score: 499.0\n",
      "Episodes: 718, AVG Score: 379.250, Score: 293.0\n",
      "Episodes: 719, AVG Score: 377.750, Score: 218.0\n",
      "Episodes: 720, AVG Score: 375.450, Score: 206.0\n",
      "Episodes: 721, AVG Score: 368.550, Score: 145.0\n",
      "Episodes: 722, AVG Score: 359.000, Score: 163.0\n",
      "Episodes: 723, AVG Score: 344.000, Score: 141.0\n",
      "Episodes: 724, AVG Score: 326.100, Score: 141.0\n",
      "Episodes: 725, AVG Score: 311.100, Score: 145.0\n",
      "Episodes: 726, AVG Score: 302.150, Score: 140.0\n",
      "Episodes: 727, AVG Score: 298.250, Score: 153.0\n",
      "Episodes: 728, AVG Score: 294.150, Score: 155.0\n",
      "Episodes: 729, AVG Score: 290.750, Score: 167.0\n",
      "Episodes: 730, AVG Score: 281.350, Score: 204.0\n",
      "Episodes: 731, AVG Score: 267.250, Score: 159.0\n",
      "Episodes: 732, AVG Score: 256.300, Score: 216.0\n",
      "Episodes: 733, AVG Score: 242.900, Score: 216.0\n",
      "Episodes: 734, AVG Score: 232.000, Score: 281.0\n",
      "Episodes: 735, AVG Score: 219.950, Score: 258.0\n",
      "Episodes: 736, AVG Score: 214.400, Score: 388.0\n",
      "Episodes: 737, AVG Score: 205.200, Score: 315.0\n",
      "Episodes: 738, AVG Score: 215.500, Score: 499.0\n",
      "Episodes: 739, AVG Score: 229.550, Score: 499.0\n",
      "Episodes: 740, AVG Score: 233.200, Score: 279.0\n",
      "Episodes: 741, AVG Score: 237.350, Score: 228.0\n",
      "Episodes: 742, AVG Score: 254.150, Score: 499.0\n",
      "Episodes: 743, AVG Score: 267.400, Score: 406.0\n",
      "Episodes: 744, AVG Score: 276.350, Score: 320.0\n",
      "Episodes: 745, AVG Score: 294.050, Score: 499.0\n",
      "Episodes: 746, AVG Score: 304.000, Score: 339.0\n",
      "Episodes: 747, AVG Score: 314.950, Score: 372.0\n",
      "Episodes: 748, AVG Score: 329.550, Score: 447.0\n",
      "Episodes: 749, AVG Score: 336.650, Score: 309.0\n",
      "Episodes: 750, AVG Score: 351.400, Score: 499.0\n",
      "Episodes: 751, AVG Score: 368.400, Score: 499.0\n",
      "Episodes: 752, AVG Score: 382.550, Score: 499.0\n",
      "Episodes: 753, AVG Score: 396.700, Score: 499.0\n",
      "Episodes: 754, AVG Score: 401.700, Score: 381.0\n",
      "Episodes: 755, AVG Score: 398.300, Score: 190.0\n",
      "Episodes: 756, AVG Score: 387.900, Score: 180.0\n",
      "Episodes: 757, AVG Score: 381.100, Score: 179.0\n",
      "Episodes: 758, AVG Score: 366.100, Score: 199.0\n",
      "Episodes: 759, AVG Score: 357.750, Score: 332.0\n",
      "Episodes: 760, AVG Score: 353.300, Score: 190.0\n",
      "Episodes: 761, AVG Score: 353.300, Score: 228.0\n",
      "Episodes: 762, AVG Score: 341.750, Score: 268.0\n",
      "Episodes: 763, AVG Score: 346.400, Score: 499.0\n",
      "Episodes: 764, AVG Score: 343.950, Score: 271.0\n",
      "Episodes: 765, AVG Score: 338.300, Score: 386.0\n",
      "Episodes: 766, AVG Score: 334.800, Score: 269.0\n",
      "Episodes: 767, AVG Score: 329.250, Score: 261.0\n",
      "Episodes: 768, AVG Score: 320.150, Score: 265.0\n",
      "Episodes: 769, AVG Score: 323.950, Score: 385.0\n",
      "Episodes: 770, AVG Score: 318.000, Score: 380.0\n",
      "Episodes: 771, AVG Score: 306.400, Score: 267.0\n",
      "Episodes: 772, AVG Score: 306.400, Score: 499.0\n",
      "Episodes: 773, AVG Score: 306.400, Score: 499.0\n",
      "Episodes: 774, AVG Score: 312.300, Score: 499.0\n",
      "Episodes: 775, AVG Score: 327.750, Score: 499.0\n",
      "Episodes: 776, AVG Score: 343.700, Score: 499.0\n",
      "Episodes: 777, AVG Score: 359.700, Score: 499.0\n",
      "Episodes: 778, AVG Score: 374.700, Score: 499.0\n",
      "Episodes: 779, AVG Score: 383.050, Score: 499.0\n",
      "Episodes: 780, AVG Score: 398.500, Score: 499.0\n",
      "Episodes: 781, AVG Score: 412.050, Score: 499.0\n",
      "Episodes: 782, AVG Score: 423.600, Score: 499.0\n",
      "Episodes: 783, AVG Score: 423.600, Score: 499.0\n",
      "Episodes: 784, AVG Score: 435.000, Score: 499.0\n",
      "Episodes: 785, AVG Score: 440.650, Score: 499.0\n",
      "Episodes: 786, AVG Score: 452.150, Score: 499.0\n",
      "Episodes: 787, AVG Score: 464.050, Score: 499.0\n",
      "Episodes: 788, AVG Score: 475.750, Score: 499.0\n",
      "Episodes: 789, AVG Score: 481.450, Score: 499.0\n",
      "Episodes: 790, AVG Score: 487.400, Score: 499.0\n",
      "Episodes: 791, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 792, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 793, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 794, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 795, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 796, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 797, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 798, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 799, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 800, AVG Score: 481.100, Score: 141.0\n",
      "Episodes: 801, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 802, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 803, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 804, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 805, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 806, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 807, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 808, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 809, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 810, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 811, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 812, AVG Score: 481.100, Score: 499.0\n",
      "Episodes: 813, AVG Score: 478.300, Score: 443.0\n",
      "Episodes: 814, AVG Score: 478.300, Score: 499.0\n",
      "Episodes: 815, AVG Score: 478.300, Score: 499.0\n",
      "Episodes: 816, AVG Score: 478.300, Score: 499.0\n",
      "Episodes: 817, AVG Score: 478.300, Score: 499.0\n",
      "Episodes: 818, AVG Score: 478.300, Score: 499.0\n",
      "Episodes: 819, AVG Score: 478.300, Score: 499.0\n",
      "Episodes: 820, AVG Score: 496.200, Score: 499.0\n",
      "Episodes: 821, AVG Score: 495.350, Score: 482.0\n",
      "Episodes: 822, AVG Score: 486.300, Score: 318.0\n",
      "Episodes: 823, AVG Score: 472.650, Score: 226.0\n",
      "Episodes: 824, AVG Score: 459.000, Score: 226.0\n",
      "Episodes: 825, AVG Score: 445.350, Score: 226.0\n",
      "Episodes: 826, AVG Score: 430.450, Score: 201.0\n",
      "Episodes: 827, AVG Score: 414.600, Score: 182.0\n",
      "Episodes: 828, AVG Score: 399.450, Score: 196.0\n",
      "Episodes: 829, AVG Score: 384.150, Score: 193.0\n",
      "Episodes: 830, AVG Score: 369.350, Score: 203.0\n",
      "Episodes: 831, AVG Score: 354.650, Score: 205.0\n",
      "Episodes: 832, AVG Score: 339.400, Score: 194.0\n",
      "Episodes: 833, AVG Score: 329.050, Score: 236.0\n",
      "Episodes: 834, AVG Score: 315.500, Score: 228.0\n",
      "Episodes: 835, AVG Score: 303.250, Score: 254.0\n",
      "Episodes: 836, AVG Score: 298.800, Score: 410.0\n",
      "Episodes: 837, AVG Score: 298.800, Score: 499.0\n",
      "Episodes: 838, AVG Score: 298.800, Score: 499.0\n",
      "Episodes: 839, AVG Score: 298.800, Score: 499.0\n",
      "Episodes: 840, AVG Score: 298.800, Score: 499.0\n",
      "Episodes: 841, AVG Score: 299.650, Score: 499.0\n",
      "Episodes: 842, AVG Score: 308.700, Score: 499.0\n",
      "Episodes: 843, AVG Score: 314.500, Score: 342.0\n",
      "Episodes: 844, AVG Score: 314.750, Score: 231.0\n",
      "Episodes: 845, AVG Score: 315.900, Score: 249.0\n",
      "Episodes: 846, AVG Score: 320.900, Score: 301.0\n",
      "Episodes: 847, AVG Score: 327.000, Score: 304.0\n",
      "Episodes: 848, AVG Score: 329.350, Score: 243.0\n",
      "Episodes: 849, AVG Score: 331.250, Score: 231.0\n",
      "Episodes: 850, AVG Score: 331.650, Score: 211.0\n",
      "Episodes: 851, AVG Score: 331.750, Score: 207.0\n",
      "Episodes: 852, AVG Score: 331.750, Score: 194.0\n",
      "Episodes: 853, AVG Score: 328.950, Score: 180.0\n",
      "Episodes: 854, AVG Score: 326.800, Score: 185.0\n",
      "Episodes: 855, AVG Score: 322.350, Score: 165.0\n",
      "Episodes: 856, AVG Score: 310.650, Score: 176.0\n",
      "Episodes: 857, AVG Score: 294.100, Score: 168.0\n",
      "Episodes: 858, AVG Score: 277.300, Score: 163.0\n",
      "Episodes: 859, AVG Score: 261.150, Score: 176.0\n",
      "Episodes: 860, AVG Score: 245.150, Score: 179.0\n",
      "Episodes: 861, AVG Score: 228.950, Score: 175.0\n",
      "Episodes: 862, AVG Score: 212.150, Score: 163.0\n",
      "Episodes: 863, AVG Score: 204.050, Score: 180.0\n",
      "Episodes: 864, AVG Score: 202.200, Score: 194.0\n",
      "Episodes: 865, AVG Score: 199.700, Score: 199.0\n",
      "Episodes: 866, AVG Score: 194.700, Score: 201.0\n",
      "Episodes: 867, AVG Score: 189.600, Score: 202.0\n",
      "Episodes: 868, AVG Score: 188.350, Score: 218.0\n",
      "Episodes: 869, AVG Score: 186.450, Score: 193.0\n",
      "Episodes: 870, AVG Score: 185.250, Score: 187.0\n",
      "Episodes: 871, AVG Score: 185.050, Score: 203.0\n",
      "Episodes: 872, AVG Score: 184.900, Score: 191.0\n",
      "Episodes: 873, AVG Score: 186.300, Score: 208.0\n",
      "Episodes: 874, AVG Score: 186.900, Score: 197.0\n",
      "Episodes: 875, AVG Score: 188.300, Score: 193.0\n",
      "Episodes: 876, AVG Score: 189.650, Score: 203.0\n",
      "Episodes: 877, AVG Score: 191.950, Score: 214.0\n",
      "Episodes: 878, AVG Score: 194.600, Score: 216.0\n",
      "Episodes: 879, AVG Score: 196.350, Score: 211.0\n",
      "Episodes: 880, AVG Score: 198.350, Score: 219.0\n",
      "Episodes: 881, AVG Score: 200.350, Score: 215.0\n",
      "Episodes: 882, AVG Score: 206.000, Score: 276.0\n",
      "Episodes: 883, AVG Score: 213.250, Score: 325.0\n",
      "Episodes: 884, AVG Score: 228.500, Score: 499.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 885, AVG Score: 236.550, Score: 360.0\n",
      "Episodes: 886, AVG Score: 238.800, Score: 246.0\n",
      "Episodes: 887, AVG Score: 242.700, Score: 280.0\n",
      "Episodes: 888, AVG Score: 243.300, Score: 230.0\n",
      "Episodes: 889, AVG Score: 247.550, Score: 278.0\n",
      "Episodes: 890, AVG Score: 253.000, Score: 296.0\n",
      "Episodes: 891, AVG Score: 267.800, Score: 499.0\n",
      "Episodes: 892, AVG Score: 269.300, Score: 221.0\n",
      "Episodes: 893, AVG Score: 271.450, Score: 251.0\n",
      "Episodes: 894, AVG Score: 275.800, Score: 284.0\n",
      "Episodes: 895, AVG Score: 276.750, Score: 212.0\n",
      "Episodes: 896, AVG Score: 276.700, Score: 202.0\n",
      "Episodes: 897, AVG Score: 275.950, Score: 199.0\n",
      "Episodes: 898, AVG Score: 276.150, Score: 220.0\n",
      "Episodes: 899, AVG Score: 276.450, Score: 217.0\n",
      "Episodes: 900, AVG Score: 277.750, Score: 245.0\n",
      "Episodes: 901, AVG Score: 283.350, Score: 327.0\n",
      "Episodes: 902, AVG Score: 285.500, Score: 319.0\n",
      "Episodes: 903, AVG Score: 284.500, Score: 305.0\n",
      "Episodes: 904, AVG Score: 281.000, Score: 429.0\n",
      "Episodes: 905, AVG Score: 287.950, Score: 499.0\n",
      "Episodes: 906, AVG Score: 300.600, Score: 499.0\n",
      "Episodes: 907, AVG Score: 311.550, Score: 499.0\n",
      "Episodes: 908, AVG Score: 325.000, Score: 499.0\n",
      "Episodes: 909, AVG Score: 336.050, Score: 499.0\n",
      "Episodes: 910, AVG Score: 346.200, Score: 499.0\n",
      "Episodes: 911, AVG Score: 346.200, Score: 499.0\n",
      "Episodes: 912, AVG Score: 360.100, Score: 499.0\n",
      "Episodes: 913, AVG Score: 372.500, Score: 499.0\n",
      "Episodes: 914, AVG Score: 383.250, Score: 499.0\n",
      "Episodes: 915, AVG Score: 397.600, Score: 499.0\n",
      "Episodes: 916, AVG Score: 412.450, Score: 499.0\n",
      "Episodes: 917, AVG Score: 427.450, Score: 499.0\n",
      "Episodes: 918, AVG Score: 441.400, Score: 499.0\n",
      "Episodes: 919, AVG Score: 455.500, Score: 499.0\n",
      "Episodes: 920, AVG Score: 468.200, Score: 499.0\n",
      "Episodes: 921, AVG Score: 476.800, Score: 499.0\n",
      "Episodes: 922, AVG Score: 485.800, Score: 499.0\n",
      "Episodes: 923, AVG Score: 495.500, Score: 499.0\n",
      "Episodes: 924, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 925, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 926, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 927, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 928, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 929, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 930, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 931, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 932, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 933, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 934, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 935, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 936, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 937, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 938, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 939, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 940, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 941, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 942, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 943, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 944, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 945, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 946, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 947, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 948, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 949, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 950, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 951, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 952, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 953, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 954, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 955, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 956, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 957, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 958, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 959, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 960, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 961, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 962, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 963, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 964, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 965, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 966, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 967, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 968, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 969, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 970, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 971, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 972, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 973, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 974, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 975, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 976, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 977, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 978, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 979, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 980, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 981, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 982, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 983, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 984, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 985, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 986, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 987, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 988, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 989, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 990, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 991, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 992, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 993, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 994, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 995, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 996, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 997, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 998, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 999, AVG Score: 499.000, Score: 499.0\n",
      "Episodes: 1000, AVG Score: 499.000, Score: 499.0\n"
     ]
    }
   ],
   "source": [
    "global_step = 0\n",
    "n_eps = 0\n",
    "queue_rewards = collections.deque(maxlen=20)\n",
    "reward_lst = []\n",
    "avg_reward_lst = []\n",
    "mva_lst = []\n",
    "mva = 0.\n",
    "\n",
    "while n_eps < args.max_eps:\n",
    "    n_eps += 1\n",
    "    next_obs = to_tensor(env.reset(), dtype=torch.float)\n",
    "    done = False\n",
    "    score = 0\n",
    "    while not done:\n",
    "        obs = next_obs\n",
    "        a_optim.zero_grad()\n",
    "        c_optim.zero_grad()\n",
    "        \n",
    "        # estimate action with policy network\n",
    "        policy = actor(obs)\n",
    "        action = select_action(policy.detach().numpy()[0])\n",
    "        \n",
    "        # interaction with environment\n",
    "        next_obs, reward, done, info = env.step(action)\n",
    "        \n",
    "        next_obs = to_tensor(next_obs, dtype=torch.float)\n",
    "        advantages = torch.zeros_like(policy)\n",
    "        ext_reward = 0. if args.sparse_mode else reward\n",
    "        \n",
    "        v = critic(obs)[0]\n",
    "        next_v = critic(next_obs)[0]\n",
    "        if done:\n",
    "            t_ext_reward = to_tensor([-100], dtype=torch.float) if score < 499 else to_tensor([ext_reward], dtype=torch.float)\n",
    "            advantages[0, action] = t_ext_reward - v\n",
    "            c_target = t_ext_reward\n",
    "        else:\n",
    "            t_ext_reward = to_tensor([ext_reward], dtype=torch.float)\n",
    "            advantages[0, action] = t_ext_reward + args.discounted_factor*next_v - v\n",
    "            c_target = t_ext_reward + args.discounted_factor*next_v\n",
    "        \n",
    "        actor_loss = pg_loss(policy, advantages.detach())\n",
    "        critic_loss = mse_loss(v, c_target.detach())\n",
    "        loss = actor_loss + critic_loss\n",
    "        loss.backward()\n",
    "        a_optim.step()\n",
    "        c_optim.step()\n",
    "        \n",
    "        if not done:\n",
    "            score += reward\n",
    "        \n",
    "        global_step += 1\n",
    "    \n",
    "    mva = 0.95*mva + 0.05*score\n",
    "    queue_rewards.append(score)\n",
    "    reward_lst.append(score)\n",
    "    avg_reward_lst.append(np.mean(queue_rewards))\n",
    "    mva_lst.append(mva)\n",
    "    print('Episodes: {}, AVG Score: {:.3f}, Score: {}'.format(n_eps, mva, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD8CAYAAABthzNFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXe8HFX5/z/P7t6actNDSAIppEEg\nlAgJhBo6SFFU/CmiXwQUVMAaQFEUFBXFgiJNRUS6SK+hhRYIpEEISUghvdd7c+/dcn5/zJzZM2fO\nzM5suXvv7vN+vfLanZmzM2d2c59nnnpICAGGYRiG0YmVewIMwzBM54QVBMMwDGOEFQTDMAxjhBUE\nwzAMY4QVBMMwDGOEFQTDMAxjhBUEwzAMY4QVBMMwDGOEFQTDMAxjJFHuCRRCv379xLBhw8o9DYZh\nmC7Fu+++u0kI0T/XuC6tIIYNG4ZZs2aVexoMwzBdCiJaEWYcu5gYhmEYI6wgGIZhGCOsIBiGYRgj\nrCAYhmEYIyVVEES0nIjmE9EcIppl7+tDRM8T0WL7tbe9n4joT0S0hIjmEdHBpZwbwzAME0xHWBDH\nCiEOFEJMtLenAZguhBgFYLq9DQCnABhl/7sIwC0dMDeGYRjGh3K4mM4EcJf9/i4AZyn7/yUs3gLQ\ni4gGlWF+DMMwDEpfByEAPEdEAsCtQojbAAwUQqy1j68DMNB+PxjASuWzq+x9a8EwTIeTyQj8843l\n2NbS7uwb0b87zjposHF8Mp3BI7NX45yDhyAtBP7+2jLEY4SvHTEc8Ri5xj7z/jos2bATDbUJHLRX\nL7y9bAtq4zF87YhhSKYFHp2zGuccMgQrNrfgv7NXAwFLIy/b3IK9+zRCXmLppmYM79cNRISzDxqM\n4f26OWNnLd+CVxdtRGNdAl89fBjqa+Khvoutze14c+lmHD9uoDM3IuuCD85aiZVbWkKdp5hMHTcQ\nE4b2Kuk1Sq0gpgghVhPRAADPE9FC9aAQQtjKIzREdBEsFxT22muv4s2UYRgXyzc34+dPLAAAEFky\nmgg4Y8KeiGkCHwDumLEMv37G+hMf3q8bfvW09X7KqH4Yu0dP19hv/Ptd4zWPHTsAD7+7Cje/tASN\ntQnM/mQr7nhtGch7OQBuvSHnqLJjdxI/O2M/Z/vG5z7CW0u3AAD2H9yEI/bp5/8FKHzznnfx1tIt\n+NwhQ/Dgu6vQWJvAaQcMQmsyjR88NM+5fkcyoGd911YQQojV9usGInoEwKEA1hPRICHEWtuFtMEe\nvhrAUOXjQ+x9+jlvA3AbAEycODGScmEYJjzpjPXndfP/OwinH7Anbn5xMW58bhHSQiAGrzTc0twG\nANjW0o4du2ud/al0+D/TdCaDTbus8+xoTSKVEehRn8D8n51kHP/U/LW45J730KMugfnXnoT/zV6N\ny++fg736NCKVzmBXW8o1PpUWaKiJY3cyjVQm/LxWb9vtet3RmrTOZ5/j6lPH4cKjRoQ+X1ehZDEI\nIupGRD3kewAnAngfwGMAzreHnQ/gUfv9YwC+YmczTQKwXXFFMQxTZuIxS1z4CXzpchECaG5P53WN\nAE9SKFTLpqE2jpZ2t4LICOG4u0QeF9M/IpVoR1sPHUUpLYiBAB6x/9MkAPxHCPEMEb0D4AEiugDA\nCgCft8c/BeBUAEsAtAD4WgnnxjBMDqQsJNtaqIlbr6lMBoC/714AaFae3KPIYf2hXghhsFWyyGPy\nYwlFQXSrS6BFU1RpgayCCD8t3+tm7AnrMZZKoWQKQgixFMAEw/7NAKYa9gsAl5ZqPgzDFIYUgr4W\nhP0qhKYgIojijKJN5FuK8HgeU8Y21MTR0uZWEKJQC0K7l7SobAXBldQMwxjJCmjrNRG3XUx+vnt7\nnIBw+f6jyGF9rECw+0Y/5rEgkkEupvDzIh87Riq0WIX6mFhBMAwTCil8LReTFylEhQB2J7NP7lGe\n01ULwsmcivD5uCcG4bYgMhkgbgvzCDFqB12pyK+CFQTDMFWFdKdI0ZfI5WJSZGQ6rbqKwktirwUh\ncriY3MdUBVGXiKEt6VZm+QapybGO3NtZF1PoU3UpKvS2GIYpNgknSJ0rBiFcY6I8qOs+/qhhAlVB\nmNxCGSGc+8jHgtBvRgap2YJgGKaq8MQg7DTXtJ+LSbUgVAVRSBYTgl1MulxWBbXlonKfMCNUJRLB\ngnA+oZ+Pg9QMw1QhWdlqCT/pYkr6ZjFlYxBuKyOKi8k7NsrDubQOrPl4r5wRoigxCHmvabYgGIZh\nsllMaT8Xky0jM8JtZRRWBwEE2RD6Ea8F4T1fPllMfjhZTGxBMAxTTThBasfFJC0IvyymLFHaWLiu\n6ZHaIpoFocUgTC6hbKFc4RpCfhVxtiAYhqlmpPvGz4KQklxAuGMQEa7hUQ850lxlhpMco8YCYjFD\nWqoQjhKJosPIcUs5PiYA2e+Cs5gYhqkqNFnoCF//GET2c6l8g9Sa1C4kiwkgjxLIZJBfmqvPfqkw\nolR7dyVYQTAME4qaHDEIiUD+dRDeLKZgF5N+yJXmSnI26vnzq6T2m5+TxcQKgmGYakQ+HTsWRK40\n1yLWQQD+bS7Ua8pPuYLUMLuY8opBaNeRpCu8WR8rCIZhjOgupppc7b5lmivyz2JSxwqRXaQoLLoF\n4U1zzY7x0XPhJwjOYmIYhgGgPK37SPzscS0GkWc3V+uzUQvl1PfkmasQwrEyolg2Tntvj0VivbKL\niWGYqkJPc9XdOTqqiHTFKfK0ILLLnOYnfAlmgZ7NYooehNA/ky2Uy2uKnR5WEAzDhCJbKW0WrMJ5\ntWIQMR+FEhS09loQwUI8OD7htSDSGeGsjJdPGYRUOPqCQexiYhimqtB7MakuJBPyadqqpBZO5bWp\nmjnXNbM7IkzYdD5tW62DyKdQzqNwuBcTwzBMbheTfPrPCIH2VAY1PoI4SCwbs5jCd/t2lFLvbrXW\n5wJabeRTKOfNirJeuRcTwzBVhZSF0o0T8xGSEqkg1m1vxfzV29FsL9bjtSACXEwZdVzuFeV0Bvdq\nwHVnjcft5x1it9rwzrGwOgh3YVymwmMQJVuTmmGYro0jyMn14hvclX2J1m1vdZ9HP2/QNQ1zCIwz\naK8A8OVJewOwhLa33bdQLIjoldR+QWp2MTEMU9WEdTHVJtxixSSk/fAsOYpoFoQKkTmLKVsoFx19\n6jIGwS4mhmGqiqyLyX51XExm0SrdLe0py5QY1FTvOo9z3sAgtbcXU5hmfX7HPN1cM0qQOo8lRx0X\nkzZftiAYhqlq1GZ8JuTTtCySO+eQIfYHwl/DuyZ1/vi12ogVEIPwttqwXtmCYBimqsimuZLr1S89\nVFoQcr0IuUSp33mN5zAcC7QS/E8F+LTayKdQTsZBPDEIx4IIfaouRYXeFsMwxSaXBSGFu3QxyfUj\nvGmu4WIQVi8mEarVhumMpGkIocUL8spi0vo3yXNyu2+GYaoMt789l2CVT9NZC8I8PjAGAXdQWqgT\niEiM3ApHKrBEQUFqd/sRvaFhpcEKgmEYI7og1wO1OlkXkztwGyWuIIRwj88VpM7RClw9lZOSKi2b\nvILU2nzBWUwMw1QxuuzLleaasi0IucCQqbbBD2+aqyigWZ+7F5M8t59lEwZdOWY4SM0wTDXiqaT2\n675nIzN6kvZjdsLnST3Yggje1glcbU6zIOS5ZGvu/Lq5uq+b0VxOlQYrCIZhQpGrkjqjxSDkAkNR\n6iCMWUxRJql9Tr2Wszyoz7zCYVZ2rCAYhqkqfLu5+oyXPv6knsUUodeGam2EWVHO1GrDOaYV9mUV\nBFzbYXB6L3ksHM5iYhiGUdaDMB/PWhDSxSTFS/g0V10hCAT3Ygqcr5Zp5HReLUazPu27qNBC6tIr\nCCKKE9FsInrC3h5ORDOJaAkR3U9Etfb+Ont7iX18WKnnxjCMP87Tsb2dDUEEu5jaHRdT9DRXz4JB\nOSyIIN2hrpFtncu2IHK0DAm6jPyI/A6yCwhVpoboCAviMgAfKtu/BnCTEGIfAFsBXGDvvwDAVnv/\nTfY4hmHKhEd8+qR6StLagYRfFlOOa6pyu6BWG44FIexXa7sY7b4l2TTX/ObY2SmpgiCiIQBOA3CH\nvU0AjgPwkD3kLgBn2e/PtLdhH59KlerYY5iuhNPuO3hJOV1x+MUgIqW55spiCtEKXGiv2ZYh0RGa\nxeDcc4VKqlJbEH8A8EMAskC9L4BtQoiUvb0KwGD7/WAAKwHAPr7dHu+CiC4iollENGvjxo2lnDvD\nVDW6MMzZ7lvTENkspvBprlF7MQWdU49BZF1M8lrRVYRHuXG77/wgotMBbBBCvFvM8wohbhNCTBRC\nTOzfv38xT80wTABhW21I/C2IgItoWUxAuF5M5mPuXlBS+eTjYvKrpK70JUdLuaLcEQDOIKJTAdQD\n6AngjwB6EVHCthKGAFhtj18NYCiAVUSUANAEYHMJ58cwTABSsDpprvZ+/zoI93ZN3OzKCW7Wp2Ux\n5QpSB+CxIOR18whSZ+cnjNuVqR5KaEEIIa4UQgwRQgwDcC6AF4UQXwLwEoBz7GHnA3jUfv+YvQ37\n+Isin1+QYZiSoAtcHd3F5BSkRWjG5A0Ch6uDMB/TLAXHZWYvRxrwWc+59OZ82nalWhDlqIP4EYDv\nEtESWDGGO+39dwLoa+//LoBpZZgbwzASRaACuYO7ehZT3Edozl65zf+SQstiEjnWpA5cUc4+hz1j\nteqZiPKLQegr1FV4O9dSupgchBAvA3jZfr8UwKGGMa0APtcR82EYJje6+NTTRnV0gesnuy++2z8s\nqZ7jqkfmAwD2H9wUPFEfPLULytO+3oYj97nMldQSTnNlGKYqcVaUs7dzVVJLClmYR2XTrraAufl/\nLqZZPGq8IEbkK+yD8I1BsIuJYZhqwtuLyZ0VpKO7mOSKo0FB6VznAIC121tDf15F77iquphA0eYl\n8cvIYguCYZiqJqYFZnV02Z6PBZErKK0T1KzPOadWB0Eg616KkALDrTYYhqlKnDRXezuXH97rYrLP\nE0VBFDFvkbJRate5iax7idbN1XxuXg+CYZhOw+xPtuLut1aU5dp6VpCOx8WUR0uLqG6fMCmwThaT\noyAsCyJakDqYSk1z7ZAsJoZhisPZf30DAHDepL1Lfi3f9SD8Kqn9FEQESaynuRaCX6EcQaa5Fn4N\nWftRofqBLQiGYcx40lwRLPD13flYENHxl8z69V0upqhBak0D6I0AK9WCYAXBMEwOtGZ9IXsxkS5F\nQ5Bv84SgZn16FlM+dRB+1+NWGwzDVCVCC8Dmkvd6qw1n5bYIGiKq0A4Vg7DPqQaUiSivBYMk+hoT\nFWpAsIJgGCYcudJWPZXU9msp01wD0eo21HnEyD8bKwq8JjXDMFWJU1gmX3Oso6C6mKSfXz1PqGtG\ntSDCHBPuN0RkWRBFiI4IVG6RHMAKgmGYkORq1pfJKGNh6KYaAgERcZ2GMM364JpHjBA5zVVHdVtV\naoAaYAXBMIwfSt2AhAKiu6plESPKWTdhvGQRU56k4Jbzclc9R0tz9dMB+voVlQYrCIZhjJgEO8Hf\nd6/WQVjVyvZ5IsYgohDUasPTzVVZAMlyC+WvjdS4RqXGHwBWEAzD5EAVf0G+e9WCIGQ1RCQxXEQT\nws/FRPYx1SWW81zattrfqXLVAysIhmF8MMnqoPoBl2VB0RvYWcVrEZv1Baa5ugv73K02ihmkrlwV\nwQqCYZhAVPlnCVYzqovJldkT0iootHjNeELAXAeBaGmuuhvJOWdGcAyCYZjqI+uSUSQg+ae5qoVy\nBIqc5ppP6mmQleL3ZC97MRWjkpotCIZhGJug2G5Gr4Ow34cVxNKCKFqzPm1eqouJKFpbD79K6gzH\nIBiGqUacQjnVgCB/i8BVKAelbiKsiyng3EGfyXXM3M21OE0EBae5MgxTjZgEe0zrYfT8gvVYtqkZ\ngDsrSDbEAyK4mOxFfNrTEdKLAq6hu7ikBywW895HvtcTQjg9pyoRXg+CYZjQ6MHdC/81CwCw/IbT\n3LEJ8j7Bhzn5ra8sLco8rdPpWUzC2R89SK3tcALfldvJFWALgmEYH8xP5f7BXY+LSQrokNcrtqD1\n1EEoFwrKxopCMp2p6CA1WxAMwwTijUF4RasQ7h5KpESpo8QgCpmb95i7F5RaKBeUjWU8l6a+BARm\nLd+C+95ZGWG2XQ+2IBiGMWJKc/WrVdCXG1W7uYYlamGd+7P++7IKynqN2YVyhZgQQgBzVm7L/wRd\nBFYQDMOExm+hHd2f7wpSh01zzceCCFAqfkHqbKFceA1hGtu3e23oz3dVWEEwDONDtvJYoj54PzZ3\njbPftFhQtj14+ErqqIRrtQHXqyzii5LEpN+fANCrgRUEwzB5sqsthXXbW8s9jbwxCdCYHaR+e9kW\nfOfe2c5+o4sp4DyScYN6Kp8pbrBXZp9m9CwmJ0gdXkOktaFCRGtj3lVhBcEwJeLTf34Nk341vdzT\nKBiXBQFL4O5qS7rGeF0w/q02Gmrizvu+3WqVTxQ2N+8xzYLQPhMlzVVfb9vaF/7zXRVWEAxTImQB\nWVfFr/jM1H9IF5Yx8rp4JJNG9DFfsMjZoroF4TTrAzmWUFhSGd3FJFxpvZUKKwiGYQJxZTHZglV3\nB+nC0r0mtfuYKmvVY3lZECGa9WlJTM7colRS6xaE1TOKFUTeEFE9Eb1NRHOJ6AMiutbeP5yIZhLR\nEiK6n4hq7f119vYS+/iwUs2NYRh/vvfAXAyb9qTS3C57zEpzFdC7S6ittK1x/oJbF6tPfHsKbvjM\n/sWPQcTcc3NcTIheKJfSTCSBaC6qrkopLYg2AMcJISYAOBDAyUQ0CcCvAdwkhNgHwFYAF9jjLwCw\n1d5/kz2OYRgDpXx6ffi9Vb7HYmT1S4p7XEzWfBK25lAP61PV5z5+cBPOPXSvkhXKpZ0gtbU/FrPi\nIy8u3ICbX1wc6jomZaAH5iuRkikIYbHL3qyx/wkAxwF4yN5/F4Cz7Pdn2tuwj0+lSl7slWEKoCO8\nG2r3U0lNgpBKC18XU9xWEDEiX+Gtzl1r31TAXL3EHReTHoPIKo8bn1sU6vy6BQEhnPM9+I3J0Sfc\nRShpDIKI4kQ0B8AGAM8D+BjANiFEyh6yCsBg+/1gACsBwD6+HUDfUs6PYboqHfHsalJCNfEY2tMZ\ng4vJPh7LihS9WV52rHn2+TwPBn1CxiDk3NQspnTEFCSPfkD2++nXvS7SuboSJVUQQoi0EOJAAEMA\nHApgbKHnJKKLiGgWEc3auHFjwXNkmK5IRwZIVbldG48haWjHLV1M8XjWxeTXzdXds0l5X5TZZnGy\nmDLuOgiAELWjuMeCQNbFVMHdvjsmi0kIsQ3ASwAmA+hFRLJJ4BAAq+33qwEMBQD7eBOAzYZz3SaE\nmCiEmNi/f/+Sz51hqhWTCqqJx9Ceynh88mknBmGJFFehnHYOfwsi+hzDxCB0CyIWMYMJgEehCJG9\nj0ru5lrKLKb+RNTLft8A4AQAH8JSFOfYw84H8Kj9/jF7G/bxF0U15JExTB507B9GVgDWxAnJtPAI\nebntBKntWgP1mETdch8qbrO+mGPB6JXUFKkPE+B1Sanda3nBoPwYBOAuIorDUkQPCCGeIKIFAO4j\nousAzAZwpz3+TgB3E9ESAFsAnFvCuTFMh7F9dxJNDTVFPWeHBKkNF6lNWDEIPwWRDVIHVCz7zD2/\nB/GAOoiYZkE4vZiip6iaMpbSjgUR7VxdiZIpCCHEPAAHGfYvhRWP0Pe3AvhcqebDMOXiy3fMxOPf\nnlLUc3ZkHyBVcNfEY9jVlvIITOmCqXFiEAQiMrpzfF1MBc5Nx9uLKfuZ6BaE1wpylCK7mBiGyZf5\nq7eXewoFoYq/WicGoSsI28UUj7k+EyMyClfnvU/AuhiQ5uLKxiCitdkAvJXi1z6+AMlUxnWdSiS0\ngiCiKUT0Nft9fyIaXrppMQwTRDFdTIvX78SwaU9i8fqdOa9RY2cx6UHbtFYoJzVELEbu1hpC4N0V\nW43zyGfBoDBprvI+VKUWtcjNNH7uqu32dSKdqksRSkEQ0U8B/AjAlfauGgD/LtWkGIbpOB6ftxYA\n8OT8tcbj6hNybSJmDFL/cbpVcFZnd2rNWhBuwawrB1cvphI160tniu9iArL3GK9gDRHWgjgbwBkA\nmgFACLEGQI9STYphmGCKGqT2OZkpzuGkuWoC84UPNwAA6hIyzdUSmnEi11i9K6pKfjGI3M36sspA\nVlJHczEJIcxBbWkssYsJ7XbKqQAAIupWuikxDJOLUgSp/Vw8rhhEgtCeznh88j3qrHwXqSBkYDoW\nI9dYGcSWqNmjhQha07fhqaRWLIgoLia/sfL7YgsCeICIboVV5HYhgBcA3F66aTEMU25MT9nxmBV0\n9gs81yXiru2YZkEkYm6RY6pQjkJgDMK+VLYXkzKnCCaEn9Uj9VkF64dwaa5CiBuJ6AQAOwCMAXCN\nEOL5ks6MYRhfSlEH4WeVqA/2VgaQ8Fx/V5vVXq2uxq0A4lqQWn/aVpfyLH4MQq+kloVyEVeT8/my\ns602KldD5FQQdqHbC0KIY2E13GMYpswUVT/4CLhsYVn2uPX07e92kS4mOE/r7hRRT4Gdcp5it9rw\nrYNAtFYbfhaE7ElVyQoip4tJCJEGkCGipg6YD8MwIeiILjTGuKydAaQKfXWNaelikkiLQ+ItsFMU\nhI/D6IkQRYamT/rVQRB56xqCMK1HDQCpNFdSS3YBmE9Ez8POZAIAIcR3SjIrhmE6DV4Xk1tBdatL\nYHcyDUAJUtvHZMxC4teiQ7+OpHdjDcYP9n82jbLkqKsXkzInIbzrW6j4WRAyflLJFkRYBfFf+x/D\nMJ2AotoPfmmuhv2yrkEtlOtWF8cme2kwGYNwspjIHYPwK7ADvFYAEfCzM/YLdw8Ggl1M2XEZAcQD\nZLyfBZGUFkQFmxBhg9R32WtHj7Z3fSSESJZuWgzDBFEKD5P+NO6XOqq7mBprs2LE42KKuQWsx8Xk\nsiCy12+sjWPBz0/OPefAGAS5rpkNUpOnqjooVTXIgqhg3QAgfCX1MQAWA/gLgL8CWERER5VwXgzD\nBFH6EITxGmRbBKrQ716XVQr1NW4Xk55SKt9ffeo4a9vHgihGbYG+YJFqQQQFznX8AvLJtKho9xIQ\n3sX0OwAnCiE+AgAiGg3gXgCHlGpiDMOUF2cNZ1cMQlYWmy2I2riW5krkSmWVwrZng/UZV7BYuU4x\nFEQ85g5SS0sgHiNXgV6uojl/BZGpaPcSEL5QrkYqBwAQQiyC1Y+JYZgyUMxK6kfmrDbul3JRdf2Y\n0ly72RZEv+51qHUqqeVntSZ59ns5zlVJrVw7EVLwhnExyam2291X62pixjn54Xc8lRYV72IKa0HM\nIqI7kG3Q9yUAs0ozJYZhclHMGMTKLbuN+02uFxmkVo91sy2IsXv08GQDWU/rijvHfl8bt5SKWkmt\nfraYLiY51zapIOJxV1whnS7AgmAXEwDgmwAuBSDTWmfAikUwDFMGShGC2NWWxI7WbO6JkxaqjCE7\nzTXlsiAsMdKzIZFd5hPZLKa0IUgtLYi0jwURdhGeMM365H2oFoRKTgvCN0jNMQh13B+FEL8HnOrq\nupLNimGYQEpRKHf7jGW4fcYyZ9skF6VATKbUGIRlDdQl4p7aAz3NNaO5mIQriyk7Lh6Ud2rAL+NK\nvY+2lFWrocdJUiEtiMNH9sUbH2929rcm057mg5VG2BjEdAANynYDrIZ9DMNUKLfPWArAG6QGsm0m\nAKDerqROxLxla1YvJtWCsF6lYHWluSqf1pv6+RG8YJD1mlEsiESMPIHlXW3BGftqcFulNZlxFF2l\nEvbu6oUQu+SG/b6xNFNiGCYXHZHlumqrNzYhhWu7oiBq7CdytTGf/Ky+YJATpLY/o/r/TYooLMGt\nNqztt5ZuNtY0bN+dCjx32kdBtCXTrCBsmonoYLlBRBMBmCNbDMOUnA5oxeSgPtlLIS79+a/96FjH\nGojFCGu2ucVCTGu1ITQXk5//P7QFEaJZn7zme59sM45T4y4mPMup2uxOpj3uqkojbAzicgAPEtEa\ne3sQgC+UZkoMw+SiFAsG+aH3YgIsC6JbbRxDejc6gjNGwJrtloL48qS9nPGqDvAGqc2V1MXIYlIr\nqYNiNjt2h1MQ+pxSGYFarXK80ghUf0T0KSLaQwjxDoCxAO4HkATwDIBlQZ9lGKbycGIQqQwS9tNz\ngx2k3t2ewTFjBgAALjpyJAC7UM6QxSTdUq5mfcp1wiqIMM36TO3JjxnT33m/ozWci6lPt1rPsWp3\nMd0KoN1+PxnAVbDabWwFcFsJ58UwTBAd6mLKoloQ0nLo3WgJzq0t7Thjwp74+JenYq++VogyFoOx\n1YZ0zUwe2S97nSJXUksv1auLNnpiD//46qcw95oTAVixhCCkG+yzBw/Brz6zv+tYXYW7mHLdXVwI\nscV+/wUAtwkhHhZC/ATAPqWdGsMwfnSgfnAh3UDJdAYJO/YwqMlKcOxu10Oowl3vxZTNYoph+veO\nxt++7IQ281IQQTEIGcd4c+lmbNzZBgD46uHDnPtorJOWTw4FYRfzJeIxnPupoa5j1W5BxIlIximm\nAnhRORY2fsEwTJHpyCC1akLIt1bKqCU+9h/ShBs/NwG/OHO856P6kqPyaTwWA0b27+7q46S6i8IW\nygURjxF+9ul9AQDb7TjDXn2yyZc18RjiMUJrKpeCsF4TMfIU5lW6gsgl5O8F8AoRbYKVtTQDAIho\nHwDbSzw3hmF86MggtYp8sG9PC8eCAIBzDhliHE9aDEK22jApgLwsiBzH+3a36nl32nGGhFbY1lAT\nx+72jOdzu9vTTmxl3ior+8lUNT37k62h5tlVCVR/QojrAXwPwD8BTBHZVIAYgG+XdmoMw3QGXGtS\nx2QldSaUEE9oaa4rNrcAMCuAvNp95xgmV7hrbrMVhJY+W18T91gQT85bi3HXPIMP1+6AEAJ/fnGJ\na04vf/8YZ+zWlspeFienm0gI8ZZh36LSTIdhGD/eWLLJed+hLiYFNQZRE6JWoVtdAss3OasU453l\nVkizvsaQHlrkNFf1Os3tZguiviaGVi0G8fyCdQCABWt24HXlO5e1H9KyAIDzJ+9dlHl2VirbgcZU\nNAvW7EBKX8OygvlkS4vzvkPbyn/yAAAgAElEQVRDEIYK5/Z0OAuiqSHh+P8B64l+30E9jQoir3bf\nOUwIeZ1djgVhcDFpWUwy4ykRJ7yp9F4aP7gnALer6fQJe4aaZ1eFFQQTmlwFRx3Jxxt34dQ/zcCv\nn1lY7qmUhOcXrPdUJacMFckdgTHNNZUJ1aiuZ30NdrQmnfmmhUDf7t56AkBTRBEtCL9vQ65wt8uJ\nQbhFXvf6BLZpbiLpEiMip17jMwcNdqwn1YKor+ZCuUIgoqFE9BIRLSCiD4joMnt/HyJ6nogW26+9\n7f1ERH8ioiVENE9t7cGEY3d7Gj/53/vYmaN1QD7saE1i5FVP4ZZXPi76ufNhk522OHdlZeZKXPiv\nWTjj5tdd+zqDco5qQfRsqEEyLdCatCy9TMD6z8VeMAjIrpEtLYga7bxjBvbAh+t2uL7bpN0f6jv3\nzsYzH6zDmIE98JtzDnCOd6v1LrFaqZTy7lIAvieE2BfAJACXEtG+AKYBmC6EGAWrS+w0e/wpAEbZ\n/y4CcEsJ51aR3DNzBe5+awVufmlJ0c+9eZdVL/nAOyuLfu5CKFc2T0ewaVeba9vd06jj5qGmdrrr\nIHKLD1kbsbMtid3tacxdtd03hbWQVht+o6UA/8tL7kCzZK++jdjWknQUGJCte5AM7dPguld1nsZY\nSgVRMgUhhFgrhHjPfr8TwIcABgM4E8Bd9rC7AJxlvz8TwL+ExVsAehHRoFLNrxKRAqQUwqMzPL2q\nBC0UU6nkWLagQ5AuprZkOBeTtATSGYGfP/EBAOA9n9RQVXYXK81VCnCpW2s0pdaj3lo5eafS8rtZ\nC1o3NZhdYur5K5UOsY+IaBiAgwDMBDBQCLHWPrQOwED7/WAA6uPpKnsfE5HOJsxLSTlv9cl5a9HS\nHtzHp5ioT7Yd2801i5TbrSE7mUpBn0oLrN7WCiC79Kf3OiXIYtJiBHrLjR62hSNjFC3tKby9bItr\nTO/GGv/zs4upMIioO4CHAVwuhNihHrPrKiL9Vyeii4hoFhHN2rhxYxFnynQlym1AzF25DZf+5z1c\n8+gHRT+3n4JXZVv5CuWsL741lfE8jZuQgj4jsgkOvvEF1YIowpKjgHd5UVkPIXFcYLaCeHeF17pp\navAqiKtPHYfaeMxVCV6JlFRBEFENLOVwjxDiv/bu9dJ1ZL9usPevBqA2Ohli73MhhLhNCDFRCDGx\nf//++mEGpXW/dDbXTrkMCBn01DONioHPEsi+ayOXGvUnV9eDCNNmIq64mCR+sQtXkLpIS3nWaXPc\npSmIHvUJ1/6lG5uhY2rFceFRI7Do+lOKZul0VkqZxUQA7gTwoVzL2uYxAOfb788H8Kiy/yt2NtMk\nANsVVxQTgVK4mDqb06rcf5aldPFk/CyIcgWp1UpqRVuEURDZltvZCftmMZUgBqE/0Ewa0ce13b3e\nbUGs2bbbY+F0r/N3MVU6pbSPjgBwHoD5RDTH3ncVgBsAPEBEFwBYAeDz9rGnAJwKYAmAFgBfK+Hc\nmDwpt2DWKXe8pRQGld8tueogin/ZUKgKQn86N5G1ILL35ediUhVRKVw3i68/xRuktoW/TA3f1ZZC\nU0MNNjdbWXv7DOiOC6YML/pcugolUxBCiNfgL0+mGsYLAJeWaj7VRGdzA5WCct9iKWMA6tP2B2u2\n47Q/vYb/XnK4Sxl2qGI0VFIDCBWklgollck49+VnHTQq9QWXTR0VbmoR/h+YYia6i6m5LYVudQlH\nQTx7+VEV70YKorJD8FVKuZ+qO5KOvtPH567BFffPyT2wSLz8kZWI8dwH633Xb+5IVIEcxsUkrQW1\ntMBP4Ep3zw9OGoNudR0T/JXXufbxBQCsFNfG2jh+ePIYXHrsyKpWDgCv6cCEpBPIJg3rD7ej5/Xt\ne2cDAD5zcOkysF3LcCrySW07Fea2hRDYsLMNA3vWFzQfdQ5q36JIQWohcv5WMiU1jOvKmZv8fxD6\nE270e2huS6F7XQKXHMProQFsQTBR6SQPVFJolVtv5WoWlw8mQSogXIojjGJ8+L3VOOyX030L0/Jh\nx+5sFlBtPHeRWEzJYpJuOb+5yx5HySgVgSG+/q9M3juUy2rJhl2Yt2o7GjvIeukKsIKoIMrtl+9I\nyn2r5935dsnO7bIglDt1p7nmFqJvL7M6kS5ev7Og+ajftdrnK5QFoWQxydvyi998eoLVOEF2Tc13\njjo/P3M8rjhhtO/xQU2WhXX871/BrrYUutdVdnV0FFhBVBCldbeU+1ndh87n+yoYY7mDyL8Xkzr2\nrjeW4/ZXl0aaj5r0oPYskjGDIOSSEal0Vi34zf2Qvftg8fWn4MhR4eubivFQpGcp1VV4h9YosIJg\nQlGmGq3qRPmuVQEYNfnA5P766WMf4PqnPkRrMngdZj8uOnqE8/6Y0bkFuWpByPsKuo0w1dmlpt2n\nFUg1Uv5fgykapXQxSbdHuV07EqG9lotSfOemQjm9J81fInTsNX1HU3/3inUshNJRb7FnfbZorH+P\nupyflRXRagyis6H3hipXxXpnhBUEE4pMJ3uoqkDPkoN6a376539z1uQ8T5DyWm23CCnkewzTyVTW\nQaSFcKzQzpaGrSsIvaFfNcMKggmFX/uHciGFTCebVlEwWhAh0kSLdS2dQqwkp1lfRjium872k2U0\nhaCvB1HNsIJguiSdTcjkQl/8JwjhF4PI866DdECpv0dpQVxx/xwnA6qzKfULjxzh2u7bPbfrrFrg\nhN8KpJRPmp2ljYd86uusfm2dS/79Hh74xuRQY00uGCGi/65hfqow59TrEp749hRs3x1uWVtpQexo\nTTldXDvbb9bUWIP5PzsR/5uzBnEiJ92WYQVRkZTCh9rZ3LKdbDo5iWJBqN91MQrxggRyLmF9wZTh\n6Kmls44f3BT62qZWFZ89eEjoz3cUPeprcN6kvcs9jU4Hu5gqkH++sbzo5+xsMYhMjhhEc1sKSzYU\nViBWLkxCO/LKWmGvleOkPzl934KsRlVBJNMZfGHiUHz/xDF5n4/pWFhBMKHoZPrBmFP/j9eX4aSb\nXgUAXHDXOzj+96+WPGOmFC43lwXhqoMI93khBFLp3IHWZZu8i+MUG3VluJ2tKRwwtMlpv8F0flhB\nMKGQgraz/Gk7KZPKvmsfX4CP7LYSby3d4hrXlfCLQYS1IW5+cQn2ufppNLcFF8Mde+PLJbcMdRfT\nhCG9Sno9priwgmAAAMs3NeP1JZt8j3c2QRs20NlZip6izMJPZoeV5fe9sxIAsG137qyhUn89Sc2S\nUQvtmM4PK4gq4h+vL8Os5VuMx4658WV86Y6Zvp/tfDEI6zWXC6nUCqIUFpVfN9fIWUwhxjw1r7Sr\n+upFaGH6NzGdB1YQVcS1jy/AOX97E7PzaP/c2RRE2NhCqpMUPUVRJO71IJRuriX4DXa0hktXzZdR\nA7q7tnuwguhSsIKoQs7+6xsYNu1Jj/kfiC2bOkkZROin6bD6IZ0RGDbtSfz22YX5TyqAKKLdXEmd\nv5IO+lSpV0xLxGP45jEjne1iN+OrsdvFjtszeotwJjeszquY9lQm8A/2pYUbMGFoL/TpVttlYxBh\nLQipLG+fsQw/OGls3vMqBn69mPSWEMWgI5bUvPDIEVi6cRf6laBCuaE2jgcunowxe/Qo+rkZtiCq\nGgFg4842DJv2pOfYztYkvvbPd/D1u94BoHZz7RwmhJT7Rn+9sjNsDEKOiyovi2VRPb9gPYZNexIr\nNjf7us+i6gdnbgGWR0dUxvfpVotbz5uI68/evyTnP3R4HzQ1cPC7FLCC6ORsaW7HiTe9gqUbd+V9\njkXrd2Lhuh2e/Rkh8MkWcy68bKwmc+U7XQwi6JhyMKzfXo6L+QjMtlQaZ/3l9bDTi8z/5qwGAMxb\ntd1XnpciBmFSiBcfNcK7k6lK2MXUyXn2g3VYtH4Xbnt1KW747AGBY/2e7k+0i8d0MhnhWT0rkxGI\nxcgRRtIF0bnUg1JJbZiZqsxSIdc3TqfdCqK5LYX2VAa9u9UCAJZvasGcldtCnWvF5mY8Mnt1qLE6\nAmZLIZ0RWL11d6RzmWpFdEwKkQvZGAkriAoiahO0VEZ4XDCpjEBtjBwXDhHhuQ/WdZomfZKbX/Rf\nMEe9pbAuJtm/St7lUb95CZub27H8htMAADXx8Pd/8h9mYHfEFdvUs6u/o/za735rRaTzAeEyvUy6\ngPUDI2EXUwcy7eF5GH3105E+Ez5bR+DPitC85eWPsWFHa87PtKf1xVKsbRm03bizDRfd/S5uedk6\ndy49saW5Pe/lLKMwf/V2AObvZ6eSuhnaxSQViX1/m5vbXcf9gvmmr8OkHHIJa/VosTJz83UL+rnZ\nmOqDFUQHct87Kz0COSymv9ltLe14xy58e3PpZuxsTTnHfv3MQnzrP7MDz5kWAm1J82paegrsypDu\njYN/8Ty+cufbocaWikOue8F5H96CsO7XTzh2VEU2QauDKOBcYRSNqfNvZ7MWmfLBCqILsmGnZRmc\n//e38bm/vYl0Rhj/0N9evgU/fGiu73lSaYH2dNqzD/AKjigPo2/7VGuXglzTkoJ94bodGDbtScxf\ntd04TgpTP/dKoQHicghdqWh2tqbQ0p4ypsmaYjRxVhCMDSuILsbrSzbh0Oun47kP1uH9NVZmUiqT\n8X3SfGDWKt9zZUR2GUiJfJLW93e2dYQludwoUkFM/3ADAOCp982tJeR9+wnyQmsQgr6/Fxeux5NK\nywu/Smqdrc3tOO/Omb6uRHmW3z77ESb9crrxIcK0L0bA81cchUcuOdz32kx1wAqik6MHnqXvfdaK\nrc7Tbiot8srHT2eEd8F2HwsiSBC3JtNobkv5Hi8pOeR2WEsoVx1EFAsiavHZ//1zlvNeILy1du87\nn2DG4k049JfTsXJLi+e4qtR2tKaMv6Fp/eVYjDBqYA8ctFfvcBNhKhZWEF2MhC180hnhpLWGTeXU\nMVkQ6YzA4vU7PSmVQT74E256Bfv99Nm85lAouVbPywp+ma5rHu9kMUWMQZjG1yUK+7NyWxD+41R5\n/+bSzYHnAczflTkGEWKSTFXAaa5dDCno0hnhRDCTmUxemScpQxZTMp3BCYa6CVXWbN+ddFWurtwS\nLT+/mOQKHuuWQd4WRAQXU6HtK/Q1oP1QLQSTC0zfkzac1/RwwVlMjIQtiC6D9UebsPPxU5mMI8zy\nzbB55v11RgvChNy7cN1OTLj2ObyyaKN3TBHjFKu2tuDy+2ajLRWcMpur11Iqk8HKLS1Yu93y0/vF\nErJ1ENEsiBcXbsDmCOtNh+Erf8+2XQ+7loPJBaZPOWn4rvxiEAwDlFBBENHfiWgDEb2v7OtDRM8T\n0WL7tbe9n4joT0S0hIjmEdHBpZpXR/PTR9/H6X+eEWpsazKN4Vc+icfnrvEdE3dcTFlhlkxncM/M\n6IVUf3hhsUfw+T296u4K07oS6pBcgj0X1zz6Af43Zw1mLPJfxAjIrRwzGeDI37zkrNPtNzrtpLm6\n97cm00imM4ExmD/rRXsF6slWJfU46LrqMdPXoCtsk7VgWpqULQhGUkoL4p8ATtb2TQMwXQgxCsB0\nexsATgEwyv53EYBbSjivDuWuN1fg/dXePkgm1m5vhRDA7577yNmny4e442LKWhCfbG7BU/PX5TU/\nXcD6CdyW9twCX32KHfPjZ1zHVmxudtJzo5BLVuWKQegWxpINu3D1I/O9FeRpcwxi7E+ewSl/nIGg\n8pUwAnX55has2x79/oMsiBWbs320TJaRNwbhvQnT711bYAyFqRxK9j9BCPEqAP0x80wAd9nv7wJw\nlrL/X8LiLQC9iGhQqebWWVGf+J55f62xQZ+0IFIZ4QizVRF79Kj86mn3+gcmN4QJk+AKCpYf/duX\ncej10yPNLQxSwPm5t579YL1r+5VFG3HPzE/wsfbdOjEIw1/Ekg27Al1ZYV0yP3vsA9f2vFXb8Nri\nYAvJZEGc+6mhAID/zVkTOE6fstGCMCgI7ozKSDr6UWGgEEImfK8DMNB+PxjASmXcKnufByK6iIhm\nEdGsjRu9fvBys6W5HdMenldQu4lkWuAb/34P/+/2rC9aPqTKGEQmIxxv+SeGFMd8KaRiuKW941Nd\n5XzfWmouzrv37U+M+xOaVM8VgwjSm2Gb2y3esNO1fcbNr+PLd7qXef1wbW5rc/RA79oH6YzAis3N\n2KK0CAljQdz/zkrPvl6NtTnnwFQHZbMlhfXIF1kaCSFuE0JMFEJM7N+/fwlmVhi/fXYh7ntnZeRu\nnvfMXIHjfvcKAGD1NssiMK34Jt0ZlgVh7dtVxBqEKKvMJdMZlw87jBuqUF5bvAmL1mcFrRTsUTOH\nEpqpUEgdRFiX/ccbza3VVW55+WPXtn7Zt66carzXjBA4+rcv4+jfvuT7WVN8yfR/p3cjWxCMRUen\nua4nokFCiLW2C2mDvX81gKHKuCH2vi5Hvok8/3h9uWdfr8YajwaV508rLqZiNscLa0HcNmMpbn7J\nHZxtLqIF4ecy0p+45Xzra6I96/jVCPjFE4IqqfNZRGl3SGWqzrN3Yw32aKo3KiSpp3e2ptCzvsbz\nWWtMuN92ZP/uuQcxVUFHWxCPATjffn8+gEeV/V+xs5kmAdiuuKK6FH4K4ol5a5Qx3kHdauOefSbk\nH3lasSD0auhCCFt0p6fHAsWphzAJsaD0WTk+V7BaR7eU0k6rjfDzkuifCTOTrS3tuQdp55KWgynY\nrSoDaYEu3uCOs4SxDh/6xmR0q+PyKMailGmu9wJ4E8AYIlpFRBcAuAHACUS0GMDx9jYAPAVgKYAl\nAG4HcEmp5tVR6HJG7axqknf6wj1+SFdHOiOcp91iWhBRBa3KgjXhsrX8uOuN5Rh51VPYqvjRL7hr\nVqiV3KLGTk646VWX4sllQQR9L7rHJ0ihyWNbmsMpCFXwO0WSxoB07vsPU4DHnVwZlZI9Kgghvuhz\naKphrABwaanm0tnICIGYpkJyxRGEsITK0/Mtw0ogK5iKaUGYLIOw7FDWYQCA6R+ux9RxA31Ge7nt\n1aUAgHmrt+Olj7IJCHN9OrCq5NNupD2dcRSzVDB+8jGoHiFGhGc/WIffP7cIT112ZODa0emMQCJO\n2L476T9IQb2stCBaDe6pML2i5DoVtfGYb9v5QqvAmcqCE57LgEmA7GzzERj2H/69b3+CSb+a7ghO\nS2BZf8wvLtxg/mwe/ObZhbkH+aBbMhfcNctYiOXHJrsi+QFDZk0uTBk6PeqDn39UxZpKB1sQUoF8\n94TRnmM7didx8d3v4qP1O3HZfbMDV5OTlkhQDKKhJmtN3vpKNmgt52ZKBghjQe22Y0RBSqCzrT3O\nlBdWECH4z8xP8NV/hFsEJ8yyn6Y/Qj+BoY5Un+6FKE1TtRWb80+ZbU2Ga+XghxTYudpnmDBd5/Lj\nvcJcRf0+w/Z0Gt6vm+fYXW9mq9ifmBccOktlBK7873zcqBRDBrFDWQRKCvYWgwIKk8nW3GZ9LhGw\nfGqyiNYo0/XhaFQIrnpkfuTPhO3CKTG5drbvTmHzLrOvOiMENu4sbg+gQnn4Pe/aE8l0BvU14eIr\nEr9A9U0vLPYdrzeiGz+4Z86CL5cFkSMGIVuH9CywiCyVzvjWZkhafdqUSAVhepjYGiKmIRWLXgOi\nErZRIFMdsAVRBkwWhMknvGlXG/443SwUZ+SowO0s/NOQvpsLkzWweMMu/Mnnu0imMx6rI5UWOVtG\nuC2I4CwmqUxyua1yEcai8vPySLl+1aljPcc2+TxIqMjYRVyrATll/B7O+7CV9Ex1wAqiDJgURFd4\ncpt7zYmRP/O75xe5tu+YsRRLNnhbiKiYLAh97WyVlGHJ1WQ6g9p48H/vNz/OrqGgrxvhd/2ehSqI\nAn5nWdy3zwBvJXUYa1LGLmptF5O0JPZoqnfGFJKkwFQerCCKTJgYnz4knREFtbiIypDeDfjxaeMi\nf66pCBW21z35Ic752xuufe+v3u5qPGcSokFPtslUxpNim8oI1OUonrvqkfmYuXQzvnzHTMdC8PO+\nXP/UhwCAHvUFupgKeEJXW3p8/0R3fGVna+6sqJtesJT1N4/dB/sPbsKEob0AuF1Oh4/sm/f8mMqD\nFUSJCKquXb6pGXe+tgxCCKzdvjtSe4ticPWp4/D1I0d06DVVtrW4hdnpf34NR//2ZWfbpCyDnmyT\nmQz+qrWoSKVFqKf9K+6fg9eWbMLKrVZwPlcdQNR4ik4hFoRqEH3ruFGuY1HarZy2/yA8/u0pGDXA\nqphOKCcuVAEylQUriAgUunC95IybX8cvnliAW175GJN/9SLmrtxWlPOGRcrAfKyIfIi6kJDpKXtX\nq78ANHcpzTgtJ4JosCvYd9rnz5UZFrSc6CDFVeNHIYWIQd/BjoBjY/dwu6Qa7XuW9x4UtGaqG1YQ\nEYiycH0YpA984bqdOUYWF2mwdJQVEVUmmiyIP2t9n1R+8cQCz75UWoTKOKqxn55lR9wYUWDthqog\nemgtKYLSR515FeBiGtK7Ma/PXXP6vq5teQ+ySFBvXMgwEv6fobBqaws+ChDWYeIEUWSh1DeFtMq4\n4TP7R/5M2GKoO8+f6LzXhUyxrmeyLkxP2UFW1tPvexdLSsQplAUhlfPsT7LnX7fDf2Ef1QX19tXH\nu47VhBC0+bqYbj3vEPz+CxNCj//pp7O/V41m9ch7kMHqMLU7THXCCkJhyq9fwkl/eNX3eKRAcgir\nXVYOBymlXIwf3OS8/9Jhe7mO/fJss/IIEth/+7K12uuVp4x1tcn42hHD8p6jSSi+tdSynkxVx8UI\n2N/z9cOcDq8H79Ur9OeEEFgdcgEmTwfZEL/56X9+zf2RkN6dk/bbAwN65HZhSfbs1eC8r/HJ5pKx\nh46OgTFdB1YQEVCfbFPpDL71n/dCLfDih3x6/W/EtSNU1LYJ15+9P5bfcBr+c+FhGNyrAZ+e4F6U\nb8o+/QAEC+CTxw/C8htOw8VHjwQA3PSFCdh/cJPz1PkrH4vlx6eNwwn7mvsumdqAn3vbWwDMwdVC\nUi271cbxyg+OwT4DeoCIsOxXp+LBbxwe+vNpkY1H5EIPaC8Nsd6DTiFey9+cc4DHzSXpVpvd75fu\nKxWHTOHdb8+e+U+GqUhYQURAFaxLNu7CE/PW4rL7Zgd8ovSYBMzhI/vh9WnHobsiPKZ/72gM6FEH\nIFpM4OyDhuDxb09xtr946F7GcceOHYBv2EpFJyi4aqoUX7opuqCVzL7mROzdN9sOg4giNaBbtaUF\nX//XrLyvXyx+dHK2GO6k/cyK9/MTh2L+tScZLaRDh/dx3g/oWee8f2Pacc77cYOs4HUqIzDjh8fi\n/osnFzxvprJgBREBVUHIwJ7uLy/kifDxb00x7v/q4cPwi7PGAwBO9HlKN6E+4Y7s393ZLnZDtiuO\nH42R/bvjkL17G48HpWCe8scZeV1zcK8Gl0vtiH364uNfnpqzejoXm0O0rHjuiqNw74WTAsfcd9Ek\nPPFt8+8ZhguPHI5HLz0CALDfnk2BY3VL5senjXN9D/26WwpiWN9Gl+vp6NH9cdMXJuA7U0dhaJ9G\n1wMFwwCsICKhKgj5VOoXdPzhQ/MwbNqToc/dUBPH/kP8BUG9/Qev/uH/3xHDPSmMOt89YTQe/qbl\nYjnRfhLdf3CwwInKmQfu6bxvMNQJNPsoCHV5zKi8Pu04fHpC9roDe9SHshQKEdqS0QN7YLJPQdnk\nEdb+SSP6YvzgJvTpZq3v/PA3oz2dJ+IxTBjaC09+ZwouPXafwLE3ff5A17bJhTjjh8fikUuOcO0j\nIpx90BBnjgyjwwoiB+5FZbzdP9MZgdXbduP7D84tKBtJCrczFKGnXlc+BW5taccVx4/GocP64JpP\n74tYjPD0ZUfiPxceZjzvd6aOcp7sT9pvDyy67hSMG2T2Ne8ZIo/fhFqx/Oi3skJIPpF+wY436OTb\nOfa0/a3YyjZlVbYrDG24TYwf3JTz6d9ETZx8XT0q93z9MCy+/hRne4+e1nfaUJPf0/l+ezblVHx7\n9W3EH889ED3qE0jECKeMt76fy48fhYn2bz+0TyN6syJgIsI2ZQ7Ujp9qCrtUEMl0Bjc8vRCPz12D\no0f3z/s60kvwpy8ehMfmrnEd292ewdA+Vg58XSKOy44fhcuOz1bS+gl8E7oL5q0rp0JAoDWZyXux\n+nplNbzRA3vg+HED8cKH63HtGfvhew/OdY298MjhuH3Gspzn/OfXPoWv/uMd47Fjxljf81Gj++Oc\nQ4bgRyePRf8edcaxJvye/v0Y1FSPN6YdZ6yyfvF7R6NHfQ0+df0LGNK7AbEYuRaDuvW8Q3DLKx9j\n9MDumDyiL95cutn1+Zu+MAFX3D9XP21kzjxwMM48cLBr3+XHj87Z8pxhgmALIoDlm5rx3QfmONuq\nBSFTA9MZ4Twp63/8UTA1ifv9562899ZkGvsM6I7fnnMAbvhs9LqHIPZoqsegpgYM79cNvRrDPWFe\nfNQIl0DWex799pwD8JvPHoDTtSwqADg+YIW5r0ze23lvWndBIi2PxtoEbvzchEjKQTKsb/iis88e\nPMS3BceI/t3Rv0cdFl13Cl7+/jGe40P7NOKXZ++PRDyGuy84FB9ddzJe/cGxaKyN44GLJ2Nk/+7O\n2FzuQobpaFhB2Pz1ZW+l7rfufQ9Pzc8WYam+XdWC6GU/ef9n5idObUNUapQq3AcunozvHLePs56B\nrBX43MShkXLhS8WVp47DQ9/I+tT19bR7d6vF5z81FHWJOC4/3t0zKKiX0dVK648BPeqxt0GIf/bg\nIfjSJHMmVRQeueQIvPDdo1z7Dh3Wxzj2Mu0eTNQmYq6eRiYS8RjqEnHs1bcRC35+Mg4d3sel3K46\ndRxm/PBYzLzKsyovw5QFVhA2v3kmu8LX3W+twAsL1kPvipA2xCNUCwKI1jRNRfbFAawUxe+eOMYR\npi2GOoJyIy2eRCw4jfSYMQOc9/93xHDfdtqApWi+eOhQnH7AIDTUxvHy94/Bzz7truD+3ecnYFBT\ng88Z/PnhyWOceABgKUBvMOgAAAjpSURBVDG9bfY3jxmJ2kQMX58yHCP7Zy0Yv0KzYjCoqQHzfnYi\n3r/2JBw1uj+G9mnEwJ7lfwhgGIBjEACA/2orof3kf+8DAPbVfPtqxpJ8n8wIVzVsvs3YTNk/0s0y\ndWz41NaOYmifRjz0jck4YEhwlXK/7lm31f5DemLcoB74/MQheGBW9jvv3ViDe75uBY5/9ZkDnP1E\nhK8eMRyPzFlTcEPDS47ZB5cc458NNLJ/Nxw7dgAWXWcFmH98+r4Y95NnOmSN5jAtQRimHLCCAPDd\nB8IFCVUXU0rJYlKXu2wNWIzexI2fm4DvPzjXqCD27NWAuT89seBFakrFRB+XjMqQ3o247qzx+NSw\nPhg90KrF+M05E/DU/HWOtXXY8L7YN6CK98GLJ2P0j5/GTwroB+XHfy48DG2pjHEdhNnXnFD06zFM\nV6JzSp4OQgiRs/+/itvFlFUQ0xducPa3JN3uoFu+dDC+ec97xvOdtN9AnHngnnht8UZ8e6rZz51r\nXeWuwJcn7e3Zd8r4PfDgu6tw74WTcEBA/Qdg+feX33BaSeZ2+Mh+vscKXfuBYbo6Va0ghl/5VOBx\nXXfcO/MTrN3WitMOGORqCT1HcX+s3OJu9KbGFnTOmzQMNfEY/nDuQRFmXRlcf/b++PFp+xZllTqG\nYUpD1SqIMB1D9TEPvrsKD767Cje/1BPLQ/YL6ubTvmBk/26YMsr/6bXSqU3ECm6LwTBMaanav9Db\nZyzNOUZdyEfNgPlw7Q5jm2oTptgCwzBMV6BqFcQNTy8MPbZf9zqn82VUVAviwKG98N5PrMDn8RGa\n7jEMw5SDqnUxRaE2Thg7qCde+mhjzrGDmuqxdnt2RbJuSgziginD0adbLd6+air6do9e/cswDNOR\nVKUFcc/MFZHG1yZiodogXDZ1FF77Ubbf/r8vOAx1tovJWsDHasQ3oGe4zqMMwzDlpCotiN0RaxVq\nEzGM6Nfd9/gPThqDw4b3wcRhfZzur2oQ+spTxuLk8XvkP2GGYZgyUJUKYg+trfXHvzwVo65+ChkB\n/OOrn8KxY632EHI9h2PGDMDYQT1w1Oj+mLV8C1psBfPdE0bjO1r9AhHhnq8fhjGKxXGxz0prDMMw\nnRkSHdBKICxEdDKAPwKIA7hDCHFD0PiJEyeKWbOiLw+ZTGfwu+cW4YuHDkVDbRwDetRDCIHlm1tc\nXURfWrgBH63fiQuPHOG4hFqTaWxubsf6Ha3Yd1BPLqZiGKbLQUTvCiEm5hzXWRQEEcUBLAJwAoBV\nAN4B8EUhxAK/z+SrIBiGYaqZsAqiMwWpDwWwRAixVAjRDuA+AGeWeU4MwzBVS2dSEIMBrFS2V9n7\nGIZhmDLQmRREKIjoIiKaRUSzNm7MXZfAMAzD5EdnUhCrAQxVtofY+1wIIW4TQkwUQkzs3z//NaAZ\nhmGYYDqTgngHwCgiGk5EtQDOBfBYmefEMAxTtXSaOgghRIqIvgXgWVhprn8XQnxQ5mkxDMNULZ1G\nQQCAEOIpAMGLNDAMwzAdQmdyMTEMwzCdiE5TKJcPRLQRQLTOe1n6AdhUxOl0BfieqwO+5+qgkHve\nWwiRM8unSyuIQiCiWWEqCSsJvufqgO+5OuiIe2YXE8MwDGOEFQTDMAxjpJoVxG3lnkAZ4HuuDvie\nq4OS33PVxiAYhmGYYKrZgmAYhmECqEoFQUQnE9FHRLSEiKaVez7FgoiGEtFLRLSAiD4gosvs/X2I\n6HkiWmy/9rb3ExH9yf4e5hHRweW9g/wgojgRzSaiJ+zt4UQ0076v++3WLSCiOnt7iX18WDnnnS9E\n1IuIHiKihUT0IRFNroLf+Ar7//T7RHQvEdVX4u9MRH8nog1E9L6yL/JvS0Tn2+MXE9H5+c6n6hSE\nvTDRXwCcAmBfAF8kon3LO6uikQLwPSHEvgAmAbjUvrdpAKYLIUYBmG5vA9Z3MMr+dxGAWzp+ykXh\nMgAfKtu/BnCTEGIfAFsBXGDvvwDAVnv/Tfa4rsgfATwjhBgLYAKse6/Y35iIBgP4DoCJQojxsFrx\nnIvK/J3/CeBkbV+k35aI+gD4KYDDYK2z81OpVCIjhKiqfwAmA3hW2b4SwJXlnleJ7vVRWCv0fQRg\nkL1vEICP7Pe3wlq1T453xnWVf7C6/k4HcByAJwAQrOKhhP57w+rzNdl+n7DHUbnvIeL9NgFYps+7\nwn9juVZMH/t3ewLASZX6OwMYBuD9fH9bAF8EcKuy3zUuyr+qsyBQJQsT2Wb1QQBmAhgohFhrH1oH\nYKD9vhK+iz8A+CGAjL3dF8A2IUTK3lbvyblf+/h2e3xXYjiAjQD+YbvV7iCibqjg31gIsRrAjQA+\nAbAW1u/2Lir7d1aJ+tsW7TevRgVR8RBRdwAPA7hcCLFDPSasR4qKSF0jotMBbBBCvFvuuXQgCQAH\nA7hFCHEQgGZkXQ4AKus3BgDbPXImLOW4J4Bu8LphqoKO/m2rUUGEWpioq0JENbCUwz1CiP/au9cT\n0SD7+CAAG+z9Xf27OALAGUS0HNYa5sfB8s/3IiLZqVi9J+d+7eNNADZ35ISLwCoAq4QQM+3th2Ap\njEr9jQHgeADLhBAbhRBJAP+F9dtX8u+sEvW3LdpvXo0KomIXJiIiAnAngA+FEL9XDj0GQGYynA8r\nNiH3f8XOhpgEYLtiynZ6hBBXCiGGCCGGwfodXxRCfAnASwDOsYfp9yu/h3Ps8V3qSVsIsQ7ASiIa\nY++aCmABKvQ3tvkEwCQiarT/j8t7rtjfWSPqb/ssgBOJqLdtfZ1o74tOuQMyZQoCnQpgEYCPAVxd\n7vkU8b6mwDI/5wGYY/87FZb/dTqAxQBeANDHHk+wMro+BjAfVpZI2e8jz3s/BsAT9vsRAN4GsATA\ngwDq7P319vYS+/iIcs87z3s9EMAs+3f+H4Delf4bA7gWwEIA7wO4G0BdJf7OAO6FFWdJwrIWL8jn\ntwXwf/b9LwHwtXznw5XUDMMwjJFqdDExDMMwIWAFwTAMwxhhBcEwDMMYYQXBMAzDGGEFwTAMwxhh\nBcEwDMMYYQXBMAzDGGEFwTAMwxj5/zBezeruYejmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f36d37b1e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(reward_lst)\n",
    "plt.ylabel('Score')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD8CAYAAABthzNFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd8W/XV+PHP8V6xY8eJne2ELCCM\nJAbCHmGVtkAns6WUFtrSBwq0/UFL26cbumh5oDShtEAHs2G00DBC2ISQkEW24+w43ntItnR+f+hK\ncRLZlh3JkqXzfr38su5XV/K5uaCj7xZVxRhjjDlYUrQDMMYYE5ssQRhjjAnKEoQxxpigLEEYY4wJ\nyhKEMcaYoCxBGGOMCcoShDHGmKAsQRhjjAnKEoQxxpigUqIdwOEoLCzUkpKSaIdhjDFDyooVK2pU\ndWRf5w3pBFFSUsLy5cujHYYxxgwpIrIjlPOsickYY0xQliCMMcYEZQnCGGNMUJYgjDHGBBXRBCEi\n20VkrYisEpHlTlmBiLwiIluc3/lOuYjIvSJSJiJrRGR2JGMzxhjTu8GoQZytqseraqlzfDuwWFWn\nAoudY4CPAVOdn+uBBwYhNmOMMT2IRhPTJcAjzuNHgEu7lT+qPkuB4SIyOgrxGWOMIfLzIBR4WUQU\nmK+qC4AiVa1wnt8HFDmPxwK7ur12t1NW0a0MEbkeXw2DCRMmRDB0Y4yJHS2uLjZXNrOzto3tta3M\nm1HEMePyIvo3I50gTlPVPSIyCnhFRDZ2f1JV1UkeIXOSzAKA0tJS21DbGBOXWl1dvLWlhiUbq1ix\ns56t1S2o84knAoU56UM7QajqHud3lYg8A5wIVIrIaFWtcJqQqpzT9wDju718nFNmjDEJY9O+Zh5+\ndxvPrNxDR6eXYRkpnFhSwCePHcPRY3IpKcxmXH4mGanJEY8lYglCRLKBJFVtdh6fD/wEeB64BrjL\n+f2c85LngW+KyOPASUBjt6YoY4yJazUtLu7670b+9eFu0lOSuOS4sVxy/BhOmFRAanJ0ZiREsgZR\nBDwjIv6/809VXSQiHwBPish1wA7g8875LwIXAWVAG3BtBGMzxpiY8daWam55YjWN7W6+evpkvn7m\nEeRnp0U7rMglCFUtB44LUl4LzAtSrsCNkYrHGGNi0ZMf7OL2hWuYMiqHv3/lRGYU50Y7pIAhvZqr\nMcYMZY8t28kdC9dy+tRC5n9hDllpsfWRHFvRGGNMgnhrSzV3PvsRZ00fyfwvzCE9JfKdzv1lazEZ\nY+JSl8dLl8cb7TCCqm52cfPjq5g6Kof7rpwdk8kBLEEYY+JQu9vDqXe/xufmvxftUIK6Y+FaWlxd\n/N8Vs8hJj92GnNiNzBhjBuiBN7ZS2eSissmFquKMpowJr2+q4tUNldzxsRlMLRoW7XB6ZTUIY0xc\nqW52seDNrYHjFldXFKM5UJfHy89f2EDJiCyuPXVStMPpkyUIY0zcqG91c+avl9DR6eXmeVMBqGp2\nRTmq/V5YW8GWqha+e+EM0lJi/+M39iM0xpgQ3b1oI21uD2dPH8msCcMBaGjrjHJUPqrKgjfLmTwy\nmwuPLo52OCGxBGGMiQtt7i5eXFtBRmoSv798FnmZqQA0tcdGgnhvay3r9jZx/emTSUqKnT6R3lgn\ntTEmLjy3ai9NHV089bWTyctMDSSIxhhJEI9/sIu8zFQunTU22qGEzGoQxpi48K8Vu5leNIzSifkA\nMZUgmjo6eWndPi4+bsygrMIaLpYgjDFD1ivrKym5/QXW7G5g+Y56Lj5+TGBIa24MJYgX11Tg6vLy\n2Tnjoh1Kv1iCMMYMWXcsXAPAfa+VAfDJY8cEnktNTiInPSUmOqlfWFvB5MJsjo3wBj/hZgnCGDMk\nebxKTYsbgJfXV3LcuDwmjMg64Jy8zNSo1yCaOjpZWl7LeUcXxdSEvVBYgjDGDEkf7qw/4Pi8o4oO\nOSc3BhLEm5ur6fQo5x55aHyxzhKEMWZIenV95QHHZ04bdcg5eZkpUR/m+ur6SvKzUpk9IT+qcQyE\nJQhjzJD0ztYaSro1KR095tCNdqLdxKSqvLWlhjOnjSR5iMx96M4ShDFmyGls62Td3ibmdWu2CTb5\nLNoJoqyqhdpWNycfMSJqMRwOmyhnjBly3t9Wi6qv38HjVT49O/jks2gniKXltQDMnWwJwhhjBsXS\n8jrSU5KYNWF4rx++eZmptHd6cHd5o7I43tJtdRTnZjChIKvvk2OQNTEZY4acZdtrmTMxv8+d2KI5\nm1pVeb+8jrmTC4bc8FY/SxDGmCGlo9PDxopmjh8/vM9zozmbend9OzUtLkpLCgb9b4eLJQhjzJCy\nvqKJLq9y7Li+E0Q0axBrdjcCDLnZ091ZgjDGDClrdjUAcNz4vj94o7nk99o9jaQmC9OLY3tb0d5Y\ngjDGDClrdjcyclg6xbkZfZ4bzRrE2j0NzCjO7bOfJJZZgjDGDCmrdzdw3Li8kDp+o5UgVJW1uxs5\nZgg3L4ElCGPMENLm7qK8ppWZY0P74I1WJ/XOujaaOro4NsQ4Y5UlCGPMkFFW1YIqTC8KrV3fv+T3\nYCeIDRVNABwVZPmPocQShDFmyNhc2QLAtH50/EZjNvUWJ84po3IG9e+GmyUIY8yQsaWymbTkJCb2\nY2ZyNJb83lLVwrj8TLLShvZiFRFPECKSLCIrReQ/zvEkEXlfRMpE5AkRSXPK053jMuf5kkjHZowZ\nWjZVNjN5ZDYpyaF/dOVlptA4yLvKbalqYeoQrz3A4NQgbgY2dDu+G7hHVacA9cB1Tvl1QL1Tfo9z\nnjHGBGypbGFaiP0PfoPdxOTxKlur+x9nLIpoghCRccDHgT87xwKcAzztnPIIcKnz+BLnGOf5eTJU\nFzAxxoRdi6uLPQ3tTCvq3zfzwU4Qu+racHd5h3z/A0S+BvF74LuA1zkeATSoapdzvBvwr9M7FtgF\n4Dzf6JxvjDGUVfk6fqfGeA1iywDjjEURSxAi8gmgSlVXhPl9rxeR5SKyvLq6OpxvbYyJYTtqWwGY\nVJjdr9cNz0qjvdNDR6cnEmEdwp/IjhjZvzhjUSRrEKcCF4vIduBxfE1LfwCGi4i/a38csMd5vAcY\nD+A8nwfUHvymqrpAVUtVtXTkyJERDN8YE0t21LYB9HtvhYLsNADq29xhjymYnXWtFOakMSwjdVD+\nXiRFLEGo6h2qOk5VS4DLgddU9SpgCfBZ57RrgOecx887xzjPv6aqGqn4jDFDy47aNopy08lI7d/a\nRvlZvgRR1zpYCaJtyG4QdLBozIP4f8CtIlKGr4/hIaf8IWCEU34rcHsUYjPGxKidda1MLOh/s82I\nnMFNEDtqLUH0i6q+rqqfcB6Xq+qJqjpFVT+nqi6nvMM5nuI8Xz4YsRljhoYdtW1MGNH/D15/E5M/\nQeyqa+OYH73EamfZ8HDq9HjZ29CeWAlCRE4TkWudxyNFZFJkwzLGmP3a3R6qml2UDCRBOE1MtS2+\nBPFOWQ3Nri5+/+rmsMYIsLehHa/C+ERJECLyI3zNQnc4RanA3yMZlDHGdLezzumgHtH/Jqa8zFSS\nkyTQSV3vzKr2RKCHc6Ad6bEqlBrEp4CLgVYAVd0LDP0BvsaYIcM/xLU/azD5JSUJ+Vmp1DpNTLvr\nfR/iLR3hnxvhT2QTB5DIYlEoCcLtjCZSABGJjys3xgwZgRrEAL+ZF2SnUec0Me2qbwdgX2NHeILr\nZlddG2kpSYwalh72946GUBLEkyIyH9/8ha8CrwIPRjYsY4zZr6Kxg8zUZIZnDWxuQX5WWqCT2l+D\nqGx24fGGt51pZ10b4/IzSUqKj1WC+lyLVlV/IyLnAU3AdOCHqvpKxCMzxhjHvsYORudlhLTNaDAj\nctLYXNmCqrKvsYO05CTcHi+1LS5GhbC3daj2NnYwdnhm2N4v2npNECKSDLyqqmcDlhSMMVGxt7Gd\n0cMH/kFekJ1GbYuLpo4u2tweTp0ygnfKaimvaQ1rgtjX2M70ovhZ4aHXJiZV9QBeERnaG6saY4a0\nioYORucN/Jv5qGEZ1Ld1stMZZXTB0cUAfLizPizxAXR5vFQ3uygOY8KJtlC2O2oB1orIKzgjmQBU\n9aaIRWWMMY4uj5eqZl8T00AV5fo6jdfs8U2OO2p0LpNHZrNie/gSRHWLC69C8WEkslgTSoJY6PwY\nY8ygq2r2ffAeVg3C+Vbvnz1dnJfBkaNzWbenMSwxgq8jHTisRBZrQumkfsTZFnSaU7RJVQd3/z5j\nTMKqaPQNSz2cPoiiYf4E0YiIr8lp7PBMXllfiaoOuPO7O/+w2eI4ShChzKQ+C9gC3A/8EdgsImdE\nOC5jTIJ5a0t10L2j9zYc/jfzUU4T06bKZgpz0klLSWJMXgbuLm9gAt3hiscaRCjzIH4LnK+qZ6rq\nGcAF+PaMNsaYHjW0ubnrvxtD2qhn+fY6vvDQMu59bcshz+0LfPAOvImpICuN1GRx3sf3AT7aGY66\nt6F9wO/b3b7GdjJSk8jLHPr7QPiFkiBSVXWT/0BVN+Nbj8kYY3r00Nvb+NMbW3ls2c4+z91Q0QTs\nn8TW3d7GdrLTksnNCKXLNLikJGFcvm8Wtn+U0dhAggjPjOqKRt9Iq3A0V8WKUBLEchH5s4ic5fw8\nCCyPdGDGmKEtPcX38eJfwK431c0uAIJNbK5s6qDoMCbJ+fmX6fDXIMaEvQbREVdDXCG0BPF1YD1w\nk/Oz3ikzxpgeNbb7+hOaQlgUr7rFlyCCbepT0+xmZM7hr23k3xfCPww1PyuV9JSksCWIisaOuOqg\nhtCGuaYAf1DV30FgdnV8rERljImYGmdxvFAWxfPXIKqaDz23usXF0WNyDzse/7pL/pqEiDB2eGag\nc/lwqCrVLa64WaTPL5QEsRg4F9+EOYBM4GXglEgFZYwZ+vwf+qF8AAcSRJPrkGGnNc0uCsNQg/jO\nBdPJSktm3pGjAmVjhmeyJww1iGZXF+4ub1jijCWhNDFlqKo/OeA8jo/dMIwxEeP/0N/b0I5vx4Ce\n+Wsbri4vTR1dgfKOTg/Nri5GhuGb+fiCLO76zLFkpCYHykbnZYSlianGuVb//tfxIpQE0Sois/0H\nIjIHCE+jnTEmbvn7FVxd3sAubsGoKtXNrkDncXW3ZiZ/kglHH0QwY4ZnUt3iwt3lPaz38c+lSMQa\nxLeAp0TkLRF5G3gC+GZkwzLGDGWdHi91rW6OGOnbX6zGSRbBNLV34fZ4A/0MVU37z/W/rnBYZL6Z\nj87LQDV430d/+GsQCZcgVPUDYAa+kUtfA45U1RWRDswYM3T5RyPNGO370O8tQVS3+D6cjxrjWzS6\nqrl7gojsN3P/qKPD3V0u0oksWnpMECJygogUAzhrL80Gfg78VkQKBik+Y8wQ5G8aOrLYt319bUvP\ny1n4E0KgBhGsiSlCo4P8s7P3NR1egqhucSPim7EdT3qrQcwH3ADO2kt3AY8CjcCCyIdmjBmq/B/s\n04tDqEE4504uzCYjNSloE9OI7NivQeRnpZGSHEqr/dDR29Ukq2qd8/gyYIGq/ktVfwBMiXxoxphY\noqos21ZHl6fvDl3/h/7UUTkkSe81CP8mPqOGZeBV+PPb2wKjnmpaXORlppKWEpkP3tyMFDJTkw97\nLkRti4vCOBvBBH0kCBHxz5OYB7zW7bmBL4pijBmSNlQ08/n57/Htp1b3ea5/BFNRbgYF2enUtvZc\ng/jtK5sByM1M4ZQjRgBww99W8Lf3trO9ti1izUvgmyw3Oi8jDDUId9x1UEPvCeIx4A0ReQ7fsNa3\nAERkCr5mJmNMAvHPF3h21d4+z61q6mBYegqZackU5qQFOpsP5nVmN6enJCEiXHNKCQAvr6/kB8+t\n483N1ZRVtQR9bbgU52UE9pwYqJqW8EzmizU91gRU9ecishgYDbys+2e6JAH/MxjBGWNiR3W3foQu\nj7fX9vZ9zgJ74Js8VttDH4S/j+HOTxwFwMSCwZ+DW5yXwfvldX2f2IuaZlfcTZKDPpqKVHVpkLLN\nkQvHGBOrqrsNP61qdgVWQw2msskVWNl0RHY6q+oagp633lnme9II33wJ/5Lc3b313bMHHHMoRudl\nUNnUgcerJCf1f8XYdreHVrcnLmsQ8dXlboyJmO4jkfpqkqls6gjs4tZbDeLZlXvITE1mzsR8gAM6\no8+YNpJLjh/D+AjXKopzM+jyao8x9iUwByLRahDGGOPXvQaxt6GDORODn+f1KlXN+2sQhTnptLo9\ntLs9ZKYlH3DuR3ubOH1q4QHlv/rssazd3chPL50Z/osIwr/8d0VjB6MGsJ9Dg7OMSH6czYGAEGsQ\nIjJRRM51HmeKyLAQXpMhIstEZLWIrBORHzvlk0TkfREpE5EnRCTNKU93jsuc50sGflnGmHCrbnYx\nc6xvXkNvC9zVtLrweDUwx8D/zTrYSKb6VjeFB41S+nzp+EFLDrB/A6GBDnVtaPd1wA9PxAQhIl8F\nnsY3cQ5gHPBsCO/tAs5R1eOA44ELRWQucDdwj6pOAeqB65zzrwPqnfJ7nPOMMTGirs3NxIJsctJT\nevww/ePrZXz36TWAb4gr7J/kdvBcCK9XqW9zR332sT+RVQ5wNnV9oAYRfzsxh1KDuBE4FWgCUNUt\nwKheX+E7T7stE57q/ChwDr6EA/AIcKnz+BLnGOf5eRJPm7saM8Q1tnUyPCu1xyWyuzxefrVoE69v\nqga6JQinBnHwbOptta14FfKzo5sgCrLSSEtOGnANorHNl/jyEjRBuFQ1kPqdyXO9L+6+/9xkEVkF\nVAGvAFuBBlX1L/i+GxjrPB4L7AJwnm8ERoTyd4wxkaWqNLQ7CaKHXdgOXtLb3wfhb9fvvggfwGsb\nqgA4e/rISIQcsqQkYVRuOvsGOBfCf93DMxOwiQnfZLnvAZkich7wFPDvUN5cVT2qejy+ZqkT8a0K\ne1hE5HoRWS4iy6urqw/37YwxIWhxdeHxKsMz0xjTw8Syg/sY/DOg/Xs5HNyEs722lbzMVCaPzIlQ\n1KEbnZcx8D6Itk6y05IjthxINIVyRbcD1cBa4AbgReDO/vwRVW0AlgAnA8O7LeExDtjjPN4DjIdA\nLSUPqA3yXgtUtVRVS0eOjO43D2MShX+kTl5WKqPzMqlpcePq8hxwTk3zgX0M/jkFaSlJFOakUdl0\nYALZ09DO+IKe51IMpuK8zAGv6NrQ5o7LDmoIYZirqnqBB52fkInISKBTVRtEJBM4D1/H8xLgs8Dj\nwDXAc85LnneO33Oef63b7G1jTBQ1tvubUVLxdwzubehgUmF24Bx/DeLb509jyqgDBzqOGpZxSA2i\nvq2Tggit0tpfo/MyeHldxyH7YYeiob2T/Oz463+AEBKEiKzl0D6HRmA58DNVPeRbvmM08IiIJOOr\nqTypqv8RkfXA4yLyM2Al8JBz/kPA30SkDKgDLu/31RhjIsJfgxielUaB06m8rablgAThnydx9dyJ\nh3yjLspNPyBBlFW1sHpXAxcfNybSoYekODcDV5eXhrbOfnea17e547L/AUKbKPdfwAP80zm+HMgC\n9gEPA58M9iJVXQPMClJejq8/4uDyDuBzoQRtjBlc+8f6pwb6FMqrWzmnW69ibaublCQhL/PQb9PF\neRms3dMUOP7FixsA2Fod2YX4QuVfNmRXfVu/E0RjWydje1l2ZCgLpQ/iXFW9Q1XXOj/fB85U1buB\nksiGZ4yJBYEaRGYq+dlp5Gel8tK6fQf0Q9S2+BasC9ZEM2pYBrWtLjqdvSQanKGhE6KwOF8wJYW+\nOHY4e1P0R32bm+FxOMQVQksQySIS+MYvIicA/nnxXcFfYoyJJ/4+iFynduDxKh9sr2f6nYv4/Pz3\nWLmzvtc9EYrzMlDd3wwlIuRmpHD3Z48dnAvow8QCX1PZ9prWfr3O61Ua2zvjcpkNCC1BfAV4SES2\nich2fH0FXxWRbOCXkQzOGBMbGtrcZKYmk5Hq+25412f2f7Av21bHLU+soqbFxYgeEkSRs3Cff6RQ\nY3snp00tJDcjNr55Z6YlU5ybwfZ+1iCaO7rwKkGb1eJBKKOYPgCOEZE857j7ZkFPRiowY0zsaHBm\nUftddMxoVv/ofJ5fvRcB7nz2IwA+PWts0Nf7l/HeXd/O7An5NLR1xtyH6sQRWWyv7V8Not5pKovX\nGkRIq7mKyMeBo4EMf/uiqv4kgnEZY2JIY/uhH+h5mal8Ye5Emjo6AwliSlHwSW/jnQSxq64NVaWp\nvTPQXBUrJhVm8+qGyn69psE//DdR+yBE5E/AZfh2kRN8I416WOjXGBOPGoIkCL/cjFSuOHECSQKn\nTwk+edW39Wg6O2vb6Oj04vZ4Y25o6MQR2dS0uGnu6Oz7ZIe/BhGvE+VC6YM4RVW/iG+l1R/jmw09\nLbJhGWNiSeNBTUwH+9mlM1l6xzyOGZfX4zkTCjLZWdcWGDIba01MJSP6HsnU2NbJXf/dSLvbN3qr\nIdDEFFvXEi6hJAj/7JY2ERkDdOKbBGeMSRAN7b1PBktOkj4325lUmMOWqubAiKiYSxDOpL/yXkYy\nPbViF396YytH/nARri5PYHmRg/e0iBehJIh/i8hw4NfAh8B29k+aM8YkgIM7qQdizsR8alrcLNno\nW2TTvw9DrJg8MpvUZGFDRVOP53Tf+e6dshqqW1ykpyQxLD0+N+fs9apEJAlY7Cy29y8R+Q+QcdBI\nJmNMHOvo9ODq8h52p/I0pwP7qRW7yExN5rhemqOiIT0lmWlFw/hoT88fby0d+6d+Ld5QRUN7JyOH\npfd7/aahotcahLNQ3/3djl2WHIxJLM3Oh2JuxuF9Sw404VS3MjY/k5Tk2Fsee+aYPD7a00hP64Q2\ndXSSkiRccHQRr6yv5M3N1Zw0KX63rQnlDi0Wkc/Y7m7GJCb/qJ5hhzmprTAnnVFOW/3oGGte8ist\nyae+rZON+5qDPl/f1smwjBTmHVlEVbOL5o4uTpua2AniBnybBLlFpElEmkWk50Y6Y0xc8dcghh1m\nDQLghjOPAHwrvsai06f6hum+teXAzci6PF4u+sNb/PP9nUwrGsY5M/bvujxnQsGgxjiYQplJPayv\nc4wx8avF5UsQOWHoiL32lBI+O3tczO7fXJyXwbSiHN7cXMP1ZxwRKN9Q0cx6p/P62xdMpzAnnU/N\nGkt5dUvMbHoUCaHsByHAVcAkVf2piIwHRqvqsohHZ4yJunA1MYFv/+dYTQ5+844sYsGb5VQ0tjM6\nz/fhv6PON/T1xZtO56gxuQDcc9nxUYtxsITSxPRHfJPjrnSOW+jWcW2MiW/hbGIaCj43Zxwer/Lq\n+v3LblQ0+KaDjc2P39pCMKEkiJNU9UacCXOqWg/E57xyY8whEi1BTB6ZQ8mILBZvrAqUNXd0IkLc\nznfoSSgJotPZNlQhsNe0N6JRGWNihj9BZCfQh+PZM0bx3tbawJIaTR1d5KSnkJSUWIM5Q0kQ9wLP\nAKNE5OfA28AvIhqVMSZmtLg6yUxNJjUG5y1EyrwZRbi6vLxdVgP4kmSs7F0xmEIZxfQPEVkBzMO3\nmuulqroh4pEZY2JCc0cXOQnSvOR34qQCRmSn8cQHOznvqCJaXJ0J08TWXSijmO4FHldV65g2JgE1\nu7oS7sMxLSWJK06cwP2vl7Gzto36ts6ErEGEUmdcAdwpIltF5DciUhrpoIwxsaO5oyssQ1yHmqvm\nTiBJhD++Xsa2mlYmOsuBJ5I+E4SqPqKqFwEnAJuAu0VkS8QjM8bEhOaOzoQbvQMwOi+Ta08p4fEP\ndlHd7GJ6ceLNGe5Pr9MUYAa+3eQ2RiYcY0ysaelIvCYmvzsuOpJPHDuatOQkzj2yKNrhDLpQ+iB+\nBXwK2Ao8AfzUWf7bGJMAmp0hnokoOUm478rZtLs9B+wFkShCuetbgZNVtSbSwRhjYk+LKzH7ILpL\nxOQAofVBzPcnBxE5QkR+ICLrIh+aMSbaPF6lxZV4w1yNT58JQkTGiMgtIvIBsM55zeURj8wYE3Wt\n7vBsFmSGph4ThIhcLyJLgNeBEcB1QIWq/lhV1w5SfMaYKPIvs5GofRCJrre7fh/wHnClqi4HEJHg\n+/AZY+JSOJf6NkNPbwliNPA54LciUgw8Cdh/JcYkkBZ/DcKamBJSj01Mqlqrqn9S1TPxrcPUAFSK\nyAYR6XOxPhEZLyJLRGS9iKwTkZud8gIReUVEtji/851yEZF7RaRMRNaIyOwwXaMxZoDCuZucGXpC\nmiinqrtV9beqWgpcgrM3RB+6gNtU9ShgLnCjiBwF3A4sVtWpwGLnGOBjwFTn53rggX5diTEm7PzL\nXWcl6DDPRNfv9XtVdbOq/iSE8ypU9UPncTOwARiLL8E84pz2CHCp8/gS4FH1WQoMF5HR/Y3PGBM+\nrU6CyE6zGkQiGpQF3kWkBJgFvA8UqWqF89Q+wD9/fSywq9vLdjtlxpgoaXOGuWalWw0iEUU8QYhI\nDvAv4Fuq2tT9OVVVnJ3q+vF+14vIchFZXl1dHcZIjTEHa3VZDSKRhbIWU7DO4kZgh6p29fHaVHzJ\n4R+qutAprhSR0apa4TQh+Td+3QOM7/bycU7ZAVR1AbAAoLS01IbdGhNBbe4uRCAjNXF2kzP7hXLX\n/wgsxfeh/CC+uRFPAZtE5PyeXiQiAjwEbFDV33V76nngGufxNcBz3cq/6Ixmmgs0dmuKMsZEQavL\nQ3ZaCr7/nU2iCSVB7AVmqWqpqs7B15dQDpwH/KqX150KfAE4R0RWOT8XAXcB5zl7SpzrHAO86Lxv\nGb5E9I2BXJAxJnza3F02gimBhdKwOE1VA4vzqep6EZmhquW9fatQ1bfx7WEdzLwg5ytwYwjxGGMG\nSavbQ7bNgUhYodz5dSLyAPC4c3wZsF5E0oHOiEVmjIm6NpfVIBJZKE1MX8LX7PMt56fcKesEzo5U\nYMaY6Gtze2wEUwLr886rajvwW+fnYC1hj8gYEzPa3F0Mz0qLdhgmSkLZD+JUZ82kzSJS7v8ZjOCM\nMdHl64OwJqZEFUrd8SHgFmAF4IlsOMaYWOLrg7AmpkQVyp1vVNX/RjwSY0zMaXV7yLZO6oQVSoJY\nIiK/BhYCLn+hfyE+Y0z8anN3kWXDXBNWKHf+JOd3abcyBc4JfzjGmFjh7vLS6VGrQSSwUEYx2VBW\nYxJQYCVX64NIWD3eeRG5WlXu5Vs2AAAXhElEQVT/LiK3Bnv+oPWVjDFxJrAXhI1iSli9fTXIdn4P\nG4xAjDGxpd1qEAmvxzuvqvOdh39UVdt4wZgE498LwpbaSFyhLLXxjoi8LCLXiUh+xCMyxsSEVqtB\nJLw+E4SqTgPuBI4GVojIf0Tk6ohHZoyJqjaX9UEkupC2iVLVZap6K3AiUAc8EtGojDFRZzUIE8pa\nTLkico2I/Bd4F6jAlyiMMXGszUYxJbxQvhqsBp4FfqKq70U4HmNMjKhs6gCsBpHIQrnzk1VVRSRH\nRHJU1Zb4NiYBvLyukimjcsjNsASRqELpgzhaRFYC6/DtJLdCRGZGOC5jTBR1dHrYVNnMhUcX09vW\nwia+hZIgFgC3qupEVZ0A3OaUGWPi1IaKJjxe5ZhxedEOxURRKAkiW1WX+A9U9XX2z7I2xsSh5dvr\nATjWEkRCC6VxsVxEfgD8zTm+Gt++1MaYOLStppWfv7gBgOLcjChHY6IplBrEl4GR+PaDWOg8/nIk\ngzLGRM+f3/J9/7vr08dY/0OCC2W573rgpkGIxRgTAyqbXMwoHsblJ06Idigmynpb7vv53l6oqheH\nPxxjTLTVtboYkZMW7TBMDOitBnEysAt4DHgfsLqmMQmgrtXNMfnDox2GiQG99UEUA98DZgJ/AM4D\nalT1DVV9YzCCM8YMLneXl70NHYzJs85p00uCUFWPqi5S1WuAuUAZ8LqIfHPQojPGDKq3y6pxe7wc\nO85qEKaPTmoRSQc+DlwBlAD3As9EPixjzGDbXNnM7f9aS3FuBvOOHBXtcEwM6K2T+lF8zUsvAj9W\n1Y8GLSpjTI8WvLmVzZUt/OZzx4X1fa/96wdUNbu44czJZKTaCq6m9xrE1UArcDNwU7fx0AKoquZG\nODZjzEHKqpr5xYsbATh7+ig+fuzow35PVeWD7fU0tXcC8NXTJx/2e5r40FsfRJKqDnN+crv9DAsl\nOYjIX0SkSkQ+6lZWICKviMgW53e+Uy4icq+IlInIGhGZHZ7LMya+PPrejsDjuxdtpMvjPez3fG1j\nFZ+f/x7Nri5uOHMyhTnph/2eJj6EtKPcAD0MXHhQ2e3AYlWdCix2jgE+Bkx1fq4HHohgXMYMWY3t\nnZSMyOJPV89hZ10bc3+5GI9XD+s9G9o6A48nF9oya2a/iCUIVX0T3/ak3V3C/u1KHwEu7Vb+qPos\nBYaLyOHXnY2Jsne31lDX6g7b+7V0dJGTkcIFRxcxo3gYNS1u/vbe9sN6T398+VmpnD3DOqfNfpGs\nQQRTpKoVzuN9QJHzeCy+SXl+u52yQ4jI9SKyXESWV1dXRy5SYw5Tl8fLlQ++z+yfvsLa3Y1hec9m\nVxc56SmICE9+7WQA/vff6/nv2oo+XnkgV5eHf63YTbvbE1iYb8Wd5zFqmM1/MPsNdoIIUFUF+l03\nVtUFqlqqqqUjR46MQGTGhEdD+/6mm0/e9/ZhNwWBU4NITwUgNyOVv37pBAB+9sKGfr3/v1bs4ban\nVnPLE6sCZUlJtliCOdBgJ4hKf9OR87vKKd8DjO923jinzAyCd8tqeG9rLUs2VeHt5UNmV10b+xo7\nBjGyoa3+oKalc3/3Rq//vn1xdXnYXtvKsG5bgJ49YxT3XTmLPQ3tvLkltBr1kk1VfO+ZtQAsWrcP\ngP934YwBx2Xi12AniOeBa5zH1wDPdSv/ojOaaS7Q2K0pyoTZjtpWLrjnTS6+720eX7aTK//8Plc8\nuJRr//oBj32wM+hrGtrcnP6rJcz95WKu+cuysIyeiXfrK5oA+NPVcxiXn8m2mlbeLqsZ8Put3tVI\nm9vDKUeMOKD8/KOKKchO46nlu3p45YGu/esHAFx+wv7vZEW5NnLJHCpiCUJEHgPeA6aLyG4RuQ64\nCzhPRLYA5zrH4JuMV45vOY8HgW9EKq5E4eryHFK2elcDtz25mjN//TqbKptZs7uR2xeuPeCcHz+/\nnhfWVNDp8fLgm+VsrW4BYEdtW+CcNzZX87elO0hkHZ0ern90OQ+9vQ1fa+mh3nGSwcyxuSy+7Uzy\ns1J5vIcEHIo9Db57MHti/gHlaSlJfGrWWF5cu49nVu4+5HU7alt5e4svliUbqwLlt5w3jX99/WTu\n/PiRXDizeMBxmfgVyo5yA6KqV/Tw1Lwg5ypwY6RiSST/fH9noPkAfCNTTigpYMzwTB5+d3ug/Cun\nTeKCmcV87k/vcfrUQr50Sgn52Wnc/PhKbvznh4Hzfv7iBi4rHc8bm33NF3+77kS+8NAy5r9Rzmfm\njCM3I3XQrq2/9ja0s7ehndKSgrC/98Z9zby8vpKX11eSmZrMlScdundCY3snU0blMC4/C4DPzB7H\nw+9up6qpg1ED2KltT307AGPyMg957ooTx/PQ29u45YnVZKamUFKYRVZqCnct2sCLa33NSC996wwW\nrvS13C69Yx5FuRkU5WYwZ2L4/31MfIhYgjCDr67VfUByAKhv6+Tl9ZUHlD1+/VzmTvY1U2y/6+MH\nPPeP6+Zyxq+XHFD2hNN0MXdyASdPHsHD157Al/76Aaf88jVW/vA8UpOjNtahR7vq2jjvnjfo6PRy\nzNg8vnxaCZ+aNS5s7//W5v3t/T/5zzrG5WdyxrT9gybW7m6kvrWT/Kz9CfTquRP5yzvbePCtcr7/\n8aP6/Te3VrcyJi+DzLRDl8GYMmoYi751Ohf+/i2+9vcVQV///WfWUtXs4mMziym21VpNCGLv/2wz\nYB/u8G00P3Z4Jq9/+yyeu/FUfn/Z8Rw7Lo+M1CROn1pI+S8uCiSHYCaMyGL7XR/ntdvOZOUPzmPB\nF+YwrSiH5248lcevP5mU5CTOmj6KORPzaXF18fqm2Bxq/O81e+no9PWTrN3TyC1PrOamx1aytLyW\nF9ZU9NgsFKrFTlPNq7eeQWpSEjf+80OaOnyjllbsqOOT973Nsu11B9SwSgqzufT4sTz41jY+2hP6\nsNd1exvZWdvGu1trmF48rMfzZhTn8tK3zjik/JlvnMJvP3ccy3fUs7OujalFPb+HMd3J4f6PEk2l\npaW6fPnyaIcRMy6b/x7vb6tj2ffmHdKE0enxkiwStqGMnR4vJ/9yMUW5Gdz58aP44XMfcfrUkRw3\nPo8jR+cyLcofQpfe/w6qysJvnMoLayu46bGVBzx/YkkBj153Yr8Xpatq6uCpFbv59UubuOLE8fzy\n08eyZncDF9/3DukpSXz8mNFsq21l5c4GAM6YNpJHv3xi4PVbq1uY99s3KM7N4NXbzqS5o5PRQZqM\nnlu1h4kjsslOS+b837+J/3/Thd84hdkT8g85v7uX1+0jOUlo6uhkS2UL33VGKN34jw9ZtG4fz3/z\nVI4ek9ev6zbxRURWqGppX+dZE1Oc2FrdwvvbfBPXg62lE+5moNRkX8fog29t44oHlwKwpaol8PyN\nZx/Bdy6IztDJp5bvYtWuBr59/jSSk4SLjxtDu7uL7z/zEfOOHMVL6ypZtr2OW55YxQNXz+nXe//f\na2WBDvqjnA/ZY8cN56qTJvDcqr2BNv6TJ49gVG465x1VdMDrjxiZw4NfLOWrjy5n5o9eQgTuv3I2\nFx2zf+EAd5eXmx/3zU/ITE0OJIc5E/P7TA4A5x8dvMP5/66YRW2rm5HDbMSSCY0liDjx8xd8s2H/\n8ZWTBm3C083nTmP5jnpW7mzgj1fNprbVzQ+e9a3NeP+Sraze1cj9V80mLzMyHdmrdzWwcmc915xS\ngojw6vpK/m9JGat3NXDEyGyuOaUkcO5lJ0zgshN8HckdnR7uXrSRv76znWdW7u5X30R5jS8JXlY6\nnk/P2j/Z/+efOoaff+oYVu6s5ztPr+G60yZx7kHJwe/cI0dx5OhcNlQ0oQrf+MeHfO3MIyjMSePs\nGaNI65bM2zt9o9FW/+h8kg/zviYliSUH0y/WxDSE1bW6efCtcp5esZvqZhe3nDuNm8+dOuhxdHq8\nB9RQVu1q4MoHl9Lm9lCUm87i284iJz2830VUlWl3/pdOj/LzT83kqpMmcsWCpbxXXktuRgoLvlja\na19Lu9vDtQ8vY8WOeh6/fm5II3mqm12c+7s3OGfGKO657PjDin9XXRtl1S2kJydx5Z/fP+C5S48f\nw7Or9vLwtSfw4c4GUpKEm+YN/n018cuamOKMquLxKh5Vvv73DxmXn3nA0s856SnccGZ01vE/uPnq\n+PHD+fAH5/H4sp3877/Xc/+SsrDP1N1V106nx/fl5ofPrWNXXTvLd9TxsZnFITUbZaYlM/8Lpcz7\n7et85oH3eOYbpzD/jXJOm1rI1XMnBn3ND579iI5OT1j+nccXZDG+wDf89eq5E/j70v3zI55dtZfh\nWamUlhRw1nRbPM9EjyWIIeDdshruXrSRDRXNuA+awXzukaO478rZuD3emNoFLCM1mS+dOom1e5p4\n6K1tfL50PJPCuJR0RaNvTsCfrp7Dn98q509vbAXgqpOCf7gHk5eZyn1Xzub6R5dz1Z/fp83tYdG6\nfZxyxAgmFWbj8SopTvLr6PTw2qYqrjppAjOKw7tX1s8uPYbvXDCD5o5OkkT41aKNnHNkUdhrXcb0\nl/0XGIM8XuVLf12GiHDO9JH877/XH/D82OGZPPjFUvY2tDP3iBFkpCbHVHLo7v9dOJ2X1u3j7N+8\nzrfOncoNZxwRdBx/f+1r8q0JdcTIbJ7++iksLa+lrtXNaVML+/U+cyeP4MeXHM0tT6wOlN321Gom\njcjmjc3V3Hb+dN7cXE15TQvuLi+n9/P9Q5WXmRroq/n95bMi8jeM6S/rg4gxXq9y8l2LqWxyHVD+\nly+VMnNMHsMyUklOEtJShs4Ulgde38rdizYGjk+bUshvP38cRQOYTez3P4+t5OV1+/jwB+eRHYZv\n2os+2kdlUwf52Wnc+sQquoIsqjdxRBb/vfl0stLse5UZ2qwPYoi69clVgeTwh8uP577XyvjYMaM5\nZ0bwETFDwXWnTaKj00Obu4ul5XW8XVbDp+5/hyduODnQDh+M16tBR2RtrW7h36v38unZY8OSHIAD\n1iIanpnKPa9uZtb4fJ5btYefXjqTc2aMIjU56bBHEhkzlFgNIoa8uLaCb/zjQ4ZlpLD6h+fH7fr8\ny7bV8ZVHPmDSyBye/trJgU7uLo+XDRXNzByby8IP93D7wjUcPSaPguw0fviJo6hucbFqZwPz3yyn\npsXFw9eeEPFOXFVFJD7vg0lcodYgLEHECI9XOes3S9hV186rt57BlFHxvRzCwg93c+uTq5kyKoer\nT5rAEaNyWL69nj8s3sLkwmzKa1p7ff2TN5zMiZNskTljBsKamIaYpeW17Kpr557Ljov75ADw6dnj\nyEhN5ltPrDqkE357bSvZacn84fJZ1LW6yctK5VeLNpKTnsIRI3O47YLpjB1+6PIUxpjwsgQRIxZ+\nuIdh6Sl8bObovk+OExcdM5pjxubx2sYq2js91DS7+MrpkwOb13Rv2rmgh+UjjDGRYwkiynbVtbGl\nqplFH1Vw0TGjY3a4aqSML8g6YEkMY0zssAQxiCoa27lj4Vq2Vrfg8She3T+eH+CyE8b38mpjjBlc\nliAG0W1PrubdrbUHlJ0+tZAzp41kdogrdRpjzGCxBDFINlc2B5LD8988lWEZqYzPzwws5WCMMbHG\nEkSEubu8/OblTSx4sxyAey47jmPHDY9yVMYY0zdLEBH2ixc38PC72wH48cVHh3VfZGOMiSRLEBHk\n9Sqvb/LtXfzQNaXMO3LoLpdhjEk8liAiYGt1C8+u3MP8N8pxe7x8/6IjLTkYY4YcSxAh8HiVsqoW\nxuVn9rk43HtbawN7NIOvWamnDWiMMSaWWYIIwa9e2sj8N8pJSRIumFnMaVMKueDoYpZtq2N3fRvj\n8rO4d/EWyqp9ewYA3HvFLGYUD2NaUfwvm2GMiU+WIPqgqryyrpIR2WmcUFLAW5ureWFNBXcsXHvI\nuTOKh5GdnsLx44dz8XFjohCtMcaEjyWIIOpa3aze1cDClXt4e0s19W2d/OZzx/HZOePo9Hj5YFsd\n898sp77NzY8vPprKpg6y0lI4Y9rIaIdujDFhYwmimz0N7dy/pIzHlu2k+yro/3POFD49aywAqclJ\nnDKlkFOmRGbrSWOMiRWWIBw1LS6uenAp22vbOG1KIZ8/YTwnTx7BsIyUhFtAzxhjwBIEAO+X13Ld\nI8tpcXXxjbOO4LsXzoh2SMYYE3UJnyD+vXov31u4FlXlZ5fO5KqTJkQ7JGOMiQkxtVKciFwoIptE\npExEbo/033t6xW5ufnwlJYXZvHzrmVw9d6LtP2yMMY6YqUGISDJwP3AesBv4QESeV9X1vb9yYBZ9\ntI/vPL2aY8fm8c+vzu1zApwxxiSaWKpBnAiUqWq5qrqBx4FLIvGHnvxgFzc9tpLjxw+35GCMMT2I\npQQxFtjV7Xi3U3YAEbleRJaLyPLq6uoB/aEJI7I4/+giHrrmBEsOxhjTgyH36aiqC4AFAKWlpdrH\n6UHNnTyCuZNHhDUuY4yJN7FUg9gDdN+UeZxTZowxJgpiKUF8AEwVkUkikgZcDjwf5ZiMMSZhxUwT\nk6p2icg3gZeAZOAvqrouymEZY0zCipkEAaCqLwIvRjsOY4wxsdXEZIwxJoZYgjDGGBOUJQhjjDFB\nWYIwxhgTlKgOaK5ZTBCRamDHAF9eCNSEMZyhwK45Mdg1J4bDueaJqtrnFphDOkEcDhFZrqql0Y5j\nMNk1Jwa75sQwGNdsTUzGGGOCsgRhjDEmqEROEAuiHUAU2DUnBrvmxBDxa07YPghjjDG9S+QahDHG\nmF4kZIIY7L2vB4uIjBeRJSKyXkTWicjNTnmBiLwiIluc3/lOuYjIvc6/wxoRmR3dKxgYEUkWkZUi\n8h/neJKIvO9c1xPO6sCISLpzXOY8XxLNuAdKRIaLyNMislFENojIyQlwj29x/pv+SEQeE5GMeLzP\nIvIXEakSkY+6lfX73orINc75W0TkmoHGk3AJotve1x8DjgKuEJGjohtV2HQBt6nqUcBc4Ebn2m4H\nFqvqVGCxcwy+f4Opzs/1wAODH3JY3Axs6HZ8N3CPqk4B6oHrnPLrgHqn/B7nvKHoD8AiVZ0BHIfv\n2uP2HovIWOAmoFRVZ+Jb7fly4vM+PwxceFBZv+6tiBQAPwJOwreV84/8SaXfVDWhfoCTgZe6Hd8B\n3BHtuCJ0rc8B5wGbgNFO2Whgk/N4PnBFt/MD5w2VH3wbSy0GzgH+Awi+yUMpB99vfEvJn+w8TnHO\nk2hfQz+vNw/YdnDccX6P/dsRFzj37T/ABfF6n4ES4KOB3lvgCmB+t/IDzuvPT8LVIAhx7+uhzqlW\nzwLeB4pUtcJ5ah9Q5DyOh3+L3wPfBbzO8QigQVW7nOPu1xS4Xuf5Ruf8oWQSUA381WlW+7OIZBPH\n91hV9wC/AXYCFfju2wri+z531997G7Z7nogJIu6JSA7wL+BbqtrU/Tn1faWIi6FrIvIJoEpVV0Q7\nlkGUAswGHlDVWUAr+5scgPi6xwBO88gl+JLjGCCbQ5thEsJg39tETBBxvfe1iKTiSw7/UNWFTnGl\niIx2nh8NVDnlQ/3f4lTgYhHZDjyOr5npD8BwEfFvhtX9mgLX6zyfB9QOZsBhsBvYrarvO8dP40sY\n8XqPAc4Ftqlqtap2Agvx3ft4vs/d9ffehu2eJ2KCiNu9r0VEgIeADar6u25PPQ/4RzJcg69vwl/+\nRWc0xFygsVtVNuap6h2qOk5VS/Ddx9dU9SpgCfBZ57SDr9f/7/BZ5/wh9U1bVfcBu0RkulM0D1hP\nnN5jx05grohkOf+N+685bu/zQfp7b18CzheRfKf2db5T1n/R7pCJUifQRcBmYCvw/WjHE8brOg1f\n9XMNsMr5uQhf++tiYAvwKlDgnC/4RnRtBdbiGyUS9esY4LWfBfzHeTwZWAaUAU8B6U55hnNc5jw/\nOdpxD/BajweWO/f5WSA/3u8x8GNgI/AR8DcgPR7vM/AYvn6WTny1xesGcm+BLzvXXwZcO9B4bCa1\nMcaYoBKxickYY0wILEEYY4wJyhKEMcaYoCxBGGOMCcoShDHGmKAsQRhjjAnKEoQxxpigLEEYY4wJ\n6v8D8q+cr95nlrMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f36d37b1e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(mva_lst)\n",
    "plt.ylabel('Moving Average Score')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('a2c-mva-2.npy', np.array(mva_lst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}