Curiosity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Curiosity-Driven Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Actor(nn.Module):\n",
    "    def __init__(self, n_actions, space_dims, hidden_dims):\n",
    "        super(Actor, self).__init__()\n",
    "        self.feature_extractor = nn.Sequential(\n",
    "            nn.Linear(space_dims, hidden_dims),\n",
    "            nn.ReLU(True),\n",
    "        )\n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(hidden_dims, n_actions),\n",
    "            nn.Softmax(dim=-1),\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        features = self.feature_extractor(x)\n",
    "        policy = self.actor(features)\n",
    "        return policy\n",
    "    \n",
    "class Critic(nn.Module):\n",
    "    def __init__(self, space_dims, hidden_dims):\n",
    "        super(Critic, self).__init__()\n",
    "        self.feature_extractor = nn.Sequential(\n",
    "            nn.Linear(space_dims, hidden_dims),\n",
    "            nn.ReLU(True),\n",
    "        )\n",
    "        self.critic = nn.Linear(hidden_dims, 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        features = self.feature_extractor(x)\n",
    "        est_reward = self.critic(features)\n",
    "        return est_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class InverseModel(nn.Module):\n",
    "    def __init__(self, n_actions, hidden_dims):\n",
    "        super(InverseModel, self).__init__()\n",
    "        self.fc = nn.Linear(hidden_dims*2, n_actions)\n",
    "        \n",
    "    def forward(self, features):\n",
    "        features = features.view(1, -1) # (1, hidden_dims)\n",
    "        action = self.fc(features) # (1, n_actions)\n",
    "        return action\n",
    "\n",
    "class ForwardModel(nn.Module):\n",
    "    def __init__(self, n_actions, hidden_dims):\n",
    "        super(ForwardModel, self).__init__()\n",
    "        self.fc = nn.Linear(hidden_dims+n_actions, hidden_dims)\n",
    "        self.eye = torch.eye(n_actions)\n",
    "        \n",
    "    def forward(self, action, features):\n",
    "        x = torch.cat([self.eye[action], features], dim=-1) # (1, n_actions+hidden_dims)\n",
    "        features = self.fc(x) # (1, hidden_dims)\n",
    "        return features\n",
    "\n",
    "class FeatureExtractor(nn.Module):\n",
    "    def __init__(self, space_dims, hidden_dims):\n",
    "        super(FeatureExtractor, self).__init__()\n",
    "        self.fc = nn.Linear(space_dims, hidden_dims)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        y = torch.tanh(self.fc(x))\n",
    "        return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class PGLoss(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(PGLoss, self).__init__()\n",
    "    \n",
    "    def forward(self, action_prob, reward):\n",
    "        loss = -torch.mean(torch.log(action_prob+1e-6)*reward)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def select_action(policy):\n",
    "    return np.random.choice(len(policy), 1, p=policy)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def to_tensor(x, dtype=None):\n",
    "    return torch.tensor(x, dtype=dtype).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ConfigArgs:\n",
    "    beta = 0.2\n",
    "    lamda = 0.1\n",
    "    eta = 100.0 # scale factor for intrinsic reward\n",
    "    discounted_factor = 0.99\n",
    "    lr_critic = 0.005\n",
    "    lr_actor = 0.001\n",
    "    lr_icm = 0.001\n",
    "    max_eps = 1000\n",
    "    sparse_mode = True\n",
    "\n",
    "args = ConfigArgs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Actor Critic\n",
    "actor = Actor(n_actions=env.action_space.n, space_dims=4, hidden_dims=32)\n",
    "critic = Critic(space_dims=4, hidden_dims=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ICM\n",
    "feature_extractor = FeatureExtractor(env.observation_space.shape[0], 32)\n",
    "forward_model = ForwardModel(env.action_space.n, 32)\n",
    "inverse_model = InverseModel(env.action_space.n, 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Actor Critic\n",
    "a_optim = torch.optim.Adam(actor.parameters(), lr=args.lr_actor)\n",
    "c_optim = torch.optim.Adam(critic.parameters(), lr=args.lr_critic)\n",
    "\n",
    "# ICM\n",
    "icm_params = list(feature_extractor.parameters()) + list(forward_model.parameters()) + list(inverse_model.parameters())\n",
    "icm_optim = torch.optim.Adam(icm_params, lr=args.lr_icm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pg_loss = PGLoss()\n",
    "mse_loss = nn.MSELoss()\n",
    "xe_loss = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 1, AVG Score: 1.350, Score: 27.0, AVG intrinsic reward: 11.123327\n",
      "Episodes: 2, AVG Score: 1.883, Score: 12.0, AVG intrinsic reward: 8.384964\n",
      "Episodes: 3, AVG Score: 2.438, Score: 13.0, AVG intrinsic reward: 2.913019\n",
      "Episodes: 4, AVG Score: 3.716, Score: 28.0, AVG intrinsic reward: 2.982554\n",
      "Episodes: 5, AVG Score: 4.281, Score: 15.0, AVG intrinsic reward: 1.379303\n",
      "Episodes: 6, AVG Score: 5.567, Score: 30.0, AVG intrinsic reward: 1.400859\n",
      "Episodes: 7, AVG Score: 7.288, Score: 40.0, AVG intrinsic reward: 1.311737\n",
      "Episodes: 8, AVG Score: 7.474, Score: 11.0, AVG intrinsic reward: 1.412290\n",
      "Episodes: 9, AVG Score: 7.950, Score: 17.0, AVG intrinsic reward: 0.792913\n",
      "Episodes: 10, AVG Score: 8.003, Score: 9.0, AVG intrinsic reward: 0.782326\n",
      "Episodes: 11, AVG Score: 8.753, Score: 23.0, AVG intrinsic reward: 1.288839\n",
      "Episodes: 12, AVG Score: 9.265, Score: 19.0, AVG intrinsic reward: 0.664575\n",
      "Episodes: 13, AVG Score: 10.252, Score: 29.0, AVG intrinsic reward: 0.745696\n",
      "Episodes: 14, AVG Score: 11.089, Score: 27.0, AVG intrinsic reward: 0.657155\n",
      "Episodes: 15, AVG Score: 11.285, Score: 15.0, AVG intrinsic reward: 0.413297\n",
      "Episodes: 16, AVG Score: 12.120, Score: 28.0, AVG intrinsic reward: 0.529824\n",
      "Episodes: 17, AVG Score: 12.664, Score: 23.0, AVG intrinsic reward: 0.664442\n",
      "Episodes: 18, AVG Score: 12.881, Score: 17.0, AVG intrinsic reward: 0.636362\n",
      "Episodes: 19, AVG Score: 12.837, Score: 12.0, AVG intrinsic reward: 0.756974\n",
      "Episodes: 20, AVG Score: 13.245, Score: 21.0, AVG intrinsic reward: 0.544958\n",
      "Episodes: 21, AVG Score: 14.333, Score: 35.0, AVG intrinsic reward: 0.493167\n",
      "Episodes: 22, AVG Score: 14.266, Score: 13.0, AVG intrinsic reward: 0.517213\n",
      "Episodes: 23, AVG Score: 15.803, Score: 45.0, AVG intrinsic reward: 0.471134\n",
      "Episodes: 24, AVG Score: 16.163, Score: 23.0, AVG intrinsic reward: 0.390786\n",
      "Episodes: 25, AVG Score: 16.905, Score: 31.0, AVG intrinsic reward: 0.758276\n",
      "Episodes: 26, AVG Score: 17.159, Score: 22.0, AVG intrinsic reward: 0.622995\n",
      "Episodes: 27, AVG Score: 17.102, Score: 16.0, AVG intrinsic reward: 0.614292\n",
      "Episodes: 28, AVG Score: 16.696, Score: 9.0, AVG intrinsic reward: 1.057892\n",
      "Episodes: 29, AVG Score: 16.362, Score: 10.0, AVG intrinsic reward: 0.572485\n",
      "Episodes: 30, AVG Score: 16.544, Score: 20.0, AVG intrinsic reward: 0.983215\n",
      "Episodes: 31, AVG Score: 17.666, Score: 39.0, AVG intrinsic reward: 0.681722\n",
      "Episodes: 32, AVG Score: 17.383, Score: 12.0, AVG intrinsic reward: 0.815615\n",
      "Episodes: 33, AVG Score: 17.314, Score: 16.0, AVG intrinsic reward: 0.325616\n",
      "Episodes: 34, AVG Score: 16.898, Score: 9.0, AVG intrinsic reward: 0.756110\n",
      "Episodes: 35, AVG Score: 18.653, Score: 52.0, AVG intrinsic reward: 0.611814\n",
      "Episodes: 36, AVG Score: 18.621, Score: 18.0, AVG intrinsic reward: 0.735630\n",
      "Episodes: 37, AVG Score: 18.290, Score: 12.0, AVG intrinsic reward: 1.016463\n",
      "Episodes: 38, AVG Score: 18.975, Score: 32.0, AVG intrinsic reward: 0.930316\n",
      "Episodes: 39, AVG Score: 18.726, Score: 14.0, AVG intrinsic reward: 1.357262\n",
      "Episodes: 40, AVG Score: 18.390, Score: 12.0, AVG intrinsic reward: 0.924086\n",
      "Episodes: 41, AVG Score: 18.221, Score: 15.0, AVG intrinsic reward: 0.520984\n",
      "Episodes: 42, AVG Score: 17.860, Score: 11.0, AVG intrinsic reward: 0.470146\n",
      "Episodes: 43, AVG Score: 18.167, Score: 24.0, AVG intrinsic reward: 0.696922\n",
      "Episodes: 44, AVG Score: 17.758, Score: 10.0, AVG intrinsic reward: 0.810040\n",
      "Episodes: 45, AVG Score: 17.520, Score: 13.0, AVG intrinsic reward: 0.451153\n",
      "Episodes: 46, AVG Score: 17.194, Score: 11.0, AVG intrinsic reward: 1.039832\n",
      "Episodes: 47, AVG Score: 16.885, Score: 11.0, AVG intrinsic reward: 0.897374\n",
      "Episodes: 48, AVG Score: 17.590, Score: 31.0, AVG intrinsic reward: 0.649015\n",
      "Episodes: 49, AVG Score: 17.311, Score: 12.0, AVG intrinsic reward: 1.785599\n",
      "Episodes: 50, AVG Score: 17.095, Score: 13.0, AVG intrinsic reward: 0.744324\n",
      "Episodes: 51, AVG Score: 16.691, Score: 9.0, AVG intrinsic reward: 1.367435\n",
      "Episodes: 52, AVG Score: 16.456, Score: 12.0, AVG intrinsic reward: 0.944542\n",
      "Episodes: 53, AVG Score: 16.183, Score: 11.0, AVG intrinsic reward: 0.518154\n",
      "Episodes: 54, AVG Score: 15.974, Score: 12.0, AVG intrinsic reward: 0.458633\n",
      "Episodes: 55, AVG Score: 16.525, Score: 27.0, AVG intrinsic reward: 0.672970\n",
      "Episodes: 56, AVG Score: 17.399, Score: 34.0, AVG intrinsic reward: 0.941206\n",
      "Episodes: 57, AVG Score: 17.079, Score: 11.0, AVG intrinsic reward: 0.858896\n",
      "Episodes: 58, AVG Score: 17.025, Score: 16.0, AVG intrinsic reward: 1.107176\n",
      "Episodes: 59, AVG Score: 16.674, Score: 10.0, AVG intrinsic reward: 0.597553\n",
      "Episodes: 60, AVG Score: 16.740, Score: 18.0, AVG intrinsic reward: 0.497582\n",
      "Episodes: 61, AVG Score: 16.403, Score: 10.0, AVG intrinsic reward: 1.183898\n",
      "Episodes: 62, AVG Score: 16.333, Score: 15.0, AVG intrinsic reward: 0.611883\n",
      "Episodes: 63, AVG Score: 15.966, Score: 9.0, AVG intrinsic reward: 1.275859\n",
      "Episodes: 64, AVG Score: 15.668, Score: 10.0, AVG intrinsic reward: 0.570453\n",
      "Episodes: 65, AVG Score: 15.785, Score: 18.0, AVG intrinsic reward: 0.377997\n",
      "Episodes: 66, AVG Score: 16.295, Score: 26.0, AVG intrinsic reward: 0.385321\n",
      "Episodes: 67, AVG Score: 16.581, Score: 22.0, AVG intrinsic reward: 0.789615\n",
      "Episodes: 68, AVG Score: 17.152, Score: 28.0, AVG intrinsic reward: 0.417168\n",
      "Episodes: 69, AVG Score: 18.044, Score: 35.0, AVG intrinsic reward: 0.510167\n",
      "Episodes: 70, AVG Score: 17.742, Score: 12.0, AVG intrinsic reward: 0.509242\n",
      "Episodes: 71, AVG Score: 19.705, Score: 57.0, AVG intrinsic reward: 0.560526\n",
      "Episodes: 72, AVG Score: 19.520, Score: 16.0, AVG intrinsic reward: 1.014375\n",
      "Episodes: 73, AVG Score: 19.194, Score: 13.0, AVG intrinsic reward: 0.538556\n",
      "Episodes: 74, AVG Score: 18.984, Score: 15.0, AVG intrinsic reward: 0.656181\n",
      "Episodes: 75, AVG Score: 19.135, Score: 22.0, AVG intrinsic reward: 0.685919\n",
      "Episodes: 76, AVG Score: 19.378, Score: 24.0, AVG intrinsic reward: 0.465877\n",
      "Episodes: 77, AVG Score: 20.009, Score: 32.0, AVG intrinsic reward: 0.445053\n",
      "Episodes: 78, AVG Score: 20.709, Score: 34.0, AVG intrinsic reward: 0.397809\n",
      "Episodes: 79, AVG Score: 20.823, Score: 23.0, AVG intrinsic reward: 0.395805\n",
      "Episodes: 80, AVG Score: 21.282, Score: 30.0, AVG intrinsic reward: 0.346466\n",
      "Episodes: 81, AVG Score: 21.068, Score: 17.0, AVG intrinsic reward: 0.300781\n",
      "Episodes: 82, AVG Score: 22.965, Score: 59.0, AVG intrinsic reward: 0.610020\n",
      "Episodes: 83, AVG Score: 22.566, Score: 15.0, AVG intrinsic reward: 0.791098\n",
      "Episodes: 84, AVG Score: 23.938, Score: 50.0, AVG intrinsic reward: 0.641309\n",
      "Episodes: 85, AVG Score: 25.091, Score: 47.0, AVG intrinsic reward: 0.704120\n",
      "Episodes: 86, AVG Score: 28.487, Score: 93.0, AVG intrinsic reward: 0.593624\n",
      "Episodes: 87, AVG Score: 29.562, Score: 50.0, AVG intrinsic reward: 0.979218\n",
      "Episodes: 88, AVG Score: 32.334, Score: 85.0, AVG intrinsic reward: 0.435396\n",
      "Episodes: 89, AVG Score: 31.617, Score: 18.0, AVG intrinsic reward: 0.326199\n",
      "Episodes: 90, AVG Score: 32.487, Score: 49.0, AVG intrinsic reward: 0.315631\n",
      "Episodes: 91, AVG Score: 33.062, Score: 44.0, AVG intrinsic reward: 0.276990\n",
      "Episodes: 92, AVG Score: 33.059, Score: 33.0, AVG intrinsic reward: 0.206769\n",
      "Episodes: 93, AVG Score: 32.906, Score: 30.0, AVG intrinsic reward: 0.345726\n",
      "Episodes: 94, AVG Score: 32.261, Score: 20.0, AVG intrinsic reward: 0.664180\n",
      "Episodes: 95, AVG Score: 33.498, Score: 57.0, AVG intrinsic reward: 0.219246\n",
      "Episodes: 96, AVG Score: 33.723, Score: 38.0, AVG intrinsic reward: 0.239671\n",
      "Episodes: 97, AVG Score: 34.487, Score: 49.0, AVG intrinsic reward: 0.401419\n",
      "Episodes: 98, AVG Score: 34.112, Score: 27.0, AVG intrinsic reward: 0.546645\n",
      "Episodes: 99, AVG Score: 34.507, Score: 42.0, AVG intrinsic reward: 0.219370\n",
      "Episodes: 100, AVG Score: 34.131, Score: 27.0, AVG intrinsic reward: 0.625159\n",
      "Episodes: 101, AVG Score: 34.625, Score: 44.0, AVG intrinsic reward: 0.222425\n",
      "Episodes: 102, AVG Score: 34.344, Score: 29.0, AVG intrinsic reward: 0.223991\n",
      "Episodes: 103, AVG Score: 34.426, Score: 36.0, AVG intrinsic reward: 0.200339\n",
      "Episodes: 104, AVG Score: 34.855, Score: 43.0, AVG intrinsic reward: 0.308655\n",
      "Episodes: 105, AVG Score: 34.412, Score: 26.0, AVG intrinsic reward: 0.409723\n",
      "Episodes: 106, AVG Score: 34.492, Score: 36.0, AVG intrinsic reward: 0.196245\n",
      "Episodes: 107, AVG Score: 34.817, Score: 41.0, AVG intrinsic reward: 0.474739\n",
      "Episodes: 108, AVG Score: 35.776, Score: 54.0, AVG intrinsic reward: 0.225523\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 109, AVG Score: 38.187, Score: 84.0, AVG intrinsic reward: 0.204245\n",
      "Episodes: 110, AVG Score: 38.328, Score: 41.0, AVG intrinsic reward: 0.249264\n",
      "Episodes: 111, AVG Score: 38.362, Score: 39.0, AVG intrinsic reward: 0.158791\n",
      "Episodes: 112, AVG Score: 39.044, Score: 52.0, AVG intrinsic reward: 0.137983\n",
      "Episodes: 113, AVG Score: 41.091, Score: 80.0, AVG intrinsic reward: 0.190762\n",
      "Episodes: 114, AVG Score: 41.987, Score: 59.0, AVG intrinsic reward: 0.334328\n",
      "Episodes: 115, AVG Score: 42.338, Score: 49.0, AVG intrinsic reward: 0.307754\n",
      "Episodes: 116, AVG Score: 45.271, Score: 101.0, AVG intrinsic reward: 0.345914\n",
      "Episodes: 117, AVG Score: 46.307, Score: 66.0, AVG intrinsic reward: 0.225329\n",
      "Episodes: 118, AVG Score: 46.342, Score: 47.0, AVG intrinsic reward: 0.347012\n",
      "Episodes: 119, AVG Score: 45.675, Score: 33.0, AVG intrinsic reward: 0.284150\n",
      "Episodes: 120, AVG Score: 45.741, Score: 47.0, AVG intrinsic reward: 0.237797\n",
      "Episodes: 121, AVG Score: 47.704, Score: 85.0, AVG intrinsic reward: 0.390616\n",
      "Episodes: 122, AVG Score: 48.469, Score: 63.0, AVG intrinsic reward: 0.301059\n",
      "Episodes: 123, AVG Score: 49.495, Score: 69.0, AVG intrinsic reward: 0.349936\n",
      "Episodes: 124, AVG Score: 48.170, Score: 23.0, AVG intrinsic reward: 1.886375\n",
      "Episodes: 125, AVG Score: 48.062, Score: 46.0, AVG intrinsic reward: 0.299634\n",
      "Episodes: 126, AVG Score: 47.509, Score: 37.0, AVG intrinsic reward: 0.331309\n",
      "Episodes: 127, AVG Score: 46.683, Score: 31.0, AVG intrinsic reward: 0.489457\n",
      "Episodes: 128, AVG Score: 45.999, Score: 33.0, AVG intrinsic reward: 0.396860\n",
      "Episodes: 129, AVG Score: 45.349, Score: 33.0, AVG intrinsic reward: 0.154066\n",
      "Episodes: 130, AVG Score: 45.032, Score: 39.0, AVG intrinsic reward: 0.240670\n",
      "Episodes: 131, AVG Score: 43.880, Score: 22.0, AVG intrinsic reward: 1.053162\n",
      "Episodes: 132, AVG Score: 43.786, Score: 42.0, AVG intrinsic reward: 0.225456\n",
      "Episodes: 133, AVG Score: 43.997, Score: 48.0, AVG intrinsic reward: 0.278935\n",
      "Episodes: 134, AVG Score: 43.047, Score: 25.0, AVG intrinsic reward: 0.329774\n",
      "Episodes: 135, AVG Score: 42.995, Score: 42.0, AVG intrinsic reward: 0.149154\n",
      "Episodes: 136, AVG Score: 43.645, Score: 56.0, AVG intrinsic reward: 0.156139\n",
      "Episodes: 137, AVG Score: 43.413, Score: 39.0, AVG intrinsic reward: 0.230869\n",
      "Episodes: 138, AVG Score: 44.492, Score: 65.0, AVG intrinsic reward: 0.267413\n",
      "Episodes: 139, AVG Score: 45.967, Score: 74.0, AVG intrinsic reward: 0.423183\n",
      "Episodes: 140, AVG Score: 46.669, Score: 60.0, AVG intrinsic reward: 0.275581\n",
      "Episodes: 141, AVG Score: 48.436, Score: 82.0, AVG intrinsic reward: 0.441805\n",
      "Episodes: 142, AVG Score: 48.414, Score: 48.0, AVG intrinsic reward: 0.159546\n",
      "Episodes: 143, AVG Score: 48.343, Score: 47.0, AVG intrinsic reward: 0.143671\n",
      "Episodes: 144, AVG Score: 48.176, Score: 45.0, AVG intrinsic reward: 0.212630\n",
      "Episodes: 145, AVG Score: 47.867, Score: 42.0, AVG intrinsic reward: 0.373819\n",
      "Episodes: 146, AVG Score: 52.124, Score: 133.0, AVG intrinsic reward: 0.285387\n",
      "Episodes: 147, AVG Score: 52.318, Score: 56.0, AVG intrinsic reward: 0.509093\n",
      "Episodes: 148, AVG Score: 51.702, Score: 40.0, AVG intrinsic reward: 0.310671\n",
      "Episodes: 149, AVG Score: 51.867, Score: 55.0, AVG intrinsic reward: 0.189218\n",
      "Episodes: 150, AVG Score: 51.273, Score: 40.0, AVG intrinsic reward: 0.309245\n",
      "Episodes: 151, AVG Score: 50.010, Score: 26.0, AVG intrinsic reward: 0.282321\n",
      "Episodes: 152, AVG Score: 50.309, Score: 56.0, AVG intrinsic reward: 0.155746\n",
      "Episodes: 153, AVG Score: 50.594, Score: 56.0, AVG intrinsic reward: 0.235617\n",
      "Episodes: 154, AVG Score: 52.414, Score: 87.0, AVG intrinsic reward: 0.199694\n",
      "Episodes: 155, AVG Score: 51.693, Score: 38.0, AVG intrinsic reward: 0.352629\n",
      "Episodes: 156, AVG Score: 52.259, Score: 63.0, AVG intrinsic reward: 0.262595\n",
      "Episodes: 157, AVG Score: 53.896, Score: 85.0, AVG intrinsic reward: 0.175075\n",
      "Episodes: 158, AVG Score: 54.451, Score: 65.0, AVG intrinsic reward: 0.400420\n",
      "Episodes: 159, AVG Score: 53.778, Score: 41.0, AVG intrinsic reward: 0.192026\n",
      "Episodes: 160, AVG Score: 54.189, Score: 62.0, AVG intrinsic reward: 0.168610\n",
      "Episodes: 161, AVG Score: 54.480, Score: 60.0, AVG intrinsic reward: 0.124412\n",
      "Episodes: 162, AVG Score: 53.456, Score: 34.0, AVG intrinsic reward: 0.351794\n",
      "Episodes: 163, AVG Score: 55.983, Score: 104.0, AVG intrinsic reward: 0.235409\n",
      "Episodes: 164, AVG Score: 55.684, Score: 50.0, AVG intrinsic reward: 0.270110\n",
      "Episodes: 165, AVG Score: 55.400, Score: 50.0, AVG intrinsic reward: 0.154602\n",
      "Episodes: 166, AVG Score: 56.080, Score: 69.0, AVG intrinsic reward: 0.346372\n",
      "Episodes: 167, AVG Score: 56.526, Score: 65.0, AVG intrinsic reward: 0.277331\n",
      "Episodes: 168, AVG Score: 55.900, Score: 44.0, AVG intrinsic reward: 0.329804\n",
      "Episodes: 169, AVG Score: 56.455, Score: 67.0, AVG intrinsic reward: 0.173070\n",
      "Episodes: 170, AVG Score: 56.782, Score: 63.0, AVG intrinsic reward: 0.144713\n",
      "Episodes: 171, AVG Score: 60.793, Score: 137.0, AVG intrinsic reward: 0.217183\n",
      "Episodes: 172, AVG Score: 59.953, Score: 44.0, AVG intrinsic reward: 0.199272\n",
      "Episodes: 173, AVG Score: 59.155, Score: 44.0, AVG intrinsic reward: 0.193877\n",
      "Episodes: 174, AVG Score: 59.898, Score: 74.0, AVG intrinsic reward: 0.239092\n",
      "Episodes: 175, AVG Score: 58.953, Score: 41.0, AVG intrinsic reward: 0.496138\n",
      "Episodes: 176, AVG Score: 57.805, Score: 36.0, AVG intrinsic reward: 0.519742\n",
      "Episodes: 177, AVG Score: 58.115, Score: 64.0, AVG intrinsic reward: 0.178828\n",
      "Episodes: 178, AVG Score: 57.359, Score: 43.0, AVG intrinsic reward: 0.186814\n",
      "Episodes: 179, AVG Score: 56.791, Score: 46.0, AVG intrinsic reward: 0.266272\n",
      "Episodes: 180, AVG Score: 55.952, Score: 40.0, AVG intrinsic reward: 0.327866\n",
      "Episodes: 181, AVG Score: 58.104, Score: 99.0, AVG intrinsic reward: 0.236138\n",
      "Episodes: 182, AVG Score: 57.949, Score: 55.0, AVG intrinsic reward: 0.119069\n",
      "Episodes: 183, AVG Score: 58.701, Score: 73.0, AVG intrinsic reward: 0.152532\n",
      "Episodes: 184, AVG Score: 58.916, Score: 63.0, AVG intrinsic reward: 0.363425\n",
      "Episodes: 185, AVG Score: 58.471, Score: 50.0, AVG intrinsic reward: 0.159206\n",
      "Episodes: 186, AVG Score: 61.997, Score: 129.0, AVG intrinsic reward: 0.201703\n",
      "Episodes: 187, AVG Score: 60.447, Score: 31.0, AVG intrinsic reward: 0.232037\n",
      "Episodes: 188, AVG Score: 61.575, Score: 83.0, AVG intrinsic reward: 0.334249\n",
      "Episodes: 189, AVG Score: 61.246, Score: 55.0, AVG intrinsic reward: 0.155689\n",
      "Episodes: 190, AVG Score: 60.234, Score: 41.0, AVG intrinsic reward: 0.185048\n",
      "Episodes: 191, AVG Score: 59.922, Score: 54.0, AVG intrinsic reward: 0.335870\n",
      "Episodes: 192, AVG Score: 59.776, Score: 57.0, AVG intrinsic reward: 0.230655\n",
      "Episodes: 193, AVG Score: 59.687, Score: 58.0, AVG intrinsic reward: 0.165016\n",
      "Episodes: 194, AVG Score: 60.253, Score: 71.0, AVG intrinsic reward: 0.137866\n",
      "Episodes: 195, AVG Score: 59.840, Score: 52.0, AVG intrinsic reward: 0.562420\n",
      "Episodes: 196, AVG Score: 60.048, Score: 64.0, AVG intrinsic reward: 0.351750\n",
      "Episodes: 197, AVG Score: 61.496, Score: 89.0, AVG intrinsic reward: 0.167896\n",
      "Episodes: 198, AVG Score: 61.871, Score: 69.0, AVG intrinsic reward: 0.195975\n",
      "Episodes: 199, AVG Score: 60.777, Score: 40.0, AVG intrinsic reward: 0.330745\n",
      "Episodes: 200, AVG Score: 62.589, Score: 97.0, AVG intrinsic reward: 0.324150\n",
      "Episodes: 201, AVG Score: 64.509, Score: 101.0, AVG intrinsic reward: 0.317650\n",
      "Episodes: 202, AVG Score: 64.234, Score: 59.0, AVG intrinsic reward: 0.278660\n",
      "Episodes: 203, AVG Score: 65.922, Score: 98.0, AVG intrinsic reward: 0.113918\n",
      "Episodes: 204, AVG Score: 66.026, Score: 68.0, AVG intrinsic reward: 0.283937\n",
      "Episodes: 205, AVG Score: 70.775, Score: 161.0, AVG intrinsic reward: 0.495377\n",
      "Episodes: 206, AVG Score: 72.186, Score: 99.0, AVG intrinsic reward: 0.447765\n",
      "Episodes: 207, AVG Score: 71.027, Score: 49.0, AVG intrinsic reward: 0.331914\n",
      "Episodes: 208, AVG Score: 70.025, Score: 51.0, AVG intrinsic reward: 0.258602\n",
      "Episodes: 209, AVG Score: 74.024, Score: 150.0, AVG intrinsic reward: 0.436055\n",
      "Episodes: 210, AVG Score: 74.423, Score: 82.0, AVG intrinsic reward: 0.393524\n",
      "Episodes: 211, AVG Score: 73.752, Score: 61.0, AVG intrinsic reward: 0.189016\n",
      "Episodes: 212, AVG Score: 72.114, Score: 41.0, AVG intrinsic reward: 0.198064\n",
      "Episodes: 213, AVG Score: 72.058, Score: 71.0, AVG intrinsic reward: 0.099677\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 214, AVG Score: 70.555, Score: 42.0, AVG intrinsic reward: 0.116316\n",
      "Episodes: 215, AVG Score: 69.928, Score: 58.0, AVG intrinsic reward: 0.294648\n",
      "Episodes: 216, AVG Score: 68.231, Score: 36.0, AVG intrinsic reward: 0.236755\n",
      "Episodes: 217, AVG Score: 67.820, Score: 60.0, AVG intrinsic reward: 0.294116\n",
      "Episodes: 218, AVG Score: 66.779, Score: 47.0, AVG intrinsic reward: 0.130523\n",
      "Episodes: 219, AVG Score: 65.890, Score: 49.0, AVG intrinsic reward: 0.237548\n",
      "Episodes: 220, AVG Score: 64.845, Score: 45.0, AVG intrinsic reward: 0.193177\n",
      "Episodes: 221, AVG Score: 63.753, Score: 43.0, AVG intrinsic reward: 0.139511\n",
      "Episodes: 222, AVG Score: 62.715, Score: 43.0, AVG intrinsic reward: 0.206082\n",
      "Episodes: 223, AVG Score: 62.630, Score: 61.0, AVG intrinsic reward: 0.318015\n",
      "Episodes: 224, AVG Score: 61.598, Score: 42.0, AVG intrinsic reward: 0.098158\n",
      "Episodes: 225, AVG Score: 61.218, Score: 54.0, AVG intrinsic reward: 0.115393\n",
      "Episodes: 226, AVG Score: 69.257, Score: 222.0, AVG intrinsic reward: 0.810353\n",
      "Episodes: 227, AVG Score: 73.794, Score: 160.0, AVG intrinsic reward: 0.334660\n",
      "Episodes: 228, AVG Score: 73.205, Score: 62.0, AVG intrinsic reward: 0.436032\n",
      "Episodes: 229, AVG Score: 72.994, Score: 69.0, AVG intrinsic reward: 0.155340\n",
      "Episodes: 230, AVG Score: 71.495, Score: 43.0, AVG intrinsic reward: 0.299260\n",
      "Episodes: 231, AVG Score: 71.870, Score: 79.0, AVG intrinsic reward: 0.229961\n",
      "Episodes: 232, AVG Score: 74.827, Score: 131.0, AVG intrinsic reward: 0.212719\n",
      "Episodes: 233, AVG Score: 74.835, Score: 75.0, AVG intrinsic reward: 0.198519\n",
      "Episodes: 234, AVG Score: 75.943, Score: 97.0, AVG intrinsic reward: 0.209766\n",
      "Episodes: 235, AVG Score: 74.596, Score: 49.0, AVG intrinsic reward: 0.305428\n",
      "Episodes: 236, AVG Score: 76.316, Score: 109.0, AVG intrinsic reward: 0.262969\n",
      "Episodes: 237, AVG Score: 74.651, Score: 43.0, AVG intrinsic reward: 0.192961\n",
      "Episodes: 238, AVG Score: 73.618, Score: 54.0, AVG intrinsic reward: 0.294215\n",
      "Episodes: 239, AVG Score: 72.887, Score: 59.0, AVG intrinsic reward: 0.202136\n",
      "Episodes: 240, AVG Score: 74.093, Score: 97.0, AVG intrinsic reward: 0.445903\n",
      "Episodes: 241, AVG Score: 75.938, Score: 111.0, AVG intrinsic reward: 0.392733\n",
      "Episodes: 242, AVG Score: 78.591, Score: 129.0, AVG intrinsic reward: 0.293828\n",
      "Episodes: 243, AVG Score: 81.662, Score: 140.0, AVG intrinsic reward: 0.308510\n",
      "Episodes: 244, AVG Score: 79.379, Score: 36.0, AVG intrinsic reward: 0.479101\n",
      "Episodes: 245, AVG Score: 79.960, Score: 91.0, AVG intrinsic reward: 0.223084\n",
      "Episodes: 246, AVG Score: 81.212, Score: 105.0, AVG intrinsic reward: 0.193600\n",
      "Episodes: 247, AVG Score: 83.251, Score: 122.0, AVG intrinsic reward: 0.646723\n",
      "Episodes: 248, AVG Score: 83.989, Score: 98.0, AVG intrinsic reward: 0.349569\n",
      "Episodes: 249, AVG Score: 81.339, Score: 31.0, AVG intrinsic reward: 0.422045\n",
      "Episodes: 250, AVG Score: 83.072, Score: 116.0, AVG intrinsic reward: 0.350893\n",
      "Episodes: 251, AVG Score: 83.519, Score: 92.0, AVG intrinsic reward: 0.314107\n",
      "Episodes: 252, AVG Score: 88.743, Score: 188.0, AVG intrinsic reward: 0.340393\n",
      "Episodes: 253, AVG Score: 99.256, Score: 299.0, AVG intrinsic reward: 0.282540\n",
      "Episodes: 254, AVG Score: 103.043, Score: 175.0, AVG intrinsic reward: 0.901238\n",
      "Episodes: 255, AVG Score: 102.941, Score: 101.0, AVG intrinsic reward: 0.696806\n",
      "Episodes: 256, AVG Score: 106.144, Score: 167.0, AVG intrinsic reward: 0.422834\n",
      "Episodes: 257, AVG Score: 106.836, Score: 120.0, AVG intrinsic reward: 0.260931\n",
      "Episodes: 258, AVG Score: 126.445, Score: 499.0, AVG intrinsic reward: 0.278883\n",
      "Episodes: 259, AVG Score: 129.472, Score: 187.0, AVG intrinsic reward: 0.272291\n",
      "Episodes: 260, AVG Score: 131.549, Score: 171.0, AVG intrinsic reward: 0.469632\n",
      "Episodes: 261, AVG Score: 149.921, Score: 499.0, AVG intrinsic reward: 0.199048\n",
      "Episodes: 262, AVG Score: 146.025, Score: 72.0, AVG intrinsic reward: 0.656218\n",
      "Episodes: 263, AVG Score: 140.674, Score: 39.0, AVG intrinsic reward: 1.208665\n",
      "Episodes: 264, AVG Score: 142.440, Score: 176.0, AVG intrinsic reward: 0.319635\n",
      "Episodes: 265, AVG Score: 140.918, Score: 112.0, AVG intrinsic reward: 0.408407\n",
      "Episodes: 266, AVG Score: 134.822, Score: 19.0, AVG intrinsic reward: 0.129819\n",
      "Episodes: 267, AVG Score: 128.831, Score: 15.0, AVG intrinsic reward: 0.112298\n",
      "Episodes: 268, AVG Score: 122.890, Score: 10.0, AVG intrinsic reward: 0.839928\n",
      "Episodes: 269, AVG Score: 117.245, Score: 10.0, AVG intrinsic reward: 0.690800\n",
      "Episodes: 270, AVG Score: 111.933, Score: 11.0, AVG intrinsic reward: 0.816512\n",
      "Episodes: 271, AVG Score: 106.836, Score: 10.0, AVG intrinsic reward: 0.414603\n",
      "Episodes: 272, AVG Score: 101.894, Score: 8.0, AVG intrinsic reward: 0.618896\n",
      "Episodes: 273, AVG Score: 97.400, Score: 12.0, AVG intrinsic reward: 0.457419\n",
      "Episodes: 274, AVG Score: 93.030, Score: 10.0, AVG intrinsic reward: 0.747719\n",
      "Episodes: 275, AVG Score: 89.078, Score: 14.0, AVG intrinsic reward: 0.300345\n",
      "Episodes: 276, AVG Score: 85.174, Score: 11.0, AVG intrinsic reward: 0.151298\n",
      "Episodes: 277, AVG Score: 82.016, Score: 22.0, AVG intrinsic reward: 0.226237\n",
      "Episodes: 278, AVG Score: 81.215, Score: 66.0, AVG intrinsic reward: 0.259188\n",
      "Episodes: 279, AVG Score: 82.254, Score: 102.0, AVG intrinsic reward: 0.156130\n",
      "Episodes: 280, AVG Score: 78.691, Score: 11.0, AVG intrinsic reward: 0.436585\n",
      "Episodes: 281, AVG Score: 76.057, Score: 26.0, AVG intrinsic reward: 0.203665\n",
      "Episodes: 282, AVG Score: 72.654, Score: 8.0, AVG intrinsic reward: 0.524562\n",
      "Episodes: 283, AVG Score: 69.671, Score: 13.0, AVG intrinsic reward: 0.225695\n",
      "Episodes: 284, AVG Score: 66.838, Score: 13.0, AVG intrinsic reward: 0.056971\n",
      "Episodes: 285, AVG Score: 64.246, Score: 15.0, AVG intrinsic reward: 0.083952\n",
      "Episodes: 286, AVG Score: 61.684, Score: 13.0, AVG intrinsic reward: 0.209155\n",
      "Episodes: 287, AVG Score: 62.749, Score: 83.0, AVG intrinsic reward: 0.152119\n",
      "Episodes: 288, AVG Score: 63.662, Score: 81.0, AVG intrinsic reward: 0.106076\n",
      "Episodes: 289, AVG Score: 61.729, Score: 25.0, AVG intrinsic reward: 0.138062\n",
      "Episodes: 290, AVG Score: 65.842, Score: 144.0, AVG intrinsic reward: 0.052300\n",
      "Episodes: 291, AVG Score: 73.000, Score: 209.0, AVG intrinsic reward: 0.052769\n",
      "Episodes: 292, AVG Score: 78.900, Score: 191.0, AVG intrinsic reward: 0.049630\n",
      "Episodes: 293, AVG Score: 84.105, Score: 183.0, AVG intrinsic reward: 0.051548\n",
      "Episodes: 294, AVG Score: 88.450, Score: 171.0, AVG intrinsic reward: 0.042989\n",
      "Episodes: 295, AVG Score: 93.227, Score: 184.0, AVG intrinsic reward: 0.135569\n",
      "Episodes: 296, AVG Score: 95.466, Score: 138.0, AVG intrinsic reward: 0.080772\n",
      "Episodes: 297, AVG Score: 99.643, Score: 179.0, AVG intrinsic reward: 0.028408\n",
      "Episodes: 298, AVG Score: 106.161, Score: 230.0, AVG intrinsic reward: 0.041326\n",
      "Episodes: 299, AVG Score: 122.503, Score: 433.0, AVG intrinsic reward: 0.024485\n",
      "Episodes: 300, AVG Score: 138.577, Score: 444.0, AVG intrinsic reward: 0.048889\n",
      "Episodes: 301, AVG Score: 148.499, Score: 337.0, AVG intrinsic reward: 0.061086\n",
      "Episodes: 302, AVG Score: 151.374, Score: 206.0, AVG intrinsic reward: 0.059420\n",
      "Episodes: 303, AVG Score: 152.105, Score: 166.0, AVG intrinsic reward: 0.085300\n",
      "Episodes: 304, AVG Score: 153.650, Score: 183.0, AVG intrinsic reward: 0.063825\n",
      "Episodes: 305, AVG Score: 159.867, Score: 278.0, AVG intrinsic reward: 0.057243\n",
      "Episodes: 306, AVG Score: 160.124, Score: 165.0, AVG intrinsic reward: 0.074491\n",
      "Episodes: 307, AVG Score: 162.918, Score: 216.0, AVG intrinsic reward: 0.040572\n",
      "Episodes: 308, AVG Score: 169.222, Score: 289.0, AVG intrinsic reward: 0.026857\n",
      "Episodes: 309, AVG Score: 168.661, Score: 158.0, AVG intrinsic reward: 0.028547\n",
      "Episodes: 310, AVG Score: 169.528, Score: 186.0, AVG intrinsic reward: 0.017585\n",
      "Episodes: 311, AVG Score: 171.051, Score: 200.0, AVG intrinsic reward: 0.041009\n",
      "Episodes: 312, AVG Score: 171.449, Score: 179.0, AVG intrinsic reward: 0.021379\n",
      "Episodes: 313, AVG Score: 170.576, Score: 154.0, AVG intrinsic reward: 0.061989\n",
      "Episodes: 314, AVG Score: 168.747, Score: 134.0, AVG intrinsic reward: 0.030680\n",
      "Episodes: 315, AVG Score: 166.910, Score: 132.0, AVG intrinsic reward: 0.023084\n",
      "Episodes: 316, AVG Score: 163.615, Score: 101.0, AVG intrinsic reward: 0.040857\n",
      "Episodes: 317, AVG Score: 163.434, Score: 160.0, AVG intrinsic reward: 0.026338\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 318, AVG Score: 163.912, Score: 173.0, AVG intrinsic reward: 0.039095\n",
      "Episodes: 319, AVG Score: 161.567, Score: 117.0, AVG intrinsic reward: 0.081321\n",
      "Episodes: 320, AVG Score: 158.838, Score: 107.0, AVG intrinsic reward: 0.075940\n",
      "Episodes: 321, AVG Score: 156.946, Score: 121.0, AVG intrinsic reward: 0.089418\n",
      "Episodes: 322, AVG Score: 153.399, Score: 86.0, AVG intrinsic reward: 0.044426\n",
      "Episodes: 323, AVG Score: 150.179, Score: 89.0, AVG intrinsic reward: 0.062204\n",
      "Episodes: 324, AVG Score: 149.970, Score: 146.0, AVG intrinsic reward: 0.017997\n",
      "Episodes: 325, AVG Score: 149.172, Score: 134.0, AVG intrinsic reward: 0.022244\n",
      "Episodes: 326, AVG Score: 149.563, Score: 157.0, AVG intrinsic reward: 0.035605\n",
      "Episodes: 327, AVG Score: 145.985, Score: 78.0, AVG intrinsic reward: 0.027488\n",
      "Episodes: 328, AVG Score: 143.786, Score: 102.0, AVG intrinsic reward: 0.032885\n",
      "Episodes: 329, AVG Score: 141.396, Score: 96.0, AVG intrinsic reward: 0.027156\n",
      "Episodes: 330, AVG Score: 139.277, Score: 99.0, AVG intrinsic reward: 0.039458\n",
      "Episodes: 331, AVG Score: 139.513, Score: 144.0, AVG intrinsic reward: 0.018509\n",
      "Episodes: 332, AVG Score: 143.487, Score: 219.0, AVG intrinsic reward: 0.044129\n",
      "Episodes: 333, AVG Score: 142.763, Score: 129.0, AVG intrinsic reward: 0.021750\n",
      "Episodes: 334, AVG Score: 144.275, Score: 173.0, AVG intrinsic reward: 0.027735\n",
      "Episodes: 335, AVG Score: 152.861, Score: 316.0, AVG intrinsic reward: 0.028422\n",
      "Episodes: 336, AVG Score: 160.918, Score: 314.0, AVG intrinsic reward: 0.013817\n",
      "Episodes: 337, AVG Score: 167.822, Score: 299.0, AVG intrinsic reward: 0.035452\n",
      "Episodes: 338, AVG Score: 184.381, Score: 499.0, AVG intrinsic reward: 0.019368\n",
      "Episodes: 339, AVG Score: 185.962, Score: 216.0, AVG intrinsic reward: 0.027940\n",
      "Episodes: 340, AVG Score: 180.914, Score: 85.0, AVG intrinsic reward: 0.036606\n",
      "Episodes: 341, AVG Score: 176.518, Score: 93.0, AVG intrinsic reward: 0.053138\n",
      "Episodes: 342, AVG Score: 172.942, Score: 105.0, AVG intrinsic reward: 0.084928\n",
      "Episodes: 343, AVG Score: 168.595, Score: 86.0, AVG intrinsic reward: 0.040596\n",
      "Episodes: 344, AVG Score: 164.065, Score: 78.0, AVG intrinsic reward: 0.048607\n",
      "Episodes: 345, AVG Score: 158.412, Score: 51.0, AVG intrinsic reward: 0.047147\n",
      "Episodes: 346, AVG Score: 154.441, Score: 79.0, AVG intrinsic reward: 0.035792\n",
      "Episodes: 347, AVG Score: 150.769, Score: 81.0, AVG intrinsic reward: 0.044673\n",
      "Episodes: 348, AVG Score: 147.781, Score: 91.0, AVG intrinsic reward: 0.029919\n",
      "Episodes: 349, AVG Score: 143.592, Score: 64.0, AVG intrinsic reward: 0.031642\n",
      "Episodes: 350, AVG Score: 139.162, Score: 55.0, AVG intrinsic reward: 0.032717\n",
      "Episodes: 351, AVG Score: 135.704, Score: 70.0, AVG intrinsic reward: 0.028336\n",
      "Episodes: 352, AVG Score: 131.319, Score: 48.0, AVG intrinsic reward: 0.022609\n",
      "Episodes: 353, AVG Score: 127.153, Score: 48.0, AVG intrinsic reward: 0.019308\n",
      "Episodes: 354, AVG Score: 122.545, Score: 35.0, AVG intrinsic reward: 0.031777\n",
      "Episodes: 355, AVG Score: 119.568, Score: 63.0, AVG intrinsic reward: 0.043573\n",
      "Episodes: 356, AVG Score: 115.790, Score: 44.0, AVG intrinsic reward: 0.020945\n",
      "Episodes: 357, AVG Score: 112.750, Score: 55.0, AVG intrinsic reward: 0.022710\n",
      "Episodes: 358, AVG Score: 110.163, Score: 61.0, AVG intrinsic reward: 0.027655\n",
      "Episodes: 359, AVG Score: 107.555, Score: 58.0, AVG intrinsic reward: 0.027273\n",
      "Episodes: 360, AVG Score: 105.027, Score: 57.0, AVG intrinsic reward: 0.009056\n",
      "Episodes: 361, AVG Score: 101.925, Score: 43.0, AVG intrinsic reward: 0.019426\n",
      "Episodes: 362, AVG Score: 98.779, Score: 39.0, AVG intrinsic reward: 0.009435\n",
      "Episodes: 363, AVG Score: 95.990, Score: 43.0, AVG intrinsic reward: 0.018748\n",
      "Episodes: 364, AVG Score: 94.141, Score: 59.0, AVG intrinsic reward: 0.027309\n",
      "Episodes: 365, AVG Score: 92.334, Score: 58.0, AVG intrinsic reward: 0.014832\n",
      "Episodes: 366, AVG Score: 90.067, Score: 47.0, AVG intrinsic reward: 0.013501\n",
      "Episodes: 367, AVG Score: 88.964, Score: 68.0, AVG intrinsic reward: 0.017569\n",
      "Episodes: 368, AVG Score: 87.065, Score: 51.0, AVG intrinsic reward: 0.054225\n",
      "Episodes: 369, AVG Score: 85.112, Score: 48.0, AVG intrinsic reward: 0.029614\n",
      "Episodes: 370, AVG Score: 83.507, Score: 53.0, AVG intrinsic reward: 0.026735\n",
      "Episodes: 371, AVG Score: 82.331, Score: 60.0, AVG intrinsic reward: 0.024342\n",
      "Episodes: 372, AVG Score: 80.865, Score: 53.0, AVG intrinsic reward: 0.014750\n",
      "Episodes: 373, AVG Score: 79.971, Score: 63.0, AVG intrinsic reward: 0.023158\n",
      "Episodes: 374, AVG Score: 78.723, Score: 55.0, AVG intrinsic reward: 0.018209\n",
      "Episodes: 375, AVG Score: 77.487, Score: 54.0, AVG intrinsic reward: 0.069387\n",
      "Episodes: 376, AVG Score: 76.462, Score: 57.0, AVG intrinsic reward: 0.031551\n",
      "Episodes: 377, AVG Score: 75.789, Score: 63.0, AVG intrinsic reward: 0.017283\n",
      "Episodes: 378, AVG Score: 75.250, Score: 65.0, AVG intrinsic reward: 0.028605\n",
      "Episodes: 379, AVG Score: 73.337, Score: 37.0, AVG intrinsic reward: 0.046302\n",
      "Episodes: 380, AVG Score: 72.420, Score: 55.0, AVG intrinsic reward: 0.012276\n",
      "Episodes: 381, AVG Score: 71.349, Score: 51.0, AVG intrinsic reward: 0.022967\n",
      "Episodes: 382, AVG Score: 70.532, Score: 55.0, AVG intrinsic reward: 0.014417\n",
      "Episodes: 383, AVG Score: 69.005, Score: 40.0, AVG intrinsic reward: 0.091861\n",
      "Episodes: 384, AVG Score: 68.255, Score: 54.0, AVG intrinsic reward: 0.014929\n",
      "Episodes: 385, AVG Score: 68.042, Score: 64.0, AVG intrinsic reward: 0.010773\n",
      "Episodes: 386, AVG Score: 66.840, Score: 44.0, AVG intrinsic reward: 0.019103\n",
      "Episodes: 387, AVG Score: 65.348, Score: 37.0, AVG intrinsic reward: 0.017291\n",
      "Episodes: 388, AVG Score: 65.481, Score: 68.0, AVG intrinsic reward: 0.012266\n",
      "Episodes: 389, AVG Score: 65.357, Score: 63.0, AVG intrinsic reward: 0.021776\n",
      "Episodes: 390, AVG Score: 64.389, Score: 46.0, AVG intrinsic reward: 0.013284\n",
      "Episodes: 391, AVG Score: 64.269, Score: 62.0, AVG intrinsic reward: 0.014370\n",
      "Episodes: 392, AVG Score: 62.856, Score: 36.0, AVG intrinsic reward: 0.015804\n",
      "Episodes: 393, AVG Score: 61.963, Score: 45.0, AVG intrinsic reward: 0.014133\n",
      "Episodes: 394, AVG Score: 60.765, Score: 38.0, AVG intrinsic reward: 0.014407\n",
      "Episodes: 395, AVG Score: 60.027, Score: 46.0, AVG intrinsic reward: 0.020912\n",
      "Episodes: 396, AVG Score: 59.125, Score: 42.0, AVG intrinsic reward: 0.010997\n",
      "Episodes: 397, AVG Score: 58.819, Score: 53.0, AVG intrinsic reward: 0.013635\n",
      "Episodes: 398, AVG Score: 58.878, Score: 60.0, AVG intrinsic reward: 0.004740\n",
      "Episodes: 399, AVG Score: 59.684, Score: 75.0, AVG intrinsic reward: 0.026976\n",
      "Episodes: 400, AVG Score: 59.150, Score: 49.0, AVG intrinsic reward: 0.016015\n",
      "Episodes: 401, AVG Score: 58.843, Score: 53.0, AVG intrinsic reward: 0.010728\n",
      "Episodes: 402, AVG Score: 60.550, Score: 93.0, AVG intrinsic reward: 0.010048\n",
      "Episodes: 403, AVG Score: 61.323, Score: 76.0, AVG intrinsic reward: 0.011467\n",
      "Episodes: 404, AVG Score: 62.407, Score: 83.0, AVG intrinsic reward: 0.004644\n",
      "Episodes: 405, AVG Score: 64.836, Score: 111.0, AVG intrinsic reward: 0.022192\n",
      "Episodes: 406, AVG Score: 66.595, Score: 100.0, AVG intrinsic reward: 0.012846\n",
      "Episodes: 407, AVG Score: 68.615, Score: 107.0, AVG intrinsic reward: 0.013269\n",
      "Episodes: 408, AVG Score: 71.334, Score: 123.0, AVG intrinsic reward: 0.013312\n",
      "Episodes: 409, AVG Score: 72.467, Score: 94.0, AVG intrinsic reward: 0.005000\n",
      "Episodes: 410, AVG Score: 72.294, Score: 69.0, AVG intrinsic reward: 0.014239\n",
      "Episodes: 411, AVG Score: 72.329, Score: 73.0, AVG intrinsic reward: 0.009806\n",
      "Episodes: 412, AVG Score: 72.563, Score: 77.0, AVG intrinsic reward: 0.012142\n",
      "Episodes: 413, AVG Score: 72.985, Score: 81.0, AVG intrinsic reward: 0.007430\n",
      "Episodes: 414, AVG Score: 73.086, Score: 75.0, AVG intrinsic reward: 0.010612\n",
      "Episodes: 415, AVG Score: 72.231, Score: 56.0, AVG intrinsic reward: 0.005283\n",
      "Episodes: 416, AVG Score: 71.570, Score: 59.0, AVG intrinsic reward: 0.005866\n",
      "Episodes: 417, AVG Score: 70.541, Score: 51.0, AVG intrinsic reward: 0.014757\n",
      "Episodes: 418, AVG Score: 72.514, Score: 110.0, AVG intrinsic reward: 0.007028\n",
      "Episodes: 419, AVG Score: 72.388, Score: 70.0, AVG intrinsic reward: 0.007746\n",
      "Episodes: 420, AVG Score: 71.969, Score: 64.0, AVG intrinsic reward: 0.004371\n",
      "Episodes: 421, AVG Score: 73.271, Score: 98.0, AVG intrinsic reward: 0.004630\n",
      "Episodes: 422, AVG Score: 75.807, Score: 124.0, AVG intrinsic reward: 0.006681\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 423, AVG Score: 84.717, Score: 254.0, AVG intrinsic reward: 0.012260\n",
      "Episodes: 424, AVG Score: 85.081, Score: 92.0, AVG intrinsic reward: 0.014911\n",
      "Episodes: 425, AVG Score: 91.877, Score: 221.0, AVG intrinsic reward: 0.007953\n",
      "Episodes: 426, AVG Score: 91.033, Score: 75.0, AVG intrinsic reward: 0.010356\n",
      "Episodes: 427, AVG Score: 91.831, Score: 107.0, AVG intrinsic reward: 0.006683\n",
      "Episodes: 428, AVG Score: 90.790, Score: 71.0, AVG intrinsic reward: 0.008596\n",
      "Episodes: 429, AVG Score: 92.300, Score: 121.0, AVG intrinsic reward: 0.014562\n",
      "Episodes: 430, AVG Score: 97.885, Score: 204.0, AVG intrinsic reward: 0.010246\n",
      "Episodes: 431, AVG Score: 104.441, Score: 229.0, AVG intrinsic reward: 0.012569\n",
      "Episodes: 432, AVG Score: 110.069, Score: 217.0, AVG intrinsic reward: 0.007736\n",
      "Episodes: 433, AVG Score: 109.265, Score: 94.0, AVG intrinsic reward: 0.008828\n",
      "Episodes: 434, AVG Score: 114.602, Score: 216.0, AVG intrinsic reward: 0.007313\n",
      "Episodes: 435, AVG Score: 113.022, Score: 83.0, AVG intrinsic reward: 0.003800\n",
      "Episodes: 436, AVG Score: 113.671, Score: 126.0, AVG intrinsic reward: 0.005960\n",
      "Episodes: 437, AVG Score: 121.087, Score: 262.0, AVG intrinsic reward: 0.005406\n",
      "Episodes: 438, AVG Score: 119.383, Score: 87.0, AVG intrinsic reward: 0.012144\n",
      "Episodes: 439, AVG Score: 117.614, Score: 84.0, AVG intrinsic reward: 0.025432\n",
      "Episodes: 440, AVG Score: 117.033, Score: 106.0, AVG intrinsic reward: 0.011403\n",
      "Episodes: 441, AVG Score: 116.282, Score: 102.0, AVG intrinsic reward: 0.014168\n",
      "Episodes: 442, AVG Score: 115.367, Score: 98.0, AVG intrinsic reward: 0.012207\n",
      "Episodes: 443, AVG Score: 116.249, Score: 133.0, AVG intrinsic reward: 0.016978\n",
      "Episodes: 444, AVG Score: 114.337, Score: 78.0, AVG intrinsic reward: 0.012365\n",
      "Episodes: 445, AVG Score: 111.970, Score: 67.0, AVG intrinsic reward: 0.011860\n",
      "Episodes: 446, AVG Score: 110.621, Score: 85.0, AVG intrinsic reward: 0.005886\n",
      "Episodes: 447, AVG Score: 108.540, Score: 69.0, AVG intrinsic reward: 0.009002\n",
      "Episodes: 448, AVG Score: 110.663, Score: 151.0, AVG intrinsic reward: 0.005675\n",
      "Episodes: 449, AVG Score: 110.130, Score: 100.0, AVG intrinsic reward: 0.005520\n",
      "Episodes: 450, AVG Score: 107.524, Score: 58.0, AVG intrinsic reward: 0.010249\n",
      "Episodes: 451, AVG Score: 108.147, Score: 120.0, AVG intrinsic reward: 0.004726\n",
      "Episodes: 452, AVG Score: 106.190, Score: 69.0, AVG intrinsic reward: 0.006021\n",
      "Episodes: 453, AVG Score: 104.581, Score: 74.0, AVG intrinsic reward: 0.008168\n",
      "Episodes: 454, AVG Score: 105.102, Score: 115.0, AVG intrinsic reward: 0.008403\n",
      "Episodes: 455, AVG Score: 101.796, Score: 39.0, AVG intrinsic reward: 0.016597\n",
      "Episodes: 456, AVG Score: 99.557, Score: 57.0, AVG intrinsic reward: 0.038739\n",
      "Episodes: 457, AVG Score: 97.829, Score: 65.0, AVG intrinsic reward: 0.023914\n",
      "Episodes: 458, AVG Score: 95.337, Score: 48.0, AVG intrinsic reward: 0.013972\n",
      "Episodes: 459, AVG Score: 92.120, Score: 31.0, AVG intrinsic reward: 0.029204\n",
      "Episodes: 460, AVG Score: 89.464, Score: 39.0, AVG intrinsic reward: 0.011911\n",
      "Episodes: 461, AVG Score: 86.741, Score: 35.0, AVG intrinsic reward: 0.066240\n",
      "Episodes: 462, AVG Score: 84.454, Score: 41.0, AVG intrinsic reward: 0.034427\n",
      "Episodes: 463, AVG Score: 83.831, Score: 72.0, AVG intrinsic reward: 0.009387\n",
      "Episodes: 464, AVG Score: 81.690, Score: 41.0, AVG intrinsic reward: 0.014873\n",
      "Episodes: 465, AVG Score: 79.655, Score: 41.0, AVG intrinsic reward: 0.011205\n",
      "Episodes: 466, AVG Score: 77.023, Score: 27.0, AVG intrinsic reward: 0.023321\n",
      "Episodes: 467, AVG Score: 75.771, Score: 52.0, AVG intrinsic reward: 0.011426\n",
      "Episodes: 468, AVG Score: 73.233, Score: 25.0, AVG intrinsic reward: 0.016398\n",
      "Episodes: 469, AVG Score: 71.271, Score: 34.0, AVG intrinsic reward: 0.009927\n",
      "Episodes: 470, AVG Score: 69.458, Score: 35.0, AVG intrinsic reward: 0.014330\n",
      "Episodes: 471, AVG Score: 72.885, Score: 138.0, AVG intrinsic reward: 0.009214\n",
      "Episodes: 472, AVG Score: 71.591, Score: 47.0, AVG intrinsic reward: 0.010099\n",
      "Episodes: 473, AVG Score: 70.511, Score: 50.0, AVG intrinsic reward: 0.006752\n",
      "Episodes: 474, AVG Score: 68.635, Score: 33.0, AVG intrinsic reward: 0.011080\n",
      "Episodes: 475, AVG Score: 68.104, Score: 58.0, AVG intrinsic reward: 0.011781\n",
      "Episodes: 476, AVG Score: 66.399, Score: 34.0, AVG intrinsic reward: 0.008740\n",
      "Episodes: 477, AVG Score: 65.679, Score: 52.0, AVG intrinsic reward: 0.016625\n",
      "Episodes: 478, AVG Score: 64.145, Score: 35.0, AVG intrinsic reward: 0.010313\n",
      "Episodes: 479, AVG Score: 63.537, Score: 52.0, AVG intrinsic reward: 0.008788\n",
      "Episodes: 480, AVG Score: 62.411, Score: 41.0, AVG intrinsic reward: 0.005030\n",
      "Episodes: 481, AVG Score: 62.540, Score: 65.0, AVG intrinsic reward: 0.011035\n",
      "Episodes: 482, AVG Score: 61.013, Score: 32.0, AVG intrinsic reward: 0.009477\n",
      "Episodes: 483, AVG Score: 59.762, Score: 36.0, AVG intrinsic reward: 0.023007\n",
      "Episodes: 484, AVG Score: 61.124, Score: 87.0, AVG intrinsic reward: 0.020894\n",
      "Episodes: 485, AVG Score: 59.718, Score: 33.0, AVG intrinsic reward: 0.062925\n",
      "Episodes: 486, AVG Score: 58.782, Score: 41.0, AVG intrinsic reward: 0.029370\n",
      "Episodes: 487, AVG Score: 58.593, Score: 55.0, AVG intrinsic reward: 0.009176\n",
      "Episodes: 488, AVG Score: 58.813, Score: 63.0, AVG intrinsic reward: 0.006530\n",
      "Episodes: 489, AVG Score: 59.873, Score: 80.0, AVG intrinsic reward: 0.009147\n",
      "Episodes: 490, AVG Score: 61.029, Score: 83.0, AVG intrinsic reward: 0.006582\n",
      "Episodes: 491, AVG Score: 61.678, Score: 74.0, AVG intrinsic reward: 0.003391\n",
      "Episodes: 492, AVG Score: 60.194, Score: 32.0, AVG intrinsic reward: 0.011572\n",
      "Episodes: 493, AVG Score: 59.934, Score: 55.0, AVG intrinsic reward: 0.013030\n",
      "Episodes: 494, AVG Score: 59.487, Score: 51.0, AVG intrinsic reward: 0.011958\n",
      "Episodes: 495, AVG Score: 59.463, Score: 59.0, AVG intrinsic reward: 0.004835\n",
      "Episodes: 496, AVG Score: 58.840, Score: 47.0, AVG intrinsic reward: 0.010847\n",
      "Episodes: 497, AVG Score: 62.198, Score: 126.0, AVG intrinsic reward: 0.003814\n",
      "Episodes: 498, AVG Score: 62.738, Score: 73.0, AVG intrinsic reward: 0.014649\n",
      "Episodes: 499, AVG Score: 62.201, Score: 52.0, AVG intrinsic reward: 0.008930\n",
      "Episodes: 500, AVG Score: 61.941, Score: 57.0, AVG intrinsic reward: 0.005937\n",
      "Episodes: 501, AVG Score: 63.194, Score: 87.0, AVG intrinsic reward: 0.004158\n",
      "Episodes: 502, AVG Score: 64.384, Score: 87.0, AVG intrinsic reward: 0.005570\n",
      "Episodes: 503, AVG Score: 65.865, Score: 94.0, AVG intrinsic reward: 0.007416\n",
      "Episodes: 504, AVG Score: 64.272, Score: 34.0, AVG intrinsic reward: 0.020356\n",
      "Episodes: 505, AVG Score: 64.058, Score: 60.0, AVG intrinsic reward: 0.010122\n",
      "Episodes: 506, AVG Score: 63.105, Score: 45.0, AVG intrinsic reward: 0.005330\n",
      "Episodes: 507, AVG Score: 62.000, Score: 41.0, AVG intrinsic reward: 0.005782\n",
      "Episodes: 508, AVG Score: 61.400, Score: 50.0, AVG intrinsic reward: 0.009764\n",
      "Episodes: 509, AVG Score: 61.680, Score: 67.0, AVG intrinsic reward: 0.012540\n",
      "Episodes: 510, AVG Score: 64.396, Score: 116.0, AVG intrinsic reward: 0.003558\n",
      "Episodes: 511, AVG Score: 63.726, Score: 51.0, AVG intrinsic reward: 0.003599\n",
      "Episodes: 512, AVG Score: 65.690, Score: 103.0, AVG intrinsic reward: 0.006506\n",
      "Episodes: 513, AVG Score: 64.955, Score: 51.0, AVG intrinsic reward: 0.010199\n",
      "Episodes: 514, AVG Score: 64.708, Score: 60.0, AVG intrinsic reward: 0.007841\n",
      "Episodes: 515, AVG Score: 65.872, Score: 88.0, AVG intrinsic reward: 0.010483\n",
      "Episodes: 516, AVG Score: 67.879, Score: 106.0, AVG intrinsic reward: 0.004430\n",
      "Episodes: 517, AVG Score: 68.285, Score: 76.0, AVG intrinsic reward: 0.002125\n",
      "Episodes: 518, AVG Score: 75.570, Score: 214.0, AVG intrinsic reward: 0.014259\n",
      "Episodes: 519, AVG Score: 73.692, Score: 38.0, AVG intrinsic reward: 0.057420\n",
      "Episodes: 520, AVG Score: 72.157, Score: 43.0, AVG intrinsic reward: 0.051685\n",
      "Episodes: 521, AVG Score: 71.999, Score: 69.0, AVG intrinsic reward: 0.020677\n",
      "Episodes: 522, AVG Score: 72.150, Score: 75.0, AVG intrinsic reward: 0.013042\n",
      "Episodes: 523, AVG Score: 72.842, Score: 86.0, AVG intrinsic reward: 0.006977\n",
      "Episodes: 524, AVG Score: 71.950, Score: 55.0, AVG intrinsic reward: 0.003092\n",
      "Episodes: 525, AVG Score: 71.552, Score: 64.0, AVG intrinsic reward: 0.008934\n",
      "Episodes: 526, AVG Score: 70.925, Score: 59.0, AVG intrinsic reward: 0.012171\n",
      "Episodes: 527, AVG Score: 70.129, Score: 55.0, AVG intrinsic reward: 0.011683\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 528, AVG Score: 70.522, Score: 78.0, AVG intrinsic reward: 0.030873\n",
      "Episodes: 529, AVG Score: 69.896, Score: 58.0, AVG intrinsic reward: 0.006658\n",
      "Episodes: 530, AVG Score: 70.801, Score: 88.0, AVG intrinsic reward: 0.005892\n",
      "Episodes: 531, AVG Score: 73.661, Score: 128.0, AVG intrinsic reward: 0.036003\n",
      "Episodes: 532, AVG Score: 73.128, Score: 63.0, AVG intrinsic reward: 0.021835\n",
      "Episodes: 533, AVG Score: 71.922, Score: 49.0, AVG intrinsic reward: 0.007029\n",
      "Episodes: 534, AVG Score: 71.476, Score: 63.0, AVG intrinsic reward: 0.004345\n",
      "Episodes: 535, AVG Score: 72.002, Score: 82.0, AVG intrinsic reward: 0.008979\n",
      "Episodes: 536, AVG Score: 72.552, Score: 83.0, AVG intrinsic reward: 0.008138\n",
      "Episodes: 537, AVG Score: 72.874, Score: 79.0, AVG intrinsic reward: 0.009660\n",
      "Episodes: 538, AVG Score: 72.330, Score: 62.0, AVG intrinsic reward: 0.010625\n",
      "Episodes: 539, AVG Score: 71.514, Score: 56.0, AVG intrinsic reward: 0.007438\n",
      "Episodes: 540, AVG Score: 70.988, Score: 61.0, AVG intrinsic reward: 0.017394\n",
      "Episodes: 541, AVG Score: 70.489, Score: 61.0, AVG intrinsic reward: 0.016216\n",
      "Episodes: 542, AVG Score: 70.764, Score: 76.0, AVG intrinsic reward: 0.004031\n",
      "Episodes: 543, AVG Score: 70.976, Score: 75.0, AVG intrinsic reward: 0.003020\n",
      "Episodes: 544, AVG Score: 71.377, Score: 79.0, AVG intrinsic reward: 0.002134\n",
      "Episodes: 545, AVG Score: 71.758, Score: 79.0, AVG intrinsic reward: 0.003862\n",
      "Episodes: 546, AVG Score: 71.921, Score: 75.0, AVG intrinsic reward: 0.003437\n",
      "Episodes: 547, AVG Score: 72.625, Score: 86.0, AVG intrinsic reward: 0.003532\n",
      "Episodes: 548, AVG Score: 72.443, Score: 69.0, AVG intrinsic reward: 0.003149\n",
      "Episodes: 549, AVG Score: 74.171, Score: 107.0, AVG intrinsic reward: 0.001451\n",
      "Episodes: 550, AVG Score: 75.013, Score: 91.0, AVG intrinsic reward: 0.003303\n",
      "Episodes: 551, AVG Score: 76.762, Score: 110.0, AVG intrinsic reward: 0.002825\n",
      "Episodes: 552, AVG Score: 77.874, Score: 99.0, AVG intrinsic reward: 0.001807\n",
      "Episodes: 553, AVG Score: 80.780, Score: 136.0, AVG intrinsic reward: 0.002239\n",
      "Episodes: 554, AVG Score: 82.141, Score: 108.0, AVG intrinsic reward: 0.002484\n",
      "Episodes: 555, AVG Score: 82.184, Score: 83.0, AVG intrinsic reward: 0.004271\n",
      "Episodes: 556, AVG Score: 87.375, Score: 186.0, AVG intrinsic reward: 0.003523\n",
      "Episodes: 557, AVG Score: 89.806, Score: 136.0, AVG intrinsic reward: 0.003966\n",
      "Episodes: 558, AVG Score: 91.366, Score: 121.0, AVG intrinsic reward: 0.009850\n",
      "Episodes: 559, AVG Score: 92.548, Score: 115.0, AVG intrinsic reward: 0.004006\n",
      "Episodes: 560, AVG Score: 92.220, Score: 86.0, AVG intrinsic reward: 0.006154\n",
      "Episodes: 561, AVG Score: 93.459, Score: 117.0, AVG intrinsic reward: 0.003083\n",
      "Episodes: 562, AVG Score: 95.036, Score: 125.0, AVG intrinsic reward: 0.006858\n",
      "Episodes: 563, AVG Score: 96.834, Score: 131.0, AVG intrinsic reward: 0.003374\n",
      "Episodes: 564, AVG Score: 99.593, Score: 152.0, AVG intrinsic reward: 0.006009\n",
      "Episodes: 565, AVG Score: 101.163, Score: 131.0, AVG intrinsic reward: 0.005481\n",
      "Episodes: 566, AVG Score: 106.305, Score: 204.0, AVG intrinsic reward: 0.004016\n",
      "Episodes: 567, AVG Score: 114.290, Score: 266.0, AVG intrinsic reward: 0.006573\n",
      "Episodes: 568, AVG Score: 112.775, Score: 84.0, AVG intrinsic reward: 0.023013\n",
      "Episodes: 569, AVG Score: 110.586, Score: 69.0, AVG intrinsic reward: 0.025599\n",
      "Episodes: 570, AVG Score: 108.757, Score: 74.0, AVG intrinsic reward: 0.004830\n",
      "Episodes: 571, AVG Score: 105.419, Score: 42.0, AVG intrinsic reward: 0.010765\n",
      "Episodes: 572, AVG Score: 107.098, Score: 139.0, AVG intrinsic reward: 0.002295\n",
      "Episodes: 573, AVG Score: 112.143, Score: 208.0, AVG intrinsic reward: 0.001317\n",
      "Episodes: 574, AVG Score: 116.786, Score: 205.0, AVG intrinsic reward: 0.005491\n",
      "Episodes: 575, AVG Score: 118.397, Score: 149.0, AVG intrinsic reward: 0.011364\n",
      "Episodes: 576, AVG Score: 121.877, Score: 188.0, AVG intrinsic reward: 0.004535\n",
      "Episodes: 577, AVG Score: 140.733, Score: 499.0, AVG intrinsic reward: 0.007083\n",
      "Episodes: 578, AVG Score: 141.547, Score: 157.0, AVG intrinsic reward: 0.005683\n",
      "Episodes: 579, AVG Score: 159.419, Score: 499.0, AVG intrinsic reward: 0.013419\n",
      "Episodes: 580, AVG Score: 176.398, Score: 499.0, AVG intrinsic reward: 0.004869\n",
      "Episodes: 581, AVG Score: 192.528, Score: 499.0, AVG intrinsic reward: 0.022396\n",
      "Episodes: 582, AVG Score: 207.852, Score: 499.0, AVG intrinsic reward: 0.011919\n",
      "Episodes: 583, AVG Score: 222.409, Score: 499.0, AVG intrinsic reward: 0.022388\n",
      "Episodes: 584, AVG Score: 236.239, Score: 499.0, AVG intrinsic reward: 0.044412\n",
      "Episodes: 585, AVG Score: 249.377, Score: 499.0, AVG intrinsic reward: 0.013005\n",
      "Episodes: 586, AVG Score: 261.858, Score: 499.0, AVG intrinsic reward: 0.008847\n",
      "Episodes: 587, AVG Score: 273.715, Score: 499.0, AVG intrinsic reward: 0.022209\n",
      "Episodes: 588, AVG Score: 284.979, Score: 499.0, AVG intrinsic reward: 0.006251\n",
      "Episodes: 589, AVG Score: 295.680, Score: 499.0, AVG intrinsic reward: 0.012141\n",
      "Episodes: 590, AVG Score: 305.846, Score: 499.0, AVG intrinsic reward: 0.026167\n",
      "Episodes: 591, AVG Score: 315.504, Score: 499.0, AVG intrinsic reward: 0.007543\n",
      "Episodes: 592, AVG Score: 324.679, Score: 499.0, AVG intrinsic reward: 0.008386\n",
      "Episodes: 593, AVG Score: 333.395, Score: 499.0, AVG intrinsic reward: 0.011689\n",
      "Episodes: 594, AVG Score: 341.675, Score: 499.0, AVG intrinsic reward: 0.005839\n",
      "Episodes: 595, AVG Score: 349.541, Score: 499.0, AVG intrinsic reward: 0.054360\n",
      "Episodes: 596, AVG Score: 357.014, Score: 499.0, AVG intrinsic reward: 0.002334\n",
      "Episodes: 597, AVG Score: 364.114, Score: 499.0, AVG intrinsic reward: 0.067265\n",
      "Episodes: 598, AVG Score: 360.908, Score: 300.0, AVG intrinsic reward: 0.009566\n",
      "Episodes: 599, AVG Score: 367.813, Score: 499.0, AVG intrinsic reward: 0.015477\n",
      "Episodes: 600, AVG Score: 374.372, Score: 499.0, AVG intrinsic reward: 0.001603\n",
      "Episodes: 601, AVG Score: 380.603, Score: 499.0, AVG intrinsic reward: 0.001010\n",
      "Episodes: 602, AVG Score: 386.523, Score: 499.0, AVG intrinsic reward: 0.000326\n",
      "Episodes: 603, AVG Score: 392.147, Score: 499.0, AVG intrinsic reward: 0.000246\n",
      "Episodes: 604, AVG Score: 397.490, Score: 499.0, AVG intrinsic reward: 0.000277\n",
      "Episodes: 605, AVG Score: 402.565, Score: 499.0, AVG intrinsic reward: 0.000874\n",
      "Episodes: 606, AVG Score: 407.387, Score: 499.0, AVG intrinsic reward: 0.000973\n",
      "Episodes: 607, AVG Score: 411.968, Score: 499.0, AVG intrinsic reward: 0.000967\n",
      "Episodes: 608, AVG Score: 416.319, Score: 499.0, AVG intrinsic reward: 0.001631\n",
      "Episodes: 609, AVG Score: 420.453, Score: 499.0, AVG intrinsic reward: 0.001332\n",
      "Episodes: 610, AVG Score: 424.381, Score: 499.0, AVG intrinsic reward: 0.001623\n",
      "Episodes: 611, AVG Score: 428.112, Score: 499.0, AVG intrinsic reward: 0.001296\n",
      "Episodes: 612, AVG Score: 431.656, Score: 499.0, AVG intrinsic reward: 0.000996\n",
      "Episodes: 613, AVG Score: 435.023, Score: 499.0, AVG intrinsic reward: 0.001286\n",
      "Episodes: 614, AVG Score: 438.222, Score: 499.0, AVG intrinsic reward: 0.001221\n",
      "Episodes: 615, AVG Score: 441.261, Score: 499.0, AVG intrinsic reward: 0.000818\n",
      "Episodes: 616, AVG Score: 444.148, Score: 499.0, AVG intrinsic reward: 0.001097\n",
      "Episodes: 617, AVG Score: 446.890, Score: 499.0, AVG intrinsic reward: 0.001113\n",
      "Episodes: 618, AVG Score: 449.496, Score: 499.0, AVG intrinsic reward: 0.000881\n",
      "Episodes: 619, AVG Score: 451.971, Score: 499.0, AVG intrinsic reward: 0.000617\n",
      "Episodes: 620, AVG Score: 454.323, Score: 499.0, AVG intrinsic reward: 0.002347\n",
      "Episodes: 621, AVG Score: 456.556, Score: 499.0, AVG intrinsic reward: 0.000263\n",
      "Episodes: 622, AVG Score: 458.679, Score: 499.0, AVG intrinsic reward: 0.000415\n",
      "Episodes: 623, AVG Score: 460.695, Score: 499.0, AVG intrinsic reward: 0.000334\n",
      "Episodes: 624, AVG Score: 462.610, Score: 499.0, AVG intrinsic reward: 0.000746\n",
      "Episodes: 625, AVG Score: 464.429, Score: 499.0, AVG intrinsic reward: 0.000652\n",
      "Episodes: 626, AVG Score: 466.158, Score: 499.0, AVG intrinsic reward: 0.000669\n",
      "Episodes: 627, AVG Score: 467.800, Score: 499.0, AVG intrinsic reward: 0.000351\n",
      "Episodes: 628, AVG Score: 469.360, Score: 499.0, AVG intrinsic reward: 0.001070\n",
      "Episodes: 629, AVG Score: 470.842, Score: 499.0, AVG intrinsic reward: 0.000424\n",
      "Episodes: 630, AVG Score: 472.250, Score: 499.0, AVG intrinsic reward: 0.000730\n",
      "Episodes: 631, AVG Score: 473.587, Score: 499.0, AVG intrinsic reward: 0.000465\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 632, AVG Score: 474.858, Score: 499.0, AVG intrinsic reward: 0.000706\n",
      "Episodes: 633, AVG Score: 476.065, Score: 499.0, AVG intrinsic reward: 0.000440\n",
      "Episodes: 634, AVG Score: 477.212, Score: 499.0, AVG intrinsic reward: 0.000685\n",
      "Episodes: 635, AVG Score: 478.301, Score: 499.0, AVG intrinsic reward: 0.000952\n",
      "Episodes: 636, AVG Score: 479.336, Score: 499.0, AVG intrinsic reward: 0.000376\n",
      "Episodes: 637, AVG Score: 480.319, Score: 499.0, AVG intrinsic reward: 0.001454\n",
      "Episodes: 638, AVG Score: 480.353, Score: 481.0, AVG intrinsic reward: 0.002852\n",
      "Episodes: 639, AVG Score: 470.936, Score: 292.0, AVG intrinsic reward: 0.001437\n",
      "Episodes: 640, AVG Score: 457.739, Score: 207.0, AVG intrinsic reward: 0.002316\n",
      "Episodes: 641, AVG Score: 446.352, Score: 230.0, AVG intrinsic reward: 0.000734\n",
      "Episodes: 642, AVG Score: 432.734, Score: 174.0, AVG intrinsic reward: 0.000991\n",
      "Episodes: 643, AVG Score: 420.248, Score: 183.0, AVG intrinsic reward: 0.001803\n",
      "Episodes: 644, AVG Score: 409.435, Score: 204.0, AVG intrinsic reward: 0.001179\n",
      "Episodes: 645, AVG Score: 400.814, Score: 237.0, AVG intrinsic reward: 0.000804\n",
      "Episodes: 646, AVG Score: 405.723, Score: 499.0, AVG intrinsic reward: 0.000783\n",
      "Episodes: 647, AVG Score: 410.387, Score: 499.0, AVG intrinsic reward: 0.000210\n",
      "Episodes: 648, AVG Score: 414.817, Score: 499.0, AVG intrinsic reward: 0.000254\n",
      "Episodes: 649, AVG Score: 419.027, Score: 499.0, AVG intrinsic reward: 0.000375\n",
      "Episodes: 650, AVG Score: 423.025, Score: 499.0, AVG intrinsic reward: 0.000680\n",
      "Episodes: 651, AVG Score: 426.824, Score: 499.0, AVG intrinsic reward: 0.000362\n",
      "Episodes: 652, AVG Score: 430.433, Score: 499.0, AVG intrinsic reward: 0.000500\n",
      "Episodes: 653, AVG Score: 433.861, Score: 499.0, AVG intrinsic reward: 0.000721\n",
      "Episodes: 654, AVG Score: 437.118, Score: 499.0, AVG intrinsic reward: 0.000696\n",
      "Episodes: 655, AVG Score: 440.212, Score: 499.0, AVG intrinsic reward: 0.000342\n",
      "Episodes: 656, AVG Score: 443.152, Score: 499.0, AVG intrinsic reward: 0.000428\n",
      "Episodes: 657, AVG Score: 445.944, Score: 499.0, AVG intrinsic reward: 0.001650\n",
      "Episodes: 658, AVG Score: 448.597, Score: 499.0, AVG intrinsic reward: 0.000794\n",
      "Episodes: 659, AVG Score: 451.117, Score: 499.0, AVG intrinsic reward: 0.000546\n",
      "Episodes: 660, AVG Score: 453.511, Score: 499.0, AVG intrinsic reward: 0.002039\n",
      "Episodes: 661, AVG Score: 455.786, Score: 499.0, AVG intrinsic reward: 0.000787\n",
      "Episodes: 662, AVG Score: 457.946, Score: 499.0, AVG intrinsic reward: 0.001230\n",
      "Episodes: 663, AVG Score: 459.999, Score: 499.0, AVG intrinsic reward: 0.003747\n",
      "Episodes: 664, AVG Score: 461.949, Score: 499.0, AVG intrinsic reward: 0.000228\n",
      "Episodes: 665, AVG Score: 463.802, Score: 499.0, AVG intrinsic reward: 0.000305\n",
      "Episodes: 666, AVG Score: 465.561, Score: 499.0, AVG intrinsic reward: 0.000565\n",
      "Episodes: 667, AVG Score: 467.233, Score: 499.0, AVG intrinsic reward: 0.001974\n",
      "Episodes: 668, AVG Score: 468.822, Score: 499.0, AVG intrinsic reward: 0.000688\n",
      "Episodes: 669, AVG Score: 470.331, Score: 499.0, AVG intrinsic reward: 0.005544\n",
      "Episodes: 670, AVG Score: 471.764, Score: 499.0, AVG intrinsic reward: 0.000684\n",
      "Episodes: 671, AVG Score: 473.126, Score: 499.0, AVG intrinsic reward: 0.000214\n",
      "Episodes: 672, AVG Score: 474.420, Score: 499.0, AVG intrinsic reward: 0.000116\n",
      "Episodes: 673, AVG Score: 475.649, Score: 499.0, AVG intrinsic reward: 0.000345\n",
      "Episodes: 674, AVG Score: 476.816, Score: 499.0, AVG intrinsic reward: 0.000685\n",
      "Episodes: 675, AVG Score: 477.925, Score: 499.0, AVG intrinsic reward: 0.003819\n",
      "Episodes: 676, AVG Score: 478.979, Score: 499.0, AVG intrinsic reward: 0.000220\n",
      "Episodes: 677, AVG Score: 479.980, Score: 499.0, AVG intrinsic reward: 0.000304\n",
      "Episodes: 678, AVG Score: 480.931, Score: 499.0, AVG intrinsic reward: 0.000222\n",
      "Episodes: 679, AVG Score: 481.835, Score: 499.0, AVG intrinsic reward: 0.001105\n",
      "Episodes: 680, AVG Score: 482.693, Score: 499.0, AVG intrinsic reward: 0.000359\n",
      "Episodes: 681, AVG Score: 483.508, Score: 499.0, AVG intrinsic reward: 0.000256\n",
      "Episodes: 682, AVG Score: 484.283, Score: 499.0, AVG intrinsic reward: 0.000569\n",
      "Episodes: 683, AVG Score: 485.019, Score: 499.0, AVG intrinsic reward: 0.000388\n",
      "Episodes: 684, AVG Score: 485.718, Score: 499.0, AVG intrinsic reward: 0.002209\n",
      "Episodes: 685, AVG Score: 486.382, Score: 499.0, AVG intrinsic reward: 0.000343\n",
      "Episodes: 686, AVG Score: 487.013, Score: 499.0, AVG intrinsic reward: 0.000208\n",
      "Episodes: 687, AVG Score: 487.612, Score: 499.0, AVG intrinsic reward: 0.000724\n",
      "Episodes: 688, AVG Score: 488.182, Score: 499.0, AVG intrinsic reward: 0.000360\n",
      "Episodes: 689, AVG Score: 488.722, Score: 499.0, AVG intrinsic reward: 0.000297\n",
      "Episodes: 690, AVG Score: 489.236, Score: 499.0, AVG intrinsic reward: 0.000156\n",
      "Episodes: 691, AVG Score: 489.725, Score: 499.0, AVG intrinsic reward: 0.000460\n",
      "Episodes: 692, AVG Score: 490.188, Score: 499.0, AVG intrinsic reward: 0.000416\n",
      "Episodes: 693, AVG Score: 490.629, Score: 499.0, AVG intrinsic reward: 0.000451\n",
      "Episodes: 694, AVG Score: 491.047, Score: 499.0, AVG intrinsic reward: 0.000382\n",
      "Episodes: 695, AVG Score: 491.445, Score: 499.0, AVG intrinsic reward: 0.000308\n",
      "Episodes: 696, AVG Score: 491.823, Score: 499.0, AVG intrinsic reward: 0.000332\n",
      "Episodes: 697, AVG Score: 492.182, Score: 499.0, AVG intrinsic reward: 0.000528\n",
      "Episodes: 698, AVG Score: 492.523, Score: 499.0, AVG intrinsic reward: 0.001394\n",
      "Episodes: 699, AVG Score: 492.846, Score: 499.0, AVG intrinsic reward: 0.002591\n",
      "Episodes: 700, AVG Score: 493.154, Score: 499.0, AVG intrinsic reward: 0.000973\n",
      "Episodes: 701, AVG Score: 493.446, Score: 499.0, AVG intrinsic reward: 0.000418\n",
      "Episodes: 702, AVG Score: 493.724, Score: 499.0, AVG intrinsic reward: 0.000424\n",
      "Episodes: 703, AVG Score: 493.988, Score: 499.0, AVG intrinsic reward: 0.000238\n",
      "Episodes: 704, AVG Score: 494.238, Score: 499.0, AVG intrinsic reward: 0.000216\n",
      "Episodes: 705, AVG Score: 494.477, Score: 499.0, AVG intrinsic reward: 0.000520\n",
      "Episodes: 706, AVG Score: 494.703, Score: 499.0, AVG intrinsic reward: 0.001217\n",
      "Episodes: 707, AVG Score: 494.918, Score: 499.0, AVG intrinsic reward: 0.000317\n",
      "Episodes: 708, AVG Score: 495.122, Score: 499.0, AVG intrinsic reward: 0.000535\n",
      "Episodes: 709, AVG Score: 495.316, Score: 499.0, AVG intrinsic reward: 0.000189\n",
      "Episodes: 710, AVG Score: 495.500, Score: 499.0, AVG intrinsic reward: 0.000631\n",
      "Episodes: 711, AVG Score: 495.675, Score: 499.0, AVG intrinsic reward: 0.000124\n",
      "Episodes: 712, AVG Score: 495.841, Score: 499.0, AVG intrinsic reward: 0.001511\n",
      "Episodes: 713, AVG Score: 495.999, Score: 499.0, AVG intrinsic reward: 0.000206\n",
      "Episodes: 714, AVG Score: 496.149, Score: 499.0, AVG intrinsic reward: 0.000197\n",
      "Episodes: 715, AVG Score: 496.292, Score: 499.0, AVG intrinsic reward: 0.000310\n",
      "Episodes: 716, AVG Score: 496.427, Score: 499.0, AVG intrinsic reward: 0.001862\n",
      "Episodes: 717, AVG Score: 496.556, Score: 499.0, AVG intrinsic reward: 0.000311\n",
      "Episodes: 718, AVG Score: 496.678, Score: 499.0, AVG intrinsic reward: 0.000801\n",
      "Episodes: 719, AVG Score: 496.794, Score: 499.0, AVG intrinsic reward: 0.000847\n",
      "Episodes: 720, AVG Score: 496.904, Score: 499.0, AVG intrinsic reward: 0.000849\n",
      "Episodes: 721, AVG Score: 497.009, Score: 499.0, AVG intrinsic reward: 0.001234\n",
      "Episodes: 722, AVG Score: 497.109, Score: 499.0, AVG intrinsic reward: 0.000857\n",
      "Episodes: 723, AVG Score: 497.203, Score: 499.0, AVG intrinsic reward: 0.003769\n",
      "Episodes: 724, AVG Score: 497.293, Score: 499.0, AVG intrinsic reward: 0.003498\n",
      "Episodes: 725, AVG Score: 494.578, Score: 443.0, AVG intrinsic reward: 0.007641\n",
      "Episodes: 726, AVG Score: 472.949, Score: 62.0, AVG intrinsic reward: 0.017606\n",
      "Episodes: 727, AVG Score: 474.252, Score: 499.0, AVG intrinsic reward: 0.002801\n",
      "Episodes: 728, AVG Score: 451.089, Score: 11.0, AVG intrinsic reward: 0.025400\n",
      "Episodes: 729, AVG Score: 429.135, Score: 12.0, AVG intrinsic reward: 0.071580\n",
      "Episodes: 730, AVG Score: 408.278, Score: 12.0, AVG intrinsic reward: 0.055722\n",
      "Episodes: 731, AVG Score: 388.414, Score: 11.0, AVG intrinsic reward: 0.071484\n",
      "Episodes: 732, AVG Score: 369.494, Score: 10.0, AVG intrinsic reward: 0.056579\n",
      "Episodes: 733, AVG Score: 351.619, Score: 12.0, AVG intrinsic reward: 0.033606\n",
      "Episodes: 734, AVG Score: 334.688, Score: 13.0, AVG intrinsic reward: 0.044766\n",
      "Episodes: 735, AVG Score: 318.604, Score: 13.0, AVG intrinsic reward: 0.020875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 736, AVG Score: 305.523, Score: 57.0, AVG intrinsic reward: 0.006507\n",
      "Episodes: 737, AVG Score: 315.197, Score: 499.0, AVG intrinsic reward: 0.002749\n",
      "Episodes: 738, AVG Score: 324.387, Score: 499.0, AVG intrinsic reward: 0.001371\n",
      "Episodes: 739, AVG Score: 315.868, Score: 154.0, AVG intrinsic reward: 0.003509\n",
      "Episodes: 740, AVG Score: 300.575, Score: 10.0, AVG intrinsic reward: 0.011394\n",
      "Episodes: 741, AVG Score: 310.496, Score: 499.0, AVG intrinsic reward: 0.000372\n",
      "Episodes: 742, AVG Score: 319.921, Score: 499.0, AVG intrinsic reward: 0.000037\n",
      "Episodes: 743, AVG Score: 328.875, Score: 499.0, AVG intrinsic reward: 0.000528\n",
      "Episodes: 744, AVG Score: 337.381, Score: 499.0, AVG intrinsic reward: 0.000388\n",
      "Episodes: 745, AVG Score: 345.462, Score: 499.0, AVG intrinsic reward: 0.000206\n",
      "Episodes: 746, AVG Score: 353.139, Score: 499.0, AVG intrinsic reward: 0.000431\n",
      "Episodes: 747, AVG Score: 360.432, Score: 499.0, AVG intrinsic reward: 0.000377\n",
      "Episodes: 748, AVG Score: 367.361, Score: 499.0, AVG intrinsic reward: 0.000361\n",
      "Episodes: 749, AVG Score: 373.942, Score: 499.0, AVG intrinsic reward: 0.000612\n",
      "Episodes: 750, AVG Score: 380.195, Score: 499.0, AVG intrinsic reward: 0.000592\n",
      "Episodes: 751, AVG Score: 386.136, Score: 499.0, AVG intrinsic reward: 0.002456\n",
      "Episodes: 752, AVG Score: 391.779, Score: 499.0, AVG intrinsic reward: 0.001085\n",
      "Episodes: 753, AVG Score: 397.140, Score: 499.0, AVG intrinsic reward: 0.001517\n",
      "Episodes: 754, AVG Score: 402.233, Score: 499.0, AVG intrinsic reward: 0.000613\n",
      "Episodes: 755, AVG Score: 407.071, Score: 499.0, AVG intrinsic reward: 0.000680\n",
      "Episodes: 756, AVG Score: 411.668, Score: 499.0, AVG intrinsic reward: 0.000825\n",
      "Episodes: 757, AVG Score: 416.034, Score: 499.0, AVG intrinsic reward: 0.001044\n",
      "Episodes: 758, AVG Score: 420.183, Score: 499.0, AVG intrinsic reward: 0.002747\n",
      "Episodes: 759, AVG Score: 424.123, Score: 499.0, AVG intrinsic reward: 0.000397\n",
      "Episodes: 760, AVG Score: 427.867, Score: 499.0, AVG intrinsic reward: 0.000346\n",
      "Episodes: 761, AVG Score: 431.424, Score: 499.0, AVG intrinsic reward: 0.000335\n",
      "Episodes: 762, AVG Score: 434.803, Score: 499.0, AVG intrinsic reward: 0.000313\n",
      "Episodes: 763, AVG Score: 438.013, Score: 499.0, AVG intrinsic reward: 0.000581\n",
      "Episodes: 764, AVG Score: 441.062, Score: 499.0, AVG intrinsic reward: 0.000616\n",
      "Episodes: 765, AVG Score: 443.959, Score: 499.0, AVG intrinsic reward: 0.002127\n",
      "Episodes: 766, AVG Score: 446.711, Score: 499.0, AVG intrinsic reward: 0.000579\n",
      "Episodes: 767, AVG Score: 449.325, Score: 499.0, AVG intrinsic reward: 0.000282\n",
      "Episodes: 768, AVG Score: 451.809, Score: 499.0, AVG intrinsic reward: 0.000260\n",
      "Episodes: 769, AVG Score: 454.169, Score: 499.0, AVG intrinsic reward: 0.000259\n",
      "Episodes: 770, AVG Score: 456.410, Score: 499.0, AVG intrinsic reward: 0.000233\n",
      "Episodes: 771, AVG Score: 458.540, Score: 499.0, AVG intrinsic reward: 0.000457\n",
      "Episodes: 772, AVG Score: 460.563, Score: 499.0, AVG intrinsic reward: 0.000113\n",
      "Episodes: 773, AVG Score: 462.485, Score: 499.0, AVG intrinsic reward: 0.000923\n",
      "Episodes: 774, AVG Score: 464.310, Score: 499.0, AVG intrinsic reward: 0.000166\n",
      "Episodes: 775, AVG Score: 466.045, Score: 499.0, AVG intrinsic reward: 0.000258\n",
      "Episodes: 776, AVG Score: 467.693, Score: 499.0, AVG intrinsic reward: 0.000255\n",
      "Episodes: 777, AVG Score: 469.258, Score: 499.0, AVG intrinsic reward: 0.000689\n",
      "Episodes: 778, AVG Score: 470.745, Score: 499.0, AVG intrinsic reward: 0.000601\n",
      "Episodes: 779, AVG Score: 472.158, Score: 499.0, AVG intrinsic reward: 0.000195\n",
      "Episodes: 780, AVG Score: 473.500, Score: 499.0, AVG intrinsic reward: 0.000066\n",
      "Episodes: 781, AVG Score: 474.775, Score: 499.0, AVG intrinsic reward: 0.000461\n",
      "Episodes: 782, AVG Score: 475.986, Score: 499.0, AVG intrinsic reward: 0.000327\n",
      "Episodes: 783, AVG Score: 477.137, Score: 499.0, AVG intrinsic reward: 0.000315\n",
      "Episodes: 784, AVG Score: 478.230, Score: 499.0, AVG intrinsic reward: 0.000793\n",
      "Episodes: 785, AVG Score: 479.269, Score: 499.0, AVG intrinsic reward: 0.000154\n",
      "Episodes: 786, AVG Score: 480.255, Score: 499.0, AVG intrinsic reward: 0.000068\n",
      "Episodes: 787, AVG Score: 481.192, Score: 499.0, AVG intrinsic reward: 0.000072\n",
      "Episodes: 788, AVG Score: 482.083, Score: 499.0, AVG intrinsic reward: 0.000208\n",
      "Episodes: 789, AVG Score: 482.929, Score: 499.0, AVG intrinsic reward: 0.003906\n",
      "Episodes: 790, AVG Score: 483.732, Score: 499.0, AVG intrinsic reward: 0.000066\n",
      "Episodes: 791, AVG Score: 484.496, Score: 499.0, AVG intrinsic reward: 0.020222\n",
      "Episodes: 792, AVG Score: 485.221, Score: 499.0, AVG intrinsic reward: 0.001241\n",
      "Episodes: 793, AVG Score: 485.910, Score: 499.0, AVG intrinsic reward: 0.000120\n",
      "Episodes: 794, AVG Score: 486.564, Score: 499.0, AVG intrinsic reward: 0.000236\n",
      "Episodes: 795, AVG Score: 487.186, Score: 499.0, AVG intrinsic reward: 0.000119\n",
      "Episodes: 796, AVG Score: 487.777, Score: 499.0, AVG intrinsic reward: 0.000135\n",
      "Episodes: 797, AVG Score: 488.338, Score: 499.0, AVG intrinsic reward: 0.000049\n",
      "Episodes: 798, AVG Score: 488.871, Score: 499.0, AVG intrinsic reward: 0.000112\n",
      "Episodes: 799, AVG Score: 489.377, Score: 499.0, AVG intrinsic reward: 0.000172\n",
      "Episodes: 800, AVG Score: 489.859, Score: 499.0, AVG intrinsic reward: 0.000366\n",
      "Episodes: 801, AVG Score: 490.316, Score: 499.0, AVG intrinsic reward: 0.000098\n",
      "Episodes: 802, AVG Score: 490.750, Score: 499.0, AVG intrinsic reward: 0.000231\n",
      "Episodes: 803, AVG Score: 491.162, Score: 499.0, AVG intrinsic reward: 0.000356\n",
      "Episodes: 804, AVG Score: 491.554, Score: 499.0, AVG intrinsic reward: 0.000140\n",
      "Episodes: 805, AVG Score: 491.927, Score: 499.0, AVG intrinsic reward: 0.000182\n",
      "Episodes: 806, AVG Score: 492.280, Score: 499.0, AVG intrinsic reward: 0.000085\n",
      "Episodes: 807, AVG Score: 492.616, Score: 499.0, AVG intrinsic reward: 0.000470\n",
      "Episodes: 808, AVG Score: 492.935, Score: 499.0, AVG intrinsic reward: 0.000080\n",
      "Episodes: 809, AVG Score: 493.239, Score: 499.0, AVG intrinsic reward: 0.000157\n",
      "Episodes: 810, AVG Score: 493.527, Score: 499.0, AVG intrinsic reward: 0.000060\n",
      "Episodes: 811, AVG Score: 493.800, Score: 499.0, AVG intrinsic reward: 0.000377\n",
      "Episodes: 812, AVG Score: 494.060, Score: 499.0, AVG intrinsic reward: 0.000226\n",
      "Episodes: 813, AVG Score: 494.307, Score: 499.0, AVG intrinsic reward: 0.000144\n",
      "Episodes: 814, AVG Score: 494.542, Score: 499.0, AVG intrinsic reward: 0.000274\n",
      "Episodes: 815, AVG Score: 494.765, Score: 499.0, AVG intrinsic reward: 0.000137\n",
      "Episodes: 816, AVG Score: 494.977, Score: 499.0, AVG intrinsic reward: 0.000255\n",
      "Episodes: 817, AVG Score: 495.178, Score: 499.0, AVG intrinsic reward: 0.000087\n",
      "Episodes: 818, AVG Score: 495.369, Score: 499.0, AVG intrinsic reward: 0.000310\n",
      "Episodes: 819, AVG Score: 495.550, Score: 499.0, AVG intrinsic reward: 0.000463\n",
      "Episodes: 820, AVG Score: 495.723, Score: 499.0, AVG intrinsic reward: 0.000182\n",
      "Episodes: 821, AVG Score: 495.887, Score: 499.0, AVG intrinsic reward: 0.000237\n",
      "Episodes: 822, AVG Score: 496.042, Score: 499.0, AVG intrinsic reward: 0.000844\n",
      "Episodes: 823, AVG Score: 496.190, Score: 499.0, AVG intrinsic reward: 0.000084\n",
      "Episodes: 824, AVG Score: 496.331, Score: 499.0, AVG intrinsic reward: 0.000398\n",
      "Episodes: 825, AVG Score: 496.464, Score: 499.0, AVG intrinsic reward: 0.000148\n",
      "Episodes: 826, AVG Score: 496.591, Score: 499.0, AVG intrinsic reward: 0.000234\n",
      "Episodes: 827, AVG Score: 496.711, Score: 499.0, AVG intrinsic reward: 0.000570\n",
      "Episodes: 828, AVG Score: 496.826, Score: 499.0, AVG intrinsic reward: 0.000165\n",
      "Episodes: 829, AVG Score: 496.935, Score: 499.0, AVG intrinsic reward: 0.000177\n",
      "Episodes: 830, AVG Score: 497.038, Score: 499.0, AVG intrinsic reward: 0.000121\n",
      "Episodes: 831, AVG Score: 497.136, Score: 499.0, AVG intrinsic reward: 0.000104\n",
      "Episodes: 832, AVG Score: 497.229, Score: 499.0, AVG intrinsic reward: 0.000246\n",
      "Episodes: 833, AVG Score: 497.318, Score: 499.0, AVG intrinsic reward: 0.000250\n",
      "Episodes: 834, AVG Score: 497.402, Score: 499.0, AVG intrinsic reward: 0.000332\n",
      "Episodes: 835, AVG Score: 497.482, Score: 499.0, AVG intrinsic reward: 0.000076\n",
      "Episodes: 836, AVG Score: 497.558, Score: 499.0, AVG intrinsic reward: 0.000078\n",
      "Episodes: 837, AVG Score: 497.630, Score: 499.0, AVG intrinsic reward: 0.000084\n",
      "Episodes: 838, AVG Score: 497.698, Score: 499.0, AVG intrinsic reward: 0.000162\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 839, AVG Score: 497.763, Score: 499.0, AVG intrinsic reward: 0.000078\n",
      "Episodes: 840, AVG Score: 497.825, Score: 499.0, AVG intrinsic reward: 0.000061\n",
      "Episodes: 841, AVG Score: 497.884, Score: 499.0, AVG intrinsic reward: 0.000322\n",
      "Episodes: 842, AVG Score: 497.940, Score: 499.0, AVG intrinsic reward: 0.000065\n",
      "Episodes: 843, AVG Score: 497.993, Score: 499.0, AVG intrinsic reward: 0.000105\n",
      "Episodes: 844, AVG Score: 498.043, Score: 499.0, AVG intrinsic reward: 0.000410\n",
      "Episodes: 845, AVG Score: 498.091, Score: 499.0, AVG intrinsic reward: 0.000013\n",
      "Episodes: 846, AVG Score: 498.136, Score: 499.0, AVG intrinsic reward: 0.000028\n",
      "Episodes: 847, AVG Score: 498.180, Score: 499.0, AVG intrinsic reward: 0.000042\n",
      "Episodes: 848, AVG Score: 498.221, Score: 499.0, AVG intrinsic reward: 0.000150\n",
      "Episodes: 849, AVG Score: 498.260, Score: 499.0, AVG intrinsic reward: 0.000193\n",
      "Episodes: 850, AVG Score: 498.297, Score: 499.0, AVG intrinsic reward: 0.000536\n",
      "Episodes: 851, AVG Score: 498.332, Score: 499.0, AVG intrinsic reward: 0.000040\n",
      "Episodes: 852, AVG Score: 498.365, Score: 499.0, AVG intrinsic reward: 0.000828\n",
      "Episodes: 853, AVG Score: 498.397, Score: 499.0, AVG intrinsic reward: 0.000449\n",
      "Episodes: 854, AVG Score: 498.427, Score: 499.0, AVG intrinsic reward: 0.000102\n",
      "Episodes: 855, AVG Score: 498.456, Score: 499.0, AVG intrinsic reward: 0.000184\n",
      "Episodes: 856, AVG Score: 498.483, Score: 499.0, AVG intrinsic reward: 0.000216\n",
      "Episodes: 857, AVG Score: 498.509, Score: 499.0, AVG intrinsic reward: 0.000142\n",
      "Episodes: 858, AVG Score: 498.533, Score: 499.0, AVG intrinsic reward: 0.000154\n",
      "Episodes: 859, AVG Score: 498.557, Score: 499.0, AVG intrinsic reward: 0.000164\n",
      "Episodes: 860, AVG Score: 498.579, Score: 499.0, AVG intrinsic reward: 0.000189\n",
      "Episodes: 861, AVG Score: 498.600, Score: 499.0, AVG intrinsic reward: 0.000347\n",
      "Episodes: 862, AVG Score: 498.620, Score: 499.0, AVG intrinsic reward: 0.000503\n",
      "Episodes: 863, AVG Score: 498.639, Score: 499.0, AVG intrinsic reward: 0.000209\n",
      "Episodes: 864, AVG Score: 498.657, Score: 499.0, AVG intrinsic reward: 0.000383\n",
      "Episodes: 865, AVG Score: 496.024, Score: 446.0, AVG intrinsic reward: 0.000439\n",
      "Episodes: 866, AVG Score: 471.673, Score: 9.0, AVG intrinsic reward: 0.002921\n",
      "Episodes: 867, AVG Score: 473.039, Score: 499.0, AVG intrinsic reward: 0.001479\n",
      "Episodes: 868, AVG Score: 464.137, Score: 295.0, AVG intrinsic reward: 0.000263\n",
      "Episodes: 869, AVG Score: 441.580, Score: 13.0, AVG intrinsic reward: 0.001659\n",
      "Episodes: 870, AVG Score: 425.051, Score: 111.0, AVG intrinsic reward: 0.001585\n",
      "Episodes: 871, AVG Score: 409.199, Score: 108.0, AVG intrinsic reward: 0.001428\n",
      "Episodes: 872, AVG Score: 394.139, Score: 108.0, AVG intrinsic reward: 0.001219\n",
      "Episodes: 873, AVG Score: 380.382, Score: 119.0, AVG intrinsic reward: 0.000454\n",
      "Episodes: 874, AVG Score: 366.563, Score: 104.0, AVG intrinsic reward: 0.002669\n",
      "Episodes: 875, AVG Score: 349.085, Score: 17.0, AVG intrinsic reward: 0.003432\n",
      "Episodes: 876, AVG Score: 337.380, Score: 115.0, AVG intrinsic reward: 0.001567\n",
      "Episodes: 877, AVG Score: 326.861, Score: 127.0, AVG intrinsic reward: 0.001427\n",
      "Episodes: 878, AVG Score: 317.718, Score: 144.0, AVG intrinsic reward: 0.000813\n",
      "Episodes: 879, AVG Score: 326.782, Score: 499.0, AVG intrinsic reward: 0.000430\n",
      "Episodes: 880, AVG Score: 335.393, Score: 499.0, AVG intrinsic reward: 0.000360\n",
      "Episodes: 881, AVG Score: 343.574, Score: 499.0, AVG intrinsic reward: 0.000365\n",
      "Episodes: 882, AVG Score: 351.345, Score: 499.0, AVG intrinsic reward: 0.001117\n",
      "Episodes: 883, AVG Score: 358.728, Score: 499.0, AVG intrinsic reward: 0.000136\n",
      "Episodes: 884, AVG Score: 365.741, Score: 499.0, AVG intrinsic reward: 0.000308\n",
      "Episodes: 885, AVG Score: 372.404, Score: 499.0, AVG intrinsic reward: 0.000448\n",
      "Episodes: 886, AVG Score: 378.734, Score: 499.0, AVG intrinsic reward: 0.001188\n",
      "Episodes: 887, AVG Score: 372.497, Score: 254.0, AVG intrinsic reward: 0.000068\n",
      "Episodes: 888, AVG Score: 354.573, Score: 14.0, AVG intrinsic reward: 0.002004\n",
      "Episodes: 889, AVG Score: 361.794, Score: 499.0, AVG intrinsic reward: 0.001121\n",
      "Episodes: 890, AVG Score: 368.654, Score: 499.0, AVG intrinsic reward: 0.001736\n",
      "Episodes: 891, AVG Score: 375.171, Score: 499.0, AVG intrinsic reward: 0.000762\n",
      "Episodes: 892, AVG Score: 381.363, Score: 499.0, AVG intrinsic reward: 0.001548\n",
      "Episodes: 893, AVG Score: 387.245, Score: 499.0, AVG intrinsic reward: 0.001029\n",
      "Episodes: 894, AVG Score: 392.833, Score: 499.0, AVG intrinsic reward: 0.000947\n",
      "Episodes: 895, AVG Score: 395.341, Score: 443.0, AVG intrinsic reward: 0.000606\n",
      "Episodes: 896, AVG Score: 389.274, Score: 274.0, AVG intrinsic reward: 0.001344\n",
      "Episodes: 897, AVG Score: 385.210, Score: 308.0, AVG intrinsic reward: 0.003533\n",
      "Episodes: 898, AVG Score: 366.500, Score: 11.0, AVG intrinsic reward: 0.004550\n",
      "Episodes: 899, AVG Score: 348.725, Score: 11.0, AVG intrinsic reward: 0.010171\n",
      "Episodes: 900, AVG Score: 331.838, Score: 11.0, AVG intrinsic reward: 0.008470\n",
      "Episodes: 901, AVG Score: 315.897, Score: 13.0, AVG intrinsic reward: 0.005299\n",
      "Episodes: 902, AVG Score: 300.552, Score: 9.0, AVG intrinsic reward: 0.007533\n",
      "Episodes: 903, AVG Score: 286.124, Score: 12.0, AVG intrinsic reward: 0.006499\n",
      "Episodes: 904, AVG Score: 272.318, Score: 10.0, AVG intrinsic reward: 0.012148\n",
      "Episodes: 905, AVG Score: 259.302, Score: 12.0, AVG intrinsic reward: 0.005298\n",
      "Episodes: 906, AVG Score: 246.937, Score: 12.0, AVG intrinsic reward: 0.002326\n",
      "Episodes: 907, AVG Score: 235.390, Score: 16.0, AVG intrinsic reward: 0.001181\n",
      "Episodes: 908, AVG Score: 224.521, Score: 18.0, AVG intrinsic reward: 0.001256\n",
      "Episodes: 909, AVG Score: 238.245, Score: 499.0, AVG intrinsic reward: 0.000485\n",
      "Episodes: 910, AVG Score: 238.032, Score: 234.0, AVG intrinsic reward: 0.000276\n",
      "Episodes: 911, AVG Score: 251.081, Score: 499.0, AVG intrinsic reward: 0.001200\n",
      "Episodes: 912, AVG Score: 263.477, Score: 499.0, AVG intrinsic reward: 0.000567\n",
      "Episodes: 913, AVG Score: 275.253, Score: 499.0, AVG intrinsic reward: 0.000144\n",
      "Episodes: 914, AVG Score: 286.440, Score: 499.0, AVG intrinsic reward: 0.000377\n",
      "Episodes: 915, AVG Score: 297.068, Score: 499.0, AVG intrinsic reward: 0.000441\n",
      "Episodes: 916, AVG Score: 307.165, Score: 499.0, AVG intrinsic reward: 0.000635\n",
      "Episodes: 917, AVG Score: 316.757, Score: 499.0, AVG intrinsic reward: 0.001333\n",
      "Episodes: 918, AVG Score: 325.869, Score: 499.0, AVG intrinsic reward: 0.000571\n",
      "Episodes: 919, AVG Score: 334.525, Score: 499.0, AVG intrinsic reward: 0.000732\n",
      "Episodes: 920, AVG Score: 342.749, Score: 499.0, AVG intrinsic reward: 0.002036\n",
      "Episodes: 921, AVG Score: 350.562, Score: 499.0, AVG intrinsic reward: 0.000372\n",
      "Episodes: 922, AVG Score: 357.983, Score: 499.0, AVG intrinsic reward: 0.001294\n",
      "Episodes: 923, AVG Score: 365.034, Score: 499.0, AVG intrinsic reward: 0.000867\n",
      "Episodes: 924, AVG Score: 371.733, Score: 499.0, AVG intrinsic reward: 0.000973\n",
      "Episodes: 925, AVG Score: 378.096, Score: 499.0, AVG intrinsic reward: 0.001701\n",
      "Episodes: 926, AVG Score: 384.141, Score: 499.0, AVG intrinsic reward: 0.004644\n",
      "Episodes: 927, AVG Score: 389.884, Score: 499.0, AVG intrinsic reward: 0.007911\n",
      "Episodes: 928, AVG Score: 395.340, Score: 499.0, AVG intrinsic reward: 0.007877\n",
      "Episodes: 929, AVG Score: 400.523, Score: 499.0, AVG intrinsic reward: 0.010526\n",
      "Episodes: 930, AVG Score: 405.447, Score: 499.0, AVG intrinsic reward: 0.008410\n",
      "Episodes: 931, AVG Score: 410.124, Score: 499.0, AVG intrinsic reward: 0.011810\n",
      "Episodes: 932, AVG Score: 414.568, Score: 499.0, AVG intrinsic reward: 0.001697\n",
      "Episodes: 933, AVG Score: 418.790, Score: 499.0, AVG intrinsic reward: 0.007509\n",
      "Episodes: 934, AVG Score: 422.800, Score: 499.0, AVG intrinsic reward: 0.001625\n",
      "Episodes: 935, AVG Score: 426.610, Score: 499.0, AVG intrinsic reward: 0.001084\n",
      "Episodes: 936, AVG Score: 430.230, Score: 499.0, AVG intrinsic reward: 0.001153\n",
      "Episodes: 937, AVG Score: 433.668, Score: 499.0, AVG intrinsic reward: 0.001636\n",
      "Episodes: 938, AVG Score: 436.935, Score: 499.0, AVG intrinsic reward: 0.001597\n",
      "Episodes: 939, AVG Score: 440.038, Score: 499.0, AVG intrinsic reward: 0.002661\n",
      "Episodes: 940, AVG Score: 437.636, Score: 392.0, AVG intrinsic reward: 0.009607\n",
      "Episodes: 941, AVG Score: 422.104, Score: 127.0, AVG intrinsic reward: 0.017631\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes: 942, AVG Score: 406.249, Score: 105.0, AVG intrinsic reward: 0.024884\n",
      "Episodes: 943, AVG Score: 386.637, Score: 14.0, AVG intrinsic reward: 0.091638\n",
      "Episodes: 944, AVG Score: 367.805, Score: 10.0, AVG intrinsic reward: 0.202953\n",
      "Episodes: 945, AVG Score: 359.915, Score: 210.0, AVG intrinsic reward: 0.038101\n",
      "Episodes: 946, AVG Score: 352.769, Score: 217.0, AVG intrinsic reward: 0.007289\n",
      "Episodes: 947, AVG Score: 344.080, Score: 179.0, AVG intrinsic reward: 0.014950\n",
      "Episodes: 948, AVG Score: 333.476, Score: 132.0, AVG intrinsic reward: 0.013527\n",
      "Episodes: 949, AVG Score: 324.703, Score: 158.0, AVG intrinsic reward: 0.014355\n",
      "Episodes: 950, AVG Score: 316.517, Score: 161.0, AVG intrinsic reward: 0.008304\n",
      "Episodes: 951, AVG Score: 308.692, Score: 160.0, AVG intrinsic reward: 0.007466\n",
      "Episodes: 952, AVG Score: 301.057, Score: 156.0, AVG intrinsic reward: 0.008680\n",
      "Episodes: 953, AVG Score: 294.104, Score: 162.0, AVG intrinsic reward: 0.009982\n",
      "Episodes: 954, AVG Score: 286.849, Score: 149.0, AVG intrinsic reward: 0.009131\n",
      "Episodes: 955, AVG Score: 281.207, Score: 174.0, AVG intrinsic reward: 0.006805\n",
      "Episodes: 956, AVG Score: 276.646, Score: 190.0, AVG intrinsic reward: 0.009255\n",
      "Episodes: 957, AVG Score: 270.714, Score: 158.0, AVG intrinsic reward: 0.007497\n",
      "Episodes: 958, AVG Score: 265.228, Score: 161.0, AVG intrinsic reward: 0.008565\n",
      "Episodes: 959, AVG Score: 260.617, Score: 173.0, AVG intrinsic reward: 0.007396\n",
      "Episodes: 960, AVG Score: 256.686, Score: 182.0, AVG intrinsic reward: 0.008135\n",
      "Episodes: 961, AVG Score: 252.352, Score: 170.0, AVG intrinsic reward: 0.010527\n",
      "Episodes: 962, AVG Score: 248.284, Score: 171.0, AVG intrinsic reward: 0.008440\n",
      "Episodes: 963, AVG Score: 245.470, Score: 192.0, AVG intrinsic reward: 0.006496\n",
      "Episodes: 964, AVG Score: 244.246, Score: 221.0, AVG intrinsic reward: 0.008143\n",
      "Episodes: 965, AVG Score: 243.584, Score: 231.0, AVG intrinsic reward: 0.008111\n",
      "Episodes: 966, AVG Score: 242.455, Score: 221.0, AVG intrinsic reward: 0.009036\n",
      "Episodes: 967, AVG Score: 250.582, Score: 405.0, AVG intrinsic reward: 0.005903\n",
      "Episodes: 968, AVG Score: 252.053, Score: 280.0, AVG intrinsic reward: 0.007867\n",
      "Episodes: 969, AVG Score: 264.400, Score: 499.0, AVG intrinsic reward: 0.008040\n",
      "Episodes: 970, AVG Score: 276.130, Score: 499.0, AVG intrinsic reward: 0.007859\n",
      "Episodes: 971, AVG Score: 287.274, Score: 499.0, AVG intrinsic reward: 0.008291\n",
      "Episodes: 972, AVG Score: 297.860, Score: 499.0, AVG intrinsic reward: 0.008262\n",
      "Episodes: 973, AVG Score: 307.917, Score: 499.0, AVG intrinsic reward: 0.008505\n",
      "Episodes: 974, AVG Score: 316.921, Score: 488.0, AVG intrinsic reward: 0.007282\n",
      "Episodes: 975, AVG Score: 314.775, Score: 274.0, AVG intrinsic reward: 0.009634\n",
      "Episodes: 976, AVG Score: 311.886, Score: 257.0, AVG intrinsic reward: 0.010835\n",
      "Episodes: 977, AVG Score: 320.492, Score: 484.0, AVG intrinsic reward: 0.007418\n",
      "Episodes: 978, AVG Score: 315.968, Score: 230.0, AVG intrinsic reward: 0.010559\n",
      "Episodes: 979, AVG Score: 314.419, Score: 285.0, AVG intrinsic reward: 0.009590\n",
      "Episodes: 980, AVG Score: 315.298, Score: 332.0, AVG intrinsic reward: 0.009772\n",
      "Episodes: 981, AVG Score: 322.383, Score: 457.0, AVG intrinsic reward: 0.010514\n",
      "Episodes: 982, AVG Score: 321.114, Score: 297.0, AVG intrinsic reward: 0.015713\n",
      "Episodes: 983, AVG Score: 318.058, Score: 260.0, AVG intrinsic reward: 0.008347\n",
      "Episodes: 984, AVG Score: 314.955, Score: 256.0, AVG intrinsic reward: 0.014943\n",
      "Episodes: 985, AVG Score: 314.958, Score: 315.0, AVG intrinsic reward: 0.014667\n",
      "Episodes: 986, AVG Score: 318.810, Score: 392.0, AVG intrinsic reward: 0.009139\n",
      "Episodes: 987, AVG Score: 325.169, Score: 446.0, AVG intrinsic reward: 0.010158\n",
      "Episodes: 988, AVG Score: 333.861, Score: 499.0, AVG intrinsic reward: 0.012790\n",
      "Episodes: 989, AVG Score: 342.118, Score: 499.0, AVG intrinsic reward: 0.012906\n",
      "Episodes: 990, AVG Score: 349.962, Score: 499.0, AVG intrinsic reward: 0.012765\n",
      "Episodes: 991, AVG Score: 357.414, Score: 499.0, AVG intrinsic reward: 0.013739\n",
      "Episodes: 992, AVG Score: 358.343, Score: 376.0, AVG intrinsic reward: 0.014663\n",
      "Episodes: 993, AVG Score: 365.376, Score: 499.0, AVG intrinsic reward: 0.014573\n",
      "Episodes: 994, AVG Score: 372.057, Score: 499.0, AVG intrinsic reward: 0.012375\n",
      "Episodes: 995, AVG Score: 378.404, Score: 499.0, AVG intrinsic reward: 0.013697\n",
      "Episodes: 996, AVG Score: 384.434, Score: 499.0, AVG intrinsic reward: 0.012960\n",
      "Episodes: 997, AVG Score: 390.162, Score: 499.0, AVG intrinsic reward: 0.013214\n",
      "Episodes: 998, AVG Score: 395.604, Score: 499.0, AVG intrinsic reward: 0.013644\n",
      "Episodes: 999, AVG Score: 400.774, Score: 499.0, AVG intrinsic reward: 0.012044\n",
      "Episodes: 1000, AVG Score: 405.685, Score: 499.0, AVG intrinsic reward: 0.021907\n"
     ]
    }
   ],
   "source": [
    "global_step = 0\n",
    "n_eps = 0\n",
    "reward_lst = []\n",
    "mva_lst = []\n",
    "mva = 0.\n",
    "avg_ireward_lst = []\n",
    "\n",
    "while n_eps < args.max_eps:\n",
    "    n_eps += 1\n",
    "    next_obs = to_tensor(env.reset(), dtype=torch.float)\n",
    "    done = False\n",
    "    score = 0\n",
    "    ireward_lst = []\n",
    "    \n",
    "    while not done:\n",
    "        obs = next_obs\n",
    "        a_optim.zero_grad()\n",
    "        c_optim.zero_grad()\n",
    "        icm_optim.zero_grad()\n",
    "        \n",
    "        # estimate action with policy network\n",
    "        policy = actor(obs)\n",
    "        action = select_action(policy.detach().numpy()[0])\n",
    "        \n",
    "        # interaction with environment\n",
    "        next_obs, reward, done, info = env.step(action)\n",
    "        next_obs = to_tensor(next_obs, dtype=torch.float)\n",
    "        advantages = torch.zeros_like(policy)\n",
    "        extrinsic_reward = to_tensor([0.], dtype=torch.float) if args.sparse_mode else to_tensor([reward], dtype=torch.float)\n",
    "        t_action = to_tensor(action)\n",
    "        \n",
    "        v = critic(obs)[0]\n",
    "        next_v = critic(next_obs)[0]\n",
    "        \n",
    "        # ICM\n",
    "        obs_cat = torch.cat([obs, next_obs], dim=0)\n",
    "        features = feature_extractor(obs_cat) # (2, hidden_dims)\n",
    "        inverse_action_prob = inverse_model(features) # (n_actions)\n",
    "        est_next_features = forward_model(t_action, features[0:1])\n",
    "\n",
    "        # Loss - ICM\n",
    "        forward_loss = mse_loss(est_next_features, features[1])\n",
    "        inverse_loss = xe_loss(inverse_action_prob, t_action.view(-1))\n",
    "        icm_loss = (1-args.beta)*inverse_loss + args.beta*forward_loss\n",
    "        \n",
    "        # Reward\n",
    "        intrinsic_reward = args.eta*forward_loss.detach()\n",
    "        if done:\n",
    "            total_reward = -100 + intrinsic_reward if score < 499 else intrinsic_reward\n",
    "            advantages[0, action] = total_reward - v\n",
    "            c_target = total_reward\n",
    "        else:\n",
    "            total_reward = extrinsic_reward + intrinsic_reward\n",
    "            advantages[0, action] = total_reward + args.discounted_factor*next_v - v\n",
    "            c_target = total_reward + args.discounted_factor*next_v\n",
    "        \n",
    "        # Loss - Actor Critic\n",
    "        actor_loss = pg_loss(policy, advantages.detach())\n",
    "        critic_loss = mse_loss(v, c_target.detach())\n",
    "        ac_loss = actor_loss + critic_loss\n",
    "        \n",
    "        # Update\n",
    "        loss = args.lamda*ac_loss + icm_loss\n",
    "        loss.backward()\n",
    "        icm_optim.step()\n",
    "        a_optim.step()\n",
    "        c_optim.step()\n",
    "        \n",
    "        if not done:\n",
    "            score += reward\n",
    "        \n",
    "        ireward_lst.append(intrinsic_reward.item())\n",
    "        \n",
    "        global_step += 1\n",
    "    avg_intrinsic_reward = sum(ireward_lst) / len(ireward_lst)\n",
    "    mva = 0.95*mva + 0.05*score\n",
    "    reward_lst.append(score)\n",
    "    avg_ireward_lst.append(avg_intrinsic_reward)\n",
    "    mva_lst.append(mva)\n",
    "    print('Episodes: {}, AVG Score: {:.3f}, Score: {}, AVG reward i: {:.6f}'.format(n_eps, mva, score, avg_intrinsic_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD8CAYAAABthzNFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXecXGW9/z/fma3pyWYTUklCQhMI\nJYSE0ANKuQpesf1UilxBLICoSEQvcq94sV0EVC4oUgQRBBQkSAsEECHJhpIASUghlSS76W032Z15\nfn+cMs/pz5mZc+acme+bV9iZU57znDLP9zzfSkIIMAzDMIydTKU7wDAMwyQTFhAMwzCMKywgGIZh\nGFdYQDAMwzCusIBgGIZhXGEBwTAMw7jCAoJhGIZxhQUEwzAM4woLCIZhGMaVukp3oBQGDx4sxowZ\nU+luMAzDpIr58+dvEkK0Bm2XagExZswYtLW1VbobDMMwqYKIVqlsxyomhmEYxhUWEAzDMIwrLCAY\nhmEYV1hAMAzDMK5EKiCIaCURLSSit4ioTV82iIieI6Kl+t+B+nIioluJaBkRLSCio6PsG8MwDONP\nHDOIU4UQRwohJunfrwUwSwgxAcAs/TsAnAVggv7vUgC3x9A3hmEYxoNKqJjOBXCv/vleAOdJy+8T\nGq8DGEBEwyrQP4ZhGAbRx0EIAM8SkQBwhxDiTgBDhRDr9fUbAAzVP48AsEbad62+bD2YRCCEwCPz\n16KzO4f/N3k06rLlf794c/VWNNRl8JHh/cvedrXw+FvrMPeDLWjp3WBZvrcnj45dezFyQLNyW8eN\na8G08YMxc8F6LNmwo9xdNanLZtCvqQ4XTB2DnBC4918rsaOzO7LjlUJdNoNJYwbi+AMGAwCeXPAh\n3t+wU1tJBAiB1r6N+OKU/UFEAIA5KzbjL/PXYnj/Jt+2J49twQkTBnuuX715Dx59Yy36NtXh4mlj\nkc1o7bet3IJZi9uxflsnRg/qBQCYfshQTBw1oNTT9SVqAXGCEGIdEQ0B8BwRLZZXCiGELjyUIaJL\noamgMHr06PL1lAnkX8s347uPLAAAdOcELjlhbNmP8cnf/gsAsPKmc8redrVw5Z/fMj/r4xMAQC4v\nLy/3Qgjg4Pc24umrTsK1jy3Azq4epf3CIveruSGLA4f2xY9nLgrV17iQ+2o8g997ZAF278s5tj3j\n0P2wny4QPnvn6+Zyr/MRAjhkWDv+ceWJnsf/09zV+L+XlgMApoxrwWEjtBelXzy7BK+v2GI5xpB+\nTekWEEKIdfrfdiL6K4DJADYS0TAhxHpdhdSub74OwChp95H6MnubdwK4EwAmTZoUSrgwpbGzq8f8\nvG3Pvgr2hAGAl757CvZv6W1+H3PtTADAo5dPxTH7Dwrc/1sPvYW2Vdqgk8sLfOXEsbjunEPL3s/3\nN+7ER29+GQCwbU838tIo/K3TD8SVp08o+zGL5ZfPLsFtLyyzLMsJgctOHofZizuwZONONNdn0dmd\nQ3cu79j/7ouPxakHDXFt+/L752N5xy7f4+fyhTa3Sr+x7lzhmt1/yXG+s5ByEpkNgoh6E1Ff4zOA\njwJ4B8ATAC7UN7sQwOP65ycAXKB7M00BsF1SRTEMU2aa6rPo3KcNSCLCV62M7ZU6ymOViqHSkbH3\n19imJx/uRDJECNpFPtZ2SQWXk3Z06WJkRDmDGArgr7qOrg7An4QQTxPRPAAPE9ElAFYB+Iy+/VMA\nzgawDMAeABdH2DeGST0Er5FCbQTp1ZBF577CrJAi0vXUxTmilUjW5RoIaNfaWGUIiJzLaO93pkSw\nzJ7ckNdu21MQEJb9qkFACCFWAJjosnwzgOkuywWAr0fVH4apFVTH+V4NWezpzkEIAQER2bhjfyuX\nB8Ek2R8AIJsN7lCdn4DwOaEMkfXkXRBCu165vLDMIHpy8gwivovGkdQMk1K8xgnV4aO5IQshNO+n\nKNU+bmqbpOI62xHatTYGf78ZhB9qMwiBBt078Jl3N2DXXm2GJ+8X59VkAcEok7S3PcYdVVVRc30W\nANBpeOhEdH/TJCD83s6NNYYQcRvs/c5UxQahbaf9XbB2O773qOY1aLFBxHg9WUAwTJWhPIMwBER3\nztSzR4FDxVQZdboSbjMIu/rNUEO5Gan9ZLPSDEJYBfyaLXsAaJ5UBnHKWxYQTFEk7YfNFFCd6Rlv\ny3khTDVKFLgZfpNK1if40ziNuoy2jbuR2t8GoaLKc7tcecuxeAbBMEwA3jYIxQFE30yTD9EZIewq\nESHr0xMmO1y9mEwbhL5NkTaIDFnP3Q0hhKuai2cQDMOEwsvWoDro2jeLatxJk5urX18NwWsIEXcv\nJu+2CQpxELY2jI+5nCxUeQbBMEzEGAONEIW35Cjwc3NNGq6BcihPHEQmo2aDsB8b4BkEwzAh8QyT\nCzmDELqCKS4jtaUPCdMxqXhc1elG6pzbYO9rpFaZQbjHo8hZPaK6T26wgGCKI2E/7FohSIcNqA8g\nxi3UZhDRvdfb9frpS7UhNBuEbRs5b5IKajYId6GZr5DdhgUEw6QIlYytyjMIQ0CE3C8sspE66e8V\n7m6uunDQO1+IpHbuH+jFFHB881hme9CPxZHUDMOUAXUVk2GDEI6BKSqSPHsA/IPQVGYQ/kbqYBuE\nVxuygOAZBMMwrli94T28mMKqmKAP3DGNPFG61JaK6wxCvzbOOIhwbRORLZ7BiSY/Cn14e+12vLNu\nO88gmOSTcO1ATaASQxB2/KjkW33SVE4qaSwK6b5dZhB+bSsFygnHNfm32/5pMYjzDIJhmKJRHT8K\nxlARar+SSe4EwjMOgqCYi8k3m6tiqg2X5fkK1YNgAcEURcJe/GoGlYQLod1cRbj9SsF+jDhdNlVQ\nccnN+hmpfU4nk1EwUnvEo1hnEKxiYhimaMLZIEJmjCiZBE8gXFxy9dlVGdxcVYzUXvaZSiU4ZAHB\nREpPLo/5q7YEb8goIRSmEGG9mIxBK2lv85WgzqdgkDOS2mUbn7aVAuVE8H1gIzVTNfzkqcX41O2v\nYcmGnZXuSlWg4gGkboPQ/ualt+S4SZqR2n71TPWbtLyuSDdXpUC5gDaCjlFuWEAwkTJvpTZ76OzO\nVbgn1Yenm6viCOKwQZShTyokPRbCC+N6Z0wjdbj9VQoGeRmp7e3EBQsIJlJ2dGl1dfs0Zivck+pA\nKZJasS051UalSNoEwn5NLVHm+jo5wNClBc+2VbyYtGMl56qwgGAiZYdUeJ2JB/XxxWaDiGlcSnKg\nnB3TSI3C0G9PUSLjew0V4iBUrg2XHGWqhp1dWtH1tKoVkoynm2toLyZDQCTnzTWJGJcn4zPz8g+U\nM/bz+TEopF3nOAgmkRQzgBh1e+N2paxWii1Z6Uah5GgJHSoCFTVZpbB3xy2RoaliCtm2yvVWMlJz\num+m2lDRvTLhKPWNv2Ck5nujQsFIrX13u25BkdSA/29BCKHg5hrQ0TLCAoKJBR6DyoOSm6tqHIQt\nUC4+G4TUh4SZqe0DfCHKvJCsT67E59hfoW1fAaHUSZWNygMLCKYowg9CLCHKgUpErbKbq90GkbDB\nOmmYAkL/7iasfdN9K3iNqZR+ZTdXpmqw+9oz5aNkN1dY34STZg+oBPIl0OpkSDmQDBWTzwzCD5X9\nVOpycKoNpmowp+Mpcm1MMipXUXmgV/GqiQCVlOVJwOuy+Lq5BsRBAMGz6aAZIM8gmKrBeJTZi6k8\nWAbXUgsG6X9NG0QpHasS5LHXLKSkLy+4ufrYIHxTbSjYIITgSGqm9mBPmfhQtw9ZZ3eVMFInmeC8\nSeHOpGCk9msTidIxsYBglCHL57CG0PL3pxaxXMYSBwrnDILnEF4QyBzgi7VBmFfXbz+lXEzhjlsK\nkQsIIsoS0ZtE9KT+fSwRzSGiZUT0EBE16Msb9e/L9PVjou4bEz3+eWuYKAibiyl2D7MEPwqykJRV\nTNo6Db+I6KBsrkCQm6sItEFUW8GgKwEskr7/FMDNQojxALYCuERffgmArfrym/XtmCohwWNCqlCK\nQlZVMdmEdywV5WydS3J6D69r7eeu6muk9ilVCgC5vMBTCzdg2559vv2qmhkEEY0EcA6A3+vfCcBp\nAB7RN7kXwHn653P179DXT6ckPz2MGsZbE+uYyoNKHERI9R9P7gpYjdQFN1eS1pkqptBt+9sgXlna\nAQDYtCtIQFTPDOJXAK4BYFTWaAGwTQjRo39fC2CE/nkEgDUAoK/frm/PpBj2YoqfsDWp4y85Knti\nJRe74CTbh/BeTMZ+7he8J5e8H0lkAoKI/g1AuxBifpnbvZSI2oioraOjo5xNMxHCcRDlwTK4eoxG\nyoOuPZKaJ+wOrG6u2vUpJOsLGUldZJI/O9Uyg5gG4BNEtBLAn6Gplm4BMICI6vRtRgJYp39eB2AU\nAOjr+wPYbG9UCHGnEGKSEGJSa2trhN1n/AibaoPVGOVBLZurahyEzQZRdK/CkZZnwWsG4Z/uu/hA\nubC/qTiITEAIIWYIIUYKIcYA+ByAF4QQXwDwIoDz9c0uBPC4/vkJ/Tv09S8Idn2pGvhOlp8SbdQO\n4c01qd1sEPpy6apSgKrIi6B032HTtMdBJeIgvgfgaiJaBs3GcJe+/C4ALfryqwFcW4G+MT4U81wa\nPyxO1lceyplqo2I2iJQ8Co4ZhM2TKXxFOe1PqQ4bccrUuuBNSkcIMRvAbP3zCgCTXbbpAvDpOPrD\nxAdnc40O72R9ql5MVuFdiZf5hE0gLGhxELILcMEGQSgi3XfA8cJ6n8UBR1IzscDioTyo5GJSjoOo\nkJE6yc+CJVDOJgGsMwgqOtVGqcTpTBDLDIKpXYxHmc1J5SEKFVPY/WoF+7Um2+ewbq5mu143MYHX\nnwUEEymmGiMfsCETmpLrQSRA/Zc011qLkdrHmOxlg/C7+ubLUqLnUFZYxcTEQnp+EsmmnG6uxpBl\nCO/43FxT8jRYUm2QU8UUcgYR5PKdLFGpwQKCiZSCp0xKBoUqoOgZRAXe5hM2gXC4uVpTmxhGakPF\nFNYGUXr/4oZVTEwssHwoDyrqibA2CGH7HjVpeRTk+tByLibAx81VpV2P5UEzv4cvm4qdXd0KRygf\nLCAYZYp6AyoyqIjxQCGba1g3V743BRzpvqViShYBAfJI9+1ng/C/3kF3bdSgZgzrPyhgq/LCKiam\nKMJpuTlZXxR4lhwtMlAutopyChlpk4Cn4NTtEaHjIIyXpSL7k62AjooFBBMp9rKWTGmU8yo64iAS\nPVzHj6UmNWzpNlB+VVnQ+J+JsxCEcczYj8jUFFxytLyoFAxSn0EYKo8SOxUalapHlcHXzbVELybP\ndhXhGQRTdXCgXPyELxgUX0W5tGEa8KlwVQ2DtWu6b784CPMCe9kg/G8AzyCYqoXlQ3lQKbYTdqDP\nS2qUWke+BnYBIBugvSKpVdouegbBAoJJC+q5660J4fzgWUZ5CBsHEWtNakqPkRrC/doYxYPcvZiK\nP1zQvnUsIJhqRWXsZ/kQjNUG4eXFpKhiMlOxl9ytqsEaKOcXs1BMTepCu8UQZx0I85ixH5GpKcJE\nUvM4FYx8jTxVTIptVcqLyXIOCZ5C2Gc6pSbrK9UpgFVMTNURpuQoq5jKQ9HlYGMYf5J/i+VAOeHq\nNUagItN9l9azCsgHFhBMPKj8mBI/diQASz0ITzfXcComM1q4tK4pk3whoSF8pjreM4jgq+j1W/Db\nM0NhkjCWDxYQTMSo67nTMnBUknJeo0rEqNjHuCQH51lSbUjLTSO1yz6+kdRGu0XUg6jLVGaoZgHB\nFIXyW6pNz+0HR1uHo9Q3Srt9KGm1GSqBNVDO+3nUvLHCeTEFqlt9Hv8KyQcWEIw6pbztsRdT8rAP\nWLGpmBCsJksCQnJjIlIzUvvjf7J+zVUiihpgAcFEDEdSJxlrdtEkD9Zx4Z9sj8xtvJP1FW+D8PuJ\nVCKKGmABwYSgFBUQ2yDKQxQ2iFzM5WDTcp+FVC/IPvAT3L2YSlEx+f2+KuHiCrCAYCLGHq3rB9sg\nginnNTICr/IVnEEkbdIi22E8vY3Iewbh23bAer/2KhFFDbCAYEJQzJtfmGjdtLxZVgt29V8lAuWS\njBCSfcbhfRW+olyQE4DfdalEFDXAAoKJCZVBIS0DRyWJQsVUyeueNLuHNVmfd3JEr3TfKjLWU8Xk\nc3NZxcQknqJmEGFUTDyFCKSsBYNQGRVTWu6zJSjR/J+e+tvLzdUv3bfRrpeR2qcvPINgqhLOxZRc\nuJiTPwLeL0WlJOvzPaAHPINgEk8p4wjHQZSHKN6+Kxkol7RIaq+Kcm4R4MWm+y7Gi4mN1ExVUqgH\nEbztA3NWRdwbRiagwBnjM2R7zSD8jdRGqx5H4zgIJs2U8vaqomL62dNLim6/ViirDcLu5lrGttU7\nUYmDeuM1oyH9v8L38Mn6Cum+wwfKcSQ1U5UkzUsl7ZTVi0n/a5Ycjc1IHc9xSkVzcy3kITHTffsk\n6/Ml4Pr6Gql5BsEknWJ+16YhlC2hiaNSBYOSjL2inBfrtnbi729/6Nxf4RjeKiY/N1eFhiMgssMS\nURMRzSWit4noXSK6QV8+lojmENEyInqIiBr05Y3692X6+jFR9Y2JHxYP5aJ8V7LUCmfF4hVbkDQs\ngXLScgJhn56fZM++Hss+/hXlCu26Hs+nL9WoYtoL4DQhxEQARwI4k4imAPgpgJuFEOMBbAVwib79\nJQC26stv1rdjEkRpkdQsIspBJIFynKzPFZW0JmHiEyjAK6CmAuWExi79a73+TwA4DcAj+vJ7AZyn\nfz5X/w59/XTiBPWJJeydUdEwNVRqHl2jOGwQlehDgn/iVjdXNQWcSqCcyvHsVJ2AAAAiyhLRWwDa\nATwHYDmAbUIIY162FsAI/fMIAGsAQF+/HUCLS5uXElEbEbV1dHRE2X1G4so/v4mv3j8/9H7mW5jC\nq++AXvWh2681/K7iOUcMC9eY6XYZdyR1PMcpFVnFBFiN1J6UFAfhTVVGUgshckKIIwGMBDAZwMFl\naPNOIcQkIcSk1tbWkvvIqPH4W06DnArGj0FlBnFAa5+ijsFo/PrzR2HFT85W3j5MIsVawWqkDraV\n2Ad7pXTfim3JVOUMwkAIsQ3AiwCmAhhARHX6qpEA1umf1wEYBQD6+v4ANsfRPyY6CgJCPd13v6a6\ngC1rF7/LSESh3CGdebJiyuYqv5XHcsTi0OpBuOVbKo4gp4CaqgdBRK1ENED/3AzgDACLoAmK8/XN\nLgTwuP75Cf079PUviLRk9WICUbmRxpss33RvylkPwrRB6AWDEmwOiA0vm4jXtbHfD5VIai9qbQYx\nDMCLRLQAwDwAzwkhngTwPQBXE9EyaDaGu/Tt7wLQoi+/GsC1EfaNiQlDxueFwIk/ewG/ev59n41t\nf5lIcRQMium48u1NulCyznacnXWqmIJPyDOS2mefSrm5RjaXF0IsAHCUy/IV0OwR9uVdAD4dVX+Y\n8qIaVCWkD2u2dOJXzy/FVacf6Lotu8IGU85LZKijcmyEMLHUg7AZqFWM1Cr262IC5Srl7cV+hUyk\nhLNBWP8yTsoqIPQxJxdzNte0aI6FpEDyeiEKdSYlXd7KXDNlAUFEJxDRxfrnViIaG123mGrB+Imp\nlRwVlr9MtBgqJmMGUZk4iAoc1AeV/sjCwv6slpTu2+exr9QkT0lAENH10GwHM/RF9QDuj6pTTPVg\nPPSvLtsUuC1rOoIpp5HaMHyyas8dOVmfqiDzD5TTvZg8K8ppyxvqnMNype6R6gzikwA+AWA3AAgh\nPgTQN6pOMdWD8Vgv3rBTeVseruLBGPTMGURcgXJyHxLs6Gp/Dt2uT5hnNaj+hiEDnrriBMe6RM8g\nAOzTXU4FABBR7+i6xFQToV58TBVTNH2pBsprgzBUTNr3pKl7KoFdYLk5sXpVnbOvc7btj9FWvUvK\nmUqpXVUFxMNEdAe0ILevAHgewO+i6xZTPag/2Kxiihenm2sFphAJRgjhKZCnHzyk+HYDlrvdh0Sr\nmIQQv4CWQO9RAAcB+E8hxG1RdoypDsI814YOtpx6dsabjE3FVAmSNmvxqgdh9/A6fvxg50YImEFQ\nQCS1j72jUvcoMA6CiLIAnhdCnAot4R7DKFOEholVTC786vn3cdZhw8qc7ptAJL2dxjaBSNMNdu9r\nwZygfi6FXEzBgXJE1t9BYm0QQogcgDwR9Y+hP0xKUH3zC6M7ZRWTO13dOfzq+aU4//Z/lX1wzRBV\n1M01aXgGyiG8C6xf264YcpqAXvVZ66oKvTWpRlLvArCQiJ6D7skEAEKIKyLpFVM1hJtBGCqm+Lno\n7rmYvaQDK286pwJH98d4w++JQIJmZQGRNH1PxREOIQEUZl5AOCO12WpAsj4iQnNDHXbvywXuEzWq\nAuIx/R/DRE8FfgyzlyS3toghFzJU/oHComKKibSoEN1qQZjfPfZRSdbnnWpD+5shoLnBqtzZIwmL\nOFESEEKIe/Xa0UYSnSVCiO7ousVUC2EGAw7Ycse4LlEUjam0iilxsxZbd9yeSApYr9y4jUJlP8IR\nIwdgzZZOc93Kzbs99ooW1UjqUwAsBfAbAL8F8D4RnRRhv5gqIYzu1DRSxzCFuPz++bjn1Q8iP045\nyEuBbOW+MhkCcpLuOw7S8hrgF9BX8Eiyp9ooJZtr4T7/4vyJ5vJPHT0Sd3zpGJUulx1VFdMvAXxU\nCLEEAIjoQAAPAqhMr5nUEMoGYfyNYQT5xzsb8I93NuCiaclPKWa84WcyVHZjZSZDBQHEZmpbnqXC\nsyhnczW+u+/v07aiiokANDcUjNS//MxE9x1iQDVQrt4QDgAghHgfWj4mhvGHVUwlk5cGjnIjq5ji\nIi0V5ewYwsMrViJ4f/+dhGPDyqM6g2gjot+jkKDvCwDaoukSkwZUn+FQQ4+w/GF0ZBtEFComM5I6\nQQNTpbCm0Sgk/LZfGjMOoohAOU/ijmhXQFVAXA7g6wAMt9ZXoNkiGMaXcHEQRi4mFhEyshtquS9N\n3EZqIrLYmJIslAR81J2epUkVbBABgXJJuiaqAqIOwC1CiP8FzOjqxsh6xVQNRdkgouhIijFtEBEM\nHJkMmQWDGB/3VXLfJlQktbGPZ6oN/z5UAlUbxCwAzdL3ZmgJ+xjGl1C5mHiccsXq5lruSOqCl1Si\nRqYEIBupAQ8hEeJ2eAXXFY6XvIBFVQHRJITYZXzRP/eKpktMNRHmDaugYoqqN+kk0hkEFWYQcem+\nrUbq5AyGdoSPo2sxY3jQuZoqpvBNR4aqgNhNREcbX4hoEoBOn+2ZKkc9F5N6mywY3MlLb5ZR2CD2\n9WgFIeqzSRqaKoP97d3tBUd4fFYl0M01QbdB1QZxFYC/ENGH+vdhAD4bTZeYaoLH/NIxCvpkMuW/\nnkIIbNyxF4B7oZoosGctTSy+qTacqbvdSoVa9jFVTO538aX3OyxtJwHfMyKiY4loPyHEPAAHA3gI\nQDeApwGkIwyVqSxFxkHE5cn09ppt+OaDb8ZyrGKJsqDPh9u7zM9xCYgkY0+j4fUY2gVGhoDLThqn\ndAyvJ9sQEAmSD4EziDsAnK5/ngrg+wC+CeBIAHcCOD+6rjHVQE8+r7yt/GMUIp63y6/ePx/rpUEy\nicg2iCjlZkNdXLk20jev1NJ9O69PociVQhuKl9fYbu5101GfqazQDhIQWSHEFv3zZwHcKYR4FMCj\nRPRWtF1j0k4+L0LVeKhEIZnGALVAErDaIKK7RnUVGIwS9LIMwLvetCwchHB3WVUOHg24hUY7Q/o2\nKbYYHUFPRJaIDCEyHcAL0jpV+wWTAibf+DxunPleWdsM618vC5O4REU2CtegMpOTkvVFSX0KhGWc\nCLjXpBZw3guVR72gInRu3J0rzLSTNL8KeiIeBPASET0OzWvpFQAgovEAtkfcNyZG2nfuxe9eUTcr\nqfwgwuT4eebdDejYuVdqP56fyfKOyqRRDkOUqTZk4vJiSrKR2p6sr7DcHcv9CDgZv9W79/aYn+3V\n5CqJr4AQQtwI4NsA7gFwgij8ajPQbBEM44n8VhTEZX+cH2FP0o3pxRTxYNrARmoHXmrPghdTeJH9\n6xeXYdpNL1iWGa7GN37yMNQl6D4EqomEEK+7LHs/mu4waUHlZ1FKltBqrE/9s6cXY9r4wZg2fnCo\n/QpG6vLHQcjE5uaakntrn+lYZgDFBMrp+7yzbodj3V5dQCRNSCerN0xVUUoN5UoYrKPmt7OX4wu/\nnxN6P4v7b4TXpTJuronTMZkIYa9J7Zw1hDFS+7kp79OniUGxFHGTrN4wVUEuLzDjsYVY3r4reGMP\n0vKWGQeyDaJt5dbIjhObDSKFN9evJrXq+TgN24X9DHVs0mYQkXkiEdEoAPcBGApttnanEOIWIhoE\nLeBuDICVAD4jhNhKmh/ZLQDOBrAHwEVCiDei6h9TGn6/iUXrd+DBuavxj3fWO9ap6tFTOIZEhqFi\nem/9Dry33qmeKBeVSBKXOCO1rRiQd7ZvZ8fDnktPXphC2bBB1NIMogfAt4UQhwKYAuDrRHQogGsB\nzBJCTICWJfZaffuzAEzQ/10K4PYI+8ZEiBFbsGdvzrEuo/grilKVkrY32Cgr7V0wdf/I2vYiNVdf\nIalgmFtjb0G20RXyYdWIgBBCrDdmAEKInQAWARgB4FwA9+qb3QvgPP3zuQDuExqvAxhARMOi6h9T\nGn4DuPEWtM/FiyknRCG9tA9RGqlTJh8QwhksNFPHtUTXeAqxD+Lyy4RnTQjF58n+biR7+dXiDMKE\niMYAOArAHABDhRCG7mEDNBUUoAmPNdJua/VlTMrwG9yFAH7y1KLANqJ8y09b7esoa0bHHSjoVboz\niQjrFMI1etpSHU/BTC1jmUHUqpGaiPoAeBTAVUIIiwJVj6sI9fQT0aVE1EZEbR0dHWXsKRMGvzE2\naEC7f84qz3XGeBXpDCK6piMhSoEWt4DwS4CXBOwpNdy6KiAsxX+KPZ3unFPFlDQjdaS9IaJ6aMLh\nASHEY/rijYbqSP/bri9fB2CUtPtIfZkFIcSdQohJQohJra2t0XWeKZqgAc0vAZk5YEU4iPAMokCm\nwqlGklQ9zQ2vMqBu3Q46Fft6OZFlzc0gdK+kuwAsMmpZ6zwB4EL984UAHpeWX0AaUwBsl1RRTIoI\nGtDqfNwpDSN2lIN4yuRDpNf+OiYMAAAgAElEQVSirgICIi2X3ytZn2WbEO3ZW+jJCQjdJldzRmoA\n0wB8CcBpRPSW/u9sADcBOIOIlkJLJX6Tvv1TAFYAWAbgdwC+FmHfmBCEtQcECQi/H4Exg4hyEPE7\nnSR6OEWqYtIHvoS/yMeGvR6E/CRajdSFoLlin5mevMCPZy7CuO8/ZUZSN9UnS0BEFgchhPgnvG1Q\n0122FwC+HlV/mOJxzWjp86MIVDH5CYg4ZhA+4ieqOhRhB5E9+3rw3Hsbce6RIyL1YjIEclxvrgTr\nvU2LXJLrQXg9I4EmattOuXwed/1TS5C5q0tL1tdUl5xEfQCn7GYUCDtY22cQDXUZcwoNBAzAkvEv\nKny9rCI6Ztjz+c/H38Uj89di9KBeSm7BxWIIiKQZRyuFtR6Ed7pv+bPq3bE/9rKRekdXNwCgKUGZ\nXAFOtcEoEHZ8sgsU++DjN1gaNohKBcpFpWIK2+oGvcrdrr09oetqhKEwg6jMu3ySVVsqacmtdgr/\n9uzr5Rep215YBiB5BayS1RsmkbjNIPzdXO3bCt/vMlSlM4iwszDjOuRFPHEQ/ZvrIztGmrDXgwiT\naiMsbunwK+1VZocFBBOI29g2c+F6z4HePqDZxzeV8S5SW3FAIF8khwzZruzNFaU9xhikBvRqiOwY\n1QB5J9tQj6S2tVBKtuO4YAHBBOI2QC3esBOvLN2ktL1dRSIg0L6zC7ukKloGxk8oykHRr+2ojhu2\nXTNgMC8inUFs79R03wN6xTeDCKOWiRtrf2yDvzS7da1JHbKiXE+OBQRTBXgNblv37HNd7phBuMwo\nJt84C5+47Z+OfU1PkWI6qkjyf5YF1U9eRBtVftzYFkwZNwg/OOfQ6A6SUrwEmRxJbXwvBjlQLqmw\nFxMTaJj1Wuu1m33GYBcwxvFWbHLWgzYHxghHRb+3+aSomEhWMUV4LXo31uHPl06NrH03wuUvqizF\n/jZUYBUTkwqCBi/h8aLjNdC6zRhk9vZ4vzllY9A5+AbKRTS/CG2k1v8KISL1YoqbNJ2J7MLqTDJY\niIlQJUjF9PoMR3hYxWEBwQT+aL0GN88ZhE0iXHbyOMv3nV1O24NBQbVSKTfXiI4ZcvuCkTpaL6ZK\nkBZ5Z7c/yLMda7yEWntugXIGDdkM9uvfVEw3I4UFRA3Rk8vj+fc2hnI7BXxmCorLv3LiOMz9vtrb\nkZGnKcpBxK/ppLi5GvkMhYhW3VZxEqZhcpYFda7zTPcdFAdh+/7jmYW0970akxUgZ8ACooa4ffZy\n/Md9bZi1qN2yPHgG4b7caz+7e3ddhjCkn9rbURwzCH8bRESBciXYIKpKxZSic/Gzlbh5MYVl7dZO\n8/PAhLoZs4CoIYwHsmPXXsvyoMHY60ftGQdhW24M+qMGNTu2nb9qK/7j3nnm97pKJ+uL7KDhNs/E\nZKSuBC6eo4nBGSjnfu3dczGFc3OV+dyxo7xXVhAWEDWErLaQCXoL8hqfPJfbVhiJ4J664kRcdfoE\ny7rL75+P56UZjZlqo2IziPiP6UahcJL7DOKXn55Yjm4xiqik2ghsw0OADO/fhEtPGue6rtKwgKgp\nilPflGqkNmYQfZvqMXpQL8s6e7qBWGwQvpHU2hv77CXtZRVSYVsys9rmgd+8uNyx/lPHjCxDrypD\nkrVM1vgGWG6cm5JJhIik9mJ0S6/EFk5iAVFDGG+lP/jbO/jbm4VifYFurh7LvQSHQ8UkPfz2bJXd\nOfdtIy056isggAfmrsZFd8/DE29/WLZjhs/FFL0tJgkkdWAErLNYLd13YXk5KsoZ2H8DSYIFRA0h\nP6C/nb3M/Bzk+++lA/cUHLbt5QRk9oIo9nQbhYJBEbq5+tWDALB26x4AwIfbusp3zNBGau2vm4A4\noLV3GXpUGfz0+knEracWL6YQ5+MmH/Zv6YXvnXlwUX2LA46kriEyZDXAuX12w2u9qpFaJqggSp1u\nKIkyC4FvNlchIil7GnZQNGTqPpegwocvizfyOUqSNn/w6o99puO2XeC5uGzw0ndPVehV5eAZRA3h\nLKeoETQQqkZMF7b3bqsxoKRiLDMIPyM1CoNzWW0QRXox7dmX81yXRsqhs48LIQo2smzC0nDHBQuI\nGoIsM4jCr9Tr93rZH9tw0d1zvY3UHvv5DayNQTOIGIzU/jOIgreJ13ZG9a8wFBsHsdtNQNToYBUH\nlt8IBDr169+rIWuLpC48p8Wm+04DLCCqgJkL1uN3L68oen+vB/yZdzdi9pKOot1c+zY5NZhBJRUz\nFL2A8PMpEhAWF1M7s5e044gfPYvXlm8OdcQw6iohBOav2gIA2O2SEr2a5EPSJ0OGgO7dUFcwUsND\nxVSkkTrJsA0iJeze24Peje636+t/egMA8JUAX2qHC5/rFydhA+WMpTO/eaJjYLQbqe3UxRJJ7b1u\nb3fezLLp1oU5H2gD9xurt2LqAS3KxwxzNvfPWY33N+4CAOx0ma2kWsWUcPWSRQ0rgM59moBubsha\nlpufQyhD03jXeAaRAmYt2oiPXP8M5q/aWlI7GQ8JIT/iC9duR1e3Va3hmWojYGbRv7keYwZbPW4G\n9fZPKZCtcCT1iT97Eb+dvVzfzrlhsT/yMPaMDzoKadC37HbW3KgmfXiS1S5CSDMIW64kt7xMST6X\nYmEBkQKMym1vrdmmvM/D89Zgq21w8TJSyw/5x3/9T3z/rwst+3m9I3ku1xskl6erV4P/pNWwQVQq\nF5N1u/Id0+2QXd05V0O/PP5v2OF0tU3xBEJLoZ3gaYT92hpOAnbvu2Lugd0T6s+XTgnfSMywgKhC\nlrXvxDWPLsBVD71lWS4bN/2M1G+v2WZZ7+VyGjSzKEYVctKEVksbUaDadjFCKpcXeNtFkLs1dfAP\nn8aMxxY6lsv3aYU+m2isK/xU06xiShMCmoqpV0MWmUxhfiC/GIURePJd+9opB2DKOHUVZaVgAZEC\nwo4HXd3aiN6x05qUL4ybq1zUp9h032GHsRlnHYzhA4yEfuV0MRXY21NQm0U5g/j1C8tw7m9exZur\nrepA+zGNmcNDbWscbcj323iD3b+lkKIkjqJKUeKWQjuJCCGwS7b9SZ0tFAySoq0DziWbda8nkWRY\nQKSIsFNz+0NIIQLl5AjnXzy7xKM/7sc1Fod9081mSIogDrWrL794dgkO+sHTpm1FVUAUE4vx3vrt\nAIAN262qIXtL3T6RgPbr1q+pzqKaS8vg4kaCtUsAnGqgHZ096N9cb1kmbG5Mqqckq6nSYq9gAZEC\nwj5MXj9Cr0Lr9oGQiLBLqvo2e0mHx3ECZhAhfwN1GYrEzfW+f60CANOn3Z4g0Is7XlqBbXucRmLA\n+9zNN8uA7f3y79ht0MP6N5vXkijZ+YvCkuQzmb9qK15dvgn9XNy1DcI8pvVZMu9tWm4hC4gqxv4Q\nygOPZbwKmEF44Z2Cw/34QdRlM+aAUU4jtfG2bggGv5rYdt5cbbUnBJ2TPaV6Ty6PxRt2OGZEPXpf\n3Nqzq5CG9m8yr0vq1UspysT053lrsG1PN/rZZhBAcQWDiMiMA0rLXWQBkQLCjgmeRU5sxVAK2zvx\nEhBy0R9vI7W2IqyKqS5DlgjVcmGkHzcEQ5jsmRffM881H5LnLE1KAw0AP39mCc781StY0bHLst0+\nI4WDQlrQPo1Z87qwgTp++jZpAqJgpPZW1wZhBoqm5D6ygEgRodM12DPYezyT9nYJ7hG89ja9jdSF\ndsIg2yDK5QrZk8ubAsEYlLtDzCAAYPPugrE/UN1n849v02NXNu2yqqp6ct5C1K5i6skJ86gpGVd8\nsdzaFJyP4UHmfu1lI3XwyZhtlaNjMcACogrxfruVt5HcWG07LG3fhQVrtyscxysOQvsb9m23Ppsp\n2CBC7enNbS8U0prv68njn0s34T/uawvVRhi1jl1F5nWNDHVXxuUX+Kvnlzrb1RtOe5Bc0o3Ubjgu\nuRBWFVOIc/IXNskjMgFBRH8gonYiekdaNoiIniOipfrfgfpyIqJbiWgZES0goqOj6lct4XgILYnI\n4PrZ4JZZzkHKtU0XijVSW72YyjOSLFxXEHT7evL48cz3wjcS4jzsb5HGWezZZ52RdXvMIBat3+Fo\nU1ZpVIOKSVaBpsGbx+2au9ekDsZIVpmG8wainUHcA+BM27JrAcwSQkwAMEv/DgBnAZig/7sUwO0R\n9it1eD1KT7+z3tVW4DW0ehmpi1XneO1lRlIXYYMw+njx3fOK6pMdOYfOPkXvpVKwGy+Nvz+euciy\nXbeHDeLT//eab7tVIB9Sh981D/vLaeAZhIYQ4mUAW2yLzwVwr/75XgDnScvvExqvAxhARMOi6luS\n6M7lsWrz7uANYX3zWta+E1+9/w1c88jbntvLz2BXdw5rt3a6t1uGl/V/LFyPZ97doPfTP+Oo17q6\nbAZGr3vKFAjRS8oeu7c7X9QbeJjiRWYtCf1eeZ2FYYOwd8fLBbdaVExA+tRMxv2XXZgLgXLh4mWM\n+5eWuxi3DWKoEGK9/nkDgKH65xEA5JDStfqyqufGmYtw8s9nO6KeZdzGNCPCds0W66C/evMetK20\ny2Xgi7+fg0fmrzW/F+tseMYhQ83P8g/98gfewGV/nA9AUw/5DcTnHul+a+UZBOBdkCgM9VKKin25\nnKvOPwg3dZdXz8xa0sY47zEa7vMoROO2tVyjohpUTDJpOh25r8XUpJa3Sct5V8xILURY844GEV1K\nRG1E1NbR4R7AlSZeXaYl4tve6R6QJeOWOdI+0J/08xcd6gyg4E3j1pbqG911Zx+CGWcfUtjP4/bl\nhf8P4GfnH4G2H5zuWN63qc6ilgoTr+BFi5Q9dl9Pvijdby6EoJJdIf3o8RAQXscyLkvaJxBJTtTn\nhf05t6tnw5xS2gR83AJio6E60v+268vXARglbTdSX+ZACHGnEGKSEGJSa2trpJ2NA5XUEn66fN+H\n028/y2e1J7x/c71lQPNL1ufX5/psBoP7NDqWHzaiv2UA7Ox2VlMLixzDsLcnX9Sbm1vuIM/rrq/v\n2LkXry7b5HllvYzU7gJCSAIiXQOMG5ZnL33yAoD7i4DKnSnYktJxH+MWEE8AuFD/fCGAx6XlF+je\nTFMAbJdUUanlPx9/B2Ounem7jfGDV/HasTyMQQNVUFvSfqovyHYjr2cpUskNUJXm+iya6q1lHTds\n78L2PeHLexrM/WAL7pAq7e3ryRf1w8zJCdkCtjX6/9OnF+MLv5/jeX+M6G5Vm0K1qJjs1yONMwoA\nReViAtKjWjKI0s31QQCvATiIiNYS0SUAbgJwBhEtBXC6/h0AngKwAsAyAL8D8LWo+hUn9722KnAb\nh846JL4TCP3v0o07ffdU/ZGefbjVb8AvWV/YgWz4gCYA1h/Q2be+gon/9WyodmQMo7nB3p68q4rm\no4cOdS6UCONya2/fa18jWM9+neSsrQZCVI+KCUBqpw2FWYP82wnZhn4j0yIoIis5KoT4vMeq6S7b\nCgBfj6ovScav/rEbf3/7Qwzt12RWuFIZ3M+4+WXf9arPuL0anJcROZ8XoQey+qz2rlIOtZK9TQPN\nBuEk6MfqZyzftbcHd760HN+cPgH12YyjLTeV0YbtXaaXlt1oPqRvI9Zs2eM5q8tUhYQokAZR4ZZb\nzM2WpTI7NVVMKfFj4kjqGPjHQqe2rH1nF+av2mK+QX7v0QV4VPIykpEfpW8++CY+c8dreMqlTbuw\n8PXfLsJIbcdrEMsH2CAMDt6vr/nZGMzdajAXS0PW2od9OXc31yAVW87nAv3imSW49YVlePytDwE4\nf/huAmLK/8xCu14pLkNk3re1W/dg3sqtjv4cO3ZQ1QTK2T1T0jiZcNSkDnES7MXEOLj8gTccy/7t\n1n/iU7e/Zj4o7364A9/+izOm4Zl3N+D1FZsBWB/M37y43LGt/Tl9c/U2T68Y4fNNFbdZz9trtuEP\nr36glA324a9Oxf99UQuar9cH85MmtOLUg8rjfNBQZ328N2zvco0zGNirkK2zqd75k/BT/xkpxG+f\nvQzjZsx0XBOveI4dejr1VZv3YNpNL3jGqcz+zim49MRxqMsYAsK7L2kkDbld3XpI5Fyp5OaKdMVB\nRKZiYvxp1+MegnIeGbEFXliNzc5HWa6kJtMjDZTFzyCcO863udP60a+pHv2bNbVVnT6DqMtm8MvP\nHImj//u54jolIQsIIuCef6103e6TR43EtPGDcc7hw3DrrKW4VcrfBKip/5brpUG32upH9HhIF/m+\nfLi9C0s37rLYTGZ9+2Ts7OrBmMG9AQC99IjwqlMxJV8+uCLfhWKM1GmZQbCASBFub1vyMjdViFdq\na9kjqdh4NLeBs6VPg8uW3hipMPYfVDDOym/0pdAg2SCCBiIjeM9tAHYPlHNvcMtuq4DIeVx/+wzw\ntRWbcPerK83vB7T2sazvo5e9TL2KSQSnmk8qZip6aVnY/psCIiVzCBYQCcZQX/gh/9iMAvcybrUM\n7MuLnea7CZawb4RHjhqA2z5/FE6XIrTL4SO+o6sbP/q7lpjvoKF9scTVk0tD1iHbDduAzY4QUK9i\nq80tVzVlyE+eWuy73qiL7DUjTCtpcHN166Jb3RK1OIh0eTGxDSLBfO/RBZbvQb+ls255xbHMy+ib\nFzBrNBcfS+HcsZgsrB+fONySVK8cPCYZ/O/98mTfbeUxvMFFQOSFFlPx4uL2wEFgR2dxAiKI1Vv2\nAHCmVkkbwlZTLvniwYnsdmx8VyUtgsGABUTCkAfdN9cE6/OXtu/yXX/aL1/yXHfeb17Vj6nYORuG\nel3uc5i0FKo89oa7d5cfnd2FGVJfj5rCV5w2HgAwpF8hqttu2AY0ofeZO17Dxfc4M8w60jDY1veE\nyCB7zP4DPdet0QVE1ZFGCSEhQlqpC3EQ6ZAULCASxstLN5mf93ZbBxcvdVGxLN6gqV2KVzFp+8lC\noZy1pA2uftjq3TXxhmdxi0tRnTtfXo6JN2iBdXI8RTZDpg5f5srTD8Sz3zoJBw4tuNu6CogQQs8u\nEMIIzN6NdWisy+DQYf0c6/796OrJXWl3E00jphOTCHcOZPubdFhAJAy5HKa97GdXRPrnUuMgZDVK\nDCUXsL2zGzc//z4ArSbGz59ZjF17e/CTpxZje2c3tu3ZZ6rPAM2usOD6j+LK6RMs7WQzZBEOADD9\nkCEYbDO0/9eT7kWGdnZ14+E26+zGno4kTA2KPo1ZLPqvMzHzihMc6y6YOka5nSTjTLVRmX6EwRAA\nZnobKTdWWNiLiSmJPpI6pMs2Y7DPKMpFV5HRy4ZqqdviERXNL/74/5mF5799ssVGIOe5WriuUInt\n9698YDHwG/mO5FnEESP7ux5nSN8mtP3gDEvb737orPIGAA/NW+NYZp/leXmRudG7oc7TjdXNNpJW\n0hoo5+Z5JCzrVdpQ3zYJVM9TlzBWby5OZ+ynronCg+WOl5bjfI8qZkEY/evJRatiArRYgUP/8xm8\ntWab6/o5ejAhAPz6xWVmvQyZJskQ/revTSu5T26nWooJpqne21Cf9viHGWcdDCD1JgcNARhDvNB0\nTMqwDaJGEULgj6+vwnbdi+Wkn79oWf/K0g687TG4ycjqGjk2AFBzew3L//zD373SD3cVU7RDgJcw\ns0dId3Y7I7k7pbrQpQ64t8xaikeLMJ778dljRwVvlFIuO/kA1+VpFRhFFwwKsW0SYAFRJt5YvRU/\n/Ns7uO6vC13Xf+muuThX9xryI6e7Bv3tzXVYaZuF/E3P91MsZ35kv5L2t2POIKRo4agFhHdfrN93\ndjkFRG8XQ7UXT11xIq458yDHcvkwhpG/XBziYpyuRuRZZlQzzriw55YKohAolw5YQATw0LzV2LYn\nuNpbl24f2LwreFs/DJ31c4s2ltSOG9lseR9L47cdh4opLPZ4BAD43LGjlfc/dHg/nDjemRPqb2+6\n1rEKRf/mekweOwjj9DQaf7hoEp791klVUW86ECEsz0sq0LtbMFJLA7wlUC6EFSIlUwgWED68v3En\nvvfoQlz10FuB26re7qDIUfMNPILfkFfaBy+CBiyjr91lSNvhxUXHj/FcN3nsIFw8zX29PaIZUC/O\nY+BWv3q1YjxCvYswNhIB/vYLR+Phy6aaVfV6N9Q5vKmqEWNMlGeZCXmfUEK+o8XaEDI8g6geDI+U\n9h17y9bmg3PX+AqJru4cVnTswkyXdN6qTLIFXP3fF48BoOVqmjquRbmdfh4BZgYFFVP5bRDzrjsd\ns79zChpd4hIMrjp9gpnEzs6H27SI48ljBlmWvzbjNLxyzalKfSgl79Ftnz/K/HzKQa147GvHm0Zo\n45zOOUIrwDRykLNIULUiYFVJpiHVhh25z1q6b/V90+bmygLCxl/fXIvH37KqEcI8wkFBM7MWbcQm\nSQ1lHwCvfvhtPDBndYgjOtlsSxhnvDnn8wL3XTIZ/33uR5Ta6d/snzTPkAWWGUSZBERr30aMGdwb\nR40e4Lr+C8eNxpSxLZ6pLHryAv9+9Ag88JXjLMuH9W/GKMUBuVh12dGjB1jcW886bD8cPXqg+cJh\nBONdMHV/vHPDxzBiQHNRx0kbxpgoq5jSJx405EA5c1modN/pkBAsIGx866G3ceWfNZVSoe6zwmNs\nbuu/2azF7Tj2xufN7/2b681c/wYdOwszlqDMpmNcSlR+sMmatM9woc8JgfpsJtBYawxgblHFMsLF\nzdWvuE4xHD7SKSBu+dyRuPGThyOTIV+12Zbd+1yT76myt8jI9ce+Ns0yk+rbpN1Du4Agco/wrnZ6\nUqZicuuiJRdTmEjqlM0gau/plHj5/Q58sGk3LnTRcwshsHh9eb1U3MgL4XhTfXVZId1GT17gspPH\n4Y6XVrju/4eLjsXQfk249rGF+PvbTi+nxroMsroy3Ri0hvW3vrE21WfQ1Z3HY187Hju7ejBiQDMW\nrd+BW2Y501nY+270sbDMd5fQyANoQzaDfbm8ZQbglwzvkhPGlnTsUgIT5X710wWEsWy/fk1FtTl5\nzCDs17+4fZOCEOHyUyUJcnkJDJ3NlW0Q6eGCP8zF9U+867rurn9+YFZ4U1I1FDkwnnnYfo5BVVYR\nnXfkCMw46xD84tMTPdvo3ViHI0cV3rTvufhY8/Mr15yKKeMG4eMTh+PH5x0GAJh6QAse/MoUc5vF\n/30WVt50Do4ePRAnH9iK8UP64OMTh3uet1Eq1IyDiEDFZCALiAum7m85vtYH7+OdOKG0ynSlqH4M\nd+URA5oxbbzV7jOgV7iaGQYPf3UqbpVsG2nDMOxaZhApUjLJaiHjc9jec7rvFLNhe5f5+W2p0pvb\nGLR5115s3FHY3qhJLACs6PDPsGrwyjWn4vqP+9sDfvQJbf35x4w0l9382YkYP0QrKGP82A4brvnQ\nXzF9Ak45aIi57ZB+TWisy+K2zx+F/Vt6m8unHqANWmMHF5bZcUsaBwCPf2MaDhra191IXWadgex5\nNOPsQ/DmD89Ar4aC0Pj3o7XrMvs7p2DCkEKRnctPcQ/MCsPoll6Yd93pRe1rXJOTDmw1B8YnvjHN\ndBioVQSELVtABTtTJALFp/s2vVxTMoeoaRWTHbn+gvwmvLR9F/b25NBYV/CYOebHmh1h5U3nAAAe\natPy8mzatdc3xbbMyIHNICK09G5wGJYN3FwzP3nUSHywaQ9unbUUA/W30ePGteBPXznON2W0nb9/\n4wSMGOj9lvzz8yfiwuPH4NO26OXGuiz6NNWZP4wojNQyl500DseOGYRshjCwt/Xt+8hRA8x78NzV\nJ6Mnl8fcD7bg+PGDy3Ls1r6NwRsB6N2QxW4p0v3g/TThevwBhdnDESMH4IiRjl1rBuNJDpOfKgn4\n2SDldSqzgkzKdEw8gwCwcUcXtuzehz1SKga7m+nNzy3Fh9s6Mebamfjrm94pFtyqunlhvFk+cvnx\nuP7jh4bq81XTJ+CNH55hGcCOP2CwRYgFcfjI/hjU21vd0dyQxbG6m2izLU9QhuLLxTTj7ENw+qFD\ngzeEVtO6XMJBRvboGtPSC1dMn4B3b/gYxg7ujV4NWcy84kSMHtQLn52kpcs4Zv+BmP+D0/HxicPL\n3pe0k0u5m6udIiYQaZEPtTmDmL9qK74lBb8d95NZgfts3NFlFud57A2rG2yx2VANxg7ujeEDRuOG\nv7unlXYjkyHfwf35q08uWwbQJ795AobY3qSJyCPVRlkOmSj+/o0TMLR/IybfqD0nIwY24+ozDgQA\nvPidU8ztXrbFV7T0UZt91BJClK/KXty4Gqnl9QrDfsGLKR0ioiYFxBurtipHxBps7+w23U/t7o/n\nKeRYCqLeLWy3BMYP6RO8kSKHjXCmxtZmENpn+Qf/9wWl5YtKIofbUoPXomtqOTDGREscRDplhauw\nUNrP9jfp1OSTHiZpm8ELi9vxwuJ2AM5CPqpJ284+fD/89gvHYFn7TkeUrpFdtLEuYwogu0Hz+atP\nRlLS9WSITO8l+QdvCFF7cZ5q4N0bPobLH3gD3z/7kEp3JbUI2FJtpMiLyUBAWGYLYdRkaZk5GNSo\ngFDX07thFxCqGBlGxw9xz7vz4/MOw5RxLfjSXXNw7JhBOPMwa/bVcs4KSiUjqZjsKrbBfRrwLV0F\nU030bqzDfV+eXOlupBZjUO222CAq1Rt1/LsYLhDCjMAuvjuxUpMColQVwa69hQHx4rvnol9THXZI\n6aXPOXyYw8g9bXwLrjvH/83zi1M0P//XZkwvqX9xUJ8lM4Ptdlvm1LS9JTHxktZkfTKyiilM/jHj\nt5EW43xNCgjZj74Y5BnEi0s6HOunjBtkCoi7LzoWO7q6ce6R1VN0HgD2b+mN11dsQT4vHCo2OVUI\nw8gIYXVzTcMw+cmjtN9uYXC3urTOWbEFAHDIfsH1PIq1XVSKmhQQbjOIUYOasWZLp9L+nQFeS5PH\ntuC+L0/G715ZgZMPbE19uUg3DhzaF53dOby6fBP+WoYaCUz1sy+Xxx0vL7elqkjuSNm3sQ4Lb/iY\n+f0QPYJfzlpw+QNvAH5eOl4AAAbYSURBVNDcoA8bESwgjNxgQWNIUqjJOIjhA5pQnyUM7lNwE335\nu5qLYkM2g+9+zFlJLIgTJ2i+9988bTwO2q8vTjqwFX+85LiqFA4AcOBQzR7ypbvmOtYZ9YcZxo4h\nDwwX7eSKB+CvX7fWLT9+/GC8cs2pOO+oEaizeR2OH9JHSbV6QKuWuWBpe/R53spBogQEEZ1JREuI\naBkRXRvVcVr6NGLJf5+FV689TT42Vt50Dt6/8SycYAu0mnvddNygp7w4+3DNcHzOEcMwXEqc9tNP\nHYGVN52Db380vHBJIx8Z3h/7S5lk//czhVxRXvWHmdrm7ouPxccnDsfVZxyIv3x1KsYP6YMzDlEL\ngIybQ4b1c3UKMRJFHtDaGz/55OHm8ju/pJZC5fxjRuKY/Qfi4mmlJZKMC0rKFI+IsgDeB3AGgLUA\n5gH4vBDCM3ps0qRJoq2traTjbt/TjZwQlqAzIQSWd+zC2MF9sG5rJ0bbUmov79iFA1q1h2f33h5s\n3bMPIwfWTtEXmXte/QAHD+uHKeNa8Mj8tdivXxNOmFD+SGaGiYv2HV3o01SnZKtct60Tg3o1oNmj\ncFVSIaL5QohJgdslSEBMBfAjIcTH9O8zAEAI8T9e+5RDQDAMw9QaqgIiSSqmEQDWSN/X6ssYhmGY\nCpAkAaEEEV1KRG1E1NbR4XQxZRiGYcpDkgTEOgCjpO8j9WUWhBB3CiEmCSEmtbaWVhCGYRiG8SZJ\nAmIegAlENJaIGgB8DsATFe4TwzBMzZKYQDkhRA8RfQPAMwCyAP4ghHCvB8owDMNETmIEBAAIIZ4C\n8FSl+8EwDMMkS8XEMAzDJAgWEAzDMIwriQmUKwYi6gCwqsjdBwPYVMbupAE+59qAz7k2KOWc9xdC\nBLqBplpAlAIRtalEElYTfM61AZ9zbRDHObOKiWEYhnGFBQTDMAzjSi0LiDsr3YEKwOdcG/A51waR\nn3PN2iAYhmEYf2p5BsEwDMP4UJMCIq7KdXFDRKOI6EUieo+I3iWiK/Xlg4joOSJaqv8dqC8nIrpV\nvw4LiOjoyp5BcRBRlojeJKIn9e9jiWiOfl4P6bm9QESN+vdl+voxlex3sRDRACJ6hIgWE9EiIppa\nA/f4W/oz/Q4RPUhETdV4n4noD0TUTkTvSMtC31siulDffikRXVhsf2pOQOiV634D4CwAhwL4PBEd\nWtlelY0eAN8WQhwKYAqAr+vndi2AWUKICQBm6d8B7RpM0P9dCuD2+LtcFq4EsEj6/lMANwshxgPY\nCuASffklALbqy2/Wt0sjtwB4WghxMICJ0M69au8xEY0AcAWASUKIw6DlavscqvM+3wPgTNuyUPeW\niAYBuB7AcQAmA7jeECqhEULU1D8AUwE8I32fAWBGpfsV0bk+Dq2E6xIAw/RlwwAs0T/fAa2sq7G9\nuV1a/kFLCz8LwGkAngRA0IKH6uz3G1oiyKn65zp9O6r0OYQ83/4APrD3u8rvsVFMbJB+354E8LFq\nvc8AxgB4p9h7C+DzAO6Qllu2C/Ov5mYQqJHKdfq0+igAcwAMFUKs11dtAGBUiq+Ga/ErANcAyOvf\nWwBsE0L06N/lczLPV1+/Xd8+TYwF0AHgbl2t9nsi6o0qvsdCiHUAfgFgNYD10O7bfFT3fZYJe2/L\nds9rUUBUPUTUB8CjAK4SQuyQ1wntlaIqXNeI6N8AtAsh5le6LzFSB+BoALcLIY4CsBsFlQOA6rrH\nAKCrR86FJhyHA+gNpxqmJoj73taigFCqXJdWiKgemnB4QAjxmL54IxEN09cPA9CuL0/7tZgG4BNE\ntBLAn6GpmW4BMICIjFT28jmZ56uv7w9gc5wdLgNrAawVQszRvz8CTWBU6z0GgNMBfCCE6BBCdAN4\nDNq9r+b7LBP23pbtnteigKjaynVERADuArBICPG/0qonABieDBdCs00Yyy/QvSGmANguTWUTjxBi\nhhBipBBiDLT7+IIQ4gsAXgRwvr6Z/XyN63C+vn2q3rSFEBsArCGig/RF0wG8hyq9xzqrAUwhol76\nM26cc9XeZxth7+0zAD5KRAP12ddH9WXhqbRBpkJGoLMBvA9gOYDrKt2fMp7XCdCmnwsAvKX/Oxua\n/nUWgKUAngcwSN+eoHl0LQewEJqXSMXPo8hzPwXAk/rncQDmAlgG4C8AGvXlTfr3Zfr6cZXud5Hn\neiSANv0+/w3AwGq/xwBuALAYwDsA/gigsRrvM4AHodlZuqHNFi8p5t4C+LJ+/ssAXFxsfziSmmEY\nhnGlFlVMDMMwjAIsIBiGYRhXWEAwDMMwrrCAYBiGYVxhAcEwDMO4wgKCYRiGcYUFBMMwDOMKCwiG\nYRjGlf8PioE1flgrDAsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9714172470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(reward_lst)\n",
    "plt.ylabel('Score')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD8CAYAAABthzNFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4XOWV+PHvkUa9V9tyk7vBptgY\nA4HQO5tAQpKFTXYJkJBsSM/uBrLJpmfTE/htwoYloYQEAgkJBAgBTG8GG4wbLnJXs3qdPvP+/rh3\nRiNpJI2kaZLO53n0eObOnZn36sr33LedV4wxKKWUUkNlpLoASiml0pMGCKWUUlFpgFBKKRWVBgil\nlFJRaYBQSikVlQYIpZRSUWmAUEopFZUGCKWUUlFpgFBKKRWVI9UFmIzKykpTW1ub6mIopdSUsnnz\n5jZjTNVY+03pAFFbW8umTZtSXQyllJpSRORQLPtpE5NSSqmoNEAopZSKSgOEUkqpqDRAKKWUiiqh\nAUJEDorINhHZIiKb7G3lIvKUiOy1/y2zt4uI3CoidSKyVUTWJrJsSimlRpeMGsQ5xpgTjTHr7Oc3\nARuMMcuADfZzgEuAZfbPDcBtSSibUkqpEaSiiely4G778d3AFRHb7zGW14BSEZmTgvIppZQi8fMg\nDPCkiBjgV8aY24FZxpgm+/VmYJb9eC5wJOK99fa2JpRSapKMMbh8ATr6vXT2++hweul1+3B5A7j9\nQdzeAG5fgAUV+Vx+4txUFzctJDpAnGGMaRCRauApEdkV+aIxxtjBI2YicgNWExQLFiyIX0mVUnHR\n5fTyxsFOdjb2sLOpm32t/XT2e7nz2pM5fl5pUsrQ6/bx9pFu3jrcye6jvRxqd3KwvZ9et3/M94rA\n6UsrqSzMSUJJ01tCA4QxpsH+t0VE/gysB46KyBxjTJPdhNRi794AzI94+zx729DPvB24HWDdunXj\nCi5KqcRo7/Pw57caeHxbE1uOdBE01oV2UUUBS6sLea7dyWNbmxIaIHrdPh55u5G/bWvm1f3tBILW\n5WFBeT61lQWsWVDKnJI8KgqyKSvIprwgi6LcLHIdmeRmZ5Cblcn2+m7+6Y6NbG/o5uwV1Qkr61SR\nsAAhIgVAhjGm1358IfAt4BHgGuD79r8P2295BPi0iNwPnAJ0RzRFKaXSUEuPm1uf2csDb9TjDQQ5\nbm4Jnz53GWcsrWRVTTEFOdYl5qwfPUtzjzshZeh2+fjlc3X8/rXD9Hr8LK4s4IYzF3Pa4gpOmF9K\nSV5WzJ9Vkm/t6/EHE1LWqSaRNYhZwJ9FJPQ9vzfGPCEibwAPiMj1wCHgQ/b+jwOXAnWAE7g2gWVT\nSk2CMYZ7Nx7me4+9gy8Q5EMnz+ea02pZMbso6v65jkzcvkDcy/Ho1ka+/vAOOp1eLju+ho+dsYjj\n55VgX3fGLceRCZCQsk5FCQsQxpj9wAlRtrcD50XZboAbE1UepVR8uH0BvvCHLfxtezPvXlbJty9f\nTW1lwajvyc3KwO2L3125PxDkO4+9w12vHOTE+aXcc/16VtWUTPpzcxzWwE6tQVimdDZXpVRy9Xn8\nXHfXG7xxsIOvXLqSj52xmIyMse/Wc7LiV4PwB4J88YG3eeTtRq4/YxE3XbKSrMz4jNjPydIAEUkD\nhFIzlMsboKnbxeKqwpj29wWCfOp3b7L5UCe3XLWG955QE/N35WVl0uXyTbSoYcYYbn5oG4+83ciX\nL17Jv569ZNKfGSnUxOTRJiZAczEpNSO5fQHO/NGznPfT52nt9cT0nh/8bRcv7Gnlu1esHldwAKuJ\nKR4X3btfOciDm+v57LlL4x4cQJuYhtIAodQM9IMndtHa68EYaO4ee3TRi3tbueOlA/zzqQu5av34\n5x/lxqGJae/RXr77+Ducf0w1nz9/+aQ+ayQaIAbTAKHUDLO1vou7XznIqppiADqc3lH3d/sC3PzQ\nNpZUFfCflx0zoe9s7nZzsN1JZ//o3zWSQNDwH3/aSmGOgx9ceXxM/R4TERr9tKupJyGfP9VogFBq\nBjHG8F8P76CiMIfvve84ADr6R29iuuPF/dR3uvj25avJzcqc0Pd22/0POxonduF95O0G3jrcxVcv\nO5aKJMxwfnLn0YR/x1SgAUKpGeSZXS1sOdLFv124nIUV+QB09I/cedzl9HLbc/u4aNUs3rW0csLf\n+63LVwPgC46/6cbrD/LTp/Zw7Jxi3rcmeTmSrJH3M5sGCKVmCGMMP396LwvK83n/2nkU2rOce90j\nB4h7Xj1EvzfAFy6YXJt/Ua71XW7v+Psh/vJWA0c6XPz7RSsS1rQU6at2M1pPDHmbpjsNEErNEK8f\n6GBbQzf/evYSsjIzcGRmUJjjoMcV/ULo8ga465WDnLuympWziyf13aGmKbd/fAHCGMNvXj7AytlF\nnL2ialJliFUoSV9bX2yju6YzDRBKzRD3bjxMca6DKyJSWRfnOugZoQbx+LYmOvq9fPzdiyf93bn2\nBLTxzqbeeKCDXc29XHt67YTTZ4xXcZ5V2+nTGoQGCKVmgrY+D09sb+LKk+aRlz3Q0Vycl0XPCBPY\nHth0hNqKfE5dXD7p78/LmliOo/teP0xJXlZS12cI1XacE2gOm240QCg1Azy2tQlfwHD1kDkMxblZ\nUWsQB9v62Xiggw+umx+XO/dwE9M4ahBOr58ndxzlsuPnTHj01ERMNJhNRxoglJoBHtvaxIpZRSyf\nNTjbanFe9D6IR7c2AvD+tfG5cw9NQBvPRfepnUdx+QLjnrU9WfnZVhOTSwOEBgilprujPW7eONTB\npccNX+J9pBrEkzuPcuJ8a4GdeBARchwZ4+qk/uvbTcwuzmV97eSbuMYjVINwaROTBgilprsntjdj\nDFx2/Oxhr0Xrg2jqdrG1vpsLV80atv9k5GZlxjzM1e0L8FJdKxetmpWUoa2RcrOty6LWIDRAKDXt\nPb+nldqKfJZWD1/MpzjXQa/HTzA4MCnsaXsW8YXHDg8okzGeNSE2HujA7Qty9srkL/upfRADNEAo\nNY15/UFe29/Ou5dFn0NQnJeFMdDnHeiHeG53Kwsr8llaHVsa8FjlZmXG3MT03O4WchwZnLa4Iq5l\niEWuNjGFaYBQahp783AnTm+AM5ZFT5NRnGutwRxqZvIHgrx+oIN3LZl4Wo2R5I0jo+vzu1s5dXFF\nUkcvhWRlZpCVKXFvYnJ6/eGcVFOFBgilprGX9raRmSGctiT6nXhoUlhoJNOOxh56Pf4R958Ma1W5\nsZuYmrvd7G/r590jBLVkyM3KjHuA+NCvXuWEbz4Z189MNA0QSk1jr+1v5/h5JeGawlDhGoQ9kunV\n/e0AcZkcN1SuIyOmGsSmQx0AnJzk0UuRxlPbiUUwaNjeYGWybep2xe1zE00DhFLTlNcfZGtDNyct\nKBtxn+K8wU1Mr+1vZ2l1IdVFuXEvj9UHMXYNYtPBTvKyMjm2ZnL5nyYjLzszrn0Q+9v6wo9Hyn2V\njjRAKDVN7WzqwesPsnbhKAEiXIPwY4zhrcNdnFw78v6TkZuVEdMw102HOjhxfilZmam7POVlZcY1\n1cabh7vCj51eDRBKqRR781AnAGtHrUEMpPw+1O6k2+Xj+HmlCSlPLKOY+jx+djb2JCxIxSrefRBb\njkQGiKkzOkoDhFLT1FtHuqgpyWV2ycjNRaE1IXpcft6uty5ix88rSUh5ch1jt+tvre8iaBi11pMM\n8e6DeOtwF2X5Vm1NA4RSKuXeOtzJmlFqD8DAmhBuH1vru8lxZAzL1xQvedljj2LaaS9JuqomMUEq\nVnnZ8Wticnr97G7u4XR7RT5tYlJKpVS3y0d9p4vVc8e+0BblOuhx+dha38WqmuKEtf3nZI09imln\nYw/VRTlUFSV+3enRFOQ44hYgdjT2EDSEhw5rDUIplVK7m3sBOGbO2LWB4twsOp0+tjf0JKz/Aawm\nJo8/OOpazzubeliVwtFLIQXZmfR54nOnv8s+F+sWWsN2++P0ucmgAUKpaWhXs9VUc8ycsS+2xXkO\ntjd04/IFEjq0NDQr2jPCUFe3L0BdS19Kh7eGFOQ44nYh39XUQ1Gug8VVBcDUSuGhAUKpaeidph7K\n8rOojqGppjg3i+YeN0DC+h9gYNnRkS6Qe4/24Q8ajp2T2v4HGGhiikxiOFG7m3tZObuIrMwMsh0Z\n9GuAUEql0jtNvaycXRzTanChyXIAy+KcoC9SeFW5EYa67mzqBkiLGkRhjlXW/kl2KBtj2H20lxWz\nrcCbn52pndRKqdQJBo111xpD/wNYKb8B5pXlUWAPe02EUA1ipJFMe4/2kePIYGF5fsLKEKvQ76Hf\nM7m7/cZuN71uPytnW0GvIDt+nd/JkPAAISKZIvKWiDxqP18kIhtFpE5E/iAi2fb2HPt5nf16baLL\nptR0dKTTicsXYOXs2AJEeYHVDLWosiCRxRpznYX9bf0sqixI+gJB0YTmh0y2o3q33RcUOhd5cahB\n+AJBvvaX7Wy2c1YlUjJqEJ8D3ol4/gPgZ8aYpUAncL29/Xqg097+M3s/pdQ47W/rB2BJVWzNRWUF\nVhNTZWFih5bmjBEg9rX2xVzmRCuw16We7MV8z1ErB9Myu2+nIA7zK450OPnta4c40Oac1OfEIqEB\nQkTmAZcBd9jPBTgX+KO9y93AFfbjy+3n2K+fJ7E0oCqlBjnQagWI2hhrBKFmlLL87ISVCaxhrhC9\nicnjD3Ckwxke6ZNqBXGqQRxs66eiIJsSu58nLzsT5ySbrQ62W+d3UWXim+ISXYP4OfAfQOgvogLo\nMsaEfuv1wFz78VzgCID9ere9/yAicoOIbBKRTa2trYksu1JT0sH2fopyHVQUxHbBf9+auayqKea6\nM2oTWq5wH0SUTurD7U6CJvZaT6IVxqkP4kBb/6BAXZDtwOmbbNCxag61FYkPpgkLECLyD0CLMWZz\nPD/XGHO7MWadMWZdVVX0ZRSVmskOtPWzuLIgphFMALNLcnnss+9mXlli70jDo5iiNLHsa7WaYtKn\nBmGPYhqhBvHMrqP0usdeHe5ge/+gC3l+jiMuNYiiHAflMd4ATEYiaxCnA+8VkYPA/VhNS7cApSIS\nGioxD2iwHzcA8wHs10uA9gSWT6lpaX9rf8I7nCditGGu++xmscVpVoOI1sS0vaGb6+7axPce3zXq\nZzi9fo72eAY1BeXHIY14qFaSjBb4hAUIY8zNxph5xpha4CrgGWPMh4FngQ/Yu10DPGw/fsR+jv36\nM2a0OflKqWHcvgCN3a6Y+x+SabRhrvta+5hVnBO+MKdafriJaXiA2GynUW/sGn1luEPtVlPQwkE1\niMxJz6042N6ftPObinkQXwa+KCJ1WH0Mv7a3/xqosLd/EbgpBWVTako73OHEmMQPWZ2I0Ya51ne4\nWFiePmXOzxq5iSl04R8rZcbBtlBnckSAsFeqm+i9r9cfpKHTxaKK5MwViSlci8gZwDJjzJ0iUgUU\nGmMOxPolxpjngOfsx/uB9VH2cQMfjPUzlVLDHYhyUUoXoSamaAvx1Hc6OXXxsDEpKZORIXbCvigd\n6h1WgAgNJx7Jgfbho8nysx34gwZvIEiOPaprPA53WJ35aVODEJGvY93132xvygLuTWShlFITc8S+\neC1Ig9nIQ+U4MsiQ4XfeXn+Q5h4388ryUlSy6EZK2Bf6Hbf1eUZNX36ozUll4eBms/xsO0hOsB/i\nUJSgk0ixNDG9D3gv0A9gjGkEEpfRSyk1YQ1dLgqyM8Pj7tOJiJCf7Rg2dLS5203QwLw0C2qFOY5h\n/QXGGA53OMm0Z3uPdqGv73Iyv3xw0MsbpRYViwa73yNZwTSWJiavMcaIiAEQkfSruyqlAKvjtKY0\nLykjXCYiLzsT15B5APWd1h15utcgfvrkbnLstaqPmVPMO0099Hv9lI0w3LSxyz0s8WCeXYOY6Eim\nhi4X2ZkZVBYkZ0GlWALEAyLyK6zhqR8HrgP+L7HFUkpNREOXi7lpdqGNFC3VRH2ndVc8P8HzMMar\nICdzUG3n1mfqwo+XVBXwTlPPiDUIYwyNXS4uOHbWoO3hGsREA0SnizmluUnLVzVmgDDG/FhELgB6\ngBXAfxljnkp4yZRS49bQ6UroqnCTlRclm2l9p5MMsSbspZPCHAeNXdY6Gd4hixwttdOij1QT6Oj3\n4vEHqRlyTKEaxFhLr46kscvF3NLk3QCMGiBEJBN42hhzDqBBQak05vT66XT6knoBGa9o6yHUd7qY\nU5KXsLWwJ6ogog+ipdc96LVQgBhpTkMosNQMORf5k2xiauxyc8ayygm9dyJGPSPGmAAQFJHUL/Gk\nlBpVY5I7MCcif4QmpnQsc2QfRFP34AARSksyUlNRqDN5aIAYbajvWHyBIEd73cM+M5Fi6YPoA7aJ\nyFPYI5kAjDGfTViplFLjFmrLT+YFZLzyszNp7fUM2tbQ5eKUxeUpKtHICnMc4VQbkbOmS/KywivO\njVQTCO0/tDYXGvLa6x7/bOrmbjfGwNzS5DXFxRIgHrJ/lFJp7KN3vgEMvyilk/whfRDBoOFoj5vZ\nxenV/wBWMHP7gvgDQZojahBl+VnkjbFeRGOXi7ysTErzBw83rrDX3Gjr80R726gaR6iVJFIsndR3\n26u+Lbc37TbGjJ3GUCmVNJGpG6qLkjMEciKGrqjW4fTiDxpmpWGACKf89gYGNTGV5GeHU3GMWIPo\ndlFTmjtsuHFBdia5WRm0TyRAdKdhgBCRs7EW8jkICDBfRK4xxryQ2KIppWLVYzdZ5Gdn4kizzt5I\nQ7OZHu2xLryzitMvqIUWDXJ6/YOamErzssgfo4mpoSt6X4GIUFmYQ1ufd9zlqWuxUqLXlKRRgAB+\nAlxojNkNICLLgfuAkxJZMKVU7FrsC+33rzw+xSUZXX6OA5fPSlYnIhEBIv1qEAURGV2bewY3MWVn\nZpCZISM2MTV3u1gxK/p6NVaAGH8N4ontzRw/ryQ8VDYZYrnVyAoFBwBjzB6sfExKqTQRuoClY1t+\npPzsTIwZSPl9tMe6UKZjgAh1RPd5AjR2ucMJEE9fWmmlDRlhbYdg0NDW56W6KPoxVRbmDOuoj0VT\nt5t1C5PbmR9LDWKTiNzBQIK+DwObElckpdR4DVxo06+pJlJoHkC/109edma4BlGVhv0mBXZHdKfT\nS1ufh4+cuoDLT5wbDhT5OZlRh7l2OL0EgmbEY6oqymbLka5xlaXP48fpDVCd5PMbS4D4V+BGIDSs\n9UXglwkrkVJq3EIX2pHuWtPF0FQTR3s8VBZmp90kORhoYjpkp/WeVZw7ZG0HB/1RAkSodjDSYIGK\nghw6+j0Egiac9G8sLeHzm34BwgHcYoz5KYRnV6dfuFdqBmvpcVOc60hq+/REDHT8hgKEO22DWigj\nbp29XnZl4eDLXl5WJq4ofRAtdoAYqQZRWZhN0Fg1k6GfOZKWcNBJ7u8qlrC9AYjsNs8Dnk5McZRS\nE9Hc4067XEbRDGQztS6sR3vcadssFsrSuveoFSCGXvALcqL3QbSOcTGvLBr/XIhwgEjy7yqWAJFr\njOkLPbEfp1faRaVmuKM9nrS9E480dP7A0R5PWnZQgzVnIStTwsNLKwsHp/XOG6GJKZS3qbIoehrw\nUK2hrTf2oa6pamKKJUD0i8ja0BMROQkYfbVupVRStfd70rKjd6j87IEmJl8gSHu/h+o0DRAiQml+\nNu391oV8aHNQ/ghNTK29HgpzHOFjHapyArOpW3s9ZDsykr4QVCx9EJ8HHhSRRqyJcrOBf0xoqZRS\n49Le56VihIVr0klkE1N7nxdj0nvmd2leFq29HopyHeFEeyH5IzQxtfR6Rj2mqgkEiJZeD1WFOUlf\nCCqWVBtviMhKrLUgQFNtKJVWnF5rCGR5YfoHiIKcgVFMoQvk0KabdFKWb5WtKkpncrTMtGDd7VeO\nEiCK8xxkZQqt4woQ7qT3P8AoTUwicrKIzAawA8Ja4LvAT0Qk/VIvKjVDtdtpG5K1DOVk5GcN5Dfq\nsJtuKmIcyZMKoWR70UYbWYkHozcxjVaDCKfbGFcfhIdZKehjGq0P4leAF0BEzgS+D9wDdAO3J75o\nSqlYDFxo0/dOPCTUxOTy+mnvt+6gy9O4aSxcg4hywQ9lew0EzaDtrb1j9wdVFeWMqwaRqtFeozUx\nZRpjOuzH/wjcboz5E/AnEdmS+KIppWIRutCm8514SLYjA0eGcKTDFW5PT+e+k4EaxPAyhmaFu3yB\ncOZXp9dPn8c/ZoCoLsqhocs96j4hLm+AHrc/JZ35owYIEXEYY/zAecANMb5PKZVEocyg6XyhjZSf\nnckfNh0BwJEhFOemb2q3UrsGEa2JKXJNiFCACDf3jRGsq4py2HKke8zvf+NgR3jWeSqGA492ob8P\neF5E2rCGtb4IICJLsZqZlFJpIHRRmgpNTAA5WZlgpycvK8gmI8Z0E6lQZtcgotUICrIHpw0Ba3Y0\nQHn+6OeiqnD0dBuBoOEHT+zi9hf2h7elVROTMea7IrIBmAM8aQZWJMkAPpOMwimlxtbe5yE/O3PE\ncffpxusPhh+ne61ntBpEOPGgZyBAhPqDysY4rqqiHILGah6MNsHxmV0tg4IDpF8NAmPMa1G27Ulc\ncZRS49Xe703rjt6hBgWINK/1HD+vJPwzVKiJyeUbGMkUqkGU5Y/ebBaqkbREmQHv9Qf5+D3DE2Yv\nrEh+AoupccuhlBpRW59nSnRQh/iDAwGiPM2H5taU5vHIp8+I+lrkehEhHf3WFLGxAnaVHRSijWQK\nBZlIZ6+oIseR/ESMGiCUmuLa+7zMmQKJ+kJ8gYFhoenexDSawhyrltDviahB9HvJEMbseA/Nk4i2\ncFB7xHKkT37hTJZVF8ajuBMSUxJ2EVkoIufbj/NEpCixxVJKxaq935P2TTUjKU5ybqF4Ksy17q/7\n3AMBosPppSx/7I73UJ9GtAARqkH89EMnsHxWESKS9BQbIWMGCBH5OPBHrIlzAPOAv8TwvlwReV1E\n3haRHSLyTXv7IhHZKCJ1IvIHEcm2t+fYz+vs12snelBKzRTGGDr6vWnfVDOS3Kz0WygoVqGhrb1D\nahBjdVCDNWGwKMcRvQZhd3QfN3d4v0eyxXJ2bgROB3oAjDF7geoY3ucBzjXGnACcCFwsIqcCPwB+\nZoxZCnQC19v7Xw902tt/Zu+nlBpFn8ePL2AoL5g6d+IPfvK08ONUtKvHSyhARNYgOp3eMYe4how0\nm7ozxpFQyRBLgPAYY8KNYiLiAMwo+wNgLKF1JLLsHwOci1UjAbgbuMJ+fLn9HPv18yRV9Sqlpohu\nl9UpWpqX+otJrE6uLeefT10IDMwlmIoyM4T87Ez6PAO5Szv7fZTFGKyrinJo7YlegxAZSPORSrEE\niOdF5CtAnohcADwI/DWWDxeRTDstRwvwFLAP6LJnZwPUA3Ptx3OBIwD2691ARZTPvEFENonIptbW\n1liKodS01eW0Lk4lYwyrTDefP38ZV6+fzxVr5o69cxorzHHQ5xncBxHrkOPZJbk09QxfWqez30tp\nXlbM61UnUiwB4iagFdgGfAJ4HPhqLB9ujAkYY07E6rdYD6ycYDkjP/N2Y8w6Y8y6qqqqyX6cUlPa\nQA1iagWIisIc/vv9xw9bY2GqKcxx0Gs3MRljrD6IGO/8a0rzaO52D0v21+n0hifopVos60EEgf+z\nfybEGNMlIs8CpwGlETme5gEN9m4NwHyg3m7GKgHaJ/qdSs0EoRpEulxQZprC3IEaRK/Hjz9oYq5B\n1JTm4QsY2voGL7va7fIlfeW4kcQyimmbiGwd8vOiiPxMRIY1AUW8r0pESu3HecAFwDvAs8AH7N2u\nAR62Hz9iP8d+/ZmI9B5KqSi6XFb3YOkUa2KaLgpzHOF5EOHO5VhrEPbclcauwc1M6RQgYpko9zcg\nAPzefn4VkA80A3cB7xnhfXOAu0UkEysQPWCMeVREdgL3i8h3gLeAX9v7/xr4rYjUAR329yilRhHu\ng0iTC8pMU5jj4HCHExjIwzSeGgRAY5ebNQsGtne7fNRWFMS3oBMUS4A43xizNuL5NhF50xizVkQ+\nMtKbjDFbgTVRtu/H6o8Yut0NfDCG8iilbD0uHzmOjCnflj9VRTYxhSa4xVqbGwgQ6VuDiKWTOlNE\nwhd0ETkZCP01Dl9vTymVNF1OnzYvpVBRxCimHpf1b6wX9+JcB4U5DhoiAkQwaOhJowARSw3iY8Bv\nRKQQEKwJcx8TkQLgvxNZOKXU6Lpc3ik1B2K6Kcx10Of2Y4yhx20198WaPkREqCnNHVSD6PP6CZr0\n6VOKZRTTG8BxIlJiP49cLOiBRBVMKTW2Lqdvys2BmE4Kchz4gwaPP0i33R80nhXyFpTnc6jdGX4e\n/owpVINARC4DVgG5ocnNxphvJbBcSqkYdLt8LChP/joBylIUysfk9tPj9pGXlUm2I/b8UosqC3hx\nbxvBoCEjQ8LzWtKliSmWYa7/C/wj1ipygtWRvDDB5VJKxUD7IFIrnNHV46fH5ac4b3wrKNRWFuDx\nB2nqcQNMvQABvMsY8y9YifS+iTXZbXlii6WUikW3y6eT5FIock2IHrdvXM1LAIvs4awH2/qBqRkg\n3Pa/ThGpAXxYcxyUUink9gVw+QJpczGZiQqHNDGN91wsqrICxP40DRCx1If+as+I/hHwJlZG1gmn\n3VBKxUdPml1MZqKiiCambpdv2PrSY5ldnEt+dib7W63E11MqQIhIBrDBGNMF/ElEHgVyh4xkUkql\nQFcoUZ/2QaRMeE0Ij48el5+lVePrgxARFlUWsL91oAaRlWmlEU8HozYx2Yn6fhHx3KPBQan0kG53\nmzNR5LKjPW7fhIanLq4qZF9EDaIkLytlS4wOFUsfxAYRuVIX71EqvfTaE7OKxtkxquInVIPocfvp\ncY2/kxpg5ewi6jtddLt8dLsmFmQSJZYA8QmsRYK8ItIjIr0i0pPgcimlxhBahyDUDq6SL9ee99DY\n5SJoJlabO36etfb0joZuup3pk2YDYptJXZSMgiilxiccIHI0QKRSSV4WRzqtdBnjnQcBsLrGChBb\nG7rpdvmoKEyfYcuxTJQTEfmIiHzNfj4/MnmfUio1BmoQ6XPHOROV5GVRb6f8nkgTU1lBNvPL83hx\nbyu7j/am1cz4WJqYfok1Oe7IUUpdAAAftElEQVSf7Od9RHRcK6VSo8/jIzNDyM2KPbWDij+rBmEH\niAk2Dx0/t5SX69rx+oOctTx9llKO5S/rFGPMjdgT5owxnUD61IGUmqF63X6Kch1pM+JlpirJy8IX\nsBa/nEgNAuA4ux8CGLT8aKrFEiB89qpwBqylRIFgQkullBpTn9sfHkWjUieyU3miHcyXrJ4dflxV\nlDPpMsVLLAHiVuDPQLWIfBd4CfheQkullBpTj9uv/Q9pIDIoTKSTGmBhRQF3XnsyF62aRWVh+gSI\nWEYx/U5ENgPnYWVzvcIY807CS6aUGlWv26cjmNJAZL/DZGp056yo5pwV1fEoUtyMeTQicitwvzFG\nO6aVSiN9Hj+z06i9eqaKrEE4MqfXgIFYjmYz8FUR2SciPxaRdYkulFJqbL1ufzjVg0qdYvscpNPo\no3gZM0AYY+42xlwKnAzsBn4gInsTXjKl1Kj6PH6dRZ0GQjWIoDEpLkn8jac+tBRYibWa3K7EFEcp\nFQtjDL1uX3jBGpU6uVlW5tUZGSBE5Id2jeFbwHZgnTHmPQkvmVJqRB5/EF/AaA0iDWRmWPNQAsHp\nFyBi+evaB5xmjGlLdGGUUrEJpdko1gCRcqtrSsjOzODGc5amuihxF8sw11+FHovIEqyUG1cZY1Yl\nsmBKqZGFUn1rJ3XqleRnsee7l6S6GAkRSxNTjYh8QUTeAHbY77kq4SVTSo2ozxPK5Kp9ECpxRgwQ\nInKDiDwLPAdUANcDTcaYbxpjtiWpfEqpKEJNTFqDUIk02l/X/wCvAv9kjNkEICLTrxdGqSlIFwtS\nyTDaX9cc4IPAT0RkNvAAoPVZpdJAeLlRbWJSCTRiE5Mxpt0Y87/GmLOw8jB1AUdF5B0RGTNZn72w\n0LMislNEdojI5+zt5SLylIjstf8ts7eLiNwqInUislVE1sbpGJWadkJ9ENrEpBIppolyxph6Y8xP\njDHrgMux14YYgx/4kjHmWOBU4EYRORa4CdhgjFkGbLCfA1wCLLN/bgBuG9eRKDWD9IcChCbrUwk0\n7sxSxpg9xphvxbBfkzHmTftxL/AOMBcrwNxt73Y3cIX9+HLgHmN5DSgVkTnjLZ9SM0G/N0BWppDt\nmF7J4VR6Scpfl4jUAmuAjcAsY0yT/VIzMMt+PBc4EvG2enubUmoIp8dPnp3iQalESXiAEJFC4E/A\n540xPZGvGWMM9kp14/i8G0Rkk4hsam1tjWNJlZo6nN4ABdq8pBIslvUgonUWdwOHjDH+Md6bhRUc\nfmeMecjefFRE5hhjmuwmpBZ7ewMwP+Lt8+xtgxhjbgduB1i3bp0Ou1UzktMbID9baxAqsWKpQfwS\neA3rovx/WHMjHgR2i8iFI71JrJXUfw28Y4z5acRLjwDX2I+vAR6O2P4v9mimU4HuiKYopVSEfq9f\naxAq4WIJEI3AGmPMOmPMSVh9CfuBC4AfjvK+04F/Bs4VkS32z6XA94EL7Ayx59vPAR63P7cOKxB9\naiIHpNRM4PRoDUIlXiy3IMuNMTtCT4wxO0VkpTFmv1VJiM4Y8xLWGtbRnBdlfwPcGEN51DQWCBra\n+jzM0qU0R9Xv1eVGVeLFUoPYISK3ichZ9s8vgZ0ikgP4Elw+NcN89r63OOV7G3D7AqkuSlpzegPk\naxOTSrBYAsRHsZp9Pm//7Le3+YBzElUwNTM9ts3qdtrX2pfikqQ3p9dPvg5zVQkWy3oQLuAn9s9Q\n+r9YJURdSx+rakpSXYy05fQEyM/RAKESK5b1IE63cybtEZH9oZ9kFE7NPGX5VvK5uha99xiJMcYa\nxZStTUwqsWL5C/s18AVgM6ANwyqh/AFrasvu5t4UlyR9efxBggatQaiEiyVAdBtj/pbwkqgZYfOh\nDgBOWlg+7LXQnTHAjsaeYa8rSyhRn9YgVKLF8hf2rIj8CHgI8IQ2hhLxKRWrO18+wDf/uhOAl286\nl7mleYNeD90Zl+Rl0dDlorPfS1lBdiqKmtacXqsir/MgVKLFMorpFGAd8D0GOqt/nMhCqenHGBMO\nDgC/33ho2D6hO+OTa63axTtNWouIZiBAaA1CJdaYAcIYc06Un3OTUTg1fbT1eQH494tWcPrSCn7x\n7D4OtzsH7dPeb+1z2pIKAHYf1X6IaELNcNoHoRJtxFsQEfmIMeZeEflitNeH5FdSalSHO/oBWDGr\niMwM4eW6dv7tj2/zwCdOC+/T0mO1YK6qKaYsP4s9GiCicnqsGoT2QahEG+0vrMD+tygZBVHT2y+e\n3QfAiQtKOWdlNZsOdrBhVwsd/V7K7X6Gll5rocLqohyWzyrSkUwjCNcgtA9CJdiIAcIY8yv74S+N\nMbrwgpqULUe6eNeSCioLcwD4/PnLefqdFp7c0cxV6xcA0NJr1SCqi3NZMbuIh95swBjDaDm/ZiKn\nHSA0m6tKtFg6qV8WkSdF5HoRKUt4idS04/T66ej3csayyvC2VTXFzC7O5eV97eFtLT0e8rMzKcxx\nsHxWEX0eP43dsSx/PrP0h5uYtAahEiuWTurlwFeBVcBmEXlURD6S8JKpaePpd6w1oRZXFoa3iQgn\n1Zbx5qHO8LajvW6qi6waxsrZVsvmHm1mGsZlj2LK0wChEiymJUeNMa8bY74IrAc6gLsTWiqVtlp6\n3Vz88xc42NYf83tueXoPS6sLOe+Y6kHbT1pQRkOXi4/csZFA0NDa46G6yEphvWyWFSB2NHbHr/DT\nxEAfhDYxqcSKJRdTsYhcIyJ/A14BmrAChZqBntxxlF3Nvfy/Z+pi2r/H7WNfaz/vWzOXrMzBf25X\nr19AZWEOL9W1sbu5l5ZeN1XFVg2iJC+LlbOL2HigI+7HMNU5vQFyszLIzNC+GZVYsdQg3gZOBL5l\njFlujPmyMWZzgsul0lRloTXi6GB7bDWIdnv+w5yS4Yvb5GVn8udPvQuAJ3Y0c7DdGW5iAlizoJSt\n9d1Ya0mpkH6PJupTyRFLgFhsjPkCsE1ECsfcW01rbl8QgM2HOmnsco25f4c9+W2klBnzy/M5Zk4x\nt27YC8DqiBTfx9aU0O3y0RDD98wkXU6fTpJTSRFLgFglIm8BO7BWktssIqsTXC6VpkJpHgAe3FQ/\n5v5dTitAlOePnFPpPSfMCT++Ys3c8ONVNcWAJu4b6sW9rdRWFIy9o1KTFEuAuB34ojFmoTFmAfAl\ne5uagUJj8AGOdDpH2dPS6bRWpS2113mI5sq181hfW87vPnbKoHb1Y2YXkyEaICL5A0F63H7WRcmG\nq1S8xdKQWWCMeTb0xBjznIjo7csMFVor+oT5pRzuGDtAhAJK4SiTumYV5/LAJ08btj0vO5PFVYXs\n1JFMYS6fZnJVyRNLDWK/iHxNRGrtn69irUutZiCnN4AjQ1hSVcCRGAJEeFLXBGf9rqop1hpEhFCA\nyNUAoZIglgBxHVCFtR7EQ/bj6xJZKJW+6lr6mFWcy4LyfJp73Hj8oy8y6PT6yRDIccQ05WaYVTXF\nNHW7w53dM53baw0SyMvSAKESb8zbOmNMJ/DZJJRFpTm3L8BLdW28f+1cFpTnYww0dLpYXDXy4LZ+\nT4CCbMeE8ymtskc17Wjs5t3Lqib0GdOJ06eJ+lTyjJbu+5HR3miMeW/8i6PS2aF2J05vgJNry8Or\nwR3ucI4aIJxe/6SGZEaOZNIAEZFmQ2sQKglGq0GcBhwB7gM2Ajptc4YLtX8X5TqYX54PMGY/RL83\nMKlJXaX52cwtzUt4P8SRDietfR7WLkjvfJSah0kl02gNw7OBrwCrgVuAC4A2Y8zzxpjnk1E4lV5C\nF6fcrEyqCnPIcWSMOZKp3zO5GgTAsTXFCc/JdO1db/D+X74SnreRrg7YM9ijzUxXKt5GDBDGmIAx\n5gljzDXAqUAd8JyIfDpppVNpJTTENS8rk4wMYX55fmwBYpJpIVbVFHOgrT+8ZnUitPVZa1Fc/PMX\n2VafvsNqn955lKqiHBbYNTilEmnUoSUikiMi7wfuBW4EbgX+nIyCqfQTamIKNW8sKM/ncMfoaTCc\n3sCk1y04bm4JxsDWBF24n9p5lC57Ql9zj5v3/M9LBIPpk//JGMO3H93J5kMdvLq/ncuOm6OLKKmk\nGDFAiMg9wKvAWuCbxpiTjTHfNsY0JK10Kq0M7SBdUJ7PkQ7nqMn0+r1+8ie58tn6ReU4MoQX9iZm\nYcPvPLYTgC9dsJxrTlsIwF2vHEybINHj8vPrlw5w5W2v4vYFWTFbVwFWyTFaDeIjwDLgc8ArItJj\n//SKiM5cmoFcvsEBorYinz6Pn1Z7qdBonJ7J1yCKcrNYu7CMF/bEP0C09Xk41G41k914zlJuvvQY\nAL716E4+94ctcf++iWjtG7yq3pJRRo0pFU+j9UFkGGOK7J/iiJ8iY0zxWB8sIr8RkRYR2R6xrVxE\nnhKRvfa/ZfZ2EZFbRaRORLaKyNr4HJ6Kp1BW1dAs3tCiPnUtfSO+p9/rj8vayWctr2JHY8+owWgi\nQivW/eO6+WRkCLlZmXzirMUA/PXtRjZHrHiXKi09g495abUGCJUcE5veGpu7gIuHbLsJ2GCMWQZs\nsJ8DXIJVW1kG3ADclsByqQnwB4L8+sUDHDunmCL7gh+6UO0dIUD4AkH6PH6KckdO1BerM+05EC/X\ntU36syKFEg5++tyl4W03XbySRz9zBgBX3vYKvW5fXL9zvCLTnWdmCOUjpE5XKt4SFiCMMS9gLU8a\n6XIGliu9G7giYvs9xvIaUCoic1Bp4y9bGvEGgrznhJpwB2l1UQ5FuY4RaxDN3W6Mgbmlkx+Suaqm\nmLL8rLg3Mx3pcJGZIYOGjYoIq+eWcO3ptQC8sq89rt85Xvvb+nFkCC/+xzm8dvN5KS2LmlkSWYOI\nZpYxpsl+3AzMsh/PxZqUF1Jvb1Np4s3DVlNL6KIJ1oV0aXUhe1t6o74ndOc7pyRv0t+fkSGcvrSS\nV/a1x3WFuSOdTmpKc3FkDv+vcPMlx1CQnclzu1vi9n0T8eyuFo6ZU8z88nyqIlbcUyrRkh0gwoz1\nv3zc/9NF5AYR2SQim1pbEzOqRQ13pMPJCfNLyR2S4mFZdSF1LdGXHw01PS2JU5v5u5ZU0tzj5kBb\nbMudxuJIh5P5ZdHnFGQ7Mrho1Wwe2dI4ZlLCRDnY1s+u5l6uXKv3Syr5kh0gjoaajux/Q7dmDcD8\niP3m2duGMcbcboxZZ4xZV1WluXmSpcvpoyzKoj9Lqwtp6/NEnYH81qFOSvOzqInTrN93L6sE4NGt\nTWPsGbsjna4RAwTAJcfNod8bSFln9W9ePkCGwAWrZqfk+9XMluwA8Qhwjf34GuDhiO3/Yo9mOhXo\njmiKUmmgy+WlLMqyocuqrZFMu5qHNzO9sLeVs5dXxW1S1/zyfM5cXsW9rx3CFwhO6rMau1zU3vQY\nrb0e5peP3AR22pIKq/1/b3w7x2Nx23P7uOfVQ1y5dl44OaJSyZSwACEi92FNtFshIvUicj3wfeAC\nEdkLnG8/B3gcaxGiOuD/gE8lqlxqYrr6fZTkDa9BrJ5rpeMemp6is99LW583nK47Xq59Vy0tvR7+\ntr15Up/z+oGB8RMXjnJ3XpjjYO3CMp7fndzmTLcvwP8+v4+C7Mzw3Aylkm3yA9RHYIy5eoSXhg3D\nsPsjbkxUWdTk9Hv89Hr8UTtIq4pymFuax9v1XYO217Va/Q/xHrN/1vIqaivyuevlA7z3hJoJf05T\ntzX57K2vXUDZGMNGz1lRzQ+e2EVTtysuHe6xeH5PK90uH/dct16HtaqUSVkntZoajva4+cfbXwVg\nUWX0pchPmF8yLEDsPZqYAJGRIVzzrlrePNzF20e6xn7DCB7f1kRZftaYwQHggmOrAfjda4cn/H3j\ntaupFxErzYhSqaIBQo3o/tcPc8r3NrC9wcqscuL80qj7HT+vlCMdLj5//1vhbXUtfeRmZSSk7fwD\nJ82jND+Lrz+yA/8E+yL2tfZxzorqmPZdUlXI2SuquPPlAzi9icsoG+ILBPnZ03uoKckbNmpMqWTS\nAKGiMsZw00Pbws+/977jqBnhYn/5iVZTz1+2NIYT3NW19rG4spCMjPhnHS3KzeI7V6xmy5Eufvnc\nvnG/3+n14/QGwqlCxiIi/OtZS+j3Bnhikn0fsQj1r3z41AUJ/y6lRqMBQg3S3O3m2jtf565XDgJQ\nlp/F1m9cyD+dMvLFak5JHj/54AkA7D5qjWY63N7PoqroTVLx8A/H13DFiTXcsmHvuIegtvdZQ3Ir\nCmNv2z+5tpwF5fn86c36cX1XpN3NvfTEkLbjrpcPUFuRzyfPXDLh71IqHjRAzDD+QJBP/W4zNz+0\nddCM5LePdPHjv+/m1P/ewLO7W/nmX60U2I9+9t0Ux5BLKdRW/sbBDoJBQ2O3m3kJHpr5zctXM6so\nh0/8dvOYE9n6PH62N3Szs7EnPCu8ZhwdzhkZwvvXzuWVfe0TmlltjOGin7/AmT98lsN29tgHNx3h\nzcOd7GzsCTeV/WlzPW8e7uJfTqtNSO1LqfFI2CgmlX76PH5Wf/3v4eflBdn8+0UrAfjh33fxct3g\nnEOXHTcn5j6EeWV51JTk8tr+di5ZPQevPzhik1S8lORl8aMPnsCH79jIt/66k+++77io++1v7eOi\nn7+ALzAQELMyhXW141t/+sq18/j503v56J1v8I33HMtHT18U83t73FbfRZfTx5k/epbTl1YM+n2f\nsqicuWV5PPRmA4urCkatsSmVLFqDmEGe3TVw5zu/PI9fPLsvnPyuqcsa9vnhUxZQ991LeP0r5/Hz\nq06M+bNFrFxJL9e1h5chTXSAADh9aSWfOHMxv9t4mL+8FX0tq/vfOIIvYPjyxSuZV2aV6Y5rTh53\nB/D8iGU+f/zknnFled1vD/utKcmlMMcRDg4rZhXx0XfV8nZ9F0/tOMrlJ9Zw+z+v085plRa0BjGD\nPGanqLj7uvWsW1jG+T99ns/c9xaPffYM6rtc3HDmYr5iT8qqLh5/eoyzVlTx4OZ6/r7D6mStiUMW\n11h84YLlPLOrhc//YQsef4APrZsfnr3t9Qf50+Z6Llo1i389ewkff/ci/EEz4Qvwo585gz+8cYR7\nNx7iB0/s4jtXRK+1tPS62dnYw9krqqlr6Q032d37sVNYVFnAw1saqS7O4V1LrPQh/3bRChz2ehRK\npQsNEDPE7zce5okdzayqKeas5VYOq3s/dgqX/8/LfPiOjXj9QaonmSn0jKXWxe72F/YDMG+UHEfx\nlJuVyYOfPI0Lf/YCX/7TNp7b3cov/mktGRnC83taae/38qF1VqovR2YGjklcg1fPLWH13BIyM4S7\nXjnI6poSrlo/uDnoYFs/l//iZbpdPq5eP5/7XrcSFV+9fgGL7dXgrlgzOPleYRwWVVIq3rSJaRrY\ne7SXl/a2hTs6+z1+am96jOVf/RubD3XwjUd28JU/W0NW775uffh9S6oK+e/3HxdecnOyqaRL87Mp\nyh240EVLzZEopfnZPPmFM7ly7Tz+tr2Zu145iNcf5Lbn6qgszObM5fFN7Pjli1dyzJxibnpoG1f8\n4mXqWnr5xG83cceL+zn7x8/R7bKan0LB4bLj5nDzpSvjWgalEk3imVs/2datW2c2bdqU6mKk3MU/\nf4Fdzb0U5jj4f1ev4VuP7oyaEvvOa0+OOjns4S0N3PPqIW656sRJ3/U/t7uFnz61h+tOXzTsLjkZ\njDF8/J7NPP3O0fC2W646kctPjH9ZWnrcrP/ehqivffGC5Vx7ei03PbSNUxdX8M+nLoz79ys1USKy\n2Rizbsz9NEBMTcYY7nz5IK/sax90MQyZW5rHbR9Zy+sHOsjMEK5ev2DGtG/3uH18+Y9b2Xyok/OP\nncV3r1gdt4yyQzV2ufjPP2/jQFs/15+xiEPtTvKyM/nShSsS8n1KxYMGiGnG6w+S7cjA7QvwjUd2\ncP8bAwvwzS7O5Y5r1tHr9vP9J3Zx/spqPnPeshSWVimVzmINENozNgVsb+jmfb98mYIcB13OgaGV\nVUU5/OXG0wfNVXj4xtNTUUSl1DSkASJNPLPrKN959B1uvvQYLjh2Vnj70zuP8m9/fBtfwISDw/JZ\nhTz6mXeT7dAxBkqpxNEAkWJ9Hj//9fB2HnrTmuT1yXs38433ruIjpyygvd/LZ+57C5cvwILyfH73\nsVNwegOsmB1bkjmllJoMDRAp9vuNh8LB4TcfXcddrxzia3/Zzq+e30dHv5eAMfz102dQW5lPUQw5\nkZRSKl40QKRQU7eLWzfUsWJWEY999gwcmRmctdxavSw02ex77zuO4+bFd9lOpZSKhQaIFDnU3s9X\n/rwNly/ATz50Ao5Mqz8hM0P4yqXH8MULliMCOZOZ9quUUpOgASLJjDG4fAG+89g7vFzXzqfOXsLq\nucNrCDNlzoJSKn1pgEgwlzfA83ta+ePmep7f0zIo5fT1ZyziPy7W9AtKqfSkASKB/IEga779JG6f\nlSOpKNfBZcdVU5Sbxco5RXzgpHkpLqFSSo1MA0QCPbu7FbcvyAnzSvjCBcs5dXGFNh0ppaYMDRAJ\n9Oe36qkuyuGBT56mnc1KqSlHp+ImyMt1bTy+rZnTl1ZqcFBKTUlag4ijbpePvKxMfvvaIb79qLWC\n2KfOXpLiUiml1MRogIjC6fXjDxreOtzFva8doqHTxZoFpZwwr5Tls4tYXFVAdmYGPW4f1UXWspoP\nvVnPFx94O/wZFQXZfOO9q1g2S9NiKKWmJg0QQ/xtWxP/9uDb9HsDg7bvau7hdxsPA5DtyMDrt0Ym\nzSnJZeXsIp7d3QrA+9fM5czlVSlZLEcppeJJA4Strc/Db146wC+f28fyWYWsX1SOxxfklMUVXLl2\nLh5/kB2N3bywp40up5ctR7pYs6CMhi4Xrx/o4MOnLOBz5y2jujg31YeilFJxoQECeGFPK1968G1a\nez0cM6eYO65ZN2iNBbBmNp+0sJyTFpanqJRKKZVcMzpAuH0B7n3tED98YjfzyvL49TXrOH5eaaqL\npZRSaSGtAoSIXAzcAmQCdxhjvp/I7/v8/Vt4YkczaxaUctdH11OSr+m0lVIqJG0ChIhkAr8ALgDq\ngTdE5BFjzM5EfN9f327kiR3NXHf6Ir72D8ckbFF7pZSaqtJpotx6oM4Ys98Y4wXuBy5PxBc9sOkI\nX3xgC8fPK+HfL1qhwUEppaJIpwAxFzgS8bze3jaIiNwgIptEZFNra+uEvmhxZQHnrZzFXdeuJy9b\nZzkrpVQ0adPEFCtjzO3A7QDr1q0zY+we1bractbV6mgkpZQaTTrVIBqA+RHP59nblFJKpUA6BYg3\ngGUiskhEsoGrgEdSXCallJqx0qaJyRjjF5FPA3/HGub6G2PMjhQXSymlZqy0CRAAxpjHgcdTXQ6l\nlFLp1cSklFIqjWiAUEopFZUGCKWUUlFpgFBKKRWVGDOhuWZpQURagUMTfHsl0BbH4kwFeswzgx7z\nzDCZY15ojKkaa6cpHSAmQ0Q2GWPWpbocyaTHPDPoMc8MyThmbWJSSikVlQYIpZRSUc3kAHF7qguQ\nAnrMM4Me88yQ8GOesX0QSimlRjeTaxBKKaVGMSMDhIhcLCK7RaRORG5KdXniRUTmi8izIrJTRHaI\nyOfs7eUi8pSI7LX/LbO3i4jcav8etorI2tQewcSISKaIvCUij9rPF4nIRvu4/mBnB0ZEcuzndfbr\ntaks90SJSKmI/FFEdonIOyJy2gw4x1+w/6a3i8h9IpI7Hc+ziPxGRFpEZHvEtnGfWxG5xt5/r4hc\nM9HyzLgAEbH29SXAscDVInJsaksVN37gS8aYY4FTgRvtY7sJ2GCMWQZssJ+D9TtYZv/cANyW/CLH\nxeeAdyKe/wD4mTFmKdAJXG9vvx7otLf/zN5vKroFeMIYsxI4AevYp+05FpG5wGeBdcaY1VjZnq9i\nep7nu4CLh2wb17kVkXLg68ApWEs5fz0UVMbNGDOjfoDTgL9HPL8ZuDnV5UrQsT4MXADsBubY2+YA\nu+3HvwKujtg/vN9U+cFaWGoDcC7wKCBYk4ccQ883Vir50+zHDns/SfUxjPN4S4ADQ8s9zc9xaDni\ncvu8PQpcNF3PM1ALbJ/ouQWuBn4VsX3QfuP5mXE1CGJc+3qqs6vVa4CNwCxjTJP9UjMwy348HX4X\nPwf+AwjazyuALmOM334eeUzh47Vf77b3n0oWAa3AnXaz2h0iUsA0PsfGmAbgx8BhoAnrvG1mep/n\nSOM9t3E75zMxQEx7IlII/An4vDGmJ/I1Y91STIuhayLyD0CLMWZzqsuSRA5gLXCbMWYN0M9AkwMw\nvc4xgN08cjlWcKwBChjeDDMjJPvczsQAMa3XvhaRLKzg8DtjzEP25qMiMsd+fQ7QYm+f6r+L04H3\nishB4H6sZqZbgFIRCS2GFXlM4eO1Xy8B2pNZ4DioB+qNMRvt53/EChjT9RwDnA8cMMa0GmN8wENY\n5346n+dI4z23cTvnMzFATNu1r0VEgF8D7xhjfhrx0iNAaCTDNVh9E6Ht/2KPhjgV6I6oyqY9Y8zN\nxph5xpharPP4jDHmw8CzwAfs3YYeb+j38AF7/yl1p22MaQaOiMgKe9N5wE6m6Tm2HQZOFZF8+288\ndMzT9jwPMd5z+3fgQhEps2tfF9rbxi/VHTIp6gS6FNgD7AP+M9XlieNxnYFV/dwKbLF/LsVqf90A\n7AWeBsrt/QVrRNc+YBvWKJGUH8cEj/1s4FH78WLgdaAOeBDIsbfn2s/r7NcXp7rcEzzWE4FN9nn+\nC1A23c8x8E1gF7Ad+C2QMx3PM3AfVj+LD6u2eP1Ezi1wnX38dcC1Ey2PzqRWSikV1UxsYlJKKRUD\nDRBKKaWi0gChlFIqKg0QSimlotIAoZRSKioNEEoppaLSAKGUUioqDRBKKaWi+v/FpRXV2HxjlAAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9713e87780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(mva_lst)\n",
    "plt.ylabel('Moving Average Score')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('curiosity-mva.npy', np.array(mva_lst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}