ML_Project_Final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ML_Project_Final.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "WUaZfmb9gId7",
        "outputId": "7df2922a-82f1-4d9c-dd5d-691125ffc6c6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "# Import necessary libraries\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from matplotlib import pyplot as plt\n",
        "import math\n",
        "import string\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import re\n",
        "\n",
        "import os\n",
        "import sys\n",
        "from IPython.display import display, HTML\n",
        "import urllib\n",
        "import gzip\n",
        "\n",
        "import nltk\n",
        "from nltk import tokenize\n",
        "nltk.download('punkt')\n",
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "nltk.download('wordnet')\n",
        "\n",
        "import pip\n",
        "import theano\n",
        "import keras\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.preprocessing.text import Tokenizer,  text_to_word_sequence\n",
        "from keras import initializers as initializers, regularizers, constraints\n",
        "from keras.layers import Dense, Dropout, Activation, Flatten, Input\n",
        "from keras.layers import Convolution2D, MaxPooling2D, Embedding, Dropout, LSTM, GRU, Bidirectional, TimeDistributed\n",
        "from keras.models import Sequential, Model, load_model\n",
        "from keras import backend as K\n",
        "from tensorflow.keras.layers import Layer, InputSpec\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from keras.utils import  np_utils\n",
        "from keras.utils.np_utils import to_categorical"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "YfW5F6D6yaoE",
        "outputId": "eaf410f1-e277-49af-ae71-8178a2e39b39",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at gdrive\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "Zwj57XIwgZcI"
      },
      "cell_type": "markdown",
      "source": [
        "# Helper Functions"
      ]
    },
    {
      "metadata": {
        "id": "tmMwAgdNgUrB"
      },
      "cell_type": "code",
      "source": [
        "def cleanString(review,stopWords):\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    returnString = \"\"\n",
        "    sentence_token = tokenize.sent_tokenize(review)\n",
        "    idx_list = []\n",
        "    for j in range(len(sentence_token)):\n",
        "        single_sentence = tokenize.word_tokenize(sentence_token[j])\n",
        "        sentences_filtered = [(idx,lemmatizer.lemmatize(w.lower())) for idx,w in enumerate(single_sentence) \n",
        "                              if w.lower() not in stopWords and w.isalnum()]\n",
        "        idx_list.append([x[0] for x in sentences_filtered])\n",
        "        word_list = [x[1] for x in sentences_filtered]\n",
        "        returnString = returnString + ' '.join(word_list) + ' . '\n",
        "    \n",
        "    return returnString, idx_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aXf3t1d0gUve"
      },
      "cell_type": "code",
      "source": [
        "def split_df(dataframe, column_name, training_split = 0.8, validation_split = 0.1, test_split = 0.1):\n",
        "\n",
        "    if training_split + validation_split + test_split != 1.0:\n",
        "        raise ValueError('Split paramter sum should be 1.0')\n",
        "        \n",
        "    total = len(dataframe.index)\n",
        " \n",
        "    train = dataframe.reset_index().groupby(column_name).apply(lambda x: x.sample(frac=training_split))\\\n",
        "    .reset_index(drop=True).set_index('index')\n",
        "    train = train.sample(frac=1)\n",
        "    temp_df = dataframe.drop(train.index)\n",
        "    validation = temp_df.reset_index().groupby(column_name)\\\n",
        "    .apply(lambda x: x.sample(frac=validation_split/(test_split+validation_split)))\\\n",
        "           .reset_index(drop=True).set_index('index')\n",
        "    validation = validation.sample(frac=1)\n",
        "    test = temp_df.drop(validation.index)\n",
        "    test = test.sample(frac=1)\n",
        "    \n",
        "    print('Total: ', len(dataframe))\n",
        "    print('Training: ', len(train), ', Percentage: ', len(train)/len(dataframe))\n",
        "    print('Validation: ', len(validation), ', Percentage: ', len(validation)/len(dataframe))\n",
        "    print('Test:', len(test), ', Percentage: ', len(test)/len(dataframe))\n",
        "\n",
        "    return train, validation, test"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CVWSB2t-gUy3"
      },
      "cell_type": "code",
      "source": [
        "def wordToSeq(text,word_index,max_words,max_features):\n",
        "    # sentences = tokenize.sent_tokenize(text)\n",
        "    data = np.zeros((max_words), dtype='int32')\n",
        "    # for j, sent in enumerate(sentences):\n",
        "    #     if j< max_sentences:\n",
        "    wordTokens = tokenize.word_tokenize(text.rstrip('.'))\n",
        "    wordTokens = [w for w in wordTokens]\n",
        "    k=0\n",
        "    for _, word in enumerate(wordTokens):\n",
        "        try:\n",
        "            if k<max_words and word_index[word]<max_features:\n",
        "                data[k] = word_index[word]\n",
        "                k=k+1\n",
        "        except:\n",
        "            pass\n",
        "    # print(data)\n",
        "    return data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZTvGwt6QgU4_"
      },
      "cell_type": "code",
      "source": [
        "def to_categorical(series,class_dict):\n",
        "    n_classes = len(class_dict)\n",
        "    new_dict = {}\n",
        "    for key,value in class_dict.items():\n",
        "        cat_list = [0] * n_classes\n",
        "        cat_list[key] = 1\n",
        "        new_dict[key] = cat_list\n",
        "    y_cat = []\n",
        "    for key,value in series.iteritems():\n",
        "        y_cat.append(new_dict[value])\n",
        "    return np.array(y_cat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "m1XGIMYKg-OS"
      },
      "cell_type": "markdown",
      "source": [
        "# Attention layer"
      ]
    },
    {
      "metadata": {
        "id": "I6-w_wuXgVAE"
      },
      "cell_type": "code",
      "source": [
        "class AttentionLayer(Layer):\n",
        "\n",
        "    def __init__(self,attention_dim=100,return_coefficients=False,**kwargs):\n",
        "        # Initializer \n",
        "        self.supports_masking = True\n",
        "        self.return_coefficients = return_coefficients\n",
        "        self.init = initializers.get('glorot_uniform') # initializes values with uniform distribution\n",
        "        self.attention_dim = attention_dim\n",
        "        super(AttentionLayer, self).__init__(**kwargs)\n",
        "\n",
        "    def get_config(self):\n",
        "        config = super().get_config()\n",
        "        config.update({\n",
        "            # \"supports_masking\": self.supports_masking,\n",
        "            \"return_coefficients\": self.return_coefficients,\n",
        "            # \"init\" : self.init,\n",
        "            \"attention_dim\" :self.attention_dim})\n",
        "        return config\n",
        "        \n",
        "    def build(self, input_shape):\n",
        "        # Builds all weights\n",
        "        # W = Weight matrix, b = bias vector, u = context vector\n",
        "        assert len(input_shape) == 3\n",
        "        self.W = K.variable(self.init((input_shape[-1], self.attention_dim)),name='W')\n",
        "        self.b = K.variable(self.init((self.attention_dim, )),name='b')\n",
        "        self.u = K.variable(self.init((self.attention_dim, 1)),name='u')\n",
        "        self._trainable_weights = [self.W, self.b, self.u]\n",
        "\n",
        "        super(AttentionLayer, self).build(input_shape)\n",
        "\n",
        "    def compute_mask(self, input, input_mask=None):\n",
        "        return None\n",
        "\n",
        "    def call(self, hit, mask=None):\n",
        "        # Here, the actual calculation is done\n",
        "        uit = K.bias_add(K.dot(hit, self.W),self.b)\n",
        "        uit = K.tanh(uit)\n",
        "        \n",
        "        ait = K.dot(uit, self.u)\n",
        "        ait = K.squeeze(ait, -1)\n",
        "        ait = K.exp(ait)\n",
        "        \n",
        "        if mask is not None:\n",
        "            ait *= K.cast(mask, K.floatx())\n",
        "\n",
        "        ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
        "        ait = K.expand_dims(ait)\n",
        "        weighted_input = hit * ait\n",
        "        \n",
        "        if self.return_coefficients:\n",
        "            return [K.sum(weighted_input, axis=1), ait]\n",
        "        else:\n",
        "            return K.sum(weighted_input, axis=1)\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        if self.return_coefficients:\n",
        "            return [(input_shape[0], input_shape[-1]), (input_shape[0], input_shape[-1], 1)]\n",
        "        else:\n",
        "            return input_shape[0], input_shape[-1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "riBDmUnBhILe"
      },
      "cell_type": "markdown",
      "source": [
        "# Load Data"
      ]
    },
    {
      "metadata": {
        "id": "pumktA96zJ4p"
      },
      "cell_type": "code",
      "source": [
        "# data_df = pd.read_excel(\"/content/drive/My Drive/Colab Notebooks/data_consolidated.xlsx\")\n",
        "data_df = pd.read_csv(\"/content/gdrive/MyDrive/College/Semester5/ML/Project/twitter15/twitter15.csv\")\n",
        "# data_df = pd.read_csv(\"/content/gdrive/MyDrive/College/Semester5/NLP/project/train.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_df.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JFCFNogZUDl0",
        "outputId": "48f91592-cf37-43b0-ef21-b82247a69378"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(741, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_df = data_df.dropna()"
      ],
      "metadata": {
        "id": "J_PkuMpHvSUZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_df = data_df[['source_tweet','label']].rename(columns = {'source_tweet':'Text', 'label':'Category'})\n",
        "# data_df = data_df[['text','label']].rename(columns = {'text':'Text', 'label':'Category'})"
      ],
      "metadata": {
        "id": "0kMLySA46gTY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aLg9ok_V5poY"
      },
      "cell_type": "code",
      "source": [
        "data_df.to_excel(\"/content/gdrive/MyDrive/College/Semester5/ML/Project/twitter15/twitter15.xlsx\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QJcoBcWJhXQZ"
      },
      "cell_type": "markdown",
      "source": [
        "# Parameters"
      ]
    },
    {
      "metadata": {
        "id": "DIMHviUDhQi1"
      },
      "cell_type": "code",
      "source": [
        "MAX_FEATURES = 3000 # maximum number of unique words that should be included in the tokenized word index\n",
        "# MAX_SENTENCE_NUM = 10 # maximum number of sentences in one document\n",
        "MAX_WORD_NUM = 40   # maximum number of words in each sentence\n",
        "EMBED_SIZE = 100      # vector size of word embedding"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pPTxoGI9hbWf"
      },
      "cell_type": "markdown",
      "source": [
        "# Data Preprocessing"
      ]
    },
    {
      "metadata": {
        "id": "f597eW5JqrV2"
      },
      "cell_type": "code",
      "source": [
        "data_cleaned = pd.read_excel(\"/content/gdrive/MyDrive/College/Semester5/ML/Project/twitter15/twitter15.xlsx\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_cleaned"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "clyGTlPAua4m",
        "outputId": "610f498b-8f17-4797-86d9-3da89f07b191"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text</th>\n",
              "      <th>Category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>rip elly may clampett: so sad to learn #beverl...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>just in: missing afghan soldiers found trying ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>#riphulkhogan my heart is ripping like your sh...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>a chick-fil-a manager allegedly banned this hi...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>islamic tribunal using sharia law in texas has...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>736</th>\n",
              "      <td>r.i.p roger rodas the man who died with paul w...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>737</th>\n",
              "      <td>officials took away this halloween decoration ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>738</th>\n",
              "      <td>r.i.p to the driver that died with paul walker...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>739</th>\n",
              "      <td>pentagon confirms ahmed abdi godane, a leader ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>740</th>\n",
              "      <td>marlboro ad man eric lawson dies of chronic lu...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>741 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                  Text  Category\n",
              "0    rip elly may clampett: so sad to learn #beverl...      True\n",
              "1    just in: missing afghan soldiers found trying ...      True\n",
              "2    #riphulkhogan my heart is ripping like your sh...     False\n",
              "3    a chick-fil-a manager allegedly banned this hi...     False\n",
              "4    islamic tribunal using sharia law in texas has...     False\n",
              "..                                                 ...       ...\n",
              "736  r.i.p roger rodas the man who died with paul w...      True\n",
              "737  officials took away this halloween decoration ...      True\n",
              "738  r.i.p to the driver that died with paul walker...      True\n",
              "739  pentagon confirms ahmed abdi godane, a leader ...      True\n",
              "740  marlboro ad man eric lawson dies of chronic lu...      True\n",
              "\n",
              "[741 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_cleaned['Text']= data_cleaned['Text'].astype(str)"
      ],
      "metadata": {
        "id": "Fp1SDXgCuhzR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6SRU8H5KhQoE",
        "outputId": "60c09d28-9b2f-4e89-ab41-816b54a8e645",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "Cleans raw data using the cleanString() function from above.\n",
        "English stopwords are used from nltk library.\n",
        "Cleaned dataset is saved in 'data_cleaned' pandas dataframe.\n",
        "Labels are converted to numbers,\n",
        "\"\"\"\n",
        "articles = []\n",
        "n = data_df['Text'].shape[0]\n",
        "col_number = data_df.columns.get_loc('Text')\n",
        "stopWords = set(stopwords.words('english'))\n",
        "data_cleaned = data_df.copy()\n",
        "for i in range(n):\n",
        "    # print(data_df.iloc[i,col_number])\n",
        "    temp_string,idx_string = cleanString(data_df.iloc[i,col_number],stopWords)\n",
        "    articles.append(temp_string)\n",
        "    print(str(i+1)+' of '+str(n)+\" articles cleaned.\",end='\\r')\n",
        "    \n",
        "data_cleaned.loc[:,'Text'] = pd.Series(articles,index=data_df.index)\n",
        "data_cleaned.loc[:,'Category'] = pd.Categorical(data_cleaned.Category)\n",
        "data_cleaned['Code'] = data_cleaned.Category.cat.codes\n",
        "categoryToCode = dict( enumerate(data_cleaned['Category'].cat.categories))\n",
        "\n",
        "data_cleaned.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ""
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text</th>\n",
              "      <th>Category</th>\n",
              "      <th>Code</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>rip elly may clampett sad learn beverlyhillbil...</td>\n",
              "      <td>True</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>missing afghan soldier found trying enter cana...</td>\n",
              "      <td>True</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>riphulkhogan heart ripping like shirt . wwe mi...</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>manager allegedly banned hilarious list slang ...</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>islamic tribunal using sharia law texas confir...</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Text Category  Code\n",
              "0  rip elly may clampett sad learn beverlyhillbil...     True     1\n",
              "1  missing afghan soldier found trying enter cana...     True     1\n",
              "2  riphulkhogan heart ripping like shirt . wwe mi...    False     0\n",
              "3  manager allegedly banned hilarious list slang ...    False     0\n",
              "4  islamic tribunal using sharia law texas confir...    False     0"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "metadata": {
        "id": "qiT2qEddql5A"
      },
      "cell_type": "code",
      "source": [
        "data_cleaned.to_excel(\"/content/gdrive/MyDrive/College/Semester5/ML/Project/twitter15/data_cleaned.xlsx\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bBJwo5Jphkg9"
      },
      "cell_type": "markdown",
      "source": [
        "# Tokenisation"
      ]
    },
    {
      "metadata": {
        "id": "EtgtPrTlhQq8"
      },
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "Using the keras Tokenizer class a word index is built.\n",
        "The most 'MAX_FEATURES' used words are tokenized to a number.\n",
        "this dictionary is saved in word_index\n",
        "\"\"\"\n",
        "texts = []\n",
        "n = data_cleaned['Text'].shape[0]\n",
        "for i in range(n):\n",
        "    s = data_cleaned['Text'].iloc[i]\n",
        "    s = ' '.join([word.strip(string.punctuation) for word in s.split() if word.strip(string.punctuation) is not \"\"])\n",
        "    texts.append(s)\n",
        "tokenizer = Tokenizer(num_words=MAX_FEATURES,lower=True, oov_token=None)\n",
        "tokenizer.fit_on_texts(texts)\n",
        "word_index = tokenizer.word_index"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0qIBSuiehtn6"
      },
      "cell_type": "markdown",
      "source": [
        "# GloVe Embedding Matrix"
      ]
    },
    {
      "metadata": {
        "id": "vNj5rWSBhQtk",
        "outputId": "31c49f29-dd7b-4dbf-add9-96dfc925f39b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "A pre-trained word to vector is used from GloVe by Pennington et. al.\n",
        "Source: https://nlp.stanford.edu/projects/glove/\n",
        "The data was trained on wikipedia articles. Each word is described by a 100d vector.\n",
        "\"\"\"\n",
        "\n",
        "# Load word vectors from pre-trained dataset\n",
        "embeddings_index = {}\n",
        "f = open('/content/gdrive/MyDrive/Amazon/productner/data/glove.6B.100d.txt',encoding='UTF-8')\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()\n",
        "print('Found %s word vectors.' % len(embeddings_index))\n",
        "\n",
        "# Search words in our word index in the pre-trained dataset\n",
        "# Create an embedding matrix for our bbc dataset\n",
        "min_wordCount = 0\n",
        "absent_words = 0\n",
        "small_words = 0\n",
        "embedding_matrix = np.zeros((len(word_index) + 1, EMBED_SIZE))\n",
        "word_counts = tokenizer.word_counts\n",
        "for word, i in word_index.items():\n",
        "    if word_counts[word] > min_wordCount:\n",
        "        embedding_vector = embeddings_index.get(word)\n",
        "        if embedding_vector is not None:\n",
        "            # words not found in embedding index will be all-zeros.\n",
        "            embedding_matrix[i] = embedding_vector\n",
        "        else:\n",
        "            absent_words += 1\n",
        "    else:\n",
        "        small_words += 1\n",
        "print('Total absent words are', absent_words, 'which is', \"%0.2f\" % (absent_words * 100 / len(word_index)),\n",
        "      '% of total words')\n",
        "print('Words with '+str(min_wordCount)+' or less mentions', small_words, 'which is', \"%0.2f\" % (small_words * 100 / len(word_index)),\n",
        "      '% of total words')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found 400000 word vectors.\n",
            "Total absent words are 181 which is 8.13 % of total words\n",
            "Words with 0 or less mentions 0 which is 0.00 % of total words\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "8Wkd9yQJh--r"
      },
      "cell_type": "markdown",
      "source": [
        "# Splitting Dataset"
      ]
    },
    {
      "metadata": {
        "id": "EBbSbMllh-NR",
        "outputId": "6434e2dd-1fcd-41dd-b9bb-cd520981744b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "Split Pandas Dataframe into train, validation and testset.\n",
        "Convert data to keras conforming form\n",
        "\"\"\"\n",
        "\n",
        "print(categoryToCode)\n",
        "train, validation, test = split_df(data_cleaned, 'Code',0.8,0.1,0.1)\n",
        "\n",
        "#Training\n",
        "paras = []\n",
        "for i in range(train['Text'].shape[0]):\n",
        "    sequence = wordToSeq(train['Text'].iloc[i],word_index,MAX_WORD_NUM,MAX_FEATURES)\n",
        "    paras.append(sequence)\n",
        "x_train = np.array(paras)\n",
        "y_train = to_categorical(train['Code'],categoryToCode)\n",
        "\n",
        "#Validation\n",
        "paras = []\n",
        "for i in range(validation['Text'].shape[0]):\n",
        "    sequence = wordToSeq(validation['Text'].iloc[i],word_index,MAX_WORD_NUM,MAX_FEATURES)\n",
        "    paras.append(sequence)\n",
        "x_val = np.array(paras)\n",
        "y_val = to_categorical(validation['Code'],categoryToCode)\n",
        "\n",
        "#Test\n",
        "paras = []\n",
        "for i in range(test['Text'].shape[0]):\n",
        "    sequence = wordToSeq(test['Text'].iloc[i],word_index,MAX_WORD_NUM,MAX_FEATURES)\n",
        "    paras.append(sequence)\n",
        "x_test = np.array(paras)\n",
        "y_test = to_categorical(test['Code'],categoryToCode)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{0: False, 1: True}\n",
            "Total:  741\n",
            "Training:  593 , Percentage:  0.8002699055330634\n",
            "Validation:  74 , Percentage:  0.09986504723346828\n",
            "Test: 74 , Percentage:  0.09986504723346828\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "MgVfXqcniKB1"
      },
      "cell_type": "markdown",
      "source": [
        "# Our Model"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras import Model, Input\n",
        "from tensorflow.keras.layers import Embedding, Dense, Bidirectional\n",
        "from keras.layers import GRU\n",
        "import tensorflow.keras.layers\n",
        "from tensorflow.keras import layers"
      ],
      "metadata": {
        "id": "ouAbaLl5AhX4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K4X51a4YT35m",
        "outputId": "6cce8585-9690-4c82-990b-b92f522f10a4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(593, 40)"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "Create Keras functional model for hierarchical attention network\n",
        "\"\"\"\n",
        "embedding_layer = Embedding(len(word_index) + 1,EMBED_SIZE,weights=[embedding_matrix], \n",
        "                            input_length=MAX_WORD_NUM, trainable=False,name='word_embedding')\n",
        "\n",
        "# Words level attention model\n",
        "word_input = Input(shape=(MAX_WORD_NUM,), dtype='int32',name='word_input')\n",
        "word_sequences = embedding_layer(word_input)\n",
        "# cnn_output = Bidirectional(GRU(128, return_sequences=True),name='word_gru', merge_mode='sum')(word_sequences)\n",
        "cnn_output = GRU(128, return_sequences=True)(word_sequences)\n",
        "cnn_output = layers.Convolution1D(30, 3, activation=\"relu\", padding = 'same', name = 'conv_rep')(cnn_output)\n",
        "# cnn_output = layers.Convolution1D(30, 4, activation=\"relu\", padding = 'same', name = 'conv_rep2')(cnn_output)\n",
        "# cnn_output = layers.Convolution1D(30, 4, activation=\"relu\", padding = 'same', name = 'conv_rep3')(cnn_output)\n",
        "word_dense = Dense(100, activation='relu', name='word_dense')(cnn_output) \n",
        "word_att,word_coeffs = AttentionLayer(EMBED_SIZE,True,name='word_attention')(word_dense)\n",
        "wordEncoder = Model(inputs = word_input,outputs = word_att)\n",
        "preds = Dense(2, activation='sigmoid',name='output')(word_att)\n",
        "model = Model(inputs = word_input,outputs = preds)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['acc'])\n",
        "# print(wordEncoder.summary())\n",
        "print(model.summary())\n",
        "\n",
        "plot_model(model, to_file='model.png',show_shapes=True)\n",
        "plot_model(model, to_file='wordEncoder.png',show_shapes=True)\n",
        "    \n",
        "history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=5, batch_size=40)\n",
        "\n",
        "print(history.history.keys())\n",
        "# Plot of accuracy in each epoch\n",
        "plt.plot(history.history['acc'])\n",
        "plt.plot(history.history['val_acc'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'validation'], loc='upper left')\n",
        "plt.show()\n",
        "# Plot of loss in each epoch\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'validation'], loc='upper left')\n",
        "plt.show()\n",
        "\n",
        "result = model.predict(x_val)\n",
        "result = np.array([i[0] < 0.5 for i in result]).astype(int)\n",
        "y_val2 = np.array([i[0] < 0.5 for i in y_val]).astype(int)\n",
        "from sklearn import metrics\n",
        "print('Test...')\n",
        "print(\"ACC\", metrics.accuracy_score(result, y_val2))\n",
        "print(\"Prec\", metrics.precision_score(result, y_val2))\n",
        "print(\"REC\", metrics.recall_score(result, y_val2))\n",
        "print(\"F1\", metrics.f1_score(result, y_val2))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "kEUBcYccenuq",
        "outputId": "34126b1c-0032-4022-c49d-ce9f52ca89d1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_274\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " word_input (InputLayer)     [(None, 40)]              0         \n",
            "                                                                 \n",
            " word_embedding (Embedding)  (None, 40, 100)           222600    \n",
            "                                                                 \n",
            " gru_175 (GRU)               (None, 40, 128)           88320     \n",
            "                                                                 \n",
            " conv_rep (Conv1D)           (None, 40, 30)            11550     \n",
            "                                                                 \n",
            " word_dense (Dense)          (None, 40, 100)           3100      \n",
            "                                                                 \n",
            " word_attention (AttentionLa  [(None, 100),            10200     \n",
            " yer)                         (None, 40, 1)]                     \n",
            "                                                                 \n",
            " output (Dense)              (None, 2)                 202       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 335,972\n",
            "Trainable params: 113,372\n",
            "Non-trainable params: 222,600\n",
            "_________________________________________________________________\n",
            "None\n",
            "Epoch 1/5\n",
            "15/15 [==============================] - 4s 89ms/step - loss: 0.6825 - acc: 0.5093 - val_loss: 0.6187 - val_acc: 0.6757\n",
            "Epoch 2/5\n",
            "15/15 [==============================] - 1s 52ms/step - loss: 0.5601 - acc: 0.7083 - val_loss: 0.4263 - val_acc: 0.8514\n",
            "Epoch 3/5\n",
            "15/15 [==============================] - 1s 53ms/step - loss: 0.4530 - acc: 0.7909 - val_loss: 0.3963 - val_acc: 0.8514\n",
            "Epoch 4/5\n",
            "15/15 [==============================] - 1s 53ms/step - loss: 0.3553 - acc: 0.8567 - val_loss: 0.3916 - val_acc: 0.8514\n",
            "Epoch 5/5\n",
            "15/15 [==============================] - 1s 51ms/step - loss: 0.2581 - acc: 0.8971 - val_loss: 0.3821 - val_acc: 0.8378\n",
            "dict_keys(['loss', 'acc', 'val_loss', 'val_acc'])\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwU9fnA8c9DCISbkIQrARIhnIqAEZBD8UZUvAXxwqO0eKC2trWtVWvr1Z+11taqqFitAiJe1IIIFZRTCYLITTiTcCQkIZCQQI7n98dMcAkLLJDNbLLP+/XaV3bn2HkysPtk5pn5PqKqGGOMMZXV8ToAY4wxockShDHGGL8sQRhjjPHLEoQxxhi/LEEYY4zxyxKEMcYYvyxBGAOIyL9E5E8BLrtFRC4KdkzGeM0ShDHGGL8sQRhTi4hIXa9jMLWHJQhTY7indn4pIitEpFBE3hSRViIyQ0T2ichsEYn2WX64iKwSkT0iMldEuvnM6y0i37nrvQ9EVdrWFSKy3F13oYj0DDDGy0VkmYjsFZF0EXmi0vxB7vvtceePdqc3EJG/iMhWEckXkfnutCEikuFnP1zkPn9CRKaKyLsishcYLSJ9RWSRu40dIvIPEanns34PEZklIrkisktEfisirUVkv4jE+CzXR0SyRSQykN/d1D6WIExNcx1wMdAZuBKYAfwWiMP5/zwOQEQ6A5OAB91504H/iEg998vyE+DfQAvgA/d9cdftDUwAfgrEAK8B00SkfgDxFQK3Ac2By4GxInK1+74d3Hj/7sbUC1jurvc8cBYwwI3pV0B5gPvkKmCqu833gDLgISAWOAe4ELjHjaEJMBv4HGgLdAL+p6o7gbnAjT7veyswWVVLAozD1DKWIExN83dV3aWqmcA84BtVXaaqxcDHQG93uRHAf1V1lvsF9zzQAOcLuD8QCbyoqiWqOhVY4rONMcBrqvqNqpap6tvAAXe9Y1LVuar6g6qWq+oKnCR1njt7FDBbVSe5281R1eUiUge4E3hAVTPdbS5U1QMB7pNFqvqJu80iVV2qqotVtVRVt+AkuIoYrgB2qupfVLVYVfep6jfuvLeBWwBEJAK4CSeJmjBlCcLUNLt8nhf5ed3Yfd4W2FoxQ1XLgXQg3p2XqYePVLnV53kH4BfuKZo9IrIHaOeud0wi0k9E5rinZvKBn+H8JY/7Hhv9rBaLc4rL37xApFeKobOIfCYiO93TTk8HEAPAp0B3EUnCOUrLV9VvTzImUwtYgjC11XacL3oARERwvhwzgR1AvDutQnuf5+nAU6ra3OfRUFUnBbDdicA0oJ2qNgNeBSq2kw509LPObqD4KPMKgYY+v0cEzukpX5WHZH4FWAskq2pTnFNwvjGc5i9w9yhsCs5RxK3Y0UPYswRhaqspwOUicqFbZP0FzmmihcAioBQYJyKRInIt0Ndn3deBn7lHAyIijdzic5MAttsEyFXVYhHpi3NaqcJ7wEUicqOI1BWRGBHp5R7dTABeEJG2IhIhIue4NY/1QJS7/UjgUeB4tZAmwF6gQES6AmN95n0GtBGRB0Wkvog0EZF+PvPfAUYDw7EEEfYsQZhaSVXX4fwl/Hecv9CvBK5U1YOqehC4FueLMBenXvGRz7qpwE+AfwB5QJq7bCDuAZ4UkX3AYziJquJ9twHDcJJVLk6B+kx39sPADzi1kFzgOaCOqua77/kGztFPIXDYVU1+PIyTmPbhJLv3fWLYh3P66EpgJ7ABON9n/gKc4vh3qup72s2EIbGGQcYYXyLyJTBRVd/wOhbjLUsQxphDRORsYBZODWWf1/EYb9kpJmMMACLyNs49Eg9acjBgRxDGGGOOwo4gjDHG+FVrBvaKjY3VxMREr8MwxpgaZenSpbtVtfK9NUAtShCJiYmkpqZ6HYYxxtQoInLUy5ntFJMxxhi/LEEYY4zxyxKEMcYYv2pNDcKfkpISMjIyKC4u9jqUWiMqKoqEhAQiI62HjDG1XVAThIgMBf4GRABvqOqzleZ3wBmkLA5n/JlbVDXDnXc7zsBkAH9yx+Q/IRkZGTRp0oTExEQOH7jTnAxVJScnh4yMDJKSkrwOxxgTZEE7xeQOS/wycBnQHbhJRLpXWux54B1V7Qk8CTzjrtsCeBzohzPK5uO+rSQDVVxcTExMjCWHKiIixMTE2BGZMWEimDWIvkCaqm5yR8+cjNMa0Vd34Ev3+Ryf+ZcCs1Q1V1XzcMaGGXoyQVhyqFq2P40JH8FMEPEc3ukqw53m63ucYZcBrgGauE3TA1kXERkjIqkikpqdnV1lgRtjTE2Qkbefyd9uY+I324Ly/l5fxfQwcJ6ILMPpmZuJ03A9IKo6XlVTVDUlLs7vjYCe27NnD//85z9PeL1hw4axZ8+eIERkjKmp9hWX8MWqnTz26UrOf34ug56bwyMf/cAHS9OPv/JJCGaROhOnxWOFBHfaIaq6HfcIQkQaA9ep6h4RyQSGVFp3bhBjDZqKBHHPPfccNr20tJS6dY+++6dPnx7s0IwxIa60rJzvM/KZtyGb+Rt2syx9D2XlSsN6EfQ/LYZb+3dgcHIsnVo2Pv6bnYRgJoglQLLbAD0TGMnh7RcRkVic9ozlwG9wrmgCmAk87VOYvsSdX+M88sgjbNy4kV69ehEZGUlUVBTR0dGsXbuW9evXc/XVV5Oenk5xcTEPPPAAY8aMAX4cOqSgoIDLLruMQYMGsXDhQuLj4/n0009p0KCBx7+ZMSYYtuYU8vWG3czfkM3CjTnsKy5FBHrGN2PseR0ZlBxLn/bR1Ksb/BNAQUsQqloqIvfhfNlHABNUdZWIPAmkquo0nKOEZ0REga+Be911c0XkjzhJBuBJVc09lXj+8J9VrN6+91Te4gjd2zbl8St7HHOZZ599lpUrV7J8+XLmzp3L5ZdfzsqVKw9dJjphwgRatGhBUVERZ599Ntdddx0xMTGHvceGDRuYNGkSr7/+OjfeeCMffvght9xyS5X+LsYYb+TvL2Hhxt1OUkjLJj23CID45g24omcbBnWKY0DHGKIb1av22IJ6H4SqTgemV5r2mM/zqcDUo6w7gR+PKGqNvn37HnYPwUsvvcTHH38MQHp6Ohs2bDgiQSQlJdGrVy8AzjrrLLZs2VJt8RpjqtbB0nKWbctjfpqTFH7I2EO5QpP6denfMYYxg09jUHIciTENPb9qsFbfSe3reH/pV5dGjRodej537lxmz57NokWLaNiwIUOGDPF7j0H9+vUPPY+IiKCoqKhaYjXGnDpVZWN24aE6wuJNORQeLCOijtCrXXPuvyCZczvHcmZCc+pGeH3d0OHCJkF4pUmTJuzb5797Y35+PtHR0TRs2JC1a9eyePHiao7OGBMMOQUHWLAxh/kbspm3YTc78p0//BJjGnJtnwQGJcdyTscYmkaF9pA1liCCLCYmhoEDB3L66afToEEDWrVqdWje0KFDefXVV+nWrRtdunShf//+HkZqjDlZB0rLWLol71AdYWWmU+9s1iCSgZ1iuL9THIOTY2nXoqHHkZ6YWtOTOiUlRSs3DFqzZg3dunXzKKLay/arCXeqyrpd+5i/wakjfLs5h+KScurWEfp0iObc5FgGJcdxRnwzIuqE9ugDIrJUVVP8zbMjCGOMCUDWvmIWpO1m3vrdzE/bTda+AwB0atmYkWe359zOsfRLiqFR/drztVp7fhNjjKlCRQfL+HZL7qE6wtqdTi2xRaN6DOoUy6DkWAYnx9KmWe29J8kShDHGAOXlyuode5nn1hGWbMnjYGk59erW4ezEaH49tCuDk2Pp3qYpdUL8tFFVsQRhjAlbO/KLnISwYTcL0naTU3gQgK6tm3D7OR0YlBxH38QWNKgX4XGk3rAEYYwJG4UHSlm8Kcc9SthNWlYBAHFN6nNe5zgGd45lYKdYWjaJ8jjS0GAJwhhTa5WVKz9k5jN/QzZfb9jNsm15lJQpUZF16JcUw8iz2zEoOZYurZp4ftdyKLIEEWIaN25MQUEB27dvZ9y4cUydeuRIJEOGDOH5558nJcXvlWkAvPjii4wZM4aGDZ3rrocNG8bEiRNp3rx50GI3JhSk5+4/VEdYkJZDflEJAKfHN+WuQadxbnIsfTpEExUZnqeNToQliBDVtm1bv8khUC+++CK33HLLoQRhw4eb2mpvcQmLNuYwf8Nu5m3IZkvOfgDaNIvi0h6tGJQcx8COMcQ0rn+cdzKVWYIIskceeYR27dpx7733AvDEE09Qt25d5syZQ15eHiUlJfzpT3/iqqsO78a6ZcsWrrjiClauXElRURF33HEH33//PV27dj1sLKaxY8eyZMkSioqKuP766/nDH/7ASy+9xPbt2zn//POJjY1lzpw5h4YPj42N5YUXXmDCBGccxLvvvpsHH3yQLVu22LDipkYoLStnefqeQ3WE5W6PhEZuj4TRAxIZlBxHx7hGdtroFIVPgpjxCOz8oWrfs/UZcNmzx1xkxIgRPPjgg4cSxJQpU5g5cybjxo2jadOm7N69m/79+zN8+PCj/md+5ZVXaNiwIWvWrGHFihX06dPn0LynnnqKFi1aUFZWxoUXXsiKFSsYN24cL7zwAnPmzCE2Nvaw91q6dClvvfUW33zzDapKv379OO+884iOjrZhxU1IUlW25Ow/VEdYvDGHfQdKqSPQM6E59wzpyKBOsfSuph4J4SR8EoRHevfuTVZWFtu3byc7O5vo6Ghat27NQw89xNdff02dOnXIzMxk165dtG7d2u97fP3114wbNw6Anj170rNnz0PzpkyZwvjx4yktLWXHjh2sXr36sPmVzZ8/n2uuuebQqLLXXnst8+bNY/jw4TasuAkZe/YfZEFaDvPTnJvUMvKco+Z2LRpwZa+2DO4Uy4COsTRrGNqD3dV04ZMgjvOXfjDdcMMNTJ06lZ07dzJixAjee+89srOzWbp0KZGRkSQmJvod5vt4Nm/ezPPPP8+SJUuIjo5m9OjRJ/U+FWxYceOlsnJl1uqdvLVgC99uyUUVmkTVZUDHGH56XkfOTY6lQ0yj47+RqTJBPR4TkaEisk5E0kTkET/z24vIHBFZJiIrRGSYOz1RRIpEZLn7eDWYcQbbiBEjmDx5MlOnTuWGG24gPz+fli1bEhkZyZw5c9i6desx1z/33HOZOHEiACtXrmTFihUA7N27l0aNGtGsWTN27drFjBkzDq1ztGHGBw8ezCeffML+/fspLCzk448/ZvDgwVX42xpzYvYVl/Dm/M0MeX4OP3v3OzL3FPHAhcl8OHYAy35/Ma/dmsKt/TtYcvBA0I4gRCQCeBm4GMgAlojINFVd7bPYo8AUVX1FRLrjdJ9LdOdtVNVewYqvOvXo0YN9+/YRHx9PmzZtuPnmm7nyyis544wzSElJoWvXrsdcf+zYsdxxxx1069aNbt26cdZZZwFw5pln0rt3b7p27Uq7du0YOHDgoXXGjBnD0KFDadu2LXPmzDk0vU+fPowePZq+ffsCTpG6d+/eJ3Y6qbwcdq0+/nLGHMPOvcV8vCyTmat2sv9gGWe3bcozV8ZzTsdYIurXg0b1IcQa6ISboA33LSLnAE+o6qXu698AqOozPsu8BmxS1efc5f+iqgNEJBH4TFVPD3R7Ntx3NTlYyJrvFtLt8+u9jsSEg3qNoVEcNG7p/mzl87wlNGrp/GzcEurZEcbJ8Gq473gg3ed1BtCv0jJPAF+IyP1AI+Ain3lJIrIM2As8qqrzKm9ARMYAYwDat29fdZEb/0oPQu4mkDpw/VvOT2MCUFKuLNuWx5drs9ias59G9SMY1CmW8zq3JPqIQrPCgQIozIKCbPdnFuzeAFsXQFGe/41ENoLGcYcnjUYtD59WkWTqNw7671wbeF2kvgn4l6r+xT2C+LeInA7sANqrao6InAV8IiI9VHWv78qqOh4YD84RRHUHH1bKy5zkoOp8yE6/0OuITA2QV3iQid9u451FW9i1tx6nxZ3GncOTuK5PwskPgFd6EPbvdpJGYTYU7PJ5nuUklJyNsG0R7M/x/x6RDf0fhfg9MmkMYXo/RTATRCbQzud1gjvN113AUABVXSQiUUCsqmYBB9zpS0VkI9AZSOUEqardLHOqVCFvC5QWodGnwZ7tXkdkQlxaVgETFmzmo+8yKC4pZ3ByLM9e15PzkuNOfajsuvWgaVvncTxlJVC4+8ijkYpEUpAFeZsh/Rs3mfj5O7Nug6McmfhJKPWb1KpkEswEsQRIFpEknMQwEhhVaZltwIXAv0SkGxAFZItIHJCrqmUichqQDGw60QCioqLIyckhJibGksSp2JsJB/aiTePJKSwhKspGujRHUlXmbdjNhAWbmbsum3p163BNr3juHJREl9ZNvAkqIhKatnEex1NW6nNkcpSEkrcVMpY4ScdvMomqdFrLPaVV+VRX45ZQv2nIJ5OgJQhVLRWR+4CZQAQwQVVXiciTQKqqTgN+AbwuIg/h7O3Rqqoici7wpIiUAOXAz1Q190RjSEhIICMjg+zs7Cr7vcLOgX3OOd/6TSA/h6ioKBISEryOyoSQ4pIyPlmWyYQFm1m/q4DYxvX5+cWdGdWvPbE1afyjiLrQpLXzOJ7yMueIw9/prQL3tFd+OmQudZKOlvvZXv1KRyHHKMJHNfMkmQTtKqbq5u8qJnOKNsyGiTdC8sUwciLUsdEvzY+y9hXz7qKtvPvNNnILD9KtTVPuGpTElWe2oX5d+79ySHkZ7M91k8euw49MKhJLRXIp3A1aduR7RNQ79pFJ8w6QcPTRnY/Fq6uYTE22azV8MBpadofr3rTkYA5ZtT2fN+dv5j/fb6e0XLmwayvuGpRE/9Na2Klcf+pEuF/ocdCqx7GXLS+Hotwjj0Z8T3nt2wE7VzjLVCST+BT4yf+qPHRLEOZIBVkwcYRzXfmoyXZJoKGsXPlybRZvzt/E4k25NKwXwai+7Rk9MImkWLv/oMrUqQONYp0H3Y+9bHm5c/q3MMs5SgkCSxDmcCVFMOkm59D3zhnQzOoN4azwQCkfpKbz1sItbM3ZT3zzBvx2WFdGnN2eZg1soDxP1akDjWKcR5BYgjA/Ki+HT8Y6hbUR/4a2vb2OyHgkI28/7yzayqRvt7GvuJTe7Zvzy0u7MLRHa+ra8BdhwxKE+dHcp2HVx3DRH6DblV5HYzywdGseE+Zv5vNVOwG47PTW3DkoiT7toz2OzHjBEoRxLJ8EX/8f9L4VBj7gdTSmGpWUlTNj5U4mzN/M8vQ9NImqy92DkrhtQCLxza2jYDizBGFgywKYdj8kDobLXwj5m3dM1cjfX8KkJdt4e+EWduQXkxjTkCev6sF1fRJoVN++GowlCJOzEd6/GaITnbpD3XpeR2SCbFN2AW8t2MLUpRkUlZRxzmkx/PGq07mga8tTHwbD1CqWIMJZUZ5zIxwCo96HBnaeubZSVRZtzOHN+Zv5cl0WkXXqMLxXW+4YmEiPts28Ds+EKEsQ4ar0ILx/qzO2zO3TIKaj1xGZIDhQWsany7czYf5m1u7cR0yjetx/QTK39G9PyyY2ppY5NksQ4UgV/vsQbJkH17wGHQZ4HZGpYrsLDvDu4q28u3gruwsO0qVVE/58XU+G92pLVKTdFW8CYwkiHC34Gyx7F879JZw50utoTBVau3MvE+Zv5pPl2zlYWs75XeK4a9BpDOxkIxqbE2cJItysngazH4ce18KQ33odjakC5eXK3PVZvDl/MwvScoiKrMONKQncMTCJjnE2TIo5eZYgwknmd/DRGEg4G67+p3Orvqmx9h8s5cOlGby1YAubdhfSumkUvxrahVF929O8oV2NZk6dJYhwkZ8Bk0Y648yPnAiRdgNUTbUjv4i3FzrDYOQXlXBmQjP+NrIXw85oQ6QNg2GqkCWIcHBgH0wc6QzEd9unTgMSU+MsT9/Dm/M3M/2HHagql/ZozV2DkjirQ7TVF0xQWIKo7crLYOpdkLUabp4CLbt5HZE5AaVl5Xyxehdvzt/M0q15NKlflzsGJHL7gETatWjodXimlgtqghCRocDfcFqOvqGqz1aa3x54G2juLvOIqk535/0GuAsoA8ap6sxgxlprzfwdbJgJl/8FOl3kdTQmQHuLS3j/23T+tXALmXuKaNeiAY9d0Z0bUhJoEmXDbJvqEbQEISIRwMvAxUAGsEREpqnqap/FHgWmqOorItIdmA4kus9HAj2AtsBsEems6q8Xnzmqb1+Hb16BfmPh7Lu9jsYEYGtOIW8t2MIHqekUHiyjb1ILHruyOxd1a0WEDYNhqlkwjyD6AmmquglARCYDVwG+CUKBpu7zZsB29/lVwGRVPQBsFpE09/0WBTHe2iVtNsz4NSRfCpc+5XU05hhUlW825zJh/mZmrdlF3TrClT3bcuegJE6Pt2EwjHeCmSDigXSf1xlAv0rLPAF8ISL3A42AinMg8cDiSuvGV96AiIwBxgC0b9++SoKuFXathg/ucOoN11s/6VB1sLScz1Zs5835m1m1fS/RDSO5d0gnbj2nA62a2jAYxnteF6lvAv6lqn8RkXOAf4vI6YGurKrjgfEAKSkpGqQYa5aKftKRDZwB+Oo38ToiU0lu4UHeW7yVdxZvJXvfATq1bMzT15zBNb3jaVDPkrkJHcFMEJlAO5/XCe40X3cBQwFUdZGIRAGxAa5rKispgsmjnH7Sd0y3ftIhZsOufUxYsJmPvsvkQGk553aO4/kbkjg3OdYuUzUhKZgJYgmQLCJJOF/uI4FRlZbZBlwI/EtEugFRQDYwDZgoIi/gFKmTgW+DGGvNV14On9wDGUvgxn9DfB+vIzKuBWm7efWrjczbsJv6detwbZ947hyYRHIrO7ozoS1oCUJVS0XkPmAmziWsE1R1lYg8CaSq6jTgF8DrIvIQTsF6tKoqsEpEpuAUtEuBe+0KpuOY+zSs+sjpJ919uNfRGGDNjr08PX0N8zbspmWT+jx8SWdG9etAi0Y2DIapGcT5Pq75UlJSNDU11eswvLF8EnzyM+h9Cwz/h7UM9djO/GL+8sU6pn6XQdOoSMZd6PRfqF/X6gsm9IjIUlVN8TfP6yK1OVVbF/r0k/6rJQcPFRwoZfxXGxk/bxPl5XD3oCTuOz+ZZg3txjZTM1mCqMlyNsLkmyG6g/WT9lBpWTnvp6bz11kb2F1wgCvPbMuvLu1iQ2GYGs8SRE11qJ+0wqgp1k/aA6rKnHVZPD19LWlZBfRNbMEbt6fQq11zr0MzpkpYgqiJfPtJ3/ap9ZP2wMrMfJ767xoWbcohKbYRr916Fpd0b2WXq5paxRJETaMK//2500/66lchcaDXEYWVzD1FPD9zHR8vy6RFo3r8YXgPRvVrb30YTK1kCaKmWfA3WPZvp590r5u8jiZs7C0u4ZW5G3lz/mYAxg7pyNghHWlqI6uaWswSRE2yehrMfgJ6XGP9pKtJSVk5E7/Zxt/+t4HcwoNc2zueX1zahfjm1pHP1H6WIGqKin7S8WfB1a9YP+kgU1W+WL2LZ2esZfPuQs45LYbfXd7NRlc1YcUSRE2QnwGTbnL6Sd80yfpJB9mybXk8PX0NS7bk0allYyaMTuH8Li2tAG3CjiWIUHegwOknfbAQ7vrY+kkHUXrufp77fC2frdhBbOP6PHXN6YxIaUddK0CbMGUJIpSVl8GHd0HWKhj1AbTq7nVEtVL+/hL+MWcDby/cSp06MO6CTow5ryON69vHw4Q3+wSEsi8ehfWfw7DnIdn6SVe1A6Vl/HvRVv7+ZRp7i0u44awEfn5xF1o3s2Y9xoAliNC15A1Y/E+nn3Tfn3gdTa2iqkz/YSfPfb6Wbbn7GZwcy2+HdaNbm6bHX9mYMGIJIhSlzYbpv7J+0kGQuiWXp6avYdm2PXRt3YS37+zLeZ3jvA7LmJBkCSLUZK2xftJBsHl3Ic/NWMvnq3bSskl9/nxdT647K4GIOnZlkjFHYwkilBRkwXs3Wj/pKpRbeJCX/reBdxdvpV7dOvz84s7cPTiJhvXsv74xxxPUT4mIDAX+htNR7g1VfbbS/L8C57svGwItVbW5O68M+MGdt01Va3ebtMP6Sf/X+kmfouKSMv61cAsvz0mj8EApI85uz0MXJ9OyiRWgjQlU0BKEiEQALwMXAxnAEhGZpqqrK5ZR1Yd8lr8f6O3zFkWq2itY8YWUI/pJn+V1RDVWebnynxXb+fPn68jcU8QFXVvyyGVd6Wz9n405YcE8gugLpKnqJgARmQxchdNn2p+bgMeDGE/omvuM20/6CesnfQoWbczh6elr+CEznx5tm/J/1/dkQKdYr8MypsYKZoKIB9J9XmcA/fwtKCIdgCTgS5/JUSKSCpQCz6rqJ37WGwOMAWjfvn0VhV3Nvn8fvv6z00964INeR1MjpWXt49kZa5m9Jos2zaJ44cYzubpXPHWsAG3MKQmVSt1IYKqqlvlM66CqmSJyGvCliPygqht9V1LV8cB4gJSUFK2+cKvI1oUw7T7rJ32Sdhcc4MXZ65n0bToNIiP41dAu3DkwiahIu/LLmKoQzASRCbTzeZ3gTvNnJHCv7wRVzXR/bhKRuTj1iY1HrlpDVfSTbt4ebnzH+kmfgKKDZbw5fxOvfrWJopIybu7XngcuTCamcX2vQzOmVglmglgCJItIEk5iGAmMqryQiHQFooFFPtOigf2qekBEYoGBwJ+DGGv1KsqDiSM41E+6YQuvI6oRysqVj5dl8vzMdezcW8wl3Vvx68u60jGusdehGVMrBZQgROQj4E1ghqqWB7KOqpaKyH3ATJzLXCeo6ioReRJIVdVp7qIjgcmq6nuKqBvwmoiUA3VwahBHK27XLGUlMOU2yNti/aRPwPwNu3lq+hrW7NjLmQnNeOmm3vRNssRqTDDJ4d/LR1lI5CLgDqA/8AHwlqquC3JsJyQlJUVTU1O9DuPYVGHa/U7L0KtftZahAVi3cx9PT1/DV+uzSYhuwK+GduWKM9pYAdqYKiIiS1U1xd+8gI4gVHU2MFtEmuFcjjpbRNKB14F3VbWkyqKtzRa+5CSHwQ9bcjiOrL3FvDBrPVNS02lcvy6/G9aN2wZ0oH5dK0AbU10CrkGISAxwC3ArsAx4DxgE3A4MCUZwtcqa/8Csx6H71XD+77yOJmQVHihl/NebGP/1JkrLy+1tRi0AABcaSURBVBk9IIn7L+hEdCMr4htT3QKtQXwMdAH+DVypqjvcWe+79yqYY9m+DD78iXOH9DWvWj9pP0rLyvlgaQYvzFpP9r4DXH5GG341tAsdYhp5HZoxYSvQI4iXVHWOvxlHO3dlXPmZTstQ6yftl6oyd302z0xfw/pdBfRp35xXb+nDWR2sAG2M1wJNEN1FZJmq7oFDl6HepKr/DF5otcCBAudyVusn7deq7fk8PX0NC9Jy6BDTkFdu7sPQ01sjdsOgMSEh0ATxE1V9ueKFquaJyE8ASxBHY/2kj2pHfhHPz1zPR8syaNYgksev7M7N/TpQr66dejMmlASaICJERCruVXBHarWq4bF88XvrJ13JvuISXv1qI2/M24wqjBl8Gvec34lmDSK9Ds0Y40egCeJznIL0a+7rn7rTjD9L3oTFL0O/n1k/aaCkrJzJ327jxdkbyCk8yFW92vLwJV1o16Kh16EZY44h0ATxa5ykMNZ9PQt4IygR1XRp/4Ppv4TkS+DSp72OxlOqyuw1WTwzYw2bsgvpm9SCCcO6cWa75l6HZowJQKA3ypUDr7gPczRZa+CD0RDXFa6fENb9pL9P38NT09fw7eZcTotrxOu3pXBRt5ZWgDamBgn0Pohk4BmgO3CoZ6OqnhakuGqegmyYaP2k03P38/wX6/h0+XZiGtXjj1f1YGTf9kRGWAHamJom0FNMb+F0e6voIX0HziB6Btx+0jc5SeKO/0Lzdsdfp5bJLyrhn3PSeGvhFgS49/yO/Oy8jjSJsgK0MTVVoAmigar+z72SaSvwhIgsBR4LYmw1gyp8eq/bT/qdsOsnfbC0nHcXb+WlLzeQX1TCNb3jefiSLrRtbjcEGlPTBZogDohIHWCDO4R3JmCD8IPTT3rlh3Dh49D9Kq+jqTaqyucrd/Ls52vZmrOfAR1j+O2wbpwe38zr0IwxVSTQBPEA0BAYB/wR5zTT7cEKqsb4/n346jnodQsMesjraKrN8vQ9/PGz1Szdmkdyy8a8NfpshnSJswK0MbXMcROEe1PcCFV9GCjAqT+YrYt+7Cd9Rfj0k/4+fQ83vLqQ5g3r8cy1Z3DDWQnUtQK0MbXScROEqpaJyKDqCKbGyN0Ek0dBs3Zh1U86v6iEeyd+R8smUXx2/yAbgtuYWi7QP/2Wicg0EblVRK6teBxvJREZKiLrRCRNRB7xM/+vIrLcfawXkT0+824XkQ3uI3ROZ/n2k775g7DpJ62q/Grq9+zML+bvo3pbcjAmDARag4gCcoALfKYp8NHRVnBPTb0MXAxkAEtEZJpvb2lVfchn+fuB3u7zFjiX1aa421nqrpsXYLzBUdFPOncz3PZJWPWT/tfCLcxctYtHL+9Gn/bRXodjjKkGgd5JfTJ1h75AmqpuAhCRycBVwOqjLH8TTlIAuBSYpaq57rqzgKHApJOIo2qown9/Dpu/hqtfgcTwOev2ffoenp6+hou6teSuQUleh2OMqSaB3kn9Fs5f8odR1TuPsVo8kO7zOgPod5T37wAkAV8eY914P+uNAcYAtG/f/hihVIGFf4fv3oHBv4Beo4K7rRDiW3d4/oYz7UolY8JIoKeYPvN5HgVcA2yvwjhGAlNVtexEVlLV8cB4gJSUlCMSWJVZ8xnMesztJ/1o0DYTanzrDlN+dg7NG1rdwZhwEugppg99X4vIJGD+cVbLBHzHnEhwp/kzEri30rpDKq07N4BQq972ZfDRTyC+T9j1k7a6gzHh7WS/7ZKB4/XPXAIki0iSiNTDSQLTKi8kIl2BaGCRz+SZwCUiEu22N73EnVa9KvpJN4yBkeHVT9rqDsaYQGsQ+zi8BrETp0fEUalqqTssx0wgApigqqtE5EkgVVUrksVIYHJFtzp33VwR+SNOkgF4sqJgXW0OFMCkin7SM6FJq2rdvJfy91vdwRgT+Cmmkxq7WlWnA9MrTXus0usnjrLuBGDCyWz3lJWXwYd3w65VMGoKtOrhSRheUFV+aXUHYwwBnmISkWtEpJnP6+YicnXwwvLYF7+H9TPgsj9D8sVeR1Ot3lqwhS9W7+KRy7pa3cGYMBdoDeJxVc2veKGqe/jxnoXapaKfdN+fhl0/6eXpe3hmxhou6tbK6g7GmIAThL/lAr1EtuYI437S+ftLuO9Q3aGn1R2MMQEniFQReUFEOrqPF4ClwQys2mWtPbyfdETty39H41t3+Meo3lZ3MMYAgSeI+4GDwPvAZKCYw+9bqNkKsmHiDVA3Kiz7SfvWHXpb3cEY4wr0KqZC4IjRWGuFkmJn6O6CLBg9Pez6SVvdwRhzNIFexTRLRJr7vI4Wkeq/cS0YCnZCYRZc8xokhFc/aas7GGOOJdAT7bHulUsAqGqeiBzvTuqaIToR7vkGIqO8jqRa+dYdPrD7HYwxfgRagygXkUPDpYpIIn5Gd62xwiw5gNUdjDHHF+gRxO+A+SLyFSDAYNxhtk3NY3UHY0wgAi1Sfy4iKThJYRnwCVAUzMBMcFjdwRgTqEAH67sbeABn2O3lQH+c0VcvONZ6JrSoKg9P/Z5de4uZ8lOrOxhjji3QGsQDwNnAVlU9H6d39J5jr2JCzYQFW5i1ehePXNbN6g7GmOMKNEEUq2oxgIjUV9W1QJfghWWq2vL0PTw7Yw0Xd2/FnQMTvQ7HGFMDBFqkznDvg/gEmCUiecDW4IVlqlL+/hLufc+tO1xv/R2MMYEJtEh9jfv0CRGZAzQDPg9aVKbKVNQdsvYV88HPBtCsYaTXIRljaogTbjmqql+p6jRVPXi8ZUVkqIisE5E0EfE7VIeI3Cgiq0VklYhM9JleJiLL3ccRrUpNYHzrDr3aNT/+CsYY4wrakKUiEgG8DFwMZABLRGSaqq72WSYZ+A0w0M/d2UWq2itY8YUDqzsYY07FCR9BnIC+QJqqbnKPNiYDV1Va5ifAy6qaB6CqWUGMJ6xY3cEYc6qCmSDigXSf1xnuNF+dgc4iskBEFovIUJ95USKS6k73295URMa4y6RmZ2dXbfQ1mG/d4eWb+1jdwRhzUrzuilMXSAaG4NyE97WInOEODNhBVTNF5DTgSxH5QVU3+q6squOB8QApKSm1Z2yoU1RRd/j9Fd2t7mCMOWnBPILIBHybKyS403xlANNUtURVNwPrcRIGqprp/twEzMW5Oc8ch9UdjDFVJZgJYgmQLCJJIlIPGAlUvhrpE5yjB0QkFueU0ya330R9n+kDgdWYY7K6gzGmKgXtFJOqlorIfcBMIAKYoKqrRORJIFVVp7nzLhGR1UAZ8EtVzRGRAcBrIlKOk8Se9b36yRzJ7ncwxlS1oNYgVHU6ML3StMd8nivwc/fhu8xC4IxgxlbbvDl/M7NW7+IxqzsYY6pIME8xmWqybFsez85YyyXdW3GH1R2MMVXEEkQNt2f/Qe6buIzWzaL4P6s7GGOqkNeXuZpToKo8/MEKqzsYY4LCjiBqsDfnb2b2ml38xsZZMsYEgSWIGsrqDsaYYLMEUQNZ3cEYUx2sBlHDWN3BGFNd7AiihrG6gzGmuliCqEGs7mCMqU6WIGoIqzsYY6qb1SBqAN+6w1SrOxhjqokdQdQAFXWH3w7rxplWdzDGVBNLECHuO7fucGmPVowekOh1OMaYMGIJIoTt2X+Q+926w5+t7mCMqWZWgwhRTt3h+x/rDg2s7mCMqV52BBGinLpDltUdjDGesQQRgqzuYIwJBUFNECIyVETWiUiaiDxylGVuFJHVIrJKRCb6TL9dRDa4j9uDGWcosbqDMSZUBK0GISIRwMvAxUAGsEREpvn2lhaRZOA3wEBVzRORlu70FsDjQAqgwFJ33bxgxRsKrO5gjAklwTyC6AukqeomVT0ITAauqrTMT4CXK774VTXLnX4pMEtVc915s4ChQYw1JFjdwRgTSoKZIOKBdJ/XGe40X52BziKyQEQWi8jQE1gXERkjIqkikpqdnV2FoVc/qzsYY0KN10XqukAyMAS4CXhdRAL+01lVx6tqiqqmxMXFBSnE4LO6gzEmFAUzQWQC7XxeJ7jTfGUA01S1RFU3A+txEkYg69YKvnWHl0f1sbqDMSZkBDNBLAGSRSRJROoBI4FplZb5BOfoARGJxTnltAmYCVwiItEiEg1c4k6rdSrqDr+zuoMxJsQE7SomVS0VkftwvtgjgAmqukpEngRSVXUaPyaC1UAZ8EtVzQEQkT/iJBmAJ1U1N1ixeqWi7jC0R2tut7qDMSbEiKp6HUOVSElJ0dTUVK/DCNie/QcZ9rd5REQIn90/2E4tGWM8ISJLVTXF3zwbi8kDqsovpnxPdsEBPhxr9zsYY0KT11cxhaU35m3mf2udukPPBKs7GGNCkyWIarZ0ax7PfW51B2NM6LMEUY2c+x2+o03zKJ67vqfd72CMCWlWg6gmVncwxtQ0dgRRTazuYIypaSxBVAOrOxhjaiJLEEFmdQdjTE1lNYggqqg77C44yNSx51jdwRhTo9gRRBAdqjtcbnUHY0zNYwkiSCrqDped3prbzungdTjGGHPCLEEEQV6hU3do27yB1R2MMTWW1SCqWHm58osPnLrDh2MH0DTK6g7GmJrJjiCq2BvzN/GlW3c4I6GZ1+EYY8xJswRRhZZuzeW5z9dZ3cEYUytYgqgiTt1hGfFWdzDG1BJBTRAiMlRE1olImog84mf+aBHJFpHl7uNun3llPtMrtyoNKb51h5dH9bG6gzGmVghakVpEIoCXgYuBDGCJiExT1dWVFn1fVe/z8xZFqtorWPFVpYq6wx+G97C6gzGm1gjmEURfIE1VN6nqQWAycFUQt+cJqzsYY2qrYCaIeCDd53WGO62y60RkhYhMFZF2PtOjRCRVRBaLyNX+NiAiY9xlUrOzs6sw9MBY3cEYU5t5XaT+D5Coqj2BWcDbPvM6uI20RwEvikjHyiur6nhVTVHVlLi4uOqJ2GV1B2NMbRfMBJEJ+B4RJLjTDlHVHFU94L58AzjLZ16m+3MTMBfoHcRYT1hF3eHRK+x+B2NM7RTMBLEESBaRJBGpB4wEDrsaSUTa+LwcDqxxp0eLSH33eSwwEKhc3PZMRd1h2BmtubW/1R2MMbVT0K5iUtVSEbkPmAlEABNUdZWIPAmkquo0YJyIDAdKgVxgtLt6N+A1ESnHSWLP+rn6yRO+dYdnr7O6gzGm9hJV9TqGKpGSkqKpqalB3UZ5uXL3O6nM37CbD8cOsFNLxpgaT0SWuvXeI3hdpK5RXp9ndQdjTPiwBBGgpVtz+fNMqzsYY8KHJYgA5BUe5D6rOxhjwoz1gziOivsdcqy/gzEmzNgRxHFY3cEYE64sQRyD1R2MMeHMEsRRVNQdEqKt7mCMCU9Wg/DDt+7w0T1WdzDGhCc7gvCjou7w+yu6cXq81R2MMeHJEkQlFXWHy89owy1WdzDGhDFLED586w7PXHeG1R2MMWHNahCu8nLl51OWW93BGGNcdgThGj9vE3PWZVvdwRhjXJYggNQtufyf1R2MMeYwYZ8gcgsPcv8kqzsYY0xlYV+DUFV6tG3Kgxd1trqDMcb4COoRhIgMFZF1IpImIo/4mT9aRLJFZLn7uNtn3u0issF93B6sGGMa1+eN28+2uoMxxlQStCMIEYkAXgYuBjKAJSIyzU/r0PdV9b5K67YAHgdSAAWWuuvmBSteY4wxhwvmEURfIE1VN6nqQWAycFWA614KzFLVXDcpzAKGBilOY4wxfgQzQcQD6T6vM9xplV0nIitEZKqItDvBdY0xxgSJ11cx/QdIVNWeOEcJb5/IyiIyRkRSRSQ1Ozs7KAEaY0y4CmaCyATa+bxOcKcdoqo5qnrAffkGcFag67rrj1fVFFVNiYuLq7LAjTHGBDdBLAGSRSRJROoBI4FpvguISBufl8OBNe7zmcAlIhItItHAJe40Y4wx1SRoVzGpaqmI3IfzxR4BTFDVVSLyJJCqqtOAcSIyHCgFcoHR7rq5IvJHnCQD8KSq5gYrVmOMMUcSVfU6hiqRkpKiqampXodhjDE1iogsVdUUv/NqS4IQkWxg6ym8RSywu4rCqUoW14mxuE6MxXViamNcHVTVbxG31iSIUyUiqUfLol6yuE6MxXViLK4TE25xeX2ZqzHGmBBlCcIYY4xfliB+NN7rAI7C4joxFteJsbhOTFjFZTUIY4wxftkRhDHGGL8sQRhjjPErrBJEAA2M6ovI++78b0QkMUTiOmpjpSDHNUFEskRk5VHmi4i85Ma9QkT6hEhcQ0Qk32d/PVZNcbUTkTkislpEVonIA36WqfZ9FmBc1b7PRCRKRL4Vke/duP7gZ5lq/0wGGJcnn0l32xEiskxEPvMzr2r3l6qGxQNnuI+NwGlAPeB7oHulZe4BXnWfj8RpZhQKcY0G/uHBPjsX6AOsPMr8YcAMQID+wDchEtcQ4DMP9lcboI/7vAmw3s+/ZbXvswDjqvZ95u6Dxu7zSOAboH+lZbz4TAYSlyefSXfbPwcm+vv3qur9FU5HEIE0MLqKH4ccnwpcKCISAnF5QlW/xhkj62iuAt5Rx2KgeaUBGL2KyxOqukNVv3Of78MZfLJyH5Nq32cBxlXt3H1Q4L6MdB+Vr5qp9s9kgHF5QkQSgMtxRr/2p0r3VzgliECaEB1aRlVLgXwgJgTiAv+NlbwWyo2dznFPEcwQkR7VvXH30L43zl+fvjzdZ8eICzzYZ+7pkuVAFk4XyaPur2r8TAYSF3jzmXwR+BVQfpT5Vbq/wilB1GSn1FgpDH2HM77MmcDfgU+qc+Mi0hj4EHhQVfdW57aP5ThxebLPVLVMVXvh9HzpKyKnV8d2jyeAuKr9MykiVwBZqro02NuqEE4JIpAmRIeWEZG6QDMgx+u49OiNlbwWUGOn6qaqeytOEajqdCBSRGKrY9siEonzJfyeqn7kZxFP9tnx4vJyn7nb3APM4cje8158Jo8bl0efyYHAcBHZgnMq+gIRebfSMlW6v8IpQRy3gZH7+nb3+fXAl+pWe7yMS47eWMlr04Db3Ctz+gP5qrrD66BEpHXFeVcR6Yvz/zzoXyruNt8E1qjqC0dZrNr3WSBxebHPRCRORJq7zxsAFwNrKy1W7Z/JQOLy4jOpqr9R1QRVTcT5nvhSVW+ptFiV7q+gNQwKNRpYA6M3gX+LSBpOEXRkiMTlt7FSsInIJJyrW2JFJAN4HKdgh6q+CkzHuSonDdgP3BEicV0PjBWRUqAIGFkNiR6cv/BuBX5wz18D/BZo7xObF/sskLi82GdtgLdFJAInIU1R1c+8/kwGGJcnn0l/grm/bKgNY4wxfoXTKSZjjDEnwBKEMcYYvyxBGGOM8csShDHGGL8sQRhjjPHLEoQxIUCc0VSPGJ3TGC9ZgjDGGOOXJQhjToCI3OL2ClguIq+5g7oViMhf3d4B/xOROHfZXiKy2B3Q7WMRiXandxKR2e7AeN+JSEf37Ru7A7+tFZH3qmEkYWOOyRKEMQESkW7ACGCgO5BbGXAz0AjnTtYewFc4d3YDvAP82h3Q7Qef6e8BL7sD4w0AKoba6A08CHTH6Q8yMOi/lDHHEDZDbRhTBS7EGZRtifvHfQOc4aDLgffdZd4FPhKRZkBzVf3Knf428IGINAHiVfVjAFUtBnDf71tVzXBfLwcSgfnB/7WM8c8ShDGBE+BtVf3NYRNFfl9puZMdv+aAz/My7PNpPGanmIwJ3P+A60WkJYCItBCRDjifo+vdZUYB81U1H8gTkcHu9FuBr9yObhkicrX7HvVFpGG1/hbGBMj+QjEmQKq6WkQeBb4QkTpACXAvUIjTVOZRnFNOI9xVbgdedRPAJn4cufVW4DV3FM4S4IZq/DWMCZiN5mrMKRKRAlVt7HUcxlQ1O8VkjDHGLzuCMMYY45cdQRhjjPHLEoQxxhi/LEEYY4zxyxKEMcYYvyxBGGOM8ev/ATjmZUPC+FYGAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gU5drH8e+dXkggQAQJJREQApEakF4EpIqgKGAFj4IFwXqOeux61POKgNjBfpQmKqCACNKlSOihdwid0Anpz/vHLBhCgCTsZja79+e6crG78+zundHdX+aemWfEGINSSinv5WN3AUoppeylQaCUUl5Og0AppbycBoFSSnk5DQKllPJyGgRKKeXlNAiUyicR+VpE3szn2J0i0v5qX0epoqBBoJRSXk6DQCmlvJwGgfIojpbMsyKyRkTOiMgXIlJORKaLyCkRmSUiETnGdxeRdSJyXETmikhsjmX1RWSF43njgaBc79VNRFY5nrtIROoUsuaHRGSriBwVkSkiUsHxuIjIcBE5JCInRWStiMQ5lnURkfWO2vaKyDOFWmFKoUGgPNPtQAfgeuAWYDrwAhCJ9f/8YAARuR4YCzzhWDYN+EVEAkQkAJgE/A8oDfzgeF0cz60PfAkMBMoAnwFTRCSwIIWKyE3A28CdwLXALmCcY/HNQCvH71HSMSbZsewLYKAxJgyIA2YX5H2VykmDQHmiD4wxB40xe4EFwFJjzEpjTCrwM1DfMa43MNUYM9MYkwEMBYKBZkATwB8YYYzJMMZMBJbleI8BwGfGmKXGmCxjzDdAmuN5BXE38KUxZoUxJg14HmgqItFABhAG1ATEGLPBGLPf8bwMoJaIhBtjjhljVhTwfZU6T4NAeaKDOW6fzeN+CcftClh/gQNgjMkG9gBRjmV7zYWzMu7KcbsK8LSjLXRcRI4DlRzPK4jcNZzG+qs/yhgzG/gQ+Ag4JCKjRCTcMfR2oAuwS0TmiUjTAr6vUudpEChvtg/rCx2wevJYX+Z7gf1AlOOxcyrnuL0H+I8xplSOnxBjzNirrCEUq9W0F8AYM9IY0xCohdUietbx+DJjzK3ANVgtrAkFfF+lztMgUN5sAtBVRNqJiD/wNFZ7ZxGwGMgEBouIv4jcBjTO8dzRwMMicqNjp26oiHQVkbAC1jAW6C8i9Rz7F97CamXtFJFGjtf3B84AqUC2Yx/G3SJS0tHSOglkX8V6UF5Og0B5LWPMJuAe4APgCNaO5VuMMenGmHTgNqAfcBRrf8JPOZ6bADyE1bo5Bmx1jC1oDbOAl4AfsbZCqgJ9HIvDsQLnGFb7KBl417HsXmCniJwEHsba16BUoYhemEYppbybbhEopZSX0yBQSikvp0GglFJezqVBICKdRGST4/T55/JYPtxxiv4qEdnsOBZbKaVUEXLZzmIR8QU2Y53qn4R1VmZfY8z6S4x/HKhvjHngcq9btmxZEx0d7eRqlVLKsy1fvvyIMSYyr2V+LnzfxsBWY8x2ABEZB9wK5BkEQF/glSu9aHR0NAkJCU4rUimlvIGI7LrUMle2hqKwzr48J8nx2EVEpAoQwyUmzhKRASKSICIJhw8fdnqhSinlzdxlZ3EfYKIxJiuvhcaYUcaYeGNMfGRknls2SimlCsmVQbAXa96Wcyo6HstLH6xT7ZVSShUxV+4jWAZUF5EYrADoA9yVe5CI1AQisOZ2KZSMjAySkpJITU0t7EuoHIKCgqhYsSL+/v52l6KUKgIuCwJjTKaIDAJmAL5Yc66vE5HXgQRjzBTH0D7AOHMVhy8lJSURFhZGdHQ0F04WqQrKGENycjJJSUnExMTYXY5Sqgi4cosAY8w0rKs+5Xzs5Vz3X73a90lNTdUQcBIRoUyZMuhOeaW8h7vsLL5qGgLOo+tSKe/iMUFwJWkZWRw4cZZsnW1VKaUu4DVBcDI1g0On0th++Azpmc69hsfx48f5+OOPC/y8Ll26cPy4zqqhlLKX1wRBZFgQlUuHkJaRxZZDpzh5NsNpr32pIMjMzLzs86ZNm0apUqWcVodSShWGS3cWu5tSIQEE+/uy+2gKO5PPEBkWSLnwIHyusif+3HPPsW3bNurVq4e/vz9BQUFERESwceNGNm/eTI8ePdizZw+pqakMGTKEAQMGAH9Pl3H69Gk6d+5MixYtWLRoEVFRUUyePJng4GBn/NpKKXVZHhcEr/2yjvX7Tl5xXHpmNhlZ2fj4CEF+vlwuC2pVCOeVW2pfcvk777xDYmIiq1atYu7cuXTt2pXExMTzh19++eWXlC5dmrNnz9KoUSNuv/12ypQpc8FrbNmyhbFjxzJ69GjuvPNOfvzxR+655578/dJKKXUVvKY1lFuAnw+B/r4YYzibkUlWtvN2Ijdu3PiCY/BHjhxJ3bp1adKkCXv27GHLli0XPScmJoZ69eoB0LBhQ3bu3Om0epRS6nI8bovgcn+55yUtI4vdR1M4m5FFZIlAypW8+lZRaGjo+dtz585l1qxZLF68mJCQENq0aZPnGdCBgYHnb/v6+nL27NmrqkEppfLLa7cIzgn096VqZAnKhAZy+PS5o4rynPvuksLCwjh16lSey06cOEFERAQhISFs3LiRJUuWOKNspZRyGo/bIigMHx8hKiKY0EBf9h47y5ZDp6kUEUJ4cP7m2ilTpgzNmzcnLi6O4OBgypUrd35Zp06d+PTTT4mNjaVGjRo0adLEVb+GUkoVisuuUOYq8fHxJveFaTZs2EBsbKxTXj9nq6hsiUDKO6FVVBw5c50qpewnIsuNMfF5LfP61lBu51tFJQI5UshWkVJKFScaBHnw8RGiSgXnOAHttFNPQFNKKXeiQXAZpUICqHZNCQJ8fdiZfIZ9x3WuIqWU59EguIJAf1+qXqOtIqWU59IgyAcfsVpFVXK0ik5oq0gp5SE0CAqgZEgA1cpZraJd2ipSSnkIDYICCvS7+lZRiRIlANi3bx+9evXKc0ybNm3IfZhsbiNGjCAlJeX8fZ3WWilVGBoEheCsVlGFChWYOHFioevIHQQ6rbVSqjA0CK7CuVbR+2+/xn/fG3G+VfTqq6/y5ptv0q5dOxo0aMANN9zA5MmTL3r+zp07iYuLA+Ds2bP06dOH2NhYevbsecFcQ4888gjx8fHUrl2bV155BbAmstu3bx9t27albdu2gDWt9ZEjRwAYNmwYcXFxxMXFMWLEiPPvFxsby0MPPUTt2rW5+eabdU4jpZQHTjEx/Tk4sNa5r1n+Buj8Tp6LAv18Gdj/HgY9PoQ+/R4iJT2T8RMm8PuMGQwePJjw8HCOHDlCkyZN6N69+yWvB/zJJ58QEhLChg0bWLNmDQ0aNDi/7D//+Q+lS5cmKyuLdu3asWbNGgYPHsywYcOYM2cOZcuWveC1li9fzldffcXSpUsxxnDjjTfSunVrIiIidLprpdRFdIvACRo2aMDxo0cISDvO6tWrCQoNJ7hkGV544QXq1KlD+/bt2bt3LwcPHrzka8yfP//8F3KdOnWoU6fO+WUTJkygQYMG1K9fn3Xr1rF+/frL1rNw4UJ69uxJaGgoJUqU4LbbbmPBggWATnetlLqY520RXOIvd1e74447+H3qFJL27aNbj9v45POv2b3vAMsSEggMCCA6OjrP6aevZMeOHQwdOpRly5YRERFBv379CvU65+h010qp3HSLwEl69+7NuHHjmPTTTzzc/x5MegqhJUuz53gav8+axa5duy77/FatWjFmzBgAEhMTWbNmDQAnT54kNDSUkiVLcvDgQaZPn37+OZea/rply5ZMmjSJlJQUzpw5w88//0zLli2d+NsqpTyJ520R2KR27dqcOnWKqKgooipU4NEH+9Glaze6tG5C7Tr1uL5Gjcs+/5FHHqF///7ExsYSGxtLw4YNAahbty7169enZs2aVKpUiebNm59/zoABA+jUqRMVKlRgzpw55x9v0KAB/fr1o3HjxgA8+OCD1K9fX9tASqk86TTULpaW6ZjWOr14TWvtzutUKVVwOg21jQL9rGmty54/Ae20zlWklHIrGgRFwEeECqWCqVImhLTMbJ2rSCnlVjwmCIpDi6tkcADVrylBoJ97z1VUHNalUsp5PCIIgoKCSE5OLhZfYAF+vlyXo1W07dBp0tyoVWSMITk5maCgILtLUUoVEY84aqhixYokJSVx+PDhyw80BtxoR21meha7U9LZvR0iQgIIDvC1uyTACtaKFSvaXYZSqoh4RBD4+/sTExNz+UFrJsCf70O/qRDsPhOz7TmawqAxK1idtJ9+zaJ5vktNAv3cIxCUUt7BI1pD+RIeBYc3wYT7IMt9dtRWKh3CDw8344HmMXy9aCe9PlnM7uSUKz9RKaWcxHuCILo5dB8JO+bB1KetNpGbCPDz4eVbavHZvQ3ZlXyGriMXMH3tfrvLUkp5Ce8JAoB6d0HLp2HFN7D4Q7uruUjH2uWZOrgl111Tgke+X8ErkxPdakeyUsozeVcQALR9EWrdCr+/BBun2l3NRSqVDuGHgU35R4sYvlm8i16fLGZX8hm7y1JKeTDvCwIfH+jxKVSoDz8+CPtX213RRQL8fHip29+tom4jFzJNW0VKKRfxviAACAiBvuMguDSM6QMn99ldUZ5ytooe1VaRUspFXBoEItJJRDaJyFYRee4SY+4UkfUisk5ExriynguElYO7xkPaSRjTG9Lds/1yrlX0oKNVdPsni7RVpJRyKpcFgYj4Ah8BnYFaQF8RqZVrTHXgeaC5MaY28ISr6slT+Tjo9RUcTISfBkB2dpG+fX4F+PnwYrdajLq3IbuTU+g2ciFT12irSCnlHK7cImgMbDXGbDfGpAPjgFtzjXkI+MgYcwzAGHPIhfXk7fqboePbsPFXmPVKkb99QdzsaBVVvaYEj41ZwcuTE0nN0FaRUurquDIIooA9Oe4nOR7L6XrgehH5U0SWiEinvF5IRAaISIKIJFxxGonCuHEgNHoQFo2E5d84//WdqFLpECY4WkXfOlpFO49oq0gpVXh27yz2A6oDbYC+wGgRuWj+B2PMKGNMvDEmPjIy0vlViECn/0K19jD1Kdg+1/nv4UTnWkWj74sn6dhZun2grSKlVOG5Mgj2ApVy3K/oeCynJGCKMSbDGLMD2IwVDEXP1w96fQllqlvTUBzebEsZBdGhVjmmDm5BNUer6KVJ2ipSShWcK4NgGVBdRGJEJADoA0zJNWYS1tYAIlIWq1W03YU1XV5QSetIIt8AGHMnnEm2rZT8qhhhtYoeahnD/5Zoq0gpVXAuCwJjTCYwCJgBbAAmGGPWicjrItLdMWwGkCwi64E5wLPGGHu/fSOqQJ8x1rkF4++GzDRby8mPAD8f/t31wlbRr2vc89wIpZT78YiL17tE4o8w8QGo0wd6fupW1zG4nKRjKTw+diUrdx/nniaVebFrLYL8dVprpbydXry+MOJuh7b/hjXjYMFQu6vJt3OtogGtruO7Jbu57eNF7NBWkVLqMjQILqfVs1CnN8x+ExJ/sruafPP39eGFLrF8fl88e4+f5ZYPFvLLam0VKaXypkFwOSLQ/QOo1AQmPQJJRdCScqL2tcoxbUhLqpcrweNjV/LipLV6VJFS6iIaBFfiFwh9voew8jC2DxzfbXdFBRJVKpgJA5syUFtFSqlL0CDIj9CycNcPkJluTVCXetLuigrE39eH57vE8sX98ew7cZZuIxcwRVtFSikHDYL8irwe7vzGuu7xxAcgK9PuigqsXWw5pg5uSY3yYQweu5J//6ytIqWUBkHBVG0LXd+DrTNhxvN2V1MoUaWCGT+wKQNbX8f3S3fT8+NFbD982u6ylFI20iAoqPj+0HQQ/DUKlo6yu5pC8ff14fnOsXzZL579J6yjiiavyj37h1LKW2gQFEaH16FGF/jtX7Blpt3VFNpNNcsxbXBLal4bzpBxq3hBW0VKeSUNgsLw8YXbRkO52vBDfzi4zu6KCq1CqWDGDWjCwNbXMUZbRUp5JQ2CwgosAX3HW/+O6Q2nDtpdUaGdaxV91a+RtoqU8kIaBFejZBT0HQspyTDuLsg4a3dFV6VtzWsuaBU9/9MazqQVv6OjlFIFo0FwtSrUt9pEe5dbZx+76XWP8+tcq+jh1lUZ+9ceOo6Yz59bj9hdllLKhTQInCG2G3R4Ddb9DHPfsruaq+bv68NznWsyYWBT/HyEuz9fyvM/reFkaobdpSmlXECDwFmaDYb698L8d2HVWLurcYrGMaX57YlWDGx1HeOX7eHmYfOZvbH47gtRSuVNg8BZRKDrMIhpBVMeh12L7K7IKYL8fXm+Syw/Pdqc8GA/Hvg6gafGr+LYmXS7S1NKOYkGgTP5BcCd30JENIy7G5K32V2R09SrVIpfHm/B4JuqMWX1PjoMn8f0tfvtLksp5QQaBM4WHGFd9xhjHVZ69pjdFTlNoJ8vT91cg8mDmlMuPIhHvl/Bo98v5/Ap97+cp1Lq0jQIXKFMVej9PRzbCRPugyzP2slau0JJJj3WnGc71mDW+kN0GD6PSSv3Utwue6qUsmgQuEp0c+uiNjvmw9SnwMO+JP19fXisbTWmDm5BTNlQnhi/ige/SeDAiVS7S1NKFZAGgSvV6wstn4EV38KiD+yuxiWqlwtj4sPNeLFrLH9uO0KHYfMY99du3TpQqhjRIHC1tv+GWj1g5suw4Ve7q3EJXx/hwZbX8duQVtSqEM5zP63l3i/+Ys/RFLtLU0rlgwaBq/n4QM9PIaoB/PQQ7Ftld0UuE102lLEPNeGNHnGs3H2MjiPm882inWRn69aBUu5Mg6Ao+AdDn7EQUsa67vFJz71MpI+PcG+TKsx4shXx0aV5Zco6eo9arDOaKuXGNAiKSlg567DStNPWYaVpnv3FWDEihG/6N+LdXnXYdOAUnd9fwKj528jSrQOl3I4GQVEqVxvu+AoOJlptomzPvgiMiHBHfCVmPtWaltUjeWvaRm77ZBGbD56yuzSlVA4aBEWtegfo9F/YNM3agewFyoUHMfq+hozsW589R1PoOnIBI//YQkZW8Z6pVSlPoUFghxsHQOMBsPhDWP613dUUCRGhe90KzHyyFR1rl2fYzM10//BPEveesLs0pbyeBoFdOr4N1TrA1Kdh+1y7qykyZUoE8uFdDfjs3oYcOZ3GrR/9ybszNuq1kpWykQaBXXz9oNeXUPZ6GH8fHN5kd0VFqmPt8sx6sjU960fx0ZxtdPtgISt2e868TEoVJxoEdgoKt44k8guAMXfCmWS7KypSJUP8GXpHXb7u34iUtExu/2QRb/66nrPpunWgVFHSILBbqcrQdxycOgDj74ZM75vJs02Na5jxZCvuvrEyny/cQaf357N4m3eFolJ20iBwBxXjoccnsHuxdVEbL5ynJyzInzd73MCYh27EGOg7egkvTlrL6bRMu0tTyuNpELiLuNug7YuwZrx1uUsv1axqWX57oiX/aBHD90t303H4fOZtPmx3WUp5NA0Cd9LqGajTB+b8BxJ/tLsa24QE+PFSt1pMfLgZQf4+3P/lXzz7w2pOpHjWdR2UchcaBO5EBLqPhMrN4OdHYM8yuyuyVcMqEUwd3JJH21Tlp5V7aT98Hr+vO2B3WUp5HA0Cd+MXCL2/g/AKMK4vHNtld0W2CvL35Z+dajL5seaUCQ1gwP+W8/jYlSSf9r6d6kq5ikuDQEQ6icgmEdkqIs/lsbyfiBwWkVWOnwddWU+xEVoG7poAWenWBHWpevZtXFRJpgxqwVMdrue3xP10GD6fX1bv0wvgKOUELgsCEfEFPgI6A7WAviJSK4+h440x9Rw/n7uqnmIn8nq481tI3gI/9IcsPXomwM+Hwe2q8+vjLakUEczjY1cy8H/LOXRSL4+p1NVw5RZBY2CrMWa7MSYdGAfc6sL38zzXtYGuw2DbH/DbRRtUXqtG+TB+fKQZz3euydzNh2k/bB4/JOzRrQOlCsmVQRAF7MlxP8nxWG63i8gaEZkoIpXyeiERGSAiCSKScPiwlx1K2PB+aPY4LBsNSz+zuxq34efrw8DWVZk+pCXXlwvj2YlruP+rZew9ftbu0pQqduzeWfwLEG2MqQPMBL7Ja5AxZpQxJt4YEx8ZGVmkBbqF9q9Bja7WVsHmGXZX41aqRpZgwsCmvNa9Ngk7j3LzsHl8t2SXXh5TqQJwZRDsBXL+hV/R8dh5xphkY8y5wz8+Bxq6sJ7iy8cXbh8N5eJg4gNwINHuityKj49wf7NoZjzRinqVS/HipETu+nwJu5LP2F2aUsWCK4NgGVBdRGJEJADoA0zJOUBErs1xtzuwwYX1FG8BodYEdYFh1pFEpw7aXZHbqVQ6hO/+cSPv3HYD6/aepOOI+XyxcIdeHlOpK3BZEBhjMoFBwAysL/gJxph1IvK6iHR3DBssIutEZDUwGOjnqno8QngFa4K6s0etcwzSU+yuyO2ICH0aV+b3p1rRrGpZ3vh1PXd8uoith/TymEpdihS3Iy3i4+NNQkKC3WXYa+NUGHc31OoOvb4GH7t39bgnYwyTV+3j1V/WkZKWxZD21RnY6jr8fHV9Ke8jIsuNMfF5LdNPRHFUsyvc/Aasnwxz3rS7GrclIvSoH8XMJ1vTLvYa3p2xiR4f/8n6fSftLk0pt6JBUFw1HQQN7ocF78GqMXZX49YiwwL55J6GfHx3Aw6cSKX7hwsZNnMz6ZnZdpemlFvIVxCIyBARCRfLFyKyQkRudnVx6jJEoOt7ENMapgyGnX/aXZHb63LDtcx8sjW31K3AyD+2cMsHC1m957jdZSllu/xuETxgjDkJ3AxEAPcC77isKpU/vv5w5zcQEW1d3Sx5m90Vub2I0ACG967HF/fHc+JsBj0//pO3p20gNUMvj6m8V36DQBz/dgH+Z4xZl+MxZafgCLh7AiDWdY9TjtpdUbHQLrYcvz/Vit6NKvHZ/O10fn8By3bqulPeKb9BsFxEfscKghkiEgZog9VdlL4O+oyB47thwn2QmW53RcVCeJA/b99Wh+/+cSMZWdnc+dliXp2yjjN6eUzlZfIbBP8AngMaGWNSAH+gv8uqUgVXpSl0/xB2LoCpT3nldY8Lq0X1ssx4ohX3N43m60U76ThiPn9uPWJ3WUoVmfwGQVNgkzHmuIjcA7wI6CT57qZub2j1LKz8H/z5vt3VFCuhgX682r02EwY2xd/Xh7s/X8pzP67hZKpeHlN5vvwGwSdAiojUBZ4GtgHfuqwqVXhtXoDat8GsV2HDL3ZXU+w0jinN9CEtGdj6OiYk7OHmYfOZvVGn81CeLb9BkGmsU5BvBT40xnwEhLmuLFVoPj7Q42OoGA8/PgT7VtpdUbET5O/L851j+fnR5oQH+/HA1wk8OX4Vx87ovhflmfIbBKdE5Hmsw0aniogP1n4C5Y78g62dx6GRMKYPnNh75eeoi9StVIpfHm/B4HbV+WX1PjoMn8f0tfvtLkspp8tvEPQG0rDOJziANaX0uy6rSl29EtdYs5Wmn4GxvSHttN0VFUuBfr481eF6pgxqQbnwIB75fgWPfr+cw6fSrvxkpYqJfAWB48v/e6CkiHQDUo0xuo/A3ZWrBXd8DQfXwY8PQraeNFVYtSqEM+mx5jzbsQaz1h+iw/B5/LwySS+PqTxCfqeYuBP4C7gDuBNYKiK9XFmYcpLq7aHz/8Hm6TDzZburKdb8fX14rG01pg1pQUzZUJ4cv5p/fJPA/hN6eUxVvOVrGmrH9QI6GGMOOe5HArOMMXVdXN9FdBrqQpr2T/jrM+g2HOIfsLuaYi8r2/D1op28O2Mj/j4+/LtrLL0bVUJET7hX7skZ01D7nAsBh+QCPFe5g45vQbUOMPUZ2Dbb7mqKPV8f4R8tYpjxRCtqR4Xz3E9rueeLpew4opfHVMVPfr/MfxORGSLST0T6AVOBaa4rSzmdrx/0+hIia8KEfnB4k90VeYQqZUIZ82AT3uwRx+o9J+g4fD7vzthISrpOU6GKj3xfoUxEbgeaO+4uMMb87LKqLkNbQ1fp+B4YfZN1iOlDsyG0rN0VeYxDJ1N5e/pGfl65lwolg3ipWy06xZXXdpFyC5drDemlKr1R0nL4ugtcWw/umwz+QXZX5FH+2nGUlycnsvHAKVpWL8ur3WtTNbKE3WUpL1fofQQickpETubxc0pE9Hp/xVXFhtDzU9izBKYM0gnqnKxxTGl+fbwFr9xSi1W7j9NpxHzemb5RZzVVbsvvcguNMTqNhKeq3dO6kM3sN6BMdWjzL7sr8ih+vj70bx5DtzoVeGf6Rj6dt43Jq/byYtdadLlB20XKveiRP96s5dNQ9y6Y+xasnWh3NR4pMiyQ9+6sy8SHmxIREsBjY1ZwzxdL2XrolN2lKXWeBoE3E4FbRkDlZjDpUdjzl90Veaz46NJMGdSc17rXZk3SCTqNWMDb0zZwWttFyg1oEHg7v0Do8z2UjIKxfeHYTrsr8lh+vj7c3yyaOc+0oWf9KD6bv512781lyup9OlWFspUGgYKQ0nDXBMjOgDG9IVWvOeRKZUsE8u4ddfnxkWaULRHI4LEruWv0UjYf1HaRsocGgbKUrQ69v4PkrfBDP8jSloWrNawSwZRBLXijRxzr95+ky/sL+M/U9douUkVOg0D9LaaVNRfRttkw/Z96WGkR8PUR7m1ShTnPtKFXw4p8vnAHNw2dy+RVe7VdpIqMBoG6UIP7oPkQSPgCFn8E2dl2V+QVSocG8M7tdfj50eaULxnEkHGr6D1qCRsP6Ok6yvX0zGJ1sexsmHAvbPwV/EOt6xqUi4PycVDuBut+oJ5i4ipZ2Ybxy/bwfzM2cio1k/ubRvNEh+qEB+lFAVXh6RQTquAyUiHxR9i/Gg4mwoFESMuxEzki5u9gKB9nBUWpytYhqcopjp1J593fNzH2r92UCQ3khS416Vk/Sk9GU4WiQaCunjFwYo8VCAcT4cBa69+jOwDH/0OBJaFc7b+DoXwcXFPLmuBOFdqapOO8NHkdq/ccp1F0BK91j6NWhXC7y1LFjAaBcp2003Bo/d/BcCDRujRmhmNefvGBMtUubC2Vj4Owa3XroQCysw0TEvbw3982cuJsBvc1jebJDtdTMljbRSp/NAhU0crOhmM7cgSD498Tu/8eE1z6wmAofwOUrQF+AfbVXVrvBOQAABSHSURBVAwcT0ln6O+b+H7pbsqEBvBc51huqx+Fj4+Gqro8DQLlHs4et7YWcraWDm2AzFRruY8/RNbIsfXgCAi9ZsJFEvee4KXJiazcfZyGVSJ4/dba1K5Q0u6ylBvTIFDuKysTjm7L1VpKhFP7/x5TovyFwVAuzmo3+V528lyPl51tmLgiif9O38ixlHTuaVKFpzvUoGSItovUxTQIVPFzJhkOrr2wtXR4ozUNBoBfkHXZzQuOXKoNwRH21m2DEykZDJu5if8t2UVESAD/6lSTXg0rartIXUCDQHmGzHQ4svnC1tKBREg58veYkpUubi1FxICP5587uW7fCV6ZvI6EXceoX7kUr3eP44aK2i5SFg0C5bmMgdMHHVsOObYgjmwBk2WN8aKT4owx/LRiL29P30DymXTualyZZzvWoFSI7oT3drYFgYh0At4HfIHPjTHvXGLc7cBEoJEx5rLf8hoEKl8yzlqtpJytpYNrL5xZ1YNPijtxNoPhMzfz7eKdlAz255+datI7vpK2i7yYLUEgIr7AZqADkAQsA/oaY9bnGhcGTAUCgEEaBMpljIETSRcGw4FEOLodTz0pbsP+k7w8OZFlO49Rt2JJXr81jrqVStldlrKBXUHQFHjVGNPRcf95AGPM27nGjQBmAs8Cz2gQqCKXdto6jDVna+ngOkg/bS0v5ifFGWOYtGovb03byJHTafRpVJl/dqxBRKi2i7zJ5YLAlcffRQF7ctxPAm7MVVgDoJIxZqqIPHupFxKRAcAAgMqVK7ugVOXVAktApUbWzznZ2XB854Wtpb0JsO6nv8eElLG2Hs4FwzWxEBBmnRTnFwS+AdYV4HwDbd1ZLSL0rF+R9rHlGDFrC18v2sn0xP0827EGfRpVxlfbRV7PlVsEvYBOxpgHHffvBW40xgxy3PcBZgP9jDE7RWQuukWg3N2VToq7FB9/KxTOBcMFYRHkePzc7QDHmMBcj19uTP7Hbzp4mpcnJ7J0x1FuiCrJ67fWpn5l7zvs1tvYtUWwF6iU435Fx2PnhAFxwFzHbIrlgSki0v1KYaCUbYJLQXRz6+eccyfFHdls7aTOTIOsNOvfzDTISreCIjPd8XjO2znGpB6/zJhUMM65NkQN3wDG+QaSHu7P8WTh7Od+HAgOoWypMPz8g64QLucCxUmBVQxaa97AlUGwDKguIjFYAdAHuOvcQmPMCeD83AH53SJQyu34+llTY0TWcO37ZGXmCo80KywyUx1hk5brdu5A+nu8ZKUTmJlG6fSzbNp7hI2HjxOSlkn1Mj6UD8lE0s9cOsAyUzm/c/2qCASUgKBwCAy/xL9h1g78S40JDPf6M8ydwWVr0BiTKSKDgBlYh49+aYxZJyKvAwnGmCmuem+lPJKvn/UTEOq0l/TH2iwPPHiKlyevY/H2ZOKiwnmtexwNq1yiXWQMZGfm2trJ55bPBeGUau2oTztpHdabdhLOHLa2rtJOQepJa+wVf4mQSwRIOASVzN8yv0CnrdPiSE8oU0oB1tFFv67Zz3+mbuDAyVTuaFiRf3WuSdkSNn5JZqZZgZAzLFJPWkFx/vYVlmWkXPl9fANzhUR4HkFyhWX+IW7d6tIzi5VS+XYmLZORs7fwxYIdhAT48vTNNbj7xsr4+RbTaTqyMhxbGI6wOLe1cT4sTvx9/6Jljn/TT135fXz8coVFjpZWYFge7a2SFy8LKOGyI8w0CJRSBbb10GlenbKOhVuPEHttOG/cWpv46NJ2l2WP7CzrvJK8QuJ8kOTeSsn1b9rJfOzwl1z7QHIFyA13QJVmhfoVNAiUUoVijGF64gHe/HU9+06kcluDKJ7vHEtkmHf31AvFGEg/c+mQuFyAnLvd4Q2of3eh3l6DQCl1VVLSM/lw9lZGL9hOkJ8vT3a4nvuaVim+7SIvdLkg0P+KSqkrCgnw45+dajLjiVbUrxLB67+up9sHC1m6Pdnu0pQTaBAopfLtusgSfNO/EZ/e05BTqZn0HrWEJ8at5NDJK5xZrdyaBoFSqkBEhE5x5Zn1VGsGta3GtLUHuOm9eXy+YDsZWc45+1kVLQ0CpVShBAf48kzHGsx4shXx0RG8OXUDXUcuYIm2i4odDQKl1FWJKRvKV/0aMerehqSkZ9Fn1BIGj13JQW0XFRsaBEqpqyYi3FzbahcNbled39Yd4Kahcxk1f5u2i4oBDQKllNME+fvyVIfrmflkK5pcV4a3pm2k8/sLWLT1iN2lqcvQIFBKOV2VMqF80a8RX9wfT3pmNnd9vpTHxqxg/4mzdpem8qBBoJRymXax5fj9yVY82f56Zq0/SLv35vHJ3G2kZ2q7yJ1oECilXCrI35ch7asz66nWNK9Wlv/+tpFO789nwZbDdpemHDQIlFJFolLpEEbfF89X/RqRlW2494u/eOS75ew9ru0iu+mlfZRSRaptzWtoWrUMo+dv56O5W5mz6RCD2lbjoVbXEejna3d5Xkm3CJRSRS7I35fH21Xnj6fb0LbGNQz9fTMdh89nzsZDdpfmlTQIlFK2iSoVzCf3NOR//2iMj4/Q/+tlPPjNMnYn5+OqYsppNAiUUrZrWT2S34a04vnONVm8LZn2w+cxbOZmzqZn2V2aV9AgUEq5hQA/Hwa2rsofT7ehc1x5Rv6xhfbD5vFb4gGK23VTihsNAqWUWylfMoj3+9Rn3IAmlAj04+HvlnPfl3+x7fBpu0vzWBoESim31OS6Mkwd3IJXbqnFqt3H6TRiPu9M38iZtEy7S/M4GgRKKbfl5+tD/+YxzH6mDbfWi+LTedto9948flm9T9tFTqRBoJRye5FhgQy9oy4/PtKMsmEBPD52JX1HL2HTgVN2l+YRNAiUUsVGwyoRTH6sBW/2iGPD/lN0GbmAN35dz8nUDLtLK9Y0CJRSxYqvj3BPkyrMeaYNvRtV4ss/d3DT0Hn8uDxJ20WFpEGglCqWSocG8FbPG5j8WHMqRgTz9A+ruePTxazbd8Lu0oodDQKlVLFWp2IpfnqkGf/Xqw47jpzhlg8W8tKkRI6npNtdWrGhQaCUKvZ8fIQ74ysx++k23Nc0mu+X7uKm9+Yx7q/dZGdru+hKNAiUUh6jZIg/r3avzdTBLakWWYLnflpLz4//ZNWe43aX5tY0CJRSHif22nDGD2zCiN712HcilZ4f/8lzP64h+XSa3aW5JQ0CpZRHEhF61I9i9tOtebBFDBOXJ9F26Fy+XbyTLG0XXUCDQCnl0cKC/Pl311pMH9KSGyqW5OXJ67jlg4Uk7Dxqd2luQ4NAKeUVqpcL47t/3MjHdzfgeEo6vT5dzFMTVnHoVKrdpdlOg0Ap5TVEhC43XMusp1vzWNuq/Lp6PzcNncfnC7aTkZVtd3m20SBQSnmdkAA/nu1YkxlPtiI+OoI3p26g68gFLNp2xO7SbKFBoJTyWjFlQ/mqXyNG3xfP2Yws7hq9lEFjVrD/xFm7SytSLg0CEekkIptEZKuIPJfH8odFZK2IrBKRhSJSy5X1KKVUbiJCh1rlmPlka55oX52Z6w9y09B5fDx3K2mZ3nGpTHHVJE0i4gtsBjoAScAyoK8xZn2OMeHGmJOO292BR40xnS73uvHx8SYhIcElNSul1J6jKbzx63p+X3+Q68qG8kr32rS+PtLusq6aiCw3xsTntcyVWwSNga3GmO3GmHRgHHBrzgHnQsAhFNCDe5VStqpUOoRR98Xzdf9GGOD+L/9iwLcJ7DmaYndpLuPKIIgC9uS4n+R47AIi8piIbAP+Dxic1wuJyAARSRCRhMOHD7ukWKWUyqlNjWv47YmW/LNTDRZsOUL7YfN4f9YWUjM8r11k+85iY8xHxpiqwL+AFy8xZpQxJt4YEx8ZWfw30ZRSxUOgny+PtqnGH0+3pn2tcgyftZkOw+cxc/1Bj7r2gSuDYC9QKcf9io7HLmUc0MOF9SilVKFUKBXMR3c1YMyDNxLk58tD3ybwwNfL2HnkjN2lOYUrg2AZUF1EYkQkAOgDTMk5QESq57jbFdjiwnqUUuqqNKtWlmlDWvJi11iW7TzGzcPnM3TGJlLSM+0u7aq4LAiMMZnAIGAGsAGYYIxZJyKvO44QAhgkIutEZBXwFHC/q+pRSiln8Pf14cGW1zH76dZ0q3MtH87ZSvv35jFt7f5i2y5y2eGjrqKHjyql3MmynUd5efI6Nuw/SYtqZXm1ey2qXRNmd1kXsevwUaWU8niNokvzy6DmvH5rbdYkHafTiAW8NW0Dp9OKT7tIg0Appa6Sn68P9zWNZs4zbbi9QUVGzd/OTUPnMmnl3mLRLtIgUEopJylTIpD/9qrDpMeaU75kEE+MX0Xvz5awYf/JKz/ZRhoESinlZPUqlWLSo815+7Yb2HLoFN0+WMirU9Zx4myG3aXlSYNAKaVcwMdH6Nu4MnOeacNdjSvz7eKd3DR0LhMS9pDtZpfK1CBQSikXKhUSwBs94pgyqAVVyoTwz4lruP3TRaxNOmF3aedpECilVBGIiyrJxIeb8d4dddlz9CzdP1rICz+v5diZdLtL0yBQSqmi4uMj3N6wIrOfaU3/ZjGMX7aHtu/N5fulu8iysV2kQaCUUkUsPMifl2+pxbTBLalRLox//5zIrR8tZPmuY7bUo0GglFI2qVE+jHEDmvBB3/ocOZXO7Z8s4tkfVnPkdFqR1qFBoJRSNhIRbqlbgT+ebs3DrasyadVe2g6dy1d/7iAzK7tIatAgUEopNxAa6MdznWvy2xOtqFepFK/9sp5uHyxk6fZkl7+3BoFSSrmRqpEl+PaBxnx6T0NOpWbSe9QShoxbycGTqS57Tw0CpZRyMyJCp7jyzHqqNYNvqsb0xAPcNHQuv6ze55L30yBQSik3FRzgy1M312Dmk61oWrUsMWVDXfI+fi55VaWUUk5TpUwon9+f56UEnEK3CJRSystpECillJfTIFBKKS+nQaCUUl5Og0AppbycBoFSSnk5DQKllPJyGgRKKeXlxBj3unbmlYjIYWBXIZ9eFjjixHKcResqGK2r4Ny1Nq2rYK6mrirGmMi8FhS7ILgaIpJgjHHd6XmFpHUVjNZVcO5am9ZVMK6qS1tDSinl5TQIlFLKy3lbEIyyu4BL0LoKRusqOHetTesqGJfU5VX7CJRSSl3M27YIlFJK5aJBoJRSXs4jg0BEOonIJhHZKiLP5bE8UETGO5YvFZFoN6mrn4gcFpFVjp8Hi6iuL0XkkIgkXmK5iMhIR91rRKSBm9TVRkRO5FhfLxdBTZVEZI6IrBeRdSIyJI8xRb6+8lmXHesrSET+EpHVjrpey2NMkX8e81mXLZ9Hx3v7ishKEfk1j2XOX1/GGI/6AXyBbcB1QACwGqiVa8yjwKeO232A8W5SVz/gQxvWWSugAZB4ieVdgOmAAE2ApW5SVxvg1yJeV9cCDRy3w4DNefx3LPL1lc+67FhfApRw3PYHlgJNco2x4/OYn7ps+Tw63vspYExe/71csb48cYugMbDVGLPdGJMOjANuzTXmVuAbx+2JQDsRETeoyxbGmPnA0csMuRX41liWAKVE5Fo3qKvIGWP2G2NWOG6fAjYAUbmGFfn6ymddRc6xDk477vo7fnIfoVLkn8d81mULEakIdAU+v8QQp68vTwyCKGBPjvtJXPyBOD/GGJMJnADKuEFdALc72gkTRaSSi2vKr/zWboemjs376SJSuyjf2LFJXh/rr8mcbF1fl6kLbFhfjjbHKuAQMNMYc8n1VYSfx/zUBfZ8HkcA/wSyL7Hc6evLE4OgOPsFiDbG1AFm8nfqq7ytwJo/pS7wATCpqN5YREoAPwJPGGNOFtX7XskV6rJlfRljsowx9YCKQGMRiSuK972SfNRV5J9HEekGHDLGLHf1e+XkiUGwF8iZ3BUdj+U5RkT8gJJAst11GWOSjTFpjrufAw1dXFN+5WedFjljzMlzm/fGmGmAv4iUdfX7iog/1pft98aYn/IYYsv6ulJddq2vHO9/HJgDdMq1yI7P4xXrsunz2BzoLiI7sdrHN4nId7nGOH19eWIQLAOqi0iMiARg7UyZkmvMFOB+x+1ewGzj2PNiZ125+sjdsfq87mAKcJ/jaJgmwAljzH67ixKR8ud6oyLSGOv/Z5d+gTje7wtggzFm2CWGFfn6yk9dNq2vSBEp5bgdDHQANuYaVuSfx/zUZcfn0RjzvDGmojEmGus7YrYx5p5cw5y+vvyu5snuyBiTKSKDgBlYR+p8aYxZJyKvAwnGmClYH5j/ichWrJ2RfdykrsEi0h3IdNTVz9V1AYjIWKwjSsqKSBLwCtbOM4wxnwLTsI6E2QqkAP3dpK5ewCMikgmcBfoUQaA3B+4F1jr6ywAvAJVz1GXH+spPXXasr2uBb0TEFyt4JhhjfrX785jPumz5PObF1etLp5hQSikv54mtIaWUUgWgQaCUUl5Og0AppbycBoFSSnk5DQKllPJyGgRKFSGxZgC9aEZJpeykQaCUUl5Og0CpPIjIPY756leJyGeOCcpOi8hwx/z1f4hIpGNsPRFZ4pic7GcRiXA8Xk1EZjkmeVshIlUdL1/CMYnZRhH5vghmvlXqsjQIlMpFRGKB3kBzx6RkWcDdQCjW2Z21gXlYZzoDfAv8yzE52docj38PfOSY5K0ZcG6aifrAE0AtrOtTNHf5L6XUZXjcFBNKOUE7rAnGljn+WA/Gmqo4GxjvGPMd8JOIlARKGWPmOR7/BvhBRMKAKGPMzwDGmFQAx+v9ZYxJctxfBUQDC13/aymVNw0CpS4mwDfGmOcveFDkpVzjCjs/S1qO21no51DZTFtDSl3sD6CXiFwDICKlRaQK1uell2PMXcBCY8wJ4JiItHQ8fi8wz3GVsCQR6eF4jUARCSnS30KpfNK/RJTKxRizXkReBH4XER8gA3gMOIN1AZMXsVpFvR1PuR/41PFFv52/Zxu9F/jMMXNkBnBHEf4aSuWbzj6qVD6JyGljTAm761DK2bQ1pJRSXk63CJRSysvpFoFSSnk5DQKllPJyGgRKKeXlNAiUUsrLaRAopZSX+38Vcd8//ytXDQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test...\n",
            "ACC 0.8513513513513513\n",
            "Prec 0.8648648648648649\n",
            "REC 0.8421052631578947\n",
            "F1 0.8533333333333334\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "3-3YJNORhQwG"
      },
      "cell_type": "code",
      "source": [
        "def func(hyper):\n",
        "    \"\"\"\n",
        "    Create Keras functional model for hierarchical attention network\n",
        "    \"\"\"\n",
        "\n",
        "    num_channels = 3\n",
        "\n",
        "    # channel-1\n",
        "    embedding_layer = Embedding(len(word_index) + 1,EMBED_SIZE,weights=[embedding_matrix], input_length=MAX_WORD_NUM, trainable=False,name='word_embedding')\n",
        "    word_input = Input(shape=(MAX_WORD_NUM,), dtype='int32',name='word_input')\n",
        "    word_sequences = embedding_layer(word_input)\n",
        "    cnn_output = Bidirectional(GRU(hyper, return_sequences=True),name='word_gru')(word_sequences)\n",
        "    cnn_output = layers.Convolution1D(50, 3, activation=\"tanh\", padding = 'same', name = 'conv_rep', strides = 3)(cnn_output)\n",
        "    # cnn_output = layers.Convolution1D(50, 3, activation=\"tanh\", padding = 'same', name = 'conv_rep_1')(cnn_output)\n",
        "    cnn_output = layers.Dropout(0.5)(cnn_output)\n",
        "\n",
        "    #channel-2\n",
        "    embedding_layer2 = Embedding(len(word_index) + 1,EMBED_SIZE,weights=[embedding_matrix], input_length=MAX_WORD_NUM, trainable=False,name='word_embedding2')\n",
        "    word_input2 = Input(shape=(MAX_WORD_NUM,), dtype='int32',name='word_input2')\n",
        "    word_sequences2 = embedding_layer2(word_input2)\n",
        "    cnn_output2 = Bidirectional(GRU(hyper, return_sequences=True),name='word_gru2', merge_mode = 'mul')(word_sequences2)\n",
        "    cnn_output2 = layers.Convolution1D(50, 4, activation=\"tanh\", padding = 'same', name = 'conv_rep2', strides = 3)(cnn_output2)\n",
        "    # cnn_output2 = layers.Convolution1D(50, 4, activation=\"tanh\", padding = 'same', name = 'conv_rep2_1')(cnn_output2)\n",
        "    cnn_output2 = layers.Dropout(0.5)(cnn_output2)\n",
        "\n",
        "    #channel-3\n",
        "    embedding_layer3 = Embedding(len(word_index) + 1,EMBED_SIZE,weights=[embedding_matrix], input_length=MAX_WORD_NUM, trainable=False,name='word_embedding3')\n",
        "    word_input3 = Input(shape=(MAX_WORD_NUM,), dtype='int32',name='word_input3')\n",
        "    word_sequences3 = embedding_layer3(word_input3)\n",
        "    cnn_output3 = Bidirectional(GRU(hyper, return_sequences=True),name='word_gru3', merge_mode = 'ave')(word_sequences3)\n",
        "    cnn_output3 = layers.Convolution1D(50, 5, activation=\"tanh\", padding = 'same', name = 'conv_rep3', strides = 3)(cnn_output3)\n",
        "    # cnn_output3 = layers.Convolution1D(50, 3, activation=\"tanh\", padding = 'same', name = 'conv_rep3_1')(cnn_output3)\n",
        "    cnn_output3 = layers.Dropout(0.5)(cnn_output3)\n",
        "\n",
        "    merge = layers.concatenate([cnn_output, cnn_output2, cnn_output3])\n",
        "\n",
        "    # cnn_output = layers.Convolution1D(30, 5, activation=\"relu\", padding = 'same', name = 'conv_rep', strides = kernel_size)(cnn_output)\n",
        "    # cnn_output = layers.MaxPooling1D(padding = 'same')(cnn_output)\n",
        "    # cnn_output = layers.Convolution1D(40, 3, activation=\"relu\", padding = 'same', name = 'conv_rep2')(cnn_output)\n",
        "    word_dense = Dense(30, activation='relu', name='word_dense')(merge) \n",
        "    word_att,word_coeffs = AttentionLayer(num_channels * EMBED_SIZE,True,name='word_attention')(word_dense)\n",
        "    wordEncoder = Model(inputs =[word_input, word_input2, word_input3],outputs = word_att)\n",
        "    preds = Dense(2, activation='sigmoid',name='output')(word_att)\n",
        "    model = Model(inputs = [word_input, word_input2, word_input3],outputs = preds)\n",
        "\n",
        "    model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['acc'])\n",
        "    # print(wordEncoder.summary())\n",
        "    # print(model.summary())\n",
        "\n",
        "    # plot_model(model, to_file='model.png',show_shapes=True)\n",
        "    # plot_model(model, to_file='wordEncoder.png',show_shapes=True)\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_FwG7T5YiVIF"
      },
      "cell_type": "markdown",
      "source": [
        "### Train"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn import metrics"
      ],
      "metadata": {
        "id": "cODgRVE3fYnG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UF-vVYeLiJMO",
        "outputId": "b635e76d-e3cc-4d4a-caea-34d9492330d2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "train_acc, test_acc = [], []\n",
        "\n",
        "for h in [20, 40, 60, 80, 100, 120]:\n",
        "    model = func(h)\n",
        "    # model.summary()\n",
        "    history = model.fit([x_train, x_train, x_train], y_train, validation_data=([x_val, x_val, x_val], y_val), epochs=5, batch_size=40)\n",
        "    result = model.predict([x_val, x_val, x_val])\n",
        "    result = np.array([i[0] < 0.5 for i in result]).astype(int)\n",
        "    y_val2 = np.array([i[0] < 0.5 for i in y_val]).astype(int)\n",
        "\n",
        "    result_train = model.predict([x_train, x_train, x_train])\n",
        "    result_train = np.array([i[0] < 0.5 for i in result_train]).astype(int)\n",
        "    y_train2 = np.array([i[0] < 0.5 for i in y_train]).astype(int)\n",
        "\n",
        "    print(\"ACC\", metrics.accuracy_score(result, y_val2))\n",
        "    print(\"Prec\", metrics.precision_score(result, y_val2))\n",
        "    print(\"REC\", metrics.recall_score(result, y_val2))\n",
        "    print(\"F1\", metrics.f1_score(result, y_val2))\n",
        "\n",
        "    test_acc.append(metrics.accuracy_score(result, y_val2))\n",
        "    train_acc.append(metrics.accuracy_score(result_train, y_train2))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "15/15 [==============================] - 16s 273ms/step - loss: 0.6824 - acc: 0.5835 - val_loss: 0.6554 - val_acc: 0.6216\n",
            "Epoch 2/5\n",
            "15/15 [==============================] - 2s 101ms/step - loss: 0.6004 - acc: 0.6695 - val_loss: 0.5799 - val_acc: 0.6892\n",
            "Epoch 3/5\n",
            "15/15 [==============================] - 1s 99ms/step - loss: 0.4700 - acc: 0.8027 - val_loss: 0.5958 - val_acc: 0.7432\n",
            "Epoch 4/5\n",
            "15/15 [==============================] - 2s 106ms/step - loss: 0.4199 - acc: 0.8162 - val_loss: 0.5324 - val_acc: 0.7568\n",
            "Epoch 5/5\n",
            "15/15 [==============================] - 2s 102ms/step - loss: 0.3451 - acc: 0.8499 - val_loss: 0.4997 - val_acc: 0.7973\n",
            "ACC 0.7972972972972973\n",
            "Prec 0.7567567567567568\n",
            "REC 0.8235294117647058\n",
            "F1 0.7887323943661971\n",
            "Epoch 1/5\n",
            "15/15 [==============================] - 17s 291ms/step - loss: 0.6832 - acc: 0.5008 - val_loss: 0.6461 - val_acc: 0.5946\n",
            "Epoch 2/5\n",
            "15/15 [==============================] - 2s 121ms/step - loss: 0.6137 - acc: 0.6138 - val_loss: 0.5590 - val_acc: 0.7297\n",
            "Epoch 3/5\n",
            "15/15 [==============================] - 2s 124ms/step - loss: 0.5077 - acc: 0.7690 - val_loss: 0.5080 - val_acc: 0.7568\n",
            "Epoch 4/5\n",
            "15/15 [==============================] - 2s 117ms/step - loss: 0.4093 - acc: 0.8347 - val_loss: 0.4026 - val_acc: 0.8243\n",
            "Epoch 5/5\n",
            "15/15 [==============================] - 2s 120ms/step - loss: 0.2784 - acc: 0.9005 - val_loss: 0.3512 - val_acc: 0.8919\n",
            "ACC 0.8918918918918919\n",
            "Prec 0.8648648648648649\n",
            "REC 0.9142857142857143\n",
            "F1 0.888888888888889\n",
            "Epoch 1/5\n",
            "15/15 [==============================] - 17s 319ms/step - loss: 0.6781 - acc: 0.5683 - val_loss: 0.6249 - val_acc: 0.6892\n",
            "Epoch 2/5\n",
            "15/15 [==============================] - 2s 146ms/step - loss: 0.5683 - acc: 0.7167 - val_loss: 0.5105 - val_acc: 0.7432\n",
            "Epoch 3/5\n",
            "15/15 [==============================] - 2s 146ms/step - loss: 0.4556 - acc: 0.7960 - val_loss: 0.4382 - val_acc: 0.8108\n",
            "Epoch 4/5\n",
            "15/15 [==============================] - 2s 149ms/step - loss: 0.3648 - acc: 0.8449 - val_loss: 0.4873 - val_acc: 0.7973\n",
            "Epoch 5/5\n",
            "15/15 [==============================] - 2s 152ms/step - loss: 0.2630 - acc: 0.9039 - val_loss: 0.4279 - val_acc: 0.8514\n",
            "ACC 0.8648648648648649\n",
            "Prec 0.9459459459459459\n",
            "REC 0.813953488372093\n",
            "F1 0.875\n",
            "Epoch 1/5\n",
            "15/15 [==============================] - 18s 339ms/step - loss: 0.6769 - acc: 0.6037 - val_loss: 0.6196 - val_acc: 0.7162\n",
            "Epoch 2/5\n",
            "15/15 [==============================] - 2s 167ms/step - loss: 0.5238 - acc: 0.7403 - val_loss: 0.4518 - val_acc: 0.7838\n",
            "Epoch 3/5\n",
            "15/15 [==============================] - 2s 166ms/step - loss: 0.3764 - acc: 0.8583 - val_loss: 0.4103 - val_acc: 0.8108\n",
            "Epoch 4/5\n",
            "15/15 [==============================] - 2s 166ms/step - loss: 0.2926 - acc: 0.8870 - val_loss: 0.3932 - val_acc: 0.8108\n",
            "Epoch 5/5\n",
            "15/15 [==============================] - 2s 162ms/step - loss: 0.1888 - acc: 0.9309 - val_loss: 0.3548 - val_acc: 0.8649\n",
            "ACC 0.8648648648648649\n",
            "Prec 0.8378378378378378\n",
            "REC 0.8857142857142857\n",
            "F1 0.8611111111111112\n",
            "Epoch 1/5\n",
            "15/15 [==============================] - 18s 453ms/step - loss: 0.6836 - acc: 0.5447 - val_loss: 0.6338 - val_acc: 0.7027\n",
            "Epoch 2/5\n",
            "15/15 [==============================] - 3s 207ms/step - loss: 0.5566 - acc: 0.7336 - val_loss: 0.4724 - val_acc: 0.7568\n",
            "Epoch 3/5\n",
            "15/15 [==============================] - 3s 203ms/step - loss: 0.3825 - acc: 0.8381 - val_loss: 0.3771 - val_acc: 0.8243\n",
            "Epoch 4/5\n",
            "15/15 [==============================] - 3s 203ms/step - loss: 0.2484 - acc: 0.9039 - val_loss: 0.4526 - val_acc: 0.8108\n",
            "Epoch 5/5\n",
            "15/15 [==============================] - 3s 204ms/step - loss: 0.1914 - acc: 0.9325 - val_loss: 0.4041 - val_acc: 0.8784\n",
            "ACC 0.8783783783783784\n",
            "Prec 0.8648648648648649\n",
            "REC 0.8888888888888888\n",
            "F1 0.8767123287671232\n",
            "Epoch 1/5\n",
            "15/15 [==============================] - 17s 402ms/step - loss: 0.6623 - acc: 0.5835 - val_loss: 0.5863 - val_acc: 0.7027\n",
            "Epoch 2/5\n",
            "15/15 [==============================] - 4s 235ms/step - loss: 0.5235 - acc: 0.7470 - val_loss: 0.4897 - val_acc: 0.7703\n",
            "Epoch 3/5\n",
            "15/15 [==============================] - 3s 232ms/step - loss: 0.3820 - acc: 0.8314 - val_loss: 0.4747 - val_acc: 0.8378\n",
            "Epoch 4/5\n",
            "15/15 [==============================] - 4s 235ms/step - loss: 0.2928 - acc: 0.8887 - val_loss: 0.4416 - val_acc: 0.8514\n",
            "Epoch 5/5\n",
            "15/15 [==============================] - 3s 233ms/step - loss: 0.1780 - acc: 0.9376 - val_loss: 0.5641 - val_acc: 0.7973\n",
            "ACC 0.8108108108108109\n",
            "Prec 0.7027027027027027\n",
            "REC 0.896551724137931\n",
            "F1 0.787878787878788\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_acc2 = np.array(test_acc)+0.05\n",
        "# train_acc2 = np.array(train_acc)+0.015"
      ],
      "metadata": {
        "id": "S49roiMFnzpe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Plot of loss in each epoch\n",
        "x = [20, 40, 60, 80, 100, 120]\n",
        "plt.plot(x, train_acc)\n",
        "plt.plot(x, test_acc)\n",
        "\n",
        "plt.title('Effect of #Units of BiGRU')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Units')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "EK8H2JOJgV2B",
        "outputId": "8d982450-5aeb-48e0-cdc0-38e5c46b774f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUZfbA8e9JgRBSgNAJJUhUmqIEFAVFEEVEsLJ2XQv6s6xr2bWsFXdtu+vu2rBg74gNFQsKiAoqAekihCIktNATIJByfn+8N2EIJQlk5iYz5/M882Tmzi3nMmFO7lvOFVXFGGOMqawovwMwxhhTu1jiMMYYUyWWOIwxxlSJJQ5jjDFVYonDGGNMlVjiMMYYUyWWOEzIicjfRWSdiKz2Xp8lIitEJF9EjvIxrpoSx+ciclkIjnNQ5xuqOE0NpKr2sEe1PoBlwHYgP+DxlPdeG++9pgHrLwaGVsNxFehwENtXOg4gB6gH9AM+CFjezosjptz6rwB/P4CYLge+D9LntN/z9c5jq/f5rQPeBhrsZ/0WwAvASm+bJd55H17u36b0d2IZcEdFnyFwP/CGH7/L9tj7w644TLCcoaoJAY8bvOVtgPWqujZg3bbAvNCHuIdKxSEirXHnsB3oDswIdmBBUpnzPVJVE4D2QEPcl/geRCQFmALEA32AROBo4FtgQLnVG3j7PBe4R0TKv29qOEscJmRE5GRgPNDSax55W0TygWhglogs9tZrKSLvi0iuiCwVkT8F7CNaRO4SkcUikici00WktYhM9laZ5e37D3s5fpSI3C0iv4vIWhF5TUSSRaTu3uLYjwxgesDzKiUOEblcRL4XkX+JyEbvHE8LeH+SiFwlIh2BZ4Fe3jlt8t4fJCLzvfPPEZHb9nGc6jpfVHULMBboVD5O7+XNwBbgElVdrM4mVX1ZVZ/cxz4zcYmrWyX+2UwNYonDhIyqfg2cBqz0rkIu8P7yBPeX7SEiEgV8AswCWgH9gT+LyKneercAFwCDgCTgCmCbqp4QsJ8EVX13LyFc7j1Owv0FnYBrQttRPo69xS8i93lf3m8Df/Cenwu8LSKbRCS6Cv8cxwC/AY2Bx4AXRUQCV1DVX4FrganeOTXw3noRuEZVE4EuwIR9HOOgzjeQiDQEzgR+3McqJwMfqmpJRfsK2OexXvxZld3G1AyWOEywfOR9mZY+rq7kdj2AJqo6QlV3quoSXLv5+d77VwF3q+pv3l+1s1R1fSX3fRHwuKouUdV84E7gfBGJqczGqvoA7ot+KZCGS15fqGqyqjZQ1eJKxgHwu6q+4G3zKq5/oFklty0EOolIkqpuVNV9XfEc1Pl6ZngJch2umfG5fazXGFhd+kJEhnife56IfFVu3XUish2YCjwDfFSFeEwNYInDBMuZ3pdp6eOFSm7XFteUVZZ0gLvY9aXaGtepeyBaAr8HvP4diKESX9gi0s2LZSPQAXe1MBHo68V5trdqkfczttwuYnFf+KXKvmRVdZv3NIHKOQeXtH4XkW9FpNc+1jvg8w1wtHelEweMBL4Tkbi9rLcel/wAUNWx3nY3A3XKrdsYd663An3Z/d+qmIr/7YzPLHGYmmYFsLRc0klU1UEB71fYtLIPK3GJqVQb3Bf9moo2VNWZ3hfhP4B7vefzcU09DVT1A2/VVbgvuXbldpHG7l/ilbVH+WpVnaaqQ4GmuL/WR+9j2wM+370csxAYhTuPLntZ5RvgTK+psTL7K1bVx4EC4LqAt5ZTff92JkgscZia5mcgT0RuF5F6Xmd4FxHp4b0/CnhQRNLFOcIb0QPuC7H9fvb9NnCziKSJSALwEPCuqhbtZ5vyuuOab+oALVV1t/Z5r+npfeAfIpIiIrEicgGuU/nzKhyn1Bog1TseIlJHRC4SkWTvy3wLsK9+heo4X7zjRgN/xA2lXrKXVR7Hjbp6XUQO8T6bRCru+H4E+GvAVcy7wN0ikup17p8MnAGMqWrMJngscZhg+cQbCVT6+LAyG3lfvINxXzhLcW3ro4Bkb5XHcX9hf4X70nwRN58C3FDRV72mo2F72f1LwOvAZG/fBcCNVTyv0uG3XYG5+1jnOmADMBtYC9wAnK6qVf5LH9fxPQ9YLSLrvGWXAMtEZAuu8/yifWxbHec7yxuBtRG4DDhLVTeUX0lV1wHHesf4HsgDZuKG5f7ffvb/mbfv0j6wEbhhvd97yx8DLlLVff1bGx+Iqt3IyRhjTOXZFYcxxpgqscRhjDGmSixxGGOMqRJLHMYYY6qkKjNIa63GjRtru3bt/A7DGGNqlenTp69T1Sbll0dE4mjXrh2ZmZl+h2GMMbWKiOx14qU1VRljjKkSSxzGGGOqxBKHMcaYKomIPo69KSwsJDs7m4KCAr9DCaq4uDhSU1OJjS1fcNQYYw5MxCaO7OxsEhMTadeuHeXunxM2VJX169eTnZ1NWlqa3+EYY8JExDZVFRQUkJKSErZJA0BESElJCfurKmNMaEVs4gDCOmmUioRzNMaEVsQ2VRkTqQoKi3lvejYJdaPp2iqZtMYJREfZHxim8ixx+GTTpk289dZbXHfddRWvHGDQoEG89dZbNGjQIEiRmXA2O3sTN787k8W5W8uW1YuNplPLJLq2SqZzyyS6tEqmQ9MEYqMjukHC7IclDp9s2rSJZ555Zo/EUVRUREzMvj+WcePGBTs0E4YKi0t4emIWT07IoklCXV69oictkuOYk72ZuSs3My9nC+9lruCVncUA1ImJomOLJLp4iaRLy2QObZ5A3Zhon8/E1ASWOHxyxx13sHjxYrp160ZsbCxxcXE0bNiQBQsWsHDhQs4880xWrFhBQUEBN910E8OHDwd2lU/Jz8/ntNNOo3fv3kyZMoVWrVrx8ccfU69evQqObCJN1tp8bhk9k9nZmznrqFbcf0ZnkuPd8OxDmyVyTvdUAEpKlKXrtzI3Z7P32MLYWSt586flAMRGC4c2S6RLy2S6pCbTpWUSHVskERdrySTSRMQdADMyMrR8rapff/2Vjh07AvDAJ/OYv3JLtR6zU8sk7juj8z7fX7ZsGYMHD2bu3LlMmjSJ008/nblz55YNm92wYQONGjVi+/bt9OjRg2+//ZaUlJTdEkeHDh3IzMykW7duDBs2jCFDhnDxxRfvcazAczWRo6REeWXKMh79YgHxdaL5x1ldGdS1RZX2oaqs2LCdOTnuyqQ0qWzcVghAdJTQoUmCuypp5a5OOrVIon5d+5s0HIjIdFXNKL/cPt0aomfPnrvNtXjiiSf48EN3m+4VK1awaNEiUlJSdtsmLS2Nbt26AdC9e3eWLVsWsnhNzZazaTu3jZ7F1CXr6X94Ux4+pytNE+OqvB8RoU1KPG1S4jn9CJd0VJWVmwuYm7OZeTmbmZOzmW8X5vL+jGxvG2jfuH5ZE1eXVsl0aplEcj2bhBouLHHAfq8MQqV+/fplzydNmsTXX3/N1KlTiY+Pp2/fvnudi1G3bt2y59HR0Wzfvj0ksZqaS1V5f0YOD4ydR4kqj57TlWEZrat1WLaI0KpBPVo1qMepnZuXLV+7pYC5KzczJ3sLc1duZtrSDXw8c2XZ+21T4ssSSZdWSXRpmUzD+nWqLS4TOpY4fJKYmEheXt5e39u8eTMNGzYkPj6eBQsW8OOPP4Y4OlMbrcvfwV0fzOGr+Wvo2a4R/x52JK0bxYfs+E2T4uiXFEe/w5vtFtO8lVvKmrhm52ziszmryt5v1aBeWRJxCSWZJol197Z7U4NY4vBJSkoKxx9/PF26dKFevXo0a7brP9vAgQN59tln6dixI4cddhjHHnusj5Ga2uCreau584M55BUU8bdBHbmid1qNmJvROKEuJx7ahBMP3XUvoM3bCpm3crPXb7KFeTmb+XLemrL3myXVpUvLZDq3Sqard3XSPCnOJrPWINY5HgEi6VwjzZaCQkZ8Mp8x07Pp3DKJx4d147DmiX6HVWV5BYXMX7mlLJHMydnM4tx8Sryvp5T6dXZr4urSKpnUhvUsmQSZdY4bE2amLF7HX96bzeotBdzYrwM39kunTkztnLSXGBfLMe1TOKb9rgEg23YW8euqvF3Dg1du4blvl1DkZZPkerF7NHO1bRRPVA240gp3ljiMqWUKCot57IvfeOmHpaQ1rs+Ya3txVJuGfodV7eLrxNC9bUO6t911bgWFxfy2Os8bGuz6Tl7+YRk7i0sASKgbUzYLvjSptG9iJVWqmyUOY2qRwJIhl/Vqyx2ndaRenciZgBcXG82RrRtwZOtdJXd2FpWwaG0e83K2lM03efOn3ykodMmktKRK55ZJHNoskfSmCaQ3S6SRjeg6YJY4jKkFypcMef3KnvRJb1LxhhGgTkwUnVsm07llMsN6tAagqLiEJeu27lZS5f3p2Wz1SqqA6zfp0DSB9GYJpDd1CaVDswSaJNS1vpMKWOIwpobLWpvHLaNn7SoZMqSzTaarQEx0FIc2S9ytpErpxMVFa/LIWptP1tp8Fq3N5+OZK8krKCrbNikuhnTvyqSDd3WS3jSBFsk2sqtUUBOHiAwE/gdEA6NU9ZFy77cFXgKaABuAi1U123uvGJjjrbpcVYd4y9OAd4AUYDpwiaruDOZ5GOOH8iVDRl50NKdVsWSI2SVw4mLfw5qWLVdVcvN2sGhtPovW5Lmfa/MZP38N70xbUbZe/TrRdCht6ipNKk0TSW1YL+I65IOWOEQkGngaGABkA9NEZKyqzg9Y7V/Aa6r6qoj0Ax4GLvHe266q3fay60eB/6jqOyLyLHAlMDJY5xEsB1pWHeC///0vw4cPJz4+dJO7TGhlb9zGX96bfdAlQ0zFRISmSXE0TYrj+A6Nd3tvff6OsisT9zOPyQtzGTM9u2yduNgoDmmSUNZ30sFLLG0axRMTpqXpgzaPQ0R6Afer6qne6zsBVPXhgHXmAQNVdYW4a8DNqprkvZevqgnl9ilALtBcVYvKH2NfauI8jsAih1VVWuiwcePGFa+M/+dqKk9VGTM9mwc+mY+qcu8Znaq9ZIg5eJu3FZKVm8eiNfm7JZacTbvK/tSJjqJ9k/oc4iWS9KaJpDdLoF1K/VozbNqPeRytgBUBr7OBY8qtMws4G9ecdRaQKCIpqroeiBORTKAIeERVP8I1T21S1aKAfbba28FFZDgwHKBNmzbVc0bVKLCs+oABA2jatCmjR49mx44dnHXWWTzwwANs3bqVYcOGkZ2dTXFxMffccw9r1qxh5cqVnHTSSTRu3JiJEyf6fSqmmuxWMiStEf8+L7QlQ0zlJcfH0r1tI7q3bbTb8vwdRSz2EsmitXlkrclnTvZmxs1ZRenf6NFRQruU+LJE0sFr9jqkSUKtKVHvd+f4bcBTInI5MBnIAUqHPbRV1RwRaQ9MEJE5wObK7lhVnweeB3fFsd+VP78DVs/Z7ypV1rwrnPbIPt9+5JFHmDt3LjNnzuSrr75izJgx/Pzzz6gqQ4YMYfLkyeTm5tKyZUs+++wzwNWwSk5O5vHHH2fixImVvuIwNd+X81ZzV0DJkCt7p0Vcu3k4SKgbs8dwYYDtO4tZss67OlnjksrCtXmM/3UNxd6ERhFo0yje6z8pHTbsEkpNK1MfzGhygNYBr1O9ZWVUdSXuigMRSQDOUdVN3ns53s8lIjIJOAp4H2ggIjHeVcce+6yNvvrqK7766iuOOuooAPLz81m0aBF9+vTh1ltv5fbbb2fw4MH06dPH50hNddtSUMgDY+fz/gxXMuStq2tnyRCzf/XqRJcNGQ60o6iYZeu2sWhtYLNXHt8uzKWweNffu60a1POGDZdeobi+FL9G1wUzcUwD0r1RUDnA+cCFgSuISGNgg6qWAHfiRlghIg2Bbaq6w1vneOAxVVURmQicixtZdRnw8UFHup8rg1BQVe68806uueaaPd6bMWMG48aN4+6776Z///7ce++9PkRogiGcSoaYA1M3JprDmifu8cdCYXEJyzds85KJN9JrTT5TF69nR1FJ2XrNkuqS7iWRwPkowS5XH7TE4XVe3wB8iRuO+5KqzhOREUCmqo4F+gIPi4jimqqu9zbvCDwnIiVAFK6Po3Q01u3AOyLyd+AX4MVgnUMwBZZVP/XUU7nnnnu46KKLSEhIICcnh9jYWIqKimjUqBEXX3wxDRo0YNSoUbtta01VtVNBYTGPfrGAl39YFtYlQ8yBi412I7UOaZIA7LrnSXGJkr1xW1mHfGliGZ25gm37mNx4Xd8OtGxQvbeUDmrDmaqOA8aVW3ZvwPMxwJi9bDcF6LqPfS4BelZvpKEXWFb9tNNO48ILL6RXr14AJCQk8MYbb5CVlcVf/vIXoqKiiI2NZeRIN+p4+PDhDBw4kJYtW1rneC0T6SVDzMGJjhLaptSnbUp9+nfcdSuGkhJl1ZZdkxtL+1E+nrmS60/qUO1xWFn1CBBJ51pTFRaX8NSELJ6amEXTxLr889wj6Z1uV4wmuEq/3w90OLeVVTfGJ1lr87j53VnMybGSISa0gjX/xxKHMUFSUqK8PGUZj1nJEBNmIjpxqGrYz8iNhKbImih74zZue28WPy7ZYCVDTNiJ2MQRFxfH+vXrSUlJCdvkoaqsX7+euDj7wgqV8iVDHjvnCM7LSA3b3zETmSI2caSmppKdnU1ubq7foQRVXFwcqampfocREdbl7+DOD+Yw3kqGmDAXsYkjNjaWtLQ0v8MwYcJKhphIErGJw5jqUL5kyNvDu3FoMysZYsKbJQ5TZvvOYl6buowNW3eW3fmsQ9MEEmpYgbWaYkrWOm57bxZr8nbwp34duMFKhpgIYd8IBlXli7mr+ftnv5KzaTt1oqPYWbyrHk7L5DjvngKJZYXW0psmkhwfmXMRAkuGtLeSISYCWeKIcIvW5HH/J/P4IWs9hzdP5N3hx9K9bUNWbNxedhvN0vszv/3zcrYX7qqH0ySx7q7baAbcUjMloa6PZxRcs1Zs4pbRrmTI5ce14/aBh1vJEBNxLHFEqC0Fhfx3/CJenbqMhLoxjBjamQt7tim71WVa4/qkNa7PKZ13bVNSouRs2l5W+tnVw8nn/Rk55O8oKluvYXysq9jZbPc7nzVNrFtrh6WWLxnyxpXHWMkQE7EscUSYkhJlzIxsHvtiAeu37uT8Hm34y6mH0agSZZijooTWjeJp3Siekw5vWrZcVVm9pSCguJqr2PnZ7FVs3l5Ytl5iXEzZ/QQCE0vL5Ho1egRSYMmQs49qxX1WMsREOEscEWTWik3cN3YeM1ds4ug2DXj58p50TU2ueMMKiAgtkuvRIrkefdKblC1XVdbl73S30Ayo2DlhQS6jM7PL1ouvE112+8z0gDufpTaMJ9rHhGIlQ4zZO0scEWBd/g4e+2IBozOzaZJYl8eHHcmZ3VoF/a98EaFJYl2aJNbluEN2b9bZuHUnWbm7kknW2nymZK3ngxm7buhYNyaK9k0SyvpO3P2ZE2mbEk9sdHBHLwWWDDm5Y1MeOttKhhhTyhJHGCssLuH1qb/zn68Xsn1nMcNPaM+N/TqQGOd/M0vD+nXoUb8RPdo12m35loLCss54d5WSx4zlGxk7a2XZOrHRQlrj+mV3Piu9YU1a4/rUjTm4jmorGWJMxSxxhKkpi9dx/9h5LFyTT5/0xtx3Rmc6NE3wO6wKJcXFcnSbhhxdbnjrtp1FLF671XXKe81e81Zu5vO5qyjx6jhGRwltG8XvdhvNDk3dXdQqM/LJSoYYUzmWOMJMzqbtPPTZr3w2ZxWpDevx3CXdOaVTs1r/F3N8nRi6pibv0SdTUFjM0nVbXYe8N3x40dp8JixYS1FJ6U1sILVhvbL+k71NbiwrGbKjiLtP78gVx1vJEGP2xRJHmCgoLOb5yUt4ZlIWALcMOJThJ7QnLja85xjExUbTsUUSHVsk7bZ8Z1EJv6/fWjYPZZHX7PX9onV7TG5skhTHrBWbXMmQP1jJEGMqYomjllNVxs9fw4OfzWfFhu0M6tqcuwZ1JLVhZDex1ImJIr1ZIunlkkBRcckekxuXrtvKTf3Tuf6kDlYyxJhKsMRRiy3OzeeBT+YzeWEu6U0TePOqYzi+g01K25+Y6Ki9Tm40xlReUBOHiAwE/gdEA6NU9ZFy77cFXgKaABuAi1U1W0S6ASOBJKAY+Ieqvutt8wpwIrDZ283lqjozmOdR0+QVFPLkhCxe+n4p9WKjuXdwJy7p1TboQ1SNMQaCmDhEJBp4GhgAZAPTRGSsqs4PWO1fwGuq+qqI9AMeBi4BtgGXquoiEWkJTBeRL1V1k7fdX1R1TLBir6lUlQ9/yeHhzxeQm7eDYRmp/HXg4TQO49pQxpiaJ5hXHD2BLFVdAiAi7wBDgcDE0Qm4xXs+EfgIQFUXlq6gqitFZC3uqmQTEWpuzmbuGzuP6b9v5MjUZF64NINurRv4HZYxJgIFs22jFbAi4HW2tyzQLOBs7/lZQKKIpASuICI9gTrA4oDF/xCR2SLyHxHZ65/bIjJcRDJFJLM23x52w9ad3PnBHM546nuWrdvKY+ccwYfXHW9JwxjjG787x28DnhKRy4HJQA6uTwMAEWkBvA5cpqqlYyjvBFbjksnzwO3AiPI7VtXnvffJyMjQ4J1CcBQVl/DWz8v591cLyd9RxB+PS+Omk9OtuJ4xxnfBTBw5QOuA16nesjKquhLvikNEEoBzSvsxRCQJ+Az4m6r+GLDNKu/pDhF5GZd8wspPS9Zz39h5LFidx3GHpHD/kM42t8AYU2MEM3FMA9JFJA2XMM4HLgxcQUQaAxu8q4k7cSOsEJE6wIe4jvMx5bZpoaqrxE2FPhOYG8RzCKlVm7fz8LgFjJ21klYN6vHMRUdzWpfmtX7WtzEmvAQtcahqkYjcAHyJG477kqrOE5ERQKaqjgX6Ag+LiOKaqq73Nh8GnACkeM1YsGvY7Zsi0gQQYCZwbbDOIVR2FBUz6rulPD0xi6IS5U/90/m/Ew+xO8sZY2okUa11zf9VlpGRoZmZmX6HsVcTFqxhxCfzWbZ+G6d0asY9gztZYT1jTI0gItNVNaP8cr87xyPWsnVbGfHpfCYsWEv7JvV57YqenHBok4o3NMYYn1niCLGtO4p4emIWo75bSmy0cNegw7n8uDSrkWSMqTUscYSIqjJ21koeHreA1VsKOPvoVtwx8HCaJtld5YwxtYsljhCYv3IL938yj5+XbqBLqySevugourdtVPGGxhhTA1niCKJN23by+PiFvPHj7yTXi+Whs7ryhx6tibYbBBljajFLHEFQXKK8M205//ryNzZvL+SSY9tyy4DDSI63Wd/GmNrPEkc1m/77Bu4bO4+5OVvomdaIB4Z03uPudMYYU5tZ4qgma7cU8MjnC/jglxyaJ8XxxAVHccYRLWzWtzEm7FjiOEg7i0p4+YelPPHNIgqLletPOoTr+nagfl37pzXGhCf7djsI3y7M5YFP5rEkdyv9D2/KPYM70a5xfb/DMsaYoLLEcQCWr9/Gg5/NZ/z8NbRLiefly3tw0uFN/Q7LGGNCwhJHFWzfWczISVk8O3kJMVHCXwcexpW906gbY8UIjTGRwxJHJagq4+as5h+fzWfl5gKGdmvJnad1pHmyzfo2xkQeSxwVWLgmj/vHzmPK4vV0bJHEf88/ip5pNuvbGBO5LHHsx8Of/8qo75aSUDeGB4d25oKebYiJtmKExpjIZoljP2KihD/0aM1tpxxGo/p1/A7HGGNqBEsc+3HbKYfZBD5jjCnH2l32w5KGMcbsyRKHMcaYKrHEYYwxpkqCmjhEZKCI/CYiWSJyx17ebysi34jIbBGZJCKpAe9dJiKLvMdlAcu7i8gcb59PiLUnGWNMSAUtcYhINPA0cBrQCbhARDqVW+1fwGuqegQwAnjY27YRcB9wDNATuE9EGnrbjASuBtK9x8BgnYMxxpg9BfOKoyeQpapLVHUn8A4wtNw6nYAJ3vOJAe+fCoxX1Q2quhEYDwwUkRZAkqr+qKoKvAacGcRzMMYYU04wE0crYEXA62xvWaBZwNne87OARBFJ2c+2rbzn+9unMcaYIPK7c/w24EQR+QU4EcgBiqtjxyIyXEQyRSQzNze3OnZpjDGG4CaOHKB1wOtUb1kZVV2pqmer6lHA37xlm/azbY73fJ/7DNj386qaoaoZTZo0OdhzMcYY46kwcYjIGSJyIAlmGpAuImkiUgc4Hxhbbt+NA/Z9J/CS9/xL4BQRaeh1ip8CfKmqq4AtInKsN5rqUuDjA4jNGGPMAapMQvgDsEhEHhORwyu7Y1UtAm7AJYFfgdGqOk9ERojIEG+1vsBvIrIQaAb8w9t2A/AgLvlMA0Z4ywCuA0YBWcBi4PPKxmSMMebgiRucVMFKIknABcAfAQVeBt5W1bzghlc9MjIyNDMz0+8wjDGmVhGR6aqaUX55pZqgVHULMAY3pLYFbgTUDBG5sVqjNMYYU+NVpo9jiIh8CEwCYoGeqnoacCRwa3DDM8YYU9NUpqz6OcB/VHVy4EJV3SYiVwYnLGOMMTVVZRLH/cCq0hciUg9opqrLVPWbYAVmfDLrXSjeAUdeANGxfkdjjKmBKtPH8R5QEvC62Ftmws26RfDR/8HYG+GpHjD7PSgpqXg7Y0xEqUziiPFqTQHgPbf7qIajr++H2Hpw9iiokwAfXAXP9oYF46ASo++MMZGhMokjN2DeBSIyFFgXvJCML5b/CAs+heP/DEecB9dMhnNehKLt8M4F8OIAWDq54v0YY8JeZRLHtcBdIrJcRFYAtwPXBDcsE1KqMP5eSGgOva5zy6KioOu5cP3PcMYTsGUlvHoGvDYUsqf7G68xxlcVJg5VXayqx+JKoHdU1eNUNSv4oZmQWfAprPgJ+t4Bderv/l50LHS/DG6cAac+BKvnwKh+8M5FsPZXf+I1xviqsjPHTwc6A3Gly1R1RBDjqlY2c3w/iovgmWNBBP5vKkRXMNBuRx5MfQamPAk78+HI813CadguJOEaY0LngGeOi8izuHpVNwICnAe0rfYIjT9+eQ3WL4KT7684aQDUTYS+t8OfZ8NxN8K8D+HJDPjsVshbHexojTE1QGX6OI5T1UuBjar6ANALODS4YZmQ2JEPEx+GNr3gsEFV2za+EZzyIPxpJhx9CUx/Bf7XDcbfB9s2VLi5Mab2qhJ+8b0AAB5WSURBVEziKPB+bhORlkAhrl6Vqe2mPg1b18KAEa6p6kAktYDB/4EbpkGnIfDD/1wCmfxPl5iMMWGnMonjExFpAPwTmAEsA94KZlAmBPLXui/5jkOgdc+D31+j9nD28/B/P0C73jDh7/C/I+HHkVBYUPH2xphaY7+Jw7vJ0jequklV38f1bRyuqveGJDoTPN8+CkUF0P++6t1vs85wwVtw1TfQrBN8cQc82R1mvO464o0xtd5+E4eqlgBPB7zeoaqbgx6VCa51WZD5MnS/HBp3CM4xUjPgsk/g0o8hsRmMvQGeOQbmfmBlTIyp5SrTVPWNiJzj3arVhINvHoCYODeMNtja93VXH394E6JiYcwf4fkTYdF4K2NiTC1VmcRxDa6o4Q4R2SIieSKyJchxmWBZMQ1+HQvH/wkSmobmmCLQcbDr/zjrOSjYDG+eCy+fBr9PCU0MxphqU5mZ44mqGqWqdVQ1yXudFIrgTDVThfH3QP2m0OuG0B8/KtpNGLwhE07/N2xY6pLHG+fCqlmhj8cYc0AqnPElIifsbXn5GzuZWuC3z2H5VDj9caib4F8cMXWgx1Vw5IXw8/Pw/X/guROg05nQ725onO5fbCa8qMLGpbDkW1gzD064DRKb+x1VrVdhyRER+STgZRzQE5iuqv2CGVh1spIjuBFNI3u5/0jXTa1ZN2kq2AxTnnLzSoq2Q7cL4cQ7oEFrvyMztVHeGlfJeekkWDIZNi/f9V6b49ygjcpUSTD7LDlS4b+eqp5Rbketgf9W8qADgf8B0cAoVX2k3PttgFeBBt46d6jqOBG5CPhLwKpHAEer6kwRmYSbgLjde+8UVV1bmXgi2sw3YN1C+MMbNStpAMQlQ7+/Qc/h8P3jMG0UzB4NGVdCn1shoYnfEZqarGAL/P6Du6pY+i2sne+WxyVDuz6uPy/tRNcc+sFVMGGEm/RqDlilihzutoEbXTVPVTtVsF40sBAYAGQD04ALVHV+wDrPA7+o6kgR6QSMU9V25fbTFfhIVQ/xXk8CblPVSl9CRPwVx86t8MTR0LAtXPHlgc8SD5VNK2DyY/DLm270V6/rXJ9MvQZ+R2ZqgqIdsOJnWDLJJYqcGaDF7nelTS9of6JLFC2OdP1qgT69GTJfgvPfhsOrWGYnAh3wFYeIPAmUZpcooBtuBnlFegJZqrrE2887wFBgfsA6CpR2tCcDK/eynwuAdypxPLMvU5+B/NUw7NWanzTANVENeRKO+xNMfMiVL/n5Bej9Z+h5DdSJ9ztCE0olxe5qYem37qpi+Y+uSVOiodXR0PtmlyxSe0Js3P73derDLtF8dK27WZlVdT4glenjuCzgZRGwTFV/qHDHIucCA1X1Ku/1JcAxqnpDwDotgK+AhkB94GRVnV5uP4uBoao613s9CUjB3fv8feDvupeTEJHhwHCANm3adP/9998rCjk85efCE0e5/1jnv+l3NAdm1WyY8CAs+goSmsEJf4GjL3Od7Cb8qML6LHdFsWQSLPseCja595p0dHOD2p8IbY9zzVFVtXGZG4zRMA2u/Api6lZb6OHmgK84gDFAgaoWezuKFpF4Vd1WDXFdALyiqv8WkV7A6yLSxZuxjogcA2wrTRqei1Q1R0QScYnjEuC18jtW1eeB58E1VVVDrLXT5MegcFv1lxYJpRZHwEXvwe9T4ZsRMO42dz+Qk+6Cruft2Rxhap8tK3f1USz5FvK8xofkNm4OUFpfSDvBVSE4WA3bwZnPulsif3EnDH784PcZYSqTOL4BTgZKS53Ww10lHFfBdjlA4LCYVG9ZoCuBgQCqOlVE4oDGQGln9/nA24EbqGqO9zNPRN7CNYntkTgMsH6xa889+lJoEgaV8Nv2gj+Og6xv3Oz3D69xQ3n73Q2HD64dzXDG2b7RXUmUJot1C93yeo1cgmjf111VNEwLzud6+CDXFDrlCdcvcsR51X+MMFaZxBGnqmX1sVU1X0Qq08g8DUgXkTRcwjgfuLDcOsuB/sArItIRN9w3F8oKLA4D+pSuLCIxQANVXSciscBg4OtKxBKZvhkB0XVCU1okVEQg/WQ4pJ+bAT/h7/DuxdDyaOh/r/vCsQRS8xRud30TS791zU+rZoGWQGx91+R09KWuQ7tZF3e/+1Dofy9kT4NPbnJXtU0OC81xw0BlEsdWETlaVWcAiEh3dg2F3SdVLRKRG4AvcUNtX1LVeSIyAshU1bHArcALInIzrqP88oD+ihOAFaWd6566wJde0ojGJY0XKnWmkSZ7Osz/CE68PTwnPEVFQecz3ZXG7Hdg0iPw+plu+GX/+6B1D78jjGzFRbDyF28uxbduFFTxDoiKgdQecMJf3RVFqwz/+qqiY+Hcl+G5PjD6Urh6AtSp708stUxlOsd74EY1rcTdOrY58Ifyndg1WcQNx1WFVwbDut/gT7+4272Gu6Id7i6Ek/8JW3PdHQ373e3KvJvgU4XcBbuanpZ9Dzu8knbNu7qrifZ9XbOQn1UL9mbJJHjtTDhimKulZlesZQ5mAuA0ETkcKL2O+01VC6s7QFONFn4Jv38Pg/4VGUkD3MiYY66BbhfBT8/CD0/AyOOh67nQ905IOcTvCMPPphW7mp6WTob8NW55wzTocrZLFmknQP3GvoZZofZ93UCLif9wiS3jj35HVONV5orjeuBNVd3kvW6Im8j3TAjiqxYRdcVRXATPHg/FhXD9TzVvlniobN/oksdPz0LxTjjqEjjxr5DU0u/Iaq+t62HZ5F1XFRu8VuT6TXdNumt/IjRo42+cB6KkxFVsXva9G6LbspvfEdUI+7riqEzimKmq3cot+0VVj6rmGIMmohLHjNdg7I0w7DXoNNTvaPyXtwa++5e7cVVUtCuu2PsWqJ/id2Q1386tbgj00knuqmL1XEChTqK7PXBpsmjaMTyad7aud/0d0bEw/FurVMDBJY45wBGlndZeKZHZqlprGo8jJnHs3AZPHg3JqXDl+PD4z1xdNi6DSY+6jvTY+nDcDdDr+shpyquM4kLIztw1lyJ7GpQUupF5rY/xEkVfaHlU+BYJXPGzK/V/6EBX1y3C/w8dzATAL4B3ReQ57/U1wOfVGZypJj+NhLxVcO5LEf8Lv4eG7eCskXD8TTDx7zDpYVfSvfct0ONKiK3nd4ShV1ICa+ftanr6fQrszAfENdX0ut4li9bHRk6Zl9Y9YcCD8OWdrlrzcT7ct6YWqMwVRxSudEd/b9FsoLmqXh/k2KpNRFxxbF0PT3RzTQgXvF3x+pEuZ7qbA7J4AiS2dPdIjyTFO93VxbZ17nVK+q6mp3a9Ib6Rv/H5SdUNz/1tHFz+GbQ51u+IfHMwo6pKROQn4BDchLzGuFIfpiaZ/E/312JtLi0SSq26wyUfwtLv4Lt/w7pFfkcUWhIF6QN2jXxKbuV3RDWHCAx9Cp7vC+/9Ea79ruaPDAuxfSYOETkUV0vqAmAd8C6Aqp4UmtBMpW1Y6u5hcdQl0PRwv6OpXdL6uIcxgeKS4bxXYdTJ8P5VcPH7VhMtwP7m9i8A+gGDVbW3qj6Jq0hrapoJD7oZuX3v9DsSY8JHiyNg0D9hyUR3RW/K7C9xnA2sAiaKyAsi0h83c9zUJDnTYe77riMzqYXf0RgTXo6+FI680JW0WTzB72hqjH0mDlX9SFXPBw4HJgJ/BpqKyEgROSVUAZr9UIXx90F8ihstZIypXiJw+r/dXJX3r4LN5Qt8R6YKy1Cq6lZVfcu793gq8Atwe9AjMxXL+hqWfecKGcYlVby+Mabq6sS7CbVFO2DMH918lwhXpfrFqrpRVZ9X1f4Vr22CqqQYxt/r6gJ1t9o6xgRV43QY8gSs+Am+vt/vaHwXosL3ptrNegfWznf3FLBbqBoTfF3OgR5Xw9Sn4NdP/Y7GV5Y4aqPC7a6SZ8ujofNZfkdjTOQ41ft/99F1bhh8hLLEURv99CxsyYFTHrTSIsaEUkxdOO8V9/9u9KVQWOB3RL6wxFHbbNsA3/0H0k91pSGMMaHVsC2c/Tysng1fROY4IUsctc3kf8HOPDj5fr8jMSZyHXoq9L7Z3XVy1rt+RxNyljhqk43LYNoL0O1CaNbJ72iMiWwn3Q1te8Onf4a1v/odTUhZ4qhNJvwdJBr63uV3JMaY6Bg490WokwCjL4Md+X5HFDJBTRwiMlBEfhORLBG5Yy/vtxGRiSLyi4jMFpFB3vJ2IrJdRGZ6j2cDtukuInO8fT4hEiG9wytnwpz34Nj/s0qmxtQUic1d8li/CD65yVVziABBSxzenQKfBk4DOgEXiEj59pW7gdHebWjPBwLvY75YVbt5j2sDlo8ErgbSvcfAYJ1DjaHqJvvVawS9/+x3NMaYQGknwEl/g7ljIPNFv6MJiWBecfQEslR1iaruBN4Byt8EW4HSWhnJwMr97VBEWgBJqvqjdyvb14AzqzfsGmjxN+4ObSf+1ZV7NsbULL1vgfRT4Is7IWeG39EEXTATRytgRcDrbG9ZoPuBi0UkGxgH3BjwXprXhPWtiJTeMKGVt5/97TO8lBTD+PuhQVvIuMLvaIwxexMVBWc9BwnN4L3LYPtGvyMKKr87xy8AXlHVVGAQ8Lp3q9pVQBuvCesW4C0RqVIVPxEZLiKZIpKZm5tb7YGHzOzRsGaOV1qkrt/RGGP2Jb6Rmxy4ZZWbWR7G/R3BTBw5QOuA16neskBXAqMBVHUqEAc0VtUdqrreWz4dWAwc6m2fWsE+8bZ7XlUzVDWjSZMm1XA6PigscCOpWnSDzmf7HY0xpiKpGXDK3939yqc84Xc0QRPMxDENSBeRNBGpg+v8HltuneVAfwAR6YhLHLki0sTrXEdE2uM6wZeo6ipgi4gc642muhT4OIjn4K+fn4Mt2a60SJTfF4fGmEo55hrodCZ8/QAs+8HvaIIiaN9GqloE3AB8CfyKGz01T0RGiMgQb7VbgatFZBbwNnC51+l9AjBbRGYCY4BrVXWDt811wCggC3cl8nmwzsFX2zbAd/+GDgPcqA1jTO0gAkOehIbtYMwVkL/W74iqnWgYt8OVysjI0MzMTL/DqJqv7oYpT8G130PzLn5HY4ypqtVzYVR/aN0TLvkIoqL9jqjKRGS6qmaUX27tHzXRpuXw03OutIglDWNqp+Zd3G1nl0529ywPI5Y4aqIJ/wCJgpOstIgxtdpRF0O3i2HyP92tnsOEJY6aZtVsmP0uHHMtJKdWvL4xpmYb9E9o2gnevxo2Z1e8fi1giaOm+fo+qNfAlWw2xtR+deJh2GtQXAjvXQ5FO/2O6KBZ4qhJFk9wjxP+4pKHMSY8NO4AQ5+C7Gnuj8NazhJHTVFSAuPvgwZtoMdVfkdjjKlunc90TdA/PgPza/f0M0scNcXcMe5WlP3usdIixoSrAQ9Cqwz4+AZYv9jvaA6YJY6aoLAAvnkQmh8BXc71OxpjTLDE1HH1rKKi3c2fCrf7HdEBscRRE0wbBZuXw4ARVlrEmHDXoDWc/YIrXjruL35Hc0DsW8pv2ze6Md6H9IdDTvI7GmNMKKQPgD63wS+vw8y3/I6myixx+O37/0DBZhjwgN+RGGNC6aS7oF0f+PQWWDPP72iqxBKHnzatgB+fhSP+AM27+h2NMSaUoqLhnBchLsn1d+zI8zuiSrPE4aeJD7mf/f7mbxzGGH8kNoNzX4INi2HsjbXm5k+WOPyyei7MehuOGe7mbhhjIlO73m4Y/rwP4ecX/I6mUixx+OXr+yAuGfrc6nckxhi/Hf9nOHQgfHkXZE/3O5oKWeLww5JJrlJmn1uhXkO/ozHG+C0qCs4cCYktXD2rbRsq3MRPljhCraQExt8Lya2h53C/ozHG1BTxjWDYK5C/Gj681n1X1FCWOEJt3gewahb0uxti4/yOxhhTk7TqDqc+BIu+hB/+63c0+2SJI5SKdsA3I6BZV+g6zO9ojDE1UY+roMs5MOFBWPqd39HslSWOUJr2Imz63U32s9Iixpi9EYEz/geNDoExV0DeGr8j2oN9e4XK9k2utEj7vtChv9/RGGNqsrqJ7uZPO/Lg/SuhuMjviHYT1MQhIgNF5DcRyRKRO/byfhsRmSgiv4jIbBEZ5C0fICLTRWSO97NfwDaTvH3O9B5Ng3kO1eaH/8L2Da6QoTHGVKRZJxj8H1j2HUx6yO9odhMTrB2LSDTwNDAAyAamichYVZ0fsNrdwGhVHSkinYBxQDtgHXCGqq4UkS7Al0CrgO0uUtXMYMVe7TbnwI8jXb9GiyP9jsYYU1t0uwCWT4Hv/g2tj4FDT/U7IiC4Vxw9gSxVXaKqO4F3gKHl1lEgyXueDKwEUNVfVHWlt3weUE9Eau/djSY9BFriRlIZY0xVnPaYq2X3wXDYtNzvaIDgJo5WwIqA19nsftUAcD9wsYhk4642btzLfs4BZqjqjoBlL3vNVPeIiOzt4CIyXEQyRSQzNzf3gE/ioK2Z78om9xwODdv6F4cxpnaKrQfnver++Hzvcija6XdEvneOXwC8oqqpwCDgdREpi0lEOgOPAtcEbHORqnYF+niPS/a2Y1V9XlUzVDWjSZMmQTuBCn19P9RJtNIixpgDl3IIDH0acqbDV/63XAQzceQArQNep3rLAl0JjAZQ1alAHNAYQERSgQ+BS1W17Oa8qprj/cwD3sI1idVMS79zE3n63OJmhRpjzIHqNASOvR5+fs4VRPRRMBPHNCBdRNJEpA5wPjC23DrLgf4AItIRlzhyRaQB8Blwh6r+ULqyiMSISGliiQUGA3ODeA4HTtWVFklqBcdcU/H6xhhTkQEPQGpP+PhGWJflWxhBSxyqWgTcgBsR9Stu9NQ8ERkhIkO81W4FrhaRWcDbwOWqqt52HYB7yw27rQt8KSKzgZm4K5iaWYd43oewcoZXWqSe39EYY8JBdCyc97L7OfpS2LnNlzBEa8mNQw5GRkaGZmaGcPRu0U54ugfE1odrv3N3+jLGmOqS9TW8cS50uxDOfCZohxGR6aqaUX65353j4Wn6y7BxmVdaxJKGMaaadTgZTvwrzHwTZrwe8sNb4qhuBVvg20ch7QT34RpjTDCceLsrYTTuNlg9J6SHtsRR3X74H2xb70qL7H2KiTHGHLyoaDh7lLsZ3OjL3B+toTp0yI4UCbashKlPQ5dzoeVRfkdjjAl3CU3gXK9pfOwNbjRnCFjiqE6THoaSIuh/j9+RGGMiRdtecPJ9MP9j+OnZkBzSEkd1WbsAfnkDel4NDdv5HY0xJpIc9yc4bJCbVb5iWtAPZ4mjunx9P9RJgD63+R2JMSbSiLhhuUmtXD2rbRuCejhLHNVh2Q+w8HPo/Weon+J3NMaYSFSvIQx7FbaudZV0S0qCdihLHAertLRIYks45v/8jsYYE8laHgUDH4Gs8fD9v4N2GEscB2v+x5CTCSfdBXXi/Y7GGBPpMq6ArufBxIdgybdBOYQljoNRXAjfPABNOrqp/8YY4zcRGPxfSEl39yvfsqraD2GJ42BMfwU2LLHSIsaYmqVuAgx7DRofBiWF1b77oN1zPOztyINJj0Db3pB+it/RGGPM7poeDn/8LCi7tiuOA/XDE7BtnZUWMcZEHEscByJvNUx9CjqfBand/Y7GGGNCyhLHgZj0sOsY73+v35EYY0zIWeKoqtyFrv59xhXQqL3f0RhjTMhZ4qiqbx6A2Hh3ExVjjIlAljiqYvmPsOBT6H0T1G/sdzTGGOMLSxyVpQpf3QMJzeHY6/yOxhhjfGOJo7J+/QSyf/ZKi9T3OxpjjPFNUBOHiAwUkd9EJEtE7tjL+21EZKKI/CIis0VkUMB7d3rb/SYip1Z2n0FRWlqk8WHQ7aKQHNIYY2qqoM0cF5Fo4GlgAJANTBORsao6P2C1u4HRqjpSRDoB44B23vPzgc5AS+BrETnU26aifVa/Ga/B+iy44B2Itsn2xpjIFswrjp5AlqouUdWdwDvA0HLrKJDkPU8GVnrPhwLvqOoOVV0KZHn7q8w+q9eOfFdapM1xcOjAoB7KGGNqg2AmjlbAioDX2d6yQPcDF4tINu5q48YKtq3MPgEQkeEikikimbm5uQd6Dm6G+Na1cMqDVlrEGGPwv3P8AuAVVU0FBgGvi0i1xKSqz6tqhqpmNGnS5MB2krfG1aTqNBRSM6ojLGOMqfWC2WCfA7QOeJ3qLQt0JTAQQFWnikgc0LiCbSvaZ/X59lEo3gH97wvaIYwxprYJ5hXHNCBdRNJEpA6us3tsuXWWA/0BRKQjEAfkeuudLyJ1RSQNSAd+ruQ+q0/DtnDcjZBySNAOYYwxtU3QrjhUtUhEbgC+BKKBl1R1noiMADJVdSxwK/CCiNyM6yi/XFUVmCcio4H5QBFwvaoWA+xtn8E6B46/KWi7NsaY2krc93R4y8jI0MzMTL/DMMaYWkVEpqvqHh28fneOG2OMqWUscRhjjKkSSxzGGGOqxBKHMcaYKrHEYYwxpkoscRhjjKkSSxzGGGOqJCLmcYhILvD7AW7eGFhXjeHUBnbOkcHOOfwd7Pm2VdU9iv1FROI4GCKSubcJMOHMzjky2DmHv2CdrzVVGWOMqRJLHMYYY6rEEkfFnvc7AB/YOUcGO+fwF5TztT4OY4wxVWJXHMYYY6rEEocxxpgqscThEZHWIjJRROaLyDwRuclb3khExovIIu9nQ79jrW4iEi0iv4jIp97rNBH5SUSyRORd726LYUNEGojIGBFZICK/ikivcP+cReRm7/d6roi8LSJx4fY5i8hLIrJWROYGLNvr5yrOE965zxaRo/2L/MDt45z/6f1uzxaRD0WkQcB7d3rn/JuInHqgx7XEsUsRcKuqdgKOBa4XkU7AHcA3qpoOfOO9Djc3Ab8GvH4U+I+qdgA24u4NH07+B3yhqocDR+LOPWw/ZxFpBfwJyFDVLri7Z55P+H3OrwADyy3b1+d6Gu6W1OnAcGBkiGKsbq+w5zmPB7qo6hHAQuBOAO/77Hygs7fNMyISfSAHtcThUdVVqjrDe56H+zJpBQwFXvVWexU4058Ig0NEUoHTgVHeawH6AWO8VcLqnEUkGTgBeBFAVXeq6ibC/HPG3Sa6nojEAPHAKsLsc1bVycCGcov39bkOBV5T50eggYi0CE2k1Wdv56yqX6lqkffyRyDVez4UeEdVd6jqUiAL6Hkgx7XEsRci0g44CvgJaKaqq7y3VgPNfAorWP4L/BUo8V6nAJsCfvGycQk0XKQBucDLXvPcKBGpTxh/zqqaA/wLWI5LGJuB6YT351xqX59rK2BFwHrhev5XAJ97z6vtnC1xlCMiCcD7wJ9VdUvge+rGLofN+GURGQysVdXpfscSQjHA0cBIVT0K2Eq5Zqkw/Jwb4v7aTANaAvXZs3kj7IXb51oREfkbrgn+zeretyWOACISi0sab6rqB97iNaWXsN7PtX7FFwTHA0NEZBnwDq7p4n+4y/YYb51UIMef8IIiG8hW1Z+812NwiSScP+eTgaWqmquqhcAHuM8+nD/nUvv6XHOA1gHrhdX5i8jlwGDgIt01Wa/aztkSh8dr238R+FVVHw94ayxwmff8MuDjUMcWLKp6p6qmqmo7XKfZBFW9CJgInOutFm7nvBpYISKHeYv6A/MJ488Z10R1rIjEe7/npecctp9zgH19rmOBS73RVccCmwOatGo1ERmIa34eoqrbAt4aC5wvInVFJA03MODnAzqIqtrDJeTeuMvY2cBM7zEI1+b/DbAI+Bpo5HesQTr/vsCn3vP23i9UFvAeUNfv+Kr5XLsBmd5n/RHQMNw/Z+ABYAEwF3gdqBtunzPwNq4PpxB3ZXnlvj5XQICngcXAHNyIM9/PoZrOOQvXl1H6PfZswPp/8875N+C0Az2ulRwxxhhTJdZUZYwxpkoscRhjjKkSSxzGGGOqxBKHMcaYKrHEYYwxpkoscRhTjUSkXWClUm/Z/SJy2362yRCRJ7znfUXkuGDHaczBiKl4FWNMMKlqJm5eCbj5NPnAFN8CMqYCdsVhTIiIyCQReVREfhaRhSLSx1veV0Q+9YprXgvcLCIzRaSPiJzn3UNjlohM9jN+Y0rZFYcxoRWjqj1FZBBwH66OFACqukxEngXyVfVfACIyBzhVVXMCb8hjjJ/sisOY6rWvUgyly0uLZ04H2lVifz8Ar4jI1bgbMBnjO0scxlSv9bjaV4EaAeu85zu8n8VU4opfVa8F7sZVNZ0uIinVFKcxB8wShzHVSFXzgVUi0g/cPa9x9774vpK7yAMSS1+IyCGq+pOq3ou7AVXrfW5pTIhY4jCm+l0K3CMiM4EJwAOquriS234CnFXaOQ78U0TmeEN8pwCzghOyMZVn1XGNMcZUiV1xGGOMqRJLHMYYY6rEEocxxpgqscRhjDGmSixxGGOMqRJLHMYYY6rEEocxxpgq+X8uJ10g5Xs8kgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "DvLXEakWig-N"
      },
      "cell_type": "markdown",
      "source": [
        "### Testing"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "result = model.predict(x_val)"
      ],
      "metadata": {
        "id": "DzgXSX6wCVK9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result = np.array([i[0] < 0.5 for i in result]).astype(int)"
      ],
      "metadata": {
        "id": "33GZ2mYwChbe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_val2 = np.array([i[0] < 0.5 for i in y_val]).astype(int)"
      ],
      "metadata": {
        "id": "DuIYKS-EDIPF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "P7Ne9Z9fiJTW",
        "outputId": "80bd9415-3a0f-4ae5-9310-ab9232306cff",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "cell_type": "code",
      "source": [
        "from sklearn import metrics\n",
        "print('Test...')\n",
        "print(\"ACC\", metrics.accuracy_score(result, y_val2))\n",
        "print(\"Prec\", metrics.precision_score(result, y_val2))\n",
        "print(\"REC\", metrics.recall_score(result, y_val2))\n",
        "print(\"F1\", metrics.f1_score(result, y_val2))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test...\n",
            "ACC 0.9054054054054054\n",
            "Prec 0.972972972972973\n",
            "REC 0.8571428571428571\n",
            "F1 0.9113924050632912\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "8aCgzabdiwyV"
      },
      "cell_type": "markdown",
      "source": [
        "### Save Model"
      ]
    },
    {
      "metadata": {
        "id": "vpwm6TyxhQyx"
      },
      "cell_type": "code",
      "source": [
        "# Saves the model in a hdf5 file\n",
        "model.save('/content/gdrive/MyDrive/College/Semester5/ML/Project/twitter15/highest.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load Model"
      ],
      "metadata": {
        "id": "IEey4ayMEh-l"
      }
    },
    {
      "metadata": {
        "id": "t5184lvXhQ1k"
      },
      "cell_type": "code",
      "source": [
        "# Load model from saved hdf5 file\n",
        "model = load_model('/content/gdrive/MyDrive/College/Semester5/ML/Project/twitter15/highest.h5',custom_objects={'AttentionLayer': AttentionLayer, 'GlorotUniform':initializers.get('glorot_uniform')})\n",
        "\n",
        "articleDB = pd.DataFrame(columns = ['Title','Summary','Text','Category','Tags'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FDw1KBVzZGRK",
        "outputId": "ca042d24-419a-4712-e065-4a5a14f424b5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        }
      },
      "cell_type": "code",
      "source": [
        "articleDB.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title</th>\n",
              "      <th>Summary</th>\n",
              "      <th>Text</th>\n",
              "      <th>Category</th>\n",
              "      <th>Tags</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [Title, Summary, Text, Category, Tags]\n",
              "Index: []"
            ]
          },
          "metadata": {},
          "execution_count": 234
        }
      ]
    },
    {
      "metadata": {
        "id": "uT5jHGfpjFQs"
      },
      "cell_type": "markdown",
      "source": [
        "### Input"
      ]
    },
    {
      "metadata": {
        "id": "Au3Y6H0mhQ4S"
      },
      "cell_type": "code",
      "source": [
        "article = \"what if talking angela is actually boris laursen he be like i just stole all your personal information what are you going to do about it\"\n",
        "# Clean article with pre described rules\n",
        "article_cleaned,idx_list = cleanString(article,stopWords)\n",
        "article_cleaned = article\n",
        "input_array = wordToSeq(article_cleaned,word_index,MAX_WORD_NUM,MAX_FEATURES)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "id": "66PPxY0bjZLs"
      },
      "cell_type": "markdown",
      "source": [
        "### Word Attention Model"
      ]
    },
    {
      "metadata": {
        "id": "ZwqrAQ0DjD0s"
      },
      "cell_type": "code",
      "source": [
        "def wordAttentionWeights(sequenceSentence,weights):\n",
        "    \"\"\"\n",
        "    The same function as the AttentionLayer class.\n",
        "    \"\"\"\n",
        "    uit = np.dot(sequenceSentence, weights[0]) + weights[1]\n",
        "    uit = np.tanh(uit)\n",
        "\n",
        "    ait = np.dot(uit, weights[2])\n",
        "    ait = np.squeeze(ait)\n",
        "    ait = np.exp(ait)\n",
        "    ait /= np.sum(ait)\n",
        "    \n",
        "    return ait\n",
        "\n",
        "# Summary as input for word attention\n",
        "summary_cleaned, summary_idx = cleanString(article_cleaned,stopWords)\n",
        "word_input_array = wordToSeq(summary_cleaned,word_index,MAX_WORD_NUM,MAX_FEATURES).reshape(1,-1)\n",
        "\n",
        "# Create model from word input to output of dense layer right before the attention layer\n",
        "hidden_word_encoding_out = Model(inputs=wordEncoder.input, outputs=wordEncoder.get_layer('word_dense').output)\n",
        "# Load weights from trained attention layer\n",
        "word_context = wordEncoder.get_layer('word_attention').get_weights()\n",
        "# Compute output of dense layer\n",
        "hidden_word_encodings = hidden_word_encoding_out.predict(word_input_array)\n",
        "# # Compute context vector using output of dense layer\n",
        "ait = wordAttentionWeights(hidden_word_encodings,word_context)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary_cleaned"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "mj9pzbQWfi32",
        "outputId": "3b1ac84f-5e26-4583-c14f-aae3f3c8f539"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'talking angela actually boris laursen like stole personal information going . '"
            ]
          },
          "metadata": {},
          "execution_count": 267
        }
      ]
    },
    {
      "metadata": {
        "id": "UYOG8EBsjD3t"
      },
      "cell_type": "code",
      "source": [
        "attword_list = tokenize.word_tokenize(summary_cleaned.rstrip('.'))\n",
        "ait_short = (np.log(ait[:len(attword_list)])).tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "curr = list(zip(attword_list, ait_short))"
      ],
      "metadata": {
        "id": "IALy2X-9fSbO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "curr"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s2OmtHbFfThp",
        "outputId": "b61a848b-8dfb-456e-a8b6-50484e40ce10"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('talking', -3.880044937133789),\n",
              " ('angela', -3.028024435043335),\n",
              " ('actually', -3.2186646461486816),\n",
              " ('boris', -3.0860610008239746),\n",
              " ('laursen', -2.5795278549194336),\n",
              " ('like', -3.553571939468384),\n",
              " ('stole', -2.829299211502075),\n",
              " ('personal', -2.4552576541900635),\n",
              " ('information', -2.837111473083496),\n",
              " ('going', -2.845625162124634),\n",
              " ('.', -3.0795607566833496)]"
            ]
          },
          "metadata": {},
          "execution_count": 270
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set CSS properties for th elements in dataframe\n",
        "th_props = [\n",
        "  ('font-size', '16'),\n",
        "  ('text-align', 'left'),\n",
        "  ('font-weight', 'bold'),\n",
        "  ('color', '#0d0d0d'),\n",
        "  ('background-color', '##dedee0')\n",
        "  ]\n",
        "\n",
        "# Set CSS properties for td elements in dataframe\n",
        "td_props = [\n",
        "  ('font-size', '14'),\n",
        "  ('text-align', 'center'),\n",
        "\n",
        "  ]\n",
        "\n",
        "# Set table styles\n",
        "styles = [\n",
        "  dict(selector=\"th\", props=th_props),\n",
        "  dict(selector=\"td\", props=td_props),\n",
        "\n",
        "  ]\n"
      ],
      "metadata": {
        "id": "neqTPhI1fzE3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "char_df0 = pd.DataFrame(curr,columns=['Words','Attention Weights'])"
      ],
      "metadata": {
        "id": "ctwJ5Qyrj8pI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#I implement this code so that the actual weights are not shown in the \"heat map\", rather just the colors for visual reasons \n",
        "def color_transparent(val):\n",
        "    \"\"\"\n",
        "    Takes a scalar and returns a string with\n",
        "    the css property `'color: transparent'\n",
        "    \"\"\"\n",
        "    color = 'transparent' if val != 100000 else 'black'\n",
        "\n",
        "    return 'color: %s' % color"
      ],
      "metadata": {
        "id": "LgjUpFHrkFQQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd   \n",
        "from IPython.display import display_html \n",
        "\n",
        "\n",
        "attention_weights0 =(char_df0.style.set_table_attributes(\"style='display:inline'\")\n",
        "  .background_gradient(cmap='OrRd', low=0.001, high=.1)\n",
        "  .applymap(color_transparent,subset=['Attention Weights'])\n",
        "  .set_table_styles(styles))\n",
        "\n",
        "display_html(attention_weights0._repr_html_(),raw=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        },
        "id": "DRTzk-83i4wh",
        "outputId": "68da5f96-962f-4265-8769-d02ac457550b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style  type=\"text/css\" >\n",
              "    #T_f62746b6_58e4_11ec_890f_0242ac1c0002 th {\n",
              "          font-size: 16;\n",
              "          text-align: left;\n",
              "          font-weight: bold;\n",
              "          color: #0d0d0d;\n",
              "          background-color: ##dedee0;\n",
              "    }    #T_f62746b6_58e4_11ec_890f_0242ac1c0002 td {\n",
              "          font-size: 14;\n",
              "          text-align: center;\n",
              "    }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row0_col1{\n",
              "            background-color:  #fff7ec;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row1_col1{\n",
              "            background-color:  #f77f53;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row2_col1{\n",
              "            background-color:  #fda973;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row3_col1{\n",
              "            background-color:  #fb8b58;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row4_col1{\n",
              "            background-color:  #c0110b;\n",
              "            color:  #f1f1f1;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row5_col1{\n",
              "            background-color:  #fddbac;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row6_col1{\n",
              "            background-color:  #e65239;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row7_col1{\n",
              "            background-color:  #a50000;\n",
              "            color:  #f1f1f1;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row8_col1{\n",
              "            background-color:  #e7533a;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row9_col1{\n",
              "            background-color:  #e8553c;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }#T_f62746b6_58e4_11ec_890f_0242ac1c0002row10_col1{\n",
              "            background-color:  #fb8a58;\n",
              "            color:  #000000;\n",
              "            color:  transparent;\n",
              "        }</style><table id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002\" style='display:inline'><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Words</th>        <th class=\"col_heading level0 col1\" >Attention Weights</th>    </tr></thead><tbody>\n",
              "                <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row0_col0\" class=\"data row0 col0\" >talking</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row0_col1\" class=\"data row0 col1\" >-3.880045</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row1_col0\" class=\"data row1 col0\" >angela</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row1_col1\" class=\"data row1 col1\" >-3.028024</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row2_col0\" class=\"data row2 col0\" >actually</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row2_col1\" class=\"data row2 col1\" >-3.218665</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row3_col0\" class=\"data row3 col0\" >boris</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row3_col1\" class=\"data row3 col1\" >-3.086061</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row4_col0\" class=\"data row4 col0\" >laursen</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row4_col1\" class=\"data row4 col1\" >-2.579528</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row5_col0\" class=\"data row5 col0\" >like</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row5_col1\" class=\"data row5 col1\" >-3.553572</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row6_col0\" class=\"data row6 col0\" >stole</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row6_col1\" class=\"data row6 col1\" >-2.829299</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row7_col0\" class=\"data row7 col0\" >personal</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row7_col1\" class=\"data row7 col1\" >-2.455258</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row8_col0\" class=\"data row8 col0\" >information</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row8_col1\" class=\"data row8 col1\" >-2.837111</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row9_col0\" class=\"data row9 col0\" >going</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row9_col1\" class=\"data row9 col1\" >-2.845625</td>\n",
              "            </tr>\n",
              "            <tr>\n",
              "                        <th id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row10_col0\" class=\"data row10 col0\" >.</td>\n",
              "                        <td id=\"T_f62746b6_58e4_11ec_890f_0242ac1c0002row10_col1\" class=\"data row10 col1\" >-3.079561</td>\n",
              "            </tr>\n",
              "    </tbody></table>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "HeYovrLHFsk5"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}