AmazonML_Roberta.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AmazonML2_Roberta.ipynb",
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wu0cKY6Wl4GB",
        "outputId": "728839e1-1089-45ad-b6d9-f422cab33894"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.9.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: huggingface-hub==0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.12)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (5.4.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GHcEhhmUl8Ei"
      },
      "source": [
        "### Necessary packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oue26IXml7w_"
      },
      "source": [
        "import torch \n",
        "import torch.nn as nn\n",
        "from torch.utils.data import Dataset,DataLoader,SubsetRandomSampler\n",
        "import torch.optim as optim\n",
        "\n",
        "import os\n",
        "import copy\n",
        "from collections import defaultdict\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "from pylab import rcParams\n",
        "import csv\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "import random\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.metrics import confusion_matrix,classification_report,accuracy_score\n",
        "from transformers import AutoTokenizer,AutoModel,AutoModelForSequenceClassification,AutoConfig,AdamW,get_linear_schedule_with_warmup\n",
        "\n",
        "seed_val = 42 \n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QhDcvnU4JaoU",
        "outputId": "01bfb48a-8873-4460-e557-4bb740191a3c"
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "    device = torch.device('cuda')\n",
        "    print(\"Running on gpu\",torch.cuda.get_device_name(0))\n",
        "else:\n",
        "    device = 'cpu'\n",
        "    print('No GPU found Running on cpu')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running on gpu Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5rIYey22l7tU",
        "outputId": "6bee00d4-e1b6-4353-a10f-060a7b1ee648"
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SVWO8IGIl7q5"
      },
      "source": [
        "dataset_dir = \"/content/drive/MyDrive/AmazonMLChallenge/dataset\"\n",
        "\n",
        "train_df = pd.read_csv(dataset_dir+\"/train.csv\",escapechar=\"\\\\\",quoting=csv.QUOTE_NONE)\n",
        "test_df = pd.read_csv(dataset_dir+\"/test.csv\",escapechar=\"\\\\\",quoting=csv.QUOTE_NONE)\n",
        "sample_df = pd.read_csv(dataset_dir+\"/sample_submission.csv\",escapechar=\"\\\\\",quoting=csv.QUOTE_NONE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bA1OlKGbxtUD",
        "outputId": "dfd1062b-2dfe-466a-b8a4-cceaf65f5226"
      },
      "source": [
        "train_df['TITLE'].isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "71"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Us7oVGbZx7a6",
        "outputId": "dd88b1ea-e3cd-4b8d-8bd0-4fd97f7fb8b5"
      },
      "source": [
        "len(train_df)-len(train_df.drop_duplicates())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "101494"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RWQ2zfmEyQ2U",
        "outputId": "0c2bb29d-7c21-4c35-f1c4-4d1e048e7b7a"
      },
      "source": [
        "train_df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2903024, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lLWud-oKyTqM",
        "outputId": "11d31be1-7974-4248-d56c-cde03a32d6d9"
      },
      "source": [
        "train_df = train_df.drop_duplicates()\n",
        "train_df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2801530, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "quB5XEQvl7oz"
      },
      "source": [
        "train_df = train_df[train_df['TITLE'].notnull()]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jKUyDnA2l7Iu",
        "outputId": "2e6c365e-99d8-413d-b9a4-692079ffcc53"
      },
      "source": [
        "train_df['TITLE'].isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L5Id6SlFTuOj",
        "outputId": "490f1a9c-2a7e-44d9-aa93-ed6056a9062b"
      },
      "source": [
        "le = LabelEncoder()\n",
        "train_df['BROWSE_NODE_ID'] = le.fit_transform(train_df['BROWSE_NODE_ID'])\n",
        "train_df['BROWSE_NODE_ID'].max()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9918"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-4SbM_wjxgK8"
      },
      "source": [
        "sentences = train_df['TITLE'].values \n",
        "labels = train_df['BROWSE_NODE_ID'].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7GzsJdf1yhiw",
        "outputId": "753cbb8b-f4a6-48aa-fbd4-b918d7df723a"
      },
      "source": [
        "print(sentences.shape,labels.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(2801467,) (2801467,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YMrrty_Oyi9c"
      },
      "source": [
        "train_sentences,val_sentences,train_labels,val_labels = train_test_split(sentences,labels,test_size = 0.1,random_state=seed_val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mFqwJozMy5KS",
        "outputId": "13d2fe36-036d-4afc-d749-99dfc26f4827"
      },
      "source": [
        "print(f\"No. of training sentences {len(train_sentences)}\")\n",
        "print(f\"No. of validation sentences {len(val_sentences)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No. of training sentences 2521320\n",
            "No. of validation sentences 280147\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M5reelgJzgsQ",
        "outputId": "6080e441-f488-47c7-fd59-befb3f8cd147"
      },
      "source": [
        "train_df.memory_usage(deep= True)*(1e-6)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index               22.411736\n",
              "TITLE              410.878601\n",
              "DESCRIPTION       1593.625925\n",
              "BULLET_POINTS     1398.700015\n",
              "BRAND              183.974600\n",
              "BROWSE_NODE_ID      22.411736\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ur2v7Gg00WLo"
      },
      "source": [
        "# indices , cnts = np.unique(labels,return_counts=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l8LCKQdV136Y"
      },
      "source": [
        "# sns.countplot(y = cnts[ (cnts >=10) & (cnts <=100)] )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pknP3ajt3GpX"
      },
      "source": [
        "model_name = 'xlm-roberta-base'\n",
        "max_input_length = 128\n",
        "batch_size = 64 "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aZd6b3hl3Oea"
      },
      "source": [
        "### Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2i-ntoUF2I7X"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MCg52pkq3Km1",
        "outputId": "de153d26-464d-416c-e02a-e7c92590fdb0"
      },
      "source": [
        "idx = 1000\n",
        "sample_text = sentences[idx]\n",
        "tokens =tokenizer.tokenize(sample_text)\n",
        "token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "print('Sample text {}'.format(sample_text))\n",
        "print('Tokens {}'.format(tokens))\n",
        "print('Token IDS {}'.format(token_ids))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample text Generic Ladies 3 x 4 Hooks Adjustable Bra Back Extenders Multicolor 5Pcs\n",
            "Tokens ['▁Gener', 'ic', '▁Ladies', '▁3', '▁x', '▁4', '▁Hoo', 'ks', '▁Ad', 'just', 'able', '▁Bra', '▁Back', '▁Ex', 'tender', 's', '▁Multi', 'color', '▁5', 'P', 'cs']\n",
            "Token IDS [88342, 1771, 190387, 138, 1022, 201, 39016, 1224, 3145, 20314, 2886, 6163, 26828, 5443, 132297, 7, 19335, 46133, 190, 683, 4439]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qcgFU6Aj5hao",
        "outputId": "03a6c997-0252-4175-cc03-cde68c45e04f"
      },
      "source": [
        "tokenizer.sep_token,tokenizer.sep_token_id"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('</s>', 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H_qmWHfS6KW6",
        "outputId": "8d238a29-3429-4933-cb06-55626eb30f01"
      },
      "source": [
        "tokenizer.cls_token,tokenizer.cls_token_id"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('<s>', 0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MzvFCZNT6ONQ",
        "outputId": "11ce7cd8-8ccd-4e7e-8501-6388ccb4da4e"
      },
      "source": [
        "tokenizer.pad_token,tokenizer.pad_token_id"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('<pad>', 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x_P7s9I96RQ_",
        "outputId": "4a02062f-6e22-4115-f308-b6bd047a8909"
      },
      "source": [
        "tokenizer.unk_token,tokenizer.unk_token_id"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('<unk>', 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yGWRjGfb6ma4",
        "outputId": "5041ad2e-010f-42ea-f0e8-38a981fef3ed"
      },
      "source": [
        "encoding = tokenizer.encode_plus(\n",
        "    sample_text,\n",
        "    max_length = max_input_length,\n",
        "    add_special_tokens = True,\n",
        "    pad_to_max_length=True,\n",
        "    return_attention_mask = True,\n",
        "    return_token_type_ids = False,\n",
        "    return_tensors = 'pt'\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2190: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gu9om6It7rtU",
        "outputId": "1882136c-b9d6-4ae2-e44f-bf2dd8a9bb99"
      },
      "source": [
        "encoding"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([[     0,  88342,   1771, 190387,    138,   1022,    201,  39016,   1224,\n",
              "           3145,  20314,   2886,   6163,  26828,   5443, 132297,      7,  19335,\n",
              "          46133,    190,    683,   4439,      2,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0]])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxXhFChq7suv",
        "outputId": "02a31c31-7cbe-45a5-b2dc-694a1216e7f9"
      },
      "source": [
        "encoding.keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'attention_mask'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xdvBrtuoRM8V",
        "outputId": "510b46a2-e31b-4590-942c-d1a727eae4df"
      },
      "source": [
        "base_model = AutoModel.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaModel: ['lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing XLMRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLMRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rpiW_ZXORSIs",
        "outputId": "1f54694a-6afa-40fd-daf3-d077494aec6d"
      },
      "source": [
        "base_model(**encoding)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BaseModelOutputWithPoolingAndCrossAttentions([('last_hidden_state',\n",
              "                                               tensor([[[ 0.0558,  0.0823,  0.0636,  ..., -0.0622,  0.0629,  0.0133],\n",
              "                                                        [-0.1108,  0.0240, -0.0389,  ..., -0.1025, -0.0662,  0.2437],\n",
              "                                                        [-0.0655,  0.0155,  0.0229,  ...,  0.2816, -0.0189,  0.1501],\n",
              "                                                        ...,\n",
              "                                                        [-0.0231,  0.0338,  0.0545,  ..., -0.0503, -0.0094, -0.0327],\n",
              "                                                        [-0.0231,  0.0338,  0.0545,  ..., -0.0503, -0.0094, -0.0327],\n",
              "                                                        [-0.0231,  0.0338,  0.0545,  ..., -0.0503, -0.0094, -0.0327]]],\n",
              "                                                      grad_fn=<NativeLayerNormBackward>)),\n",
              "                                              ('pooler_output',\n",
              "                                               tensor([[-0.0459,  0.2486,  0.1369,  0.4854,  0.0185,  0.3532,  0.4187, -0.4325,\n",
              "                                                         0.1371, -0.1379,  0.0991,  0.1214,  0.3860,  0.3590, -0.2255, -0.1998,\n",
              "                                                         0.1948,  0.3975, -0.0617, -0.2187, -0.2706,  0.3427, -0.6698, -0.5291,\n",
              "                                                        -0.2340,  0.5856,  0.1195, -0.3069, -0.1434,  0.6438,  0.1292,  0.4726,\n",
              "                                                        -0.2648,  0.0860,  0.1161, -0.1888,  0.3877,  0.2125,  0.4880,  0.3225,\n",
              "                                                         0.1053, -0.1281, -0.1133,  0.2465,  0.2147, -0.2349,  0.1790, -0.0488,\n",
              "                                                        -0.2107,  0.4003,  0.5604, -0.2498,  0.0510,  0.0584,  0.1785,  0.1526,\n",
              "                                                         0.3077, -0.3163, -0.2597, -0.5194,  0.0136,  0.6352,  0.4502,  0.3309,\n",
              "                                                         0.3203, -0.5465,  0.1040, -0.0600, -0.6277,  0.1879,  0.0521, -0.3244,\n",
              "                                                         0.3571,  0.0813,  0.1440,  0.0498,  0.3833,  0.1571,  0.3680,  0.2903,\n",
              "                                                         0.2984, -0.3421,  0.1160, -0.2226,  0.1010,  0.6565, -0.3813,  0.7650,\n",
              "                                                         0.3068,  0.3197, -0.1811, -0.0362, -0.2919,  0.2812, -0.3836,  0.3644,\n",
              "                                                        -0.3727, -0.5937,  0.1290, -0.0044, -0.3026, -0.3916,  0.2276, -0.4546,\n",
              "                                                        -0.0956, -0.7341,  0.5118,  0.4600, -0.1713,  0.3846,  0.1632,  0.1302,\n",
              "                                                         0.3551, -0.6614,  0.0774,  0.3543,  0.5418,  0.1345,  0.0230, -0.2773,\n",
              "                                                        -0.2443,  0.5515, -0.3544, -0.1386,  0.3530,  0.0810, -0.3182,  0.6298,\n",
              "                                                         0.3304, -0.0510,  0.4566, -0.3152,  0.1960, -0.5724, -0.0043, -0.2504,\n",
              "                                                         0.5035, -0.3983, -0.0321, -0.5638,  0.3630, -0.0710, -0.2007,  0.4523,\n",
              "                                                        -0.1900,  0.4868,  0.5027,  0.0957, -0.2993,  0.0254,  0.6214, -0.2887,\n",
              "                                                         0.1663, -0.1770,  0.1394,  0.4811, -0.0353,  0.2283, -0.1889,  0.1689,\n",
              "                                                        -0.1293,  0.2963, -0.1818, -0.3318, -0.4427,  0.2335,  0.1786,  0.7476,\n",
              "                                                        -0.1028,  0.2731, -0.2747,  0.0470,  0.4740, -0.1590, -0.2175, -0.5505,\n",
              "                                                        -0.1216, -0.3588,  0.3505, -0.0334, -0.3506, -0.2642,  0.7233,  0.3071,\n",
              "                                                        -0.1594,  0.0622, -0.1870, -0.3654, -0.0329, -0.0621, -0.3608, -0.1529,\n",
              "                                                         0.2850, -0.5232, -0.1189, -0.3678, -0.0994, -0.2221, -0.3450,  0.4868,\n",
              "                                                         0.3862,  0.0794, -0.4022, -0.1656,  0.0104,  0.0356, -0.2567,  0.3846,\n",
              "                                                        -0.0266,  0.1193,  0.0604,  0.2887, -0.0448, -0.6707, -0.3793,  0.2160,\n",
              "                                                        -0.7476,  0.3975, -0.2677,  0.2795, -0.5316,  0.3818, -0.1025, -0.5517,\n",
              "                                                         0.2847,  0.5860,  0.5732, -0.5149,  0.2293, -0.0729,  0.6767, -0.0192,\n",
              "                                                        -0.2386, -0.1964, -0.0674, -0.5023,  0.3669,  0.3465, -0.5116,  0.4617,\n",
              "                                                        -0.2372,  0.0448,  0.3218,  0.1006, -0.0168, -0.4761,  0.3197, -0.4729,\n",
              "                                                         0.2820, -0.0323, -0.7533, -0.3174,  0.0793,  0.6207, -0.3672, -0.4185,\n",
              "                                                        -0.0433,  0.4350, -0.5294, -0.1923,  0.1308,  0.0823, -0.0757, -0.6004,\n",
              "                                                        -0.4567,  0.0816, -0.0425,  0.1946, -0.2351, -0.1475,  0.5065, -0.0033,\n",
              "                                                         0.2519,  0.0951, -0.3399,  0.0202,  0.1293, -0.5560,  0.2665,  0.0708,\n",
              "                                                         0.7245, -0.3586,  0.3514, -0.1961,  0.1369, -0.0653,  0.4021,  0.0887,\n",
              "                                                        -0.2232,  0.4648, -0.5016, -0.1686,  0.1397,  0.1422, -0.3363,  0.2788,\n",
              "                                                        -0.2920, -0.7169,  0.1774, -0.0828, -0.7531, -0.7762, -0.2909, -0.2929,\n",
              "                                                         0.5308,  0.2487, -0.5176, -0.0124, -0.1459, -0.0064,  0.4950,  0.2538,\n",
              "                                                         0.2088,  0.0312, -0.5099,  0.3554, -0.5607,  0.0082, -0.1092,  0.2839,\n",
              "                                                         0.1456,  0.1986,  0.0691,  0.6944,  0.0464, -0.2018, -0.4234, -0.1185,\n",
              "                                                        -0.0252,  0.0858,  0.4346, -0.1350, -0.3880, -0.3272,  0.1417,  0.3782,\n",
              "                                                         0.5255,  0.2346, -0.3114, -0.0383, -0.0939, -0.4821, -0.5623,  0.3276,\n",
              "                                                        -0.5241, -0.4073,  0.2116,  0.0342, -0.4311, -0.0816, -0.4786,  0.4339,\n",
              "                                                        -0.0323, -0.4674, -0.1096,  0.2242,  0.5355,  0.0249,  0.2400,  0.1313,\n",
              "                                                        -0.0113,  0.0120,  0.1573,  0.0521,  0.1257, -0.5642, -0.0569, -0.2120,\n",
              "                                                         0.6012,  0.1038,  0.0838,  0.0764,  0.2111, -0.3311,  0.2766, -0.0696,\n",
              "                                                        -0.3432,  0.4474, -0.2266, -0.2059,  0.5054,  0.1938, -0.0622, -0.0101,\n",
              "                                                         0.1695,  0.3497,  0.5429, -0.0512,  0.6448, -0.1125,  0.5253,  0.1279,\n",
              "                                                        -0.1765, -0.2935,  0.0424, -0.5830,  0.5333, -0.4731,  0.3409, -0.0350,\n",
              "                                                        -0.3873, -0.4427,  0.6292,  0.4113,  0.4023,  0.1874,  0.1152,  0.5301,\n",
              "                                                        -0.6629,  0.2840,  0.3204,  0.1277,  0.1624, -0.3982, -0.1674, -0.1607,\n",
              "                                                         0.0047, -0.1982, -0.3010, -0.1468,  0.3097, -0.5974,  0.2252,  0.7473,\n",
              "                                                         0.6738, -0.1730,  0.1268, -0.1024, -0.2524, -0.6817,  0.0077, -0.1279,\n",
              "                                                        -0.4751,  0.1434, -0.1672, -0.5200,  0.0174,  0.1702, -0.0292,  0.0671,\n",
              "                                                        -0.1631,  0.1901,  0.4043, -0.2486, -0.0876, -0.2647,  0.1543,  0.3496,\n",
              "                                                        -0.1061,  0.0386,  0.2345, -0.3222,  0.5657,  0.1824,  0.6262,  0.3380,\n",
              "                                                         0.0232,  0.2383,  0.1139, -0.3725,  0.2879, -0.3997, -0.1220,  0.0966,\n",
              "                                                        -0.4469,  0.1751, -0.2643, -0.7291,  0.2380,  0.3791, -0.2631, -0.4299,\n",
              "                                                         0.0064,  0.0636, -0.4532, -0.1191,  0.2008, -0.1206,  0.0405,  0.2506,\n",
              "                                                         0.2405, -0.0106, -0.3339,  0.2976, -0.3568,  0.6585, -0.0421,  0.0415,\n",
              "                                                        -0.1434, -0.4007, -0.1297,  0.0196,  0.1846,  0.6506, -0.1743,  0.2531,\n",
              "                                                         0.1381, -0.3513, -0.4091, -0.5929, -0.1323,  0.0954, -0.3530,  0.2469,\n",
              "                                                         0.4258, -0.0877, -0.0702, -0.1025,  0.1629,  0.0538,  0.3004,  0.1149,\n",
              "                                                        -0.5804, -0.3250, -0.0966, -0.2859,  0.5212,  0.3771,  0.0542,  0.1817,\n",
              "                                                         0.5757, -0.3493, -0.0045,  0.2106, -0.1575, -0.1883, -0.3168, -0.0184,\n",
              "                                                         0.4663,  0.1526, -0.3722, -0.5258,  0.3294,  0.3069, -0.2491, -0.2298,\n",
              "                                                         0.6487,  0.1849, -0.0827,  0.1910,  0.4034,  0.0054,  0.3199,  0.2435,\n",
              "                                                         0.7535,  0.1103, -0.1206,  0.2774, -0.5055, -0.3870, -0.3390,  0.6783,\n",
              "                                                         0.0942,  0.3610,  0.1387,  0.1246,  0.1749,  0.2728, -0.2938, -0.2765,\n",
              "                                                         0.1453,  0.4732, -0.0079, -0.2924, -0.2239,  0.4038,  0.1557, -0.4477,\n",
              "                                                        -0.3975,  0.4214,  0.0671, -0.0893, -0.2465,  0.2768,  0.2030,  0.2895,\n",
              "                                                        -0.4079, -0.5119,  0.3087, -0.3122, -0.2241, -0.0148,  0.7475,  0.1860,\n",
              "                                                        -0.0497,  0.0456, -0.0560, -0.4701, -0.4404,  0.1537, -0.7405,  0.1877,\n",
              "                                                         0.4895, -0.1089, -0.2475, -0.3035,  0.0591, -0.4603,  0.2049, -0.1554,\n",
              "                                                        -0.4488,  0.2699, -0.3236, -0.0532, -0.2827, -0.1718,  0.2997,  0.7119,\n",
              "                                                        -0.4965, -0.2873, -0.5809,  0.1692, -0.1287,  0.0049, -0.6253, -0.7008,\n",
              "                                                         0.0086,  0.1172,  0.5913, -0.1588,  0.1818,  0.0454,  0.1087, -0.2930,\n",
              "                                                         0.8000,  0.4565, -0.3003, -0.3477, -0.5571, -0.2565, -0.3447, -0.3890,\n",
              "                                                        -0.1407, -0.3994, -0.0472, -0.1275,  0.8144, -0.2926, -0.3699, -0.0142,\n",
              "                                                        -0.0355, -0.0941,  0.0660,  0.0254, -0.2096, -0.2316,  0.2879,  0.0940,\n",
              "                                                        -0.0268, -0.3528, -0.2813, -0.5890,  0.0669,  0.2021, -0.2125, -0.4352,\n",
              "                                                         0.5666,  0.1266,  0.1498, -0.4914,  0.4506, -0.3843,  0.2008, -0.2441,\n",
              "                                                        -0.0443, -0.2943, -0.2093, -0.1082,  0.1521, -0.0358,  0.4177, -0.1541,\n",
              "                                                         0.4614,  0.1261,  0.0074,  0.1933,  0.4972,  0.0642, -0.6608, -0.2296,\n",
              "                                                        -0.7613,  0.0846,  0.0179,  0.4298, -0.6124, -0.0791,  0.4137,  0.0899,\n",
              "                                                        -0.2090,  0.1925, -0.3441,  0.6291,  0.4326,  0.2989, -0.0196,  0.3552,\n",
              "                                                         0.6149,  0.1175, -0.1040, -0.6377,  0.0124,  0.2358,  0.1382, -0.1665,\n",
              "                                                         0.1528, -0.6647, -0.3309,  0.3772, -0.5186, -0.1203,  0.3435, -0.1365,\n",
              "                                                         0.2880, -0.5869, -0.4083, -0.3733, -0.1768,  0.2376, -0.4213,  0.0854,\n",
              "                                                        -0.4617, -0.1371, -0.0094,  0.0087, -0.2773, -0.2103, -0.3535,  0.2737,\n",
              "                                                        -0.0985, -0.2017,  0.0611, -0.1443,  0.0261, -0.2936, -0.1307, -0.3029,\n",
              "                                                        -0.3569, -0.2242, -0.0975, -0.2254,  0.3338, -0.4062, -0.0068, -0.0786,\n",
              "                                                         0.1732, -0.2761,  0.8149,  0.0463, -0.1177, -0.5099,  0.2503, -0.0179,\n",
              "                                                        -0.4455, -0.0650, -0.1372, -0.1805,  0.3566,  0.5772,  0.5961, -0.5307,\n",
              "                                                        -0.1141,  0.6141, -0.2510,  0.4974, -0.3952, -0.0891, -0.0840,  0.1377]],\n",
              "                                                      grad_fn=<TanhBackward>))])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LOFoHmxclt4K",
        "outputId": "851a7c99-b6a0-4a49-9960-fcf25313ec31"
      },
      "source": [
        "config = AutoConfig.from_pretrained(pretrained_model_name_or_path=model_name,\n",
        "                                    num_labels=len(np.unique(labels)))\n",
        "\n",
        "print('Config type:', str(type(config)), '\\n')\n",
        "\n",
        "\n",
        "\n",
        "# Load the model from the transformers library using AutoModelForSequenceClassification\"\n",
        "\n",
        "# Load the pre-trained model for classification, passing in the `config` from above.\n",
        "model = AutoModelForSequenceClassification.from_pretrained(\n",
        "                                            pretrained_model_name_or_path=model_name,\n",
        "                                            config = config\n",
        "                                        )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Config type: <class 'transformers.models.xlm_roberta.configuration_xlm_roberta.XLMRobertaConfig'> \n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CbDzGp5o8e2v"
      },
      "source": [
        "## Choosing token length"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DjexYiW9mBdD",
        "outputId": "5e124059-ea95-493c-f76d-6e15b44f4524"
      },
      "source": [
        "print(model(**encoding))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SequenceClassifierOutput(loss=None, logits=tensor([[ 0.0262, -0.0387,  0.4675,  ..., -0.1051, -0.1080,  0.0812]],\n",
            "       grad_fn=<AddmmBackward>), hidden_states=None, attentions=None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pl1hW5vd7z51"
      },
      "source": [
        "# token_lens = []\n",
        "# for txt in sentences:\n",
        "#     tokens = tokenizer.encode(txt,max_length=512)\n",
        "#     token_lens.append(len(tokens))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U-zSVzE18xzF"
      },
      "source": [
        "# sns.displot(token_lens)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PZqbay3_81xn"
      },
      "source": [
        "class AmazonDataset(Dataset):\n",
        "\n",
        "  def __init__(self, sentences, labels, tokenizer, max_length,with_labels=True):\n",
        "    self.sentences = sentences\n",
        "    self.labels = labels\n",
        "    self.tokenizer = tokenizer\n",
        "    self.max_length = max_length\n",
        "    self.with_labels = with_labels\n",
        "  \n",
        "  def __len__(self):\n",
        "    return len(self.sentences)\n",
        "  \n",
        "  def __getitem__(self, idx):\n",
        "    sentence = str(self.sentences[idx])\n",
        "    encoding = self.tokenizer.encode_plus(\n",
        "      sentence,\n",
        "      add_special_tokens=True,\n",
        "      max_length=self.max_length,\n",
        "      return_token_type_ids=False,\n",
        "      pad_to_max_length=True,\n",
        "      return_attention_mask=True,\n",
        "      return_tensors='pt',\n",
        "    )\n",
        "\n",
        "    if self.with_labels:\n",
        "        \n",
        "        label = self.labels[idx]\n",
        "\n",
        "        return {\n",
        "            'sentence': sentence,\n",
        "            'input_ids': encoding['input_ids'].flatten(),\n",
        "            'attention_mask': encoding['attention_mask'].flatten(),\n",
        "            'labels': torch.tensor(label, dtype=torch.long)\n",
        "        }\n",
        "    else:\n",
        "        return {\n",
        "            'sentence': sentence,\n",
        "            'input_ids': encoding['input_ids'].flatten(),\n",
        "            'attention_mask': encoding['attention_mask'].flatten(),\n",
        "        }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P9NnNAOs_VXd"
      },
      "source": [
        "def create_data_loaders(sentences,labels,tokenizer,max_input_length,batch_size,with_labels):\n",
        "    ds = AmazonDataset(\n",
        "        sentences =sentences,\n",
        "        labels=labels,\n",
        "        tokenizer=tokenizer,\n",
        "        max_length=max_input_length,\n",
        "        with_labels = with_labels\n",
        "    )\n",
        "\n",
        "    return DataLoader(\n",
        "        ds,\n",
        "        batch_size=batch_size\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hyta-on7CL5j"
      },
      "source": [
        "train_loader = create_data_loaders(\n",
        "    train_sentences,\n",
        "    train_labels,\n",
        "    tokenizer,\n",
        "    max_input_length=max_input_length,\n",
        "    batch_size=batch_size,\n",
        "    with_labels = True\n",
        ")\n",
        "\n",
        "val_loader = create_data_loaders(\n",
        "    val_sentences,\n",
        "    val_labels,\n",
        "    tokenizer,\n",
        "    max_input_length=max_input_length,\n",
        "    batch_size=batch_size,\n",
        "    with_labels = True\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7t9mcWUOId8b"
      },
      "source": [
        "class AmazonClassifier(nn.Module):\n",
        "\n",
        "  def __init__(self,base_model_name, n_classes):\n",
        "    super(AmazonClassifier, self).__init__()\n",
        "    self.base_model = AutoModel.from_pretrained(base_model_name)\n",
        "    self.drop = nn.Dropout(p=0.3)\n",
        "    self.out = nn.Linear(self.base_model.config.hidden_size, n_classes)\n",
        "  \n",
        "  def forward(self, input_ids, attention_mask):\n",
        "    pooled_output = self.base_model(\n",
        "      input_ids=input_ids,\n",
        "      attention_mask=attention_mask\n",
        "    )['last_hidden_state']\n",
        "    output = self.drop(pooled_output)\n",
        "    return self.out(output)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1JCWdiwAFNIM"
      },
      "source": [
        "# from transformers import AutoConfig\n",
        "# config = AutoConfig.from_pretrained(pretrained_model_name_or_path=model_name,num_labels = len(np.unique(labels)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXXmbmlAI6Bb"
      },
      "source": [
        "# model = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path=model_name,config = config)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WRbEwdMmgXNu"
      },
      "source": [
        "# model = AmazonClassifier(base_model_name=model_name,n_classes=len(np.unique(labels)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pw0Pw3VcJD7c",
        "outputId": "aacd3af1-a343-4fb0-8c04-3a35a3fbb465"
      },
      "source": [
        "model.to(device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "XLMRobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(250002, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=9919, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R46XeyKCJHaA"
      },
      "source": [
        "num_epochs = 4\n",
        "\n",
        "optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)\n",
        "\n",
        "total_steps = len(train_loader) * num_epochs\n",
        "\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "  optimizer,\n",
        "  num_warmup_steps=0,\n",
        "  num_training_steps=total_steps\n",
        ")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tfNaSuElpDFr"
      },
      "source": [
        "import numpy as np\n",
        "def check_accuracy(predictions,true_labels):\n",
        "    \"\"\"\n",
        "    Used for checking accuracy across each epoch\n",
        "    \"\"\"\n",
        "    # Combine the results across the batches.\n",
        "    predictions = np.concatenate(predictions, axis=0)\n",
        "    true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "    # Choose the label with the highest score as our prediction.\n",
        "    preds = np.argmax(predictions, axis=1).flatten()\n",
        "\n",
        "    # Calculate simple flat accuracy -- number correct over total number.\n",
        "    accuracy = (preds == true_labels).mean()\n",
        "\n",
        "    return accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w12Iv6sxJSLG"
      },
      "source": [
        "def train_epoch(\n",
        "  model, \n",
        "  data_loader, \n",
        "  optimizer, \n",
        "  device, \n",
        "  scheduler, \n",
        "  n_examples\n",
        "):\n",
        "  model = model.train()\n",
        "\n",
        "  total_train_loss = 0\n",
        "  correct_predictions = 0\n",
        "#   predictions = []\n",
        "#   true_labels = []\n",
        "  for i,d in enumerate(data_loader):\n",
        "    if i%100 == 0:\n",
        "        print(f\"Processing batch {i+1}/{len(data_loader)}\")\n",
        "    input_ids = d[\"input_ids\"].to(device)\n",
        "    attention_mask = d[\"attention_mask\"].to(device)\n",
        "    labels = d[\"labels\"].to(device)\n",
        "\n",
        "    outputs = model(\n",
        "      input_ids=input_ids,\n",
        "      attention_mask=attention_mask,\n",
        "      labels = labels\n",
        "    )\n",
        "    # print(outputs.size())\n",
        "    # print(labels.size())\n",
        "    loss = outputs.loss\n",
        "    logits = outputs.logits\n",
        "    # _, preds = torch.max(outputs, dim=1)\n",
        "    # loss = loss_fn(outputs, labels)\n",
        "    # print(logits)\n",
        "    # print(labels)\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    labels = labels.to('cpu').numpy()\n",
        "    \n",
        "    # Store predictions and true labels\n",
        "    # predictions.append(logits)\n",
        "    # true_labels.append(labels)\n",
        "    # Combine the results across the batches.\n",
        "    # predictions = np.concatenate(predictions, axis=0)\n",
        "    # true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "    # Choose the label with the highest score as our prediction.\n",
        "    preds = np.argmax(logits, axis=1).flatten()\n",
        "\n",
        "    # Calculate simple flat accuracy -- number correct over total number.\n",
        "    correct_predictions += (preds == labels).sum()\n",
        "    \n",
        "    # Accumulate the training loss over all of the batches so that we can\n",
        "    # calculate the average loss at the end.\n",
        "    total_train_loss += loss.item()\n",
        "\n",
        "    loss.backward()\n",
        "    nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
        "    optimizer.step()\n",
        "    scheduler.step()\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "#   training_accuracy = check_accuracy(predictions,true_labels)       \n",
        "  \n",
        "  return correct_predictions/n_examples, total_train_loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3I1wdUl1JvLw"
      },
      "source": [
        "def eval_model(model, data_loader, device, n_examples):\n",
        "  model = model.eval()\n",
        "\n",
        "  total_val_loss = 0\n",
        "  correct_predictions = 0\n",
        "#   predictions = []\n",
        "#   true_labels = []\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for i,d in enumerate(data_loader):\n",
        "      if i%100 == 0:\n",
        "          print(f\"Processing batch {i+1}/{len(data_loader)}\")\n",
        "      input_ids = d[\"input_ids\"].to(device)\n",
        "      attention_mask = d[\"attention_mask\"].to(device)\n",
        "      labels = d[\"labels\"].to(device)\n",
        "\n",
        "      outputs = model(\n",
        "        input_ids=input_ids,\n",
        "        attention_mask=attention_mask,\n",
        "        labels = labels\n",
        "      )\n",
        "      \n",
        "      loss = outputs.loss\n",
        "      logits = outputs.logits\n",
        "      \n",
        "      logits = logits.detach().cpu().numpy()\n",
        "      labels = labels.to('cpu').numpy()\n",
        "    \n",
        "    #   # Store predictions and true labels\n",
        "    #   predictions.append(logits)\n",
        "    #   true_labels.append(labels)\n",
        "      preds = np.argmax(logits, axis=1).flatten()\n",
        "\n",
        "      # Calculate simple flat accuracy -- number correct over total number.\n",
        "      correct_predictions += (preds == labels).sum()\n",
        "   \n",
        "      total_val_loss += loss.item()\n",
        "  \n",
        "#   val_accuracy = check_accuracy(predictions,true_labels)       \n",
        " \n",
        "  return correct_predictions/n_examples, total_val_loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oUrVRzRnTzgv",
        "outputId": "bb7e6724-564e-467f-aa08-ea8d22bd039b"
      },
      "source": [
        "val_acc, val_loss = eval_model(\n",
        "    model,\n",
        "    val_loader,\n",
        "    device, \n",
        "    len(val_labels)\n",
        ")\n",
        "\n",
        "print(f'Val   loss {val_loss} accuracy {val_acc}')\n",
        "print()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2190: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing batch 1/4378\n",
            "Processing batch 101/4378\n",
            "Processing batch 201/4378\n",
            "Processing batch 301/4378\n",
            "Processing batch 401/4378\n",
            "Processing batch 501/4378\n",
            "Processing batch 601/4378\n",
            "Processing batch 701/4378\n",
            "Processing batch 801/4378\n",
            "Processing batch 901/4378\n",
            "Processing batch 1001/4378\n",
            "Processing batch 1101/4378\n",
            "Processing batch 1201/4378\n",
            "Processing batch 1301/4378\n",
            "Processing batch 1401/4378\n",
            "Processing batch 1501/4378\n",
            "Processing batch 1601/4378\n",
            "Processing batch 1701/4378\n",
            "Processing batch 1801/4378\n",
            "Processing batch 1901/4378\n",
            "Processing batch 2001/4378\n",
            "Processing batch 2101/4378\n",
            "Processing batch 2201/4378\n",
            "Processing batch 2301/4378\n",
            "Processing batch 2401/4378\n",
            "Processing batch 2501/4378\n",
            "Processing batch 2601/4378\n",
            "Processing batch 2701/4378\n",
            "Processing batch 2801/4378\n",
            "Processing batch 2901/4378\n",
            "Processing batch 3001/4378\n",
            "Processing batch 3101/4378\n",
            "Processing batch 3201/4378\n",
            "Processing batch 3301/4378\n",
            "Processing batch 3401/4378\n",
            "Processing batch 3501/4378\n",
            "Processing batch 3601/4378\n",
            "Processing batch 3701/4378\n",
            "Processing batch 3801/4378\n",
            "Processing batch 3901/4378\n",
            "Processing batch 4001/4378\n",
            "Processing batch 4101/4378\n",
            "Processing batch 4201/4378\n",
            "Processing batch 4301/4378\n",
            "Val   loss 6816.005071520805 accuracy 0.7060900170267752\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "BhZqTN1TJxl3",
        "outputId": "3e80659e-9d94-4de0-d0cb-7fabaab463c2"
      },
      "source": [
        "%%time\n",
        "\n",
        "history = defaultdict(list)\n",
        "best_accuracy = 0\n",
        "\n",
        "for epoch in tqdm(range(num_epochs)):\n",
        "\n",
        "  print(f'Epoch {epoch + 1}/{num_epochs}')\n",
        "  print('-' * 10)\n",
        "\n",
        "  train_acc, train_loss = train_epoch(\n",
        "    model,\n",
        "    train_loader,     \n",
        "    optimizer, \n",
        "    device, \n",
        "    scheduler, \n",
        "    len(train_labels)\n",
        "  )\n",
        "\n",
        "  print(f'Train loss {train_loss} accuracy {train_acc}')\n",
        "\n",
        "  val_acc, val_loss = eval_model(\n",
        "    model,\n",
        "    val_loader,\n",
        "    device, \n",
        "    len(val_labels)\n",
        "  )\n",
        "\n",
        "  print(f'Val   loss {val_loss} accuracy {val_acc}')\n",
        "  print()\n",
        "\n",
        "  history['train_acc'].append(train_acc)\n",
        "  history['train_loss'].append(train_loss)\n",
        "  history['val_acc'].append(val_acc)\n",
        "  history['val_loss'].append(val_loss)\n",
        "\n",
        "  if val_acc > best_accuracy:\n",
        "    torch.save(model.state_dict(), 'best_model_state.bin')\n",
        "    best_accuracy = val_acc"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/4 [00:00<?, ?it/s]\u001b[A/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2190: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/4\n",
            "----------\n",
            "Processing batch 1/39396\n",
            "Processing batch 101/39396\n",
            "Processing batch 201/39396\n",
            "Processing batch 301/39396\n",
            "Processing batch 401/39396\n",
            "Processing batch 501/39396\n",
            "Processing batch 601/39396\n",
            "Processing batch 701/39396\n",
            "Processing batch 801/39396\n",
            "Processing batch 901/39396\n",
            "Processing batch 1001/39396\n",
            "Processing batch 1101/39396\n",
            "Processing batch 1201/39396\n",
            "Processing batch 1301/39396\n",
            "Processing batch 1401/39396\n",
            "Processing batch 1501/39396\n",
            "Processing batch 1601/39396\n",
            "Processing batch 1701/39396\n",
            "Processing batch 1801/39396\n",
            "Processing batch 1901/39396\n",
            "Processing batch 2001/39396\n",
            "Processing batch 2101/39396\n",
            "Processing batch 2201/39396\n",
            "Processing batch 2301/39396\n",
            "Processing batch 2401/39396\n",
            "Processing batch 2501/39396\n",
            "Processing batch 2601/39396\n",
            "Processing batch 2701/39396\n",
            "Processing batch 2801/39396\n",
            "Processing batch 2901/39396\n",
            "Processing batch 3001/39396\n",
            "Processing batch 3101/39396\n",
            "Processing batch 3201/39396\n",
            "Processing batch 3301/39396\n",
            "Processing batch 3401/39396\n",
            "Processing batch 3501/39396\n",
            "Processing batch 3601/39396\n",
            "Processing batch 3701/39396\n",
            "Processing batch 3801/39396\n",
            "Processing batch 3901/39396\n",
            "Processing batch 4001/39396\n",
            "Processing batch 4101/39396\n",
            "Processing batch 4201/39396\n",
            "Processing batch 4301/39396\n",
            "Processing batch 4401/39396\n",
            "Processing batch 4501/39396\n",
            "Processing batch 4601/39396\n",
            "Processing batch 4701/39396\n",
            "Processing batch 4801/39396\n",
            "Processing batch 4901/39396\n",
            "Processing batch 5001/39396\n",
            "Processing batch 5101/39396\n",
            "Processing batch 5201/39396\n",
            "Processing batch 5301/39396\n",
            "Processing batch 5401/39396\n",
            "Processing batch 5501/39396\n",
            "Processing batch 5601/39396\n",
            "Processing batch 5701/39396\n",
            "Processing batch 5801/39396\n",
            "Processing batch 5901/39396\n",
            "Processing batch 6001/39396\n",
            "Processing batch 6101/39396\n",
            "Processing batch 6201/39396\n",
            "Processing batch 6301/39396\n",
            "Processing batch 6401/39396\n",
            "Processing batch 6501/39396\n",
            "Processing batch 6601/39396\n",
            "Processing batch 6701/39396\n",
            "Processing batch 6801/39396\n",
            "Processing batch 6901/39396\n",
            "Processing batch 7001/39396\n",
            "Processing batch 7101/39396\n",
            "Processing batch 7201/39396\n",
            "Processing batch 7301/39396\n",
            "Processing batch 7401/39396\n",
            "Processing batch 7501/39396\n",
            "Processing batch 7601/39396\n",
            "Processing batch 7701/39396\n",
            "Processing batch 7801/39396\n",
            "Processing batch 7901/39396\n",
            "Processing batch 8001/39396\n",
            "Processing batch 8101/39396\n",
            "Processing batch 8201/39396\n",
            "Processing batch 8301/39396\n",
            "Processing batch 8401/39396\n",
            "Processing batch 8501/39396\n",
            "Processing batch 8601/39396\n",
            "Processing batch 8701/39396\n",
            "Processing batch 8801/39396\n",
            "Processing batch 8901/39396\n",
            "Processing batch 9001/39396\n",
            "Processing batch 9101/39396\n",
            "Processing batch 9201/39396\n",
            "Processing batch 9301/39396\n",
            "Processing batch 9401/39396\n",
            "Processing batch 9501/39396\n",
            "Processing batch 9601/39396\n",
            "Processing batch 9701/39396\n",
            "Processing batch 9801/39396\n",
            "Processing batch 9901/39396\n",
            "Processing batch 10001/39396\n",
            "Processing batch 10101/39396\n",
            "Processing batch 10201/39396\n",
            "Processing batch 10301/39396\n",
            "Processing batch 10401/39396\n",
            "Processing batch 10501/39396\n",
            "Processing batch 10601/39396\n",
            "Processing batch 10701/39396\n",
            "Processing batch 10801/39396\n",
            "Processing batch 10901/39396\n",
            "Processing batch 11001/39396\n",
            "Processing batch 11101/39396\n",
            "Processing batch 11201/39396\n",
            "Processing batch 11301/39396\n",
            "Processing batch 11401/39396\n",
            "Processing batch 11501/39396\n",
            "Processing batch 11601/39396\n",
            "Processing batch 11701/39396\n",
            "Processing batch 11801/39396\n",
            "Processing batch 11901/39396\n",
            "Processing batch 12001/39396\n",
            "Processing batch 12101/39396\n",
            "Processing batch 12201/39396\n",
            "Processing batch 12301/39396\n",
            "Processing batch 12401/39396\n",
            "Processing batch 12501/39396\n",
            "Processing batch 12601/39396\n",
            "Processing batch 12701/39396\n",
            "Processing batch 12801/39396\n",
            "Processing batch 12901/39396\n",
            "Processing batch 13001/39396\n",
            "Processing batch 13101/39396\n",
            "Processing batch 13201/39396\n",
            "Processing batch 13301/39396\n",
            "Processing batch 13401/39396\n",
            "Processing batch 13501/39396\n",
            "Processing batch 13601/39396\n",
            "Processing batch 13701/39396\n",
            "Processing batch 13801/39396\n",
            "Processing batch 13901/39396\n",
            "Processing batch 14001/39396\n",
            "Processing batch 14101/39396\n",
            "Processing batch 14201/39396\n",
            "Processing batch 14301/39396\n",
            "Processing batch 14401/39396\n",
            "Processing batch 14501/39396\n",
            "Processing batch 14601/39396\n",
            "Processing batch 14701/39396\n",
            "Processing batch 14801/39396\n",
            "Processing batch 14901/39396\n",
            "Processing batch 15001/39396\n",
            "Processing batch 15101/39396\n",
            "Processing batch 15201/39396\n",
            "Processing batch 15301/39396\n",
            "Processing batch 15401/39396\n",
            "Processing batch 15501/39396\n",
            "Processing batch 15601/39396\n",
            "Processing batch 15701/39396\n",
            "Processing batch 15801/39396\n",
            "Processing batch 15901/39396\n",
            "Processing batch 16001/39396\n",
            "Processing batch 16101/39396\n",
            "Processing batch 16201/39396\n",
            "Processing batch 16301/39396\n",
            "Processing batch 16401/39396\n",
            "Processing batch 16501/39396\n",
            "Processing batch 16601/39396\n",
            "Processing batch 16701/39396\n",
            "Processing batch 16801/39396\n",
            "Processing batch 16901/39396\n",
            "Processing batch 17001/39396\n",
            "Processing batch 17101/39396\n",
            "Processing batch 17201/39396\n",
            "Processing batch 17301/39396\n",
            "Processing batch 17401/39396\n",
            "Processing batch 17501/39396\n",
            "Processing batch 17601/39396\n",
            "Processing batch 17701/39396\n",
            "Processing batch 17801/39396\n",
            "Processing batch 17901/39396\n",
            "Processing batch 18001/39396\n",
            "Processing batch 18101/39396\n",
            "Processing batch 18201/39396\n",
            "Processing batch 18301/39396\n",
            "Processing batch 18401/39396\n",
            "Processing batch 18501/39396\n",
            "Processing batch 18601/39396\n",
            "Processing batch 18701/39396\n",
            "Processing batch 18801/39396\n",
            "Processing batch 18901/39396\n",
            "Processing batch 19001/39396\n",
            "Processing batch 19101/39396\n",
            "Processing batch 19201/39396\n",
            "Processing batch 19301/39396\n",
            "Processing batch 19401/39396\n",
            "Processing batch 19501/39396\n",
            "Processing batch 19601/39396\n",
            "Processing batch 19701/39396\n",
            "Processing batch 19801/39396\n",
            "Processing batch 19901/39396\n",
            "Processing batch 20001/39396\n",
            "Processing batch 20101/39396\n",
            "Processing batch 20201/39396\n",
            "Processing batch 20301/39396\n",
            "Processing batch 20401/39396\n",
            "Processing batch 20501/39396\n",
            "Processing batch 20601/39396\n",
            "Processing batch 20701/39396\n",
            "Processing batch 20801/39396\n",
            "Processing batch 20901/39396\n",
            "Processing batch 21001/39396\n",
            "Processing batch 21101/39396\n",
            "Processing batch 21201/39396\n",
            "Processing batch 21301/39396\n",
            "Processing batch 21401/39396\n",
            "Processing batch 21501/39396\n",
            "Processing batch 21601/39396\n",
            "Processing batch 21701/39396\n",
            "Processing batch 21801/39396\n",
            "Processing batch 21901/39396\n",
            "Processing batch 22001/39396\n",
            "Processing batch 22101/39396\n",
            "Processing batch 22201/39396\n",
            "Processing batch 22301/39396\n",
            "Processing batch 22401/39396\n",
            "Processing batch 22501/39396\n",
            "Processing batch 22601/39396\n",
            "Processing batch 22701/39396\n",
            "Processing batch 22801/39396\n",
            "Processing batch 22901/39396\n",
            "Processing batch 23001/39396\n",
            "Processing batch 23101/39396\n",
            "Processing batch 23201/39396\n",
            "Processing batch 23301/39396\n",
            "Processing batch 23401/39396\n",
            "Processing batch 23501/39396\n",
            "Processing batch 23601/39396\n",
            "Processing batch 23701/39396\n",
            "Processing batch 23801/39396\n",
            "Processing batch 23901/39396\n",
            "Processing batch 24001/39396\n",
            "Processing batch 24101/39396\n",
            "Processing batch 24201/39396\n",
            "Processing batch 24301/39396\n",
            "Processing batch 24401/39396\n",
            "Processing batch 24501/39396\n",
            "Processing batch 24601/39396\n",
            "Processing batch 24701/39396\n",
            "Processing batch 24801/39396\n",
            "Processing batch 24901/39396\n",
            "Processing batch 25001/39396\n",
            "Processing batch 25101/39396\n",
            "Processing batch 25201/39396\n",
            "Processing batch 25301/39396\n",
            "Processing batch 25401/39396\n",
            "Processing batch 25501/39396\n",
            "Processing batch 25601/39396\n",
            "Processing batch 25701/39396\n",
            "Processing batch 25801/39396\n",
            "Processing batch 25901/39396\n",
            "Processing batch 26001/39396\n",
            "Processing batch 26101/39396\n",
            "Processing batch 26201/39396\n",
            "Processing batch 26301/39396\n",
            "Processing batch 26401/39396\n",
            "Processing batch 26501/39396\n",
            "Processing batch 26601/39396\n",
            "Processing batch 26701/39396\n",
            "Processing batch 26801/39396\n",
            "Processing batch 26901/39396\n",
            "Processing batch 27001/39396\n",
            "Processing batch 27101/39396\n",
            "Processing batch 27201/39396\n",
            "Processing batch 27301/39396\n",
            "Processing batch 27401/39396\n",
            "Processing batch 27501/39396\n",
            "Processing batch 27601/39396\n",
            "Processing batch 27701/39396\n",
            "Processing batch 27801/39396\n",
            "Processing batch 27901/39396\n",
            "Processing batch 28001/39396\n",
            "Processing batch 28101/39396\n",
            "Processing batch 28201/39396\n",
            "Processing batch 28301/39396\n",
            "Processing batch 28401/39396\n",
            "Processing batch 28501/39396\n",
            "Processing batch 28601/39396\n",
            "Processing batch 28701/39396\n",
            "Processing batch 28801/39396\n",
            "Processing batch 28901/39396\n",
            "Processing batch 29001/39396\n",
            "Processing batch 29101/39396\n",
            "Processing batch 29201/39396\n",
            "Processing batch 29301/39396\n",
            "Processing batch 29401/39396\n",
            "Processing batch 29501/39396\n",
            "Processing batch 29601/39396\n",
            "Processing batch 29701/39396\n",
            "Processing batch 29801/39396\n",
            "Processing batch 29901/39396\n",
            "Processing batch 30001/39396\n",
            "Processing batch 30101/39396\n",
            "Processing batch 30201/39396\n",
            "Processing batch 30301/39396\n",
            "Processing batch 30401/39396\n",
            "Processing batch 30501/39396\n",
            "Processing batch 30601/39396\n",
            "Processing batch 30701/39396\n",
            "Processing batch 30801/39396\n",
            "Processing batch 30901/39396\n",
            "Processing batch 31001/39396\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-56-32c1cf413fa3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"\\nhistory = defaultdict(list)\\nbest_accuracy = 0\\n\\nfor epoch in tqdm(range(num_epochs)):\\n\\n  print(f'Epoch {epoch + 1}/{num_epochs}')\\n  print('-' * 10)\\n\\n  train_acc, train_loss = train_epoch(\\n    model,\\n    train_loader,     \\n    optimizer, \\n    device, \\n    scheduler, \\n    len(train_labels)\\n  )\\n\\n  print(f'Train loss {train_loss} accuracy {train_acc}')\\n\\n  val_acc, val_loss = eval_model(\\n    model,\\n    val_loader,\\n    device, \\n    len(val_labels)\\n  )\\n\\n  print(f'Val   loss {val_loss} accuracy {val_acc}')\\n  print()\\n\\n  history['train_acc'].append(train_acc)\\n  history['train_loss'].append(train_loss)\\n  history['val_acc'].append(val_acc)\\n  history['val_loss'].append(val_loss)\\n\\n  if val_acc > best_accuracy:\\n    torch.save(model.state_dict(), 'best_model_state.bin')\\n    best_accuracy = val_acc\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2115\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2116\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2117\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2118\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<decorator-gen-53>\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/IPython/core/magics/execution.py\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m             \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1193\u001b[0;31m             \u001b[0mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1194\u001b[0m             \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m<ipython-input-52-908dd25d7bf7>\u001b[0m in \u001b[0;36mtrain_epoch\u001b[0;34m(model, data_loader, optimizer, device, scheduler, n_examples)\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0;31m# print(logits)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;31m# print(labels)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m     \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogits\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m     \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cpu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "opu4xshnyFgM",
        "outputId": "87c4c33c-0eae-4b8d-842c-f7387ff068e1"
      },
      "source": [
        "val_acc, val_loss = eval_model(\n",
        "model,\n",
        "val_loader,\n",
        "device, \n",
        "len(val_labels)\n",
        ")\n",
        "\n",
        "print(f'Val   loss {val_loss} accuracy {val_acc}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2190: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing batch 1/4378\n",
            "Processing batch 101/4378\n",
            "Processing batch 201/4378\n",
            "Processing batch 301/4378\n",
            "Processing batch 401/4378\n",
            "Processing batch 501/4378\n",
            "Processing batch 601/4378\n",
            "Processing batch 701/4378\n",
            "Processing batch 801/4378\n",
            "Processing batch 901/4378\n",
            "Processing batch 1001/4378\n",
            "Processing batch 1101/4378\n",
            "Processing batch 1201/4378\n",
            "Processing batch 1301/4378\n",
            "Processing batch 1401/4378\n",
            "Processing batch 1501/4378\n",
            "Processing batch 1601/4378\n",
            "Processing batch 1701/4378\n",
            "Processing batch 1801/4378\n",
            "Processing batch 1901/4378\n",
            "Processing batch 2001/4378\n",
            "Processing batch 2101/4378\n",
            "Processing batch 2201/4378\n",
            "Processing batch 2301/4378\n",
            "Processing batch 2401/4378\n",
            "Processing batch 2501/4378\n",
            "Processing batch 2601/4378\n",
            "Processing batch 2701/4378\n",
            "Processing batch 2801/4378\n",
            "Processing batch 2901/4378\n",
            "Processing batch 3001/4378\n",
            "Processing batch 3101/4378\n",
            "Processing batch 3201/4378\n",
            "Processing batch 3301/4378\n",
            "Processing batch 3401/4378\n",
            "Processing batch 3501/4378\n",
            "Processing batch 3601/4378\n",
            "Processing batch 3701/4378\n",
            "Processing batch 3801/4378\n",
            "Processing batch 3901/4378\n",
            "Processing batch 4001/4378\n",
            "Processing batch 4101/4378\n",
            "Processing batch 4201/4378\n",
            "Processing batch 4301/4378\n",
            "Val   loss 5969.303878962994 accuracy 0.7317836707157314\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ZiIvRraJ0eE"
      },
      "source": [
        "plt.plot(history['train_acc'], label='train accuracy')\n",
        "plt.plot(history['val_acc'], label='validation accuracy')\n",
        "\n",
        "plt.title('Training history')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend()\n",
        "plt.ylim([0, 1]);"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "86fTGiDOTVr9"
      },
      "source": [
        "torch.save(model.state_dict(),'/content/drive/MyDrive/AmazonMLChallenge/xlm_roberta_model_2.pth')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "77yq3-ipJW6k"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0ZY41qm2kFr"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zcN3dAK2iurR"
      },
      "source": [
        "class AmazonDataset_test(Dataset):\n",
        "\n",
        "  def __init__(self,ids, sentences, tokenizer, max_length):\n",
        "    self.sentences = sentences\n",
        "    self.ids = ids\n",
        "    self.tokenizer = tokenizer\n",
        "    self.max_length = max_length\n",
        "  \n",
        "  def __len__(self):\n",
        "    return len(self.sentences)\n",
        "  \n",
        "  def __getitem__(self, idx):\n",
        "    sentence = str(self.sentences[idx])\n",
        "    encoding = self.tokenizer.encode_plus(\n",
        "      sentence,\n",
        "      add_special_tokens=True,\n",
        "      max_length=self.max_length,\n",
        "      return_token_type_ids=False,\n",
        "      pad_to_max_length=True,\n",
        "      return_attention_mask=True,\n",
        "      return_tensors='pt',\n",
        "    )\n",
        "\n",
        "\n",
        "    return {\n",
        "        'sentence': sentence,\n",
        "        'id':self.ids[idx],\n",
        "        'input_ids': encoding['input_ids'].flatten(),\n",
        "        'attention_mask': encoding['attention_mask'].flatten(),\n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0QbZMCJRjUAh"
      },
      "source": [
        "ds_test = AmazonDataset_test(\n",
        "        sentences =test_df['TITLE'].values,\n",
        "        ids = test_df['PRODUCT_ID'].values,\n",
        "        tokenizer=tokenizer,\n",
        "        max_length=max_input_length,\n",
        "    )\n",
        "test_loader = DataLoader(\n",
        "        ds_test,\n",
        "        batch_size=batch_size\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jG6eE5JB2kTC"
      },
      "source": [
        "def get_predictions(model, data_loader):\n",
        "  model = model.eval()\n",
        "  ids = []\n",
        "  sentences = []\n",
        "  predictions = []\n",
        "  prediction_probs = []\n",
        "  #real_values = []\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for i,d in enumerate(data_loader):\n",
        "      if i%100 == 0:\n",
        "        print(f\"Processing batch {i+1}/{len(data_loader)}\")\n",
        "\n",
        "      sents = d[\"sentence\"]\n",
        "      input_ids = d[\"input_ids\"].to(device)\n",
        "      attention_mask = d[\"attention_mask\"].to(device)\n",
        "      id = d['id'].detach().cpu().numpy()\n",
        "    #   labels = d[\"labels\"].to(device)\n",
        "\n",
        "      outputs = model(\n",
        "        input_ids=input_ids,\n",
        "        attention_mask=attention_mask\n",
        "      )\n",
        "      \n",
        "      logits = outputs.logits\n",
        "      \n",
        "      logits = logits.detach().cpu().numpy()\n",
        "      \n",
        "    #   # Store predictions and true labels\n",
        "    #   predictions.append(logits)\n",
        "    #   true_labels.append(labels)\n",
        "      preds = np.argmax(logits, axis=1).flatten()\n",
        "\n",
        "      # Calculate simple flat accuracy -- number correct over total number.\n",
        "    #   correct_predictions += (preds == labels).sum()\n",
        "    #   _, preds = torch.max(outputs, dim=1)\n",
        "\n",
        "    #   probs = torch.nn.functional.softmax(outputs, dim=1)\n",
        "\n",
        "      sentences.extend(sents)\n",
        "      predictions.extend(preds)\n",
        "\n",
        "      ids.extend(id)\n",
        "    #   real_values.extend(labels)\n",
        "\n",
        "#   predictions = torch.stack(predictions).cpu()\n",
        "\n",
        "#   real_values = torch.stack(real_values).cpu()\n",
        "  return ids,sentences, predictions#, prediction_probs,# real_values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EK_ppNJug__f",
        "outputId": "e47345e0-9c5a-46fc-9abc-24f6ff4f2b88"
      },
      "source": [
        "ids,sents,predictions = get_predictions(model,test_loader)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2190: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing batch 1/1731\n",
            "Processing batch 101/1731\n",
            "Processing batch 201/1731\n",
            "Processing batch 301/1731\n",
            "Processing batch 401/1731\n",
            "Processing batch 501/1731\n",
            "Processing batch 601/1731\n",
            "Processing batch 701/1731\n",
            "Processing batch 801/1731\n",
            "Processing batch 901/1731\n",
            "Processing batch 1001/1731\n",
            "Processing batch 1101/1731\n",
            "Processing batch 1201/1731\n",
            "Processing batch 1301/1731\n",
            "Processing batch 1401/1731\n",
            "Processing batch 1501/1731\n",
            "Processing batch 1601/1731\n",
            "Processing batch 1701/1731\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "FL_O00lnj-u-",
        "outputId": "d411e2d7-0a09-45c7-c45b-27edd55c9333"
      },
      "source": [
        "df_sub = pd.DataFrame({\n",
        "    'PRODUCT_ID':ids,\n",
        "    'BROWSE_NODE_ID':le.inverse_transform(predictions)\n",
        "})\n",
        "df_sub"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>PRODUCT_ID</th>\n",
              "      <th>BROWSE_NODE_ID</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1140</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>15772</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>125</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>8915</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110770</th>\n",
              "      <td>110771</td>\n",
              "      <td>4368</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110771</th>\n",
              "      <td>110772</td>\n",
              "      <td>13568</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110772</th>\n",
              "      <td>110773</td>\n",
              "      <td>13520</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110773</th>\n",
              "      <td>110774</td>\n",
              "      <td>800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110774</th>\n",
              "      <td>110775</td>\n",
              "      <td>800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>110775 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        PRODUCT_ID  BROWSE_NODE_ID\n",
              "0                1            1140\n",
              "1                2           15772\n",
              "2                3             113\n",
              "3                4             125\n",
              "4                5            8915\n",
              "...            ...             ...\n",
              "110770      110771            4368\n",
              "110771      110772           13568\n",
              "110772      110773           13520\n",
              "110773      110774             800\n",
              "110774      110775             800\n",
              "\n",
              "[110775 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kBQin7KEkQrH"
      },
      "source": [
        "df_sub.to_csv('/content/drive/MyDrive/AmazonMLChallenge/submission_xlm_roberta.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0c25IG_H7PXS"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}