sentiment-analysis-using-roberta-lstm.ipynb

{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"gpuType":"T4","toc_visible":true},"accelerator":"GPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"source":"<a href=\"https://www.kaggle.com/code/minhcng/sentiment-analysis-using-roberta-lstm?scriptVersionId=137354058\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"code","source":"!pip install evaluate\n!pip install py_vncorenlp\n! pip install -U git+https://github.com/huggingface/transformers.git\n! pip install -U git+https://github.com/huggingface/accelerate.git\n","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"b8tEZpMFLS7R","outputId":"7362c94a-f504-4f28-9b4e-f885505f8138","execution":{"iopub.status.busy":"2023-07-20T07:58:42.599069Z","iopub.execute_input":"2023-07-20T07:58:42.599735Z","iopub.status.idle":"2023-07-20T08:00:34.58608Z","shell.execute_reply.started":"2023-07-20T07:58:42.599696Z","shell.execute_reply":"2023-07-20T08:00:34.584677Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting evaluate\n  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: datasets>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.1.0)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.23.5)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.6)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.5.3)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.28.2)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.64.1)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.2.0)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.14)\nRequirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2023.6.0)\nRequirement already satisfied: huggingface-hub>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.15.1)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from evaluate) (21.3)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.18.0)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (11.0.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.8.4)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (3.12.0)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (5.4.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->evaluate) (3.0.9)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2023.5.7)\nRequirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.1)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\nInstalling collected packages: evaluate\nSuccessfully installed evaluate-0.4.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mCollecting py_vncorenlp\n  Downloading py_vncorenlp-0.1.4.tar.gz (3.9 kB)\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hCollecting pyjnius (from py_vncorenlp)\n  Downloading pyjnius-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.4 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m18.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n\u001b[?25hBuilding wheels for collected packages: py_vncorenlp\n  Building wheel for py_vncorenlp (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for py_vncorenlp: filename=py_vncorenlp-0.1.4-py3-none-any.whl size=4324 sha256=419bee6c605aad9e1896ae8a3b14e07f5a4d61d978a44b7671e7f4201126013d\n  Stored in directory: /root/.cache/pip/wheels/d5/d9/bf/62632cdb007c702a0664091e92a0bb1f18a2fcecbe962d9827\nSuccessfully built py_vncorenlp\nInstalling collected packages: pyjnius, py_vncorenlp\nSuccessfully installed py_vncorenlp-0.1.4 pyjnius-1.5.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mCollecting git+https://github.com/huggingface/transformers.git\n  Cloning https://github.com/huggingface/transformers.git to /tmp/pip-req-build-a2sskig_\n  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-req-build-a2sskig_\n  Resolved https://github.com/huggingface/transformers.git to commit 6112b1c6442aaf7affd2b0676a1cd4eee30c45cf\n  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (3.12.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (0.15.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (5.4.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (2023.5.5)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (2.28.2)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (0.13.3)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (0.3.1)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.32.0.dev0) (4.64.1)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.32.0.dev0) (2023.6.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.32.0.dev0) (4.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers==4.32.0.dev0) (3.0.9)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.32.0.dev0) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.32.0.dev0) (3.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.32.0.dev0) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.32.0.dev0) (2023.5.7)\nBuilding wheels for collected packages: transformers\n  Building wheel for transformers (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for transformers: filename=transformers-4.32.0.dev0-py3-none-any.whl size=7404206 sha256=8f63fa0d5716e5b2b95fda95acad4743f7fe5658a1ac621087737c6230678bab\n  Stored in directory: /tmp/pip-ephem-wheel-cache-rk41s8v_/wheels/e7/9c/5b/e1a9c8007c343041e61cc484433d512ea9274272e3fcbe7c16\nSuccessfully built transformers\nInstalling collected packages: transformers\n  Attempting uninstall: transformers\n    Found existing installation: transformers 4.30.1\n    Uninstalling transformers-4.30.1:\n      Successfully uninstalled transformers-4.30.1\nSuccessfully installed transformers-4.32.0.dev0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mCollecting git+https://github.com/huggingface/accelerate.git\n  Cloning https://github.com/huggingface/accelerate.git to /tmp/pip-req-build-gh4z0kii\n  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/accelerate.git /tmp/pip-req-build-gh4z0kii\n  Resolved https://github.com/huggingface/accelerate.git to commit cafc7f785f25718b992d2e63d972f3d22b72617b\n  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.22.0.dev0) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.22.0.dev0) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate==0.22.0.dev0) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate==0.22.0.dev0) (5.4.1)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.22.0.dev0) (2.0.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate==0.22.0.dev0) (3.0.9)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.22.0.dev0) (3.12.0)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.22.0.dev0) (4.5.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.22.0.dev0) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.22.0.dev0) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.22.0.dev0) (3.1.2)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate==0.22.0.dev0) (2.1.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate==0.22.0.dev0) (1.3.0)\nBuilding wheels for collected packages: accelerate\n  Building wheel for accelerate (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for accelerate: filename=accelerate-0.22.0.dev0-py3-none-any.whl size=245637 sha256=9fa15ff424cc6f10766db232198912d164c497715e0cd82241ef38c9e12fa0b5\n  Stored in directory: /tmp/pip-ephem-wheel-cache-advfny8a/wheels/9c/a3/1e/47368f9b6575655fe9ee1b6350cfa7d4b0befe66a35f8a8365\nSuccessfully built accelerate\nInstalling collected packages: accelerate\n  Attempting uninstall: accelerate\n    Found existing installation: accelerate 0.12.0\n    Uninstalling accelerate-0.12.0:\n      Successfully uninstalled accelerate-0.12.0\nSuccessfully installed accelerate-0.22.0.dev0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import py_vncorenlp\npy_vncorenlp.download_model(save_dir=\"./\")","metadata":{"id":"vfsv3vbILS7U","execution":{"iopub.status.busy":"2023-07-20T08:00:34.592863Z","iopub.execute_input":"2023-07-20T08:00:34.595986Z","iopub.status.idle":"2023-07-20T08:00:34.612936Z","shell.execute_reply.started":"2023-07-20T08:00:34.595941Z","shell.execute_reply":"2023-07-20T08:00:34.611796Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"VnCoreNLP model folder . already exists! Please load VnCoreNLP from this folder!\n","output_type":"stream"}]},{"cell_type":"code","source":"!mkdir dataset\n!mkdir dataset/training\n!mkdir output\n!mkdir output/checkpoint\n!mkdir output/logging","metadata":{"id":"zsO4UCA8dPnz","execution":{"iopub.status.busy":"2023-07-20T08:00:34.614427Z","iopub.execute_input":"2023-07-20T08:00:34.615094Z","iopub.status.idle":"2023-07-20T08:00:39.953946Z","shell.execute_reply.started":"2023-07-20T08:00:34.615059Z","shell.execute_reply":"2023-07-20T08:00:39.952548Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"mkdir: cannot create directory ‘dataset’: File exists\nmkdir: cannot create directory ‘dataset/training’: File exists\nmkdir: cannot create directory ‘output’: File exists\nmkdir: cannot create directory ‘output/checkpoint’: File exists\nmkdir: cannot create directory ‘output/logging’: File exists\n","output_type":"stream"}]},{"cell_type":"code","source":"!cp ../input/sentiment-dataset/*.json dataset/training","metadata":{"execution":{"iopub.status.busy":"2023-07-20T08:02:57.351955Z","iopub.execute_input":"2023-07-20T08:02:57.352446Z","iopub.status.idle":"2023-07-20T08:02:58.828159Z","shell.execute_reply.started":"2023-07-20T08:02:57.352405Z","shell.execute_reply":"2023-07-20T08:02:58.826456Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"import os\n\nCURRENT_PATH = os.getcwd()\nTRAINING_DATASET_PATH = f\"{CURRENT_PATH}/dataset\"\nOUTPUT_DIR = f\"{CURRENT_PATH}/output\"\nRAW_DATASET_PATH = \"../input/vietnamese-sentiment-analyst/data - data.csv\"\n","metadata":{"id":"8crUd7FBLS7U","execution":{"iopub.status.busy":"2023-07-20T08:05:36.049621Z","iopub.execute_input":"2023-07-20T08:05:36.050032Z","iopub.status.idle":"2023-07-20T08:05:36.057245Z","shell.execute_reply.started":"2023-07-20T08:05:36.050001Z","shell.execute_reply":"2023-07-20T08:05:36.055641Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning Dataset","metadata":{"id":"5_jXF8_1LS7V"}},{"cell_type":"code","source":"import pandas as pd\ndataset = pd.read_csv(RAW_DATASET_PATH)","metadata":{"id":"Ha4gopWxLS7W","execution":{"iopub.status.busy":"2023-07-20T03:11:55.857191Z","iopub.execute_input":"2023-07-20T03:11:55.857589Z","iopub.status.idle":"2023-07-20T03:11:56.151402Z","shell.execute_reply.started":"2023-07-20T03:11:55.857552Z","shell.execute_reply":"2023-07-20T03:11:56.150454Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"dataset.shape","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cb4WBFMxLS7W","outputId":"6f92cb10-1fdb-425e-cf8e-2b2e87692ff1","execution":{"iopub.status.busy":"2023-07-20T03:11:56.152989Z","iopub.execute_input":"2023-07-20T03:11:56.153358Z","iopub.status.idle":"2023-07-20T03:11:56.162239Z","shell.execute_reply.started":"2023-07-20T03:11:56.153324Z","shell.execute_reply":"2023-07-20T03:11:56.161119Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"(31460, 4)"},"metadata":{}}]},{"cell_type":"code","source":"dataset.columns","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ry6JVlHvLS7X","outputId":"56f21252-3a83-4403-e97a-25fc420122c4","execution":{"iopub.status.busy":"2023-07-20T03:11:56.16387Z","iopub.execute_input":"2023-07-20T03:11:56.164753Z","iopub.status.idle":"2023-07-20T03:11:56.175298Z","shell.execute_reply.started":"2023-07-20T03:11:56.16472Z","shell.execute_reply":"2023-07-20T03:11:56.174129Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"Index(['comment', 'label', 'rate', 'Unnamed: 3'], dtype='object')"},"metadata":{}}]},{"cell_type":"code","source":"dataset.drop(columns=[\"Unnamed: 3\", \"rate\"], inplace=True, axis=1)\ndataset.columns","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4U9VakgkLS7X","outputId":"094405c6-a7f6-4a6c-8489-b3367b50b532","execution":{"iopub.status.busy":"2023-07-20T03:11:56.176912Z","iopub.execute_input":"2023-07-20T03:11:56.177277Z","iopub.status.idle":"2023-07-20T03:11:56.194969Z","shell.execute_reply.started":"2023-07-20T03:11:56.177245Z","shell.execute_reply":"2023-07-20T03:11:56.193764Z"},"trusted":true},"execution_count":8,"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"Index(['comment', 'label'], dtype='object')"},"metadata":{}}]},{"cell_type":"code","source":"dataset[\"label\"].value_counts()","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hNXC3PPnLS7X","outputId":"c8731f87-6942-4911-874a-fcabc0657bcf","execution":{"iopub.status.busy":"2023-07-20T03:11:56.196984Z","iopub.execute_input":"2023-07-20T03:11:56.19739Z","iopub.status.idle":"2023-07-20T03:11:56.217892Z","shell.execute_reply.started":"2023-07-20T03:11:56.197348Z","shell.execute_reply":"2023-07-20T03:11:56.216936Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"POS    20093\nNEG     6669\nNEU     4698\nName: label, dtype: int64"},"metadata":{}}]},{"cell_type":"code","source":"dataset=dataset.drop_duplicates(\"comment\")\ndataset = dataset.dropna()","metadata":{"id":"rHUSfZanLS7Y","execution":{"iopub.status.busy":"2023-07-20T03:11:56.223515Z","iopub.execute_input":"2023-07-20T03:11:56.223862Z","iopub.status.idle":"2023-07-20T03:11:56.255029Z","shell.execute_reply.started":"2023-07-20T03:11:56.223829Z","shell.execute_reply":"2023-07-20T03:11:56.254092Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"dataset[\"label\"].value_counts()","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cciAmTG4LS7Y","outputId":"7c6b68b3-c2e7-41a8-cd2d-b28027814973","execution":{"iopub.status.busy":"2023-07-20T03:11:56.256245Z","iopub.execute_input":"2023-07-20T03:11:56.257234Z","iopub.status.idle":"2023-07-20T03:11:56.269454Z","shell.execute_reply.started":"2023-07-20T03:11:56.257199Z","shell.execute_reply":"2023-07-20T03:11:56.268493Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"POS    16222\nNEG     6330\nNEU     4251\nName: label, dtype: int64"},"metadata":{}}]},{"cell_type":"code","source":"dataset[\"comment\"].sample(40)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"SUwa4NNzLS7Y","outputId":"85c43fd2-2edc-4165-c9a4-e2560f0c77f2","execution":{"iopub.status.busy":"2023-07-20T03:11:56.271272Z","iopub.execute_input":"2023-07-20T03:11:56.271779Z","iopub.status.idle":"2023-07-20T03:11:56.284212Z","shell.execute_reply.started":"2023-07-20T03:11:56.271745Z","shell.execute_reply":"2023-07-20T03:11:56.283081Z"},"trusted":true},"execution_count":12,"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"17130    Sản phẩm không đẹp,bị trầy xướt tùm lùm hết,kh...\n17114    Chất lượng sản phẩm tuyệt vời chất lượng đi đô...\n6148                                              Hơi mặn.\n25175                                                  💗💗.\n17666                                Áo đẹp xuất sắc luôn.\n17658             Áo bị lỗi mong shop kiểm tra lại kĩ hơn.\n7336     Áo đẹp lắm ạ ,chất vải ok , logo và chữ in rất...\n11707       Đóng gói sản phẩm rất đẹp và chắc chắn, nhanh.\n26065    Thời gian giao hàng rất nhanh Shop phục vụ rất...\n6139        Áo đẹp nhưng có điều tay hơi ngắn so với hình.\n1237     Hàng ngon, balo handmade đấy các ô (đọc trên c...\n5075                          Sao check mã ko ra shop nhỉ.\n4567       Form đẹp, shop còn tặng 1 gương mini dễ thương.\n14953                     Máy ồn hàng lắp ráp k chắc chắn.\n7067     K biết có phải nguyên chất k Chất lượng sản ph...\n6980                                 Nhưng màu in hơi lem.\n15712    Chất lượng sản phẩm tuyệt vời găng tay đẹp cơ ...\n8608     giao hàng hơi chậm, nhưng chất lượng rất tốt l...\n17536                                   Mình thấy rất ưng.\n29721                               Đàn đẹp Rất đáng tiền.\n17981                         Vải k thấm mồ hôi, hơi mỏng.\n18844                 Mình không thích màu son và mùi lắm.\n21648    Áo xinh lắm ạ, thích shop cập nhật nhiều mẫu q...\n111                Đóng gói sản phẩm rất đẹp và chắc chắc.\n3354     sản phẩm y hình, chất lượng rất đẹp, giao hàng...\n23431                                            K lộ béo.\n2632                   Sẽ theo dõi shop để ủng hộ lần sau.\n22334                                                Đựơc.\n5707                                                 hinh.\n12762                                     Tào lao chị nào.\n9624     Chất lượng sản phẩm tuyệt vời, vải tốt tiếp tụ...\n149      sản phẩm rất ok, kể cả áo trong của nữ chất vả...\n25503    Quần rất tốt mát đẹp giao hàng nhanh Chất lượn...\n27398    Áo đẹp, dày dặn cầm nặng tay, đặc biệt là lông...\n9329     Mình lấy cái màu be mà ngoài này nhìn thành mà...\n6836     Vỏ chặt nên còn chưa lấy ra, bên ngoài thì hộp...\n10772    đặt 2c và thanh toán tiền 2c nhưng khi mở ra t...\n10313    Lần đầu tiên mình đánh giá 5* cho 1 shop ơ sho...\n20024                                           Ung y quá.\n20555                             Hình sao hàng vậy 😍👏🏻👏🏻.\nName: comment, dtype: object"},"metadata":{}}]},{"cell_type":"code","source":"import regex as re\nimport string\nimport json\n\nemoji_pattern = re.compile(\"[\"\n                u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                u\"\\U00002702-\\U000027B0\"\n                u\"\\U000024C2-\\U0001F251\"\n                u\"\\U0001f926-\\U0001f937\"\n                u'\\U00010000-\\U0010ffff'\n                u\"\\u200d\"\n                u\"\\u2640-\\u2642\"\n                u\"\\u2600-\\u2B55\"\n                u\"\\u23cf\"\n                u\"\\u23e9\"\n                u\"\\u231a\"\n                u\"\\u3030\"\n                u\"\\ufe0f\"\n    \"]+\", flags=re.UNICODE)\n\ndef clean_text(text):\n    text = text.lower()\n    text = re.sub(emoji_pattern, \" \", text)\n    text = re.sub(r'([a-z]+?)\\1+',r'\\1', text)\n    text = re.sub(r\"(\\w)\\s*([\" + string.punctuation + \"])\\s*(\\w)\", r\"\\1 \\2 \\3\", text)\n    text = re.sub(r\"(\\w)\\s*([\" + string.punctuation + \"])\", r\"\\1 \\2\", text)\n    # text = re.sub(r\"(\\d)([^\\d.])\", r\"\\1 \\2\", text)\n    # text = re.sub(r\"([^\\d.])(\\d)\", r\"\\1 \\2\", text)\n    text = re.sub(f\"([{string.punctuation}])([{string.punctuation}])+\",r\"\\1\", text)\n    text = text.strip()\n    while text.endswith(tuple(string.punctuation+string.whitespace)):\n        text = text[:-1]\n    while text.startswith(tuple(string.punctuation+string.whitespace)):\n        text = text[1:]\n    text = re.sub(r\"\\s+\", \" \", text)\n    return text\n\ndef map_label(label):\n    label_map = {\n        \"POS\": 0,\n        \"NEG\": 1,\n        \"NEU\": 2\n    }\n    return label_map[label]\n","metadata":{"id":"uuMOXnQaN3x9","execution":{"iopub.status.busy":"2023-07-20T03:11:56.286407Z","iopub.execute_input":"2023-07-20T03:11:56.286812Z","iopub.status.idle":"2023-07-20T03:11:56.323274Z","shell.execute_reply.started":"2023-07-20T03:11:56.286769Z","shell.execute_reply":"2023-07-20T03:11:56.322325Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"dataset[\"comment\"] = dataset[\"comment\"].map(lambda text: clean_text(text))\ndataset[\"label\"] = dataset[\"label\"].map(lambda label: map_label(label))","metadata":{"id":"nN4pjCbuNjhm","execution":{"iopub.status.busy":"2023-07-20T03:11:56.324793Z","iopub.execute_input":"2023-07-20T03:11:56.325121Z","iopub.status.idle":"2023-07-20T03:12:00.486735Z","shell.execute_reply.started":"2023-07-20T03:11:56.325092Z","shell.execute_reply":"2023-07-20T03:12:00.4855Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"dataset[\"comment\"].sample(40)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"jdsqFxP5Ny2e","outputId":"d8614007-70c1-4a16-e484-883f96f38fa6","execution":{"iopub.status.busy":"2023-07-20T03:12:00.488284Z","iopub.execute_input":"2023-07-20T03:12:00.48891Z","iopub.status.idle":"2023-07-20T03:12:00.500219Z","shell.execute_reply.started":"2023-07-20T03:12:00.488869Z","shell.execute_reply":"2023-07-20T03:12:00.499126Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"676                                     vải áo rất mềm mại\n8395     toi k ưng áo màu đỏ bn gửi cho t . đât hai ảo ...\n11763        kiến gió nhà mình k ăn , nên k đuổi được kiến\n19105    không giống hình , bán xong hỏi shop hok muốn ...\n8834                        51 kg mà 2xl vẫn ko kéo lên đc\n30532                                              rất ưng\n27403                                             điên máu\n12840                               form rộng như hình nhé\n7171                                               màu đẹp\n7914     đã nt yêu cầu shop gửi sớm vì có việc cần nhưn...\n15966                          lần sau ủng hộ tiếp dài dài\n23822         áo đẹp hơn mong đợi , đường may cung rất đẹp\n27560                     áo đẹp . chị chủ shop nhiệt tình\n14928                                  chất vải ok mềm mịn\n25806    giao k đúng màu , có 1cái giao size nhỏ hơn k ...\n21406                                   còn thừa nhiều quá\n28128                                            k bám môi\n1761                                           son hơi khô\n26394                        in áo k đẹp chỉ thừa rất nhìu\n7428                            vải rất ổn và may kiểu đẹp\n30238    áo mình đặt màu đen nhưng lại giao màu trắng c...\n18008    nhưng mà thôi cũng được . chất lượng sản phẩm ...\n18449    chị chủ shop nhiệt tình kinh khủng luôn nè vải...\n9410                       mua hàng lần nào cũng ưng ý lắm\n13090    chất lượng sản phẩm tuyệt vời chất lượng sản p...\n24318                                    hàng đẹp như hình\n19465    shop phục vụ rất tốt đóng gói sản phẩm rất đẹp...\n2466                                             đúng size\n18818                                   cảm thấy thất vọng\n26581                                         chị chủ cute\n17227                     ko giống mấy áo kaki bình thường\n7893     màu đậm hơn so vs ảnh nhưng chất vải rất mát ,...\n8825     tiếp nhận đơn hàng và đưa cho bên vận chuyển c...\n8011     giao thiếu hàng , rõ ràng bộ sản phẩm nhưng ch...\n18401                                    nhưng vừa với giá\n343      chất lượng sản phẩm tuyệt vời chất lượng sản p...\n30859                                    nhận hàng rất ưng\n713                                     chuẩn ảnh , rất ok\n1541                                       mua thẻ nhớ ntn\n27143                        mua về vất đi không đáng tiền\nName: comment, dtype: object"},"metadata":{}}]},{"cell_type":"code","source":"dataset=dataset.drop_duplicates(\"comment\")\ndataset = dataset.dropna()","metadata":{"id":"cQIXiTxIMyd4","execution":{"iopub.status.busy":"2023-07-20T03:12:00.502053Z","iopub.execute_input":"2023-07-20T03:12:00.502415Z","iopub.status.idle":"2023-07-20T03:12:00.534937Z","shell.execute_reply.started":"2023-07-20T03:12:00.502384Z","shell.execute_reply":"2023-07-20T03:12:00.53403Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"dataset = dataset.drop(dataset[dataset[\"comment\"].map(len) < 2].index)","metadata":{"id":"mkqZanvrS1bP","execution":{"iopub.status.busy":"2023-07-20T03:12:00.53627Z","iopub.execute_input":"2023-07-20T03:12:00.536628Z","iopub.status.idle":"2023-07-20T03:12:00.559917Z","shell.execute_reply.started":"2023-07-20T03:12:00.536597Z","shell.execute_reply":"2023-07-20T03:12:00.55895Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"dataset[\"label\"].value_counts()","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"LrPkNzOtTtpw","outputId":"d405b396-de3c-4954-db81-27d9ff2fbdbf","execution":{"iopub.status.busy":"2023-07-20T03:12:00.563354Z","iopub.execute_input":"2023-07-20T03:12:00.563761Z","iopub.status.idle":"2023-07-20T03:12:00.57325Z","shell.execute_reply.started":"2023-07-20T03:12:00.56373Z","shell.execute_reply":"2023-07-20T03:12:00.572279Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"0    15248\n1     6213\n2     4138\nName: label, dtype: int64"},"metadata":{}}]},{"cell_type":"code","source":"import random\n_dataset = {\n    \"train\": [],\n    \"eval\": [],\n    \"test\": []\n}\ntemp_dataset = [(data[\"comment\"], data[\"label\"]) for _, data in dataset.iterrows()]\nrandom.seed(1234)\nrandom.shuffle(temp_dataset)","metadata":{"id":"vzonYcV1UKP3","execution":{"iopub.status.busy":"2023-07-20T03:12:00.574613Z","iopub.execute_input":"2023-07-20T03:12:00.575589Z","iopub.status.idle":"2023-07-20T03:12:02.143619Z","shell.execute_reply.started":"2023-07-20T03:12:00.575557Z","shell.execute_reply":"2023-07-20T03:12:02.142554Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"temp_dataset[:5]","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7_0bmgSKVS3o","outputId":"15265280-5b94-4241-aeca-8994841f675b","execution":{"iopub.status.busy":"2023-07-20T03:12:02.145107Z","iopub.execute_input":"2023-07-20T03:12:02.145459Z","iopub.status.idle":"2023-07-20T03:12:02.154572Z","shell.execute_reply.started":"2023-07-20T03:12:02.145414Z","shell.execute_reply":"2023-07-20T03:12:02.153618Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"[('vì mấy đánh giá 5 * mà mình tin tưởng lầm', 1),\n ('đổi sang rêu', 2),\n ('ngon mà', 0),\n ('chất lượng sản phẩm tuyệt vời , đẹp xuất sắc', 0),\n ('áo size s rồi nhưng thật sự còn quá rộng so với mình , chất đẹp , mát , nhân viên phục vụ tốt , giao hàng chậm , chủ shop đáng yêu',\n  0)]"},"metadata":{}}]},{"cell_type":"code","source":"split_ratio = {\n    \"train\": 0.8,\n    \"eval\": 0.9,\n    \"test\": 1\n}\n\n_i = 0\n_len = len(temp_dataset)\nfor mode in _dataset.keys():\n    _dataset[mode] = temp_dataset[_i:int(_len*split_ratio[mode])+1]\n    _i = int(_len*split_ratio[mode])+1","metadata":{"id":"RG-UZF0wV3-5","execution":{"iopub.status.busy":"2023-07-20T03:12:02.156272Z","iopub.execute_input":"2023-07-20T03:12:02.157048Z","iopub.status.idle":"2023-07-20T03:12:02.164176Z","shell.execute_reply.started":"2023-07-20T03:12:02.156884Z","shell.execute_reply":"2023-07-20T03:12:02.163017Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\nimport json\n\nfor mode in _dataset.keys():\n    print(len(_dataset[mode]))\n    a = Counter([e[1] for e in _dataset[mode]])\n    print(a)\n    json.dump({\"data\":_dataset[mode]}, open(f\"{TRAINING_DATASET_PATH}/training/{mode}.json\", \"w\", encoding=\"utf8\"), indent=4, ensure_ascii=False)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VxyyiYuJYsTS","outputId":"980283b7-875c-4ff5-a5df-7403a81bb823","execution":{"iopub.status.busy":"2023-07-20T03:12:02.165491Z","iopub.execute_input":"2023-07-20T03:12:02.166064Z","iopub.status.idle":"2023-07-20T03:12:02.356009Z","shell.execute_reply.started":"2023-07-20T03:12:02.166027Z","shell.execute_reply":"2023-07-20T03:12:02.355077Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"20480\nCounter({0: 12203, 1: 4994, 2: 3283})\n2560\nCounter({0: 1492, 1: 617, 2: 451})\n2559\nCounter({0: 1553, 1: 602, 2: 404})\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Training","metadata":{"id":"35fpS_g4LS7Y"}},{"cell_type":"markdown","source":"## Import packages","metadata":{"id":"8S6EuVix8wy4"}},{"cell_type":"code","source":"from transformers import AutoModel, AutoTokenizer, RobertaForSequenceClassification\nfrom transformers import Trainer, SchedulerType\nfrom transformers.training_args import TrainingArguments, OptimizerNames\nfrom transformers.trainer_utils import IntervalStrategy\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset\nimport py_vncorenlp\nimport json\nimport random","metadata":{"id":"UIFYf7fvLS7Y","execution":{"iopub.status.busy":"2023-07-20T08:03:24.561788Z","iopub.execute_input":"2023-07-20T08:03:24.562198Z","iopub.status.idle":"2023-07-20T08:03:41.639016Z","shell.execute_reply.started":"2023-07-20T08:03:24.562167Z","shell.execute_reply":"2023-07-20T08:03:41.637871Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n  warnings.warn(f\"file system plugins are not loaded: {e}\")\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Config","metadata":{"id":"0W4o8I_fLS7Y"}},{"cell_type":"code","source":"!pip install vncorenlp","metadata":{"execution":{"iopub.status.busy":"2023-07-20T08:03:41.641431Z","iopub.execute_input":"2023-07-20T08:03:41.642416Z","iopub.status.idle":"2023-07-20T08:03:56.657253Z","shell.execute_reply.started":"2023-07-20T08:03:41.642354Z","shell.execute_reply":"2023-07-20T08:03:56.655962Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stdout","text":"Collecting vncorenlp\n  Downloading vncorenlp-1.0.3.tar.gz (2.6 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m25.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from vncorenlp) (2.28.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->vncorenlp) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->vncorenlp) (3.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->vncorenlp) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->vncorenlp) (2023.5.7)\nBuilding wheels for collected packages: vncorenlp\n  Building wheel for vncorenlp (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for vncorenlp: filename=vncorenlp-1.0.3-py3-none-any.whl size=2645951 sha256=09741f6b15ff49fecb84be20875e90a89a285626cc5861b784bdcc3980f0be48\n  Stored in directory: /root/.cache/pip/wheels/5d/d9/b3/41f6c6b1ab758561fd4aab55dc0480b9d7a131c6aaa573a3fa\nSuccessfully built vncorenlp\nInstalling collected packages: vncorenlp\nSuccessfully installed vncorenlp-1.0.3\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"from vncorenlp import VnCoreNLP\nclass PipelineConfig:\n    rdrsegmenter = VnCoreNLP(\"./VnCoreNLP-1.2.jar\", annotators=\"wseg\")","metadata":{"id":"c7ZwDVDw0Fxh","execution":{"iopub.status.busy":"2023-07-20T08:08:42.051349Z","iopub.execute_input":"2023-07-20T08:08:42.051748Z","iopub.status.idle":"2023-07-20T08:08:47.303393Z","shell.execute_reply.started":"2023-07-20T08:08:42.051715Z","shell.execute_reply":"2023-07-20T08:08:47.302146Z"},"trusted":true},"execution_count":46,"outputs":[]},{"cell_type":"code","source":"PipelineConfig.rdrsegmenter.tokenize(\"Tôi là một kỹ sư AI\")","metadata":{"execution":{"iopub.status.busy":"2023-07-20T08:08:47.306326Z","iopub.execute_input":"2023-07-20T08:08:47.306638Z","iopub.status.idle":"2023-07-20T08:08:47.335259Z","shell.execute_reply.started":"2023-07-20T08:08:47.306611Z","shell.execute_reply":"2023-07-20T08:08:47.334345Z"},"trusted":true},"execution_count":47,"outputs":[{"execution_count":47,"output_type":"execute_result","data":{"text/plain":"[['Tôi', 'là', 'một', 'kỹ_sư', 'AI']]"},"metadata":{}}]},{"cell_type":"code","source":"class SentimentConfig:\n    def __init__(\n            self,\n            pretrained_path: str = None,\n            dataset_path: str = None,\n            training_output_dir: str = None,\n            training_learning_rate: float = None,\n            training_weight_decay: float = None,\n            training_batch_size: int = None,\n            training_num_epochs: int = None,\n            training_save_total_limit: int = None,\n            training_gradient_accumulation_steps: int = None,\n            training_eval_steps: int = None,\n            training_logging_steps: int = None,\n            training_save_steps: int = None,\n            training_logging_dir: str = None,\n            training_warm_up_ratio: float = None,\n            training_device: str = None,\n            training_metrics: str = None,\n            segmentor_dir:str = None,\n            model_input_max_length:int= None,\n            model_num_labels:int=None\n    ):\n        self.pretrained_path = pretrained_path if pretrained_path is not None else \"vinai/phobert-base-v2\"\n        self.dataset_path = dataset_path if dataset_path is not None else f\"{TRAINING_DATASET_PATH}/training\"\n        self.training_output_dir = training_output_dir if training_output_dir is not None else f\"{OUTPUT_DIR}/checkpoint/checkpoint20_1\"\n        self.training_logging_dir = training_logging_dir if training_logging_dir is not None else f\"{OUTPUT_DIR}/logging\"\n        self.training_learning_rate = training_learning_rate if training_learning_rate is not None else 1.5e-5\n        self.training_weight_decay = training_weight_decay if training_weight_decay is not None else 0.01\n        self.training_batch_size = training_batch_size if training_batch_size is not None else 32\n        self.training_num_epochs = training_num_epochs if training_num_epochs is not None else 10\n        self.training_save_total_limit = training_save_total_limit if training_save_total_limit is not None else 3\n        self.training_gradient_accumulation_steps = training_gradient_accumulation_steps if training_gradient_accumulation_steps is not None else 2\n        self.training_eval_steps = training_eval_steps if training_eval_steps is not None else 200\n        self.training_logging_steps = training_logging_steps if training_logging_steps is not None else 200\n        self.training_save_steps = training_save_steps if training_save_steps is not None else 200\n        self.training_warm_up_ratio = training_warm_up_ratio if training_warm_up_ratio is not None else 0.05\n        self.training_device = training_device if training_device is not None else \"cuda\"\n        self.training_metrics = training_metrics if training_metrics is not None else \"f1\"\n        self.model_input_max_length = model_input_max_length if model_input_max_length is not None else 256\n        self.model_num_labels = model_num_labels if model_num_labels is not None else 3","metadata":{"id":"TKNNquc-LS7Z","execution":{"iopub.status.busy":"2023-07-20T08:04:02.333013Z","iopub.execute_input":"2023-07-20T08:04:02.335352Z","iopub.status.idle":"2023-07-20T08:04:02.360451Z","shell.execute_reply.started":"2023-07-20T08:04:02.335318Z","shell.execute_reply":"2023-07-20T08:04:02.359444Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{"id":"Nq4P2WLvLS7Z"}},{"cell_type":"code","source":"random.seed(2023)\n\nclass SentimentDataset(Dataset):\n    def __init__(self, config, mode, tokenizer):\n        super().__init__()\n        self.config = config\n        self.tokenizer = tokenizer\n        self.data = self.load_dataset(path=f\"{self.config.dataset_path}/{mode}.json\")\n        random.shuffle(self.data)\n        print(Counter([e[1] for e in self.data]))\n        print(f\"Load {len(self.data)} examples for {mode} dataset\")\n\n    def load_dataset(self, path: str):\n        data = json.load(open(path, \"r\", encoding=\"utf8\"))\n        return data[\"data\"]\n\n    def __getitem__(self, idx):\n        data_item = self.data[idx]\n        labels = torch.zeros(self.config.model_num_labels)\n        labels[data_item[-1]] = 1\n        text = data_item[0].lower()\n        segmented_text = PipelineConfig.rdrsegmenter.tokenize(text)\n        tokenized_text = self.tokenizer([\" \".join(segmented_text[0])], padding=\"max_length\", truncation=True, max_length=self.config.model_input_max_length, return_tensors=\"pt\")\n        return {\n            **tokenized_text,\n            \"labels\": labels\n        }\n\n    def __len__(self):\n        return len(self.data)\n","metadata":{"id":"od47hPjtLS7Z","execution":{"iopub.status.busy":"2023-07-20T08:06:37.936946Z","iopub.execute_input":"2023-07-20T08:06:37.93735Z","iopub.status.idle":"2023-07-20T08:06:37.950529Z","shell.execute_reply.started":"2023-07-20T08:06:37.937321Z","shell.execute_reply":"2023-07-20T08:06:37.949205Z"},"trusted":true},"execution_count":33,"outputs":[]},{"cell_type":"markdown","source":"## Define Trainer for training model","metadata":{"id":"hkezRT6HLS7Z"}},{"cell_type":"code","source":"import os\nos.environ[\"WANDB_DISABLED\"] = \"true\"","metadata":{"execution":{"iopub.status.busy":"2023-07-20T08:06:40.569877Z","iopub.execute_input":"2023-07-20T08:06:40.570299Z","iopub.status.idle":"2023-07-20T08:06:40.575452Z","shell.execute_reply.started":"2023-07-20T08:06:40.570268Z","shell.execute_reply":"2023-07-20T08:06:40.574223Z"},"trusted":true},"execution_count":34,"outputs":[]},{"cell_type":"markdown","source":"### Trainer class","metadata":{"id":"1ON31HwO8naz"}},{"cell_type":"code","source":"from typing import Optional\nfrom torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss\nfrom transformers.modeling_outputs import SequenceClassifierOutput\n\n\nclass SentimentRobertaClassificationHead(nn.Module):\n    \"\"\"Head for sentence-level classification tasks.\"\"\"\n\n    def __init__(self, config):\n        super().__init__()\n        self.lstm = nn.LSTM(config.hidden_size, config.hidden_size, 2, bidirectional=True)\n        self.dense = nn.Linear(2 * config.hidden_size, config.hidden_size)\n        classifier_dropout = (\n            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob\n        )\n        self.dropout = nn.Dropout(classifier_dropout)\n        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)\n\n    def forward(self, features, **kwargs):\n        # x = features[:, 0, :]  # take <s> token (equiv. to [CLS])\n        x = features.mean(dim=1)\n        x, (h, c) = self.lstm(x)\n        x = self.dropout(x)\n        x = self.dense(x)\n        x = torch.tanh(x)\n        x = self.dropout(x)\n        x = self.out_proj(x)\n        return x\n\n\nclass SentimentModel(RobertaForSequenceClassification):\n  def __init__(self, config):\n    super().__init__(config)\n    self.classifier = SentimentRobertaClassificationHead(config)\n\n  def forward(\n        self,\n        input_ids: Optional[torch.LongTensor] = None,\n        attention_mask: Optional[torch.FloatTensor] = None,\n        token_type_ids: Optional[torch.LongTensor] = None,\n        position_ids: Optional[torch.LongTensor] = None,\n        head_mask: Optional[torch.FloatTensor] = None,\n        inputs_embeds: Optional[torch.FloatTensor] = None,\n        labels: Optional[torch.LongTensor] = None,\n        output_attentions: Optional[bool] = None,\n        output_hidden_states: Optional[bool] = None,\n        return_dict: Optional[bool] = None,\n    ):\n\n      return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n      outputs = self.roberta(\n          input_ids,\n          attention_mask=attention_mask,\n          token_type_ids=token_type_ids,\n          position_ids=position_ids,\n          head_mask=head_mask,\n          inputs_embeds=inputs_embeds,\n          output_attentions=output_attentions,\n          output_hidden_states=output_hidden_states,\n          return_dict=return_dict,\n      )\n      sequence_output = outputs[0]\n      logits = self.classifier(sequence_output)\n\n      loss = None\n      if labels is not None:\n          # move labels to correct device to enable model parallelism\n          labels = labels.to(logits.device)\n          if self.config.problem_type is None:\n              if self.num_labels == 1:\n                  self.config.problem_type = \"regression\"\n              elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):\n                  self.config.problem_type = \"single_label_classification\"\n              else:\n                  self.config.problem_type = \"multi_label_classification\"\n\n          if self.config.problem_type == \"regression\":\n              loss_fct = MSELoss()\n              if self.num_labels == 1:\n                  loss = loss_fct(logits.squeeze(), labels.squeeze())\n              else:\n                  loss = loss_fct(logits, labels)\n          elif self.config.problem_type == \"single_label_classification\":\n              loss_fct = CrossEntropyLoss()\n              loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n          elif self.config.problem_type == \"multi_label_classification\":\n              loss_fct = BCEWithLogitsLoss()\n              loss = loss_fct(logits, labels)\n\n      if not return_dict:\n          output = (logits,) + outputs[2:]\n          return ((loss,) + output) if loss is not None else output\n\n      return SequenceClassifierOutput(\n          loss=loss,\n          logits=logits,\n          hidden_states=outputs.hidden_states,\n          attentions=outputs.attentions,\n      )","metadata":{"id":"xcDhApTGCy7p","execution":{"iopub.status.busy":"2023-07-20T08:06:41.854295Z","iopub.execute_input":"2023-07-20T08:06:41.854721Z","iopub.status.idle":"2023-07-20T08:06:41.877359Z","shell.execute_reply.started":"2023-07-20T08:06:41.854688Z","shell.execute_reply":"2023-07-20T08:06:41.87601Z"},"trusted":true},"execution_count":35,"outputs":[]},{"cell_type":"code","source":"from evaluate import load\nfrom transformers.trainer_utils import EvalPrediction\nfrom collections import Counter\n\n\nclass SentimentTrainer:\n    def __init__(self, config: SentimentConfig = None):\n        self.config = config if config is not None else SentimentConfig()\n        self.model = SentimentModel.from_pretrained(self.config.pretrained_path, num_labels=self.config.model_num_labels)\n        self.model.to(self.config.training_device)\n        self.tokenizer = AutoTokenizer.from_pretrained(self.config.pretrained_path)\n        self.metrics = {\n            \"precision\": load(\"precision\"),\n            \"recall\": load(\"recall\"),\n            \"f1\": load(\"f1\"),\n            \"accuracy\": load(\"accuracy\")\n        }\n        self.train_dataset, self.eval_dataset = self.load_dataset()\n\n        self.init_trainer()\n\n    def load_dataset(self):\n        train_dataset = SentimentDataset(config=self.config, mode=\"train\", tokenizer=self.tokenizer)\n        eval_dataset = SentimentDataset(config=self.config, mode=\"eval\", tokenizer=self.tokenizer)\n        return train_dataset, eval_dataset\n\n    def collator(self, data):\n        batch = {}\n        _keys = data[0].keys()\n        for key in _keys:\n            batch[key] = torch.vstack([ele[key] for ele in data]).to(self.config.training_device)\n\n        return batch\n\n    def compute_metrics(self, eval_predictions: EvalPrediction):\n        predictions, labels = eval_predictions\n        predictions = torch.from_numpy(predictions).softmax(dim=-1).argmax(dim=-1)\n        labels = torch.from_numpy(labels).argmax(dim=-1)\n\n        output =  {\n            \"precision\": self.metrics[\"precision\"].compute(predictions=predictions, references=labels,\n                                                           average=\"weighted\")[\"precision\"],\n            \"recall\": self.metrics[\"recall\"].compute(predictions=predictions, references=labels, average=\"weighted\")[\"recall\"],\n            \"f1\": self.metrics[\"f1\"].compute(predictions=predictions, references=labels, average=\"weighted\")[\"f1\"],\n            \"accuracy\": self.metrics[\"accuracy\"].compute(predictions=predictions, references=labels)[\"accuracy\"]\n        }\n        return output\n\n    def init_trainer(self):\n        training_args = TrainingArguments(\n            output_dir=self.config.training_output_dir,\n            evaluation_strategy=IntervalStrategy.STEPS,\n            save_strategy=IntervalStrategy.STEPS,\n            per_device_train_batch_size=self.config.training_batch_size,\n            per_device_eval_batch_size=self.config.training_batch_size,\n            learning_rate=self.config.training_learning_rate,\n            weight_decay=self.config.training_weight_decay,\n            save_total_limit=self.config.training_save_total_limit,\n            num_train_epochs=self.config.training_num_epochs,\n            gradient_accumulation_steps=2,\n            eval_steps=self.config.training_eval_steps,\n            fp16=self.config.training_device == \"cuda\",\n            push_to_hub=False,\n            dataloader_num_workers=0,\n            logging_dir=self.config.training_logging_dir,\n            logging_strategy=IntervalStrategy.STEPS,\n            logging_steps=self.config.training_logging_steps,\n            save_steps=self.config.training_save_steps,\n            logging_first_step=False,\n            optim=OptimizerNames.ADAMW_TORCH,\n            load_best_model_at_end=True,\n            metric_for_best_model=self.config.training_metrics,\n            warmup_ratio=self.config.training_warm_up_ratio,\n            lr_scheduler_type=SchedulerType.COSINE_WITH_RESTARTS,\n            dataloader_pin_memory=False\n        )\n        self.trainer = Trainer(\n            model=self.model,\n            args=training_args,\n            train_dataset=self.train_dataset,\n            eval_dataset=self.eval_dataset,\n            tokenizer=self.tokenizer,\n            data_collator=self.collator,\n            compute_metrics=self.compute_metrics\n        )\n\n    def train(self):\n        self.trainer.train()\n\n    def eval(self):\n        print(self.trainer.evaluate(eval_dataset=self.eval_dataset))\n","metadata":{"id":"9QPS4PbDLS7Z","execution":{"iopub.status.busy":"2023-07-20T08:06:43.287527Z","iopub.execute_input":"2023-07-20T08:06:43.288825Z","iopub.status.idle":"2023-07-20T08:06:43.320164Z","shell.execute_reply.started":"2023-07-20T08:06:43.288777Z","shell.execute_reply":"2023-07-20T08:06:43.318978Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"markdown","source":"### Training","metadata":{"id":"X1RgBvlu8oey"}},{"cell_type":"code","source":"trainer = SentimentTrainer()","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4ngQ-ZzcamgL","outputId":"71488d2d-51e2-4eaa-a113-2dc1710c8b15","execution":{"iopub.status.busy":"2023-07-20T08:08:24.137779Z","iopub.execute_input":"2023-07-20T08:08:24.138983Z","iopub.status.idle":"2023-07-20T08:08:27.687351Z","shell.execute_reply.started":"2023-07-20T08:08:24.138933Z","shell.execute_reply":"2023-07-20T08:08:27.686158Z"},"trusted":true},"execution_count":44,"outputs":[{"name":"stderr","text":"Some weights of SentimentModel were not initialized from the model checkpoint at vinai/phobert-base-v2 and are newly initialized: ['classifier.lstm.bias_hh_l0', 'classifier.lstm.bias_ih_l0', 'classifier.lstm.weight_ih_l0_reverse', 'classifier.lstm.weight_hh_l0', 'classifier.dense.bias', 'classifier.lstm.weight_hh_l1_reverse', 'classifier.out_proj.bias', 'classifier.lstm.bias_hh_l1_reverse', 'classifier.lstm.bias_hh_l0_reverse', 'classifier.out_proj.weight', 'classifier.lstm.bias_ih_l0_reverse', 'classifier.lstm.weight_hh_l0_reverse', 'classifier.lstm.weight_ih_l0', 'classifier.lstm.weight_hh_l1', 'classifier.lstm.bias_ih_l1_reverse', 'classifier.lstm.bias_hh_l1', 'classifier.dense.weight', 'classifier.lstm.bias_ih_l1', 'classifier.lstm.weight_ih_l1', 'classifier.lstm.weight_ih_l1_reverse']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\nUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n","output_type":"stream"},{"name":"stdout","text":"Counter({2: 13161, 0: 12122, 1: 5075})\nLoad 30358 examples for train dataset\nCounter({2: 2434, 0: 1515, 1: 594})\nLoad 4543 examples for eval dataset\n","output_type":"stream"}]},{"cell_type":"code","source":"# bilstm-2, lr=1.5, \n# use mean(output_of_roberta) instead of hidden of first <s> token\ntrainer.train()","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"Gi46futy0W4G","outputId":"e6c40cd8-5fe0-41a8-f9ef-4dd2e6c2ea37","execution":{"iopub.status.busy":"2023-07-20T08:08:48.382107Z","iopub.execute_input":"2023-07-20T08:08:48.382495Z","iopub.status.idle":"2023-07-20T09:29:18.164394Z","shell.execute_reply.started":"2023-07-20T08:08:48.382468Z","shell.execute_reply":"2023-07-20T09:29:18.162707Z"},"trusted":true},"execution_count":48,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"\n    <div>\n      \n      <progress value='2406' max='4740' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [2406/4740 1:20:25 < 1:18:04, 0.50 it/s, Epoch 5.07/10]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Step</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n      <th>Precision</th>\n      <th>Recall</th>\n      <th>F1</th>\n      <th>Accuracy</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>200</td>\n      <td>0.627600</td>\n      <td>0.526083</td>\n      <td>0.614780</td>\n      <td>0.685230</td>\n      <td>0.643013</td>\n      <td>0.685230</td>\n    </tr>\n    <tr>\n      <td>400</td>\n      <td>0.371500</td>\n      <td>0.208361</td>\n      <td>0.886912</td>\n      <td>0.879815</td>\n      <td>0.880892</td>\n      <td>0.879815</td>\n    </tr>\n    <tr>\n      <td>600</td>\n      <td>0.231300</td>\n      <td>0.165286</td>\n      <td>0.902094</td>\n      <td>0.901387</td>\n      <td>0.901591</td>\n      <td>0.901387</td>\n    </tr>\n    <tr>\n      <td>800</td>\n      <td>0.213600</td>\n      <td>0.155113</td>\n      <td>0.905874</td>\n      <td>0.899846</td>\n      <td>0.900781</td>\n      <td>0.899846</td>\n    </tr>\n    <tr>\n      <td>1000</td>\n      <td>0.205200</td>\n      <td>0.152411</td>\n      <td>0.906155</td>\n      <td>0.902487</td>\n      <td>0.903091</td>\n      <td>0.902487</td>\n    </tr>\n    <tr>\n      <td>1200</td>\n      <td>0.184500</td>\n      <td>0.152408</td>\n      <td>0.907138</td>\n      <td>0.907330</td>\n      <td>0.907116</td>\n      <td>0.907330</td>\n    </tr>\n    <tr>\n      <td>1400</td>\n      <td>0.178600</td>\n      <td>0.147963</td>\n      <td>0.909081</td>\n      <td>0.908210</td>\n      <td>0.908410</td>\n      <td>0.908210</td>\n    </tr>\n    <tr>\n      <td>1600</td>\n      <td>0.168200</td>\n      <td>0.159064</td>\n      <td>0.909456</td>\n      <td>0.907110</td>\n      <td>0.907254</td>\n      <td>0.907110</td>\n    </tr>\n    <tr>\n      <td>1800</td>\n      <td>0.163300</td>\n      <td>0.153516</td>\n      <td>0.911560</td>\n      <td>0.909531</td>\n      <td>0.909871</td>\n      <td>0.909531</td>\n    </tr>\n    <tr>\n      <td>2000</td>\n      <td>0.157800</td>\n      <td>0.160224</td>\n      <td>0.906501</td>\n      <td>0.906229</td>\n      <td>0.906310</td>\n      <td>0.906229</td>\n    </tr>\n    <tr>\n      <td>2200</td>\n      <td>0.148500</td>\n      <td>0.159105</td>\n      <td>0.910787</td>\n      <td>0.907990</td>\n      <td>0.908771</td>\n      <td>0.907990</td>\n    </tr>\n    <tr>\n      <td>2400</td>\n      <td>0.144200</td>\n      <td>0.163204</td>\n      <td>0.908789</td>\n      <td>0.908871</td>\n      <td>0.908815</td>\n      <td>0.908871</td>\n    </tr>\n  </tbody>\n</table><p>"},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n  _warn_prf(average, modifier, msg_start, len(result))\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","Cell \u001b[0;32mIn[48], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# bilstm-2, lr=1.5, \u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m# use mean(output_of_roberta) instead of hidden of first <s> token\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n","Cell \u001b[0;32mIn[36], line 89\u001b[0m, in \u001b[0;36mSentimentTrainer.train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtrain\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m---> 89\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1526\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1521\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_wrapped \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\n\u001b[1;32m   1523\u001b[0m inner_training_loop \u001b[38;5;241m=\u001b[39m find_executable_batch_size(\n\u001b[1;32m   1524\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_inner_training_loop, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_train_batch_size, args\u001b[38;5;241m.\u001b[39mauto_find_batch_size\n\u001b[1;32m   1525\u001b[0m )\n\u001b[0;32m-> 1526\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1527\u001b[0m \u001b[43m    \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1528\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1530\u001b[0m \u001b[43m    \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1531\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1796\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   1793\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_step_begin(args, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n\u001b[1;32m   1795\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maccelerator\u001b[38;5;241m.\u001b[39maccumulate(model):\n\u001b[0;32m-> 1796\u001b[0m     tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1798\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m   1799\u001b[0m     args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   1800\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_tpu_available()\n\u001b[1;32m   1801\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch\u001b[38;5;241m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   1802\u001b[0m ):\n\u001b[1;32m   1803\u001b[0m     \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   1804\u001b[0m     tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2652\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   2650\u001b[0m         scaled_loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m   2651\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 2652\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maccelerator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2654\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loss\u001b[38;5;241m.\u001b[39mdetach() \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mgradient_accumulation_steps\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:1900\u001b[0m, in \u001b[0;36mAccelerator.backward\u001b[0;34m(self, loss, **kwargs)\u001b[0m\n\u001b[1;32m   1898\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m   1899\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscaler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1900\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscaler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1901\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1902\u001b[0m     loss\u001b[38;5;241m.\u001b[39mbackward(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    478\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    479\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    480\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    485\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m    486\u001b[0m     )\n\u001b[0;32m--> 487\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    488\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m    489\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    195\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    197\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    198\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    201\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    202\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "],"ename":"KeyboardInterrupt","evalue":"","output_type":"error"}]},{"cell_type":"markdown","source":"# EXPORT model to ONNX","metadata":{"id":"bHa-_Zxe-l53"}},{"cell_type":"code","source":"!pip install onnx\n!pip install onnxruntime","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"w8uWKyq_h6Vq","outputId":"9ba1d52c-bf15-4a6b-c6a7-deb71e1f61c0","execution":{"iopub.status.busy":"2023-07-20T09:29:26.510948Z","iopub.execute_input":"2023-07-20T09:29:26.511349Z","iopub.status.idle":"2023-07-20T09:29:56.372328Z","shell.execute_reply.started":"2023-07-20T09:29:26.511319Z","shell.execute_reply":"2023-07-20T09:29:56.370942Z"},"trusted":true},"execution_count":49,"outputs":[{"name":"stdout","text":"Requirement already satisfied: onnx in /opt/conda/lib/python3.10/site-packages (1.14.0)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from onnx) (1.23.5)\nRequirement already satisfied: protobuf>=3.20.2 in /opt/conda/lib/python3.10/site-packages (from onnx) (3.20.3)\nRequirement already satisfied: typing-extensions>=3.6.2.1 in /opt/conda/lib/python3.10/site-packages (from onnx) (4.5.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mCollecting onnxruntime\n  Downloading onnxruntime-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.9 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m23.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting coloredlogs (from onnxruntime)\n  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: flatbuffers in /opt/conda/lib/python3.10/site-packages (from onnxruntime) (23.3.3)\nRequirement already satisfied: numpy>=1.21.6 in /opt/conda/lib/python3.10/site-packages (from onnxruntime) (1.23.5)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from onnxruntime) (21.3)\nRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from onnxruntime) (3.20.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from onnxruntime) (1.12)\nCollecting humanfriendly>=9.1 (from coloredlogs->onnxruntime)\n  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->onnxruntime) (3.0.9)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->onnxruntime) (1.3.0)\nInstalling collected packages: humanfriendly, coloredlogs, onnxruntime\nSuccessfully installed coloredlogs-15.0.1 humanfriendly-10.0 onnxruntime-1.15.1\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import onnx\nfrom transformers import RobertaForSequenceClassification\nfrom onnxruntime.transformers.optimizer import optimize_model\nfrom transformers.convert_graph_to_onnx import quantize\nimport onnxruntime as ort\nimport torch\nfrom functools import reduce","metadata":{"id":"oDBkp6FpYCZd","execution":{"iopub.status.busy":"2023-07-20T09:29:56.379023Z","iopub.execute_input":"2023-07-20T09:29:56.381863Z","iopub.status.idle":"2023-07-20T09:29:56.657873Z","shell.execute_reply.started":"2023-07-20T09:29:56.381795Z","shell.execute_reply":"2023-07-20T09:29:56.65666Z"},"trusted":true},"execution_count":50,"outputs":[]},{"cell_type":"code","source":"best_checkpoint_path = f\"{trainer.config.training_output_dir}/checkpoint-1800\"\n# best_model = RobertaForSequenceClassification.from_pretrained(best_checkpoint_path)\nbest_model = SentimentModel.from_pretrained(best_checkpoint_path)\nbest_model.eval()\ntokenizer = AutoTokenizer.from_pretrained(best_checkpoint_path)","metadata":{"id":"dygAMuqf_rGt","colab":{"base_uri":"https://localhost:8080/"},"outputId":"620ec86b-e9db-4bfb-8444-b071b5676569","execution":{"iopub.status.busy":"2023-07-20T09:29:56.664093Z","iopub.execute_input":"2023-07-20T09:29:56.666793Z","iopub.status.idle":"2023-07-20T09:30:02.519159Z","shell.execute_reply.started":"2023-07-20T09:29:56.66675Z","shell.execute_reply":"2023-07-20T09:30:02.517585Z"},"trusted":true},"execution_count":51,"outputs":[]},{"cell_type":"markdown","source":"## Export","metadata":{"id":"YMKlc-NqoXQv"}},{"cell_type":"markdown","source":"### Make saved path","metadata":{"id":"4srin5A0V7Mw"}},{"cell_type":"code","source":"ONNX_FOLDER = f\"{trainer.config.training_output_dir}/onnx\"\n!mkdir \"{ONNX_FOLDER}\"","metadata":{"id":"Biz9CqSuG-Y9","execution":{"iopub.status.busy":"2023-07-20T09:30:02.522014Z","iopub.execute_input":"2023-07-20T09:30:02.522737Z","iopub.status.idle":"2023-07-20T09:30:03.709779Z","shell.execute_reply.started":"2023-07-20T09:30:02.522697Z","shell.execute_reply":"2023-07-20T09:30:03.708243Z"},"trusted":true},"execution_count":52,"outputs":[]},{"cell_type":"markdown","source":"### Class for converter","metadata":{"id":"NGheg42_V-tZ"}},{"cell_type":"code","source":"import logging\nclass ONNXFunc:\n    def __init__(self, dummy_inputs, model, input_names, output_names, dynamic_axes):\n        self.dummy_inputs = dummy_inputs\n        self.model = model\n        self.input_names = input_names\n        self.output_names = output_names\n        self.dynamic_axes = dynamic_axes\n        self.logger = logging.getLogger(self.__class__.__name__)\n\n    @staticmethod\n    def compose(*functions):\n        return reduce(lambda f, g: lambda x: g(f(x)), functions)\n\n    @staticmethod\n    def check_onnx_model(path):\n        model = onnx.load(path)\n        onnx.checker.check_model(model)\n\n    def onnx_only(self, saved_path):\n      torch.onnx.export(\n          self.model,\n          self.dummy_inputs,\n          saved_path,\n          verbose=True,\n          input_names=self.input_names,\n          output_names=self.output_names,\n          dynamic_axes=self.dynamic_axes\n      )\n      return saved_path\n\n    def apply_optimized_func(self, saved_path: str):\n      use_gpu = trainer.config.training_device==\"cuda\" and \"CUDAExecutionProvider\" in ort.get_available_providers()\n      optimized_model = optimize_model(saved_path, use_gpu=use_gpu, opt_level=99, verbose=True)\n      new_onnx_path = saved_path.replace(\".onnx\", \"_optimized.onnx\")\n      optimized_model.save_model_to_file(new_onnx_path)\n      self.logger.info(f\"EXPORT model to ONNX + Optimized - DONE at {new_onnx_path}\")\n      return new_onnx_path\n\n    def apply_quantized_func(self, saved_path: str):\n        from pathlib import Path\n        quantized_model_path = quantize(Path(saved_path))\n        self.logger.info(f\"EXPORT model to ONNX + Quantized - DONE at {quantized_model_path}\")\n        return quantized_model_path\n\n    @property\n    def mapping_func(self):\n        return {\n            \"ONNX\": self.onnx_only,\n            \"optimized_ONNX\": self.compose(self.onnx_only, self.apply_optimized_func),\n            \"quantized_optimized_ONNX\": self.compose(\n                self.onnx_only, self.apply_optimized_func, self.apply_quantized_func)\n        }\n\n    def convert(self, option, saved_path):\n        converter = self.mapping_func[option]\n        converter(saved_path)","metadata":{"id":"c0UVmD2SqkFn","execution":{"iopub.status.busy":"2023-07-20T09:30:03.712167Z","iopub.execute_input":"2023-07-20T09:30:03.712938Z","iopub.status.idle":"2023-07-20T09:30:03.729901Z","shell.execute_reply.started":"2023-07-20T09:30:03.712894Z","shell.execute_reply":"2023-07-20T09:30:03.728756Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"markdown","source":"### Convert","metadata":{"id":"0OqKeNiSWD4g"}},{"cell_type":"code","source":"import torch\ntext = [\"Tôi là một kĩ sư AI\",\"Tôi là một kĩ sư AI\"]\ndummy_inputs = tokenizer(text, return_tensors=\"pt\", padding=True)\ninput_keys = [\"input_ids\",\"attention_mask\"]\ndummy_inputs = tuple(dummy_inputs[i] for i in input_keys)","metadata":{"id":"cqDbWjP3AP3e","execution":{"iopub.status.busy":"2023-07-20T09:30:03.73143Z","iopub.execute_input":"2023-07-20T09:30:03.732002Z","iopub.status.idle":"2023-07-20T09:30:03.754308Z","shell.execute_reply.started":"2023-07-20T09:30:03.731966Z","shell.execute_reply":"2023-07-20T09:30:03.753077Z"},"trusted":true},"execution_count":54,"outputs":[]},{"cell_type":"code","source":"onnx_conn = ONNXFunc(\n    dummy_inputs=dummy_inputs, model=best_model, input_names=[\"input_ids\",\"attention_mask\"], output_names=[\"logits\"], dynamic_axes={\n        \"input_ids\":{\n            0: \"batch_size\",\n            1: \"dim\"\n        },\n        \"attention_mask\":{\n            0: \"batch_size\",\n            1: \"dim\"\n        },\n        \"logits\": {\n            0: \"batch_size\"\n        }\n    }\n)","metadata":{"id":"KntwnITdqmPF","execution":{"iopub.status.busy":"2023-07-20T09:30:03.755961Z","iopub.execute_input":"2023-07-20T09:30:03.75671Z","iopub.status.idle":"2023-07-20T09:30:03.766893Z","shell.execute_reply.started":"2023-07-20T09:30:03.756673Z","shell.execute_reply":"2023-07-20T09:30:03.765862Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"onnx_conn.convert(option=\"quantized_optimized_ONNX\", saved_path=f\"{ONNX_FOLDER}/model.onnx\")","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":452},"id":"Bd0lZV9wq7_-","outputId":"07d4f7b7-df9b-440b-ef7b-488337fec976","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test and compare model","metadata":{"id":"gUlgbrI2mQRi"}},{"cell_type":"code","source":"examples = [\"hôm nay tôi vui lắm\",\n            \"hôm nay trời thật đẹp\"]\nsegmented_text = [\" \".join(PipelineConfig.rdrsegmenter.word_segment(example)) for example in examples ]\n_inputs = tokenizer(examples, return_tensors=\"pt\", padding=True)\n","metadata":{"id":"kh_wdL3zbJP2","execution":{"iopub.status.busy":"2023-07-20T10:18:38.889974Z","iopub.status.idle":"2023-07-20T10:18:38.892686Z","shell.execute_reply.started":"2023-07-20T10:18:38.892381Z","shell.execute_reply":"2023-07-20T10:18:38.892409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load dataset for testing","metadata":{"id":"kLWi88eEWJQK"}},{"cell_type":"code","source":"import json\ntest_dataset = json.load(open(f\"{TRAINING_DATASET_PATH}/training/test.json\", \"r\", encoding=\"utf8\"))\neval_dataset = json.load(open(f\"{TRAINING_DATASET_PATH}/training/eval.json\", \"r\", encoding=\"utf8\"))\ntrain_dataset = json.load(open(f\"{TRAINING_DATASET_PATH}/training/train.json\", \"r\", encoding=\"utf8\"))\nlen(test_dataset[\"data\"])","metadata":{"id":"0rtWHVZTxkT3","colab":{"base_uri":"https://localhost:8080/"},"outputId":"d1b0b933-7d4e-49e6-fee0-0f054de1f5f7","execution":{"iopub.status.busy":"2023-07-20T10:18:38.897275Z","iopub.status.idle":"2023-07-20T10:18:38.899935Z","shell.execute_reply.started":"2023-07-20T10:18:38.899641Z","shell.execute_reply":"2023-07-20T10:18:38.899667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Init metrics","metadata":{"id":"wHVICnY3WOi5"}},{"cell_type":"code","source":"from evaluate import load\ndef custom_metrics(predictions, labels):\n    precision = load(\"precision\")\n    recall = load(\"recall\")\n    f1 = load(\"f1\")\n    accuracy = load(\"accuracy\")\n\n    _precision = precision.compute(predictions=predictions, references=labels, average=\"weighted\")\n    _recall = recall.compute(predictions=predictions, references=labels, average=\"weighted\")\n    _f1 = f1.compute(predictions=predictions, references=labels, average=\"weighted\")\n    _accuracy = accuracy.compute(predictions=predictions, references=labels)\n    return {\n        \"precision\":_precision,\n        \"recall\":_recall,\n        \"f1\":_f1,\n        \"accuracy\":_accuracy\n    }","metadata":{"id":"lalgdlUURB4B","execution":{"iopub.status.busy":"2023-07-20T03:17:21.370296Z","iopub.status.idle":"2023-07-20T03:17:21.370819Z","shell.execute_reply.started":"2023-07-20T03:17:21.370542Z","shell.execute_reply":"2023-07-20T03:17:21.370565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Init ONNX sessions","metadata":{"id":"OsuEMvSXnZXB"}},{"cell_type":"code","source":"options = ort.SessionOptions()\n# options.intra_op_num_threads = 1\noptions.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL\nproviders = [\"CPUExecutionProvider\"]\nonnx_path=f\"{ONNX_FOLDER}/model.onnx\"","metadata":{"id":"VQsP_BESTE5Q","execution":{"iopub.status.busy":"2023-07-20T03:17:21.375154Z","iopub.status.idle":"2023-07-20T03:17:21.37632Z","shell.execute_reply.started":"2023-07-20T03:17:21.376051Z","shell.execute_reply":"2023-07-20T03:17:21.376076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import onnxruntime as ort\nort_session = ort.InferenceSession(onnx_path, providers=providers, sess_options=options)\nquantized_model = ort.InferenceSession(f\"{ONNX_FOLDER}/model_optimized-quantized.onnx\", providers=providers, sess_options=options)","metadata":{"id":"qHKcy651a9Du","execution":{"iopub.status.busy":"2023-07-20T03:17:21.382561Z","iopub.status.idle":"2023-07-20T03:17:21.383315Z","shell.execute_reply.started":"2023-07-20T03:17:21.38295Z","shell.execute_reply":"2023-07-20T03:17:21.382974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test time of sessions with one example","metadata":{"id":"cD8_Ti0hnpgG"}},{"cell_type":"code","source":"import time","metadata":{"id":"KzIPmg8K85gS","execution":{"iopub.status.busy":"2023-07-20T03:17:21.385175Z","iopub.status.idle":"2023-07-20T03:17:21.385765Z","shell.execute_reply.started":"2023-07-20T03:17:21.385453Z","shell.execute_reply":"2023-07-20T03:17:21.385482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t1 = time.time()\nbest_model(**_inputs)\nprint(\"Original model time\\t\\t\", time.time() - t1)\n\nort_input = {key: _inputs[key].detach().numpy() for key in _inputs.keys() if key!=\"token_type_ids\"}\n\nt2 = time.time()\nort_session.run([\"logits\"], ort_input)\nprint(\"ONNX model time\\t\\t\", time.time() - t2)\n\n\nt4 = time.time()\nquantized_model.run([\"logits\"], ort_input)\nprint(\"optimized + quantized ONNX model time\\t\\t\", time.time() - t4)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Po9EcRFVIyvg","outputId":"bfbfdca4-8b36-463f-89d8-f5411f0597c0","execution":{"iopub.status.busy":"2023-07-20T03:17:21.388455Z","iopub.status.idle":"2023-07-20T03:17:21.38983Z","shell.execute_reply.started":"2023-07-20T03:17:21.389007Z","shell.execute_reply":"2023-07-20T03:17:21.389072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference accuracy and time of sessions vs base model","metadata":{"id":"OrRBPlG9nx8p"}},{"cell_type":"code","source":"from tqdm import tqdm\nimport time\nimport scipy\nimport random","metadata":{"id":"Xa7q_kPJmIDe","execution":{"iopub.status.busy":"2023-07-20T03:17:21.392974Z","iopub.status.idle":"2023-07-20T03:17:21.394029Z","shell.execute_reply.started":"2023-07-20T03:17:21.393425Z","shell.execute_reply":"2023-07-20T03:17:21.393497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modes = {\n    \"quantized_opt_onnx\":quantized_model,\n    \"onnx_only\": ort_session,\n    \"base\": best_model\n}","metadata":{"id":"eE35vc7Ux_ay","execution":{"iopub.status.busy":"2023-07-20T03:17:21.395583Z","iopub.status.idle":"2023-07-20T03:17:21.396631Z","shell.execute_reply.started":"2023-07-20T03:17:21.396326Z","shell.execute_reply":"2023-07-20T03:17:21.396351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n_len = len(eval_dataset[\"data\"])\nsubset = random.choices(eval_dataset[\"data\"], k=_len)\nonnx_inputs = [_in.name for _in in quantized_model.get_inputs()]\nonnx_outputs = [_out.name for _out in quantized_model.get_outputs()]\n\nfor mode, sess in modes.items():\n    print(f\"PROCESSING {mode}\")\n    t = time.time()\n    acc = 0\n    labelsss = []\n    predsss = []\n    for example, label in tqdm(subset):\n        labelsss.append(label)\n        segmented_text = [\" \".join(PipelineConfig.rdrsegmenter.word_segment(example))]\n        _inputs = tokenizer(segmented_text, return_tensors=\"pt\", padding=True)\n        if \"onnx\" in mode:\n            ort_input = {key: _inputs[key].detach().numpy() for key in onnx_inputs}\n            output = sess.run(onnx_outputs, ort_input)[0]\n            output = scipy.special.softmax(output, axis=-1)\n            pred = output.argmax(-1).tolist()[0]\n        else:\n            output = sess(**_inputs)\n            output = output.logits.argmax(dim=-1)\n            pred = output.item()\n        predsss.append(pred)\n        if label == pred:\n            acc += 1\n    end_time=time.time()\n    print(f\"{mode} DONE with avg time={(end_time-t)/_len}\")\n    print(json.dumps(custom_metrics(predsss, labelsss), indent=4, ensure_ascii=False))","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":860},"id":"UT3EDLmzXZaJ","outputId":"b51a9695-35b8-454a-9c49-df6aae9f07dc","execution":{"iopub.status.busy":"2023-07-20T03:17:21.401526Z","iopub.status.idle":"2023-07-20T03:17:21.402491Z","shell.execute_reply.started":"2023-07-20T03:17:21.401961Z","shell.execute_reply":"2023-07-20T03:17:21.402021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Model without ONNX:\n    - accuracy = 0.857\n    - f1 = 0.856\n    - avg time = 158.717 ms/example\n- ONNX only:\n    - accuracy = 0.857\n    - f1 = 0.856\n    - time = 58.079 ms/example\n- ONNX and quantize:\n    - accuracy = 0.858\n    - f1 = 0.856\n    - avg time = 29.912 ms/example\n","metadata":{"id":"TSJTvVqzomWn"}},{"cell_type":"markdown","source":"#### Test quantized model","metadata":{"id":"YrW3kPQvqN3r"}},{"cell_type":"code","source":"def reverse_label(label):\n    label_map = {\n        0: \"POS\",\n        1: \"NEG\",\n        2: \"NEU\"\n    }\n    return label_map[label]","metadata":{"id":"jSN9dNbDqerR","execution":{"iopub.status.busy":"2023-07-20T03:17:21.404556Z","iopub.status.idle":"2023-07-20T03:17:21.405859Z","shell.execute_reply.started":"2023-07-20T03:17:21.405369Z","shell.execute_reply":"2023-07-20T03:17:21.4054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_len = 1000\nsubset = random.choices(eval_dataset[\"data\"], k=_len)\nacc=0\nfor example, label in subset:\n    segmented_text = [\" \".join(PipelineConfig.rdrsegmenter.word_segment(example))]\n    _inputs = tokenizer(segmented_text, return_tensors=\"pt\", padding=True)\n\n    ort_input = {key: _inputs[key].detach().numpy() for key in onnx_inputs}\n    output = quantized_model.run(onnx_outputs, ort_input)[0]\n    output = scipy.special.softmax(output, axis=-1)\n    pred = output.argmax(-1).tolist()[0]\n\n    if label == pred:\n        acc += 1\n    else:\n        print(\"==============\")\n        print(example)\n        print(reverse_label(label), \"!=\", reverse_label(pred))","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7UUmXrxfoi75","outputId":"1f417544-58bf-4d90-f32b-df1a64e1b953","execution":{"iopub.status.busy":"2023-07-20T03:17:21.408743Z","iopub.status.idle":"2023-07-20T03:17:21.409544Z","shell.execute_reply.started":"2023-07-20T03:17:21.409251Z","shell.execute_reply":"2023-07-20T03:17:21.409281Z"},"trusted":true},"execution_count":null,"outputs":[]}]}