Logistic Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from bot import BertGoggles\n",
    "\n",
    "import pickle as pkl\n",
    "import numpy as np\n",
    "\n",
    "import sklearn as sk\n",
    "from sklearn import linear_model\n",
    "from sklearn import preprocessing as pre\n",
    "\n",
    "from logistic import Logistic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('squad_val_questions.pkl', 'rb') as file:\n",
    "    questions = pkl.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Second quarter for training Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = questions[len(questions)//4:2*len(questions)//4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keep Possible Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_questions = []\n",
    "\n",
    "for q in questions:\n",
    "    if not q['label']:\n",
    "        new_questions.append(q)\n",
    "        \n",
    "questions = new_questions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bobo(q, true_id, answerini, bert):\n",
    "    trus = []\n",
    "    fals = []\n",
    "    \n",
    "    anserini_scores = [0]\n",
    "    contexts = [0]\n",
    "    \n",
    "    hits = answerini.search(q, top_n=100)\n",
    "    \n",
    "    true_found = False\n",
    "    i = 0\n",
    "    \n",
    "    while len(fals) < 10 or not true_found:\n",
    "        hit = hits[i]\n",
    "        if true_id == int(hit.id):\n",
    "            true_found = True\n",
    "            contexts[0] = hit\n",
    "            anserini_scores[0] = hit.score\n",
    "            \n",
    "        elif len(contexts) < 11:\n",
    "            contexts.append(hit)\n",
    "            anserini_scores.append(hit.score)\n",
    "            \n",
    "        i += 1\n",
    "        \n",
    "        if i >= len(hits):\n",
    "            break\n",
    "    \n",
    "    if true_found:\n",
    "        bert_hits = bert.search(q, contexts)\n",
    "        \n",
    "        trus.append([anserini_scores[0], bert_hits[0].score, bert_hits[0].null_score])\n",
    "        \n",
    "        for i in range(1, len(bert_hits)):\n",
    "            fals.append([anserini_scores[i], bert_hits[i].score, bert_hits[i].null_score])\n",
    "        \n",
    "    else:\n",
    "        bert_hits = bert.search(q, contexts[1:])\n",
    "        \n",
    "        for i in range(len(bert_hits)):\n",
    "            fals.append([anserini_scores[i + 1], bert_hits[i].score, bert_hits[i].null_score])\n",
    "        \n",
    "    return trus, fals, true_found\n",
    "\n",
    "def upper_bobo(bert_goggles, collection):\n",
    "    result = []\n",
    "    trus = []\n",
    "    fals = []\n",
    "    not_found = []\n",
    "    \n",
    "    from time import time\n",
    "    start_time = time()\n",
    "    avg = 0\n",
    "    wait_iter = 5\n",
    "\n",
    "    for i, c in enumerate(collection):\n",
    "\n",
    "        question = c['question']\n",
    "\n",
    "        correct_id = c['id']\n",
    "\n",
    "        tru, fal, found = bobo(question, correct_id, bert_goggles.answerini, bert_goggles.bert_model)\n",
    "\n",
    "        if found:\n",
    "            trus.extend(tru)\n",
    "            fals.extend(fal)\n",
    "        else:\n",
    "            not_found.append(i)\n",
    "            \n",
    "        if (i + 1) % wait_iter == 0:\n",
    "            from IPython.display import clear_output\n",
    "            clear_output(wait=True)\n",
    "            \n",
    "            if avg == 0:\n",
    "                avg = (time() - start_time) / wait_iter\n",
    "            else:\n",
    "                avg = (((time() - start_time) / wait_iter) + avg) / 2\n",
    "            \n",
    "            start_time = time()\n",
    "            print('{0:.2f}%'.format(round(i / len(collection), 4) * 100))\n",
    "            \n",
    "            time_left = (len(collection) - i) * avg / 60 \n",
    "            \n",
    "            print('Time remaining: {0:.2f} mins'.format(time_left))\n",
    "            \n",
    "    return trus, fals, not_found"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Index Dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INDEX = 'Index'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Dirs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_MODEL = 'Pret-trained/model'\n",
    "BERT_MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_CHKPT = 'Fine-truned/checkpoint'\n",
    "BERT_CHKPT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAXLEN = 350"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_goggles = BertGoggles(BERT_MODEL, BERT_CHKPT, MAXLEN, INDEX, top_n=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_results = upper_bobo(bert_goggles, questions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('results_log.pkl', 'wb') as file:\n",
    "    pkl.dump(BERT_results, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('results_log.pkl', 'rb') as file:\n",
    "    BERT_results = pkl.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trus = np.asarray(BERT_results[0])\n",
    "fals = np.asarray(BERT_results[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log = Logistic()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train with S_null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log.train(trus, fals)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('log_1.pkl', 'wb') as file:\n",
    "    pkl.dump(log, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train without S_null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to remove s_null\n",
    "trus = trus[:, :2]\n",
    "fals = fals[:, :2]\n",
    "\n",
    "# train\n",
    "log.train(trus, fals, s_null=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('log_2.pkl', 'wb') as file:\n",
    "    pkl.dump(log, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "goggles",
   "language": "python",
   "name": "goggles"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}