iii_parse_htmls.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "525f246f-4d92-4328-8963-3001b5a64e1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36499\n",
      "122\n",
      "36499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/bunds/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/bunds/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/bunds/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/bunds/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re, os, glob, csv, math, requests, time, sys, random, json, datetime, urllib, nltk\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from selenium import webdriver\n",
    "from sklearn.utils import shuffle      \n",
    "import numpy as np\n",
    "from nordvpn_switcher import initialize_VPN,rotate_VPN,terminate_VPN\n",
    "import multiprocessing as mp\n",
    "from tqdm import tqdm\n",
    "from webscraping_kit import write_htmlfile, write_json_tofile, write_driverhtmlfile, rmnl\n",
    "from webscraping_kit import read_jsoncsv, read_htmlfile, read_htmlresponse, read_driverresponse, get_cases\n",
    "from string import digits\n",
    "from stop_words import get_stop_words\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.util import everygrams\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from htmlparsing_kit import *\n",
    "\n",
    "def update(*a):\n",
    "    pbar.update()\n",
    "    \n",
    "def get_errortype(x):\n",
    "    if \"404 Error\" in x:\n",
    "        tag = 'not_found'\n",
    "    elif x.startswith('ERROR_Request failed'):\n",
    "        tag = 'failed_request'\n",
    "    elif x.startswith('ERROR_Timed out waiting'):\n",
    "        tag = 'timed_out'\n",
    "    elif x == 'no_error':\n",
    "        tag = 'no_error'\n",
    "    else:\n",
    "        tag = 'parsing_error'\n",
    "    return tag\n",
    "\n",
    "def get_cleanjudloc(texts,ldict,jdict):\n",
    "    cjllist = []\n",
    "    for t in texts:\n",
    "        k = t.split('_',1)[0].strip()\n",
    "        cjllist.append('|'.join([k+'_'+map_cleanjl(t,ldict),k+'_'+map_cleanjl(t,jdict)]))\n",
    "    return cjllist\n",
    "\n",
    "def parse_parallelresults(jlresults):\n",
    "    jldict = {}\n",
    "    for jl in jlresults:\n",
    "        v = [j.split('_',1)[-1].strip() for j in jl.split('|')]\n",
    "        v = [re.sub('|','',j).strip() for j in v]\n",
    "        v = '|'.join([j for j in v if j])\n",
    "        k = jl.split('_',1)[0].strip()\n",
    "        jldict[k] = v\n",
    "    return jldict\n",
    "\n",
    "def preprocess_list(textchunk,stopwordslist,pattern):\n",
    "    tres = []\n",
    "    for t in textchunk:\n",
    "        tres.append(t.split('_',1)[0].strip() + '_' + preprocess(t,stopwordslist,pattern))\n",
    "    return tres\n",
    "\n",
    "def process_parallel(chunks,functiontype,ldict,jdict,stopwordslist,pattern,pbar):\n",
    "    pool = mp.Pool(processes=10)\n",
    "    # pbar = tqdm(textchunks)\n",
    "    jlresults = []\n",
    "    for i in range(pbar.total): \n",
    "        if functiontype == \"get_cleanjudloc\":\n",
    "            jlresults.append(pool.apply_async(get_cleanjudloc, args=(textchunks[i],ldict,jdict), callback=update))\n",
    "        else:\n",
    "            jlresults.append(pool.apply_async(preprocess_list, args=(textchunks[i],stopwordslist,pattern), callback=update))\n",
    "            continue\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return jlresults\n",
    "\n",
    "def check_membertype(member):\n",
    "    m = member.lower()\n",
    "    if 'vice chair' in m:\n",
    "        m = '_vicechair'\n",
    "    elif 'vice-chair' in m:\n",
    "        m = '_vicechair'\n",
    "    elif 'vice char' in m:\n",
    "        m ='_vicechair'\n",
    "    elif 'vice cha' in m:\n",
    "        m = '_vicechair'\n",
    "    elif 'member' in m:\n",
    "        m = '_member'\n",
    "    elif 'dispute resolution officer' in m:\n",
    "        m = '_disputeresolutionofficer'\n",
    "    elif 'hearing officer' in m:\n",
    "        m = '_hearingofficer'\n",
    "    elif 'hearings officer' in m:\n",
    "        m = '_hearingofficer'\n",
    "    elif 'associate chair' in m:\n",
    "        m = '_associatechair'\n",
    "    else:\n",
    "        m = 'missing'\n",
    "    return m\n",
    "\n",
    "mainpath = str(os.getcwd())+'/'\n",
    "metadataoutfile =  mainpath + 'output/3_metadata.csv'\n",
    "metadatamissingoutfile =  mainpath + 'output/3_metadata_missing.csv'\n",
    "df = pd.read_csv(mainpath + 'output/2_casefileidx.csv',sep='\\t')\n",
    "df = df[['cid','year','source','caseurl','goodcasefiles']]\n",
    "\n",
    "porter = PorterStemmer() \n",
    "remove_digits = str.maketrans('', '', digits)\n",
    "stopwordslist = create_stopwordslist()\n",
    "pattern = re.compile(r'\\b(' + r'|'.join(stopwordslist) + r')\\b\\s*')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "goodhtmlfiles = df['goodcasefiles'].tolist()\n",
    "print(len(goodhtmlfiles))\n",
    "\n",
    "htmlchunks = list(divide_chunks(goodhtmlfiles,300))\n",
    "print(len(htmlchunks))\n",
    "print(len([item for sublist in htmlchunks for item in sublist]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba7bd31f-3293-4846-aaa8-117460ba6d51",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "\"\"\" using parallel processing to extract the metadata from the htmls\"\"\"\n",
    "# pool = mp.Pool(processes=10)\n",
    "# pbar = tqdm(htmlchunks)\n",
    "    \n",
    "# for i in range(pbar.total):\n",
    "#     pool.apply_async(read_htmlfiles, args=(htmlchunks[i],metadataoutfile), callback=update)\n",
    "    \n",
    "# pool.close()\n",
    "# pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bd9fd145-f0e2-425c-a14e-d6f3984a0009",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36499\n",
      "There should be 36499 cases.\n",
      "There are 36499 cases.\n",
      "present    36499\n",
      "Name: missingcases, dtype: int64\n",
      "no_error          36417\n",
      "not_found            36\n",
      "timed_out            32\n",
      "failed_request       12\n",
      "parsing_error         2\n",
      "Name: errtype, dtype: int64\n",
      "44\n"
     ]
    }
   ],
   "source": [
    "metadicts = read_jsoncsv(metadataoutfile)\n",
    "print(len(metadicts))\n",
    "mdf = pd.DataFrame(metadicts)\n",
    "# mdf['tablefileno'] = mdf['cid'].map(dict(zip(df['cid'],df['fileno'])))\n",
    "mdf['caseurl'] = mdf['cid'].map(dict(zip(df['cid'],df['caseurl'])))\n",
    "mdf['file'] = mdf['cid'].map(dict(zip(df['cid'],df['goodcasefiles'])))\n",
    "mdf.replace(r'^\\s*$', np.nan, regex=True,inplace=True)\n",
    "mdf.fillna('missing',inplace=True)\n",
    "print(\"There should be \"+ str(df.shape[0])+\" cases.\")\n",
    "print(\"There are \"+ str(mdf.shape[0])+\" cases.\")\n",
    "\n",
    "df['missingcases'] = np.where(df['cid'].isin(mdf['cid'].tolist()),'present','missing')\n",
    "print(df['missingcases'].value_counts())\n",
    "\n",
    "mdf['errtype'] = mdf['error'].apply(lambda x: get_errortype(x))\n",
    "print(mdf['errtype'].value_counts())\n",
    "# parsing_error cases are in french ['2020canlii122218','2021canlii114717']\n",
    "\n",
    "missing = mdf[mdf['errtype'].isin(['timed_out','failed_request'])]\n",
    "missing = missing.copy()\n",
    "missing['file'] = missing['cid'].apply(lambda x: mainpath + 'data/errorcases/'+x + '.html')\n",
    "caselinkdict = dict(zip(missing['file'],missing['caseurl']))\n",
    "print(len(caselinkdict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "009b410d-d69e-46e8-8dbc-e89a48d75806",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# settings = initialize_VPN(save=0,area_input=['complete rotation'],skip_settings=1)\n",
    "\n",
    "# rotate_VPN(settings,google_check=1)\n",
    "# get_cases(caselinkdict,settings)\n",
    "# terminate_VPN(settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac0ef5ba-c16a-4744-a7ec-f3504b994790",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read_htmlfiles(list(caselinkdict.keys()),metadatamissingoutfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d212ced0-98cb-4238-a1bc-ff41004b3fac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44\n",
      "(36499, 13)\n",
      "no_error         36461\n",
      "not_found           36\n",
      "parsing_error        2\n",
      "Name: errtype, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "metadictsmissing = read_jsoncsv(metadatamissingoutfile)\n",
    "print(len(metadictsmissing))\n",
    "mdfm = pd.DataFrame(metadictsmissing)\n",
    "mdfm['caseurl'] = mdfm['cid'].map(dict(zip(missing['cid'],missing['caseurl'])))\n",
    "mdfm['file'] = mdfm['cid'].map(dict(zip(missing['cid'],missing['file'])))\n",
    "mdfm.replace(r'^\\s*$', np.nan, regex=True,inplace=True)\n",
    "mdfm.fillna('missing',inplace=True)\n",
    "mdfm['errtype'] = mdfm['error'].apply(lambda x: get_errortype(x))\n",
    "metadf = pd.concat([mdfm,mdf[~mdf['errtype'].isin(['timed_out','failed_request'])]])\n",
    "print(metadf.shape)\n",
    "print(metadf['errtype'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ec26a35f-6e16-4015-bafa-28fe47f93cd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36499"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "36461+36+2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0b6d4aa9-0263-463a-ab90-dacc41d3e4c5",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "both         33139\n",
      "webscrape     3295\n",
      "api             65\n",
      "Name: source, dtype: int64\n",
      "\n",
      "errtype        source   \n",
      "no_error       both         33117\n",
      "               webscrape     3293\n",
      "               api             51\n",
      "not_found      both            20\n",
      "               api             14\n",
      "               webscrape        2\n",
      "parsing_error  both             2\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "metadf['source'] = metadf['cid'].map(dict(zip(df['cid'],df['source'])))\n",
    "print(metadf['source'].value_counts())\n",
    "print()\n",
    "print(metadf[['errtype','source']].value_counts())\n",
    "# tablecaseidx.csv should be 36,437, -1 for the case from 2005, and -2 for the cases missing from the canlii tables = 36434\n",
    "# (both) 33139 + (webscrape) 3295 = 36434"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "11eef14e-a829-421d-bff3-dee408552a6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33168"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "33117+51"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bcd53b56-2acd-4035-8564-e07b0eacb4de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33168"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "33139+65-2-20-14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d0fb9f46-eeec-49ee-bed5-1921f234122c",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "ldict = {'Central-RO': 'Central_Mississauga', '3 Robert Speck Pkwy': 'Central_Mississauga', 'Central Regional Office': 'Central_Mississauga', 'Eastern-RO': 'Eastern_Ottawa', '255 Albert St': 'Eastern_Ottawa', 'Eastern Regional Office': 'Eastern_Ottawa', 'Head Office': 'HeadOffice_Toronto', '777 Bay St': 'HeadOffice_Toronto', 'Northern-RO': 'Northern_Sudbury', '199 Larch St': 'Northern_Sudbury', 'Northern Regional Office': 'Northern_Sudbury', 'South West-RO': 'SouthWest_London', '150 Dufferin Ave': 'SouthWest_London', 'South West Regional Office': 'SouthWest_London', 'Southern-RO': 'Southern_Hamilton', 'Southern Regional Office': 'Southern_Hamilton', '119 King St': 'Southern_Hamilton', 'Toronto East-RO': 'TOEast', 'Toronto East Regional Office': 'TOEast', '2275 Midland Ave': 'TOEast', 'Toronto North-RO': 'TONorth', 'Toronto North Regional Office': 'TONorth', '47 Sheppard Ave': 'TONorth', 'Toronto South-RO': 'TOSouth', 'Toronto South Regional Office': 'TOSouth', '25 Grosvenor St': 'TOSouth', '15 Grosvenor St': 'TOSouth', '79 St. Clair Ave': 'TOSouth'}\n",
    "\n",
    "ajdf = pd.read_csv(mainpath+'output/annotated_judges.csv',sep='\\t')\n",
    "ajdf = ajdf[ajdf['jnew']!='missing']\n",
    "jdict = dict(zip(ajdf['judge'],ajdf['jnew']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4042bd4b-c929-4d77-b88c-7c82904e1a29",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadf['jltext'] = metadf['cid'] + '_' + metadf['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6f147287-ff63-471a-aca0-1e40311a3e35",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                   | 0/365 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "textchunks = list(divide_chunks(metadf['jltext'].tolist(),100))\n",
    "pbar = tqdm(textchunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c1757e6d-581a-438a-bf66-099cd809787f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|████████████████████████████████████████▊| 363/365 [00:34<00:00, 17.77it/s]"
     ]
    }
   ],
   "source": [
    "\"\"\" using parallel processing to preprocess the case texts\"\"\"\n",
    "ctresults = process_parallel(textchunks,\"preprocess_list\",ldict,jdict,stopwordslist,pattern,pbar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fb797b07-cab6-4457-98e3-81d321e55471",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 365/365 [00:50<00:00, 17.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 16)\n"
     ]
    }
   ],
   "source": [
    "ctresults = [x.get() for x in ctresults]\n",
    "ctresults = [item for sublist in ctresults for item in sublist]\n",
    "ctdict = parse_parallelresults(ctresults)\n",
    "print(len(ctdict))\n",
    "metadf['cleantext'] = metadf['cid'].map(ctdict)\n",
    "metadf.replace(r'^\\s*$', np.nan, regex=True,inplace=True)\n",
    "metadf.fillna('missing',inplace=True)\n",
    "print(metadf[metadf['cleantext']=='missing'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "56066e7d-28cb-4d95-9151-e8946a7de52c",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|█████████████████████████████████████████| 365/365 [00:59<00:00,  6.15it/s]\u001b[A\n"
     ]
    }
   ],
   "source": [
    "textchunks = list(divide_chunks(metadf['jltext'].tolist(),100))\n",
    "pbar = tqdm(textchunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5181c08b-2d77-4ae0-b160-25e5c9036b9e",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|█████████████████████████████████████████| 365/365 [07:05<00:00,  1.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36499\n"
     ]
    }
   ],
   "source": [
    "\"\"\" using parallel processing to extract judge names and locations\"\"\"\n",
    "jlresults = process_parallel(textchunks,\"get_cleanjudloc\",ldict,jdict,stopwordslist,pattern,pbar)\n",
    "jlresults = [x.get() for x in jlresults]\n",
    "jlresults = [item for sublist in jlresults for item in sublist]\n",
    "jldict = parse_parallelresults(jlresults)\n",
    "print(len(jldict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0638355e-72c9-4885-a940-2411f5a36030",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6261, 17)\n",
      "(3487, 17)\n"
     ]
    }
   ],
   "source": [
    "metadf['judgeloc'] = metadf['cid'].map(jldict)\n",
    "metadf.replace(r'^\\s*$', np.nan, regex=True,inplace=True)\n",
    "metadf['judgeloc'].fillna('',inplace=True)\n",
    "metadf['member'] = metadf['judgeloc'].apply(lambda x: '_'.join([i for i in x.split('|') if i not in list(ldict.values())]))\n",
    "metadf['loc'] = metadf['judgeloc'].apply(lambda x: '_'.join([i for i in x.split('|') if i in list(ldict.values())]))\n",
    "metadf.replace(r'^\\s*$', np.nan, regex=True,inplace=True)\n",
    "metadf.fillna('missing',inplace=True)\n",
    "print(metadf[metadf['member']=='missing'].shape)\n",
    "print(metadf[metadf['loc']=='missing'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bcb4e8af-c891-4e90-be68-d8071297acf9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3022, 19)\n"
     ]
    }
   ],
   "source": [
    "metadf['membertype'] = metadf['text'].apply(lambda x: check_membertype(x[-math.ceil(len(x)/2):]))\n",
    "metadf.fillna('missing',inplace=True)\n",
    "print(metadf[metadf['membertype']=='missing'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f282877f-8042-4134-89d0-ca079c21109a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 19)\n"
     ]
    }
   ],
   "source": [
    "apidf = pd.read_csv(mainpath + 'output/1_apicaseidx.csv',sep='\\t')\n",
    "webdf = pd.read_csv(mainpath + 'output/1_tablecaseidx.csv',sep='\\t')\n",
    "apidf = apidf[~apidf['cid'].isin(webdf['cid'].tolist())]\n",
    "apiwebdf = pd.concat([apidf[['cid','fileno']],webdf[['cid','fileno']]])\n",
    "filenodict = dict(zip(apiwebdf['cid'],apiwebdf['fileno']))\n",
    "metadf['fileno'] = metadf['cid'].map(filenodict)\n",
    "metadf.fillna('missing',inplace=True)\n",
    "print(metadf[metadf['fileno']=='missing'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7f8ee1f6-e1f5-4ba4-9558-cfa2408f4266",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadf.to_csv(mainpath + 'output/3_metaclean.csv',sep='\\t',index=False,quoting=csv.QUOTE_ALL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}