model_performance_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# compare the accuracy of various models to select the best one"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The basic idea in this notebook is to get a sense for how various models perform.  The assessment is based simply on the train and test accuracy.\n",
    "\n",
    "In another notebook the models will be evaluated with cross validation and compared with grid searching."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the usual\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# some other useful things\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import pickle\n",
    "import datetime\n",
    "from os import path\n",
    "\n",
    "# my utilities\n",
    "from crash_utils.zip_code_and_borough_from_coords import zip_code_and_borough_from_coords\n",
    "from crash_utils.fix_vehicle_names import fix_vehicle_names\n",
    "from crash_utils.make_crash_features import make_crash_features\n",
    "from crash_utils.basic_cleaning import basic_cleaning\n",
    "from crash_utils.prepare_data_for_modelling import prepare_data_for_modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"/Users/Mark/brainstation/capstone/nyc_bike_crash_analysis/data/\"\n",
    "df = pd.read_csv(data_path + \"Motor_Vehicle_Collisions_-_Crashes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fill in missing zip coded and boroughs using lat/lon\n",
    "df = zip_code_and_borough_from_coords(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## clean up the VEHICLE TYPE CODE columns\n",
    "df = fix_vehicle_names(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# perform some basic data munging operations (see `crash_utils/basic_cleaning.py` for details)\n",
    "df = basic_cleaning(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare the data for modelling\n",
    "# drop columns\n",
    "# set up target\n",
    "# run \"make_crash_features.py\"\n",
    "# OHE the text columns\n",
    "# count-vectorize the vehicles and crash factors\n",
    "\n",
    "df = prepare_data_for_modelling(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# extract the features and targets from the big dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# machine learning stuff\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract target and features and then train-test-split\n",
    "# also scale the data for those algorithms which would benefit (e.g., KNN)\n",
    "\n",
    "X = df.iloc[:,1:]\n",
    "y = df.iloc[:,0]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(y_train.value_counts().sort_index())\n",
    "n_min = np.sum(y_train == 0)\n",
    "n_maj = np.sum(y_train == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fix the class imbalance with upsampling\n",
    "from sklearn.utils import resample\n",
    "\n",
    "minority_mask = y_train == 0\n",
    "\n",
    "X_upsampled, y_upsampled = resample(X_train.loc[minority_mask], \n",
    "                                    y_train.loc[minority_mask], \n",
    "                                    replace = True, \n",
    "                                    n_samples = n_maj)\n",
    "\n",
    "X_train_bal = np.vstack((X_train[y_train == 1], X_upsampled))\n",
    "y_train_bal = np.hstack((y_train[y_train == 1], y_upsampled))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# now build the pipeline for grid search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## set up models and parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# 2020-12-11, PCA(n_components=20), RandomForestClassifier(max_depth=40, n_estimators=200), scaler: None\n",
    "# upsampled the no-injury class in the training set\n",
    "# accuracy score of 91%\n",
    "# file: 2020-12-11 01:18:41 GridCVresults.pkl\n",
    "\n",
    "file_name = \"2020-12-11 01:18:41 GridCVresults.pkl\"\n",
    "\n",
    "if path.exists(file_name):\n",
    "\n",
    "    infile = open(file_name,\"rb\")\n",
    "    grid_out = pickle.load(infile)\n",
    "    infile.close()\n",
    "\n",
    "else:\n",
    "\n",
    "    # pipeline initiation\n",
    "    steps = [('scaler', StandardScaler()),        # step 1: scale\n",
    "             ('dim_reduction', PCA()),            # step 2: PCA\n",
    "             ('model', RandomForestClassifier())] # step 3: fit a regressor model\n",
    "        \n",
    "    model_pipeline = Pipeline(steps)\n",
    "    \n",
    "    # model parameters\n",
    "    forest_params = {'model': [RandomForestClassifier()],\n",
    "                     'model__max_depth': [1, 3, 10, 40, 80],\n",
    "                     'model__n_estimators': [50, 100, 200],\n",
    "                     'scaler': [None, StandardScaler()],\n",
    "                     'dim_reduction': [PCA()],\n",
    "                     'dim_reduction__n_components':[5, 20, 50, 100, 200]\n",
    "                    }\n",
    "   \n",
    "\n",
    "    # create our grid\n",
    "    parameter_grid = [forest_params]\n",
    "    grid_out = GridSearchCV(model_pipeline, parameter_grid, cv=5, verbose=1, n_jobs = 6, \n",
    "                            scoring = \"accuracy\")\n",
    "    \n",
    "    # fit\n",
    "    grid_out.fit(X_train_bal, y_train_bal)\n",
    "     \n",
    "    # write results    \n",
    "    file_name = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\") + \" GridCVresults.pkl\"\n",
    "    pickle.dump(grid_out, open(file_name, \"wb\") )\n",
    "    print(\"wrote:\",file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_out.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_out.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_out.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# view performance of all models\n",
    "grid_out.cv_results_.keys()\n",
    "#grid_out.cv_results_[\"params\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#grid_out.cv_results_[\"mean_test_score\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(grid_out.cv_results_[\"mean_test_score\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Previous runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# 2020-12-08, PCA(n_components=20), RandomForestClassifier(max_depth=40, n_estimators = 100, 'scaler': None}\n",
    "# accuracy score of 73.9%\n",
    "# file: 2020-12-10 00:30:04 GridCVresults.pkl\n",
    "\n",
    "file_name = \"2020-12-10 00:30:04 GridCVresults.pkl\"\n",
    "\n",
    "if path.exists(file_name):\n",
    "\n",
    "    infile = open(file_name,\"rb\")\n",
    "    grid_out = pickle.load(infile)\n",
    "    infile.close()\n",
    "\n",
    "else:\n",
    "\n",
    "    # pipeline initiation\n",
    "    steps = [('scaler', StandardScaler()),        # step 1: scale\n",
    "             ('dim_reduction', PCA()),            # step 2: PCA\n",
    "             ('model', RandomForestClassifier())] # step 3: fit a regressor model\n",
    "        \n",
    "    model_pipeline = Pipeline(steps)\n",
    "    \n",
    "    # model parameters\n",
    "    forest_params = {'model': [RandomForestClassifier()],\n",
    "                     'model__max_depth': [1, 10, 40, 80],\n",
    "                     'model__n_estimators': [10, 50, 100],\n",
    "                     'scaler': [None, StandardScaler()],\n",
    "                     'dim_reduction': [PCA()],\n",
    "                     'dim_reduction__n_components':[5, 20, 50, 100],\n",
    "                    }\n",
    "\n",
    "    boost_params = {'model': [GradientBoostingClassifier()],\n",
    "                    'model__max_depth': [1, 3, 80],\n",
    "                    'model__min_samples_leaf': [1, 50, 200],\n",
    "                    'model__learning_rate': [0.1, 0.5],\n",
    "                    'model__n_estimators': [10, 50, 100],\n",
    "                    'scaler': [None, StandardScaler()],\n",
    "                    'dim_reduction': [PCA()],\n",
    "                    'dim_reduction__n_components': [5, 20, 50, 100]\n",
    "                   }\n",
    "\n",
    "    # create our grid\n",
    "    parameter_grid = [forest_params, boost_params]\n",
    "    grid_out = GridSearchCV(model_pipeline, parameter_grid, cv=5, verbose=1, n_jobs = 6, \n",
    "                            scoring = \"accuracy\")\n",
    "    \n",
    "    # fit\n",
    "    grid_out.fit(X_train_bal, y_train_bal)\n",
    "     \n",
    "    # write results    \n",
    "    file_name = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\") + \" GridCVresults.pkl\"\n",
    "    pickle.dump(grid_out, open(file_name, \"wb\") )\n",
    "    print(\"wrote:\",file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# try to reproduce the highest accuracy score\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "pca = PCA(n_components=20).fit(X_train_bal)\n",
    "X_test_pca = pca.transform(X_test)\n",
    "\n",
    "print(X_train_bal.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n",
    "print(X_test_pca.shape)\n",
    "\n",
    "y_pred_test = grid_out.predict(X_test_pca)\n",
    "#print(f\"Accuracy: {accuracy_score(y_test, y_pred_test)}\")"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "deeplearning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}