math2_project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Учебный проект курса \"Теория вероятностей и математическая статистика\"\n",
    "## Оценка возможности предсказания вероятности заключения сделки на предоставленных данных с достаточной надежностью \n",
    "**(по проектам малого производственного предприятия типа job-shop )**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 560,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "pd.options.display.max_columns = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 561,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('job-projects.csv', sep=\";\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>project_id</th>\n",
       "      <th>in_work</th>\n",
       "      <th>Summ</th>\n",
       "      <th>work_summ</th>\n",
       "      <th>Sebest</th>\n",
       "      <th>o_pl</th>\n",
       "      <th>o_rent</th>\n",
       "      <th>Manager</th>\n",
       "      <th>client_projects</th>\n",
       "      <th>client_projects_calc</th>\n",
       "      <th>client_orders</th>\n",
       "      <th>client_konv</th>\n",
       "      <th>client_bill_middle</th>\n",
       "      <th>client_bill_middle_dc</th>\n",
       "      <th>client_pl_middle</th>\n",
       "      <th>client_projects_rent_middle</th>\n",
       "      <th>client_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12087</td>\n",
       "      <td>0</td>\n",
       "      <td>62135.88</td>\n",
       "      <td>45976.02</td>\n",
       "      <td>19393.98</td>\n",
       "      <td>26913.70</td>\n",
       "      <td>58.5</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>71.43</td>\n",
       "      <td>46388.94</td>\n",
       "      <td>33679.28</td>\n",
       "      <td>18724.43</td>\n",
       "      <td>54.92</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11259</td>\n",
       "      <td>1</td>\n",
       "      <td>225455.78</td>\n",
       "      <td>120758.54</td>\n",
       "      <td>49676.97</td>\n",
       "      <td>74720.11</td>\n",
       "      <td>61.9</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.00</td>\n",
       "      <td>225455.78</td>\n",
       "      <td>120758.54</td>\n",
       "      <td>74720.11</td>\n",
       "      <td>61.90</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12089</td>\n",
       "      <td>1</td>\n",
       "      <td>3700.00</td>\n",
       "      <td>3083.33</td>\n",
       "      <td>2228.09</td>\n",
       "      <td>855.24</td>\n",
       "      <td>27.7</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>91.43</td>\n",
       "      <td>50717.51</td>\n",
       "      <td>41792.53</td>\n",
       "      <td>21277.71</td>\n",
       "      <td>18.80</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11803</td>\n",
       "      <td>1</td>\n",
       "      <td>4618.99</td>\n",
       "      <td>3849.16</td>\n",
       "      <td>1966.02</td>\n",
       "      <td>1883.14</td>\n",
       "      <td>48.9</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>91.43</td>\n",
       "      <td>50717.51</td>\n",
       "      <td>41792.53</td>\n",
       "      <td>21277.71</td>\n",
       "      <td>18.80</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12150</td>\n",
       "      <td>1</td>\n",
       "      <td>16167.78</td>\n",
       "      <td>8478.55</td>\n",
       "      <td>4453.69</td>\n",
       "      <td>4310.27</td>\n",
       "      <td>50.8</td>\n",
       "      <td>23.0</td>\n",
       "      <td>74</td>\n",
       "      <td>70</td>\n",
       "      <td>64</td>\n",
       "      <td>85.71</td>\n",
       "      <td>16673.53</td>\n",
       "      <td>9724.47</td>\n",
       "      <td>4786.17</td>\n",
       "      <td>49.40</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   project_id  in_work       Summ  work_summ    Sebest      o_pl  o_rent  \\\n",
       "0       12087        0   62135.88   45976.02  19393.98  26913.70    58.5   \n",
       "1       11259        1  225455.78  120758.54  49676.97  74720.11    61.9   \n",
       "2       12089        1    3700.00    3083.33   2228.09    855.24    27.7   \n",
       "3       11803        1    4618.99    3849.16   1966.02   1883.14    48.9   \n",
       "4       12150        1   16167.78    8478.55   4453.69   4310.27    50.8   \n",
       "\n",
       "   Manager  client_projects  client_projects_calc  client_orders  client_konv  \\\n",
       "0     23.0                1                     1              1        71.43   \n",
       "1     23.0                0                     0              0        20.00   \n",
       "2     22.0                0                     0              0        91.43   \n",
       "3     22.0                1                     0              0        91.43   \n",
       "4     23.0               74                    70             64        85.71   \n",
       "\n",
       "   client_bill_middle  client_bill_middle_dc  client_pl_middle  \\\n",
       "0            46388.94               33679.28          18724.43   \n",
       "1           225455.78              120758.54          74720.11   \n",
       "2            50717.51               41792.53          21277.71   \n",
       "3            50717.51               41792.53          21277.71   \n",
       "4            16673.53                9724.47           4786.17   \n",
       "\n",
       "   client_projects_rent_middle  client_valid  \n",
       "0                        54.92             1  \n",
       "1                        61.90             1  \n",
       "2                        18.80             0  \n",
       "3                        18.80             0  \n",
       "4                        49.40             1  "
      ]
     },
     "execution_count": 562,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 563,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1150, 17)"
      ]
     },
     "execution_count": 563,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Разбиение на тестовый и валидационный сеты, подготовка данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = data.loc[:]\n",
    "df.loc[df[\"in_work\"]==0, \"in_work\"]=-1\n",
    "#df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 587,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Не берем поле Менеджер - оно категориальное и требует дополнительной обработки\n",
    "fields=[\"Summ\", \"work_summ\", \"Sebest\", \"o_pl\", \"o_rent\", \"client_projects\", \"client_projects_calc\", \"client_orders\", \"client_konv\", \"client_bill_middle\", \"client_pl_middle\", \"client_projects_rent_middle\", \"client_valid\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Разобьем данные на обучающий и валидационный сеты"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 588,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((805, 13), (345, 13), (805, 1), (345, 1))"
      ]
     },
     "execution_count": 588,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train, valid, y_train, y_valid = train_test_split(df.loc[:, fields], df.loc[:, [\"in_work\"]], test_size=0.3, random_state=7)\n",
    "train.shape, valid.shape,  y_train.shape, y_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 589,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Нормализуем значения параметров"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 590,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "\n",
    "def prepare_data(data, isTrain):\n",
    "    if isTrain is True:\n",
    "        scaled = scaler.fit_transform(data)\n",
    "    else: \n",
    "        scaled = scaler.transform(data)\n",
    "        \n",
    "    ones = np.ones((scaled.shape[0], 1))\n",
    "    x = np.hstack([ones, scaled])  \n",
    "    return  x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 591,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/data.py:625: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n",
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:8: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((805, 14), (805,), (345, 14), (345,))"
      ]
     },
     "execution_count": 591,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x=prepare_data(train.loc[:, fields], True)\n",
    "y=y_train.values.T[0]\n",
    "\n",
    "x2=prepare_data(valid.loc[:, fields], False)\n",
    "y2=y_valid.values.T[0]\n",
    "x.shape, y.shape, x2.shape, y2.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Построим модель логистической регрессии методом градиентного спуска."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 592,
   "metadata": {},
   "outputs": [],
   "source": [
    "#В качестве расстояния берем евклидово между векторами\n",
    "from scipy.spatial.distance import euclidean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 593,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    \"\"\"Сигмоида.\n",
    "    \"\"\"\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 594,
   "metadata": {},
   "outputs": [],
   "source": [
    "def partial_derivative(j, b, x, y):\n",
    "    \"\"\"Частная производная функционала Q по переменной b_j.\n",
    "    \"\"\"\n",
    "    return -sum(\n",
    "        x[i, j] * y[i] * (1 - sigmoid(b.dot(x[i]) * y[i]))\n",
    "        for i in range(x.shape[0])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 595,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient(b, x, y):\n",
    "    \"\"\"Вектор градиента.\n",
    "    \"\"\"\n",
    "   \n",
    "    return np.array([\n",
    "        partial_derivative(j, b, x, y)\n",
    "        for j in range(b.shape[0])\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 596,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent_step(lambda_, b, x, y):\n",
    "    \"\"\"Один шаг градиентного спуска.\n",
    "    \"\"\"\n",
    "    minus=lambda_ * gradient(b, x, y)\n",
    "    \n",
    "    return b - minus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 597,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success on step 90\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-0.3900818 ],\n",
       "       [-0.53387022],\n",
       "       [-0.49039695],\n",
       "       [-0.28148672],\n",
       "       [-0.61341543],\n",
       "       [-0.30067757],\n",
       "       [-0.02913996],\n",
       "       [ 0.03231162],\n",
       "       [-0.02741305],\n",
       "       [ 2.69923751],\n",
       "       [ 0.21203123],\n",
       "       [ 0.0660471 ],\n",
       "       [ 0.04790598],\n",
       "       [ 0.14083599]])"
      ]
     },
     "execution_count": 597,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#В качестве стартового берем вектор с нулями\n",
    "b_0 = np.zeros(x.shape[1])\n",
    "b = b_0\n",
    "\n",
    "for i in range(1, 10 ** 5):\n",
    "    lambda_=0.01/i\n",
    "    \n",
    "    b_new = gradient_descent_step(lambda_, b, x, y)\n",
    "    diff = euclidean(b, b_new)\n",
    "    if diff < 0.001:\n",
    "        print(f'success on step {i}\\n')\n",
    "        \n",
    "        break\n",
    "    b = b_new\n",
    "\n",
    "b = b_new.reshape(-1, 1)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 598,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = np.array([sigmoid(z_i) for z_i in x.dot(b)])\n",
    "z2 = np.array([sigmoid(z_i) for z_i in x2.dot(b)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 599,
   "metadata": {},
   "outputs": [],
   "source": [
    "z_train = np.array([\n",
    "    1 if z_ > 0.5 else -1\n",
    "    for z_ in z\n",
    "])\n",
    "z_valid = np.array([\n",
    "    1 if z_ > 0.5 else -1\n",
    "    for z_ in z2\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 600,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 601,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8472049689440994, 0.8521739130434782)"
      ]
     },
     "execution_count": 601,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y, z_train), accuracy_score(y2, z_valid)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Модель выдает хорошие показатели как на учебном сете, так и для валидационных данных. \n",
    "Более подробный анализ прогностической модели на отдельных проектах показал, что для новых клиентов вероятность продажи проекта оценивается как низкая и уровень выставляемой цены большого влияния не оказывает - возможно, будет правильно сделать отдельные модели для новых клиентов и клиентов заказывавших ранее."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}