Notebooks/02_Feature_Engineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "744dc637",
   "metadata": {},
   "source": [
    "## <span style=\"color : green\"> Feature Engineering </span>\n",
    "\n",
    "# <center> Table of Contents </center>\n",
    "\n",
    "1. Split data into training and testing \n",
    "2. Drop independent variable based on - mean, median, variance\n",
    "1. Finding outliers based on z-score and box-plot\n",
    "1. correlation between independent variables\n",
    "1. Correlation between independent variable and dependent\n",
    "1. Select Features based on Chi-Square Test, Information Gain, VarianceThreshold\n",
    "1. Feature Extraction using PCA and LDA\n",
    "1. Mean of Pixel value of Image "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "48c348b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.stats import pearsonr\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.feature_selection import chi2, SelectKBest, mutual_info_classif, VarianceThreshold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from warnings import filterwarnings\n",
    "\n",
    "\n",
    "filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "0ddd017a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Price</th>\n",
       "      <th>Age</th>\n",
       "      <th>Mfg_Month</th>\n",
       "      <th>Mfg_Year</th>\n",
       "      <th>KM</th>\n",
       "      <th>Fuel_Type</th>\n",
       "      <th>HP</th>\n",
       "      <th>Met_Color</th>\n",
       "      <th>Automatic</th>\n",
       "      <th>cc</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Gears</th>\n",
       "      <th>Quarterly_Tax</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Mfr_Guarantee</th>\n",
       "      <th>BOVAG_Guarantee</th>\n",
       "      <th>Guarantee_Period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12950</td>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>2002</td>\n",
       "      <td>46986</td>\n",
       "      <td>Diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Five</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12950</td>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>2002</td>\n",
       "      <td>72937</td>\n",
       "      <td>Diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Five</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12950</td>\n",
       "      <td>24</td>\n",
       "      <td>9</td>\n",
       "      <td>2002</td>\n",
       "      <td>41711</td>\n",
       "      <td>Diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Five</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13950</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>2002</td>\n",
       "      <td>48000</td>\n",
       "      <td>Diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Five</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17950</td>\n",
       "      <td>30</td>\n",
       "      <td>3</td>\n",
       "      <td>2002</td>\n",
       "      <td>38500</td>\n",
       "      <td>Diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Five</td>\n",
       "      <td>210</td>\n",
       "      <td>1170</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Price  Age  Mfg_Month  Mfg_Year     KM Fuel_Type  HP  Met_Color  Automatic  \\\n",
       "0  12950   23         10      2002  46986    Diesel  90          1          0   \n",
       "1  12950   23         10      2002  72937    Diesel  90          1          0   \n",
       "2  12950   24          9      2002  41711    Diesel  90          1          0   \n",
       "3  13950   26          7      2002  48000    Diesel  90          0          0   \n",
       "4  17950   30          3      2002  38500    Diesel  90          0          0   \n",
       "\n",
       "     cc  Doors Gears  Quarterly_Tax  Weight  Mfr_Guarantee  BOVAG_Guarantee  \\\n",
       "0  2000    3.0  Five            210    1165            0.0                1   \n",
       "1  2000    3.0  Five            210    1165            0.0                1   \n",
       "2  2000    3.0  Five            210    1165            1.0                1   \n",
       "3  2000    3.0  Five            210    1165            1.0                1   \n",
       "4  2000    3.0  Five            210    1170            1.0                1   \n",
       "\n",
       "   Guarantee_Period  \n",
       "0                 3  \n",
       "1                 3  \n",
       "2                 3  \n",
       "3                 4  \n",
       "4                 5  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('../Datasets/Toyato.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "a007b5cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50, 17)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "205fc680",
   "metadata": {},
   "source": [
    "## Split data into training and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "73a8a25a",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = df.drop('Fuel_Type', axis=1)\n",
    "x = x.select_dtypes(include=['int64', 'float64'])\n",
    "y = df[['Fuel_Type']]\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x,y, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8cc4d925",
   "metadata": {},
   "source": [
    "## Drop independent variable based on - \n",
    "\n",
    "    1. If mean less than median \n",
    "    2. bIf variance is less than the theshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "1a6d41cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Age', 'Met_Color', 'Mfr_Guarantee', 'BOVAG_Guarantee']\n"
     ]
    }
   ],
   "source": [
    "# Checking if mean is less than median\n",
    "\n",
    "low_mean_features = list()\n",
    "\n",
    "for col in x.columns:\n",
    "    mean = x[col].mean()\n",
    "    median = x[col].median()\n",
    "    if mean < median :\n",
    "        low_mean_features.append(col)\n",
    "\n",
    "print(low_mean_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "e931d706",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Mfg_Year', 'Automatic', 'BOVAG_Guarantee']\n"
     ]
    }
   ],
   "source": [
    "# Checking if variance is less than threshold\n",
    "\n",
    "low_variance_features = list()\n",
    "threshold = 0.1\n",
    "\n",
    "for col in x.columns:\n",
    "    variance = x[col].var()\n",
    "    if variance < threshold :\n",
    "        low_variance_features.append(col)\n",
    "\n",
    "print(low_variance_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbfdafda",
   "metadata": {},
   "source": [
    "## Finding outliers based on z-score and box-plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "74bdce1d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Price</th>\n",
       "      <th>Age</th>\n",
       "      <th>Mfg_Month</th>\n",
       "      <th>Mfg_Year</th>\n",
       "      <th>KM</th>\n",
       "      <th>HP</th>\n",
       "      <th>Met_Color</th>\n",
       "      <th>Automatic</th>\n",
       "      <th>cc</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Quarterly_Tax</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Mfr_Guarantee</th>\n",
       "      <th>BOVAG_Guarantee</th>\n",
       "      <th>Guarantee_Period</th>\n",
       "      <th>z-score-price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12950</td>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>2002</td>\n",
       "      <td>46986</td>\n",
       "      <td>90</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.849159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12950</td>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>2002</td>\n",
       "      <td>72937</td>\n",
       "      <td>90</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.849159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12950</td>\n",
       "      <td>24</td>\n",
       "      <td>9</td>\n",
       "      <td>2002</td>\n",
       "      <td>41711</td>\n",
       "      <td>90</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.849159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13950</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>2002</td>\n",
       "      <td>48000</td>\n",
       "      <td>90</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>210</td>\n",
       "      <td>1165</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.454219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17950</td>\n",
       "      <td>30</td>\n",
       "      <td>3</td>\n",
       "      <td>2002</td>\n",
       "      <td>38500</td>\n",
       "      <td>90</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>210</td>\n",
       "      <td>1170</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1.125537</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Price  Age  Mfg_Month  Mfg_Year     KM  HP  Met_Color  Automatic    cc  \\\n",
       "0  12950   23         10      2002  46986  90          1          0  2000   \n",
       "1  12950   23         10      2002  72937  90          1          0  2000   \n",
       "2  12950   24          9      2002  41711  90          1          0  2000   \n",
       "3  13950   26          7      2002  48000  90          0          0  2000   \n",
       "4  17950   30          3      2002  38500  90          0          0  2000   \n",
       "\n",
       "   Doors  Quarterly_Tax  Weight  Mfr_Guarantee  BOVAG_Guarantee  \\\n",
       "0    3.0            210    1165            0.0                1   \n",
       "1    3.0            210    1165            0.0                1   \n",
       "2    3.0            210    1165            1.0                1   \n",
       "3    3.0            210    1165            1.0                1   \n",
       "4    3.0            210    1170            1.0                1   \n",
       "\n",
       "   Guarantee_Period  z-score-price  \n",
       "0                 3      -0.849159  \n",
       "1                 3      -0.849159  \n",
       "2                 3      -0.849159  \n",
       "3                 4      -0.454219  \n",
       "4                 5       1.125537  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x['z-score-price'] = ( x.Price - x.Price.mean() ) / x.Price.std()\n",
    "x = x[(x['z-score-price'] > -3) & (x['z-score-price'] < 3) ]\n",
    "x.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b89dad75",
   "metadata": {},
   "source": [
    "## correlation between independent variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "62ee8809",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Mfg_Month', 'Price', 'Age', 'Quarterly_Tax', 'cc']\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.75\n",
    "\n",
    "correlated_features = []\n",
    "\n",
    "correlation_matrix = x.corr()\n",
    "\n",
    "for i in range(correlation_matrix.shape[0]):\n",
    "    for j in range(i+1, correlation_matrix.shape[1]):\n",
    "        if abs(correlation_matrix.iloc[i,j]) > threshold:\n",
    "            correlated_features.append(correlation_matrix.columns[i])\n",
    "\n",
    "print(list(set(correlated_features)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "001e0e03",
   "metadata": {},
   "source": [
    "## Correlation between independent variable and dependent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "f8a1d928",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = LabelEncoder()\n",
    "labels = encoder.fit_transform(y)\n",
    "\n",
    "x.fillna(x.mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "58c66f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Met_Color', 'Automatic']\n"
     ]
    }
   ],
   "source": [
    "# To find out the unrelated attribute w/r to target \n",
    "\n",
    "threshold = 0.1\n",
    "\n",
    "uncorrelated_features = []\n",
    "\n",
    "for feature in x.columns:\n",
    "    correlation, _ = pearsonr(x[feature] ,labels )\n",
    "    if abs(correlation) < threshold:\n",
    "        uncorrelated_features.append(feature)\n",
    "        \n",
    "print(uncorrelated_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee3533c9",
   "metadata": {},
   "source": [
    "## Select Features based on Chi-Square Test, SelectPercentile, VarianceThreshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "b40f01de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The selected attributes were : ['KM', 'cc', 'Quarterly_Tax']\n"
     ]
    }
   ],
   "source": [
    "# Using Chi-Square Test \n",
    "\n",
    "selector = SelectKBest(score_func=chi2, k=3)\n",
    "new_df = selector.fit_transform(abs(x), labels)\n",
    "\n",
    "print(f'The selected attributes were : {list(x.columns[selector.get_support()])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "64ae2a04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The selected attributes were : ['HP', 'cc', 'Quarterly_Tax']\n"
     ]
    }
   ],
   "source": [
    "# Using Information Gain \n",
    "\n",
    "selector = SelectKBest(score_func=mutual_info_classif, k=3)\n",
    "new_df = selector.fit_transform(abs(x), labels)\n",
    "\n",
    "print(f'The selected attributes were : {list(x.columns[selector.get_support()])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "39f58d9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Old Feature shape = (50, 16)\n",
      "New Feature Shape = (50, 11)\n"
     ]
    }
   ],
   "source": [
    "# Using Variance Threshold\n",
    "\n",
    "selector = VarianceThreshold(0.4)\n",
    "new_df = selector.fit_transform(x)\n",
    "print(f'Old Feature shape = {x.shape}\\nNew Feature Shape = {new_df.shape}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d3529ea",
   "metadata": {},
   "source": [
    "## Feature Extraction using PCA and LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "9d109fbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 7)\n"
     ]
    }
   ],
   "source": [
    "# Principle Component Analysis\n",
    "\n",
    "analyser = PCA(n_components=7)\n",
    "extracted = analyser.fit_transform(x)\n",
    "\n",
    "print(np.shape(extracted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "df57f2aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 1)\n"
     ]
    }
   ],
   "source": [
    "# Linear Discriminant Analysis\n",
    "\n",
    "analyser = LinearDiscriminantAnalysis(n_components=1)\n",
    "extracted = analyser.fit_transform(x, y)\n",
    "print(np.shape(extracted))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "132849ca",
   "metadata": {},
   "source": [
    "## Mean of Pixel value of Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "79f5d7d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean pixel value: 72.43\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "\n",
    "image = Image.open(\"image.png\")\n",
    "\n",
    "image_array = np.array(image)\n",
    "mean_pixels= np.mean(image_array)\n",
    "\n",
    "print(f\"Mean pixel value: {round(mean_pixels,2)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}