nus_datathon_submission.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Set-up and Installing of Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MsDZzX1vdea5",
        "outputId": "31077c2f-621f-423f-d68d-e04f6910cab1"
      },
      "outputs": [],
      "source": [
        "# Installing required packages\n",
        "%pip install tableone\n",
        "%pip install shap\n",
        "\n",
        "# Importing dependencies for this notebook\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import warnings\n",
        "import shap\n",
        "from tableone import TableOne\n",
        "from datetime import datetime\n",
        "from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder, StandardScaler\n",
        "from sklearn.compose import make_column_transformer\n",
        "from sklearn.impute import KNNImputer\n",
        "from sklearn.model_selection import cross_validate, StratifiedKFold, GridSearchCV\n",
        "from sklearn.utils import resample\n",
        "from xgboost import XGBClassifier\n",
        "import pickle\n",
        "\n",
        "#Configurations\n",
        "warnings.filterwarnings(action='ignore')\n",
        "filepath = \"./data/catB_train.parquet\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ykMse5xdL4JK"
      },
      "source": [
        "## 1.0 Dataset Cleaning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "C9cYVMREgEFh",
        "outputId": "1c69114b-ca55-4f22-f354-b67b5433383c"
      },
      "outputs": [],
      "source": [
        "# Opening the train dataset\n",
        "df = pd.read_parquet(filepath)\n",
        "\n",
        "# Replacing all missing data as NaN (some are encoded as None)\n",
        "df = df.fillna(value = np.nan)\n",
        "display(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5TaOKJFpL0ev"
      },
      "source": [
        "## 1.1 Dropping rows and columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TZp_2bRaXN4V"
      },
      "outputs": [],
      "source": [
        "# Replacing NAs in target column with 0 so they don't accidentally get dropped\n",
        "df['f_purchase_lh'] = df['f_purchase_lh'].fillna(0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A2eWUHJJUwcA",
        "outputId": "bc0d437f-e469-4a9e-d17e-88eb6a50293d"
      },
      "outputs": [],
      "source": [
        "# Dropping columns where more than half the values are NA\n",
        "null_thresh = 0.5 * len(df)\n",
        "col_dropped = df.columns[df.isna().sum() >= null_thresh]\n",
        "df = df.dropna(axis=1, thresh = null_thresh)\n",
        "\n",
        "print(\"A total of\",304-df.shape[1],\"columns dropped. Columns dropped are\", list(col_dropped),\"\\n\")\n",
        "print(f\"The current shape of the df is as follows: {df.shape}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Xe1Cz-6VO3e",
        "outputId": "807b6249-c909-40e8-89db-209c0914930b"
      },
      "outputs": [],
      "source": [
        "# Dropping columns with only one unique value (they add nothing to the model)\n",
        "col_no = df.shape[1]\n",
        "\n",
        "columns_to_drop = df.columns[df.nunique() == 1]\n",
        "df = df.drop(columns = columns_to_drop)\n",
        "\n",
        "print(\"A total of\", col_no - df.shape[1],\"columns dropped. Columns dropped are\", columns_to_drop,\"\\n\")\n",
        "print(f\"The current shape of the df is as follows: {df.shape}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "id": "eshU1ODPTk_V",
        "outputId": "3a0551f4-9b84-4e8a-9d46-74a811e145e8"
      },
      "outputs": [],
      "source": [
        "# Dropping race as well due to ethical and social implications of using race to drive actions;\n",
        "# also since chinese are the majority @ 58.5% and NaN making up 22.2% with the rest <10%\n",
        "# race also contains missing values that are problematic to impute from statistical POV\n",
        "\n",
        "display(df[\"race_desc\"].value_counts(dropna=False).divide(df.shape[0]).multiply(100).apply(lambda x: '{:.1f}%'.format(x)))\n",
        "df = df.drop('race_desc', axis=1)\n",
        "print()\n",
        "\n",
        "# Also dropped client number as that won't be used in model\n",
        "df = df.drop('clntnum', axis=1)\n",
        "print(f\"The current shape of the df is as follows: {df.shape}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8qo_kMSG3z3N",
        "outputId": "d2d5c2c5-aea1-41f9-d52a-1c2c2a5c4cd4"
      },
      "outputs": [],
      "source": [
        "# Displaying list of all columns with at least one NA\n",
        "null_values_per_column = df.isna().sum().sort_values(ascending=False)\n",
        "columns_with_null_values = null_values_per_column[null_values_per_column > 0]\n",
        "\n",
        "pd.set_option('display.max_rows', None)\n",
        "print(columns_with_null_values)\n",
        "pd.reset_option('display.max_rows')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9xsxgMhXBUot",
        "outputId": "bcaed29d-7579-413f-fbb3-9e07ce8bd157"
      },
      "outputs": [],
      "source": [
        "# Note that across quite a few key client status indicators, there are exactly 1,014 NAs for all of them\n",
        "# suspect that they belong to the same individuals, thus will drop all individuals with NAs across these columns\n",
        "# drop observations which are NULL across key status columns (these individuals don't provide much useful info)\n",
        "null_values_per_column = df.isna().sum()\n",
        "columns_to_check = null_values_per_column[null_values_per_column == 1014]\n",
        "list_of_columns = columns_to_check.index.tolist()\n",
        "\n",
        "orig_len = len(df)\n",
        "\n",
        "df = df.dropna(subset = list_of_columns)\n",
        "\n",
        "print('Number of observations dropped =', orig_len - len(df))\n",
        "print(f\"The current shape of the df is as follows: {df.shape}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uznkV3-mHmrL",
        "outputId": "6685079b-db5a-4907-b668-0e0a1b67fa0a"
      },
      "outputs": [],
      "source": [
        "# Since only 23 individuals with missing data for sex, propose to exclude them from analysis\n",
        "# same for country - only 20 individuals missing, so exclude them as well\n",
        "# as there is no reliable way to impute a client's sex and country without introducing substantial bias\n",
        "orig_len = len(df)\n",
        "\n",
        "df = df.dropna(subset = ['cltsex_fix', 'ctrycode_desc'])\n",
        "\n",
        "print('Number of observations dropped =', orig_len - len(df))\n",
        "print(f\"The current shape of the df is as follows: {df.shape}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IDMK97RncR7u",
        "outputId": "4ae3eea6-db3b-4ddb-c90e-4a52c68e08d8"
      },
      "outputs": [],
      "source": [
        "# Observed that hh_size_est is derived from hh_size, which is calculated from hh_20 and pop_20\n",
        "# thus drop hh_size, hh_20, and pop_20\n",
        "df = df.drop(columns = ['hh_20', 'pop_20', 'hh_size'], axis = 1)\n",
        "print(f\"The current shape of the df is as follows: {df.shape}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yGsn7WcLOP0A"
      },
      "source": [
        "## 1.2 Label Encoding\n",
        "\n",
        "\n",
        "Numerize all columns.\n",
        "\n",
        "Step 1: Label encode ordinal variables and one-hot encode nominal variables.\n",
        "\n",
        "Step 2: Convert `cltdob_fix` to current age and `min_occ_date` to duration since person became customer in months.\n",
        "\n",
        "Step 3: Convert all columns stored as object into numeric (float64/int8/int16 etc) columns.\n",
        "\n",
        "Step 4: Convert all numeric columns into int64 format."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3VHeRDB1BQaH",
        "outputId": "3ddc00fb-4cfc-40a7-f9fb-0891ec6174c4"
      },
      "outputs": [],
      "source": [
        "# Examining data types of all columns in dataframe\n",
        "pd.set_option('display.max_rows', None)\n",
        "print(df.dtypes)\n",
        "pd.reset_option('display.max_rows')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J77LQIcksl9g"
      },
      "outputs": [],
      "source": [
        "# Label encoding following columns: hh_size_est, annual_income_est\n",
        "ordinal_cols = ['hh_size_est', 'annual_income_est']\n",
        "\n",
        "oe = OrdinalEncoder()\n",
        "df[ordinal_cols] = oe.fit_transform(df[ordinal_cols])\n",
        "\n",
        "# One-hot encoding nominal variables: ctrycode_desc, clttype, stat_flag, cltsex_fix\n",
        "nominal_cols = ['ctrycode_desc', 'clttype', 'stat_flag', 'cltsex_fix']\n",
        "\n",
        "transformer = make_column_transformer((OneHotEncoder(), nominal_cols),\n",
        "                                      remainder = 'passthrough')\n",
        "\n",
        "transformed = transformer.fit_transform(df)\n",
        "df = pd.DataFrame(transformed, columns = transformer.get_feature_names_out())\n",
        "df.columns = df.columns.str.replace('onehotencoder__|remainder__', '')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "robdft6deiQx"
      },
      "outputs": [],
      "source": [
        "# Converting dob to current age\n",
        "df['cltdob_fix'] = pd.to_datetime(df['cltdob_fix'])\n",
        "df['current_age'] = 2024 - df['cltdob_fix'].dt.year"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6pKXiz8DCxo-"
      },
      "outputs": [],
      "source": [
        "# Converting 'date of client's first interaction or policy purchase' to 'duration since becoming customer (in months)'\n",
        "df = df[df[\"min_occ_date\"] != \"None\"] # drop the 7 rows with missing occ date\n",
        "df['min_occ_date'] = pd.to_datetime(df['min_occ_date'], format = '%Y-%m-%d')\n",
        "today = datetime.today()\n",
        "df['cust_duration_mths'] = (today - df['min_occ_date']) // pd.Timedelta('30D')\n",
        "\n",
        "# Droping dob and min_occ_date columns\n",
        "df = df.drop(columns = ['cltdob_fix', 'min_occ_date'], axis = 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0Qxt1QK3Bq-a"
      },
      "outputs": [],
      "source": [
        "# Converting entire dataframe to numeric\n",
        "df = df.apply(pd.to_numeric, downcast = 'integer')\n",
        "\n",
        "# Identifying all numeric columns\n",
        "numeric_columns = df.select_dtypes(include = 'number').columns\n",
        "\n",
        "# Converting all numeric columns to int64\n",
        "df[numeric_columns] = df[numeric_columns].astype('int64', errors = 'ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "Eh6X8-sBO6_T",
        "outputId": "5d5b1cd7-8582-44e9-a690-6306d0e38cba"
      },
      "outputs": [],
      "source": [
        "# These are the columns that are not int64 yet\n",
        "display(df.select_dtypes(exclude='int').columns)\n",
        "\n",
        "# Double checking every variables have been converted to numerical\n",
        "print(\"There are a total of\", (df.dtypes != int).sum(), \"non-integer variables.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y8TRKnPjO4LN"
      },
      "source": [
        "## 1.3 Missing data imputation\n",
        "\n",
        "Variables to impute for: `hh_size_est` and `annual_income_est`."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "qmFLosgEFTMB",
        "outputId": "e7500b20-35a2-4084-ebd2-3aeb23b55d12"
      },
      "outputs": [],
      "source": [
        "# Checking the columns still with NA\n",
        "display(df.columns[df.isna().any()].tolist())\n",
        "\n",
        "print()\n",
        "\n",
        "# Checking how many NA are there before KNN imputation\n",
        "display(df[[\"hh_size_est\", \"annual_income_est\"]].isna().sum())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xWSX-VGaL47D"
      },
      "outputs": [],
      "source": [
        "# Imputing using KNN imputation\n",
        "\n",
        "# Specifying the columns with missing values\n",
        "columns_with_missing_values = df.columns[df.isna().any()].tolist()\n",
        "\n",
        "# Initializing KNN imputer\n",
        "knn_imputer = KNNImputer(n_neighbors=5)\n",
        "\n",
        "# Performing imputation on the specified columns\n",
        "df[columns_with_missing_values] = knn_imputer.fit_transform(df[columns_with_missing_values])\n",
        "df[columns_with_missing_values] = df[columns_with_missing_values].round()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FK_2v2MMM5_E",
        "outputId": "cad3eb64-e2c2-40bf-c8c5-bbcdf0e79ce5"
      },
      "outputs": [],
      "source": [
        "# Checking how many NAs are there after KNN imputation\n",
        "df[[\"hh_size_est\", \"annual_income_est\"]].isna().sum()\n",
        "\n",
        "print(\"There are currently\", df.isna().sum().sum(), \"NAs in the dataset.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_PzSJruQJiV"
      },
      "source": [
        "## 2.0 Feature Selection\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qsd9nb_mPPyj"
      },
      "source": [
        "## 2.1 **Trimming** features based on domain knowledge\n",
        "\n",
        "*   Remove `n_months_last_bought_` set of features as max value (indicating customers who never bought policy) is 9999 which will wildly skew model estimates. Likely to be highly correlated with `f_ever_bought_` set of features anyway, since 9999 for the former corresponds to 0 for the latter.\n",
        "*   Aggregate ape, sumins, and prempaid across all products per client to reduce overgranularity of data and provide more high-level insights.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pLHB0AZZW9AQ",
        "outputId": "3f952e9d-b896-4bd4-b18d-3143895d0f45"
      },
      "outputs": [],
      "source": [
        "# Dropping columns with max value of 9999\n",
        "col_9999 = [x for x in df.columns if getattr(df, x).max() == 9999]\n",
        "df = df.drop(columns = col_9999, axis = 1)\n",
        "\n",
        "print(len(col_9999), 'columns were dropped. The dataframe now has shape ', df.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yEA5eS1Ddjhv",
        "outputId": "c6a51efd-6a29-4712-c379-8a47b94bc3a9"
      },
      "outputs": [],
      "source": [
        "# Aggregating ape, sumins, and prempaid columns to obtain total sum of ape, sumins, and prempaid per client\n",
        "metx = ['ape', 'sumins', 'prempaid']\n",
        "\n",
        "for i in metx:\n",
        "  new_colname = i + '_total'\n",
        "  rel_cols = [col for col, is_rel in zip(df.columns, [col.split('_')[0] == i for col in df.columns]) if is_rel]\n",
        "  df[new_colname] = df[rel_cols].sum(axis = 1)\n",
        "  df = df.drop(columns = rel_cols, axis = 1)\n",
        "\n",
        "print(f\"The current shape of the df is as follows: {df.shape}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5mzwT8fuQSKq"
      },
      "source": [
        "## 2.2 Table One\n",
        "Create a table of descriptive statistics comparing between target and control population for each feature. Drop features which are not significantly different between target_variable == 0 and target_variable == 1 as they are not statistically important for the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DWPHdkKc8G2w",
        "outputId": "9d36d6b3-ebbd-464c-aa65-6e499de07a7f"
      },
      "outputs": [],
      "source": [
        "# Balancing the cohort\n",
        "# Calculating class proportions\n",
        "class_proportions = df['f_purchase_lh'].value_counts(normalize=True)\n",
        "\n",
        "# Determining desired size of the majority class (e.g., 100% of the minority class)\n",
        "desired_majority_size = int(class_proportions.min() * 1.0 * len(df[df['f_purchase_lh'] == 0]))\n",
        "\n",
        "# Undersampling the majority class while maintaining distribution\n",
        "undersampled_majority = df[df['f_purchase_lh'] == 0].sample(desired_majority_size, replace=False, random_state=42)\n",
        "\n",
        "# Combining the undersampled majority class with the minority class\n",
        "undersampled_df = pd.concat([undersampled_majority, df[df['f_purchase_lh'] == 1]])\n",
        "\n",
        "# Shuffling the combined DataFrame\n",
        "undersampled_df = undersampled_df.sample(frac=1, random_state=42).reset_index(drop=True)\n",
        "\n",
        "# Checking if undersampling is done\n",
        "print(f\"After balancing the the distribution between those that bought and those that didnt bought within 3 months are as follow:\\n{undersampled_df.f_purchase_lh.value_counts()}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "u0pQYYPBvuEX",
        "outputId": "0a7e7faa-4e89-467d-e3a8-acdae5ece980"
      },
      "outputs": [],
      "source": [
        "# Creating tableone\n",
        "groupby=[\"f_purchase_lh\"]\n",
        "non_normal = [x for x in undersampled_df.columns if getattr(undersampled_df, x).max() > 10]\n",
        "categories = [column for column in list(undersampled_df.columns) if column not in non_normal and column not in groupby]\n",
        "\n",
        "mytable = TableOne(undersampled_df, columns= list(undersampled_df.columns), categorical=categories, groupby=groupby, nonnormal=non_normal, pval=True, htest_name = True, sort = \"P-Value\" )\n",
        "pd.set_option(\"display.max_rows\", None)\n",
        "display(mytable)\n",
        "pd.reset_option(\"display.max_rows\")\n",
        "\n",
        "summary_table_data = mytable.tableone"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7BzVKbjGVXWC"
      },
      "outputs": [],
      "source": [
        "# Converting the summary table dictionary to a pandas dataframe and reset index\n",
        "tableone_df = pd.DataFrame(summary_table_data)\n",
        "tableone_df = tableone_df.reset_index()\n",
        "tableone_df.columns = tableone_df.columns.get_level_values(1)\n",
        "\n",
        "# Converting p-value column to numeric and keep rows with p-value > 0.05 (these variables are not significant)\n",
        "tableone_df['P-Value'] = pd.to_numeric(tableone_df['P-Value'], errors = 'coerce')\n",
        "cols_to_drop = tableone_df[tableone_df['P-Value'] > 0.05]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mxRdloCYCiiS",
        "outputId": "6484ed9b-05c2-4b0c-e3dc-112224a96be9"
      },
      "outputs": [],
      "source": [
        "# Obtaining column names in a list and drop these columns from main dataset\n",
        "cols_to_drop = cols_to_drop.iloc[:,[0]]\n",
        "cols_to_drop = cols_to_drop.rename(columns={'':'names'})\n",
        "list_to_drop = list(cols_to_drop['names'].str.split(',').str[0])\n",
        "\n",
        "df = df.drop(columns = list_to_drop, axis = 1)\n",
        "print(len(list_to_drop), 'columns were dropped. The dataset now has', df.shape[1], 'features.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cFKgARy3RRxf",
        "outputId": "757cbc55-7291-4478-d8e3-4af727df30a6"
      },
      "outputs": [],
      "source": [
        "# Condensing nominal variables to x or not x. For example, client type is majority P,\n",
        "# so we only need column for P with 1 being P and 0 being not P\n",
        "redundant = ['clttype_G', 'stat_flag_LAPSED']\n",
        "df = df.drop(columns = redundant, axis = 1)\n",
        "\n",
        "df = df.rename(columns={'ctrycode_desc_Singapore': 'ctry_is_sg', 'stat_flag_ACTIVE': 'stat_flag_is_active', 'clttype_P' : 'clttype_is_p'})\n",
        "\n",
        "print(len(redundant), 'columns were dropped. The dataset now has', df.shape[1], 'features.')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GLrg8h85Rs1S"
      },
      "source": [
        "## 2.3 Correlation Matrix\n",
        "\n",
        "Check for multicollinearity using a correlation matrix between variables."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 917
        },
        "id": "kljf4jvxDDP4",
        "outputId": "ca5b82ac-1dd6-45bd-ddf8-448bb1bf52b3"
      },
      "outputs": [],
      "source": [
        "# Setting figure size\n",
        "plt.figure(figsize=(20, 8))\n",
        "\n",
        "# Plotting correlation heatmap\n",
        "heatmap = sns.heatmap(df.corr(), vmin=-1, vmax=1, cmap='BrBG')\n",
        "heatmap.set_title('Correlation Heatmap', fontdict={'fontsize':12}, pad=12)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o03BfEWdStfz"
      },
      "source": [
        "Note that `f_hold_xxx` is highly correlated with `f_ever_bought_xxx`. Since it is not stated in the GitHub documentation what the variable `f_hold_xxx` means, we shall exclude it from analysis due to its lack of interpretability and multicollinearity with other variables.\n",
        "\n",
        "Also drop `f_ever_bought_ltc_43b9d5` as it is highly correlated with `f_ever_bought_ltc`."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QV6thg60SqMm",
        "outputId": "8b2db262-9a61-478c-fb4e-80e42a3e68da"
      },
      "outputs": [],
      "source": [
        "# Removing `f_hold_xxx` variables\n",
        "cols_to_drop = [col for col in df.columns if col.startswith('f_hold_')]\n",
        "cols_to_drop.append('f_ever_bought_ltc_43b9d5')\n",
        "df = df.drop(columns = cols_to_drop, axis = 1)\n",
        "\n",
        "print(len(cols_to_drop), 'columns were dropped. The dataset now has', df.shape[1], 'features.')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I8MNumrfSKFf"
      },
      "source": [
        "## 3.0 Balancing the dataset\n",
        "\n",
        "- Downsample majority by 50%\n",
        "- Upsample minority to match number of majority"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ldvfmuux3hSb"
      },
      "outputs": [],
      "source": [
        "# Downsampling the majority class\n",
        "majority = df[df['f_purchase_lh'] == 0]\n",
        "majority_downsampled = majority.sample(frac=0.25, random_state=42)\n",
        "\n",
        "# Upsampling the minority class\n",
        "minority = df[df['f_purchase_lh'] == 1]\n",
        "minority_upsampled = resample(minority,\n",
        "                              replace=True,\n",
        "                              n_samples=len(majority_downsampled),\n",
        "                              random_state=42)\n",
        "\n",
        "# Concatenatng the majority_downsampled and minority_upsampled to get the balanced dataset\n",
        "df_balanced = pd.concat([majority_downsampled, minority_upsampled])\n",
        "\n",
        "# Shuffling the dataset\n",
        "df_balanced = df_balanced.sample(frac=1, random_state=42).reset_index(drop=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H9N-R9NR5KVM",
        "outputId": "0992a2b7-8cbb-40cc-ea4d-167128112969"
      },
      "outputs": [],
      "source": [
        "# Checking if df is properly balanced\n",
        "df_balanced['f_purchase_lh'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_I4drkN7ShRe"
      },
      "source": [
        "## 4.0 Model Building\n",
        "\n",
        "* Estimator: XGBoost classifier."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IwWpLPmC0RGN"
      },
      "source": [
        "## 4.1 Model Training and Hyperparameter Tuning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0vm8vuruWhdX"
      },
      "outputs": [],
      "source": [
        "# Split data into features and label\n",
        "df_balanced_features = df_balanced.drop(['f_purchase_lh'], axis = 1)\n",
        "labels = df_balanced['f_purchase_lh']\n",
        "\n",
        "# Note: train-test split not used here since the model will already be tested on hidden data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QFJDD2pKN1Rc"
      },
      "outputs": [],
      "source": [
        "# Scale continuous variables\n",
        "scaler = StandardScaler()\n",
        "\n",
        "continuous_cols = [x for x in df_balanced.columns if getattr(df_balanced, x).max() > 10]\n",
        "df_balanced_features[continuous_cols] = scaler.fit_transform(df_balanced_features[continuous_cols])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VdrRkE8ZQfe9"
      },
      "outputs": [],
      "source": [
        "#################################################################################\n",
        "#   WARNING: GRIDSEARCHCV WILL TAKE APPROX 45MINS, PLEASE DO NOT RUN THIS CELL. #\n",
        "#     WE HAVE RAN IT AND TAKEN DOWN THE BEST HYPERPARAMS FOR THE MODEL          #\n",
        "#################################################################################\n",
        "\n",
        "# # Initialize XGBoost classifier\n",
        "# xgb_model = XGBClassifier(\n",
        "#     objective='binary:logistic',\n",
        "#     eval_metric='logloss'  # Choose the appropriate evaluation metric\n",
        "# )\n",
        "\n",
        "# # Define hyperparameters to search through\n",
        "# param_grid = {\n",
        "#     'n_estimators': [50, 100, 200, 300],\n",
        "#     'max_depth': [4, 5, 7, 8],\n",
        "#     'learning_rate': [0.01, 0.05, 0.1, 0.2],\n",
        "#     'lambda': np.power(10., np.arange(0,3)),\n",
        "#     'alpha': np.power(10., np.arange(0,3))\n",
        "# }\n",
        "\n",
        "# # Initialize StratifiedKFold for cross-validation\n",
        "# cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)\n",
        "\n",
        "# # Initialize GridSearchCV for hyperparameter tuning\n",
        "# grid_search = GridSearchCV(\n",
        "#     estimator = xgb_model,\n",
        "#     param_grid = param_grid,\n",
        "#     scoring = ('precision', 'recall', 'roc_auc'),\n",
        "#     cv = cv,\n",
        "#     verbose = 3,\n",
        "#     refit = 'precision'\n",
        "# )\n",
        "\n",
        "# # Fit the GridSearchCV to find the best hyperparameters\n",
        "# grid_search.fit(df_balanced_features, labels)\n",
        "\n",
        "# # Get the best hyperparameters\n",
        "# best_params = grid_search.best_params_\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "96nFoL8bUGF_"
      },
      "source": [
        "After running the cell above, we derived the following optimal hyperparameters:\n",
        "\n",
        "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQ0AAAB4CAYAAADoiqfOAAABUWlDQ1BJQ0MgUHJvZmlsZQAAGJV10DFLw2AQBuA3WKmoaAu6KURwUaKUWARxqhWq4BBqxeqWpjFR0viRpEg3BxcXR3FwFCdxzVrQXUFQ6A8QBzcliy3x0qhpFQ+OezjehI8DuKbMmBEDUDEdK59b4otb23z8BRzGMYR5JGTFZhlJWqMIvmd3eY+UpnqYCf61XN3LDZ88C8du7UKfvjn/m++q/rJqKzSb1JMKsxyA48nSgcMCM/KIRY8iHwXWQp8FLoW+amcK+Sy5Tk4oulwm35OFUsde63DFqCpfbwheP6iaG+s0+6jHUISIBaT+yaTbmSz2wVCDhV1o0OGAR4Y2DAZU8ipMKJiFQBaRok4Ht/19s2h3mAQWVwiv0a5wB1y/A8lGtJvwgNFboK4z2ZJ/Lsl5MXtnTgw94AK9p77/tgnEp4DWk+9/uL7fugR6GvSt9wnyi2NF8E/5XAAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAABDaADAAQAAAABAAAAeAAAAABUYuPKAAAfQklEQVR4Ae2dCbiV0/fHdzIPP5UxGUopCSUJCZfQJEKESEQpFYlknuVRUZEQjYrQ8OCJCJXwpDQZUmkgFCrzGPr32f/fOr99396xc+7tDGs9z73nPe+7x+9599prr73WXmX23XffDUZJEVAEFIGYCGwVM50mUwQUAUXAIqBMQ18ERUARSISAMo1EcGliRUARUKah74AioAgkQkCZRiK4NLEioAgo09B3QBFQBBIhoEwjEVyaWBFQBLYuBAh22mmnQuhm2n389ddf0y5DC8h/BCKZxvbbb28qVqxokVi1apX5448/QlHZbrvtzD777BM7fWhh+lARUASyDoHI5UnVqlVN165dzYUXXmh23333yA6UL1/epiVP9erVI9Onk2CrrbYytWrVMvvtt19oMWXKlDF9+/Y1e+65Z2g6fagIKALRCEQyDYr4559/zL333mu+/PLLyBJXr15t05KnJOmYY46x9bRv3960aNEitCqYRqVKlczOO+8cmi7pw6uvvtoMGDAgaTZNrwjkNAKRy5Ns7V2NGjXM1KlTTc2aNWM3EUmjfv36lvnNmDEjla9s2bLm5JNPtkzl5ZdfNuvXr08923HHHc2JJ55oKlSoYObMmWMWLlxoWII1bNjQLtvQlzRq1MjmmT59eiqf3wWSGtLX999/b4444ggza9Yss2TJEpuUNjRo0MAceOCB5ttvvzWU5eoYjj32WLN06VL7vFq1ambatGlm5cqVZqPvkDn66KNtm+bPn28+/vhjW16dOnXM1ltvbdNT1vHHH29eeeUV88svv9jnfv3ya7PeUwS8COQs0xgxYoTtS5IlULdu3exAROKoXbu2GTRokC1j8ODBhkHEID3nnHPs/ffee8/AEIYMGWJ+/PFH8/vvv5tTTz3VtGvXzt4//fTTDUsxBibXSFZRTKNevXrmkksuMRs2bLC6ISSkfv36mdmzZ9v7RUVFZs2aNZZBtW7d2nTp0iU1yDt16mT+/vtvg47p559/Nocffrjp2bOn6d27t/1O/bTjtddeM8OHDzcdO3Y0u+yyi+1fy5YtbX1NmzY1l112WWC/vC+HflcE/BDIWabh15moe++//77p37+/adu2rWGAQkgeMIwrrrjCMobu3bubVq1aGZgGEgTUuXNnO9C32WYb+33dunWGdNdcc42dyblOQrSBtsAwGjdubJnG2LFjzdChQ20x1ANTPOGEE8ykSZNSRf/777+G5RgMDOYBwUxEeoApkgemAU2ZMsX2DQZz6623moEDB9r7Qf2yD/WfIhCBQEExDZYD0MyZMw2zLkuO0047zbA0eOqpp1JQMaNDLGHOO+88M2rUKLukGTNmjFmwYEEq3eZeIFlAixYtMuhmIHacrrzySrPXXnsZFLzoYf7zn//YZ/Jv7ty5lmHwXXaxLrroIoMEA+ODpO1cCzP566+/rDTCPRhSSfWL8pXyH4FYitB8gUHEdbHb+O2338y2225rBxe7Q/KHJAIhUTCzjx492uywww6mV69eqcEpmDC4kxI6EYgyGdAQUgLMgiUJjIAlDN9d8iqi69ata/Utw4YNM23atLHMwG0PZbh/Ulacfkla/VQEvAgUfyu9T7P4OwOKQYeUwJ9chzUZZSfpmjVrZvUUzNYsE9BxoGiEypUrZ5WGXKNw3GOPPawC8bHHHrOzv9ig8HzFihU2/a677srX2IT0Qp5DDz3UKlbJiG4EBSj6k4svvtjWFVUgzAemgPITWxokjjgU1a84ZWiawkUgZ5cnxx13nDnrrLNSvxxbws8884zdkUjd9FzAANAbMNBGjhxpn7JbwpqfmR7bEmZqBuHbb79tdzjOPvtsq+SESX300Ufms88+S5WKzqB58+YGhoIiEgkhiqgbhWqTJk3sUmLChAk2C59IOEg1SB8sM0jrkvc7eheko0cffdS2m10Z0btIPm8e7rNzE9YvyaufioAfAmWijvtja7NDhw4GJRwv5/Lly/3KSd074IADrIjNIGOAyhZgKsEWuJDlCFWzRfndd9+ZP//8s1hLmLV5hp2Ju9WJZIJ0gVQRZXsCw9l7772LlStfqA+lK8sIdlAwSPviiy+KMQYwoy7vMkTKCPpkKxn9BcutuOTXL7ffccvRdIWHQCTTyAdIXKZRkv1hacP2rR/99NNPBmkCpsHyIxtJmUY2/irZ16acXZ4kgfLgKhvMp8vLmNo1Npgrzv1n4+xuzKAxZQPvPXILS4PwNH5lHbDX92btx+cG5v157b5m6usbzOaW71en995225pYffTmAx8lRSAOAgUhaVx9yfam7EaV7+8bfe1GvVTWMLAuavFP4L0nno9OE7csbzo2S+K0wZsvbvvTKf/BYeHOiHFeKE2T/wgUBNPI/59Re6gIlB4CObvlWnoQaU2KgCLgIqBMw0VDrxUBRSASAWUakRBpAkVAEXARUKbhoqHXioAiEImAMo1IiDSBIqAIuAgo03DR0GtFQBGIRECZRiREmkARUARcBLbGiUtJEVAEFIG4CKikERcpTacIKAIWAWUa+iIoAopAIgSUaSSCSxMrAoqAMg19BxQBRSARAso0EsGliRUBRUCZhr4DioAikAiBrD6Eh9genGfJeZkSfiBR7zSxIqAIZByBshtPsb4j46VmqECCNh911FH20FyJFULRRBLjRO+oiGbSDA4gJrIYB/py/iZRzKKIcI/EIYlbh7c8DjrmgGDOAc0EcXI5wbiTnB9KuEbCMRJciUOHlRSBTCCQtZIGB98SLIhoZN4ARcQ75QDjuPTBBx9YRkFIReKpxiHSVa5cOU5S3zSEFIiKZu+b0XMTZnH55ZdbLJC4OIE8DhE/BSmNA40JCEVsFE5YV1IE0kUga3UaBDGCiAUSh4h9QuwSjvRHonAPE2a2Z8B4j/Pn9G+CO++///42FCMzs5c4oZzj/l3L2bC6vPm93wmXwOwvYRW9z73fDznkEBvomfgscYm+wzDGjRtnT4aH2RCjVkkRyAQCWStpSFAib6gBOs2p2d77F1xwgR2Ma9euteEWGSQ9evQoFo7ACxgBiiQgM2USNIlByqwMwVRuv/12K96zJOrTp48NbhSnLsId+IUUYJmEBAUj+/zzz71N2uT7c889Z+9RZ1xCOiGcwptvvmljsdAPb4jHuGVpOkXAi0DWMQ1mYJYRvPis3/3W4m7cVekQ4QGefvpp+xVm8PjjjxsCKhFFPYpQshLwiGBHxFYVpkG+AQMG2CBJDz/8sJVkFi5caEMRRNVFQGk/IpQBcVfXr1/v9zgj93bbbTcrVRFSgYDQL730kjnzzDNtFHkizispAukgkHVMgwhhxFwleBEKvLhEkCJmcQIoQ8y0cWfXOXPm2DzoPgjdiMQBsZwhqhqE8hQ9BZROXbfddpstoyT/sRyBiGpPn0R5WpKMqiT7o2VnFwJZp9NgJmQZwMx/0EEH2dircSDr1KmTXU6wJGGWJyIcjMMlmABSiJdE/yGR1yUos6sDccuLU5e3DvlONDSUuOhFMkHocdgdchXD6IHoO/Fin3jiCVOpUiXLACXSPPXC+IKiwWWiXVpG/iKQdUxDoJaYqXGDK8MMCLcI02H9zzreSyx1UHZ6mUlRUZFVTKJAJQCzMA1vfvkepy5J6/286aabrJ4EBWtcQvISxal7TX50Ny1atEgFsObe/PnzLdNcunSpZRYoe7/55hsepeiee+4xbAsrKQJJEdh0ZCUtoYTSixIRaSMOvfjii1aJOWTIELuOR0R3JQXKGD9+vN0GRSfihkZk9idOLXWx/vcjykLagOLU5VeGe8/bNveZe03b0Kuw4wKz4pplh5fc8rgeO3aswdYEPFhuoeMRgmnyJ/2R+/qpCMRBoMzGrbmNAQizk+688047yFmTx9EFMBDQO3z99dexOsS2LsugO+64w1qdoreIkjKk4KR1Sb7S/EQ/hI5HdBpSN9uxXbt2tdvQrtJXnuunIhCGwKYL/LDUpfyM7U4Gp+gaoqpnho3LMLxlJc2XTl3eukvqO4pPL8OgLozjiDI/atSokqpay81jBLJa0ihp3GFIZ5xxhpk8ebJxlYQlXa+WrwjkMgIFzTRy+YfTtisCWwqBrFWEhgGCdeYtt9xSLAnGYA888ID9w5ApXUKJePPNN8cq5sYbb7TGaHESV69e3fTu3Tu2D0xUmUhLGLHR3riELgd/FLZqK1SoEDebplMELAJZrdMI+o38HNYwyWbngy1InqdLKBArx3RYoz7XNyWsbqw1KRvbkDjetmFlwTzZbkXhuXLlSruVG5aeZ2xFw1xxCIThNGvWzNx6661m9erVUVn1uSJgEchJScPvt8M+A6c0fEi8xJYjMyvObBhDCWEQVbduXeushq8Lg9A1kiIdksG5555rt2MlH59sz3K/Ro0a7m27velXV7FEAV8aNmxo6tevH/B009v4ycyYMSPRgG/QoIG1lEVSw04DIzOOGVBSBOIikJOShp/DWliHMajCKIodA8zEmzdvbpc3zNIwDZSgTZs2tQ5meLRed911tjgGVM+ePVPP2Z7F25QzPrAKxZakcePGxQzJguqS9rGli30EDm1ewgye+3E9WpEYIAy14lKdOnUsY8UQjmUSbalSpUrc7JpOETA5yTT8HNbCfktmVJFAjjzySHPVVVelXOdXrVplDaEwmOrWrZsZPHiwlS6kPMywGcQMTBgO13ySD52HMBBJH1SX1I8vCOdj+BEMJa6diF/+OPfwx4FJYv1avnx5g98N+iAlRSAuAjnJNOJ2TtLhuYp04TqwiXk6AwgJBLsL/ph5sQsRa0kGObR48eLU0gZdhxyGI85uUXUJ05B0fp+cFFbShKUszoAsrWC+SFpKikASBPJGp+F2GgWfEDsF+KLg0t65c2czcOBA+0h8U4RR8AnBLNz84vPBQBMvUc7ykPtyWBB5o+oiTRhxGJCcIxKWLu4zdDTskIhDHvnwv0HHw3keM2fONChmXYaGUpU2IIUoKQJ+COQd08BJC0lABgrMASaAAxw6CgZREmrZsqXVhxx22GFW2iAvZSHSoydBByKUbl2Ys2M6H5foD22gXrlm0AvRV/Q27hGHU6dOtY/fffddyxjAat68eZLFYsdSLO52cyqjXhQMAnnHNCZNmmT1Ahyag36B5QdLDA7YeeSRR+wg49eV5QfXImVwDcl3PvEQxUkMyUKc2XBY4zv3eS7p49T1/zUE/5eyglP87wnWrLQBF3ekA64bNWr0vwT/vXLLXLJkifWCBY9+/fqZH374wR4LKJlkCScOg3JfPxUBQaBgLEKRPFhirFu3Tvoe6xMphQOC8eFwGQ2zO+7t2Ee4g5JCN7euWA3KUCLaiO6GXRSXOnToYE8ve/DBB1MHELnP9VoRKAhFKD8z63Z37R73p4ch+IUhgIH43U+nrrhtykS6IDxYyixbtkwZRiZAztMyCkbSyNPfT7ulCJQ6Anmn0yh1BLVCRaDAEFCmUWA/uHZXEUgXgawOyxjUOewPvGEZ2QLF/BtLRxR8ixYtCspeKvdpI9aiH374YanUVxqV4IcDxpxBmondFfx38NPh9/rqq68iu4DymTNe2SFilwcltFLpI5CTkkaYlysGVpnwck33p0ChiOFUttLQoUPtmapJ2ud66CbJ55eWrWqOFMBaF0tYDO+iiHNdOWmeKHVcY+IvRnZRefV55hDIm90T8XJt0qSJLzp4rzLzM0tiqyCEfQP3YTbEOMFyVAh3d6w0cbvnYF8CHb311lv2Htu3/GEcRUAmbDQgDv+lPCQd7wngDJRPPvnEDhSM0LgWgskQ2Ig8hCDA6jRod0by8IlX7PLly207YJbvvPOO3R4O6hdeuRK6AD8c+khb5FhAzOvpK/YbeNDGJTx08ZuJ62zHEQb8DjjN3XXXXYa2sA0ctsMF9i+88IJNgxEbfkKtWrVKBcmK21ZNlx4COck0eLG8YRnDYGDJgik5gx6vVJzN5KBirDDxPWELFYZDKMPRo0fb4hgI+KxADAgYAgOUQ35q1apl87Ali9Vlx44d7XdefKxEGXycUSH1UAYhIPFiJQgU1ppvvPGGrYs8DB6x98C6E4aC8VUUtW/f3pYJA4Nx0i7qDOoXTIY0EAOV6+nTp1umgd0J+TCXpy1t2rSxXr6UC4EBOGXCQxdTdszYWVbC4LCHoS1hTMdlorSRdnAuiFLpIpCTTCOplyvMAAczjvPnJcMylPU0M1337t1TsxtiMudNCNPgp2AdTTzVV1991R524xp44R7PjEf4A862QFIhbsr1119v1/5iXen+pLSDpQH1ImZTFzM7hPctg4fykhCDh7xIOyKuB/VLDhOmDU8++WQxewz0MPim0H6of//+lkFKCMpMeeiKsyCGdjBbsOVoAtfcPar/WLTCXCdOnBiVVJ9nGIGcZBpJMOAFxfmKtbMbFZ4ZF6bRunVrG2EdZRwkIQ2lDmbc119/3X4VhzW+MLgYrPxxv/LGZYqEcJS8fp8yk65YsSLlgs+Mj2WmlI/kk4QWLFiQWh7JMimqX37lw8RgOjATCAaG8jMOJfHQFfd/pC3wZsnBCWLS/6j6kABPOukky3C9Fq1RefV5+gjkJdPgZfcSM6w4a8kzDqThBZQzMxD10Ue4xAsOY/CSvPje+1HfJT6tK7EgnbjHBbqes1Hl8dy78xCnX37lghuMDxPypITuBwYQJxQEGMCMMc/HQU/OKXV1QEg9SHHoi1w9BzFbLr30UoOPEUtJl0gPjpQPU1cqGQRycvckDAqvlysDkpeIrT0Ri5E4eFEZnLy8KD+JZMZg2xKE8hJvVaQhDgmWJcbmtiVOv9AJ1atXr1gVKHyRLOTIQ6Qnll1xCF1IEg9dGAQSFXoKpCIYsyup+XnoosTt0qWLmT17tpkyZYr9PV2dBu1VD904v1Z6afJO0mAG4kXHy1VO10LLzsv20EMPWUUekKGxZ6nAYTTMrMyy7BgwW7kEU/GSKyXwjDRyD90CsyG6EAhRH0aGojOI2NFh8OAsxuBhieFdJgXl9bsfp1/Tpk2zNi3oU2Ba6DjAiUOGCVJFf8BkwoQJxXZ5/OqTe35YyTPvJ9IdW66yFBozZkwKQzetWyZKbNqENCgSoXugsuiQMmFD4rZBr4sjUFC+J2w1MpjZ1ZBBDhyItMx6snQoDlHpfWNAMEgYUCNGjLDbw+wy+BFKxKgl0ub2C6kHmwy2YdNhXn7t9t5jxwbm7rcE9KaN+q4eulEIZeZ53kkaYbAEHdOfDco0AjTTPgY6A5VDcm644YZARSRMBWkhjDa3X2yxyjZrWPmZeCb2IZkoSz10M4FidBkFJWlEw7HlUrCbU61aNWsUJueSbrnWaM2KQDACyjSCsdEnioAi4INA3u2e+PRRbykCikAGEchJpsEevjeWawYxSVxUkliuSQvHRL1du3ah2TgLFXuTTBDb0mx3Yjzl3UnKRPlaRu4jkJNMw8/LdUv+FLQHBWa65Od5ip4Dk/cwqlixojWUCksT5xl2KmxLw4BwBBs0aJA11Y6TV9MUDgI5yTT8fh7sDTCOwogL4yR8GcRWIiiWK9t9bmxXHMfwRHUNhvzq4l5QLFeeUT+Dzh3stIWysZzkmWvSjtESzyBMyrmmbS5h4s3p40kMv8gDLnHzEK4BY7gePXrYP3w7cPZTUgRcBHJyy9XPyxXTYnHYYssSewdilRDvNCi+KlaRmI5jzoxTGrE+GGAEkg4jDIuCYrkGedTiIYuXK+2i/TAyjNCGDRtm3duDPE9pB1HmMU4jr+tRK23ExsHPoInYsBg8YTiGtWcUkQ5mxhJFPE+5p6QIuAjkJNMI8nLt27evHcy86PwVFRXZvgbFV8WOgUHL4GJmx5S8V69exQy/XLDkOiyWa5BHrQzaWbNmGQJJ46WJZET9YZ6nUqefR60842AaP+IoABzx4jqCjRw50lqzskSBMOt2z/zwq0PvFR4COck0gn4mDJIYIMzkzLzM7hCD0y+WK+kwocYdnnMdcAFfs2ZNUPGp+5U3+jj4xXIN86gVpiGxX7HFgPkgcUR5tW6uR617lkeq8SEXMB/O5Rg7dqx1U+dYPaQqGJ2SIiAI5BXTQHznD+ITs2yctziAhwNfsKI8+OCDDRHiRd+B/qJq1apWukAy8XpOClDuJ8sa0RP4eaT6edRKOvQmkLjiR5mCkzZOGtJ5CcmJ/mF1GcdMmyXSp59+aiZPnmyLgtmCiTANnPzoN85mccrztke/5wcCW+VHN4J7AXOAeQTFcmW7FF0IszLreA4sjqKgWK5hHrVSJoOQgYfug/QuQ/DzPJV8m/OJLgfnM69SNagszNiRomBsYAFjc828OVQYL1J0RUqFi0DeMQ2vtAFDCIrlCoNgcNx///32HIhnn33WHgcYdfBMUCxXXiM8RZmR0QvgwYnI7zqdMftzMhc7KxIbVl4/fEnY7UBng54Fch3r+E7/vPe4H0YifYWl4RnLEmjgwIHm7rvvthLF888/b+/xj6ULpGdVWBgK9l/BmJEze/LSx4nlylJCDobxvhnoTdCFIMEwg/vFciWP16OWMlGAcu4EuzvoTlwpw1vPlvxO39EJwXCFkNZggrQbRzqlwkUgr3QaYT8jA52/OHTKKadYWwq/tJx9QXR2Zvuw7cggj1rKjHO6lV/dpXXPj7FiwAazGz58eGk1Q+vJUgQKRtLY0vgzU2OchZLRncG3dLu0fkUgKQLKNJIipukVgQJHIO8UoQX+e+Z199l1YukYpajOaxCyoHN5E8s1C7As1oRcjuWKyTr2LOhv0iEGOQZiOMCh9PUazrGDFBTLFXN2dpLck8U5hhB3Abaruc6nOLnp4FzaeXNS0sgGL1dO2pJDcf1+tEzGco2qy6/+dO7hWZuJ2ZydIpzzsOvAxoOtbSGc8oJiucK0OO+TMJX4A8E8ILZ6CeSEAhonP6Utg0De7J5glMSWKn9cu/FVo6Dl9HCsQrEGdXcOUF5iRs0gIhwjVqXsmjCIsZ7kuXincqw+uzNBsVzxfOWwXhge53/WrVvXzpRiXo7zGmUyI1OP7L6E1UW/yMegZEfGdbSjHYRDoF2YyWMrIjFEgvoVhRNes5jKYyEaR5mLX83ixYutbQlm+tdee621TyFIVVAsV9rA7zFu3DgbeY1tatK6fWPbOxNHEUT1V5/7I5CTTIPBifWkSxwcw6BjUGPM5OcN6qaXa8IXMuiwzsTZjJcTc3OIrVUOooGRwBwwn162bJldV2OHweDDBR/CcpJQBdiD+MVypT0wCgYbebCDIB0zMBHgGFBr1661Zt/sshCCYd68eaF1SWwQfFdgCszMOOdBlIlXLSEasBHBJwcLT8JABvXLZtz4jyWB3/Z0Uq9ZgmALydJEzOdpr18sVzDlDwaOUx/2MBKaQMqCaVSpUkW+6mcpI5CTTCPIyxXswrxBvdgyGzMDMohYvzOrY8GJ7wjLC15sPGfx9OTlFcvK++67z6YlDilesS7BfIJiuXJUP1aX+L5069bNWo+yDGDmRiSX2ZvAz6z1YRphdaErmDt3rmUwSBUMagactJN2YXzWp08fy0zRMWCRGtQv6QfLBj9K6jXrltG1a1frmIceAmkFghn7xXKl/eXKlbNMEKtZAm6jw4BZQFOnTrX4s/zBcVB8ZexD/VfiCOQk0whCJak3KEfaMch4oYX4znIEsZryOJCGl5sI7wQqTodgCkgFDAr+kIqYeTnsBomDcz1gZLQhyvOVdAwslh8Qs7aElZR4sdxnQFEPRP38bW6/knrN2ko3/oORcyYIg5y2iCUskhIGY24sV/oOwVgx/xffF9e9H0mI7/QfBqhUugjkFdOQlzEuhOJLAdPwC5QEw0CnUVRUZB3ZGMgzZsyIW/wm6YRR8AkxgBgkLCM40YuobwwSJAMZPJsU8t8b5KUcGTRy2hhR4lxCF+Olze1XUq9Z6m3Tpo09jYwodsIAwJq2+8VyFQdDpBGCRnGkAWlFCqNMgkVDMBal0kcgJ3dPMgWTzNK4zqO74A87AAhxuHbt2nbpIGbjEuOU53i6MrBr1qzJ17QInQM6GgI5o2xEYeqSX10wDXQsMDSWHexS4K6OhBRGUf0Ky5vUa5ZlBVuunFNC32AESEhQUCxXlon0Df0QzEJ0SW67YJQsA5W2DAJ5I2nwornEC+e95z7nmh2KiRMn2jUzOgEIBSDBhXHaQu9AOTAHXlI0+kIsWVCKor8gDZ6hvPBBsVxlPU5al/hOvFREf9nCpQ1uuqC6xo8fb0V/vGYhllBe8mIQ1S9vfr/vbtv8nss9ln9Q27Zt7R/XxNplORIWyxW9z/nnn2+GDBlily9g6xIST9TyzU2v15lFIO/NyBkkzOR+JNuaiMSIygxsBqgQugY8WdH8++0mSLpMfeKmj7I07qCkXhgaSxtmbleED2tTafcrrC3gS5+9h/og9aEHkSUNZYAPSziIIwg0Ep2FotT/5T3T4Bh+We970UVxmGSAevPr99JHgKWJShmlj7tbY94zDbezeq0IKALpI1DQitD04dMSFIHCQ0CZRuH95tpjRSAtBJRppAWfZlYECg8BZRqF95trjxWBtBBQppEWfJpZESg8BJRpFN5vrj1WBNJCQJlGWvBpZkWg8BBQplF4v7n2WBFICwFlGmnBp5kVgcJDQJlG4f3m2mNFIC0ElGmkBZ9mVgQKDwFlGoX3m2uPFYG0EFCmkRZ8mlkRKDwElGkU3m+uPVYE0kLg/wAJLEJWRotATAAAAABJRU5ErkJggg==)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        },
        "id": "V5QzUxbmT8Rx",
        "outputId": "9a906db3-9d5d-4be6-fb21-533a3c4793c5"
      },
      "outputs": [],
      "source": [
        "best_params = {'alpha': 1.0,\n",
        "               'lamda': 1.0,\n",
        "               'learning_rate': 0.2,\n",
        "               'max_depth': 8,\n",
        "               'n_estimators': 200}\n",
        "\n",
        "# Train the final XGBoost model with optimal hyperparameters\n",
        "final_xgb_model = XGBClassifier(\n",
        "    objective='binary:logistic',\n",
        "    eval_metric='logloss',\n",
        "    **best_params\n",
        ")\n",
        "\n",
        "final_xgb_model.fit(df_balanced_features, labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M3qNyHcdqL-p",
        "outputId": "405d419c-81f1-463b-e9e9-21a1cc3cea36"
      },
      "outputs": [],
      "source": [
        "# Evaluate this model on the training dataset via 10-fold cross-validation\n",
        "cv = StratifiedKFold(n_splits = 10, shuffle = True, random_state = 42)\n",
        "scores = cross_validate(final_xgb_model, df_balanced_features, labels, scoring = ('precision', 'recall', 'roc_auc'), cv = cv, n_jobs = -1)\n",
        "\n",
        "print('Precision:', round(np.mean(scores['test_precision']), 3))\n",
        "print('Recall:', round(np.mean(scores['test_recall']), 3))\n",
        "print('ROC score:', round(np.mean(scores['test_roc_auc']), 3))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SUVMKTK3Or-K"
      },
      "outputs": [],
      "source": [
        "# Saving the model as a file for testing hidden_data later\n",
        "with open('final_xgb_model', 'wb') as files:\n",
        "  pickle.dump(final_xgb_model, files)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_U8-hvOGz_Xb"
      },
      "source": [
        "## 4.2 Feature Importance and Output Analysis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 872
        },
        "id": "cNSJwyUOpCEs",
        "outputId": "f404b24d-715e-4cd0-a02d-45f81a4f905a"
      },
      "outputs": [],
      "source": [
        "# Feature importance graph from XGBoost\n",
        "from xgboost import plot_importance\n",
        "\n",
        "plt.rcParams[\"figure.figsize\"] = (15, 10)\n",
        "plot_importance(final_xgb_model)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cm21V0sakPQI"
      },
      "source": [
        "From the above plot, we can see that there are 9 relatively important features in the model based on their F score. The top 3 most important features are `cust_duration_mths`, `n_months_last_bought_products`, and `current_age`.\n",
        "\n",
        "However, while this plot tells us the absolute importance of these features, they do not tell us anything about the direction of importance. For example, are clients who have been with Singlife for a longer duration more or less likely to purchase a policy within the next 3 months?\n",
        "\n",
        "To identify the direction of importance, we plotted the SHAP values for the model, which gives us directionality and thus much more interpretability.\n",
        "\n",
        "That said, this plot is nonetheless still valuable as it tells us which variables are NOT important in influencing clients' propensity to purchase (such as client type), and should therefore not be the main focus for Singlife's client retention strategy."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 957
        },
        "id": "Q7xfVFJxpaoz",
        "outputId": "2bec2bd9-664d-4d29-c26d-79a82d719492"
      },
      "outputs": [],
      "source": [
        "# Using SHAP to see the directionality for the different features\n",
        "explainer = shap.TreeExplainer(final_xgb_model)\n",
        "shap_values = explainer.shap_values(df_balanced_features)\n",
        "shap.summary_plot(shap_values, df_balanced_features, max_display = 20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pOjjIC-wmXX1"
      },
      "source": [
        "This plot allows us to distill our findings into three key high-level insights, corresponding with the top 9 most important features identified previously:\n",
        "\n",
        "1. The higher the annual income of the client, the less likely they are to buy a product in the near future.\n",
        "2. Clients in smaller households are less likely to buy products in the near future.\n",
        "3. Clients who consent to SMS and/or mail are less likely to buy products in the near future. This is counter-intuitive as clients who consent to SMS and/or mail are normally perceived to be more interested in the company's products.\n",
        "\n",
        "It is however interesting to note that the top 3 most important features for the SHAP plot do not exactly correspond with the top 3 for the F score chart."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iUEoUwiwrD7a"
      },
      "source": [
        "## 5.0 Conclusion and Recommendations\n",
        "\n",
        "From our findings, we suggest two broad approaches for Singlife in their overall client retention strategy:\n",
        "\n",
        "1. Effective client segmentation based on important characteristics should be the key priority for targeted engagement efforts. Here, we show that high-income clients, as well as clients in smaller households, are especially at risk of churning, and thus should be a focal demographic for engagement (for example, organising more frequent touchpoints & call-ups).\n",
        "2. As clients who consent to SMS and/or email are less likely to buy products in the near future, this could imply that Singlife's communication efforts may be counterproductive. One potential reason could be the lack of personalisation in SMSes or emails, which may cause clients to perceive the messages as spam. Thus, Singlife should look into ways in which they can revamp their communications strategy and increase engagement with clients via such channels.\n",
        "\n",
        "Ultimately, while our model attempts to predict clients' future purchase behaviour based on their characteristics and past behaviour, it does not take into account factors such as client sentiment and competitor analysis, which may be more important in elucidating overall client satisfaction. More work has to be done and more data should be collected to build upon this expansive dataset, which will lead to an even better-performing predictive model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3psDj95Ldea7"
      },
      "source": [
        "## Testing Hidden Data \n",
        "\n",
        "The `testing_hidden_data` function below will be used by the datathon organisers to evaluate our model on a hidden test dataset. The function carries out the necessary data pre-processing steps that were determined in the analysis section above, before importing our trained model and predicting labels on this test set. Model performance was evaluated based on precision and recall."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l2h3WFs7dea8"
      },
      "outputs": [],
      "source": [
        "def testing_hidden_data(hidden_data: pd.DataFrame) -> list:\n",
        "\n",
        "  # first, replace all NA values with NaN\n",
        "  hidden_data = hidden_data.fillna(value = np.nan)\n",
        "\n",
        "  # then, aggregate the ape, sumins, and prempaid columns\n",
        "  metx = ['ape', 'sumins', 'prempaid']\n",
        "\n",
        "  for i in metx:\n",
        "    new_colname = i + '_total'\n",
        "    rel_cols = [col for col, is_rel in zip(hidden_data.columns, [col.split('_')[0] == i for col in hidden_data.columns]) if is_rel]\n",
        "    hidden_data[new_colname] = hidden_data[rel_cols].sum(axis = 1)\n",
        "    hidden_data = hidden_data.drop(columns = rel_cols, axis = 1)\n",
        "\n",
        "  # then, create cust_duration_mths and current_age columns\n",
        "  hidden_data['cltdob_fix'] = pd.to_datetime(hidden_data['cltdob_fix'], errors = 'coerce')\n",
        "  hidden_data['current_age'] = 2024 - hidden_data['cltdob_fix'].dt.year\n",
        "\n",
        "  hidden_data = hidden_data[hidden_data[\"min_occ_date\"] != \"None\"]\n",
        "  hidden_data['min_occ_date'] = pd.to_datetime(hidden_data['min_occ_date'], format = '%Y-%m-%d', errors = 'coerce')\n",
        "  today = datetime.today()\n",
        "  hidden_data['cust_duration_mths'] = (today - hidden_data['min_occ_date']) // pd.Timedelta('30D')\n",
        "\n",
        "  # then, drop dob and min_occ_date columns\n",
        "  hidden_data = hidden_data.drop(columns = ['cltdob_fix', 'min_occ_date'], axis = 1)\n",
        "\n",
        "  # then, create ctry_is_sg, clttype_is_p, and stat_flag_is_active columns\n",
        "  hidden_data['ctry_is_sg'] = np.where(hidden_data['ctrycode_desc'] == 'Singapore', 1, 0)\n",
        "  hidden_data['clttype_is_p'] = np.where(hidden_data['clttype'] == 'P', 1, 0)\n",
        "  hidden_data['stat_flag_is_active'] = np.where(hidden_data['stat_flag'] == 'ACTIVE', 1, 0)\n",
        "\n",
        "  # then, drop other columns\n",
        "  cols_to_drop = ['clntnum', 'race_desc', 'ctrycode_desc', 'clttype', 'stat_flag', 'cltsex_fix', 'flg_is_borderline_standard', 'flg_is_revised_term',\n",
        "                  'flg_has_health_claim', 'flg_has_life_claim', 'flg_with_preauthorisation', 'is_consent_to_email', 'is_dependent_in_at_least_1_policy',\n",
        "                  'f_ever_declined_la', 'hh_20', 'pop_20', 'hh_size', 'flg_latest_being_cancel', 'recency_lapse', 'recency_cancel', 'tot_cancel_pols',\n",
        "                  'f_hold_839f8a', 'f_hold_e22a6a', 'f_hold_d0adeb', 'f_hold_c4bda5', 'f_hold_ltc', 'f_hold_507c37', 'f_hold_gi', 'lapse_ape_ltc_1280bf',\n",
        "                  'lapse_ape_grp_6fc3e6', 'lapse_ape_grp_de05ae', 'lapse_ape_inv_dcd836', 'lapse_ape_grp_945b5a', 'lapse_ape_grp_6a5788', 'lapse_ape_ltc_43b9d5',\n",
        "                  'lapse_ape_grp_9cdedf', 'lapse_ape_lh_d0adeb', 'lapse_ape_grp_1581d7', 'lapse_ape_grp_22decf', 'lapse_ape_lh_507c37', 'lapse_ape_lh_839f8a',\n",
        "                  'lapse_ape_inv_e9f316', 'lapse_ape_grp_caa6ff', 'lapse_ape_grp_fd3bfb', 'lapse_ape_lh_e22a6a', 'lapse_ape_grp_70e1dd', 'lapse_ape_grp_e04c3a',\n",
        "                  'lapse_ape_grp_fe5fb8', 'lapse_ape_grp_94baec', 'lapse_ape_grp_e91421', 'lapse_ape_lh_f852af', 'lapse_ape_lh_947b15', 'lapse_ape_32c74c',\n",
        "                  'n_months_since_lapse_ltc_1280bf', 'n_months_since_lapse_grp_6fc3e6', 'n_months_since_lapse_grp_de05ae', 'n_months_since_lapse_inv_dcd836',\n",
        "                  'n_months_since_lapse_grp_945b5a', 'n_months_since_lapse_grp_6a5788', 'n_months_since_lapse_ltc_43b9d5', 'n_months_since_lapse_grp_9cdedf',\n",
        "                  'n_months_since_lapse_lh_d0adeb', 'n_months_since_lapse_grp_1581d7', 'n_months_since_lapse_grp_22decf', 'n_months_since_lapse_lh_507c37',\n",
        "                  'n_months_since_lapse_lh_839f8a', 'n_months_since_lapse_inv_e9f316', 'n_months_since_lapse_grp_caa6ff', 'n_months_since_lapse_grp_fd3bfb',\n",
        "                  'n_months_since_lapse_lh_e22a6a', 'n_months_since_lapse_grp_70e1dd', 'n_months_since_lapse_grp_e04c3a', 'n_months_since_lapse_grp_fe5fb8',\n",
        "                  'n_months_since_lapse_grp_94baec', 'n_months_since_lapse_grp_e91421', 'n_months_since_lapse_lh_f852af', 'n_months_since_lapse_lh_947b15',\n",
        "                  'n_months_since_lapse_32c74c', 'f_ever_bought_e22a6a', 'f_ever_bought_d0adeb', 'f_ever_bought_c4bda5', 'n_months_last_bought_839f8a',\n",
        "                  'n_months_last_bought_e22a6a', 'n_months_last_bought_d0adeb', 'n_months_last_bought_c4bda5', 'n_months_last_bought_ltc', 'n_months_last_bought_507c37',\n",
        "                  'n_months_last_bought_gi', 'f_ever_bought_ltc_1280bf', 'f_ever_bought_grp_de05ae', 'f_ever_bought_inv_dcd836', 'f_ever_bought_grp_6a5788',\n",
        "                  'f_ever_bought_ltc_43b9d5', 'f_ever_bought_lh_d0adeb', 'f_ever_bought_grp_22decf', 'f_ever_bought_inv_e9f316', 'f_ever_bought_lh_e22a6a',\n",
        "                  'f_ever_bought_grp_e04c3a', 'f_ever_bought_grp_fe5fb8', 'f_ever_bought_grp_94baec', 'f_ever_bought_32c74c', 'n_months_last_bought_ltc_1280bf',\n",
        "                  'n_months_last_bought_grp_6fc3e6', 'n_months_last_bought_grp_de05ae', 'n_months_last_bought_inv_dcd836', 'n_months_last_bought_grp_945b5a',\n",
        "                  'n_months_last_bought_grp_6a5788', 'n_months_last_bought_ltc_43b9d5', 'n_months_last_bought_grp_9cdedf', 'n_months_last_bought_lh_d0adeb',\n",
        "                  'n_months_last_bought_grp_1581d7', 'n_months_last_bought_grp_22decf', 'n_months_last_bought_lh_507c37', 'n_months_last_bought_lh_839f8a',\n",
        "                  'n_months_last_bought_inv_e9f316', 'n_months_last_bought_grp_caa6ff', 'n_months_last_bought_grp_fd3bfb', 'n_months_last_bought_lh_e22a6a',\n",
        "                  'n_months_last_bought_grp_70e1dd', 'n_months_last_bought_grp_e04c3a', 'n_months_last_bought_grp_fe5fb8', 'n_months_last_bought_grp_94baec',\n",
        "                  'n_months_last_bought_grp_e91421', 'n_months_last_bought_lh_f852af', 'n_months_last_bought_lh_947b15', 'n_months_last_bought_32c74c',\n",
        "                  'flg_affconnect_show_interest_ever', 'flg_affconnect_ready_to_buy_ever', 'flg_affconnect_lapse_ever', 'affcon_visit_days', 'n_months_since_visit_affcon',\n",
        "                  'clmcon_visit_days', 'recency_clmcon', 'recency_clmcon_regis', 'hlthclaim_amt', 'recency_hlthclaim', 'hlthclaim_cnt_success', 'recency_hlthclaim_success',\n",
        "                  'hlthclaim_cnt_unsuccess', 'recency_hlthclaim_unsuccess', 'flg_hlthclaim_839f8a_ever', 'recency_hlthclaim_839f8a', 'flg_hlthclaim_14cb37_ever',\n",
        "                  'recency_hlthclaim_14cb37', 'giclaim_amt', 'recency_giclaim', 'giclaim_cnt_success', 'recency_giclaim_success', 'giclaim_cnt_unsuccess', 'recency_giclaim_unsuccess',\n",
        "                  'flg_gi_claim_29d435_ever', 'flg_gi_claim_058815_ever', 'flg_gi_claim_42e115_ever', 'flg_gi_claim_856320_ever']\n",
        "\n",
        "  hidden_data = hidden_data.drop(columns = cols_to_drop, axis = 1)\n",
        "\n",
        "  # then, drop all rows with NAs except for those in hh_size_est and annual_income_est\n",
        "  ordinal_cols = ['hh_size_est', 'annual_income_est']\n",
        "  hidden_data = hidden_data.dropna(subset=[col for col in hidden_data.columns if col not in ordinal_cols])\n",
        "\n",
        "  # then, label encode hh_size_est and annual_income_est\n",
        "  oe = OrdinalEncoder()\n",
        "  hidden_data[ordinal_cols] = oe.fit_transform(hidden_data[ordinal_cols])\n",
        "\n",
        "  # then, impute hh_size_est and annual_income_est w KNNImputer\n",
        "  knn_imputer = KNNImputer(n_neighbors=5)\n",
        "  hidden_data[ordinal_cols] = knn_imputer.fit_transform(hidden_data[ordinal_cols])\n",
        "  hidden_data[ordinal_cols] = hidden_data[ordinal_cols].round()\n",
        "\n",
        "  # then, convert all variables to numeric\n",
        "  hidden_data = hidden_data.apply(pd.to_numeric, downcast = 'integer')\n",
        "  numeric_columns = hidden_data.select_dtypes(include = 'number').columns\n",
        "  hidden_data[numeric_columns] = hidden_data[numeric_columns].astype('int64', errors = 'ignore')\n",
        "\n",
        "  # then, normalise continuous variables in dataset\n",
        "  scaler = StandardScaler()\n",
        "  continuous_cols = [x for x in hidden_data.columns if getattr(hidden_data, x).max() > 10]\n",
        "  hidden_data[continuous_cols] = scaler.fit_transform(hidden_data[continuous_cols])\n",
        "\n",
        "  # then, use tuned model above to predict test data\n",
        "  hidden_data = hidden_data[df_balanced_features.columns]  # to ensure that the order of columns are the same between the training dataset and test dataset\n",
        "  with open('final_xgb_model', 'rb') as f:                 # load model that was trained on training dataset\n",
        "    final_xgb_model = pickle.load(f)\n",
        "\n",
        "  predictions = final_xgb_model.predict(hidden_data)\n",
        "\n",
        "  # finally, return list of predictions\n",
        "  return list(predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h_hEvzJldea8"
      },
      "source": [
        "##### Cell to check testing_hidden_data function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aG-yKtA3dea8",
        "outputId": "a01367a5-5542-40e5-e58a-4827dfc6b218"
      },
      "outputs": [],
      "source": [
        "# This cell should output a list of predictions.\n",
        "test_df = pd.read_parquet(filepath)\n",
        "test_df = test_df.drop(columns=[\"f_purchase_lh\"])\n",
        "print(testing_hidden_data(test_df))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}