03b_DistanceMatrix_exclude_target_tqv3_av78.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"},"colab":{"name":"03_DistanceMatrix_exclude_target_tqv3_av78.ipynb","version":"0.3.2","provenance":[]}},"cells":[{"cell_type":"markdown","metadata":{"id":"axcaGD3jhWpD","colab_type":"text"},"source":["Note: All code in script is sourced from various internet sources, from mentor Tomasz Popiel and from Alan Chalk at Sabre Insurance."]},{"cell_type":"markdown","metadata":{"id":"iN7gPy8XhWpE","colab_type":"text"},"source":["## Distance Matrix\n","\n","Contents:\n","\n"," - Start_: import modules, set directories, load data ('02_df_all.pickle') \n"," \n"," - Sample non-scaled data\n"," \n"," - Scaling (numerical and histogram variables)\n"," \n"," - Distance matrix components (Numerical, Categorical, Histogram)\n"," \n"," -  Combining into final distance matrix\n"," \n"," -  Save ('03_DistanceMatrix_20%_exclude_target_tqv3_av78.hkl')\n","\n","Notes: \n","\n","- Exclude target for target analysis \n","- Exclude tq_v3 & a_v7_8 to try to reduce the outliers "]},{"cell_type":"markdown","metadata":{"id":"RSvc-OoehWpF","colab_type":"text"},"source":["### Start_:"]},{"cell_type":"markdown","metadata":{"id":"WWOdBOMNhWpF","colab_type":"text"},"source":["Import modules"]},{"cell_type":"code","metadata":{"id":"OjYaschthWpG","colab_type":"code","colab":{}},"source":["import os\n","import pickle\n","import gc\n","import sys\n","\n","import pandas as pd\n","import numpy as np\n","\n","from scipy.spatial.distance import pdist\n","from scipy.spatial.distance import squareform\n","from scipy.spatial.distance import jensenshannon\n","\n","from sklearn.preprocessing import normalize"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"bpHv96PYhWpI","colab_type":"text"},"source":["Set directories"]},{"cell_type":"code","metadata":{"id":"ZvBXmeIkhWpJ","colab_type":"code","outputId":"14f9b035-9046-4b50-ee9d-794a2bd34fbd","colab":{}},"source":["print(os.getcwd())\n","\n","dirRawData = \"../RawData/\"\n","dirPData = \"../PData/\"\n","dirPOutput = \"../POutput/\""],"execution_count":0,"outputs":[{"output_type":"stream","text":["/Users/hujuiting/Desktop/GROUP PROJECT/PCode\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"yOABHfCHhWpM","colab_type":"text"},"source":["Load data"]},{"cell_type":"code","metadata":{"id":"R1G4qFZdhWpN","colab_type":"code","outputId":"e94eec86-6732-4711-f9ff-67c6863169fb","colab":{}},"source":["fname = dirPData + '02_df_all.pickle' \n","with open(fname, 'rb') as f:\n","    dict_ = pickle.load(f)\n","\n","df_all = dict_['df_all']\n","del fname\n","df_all.shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(65196, 643)"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"markdown","metadata":{"id":"zP7On85dhWpP","colab_type":"text"},"source":["### Sample non-scaled data"]},{"cell_type":"code","metadata":{"id":"sFR-Qq8ThWpP","colab_type":"code","colab":{}},"source":["df_all_2 = df_all.sample(frac=0.2, random_state=123)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"iSgUGCiehWpR","colab_type":"code","outputId":"af5f93d6-5c20-49f3-b3b1-0b0da0cc7a11","colab":{}},"source":["df_all_2.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>tq_db11_mean</th>\n","      <th>tq_db12_mean</th>\n","      <th>tq_db13_mean</th>\n","      <th>tq_db14_mean</th>\n","      <th>tq_dt1_mean</th>\n","      <th>tq_dt2_mean</th>\n","      <th>tq_dt3_mean</th>\n","      <th>tq_dt4_mean</th>\n","      <th>tq_db15_mean</th>\n","      <th>tq_db16_mean</th>\n","      <th>...</th>\n","      <th>a_v7_8__97_431</th>\n","      <th>a_v7_8__97_46</th>\n","      <th>a_v7_8__97_647</th>\n","      <th>a_v7_8__97_770</th>\n","      <th>a_v7_8__97_875</th>\n","      <th>a_v7_8__97_909</th>\n","      <th>a_v7_8__99_37</th>\n","      <th>a_v7_8__99_995</th>\n","      <th>a_v7_8__99_996</th>\n","      <th>a_v7_8_other</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>12437</th>\n","      <td>15.9672</td>\n","      <td>7.3934</td>\n","      <td>1.4918</td>\n","      <td>0.0164</td>\n","      <td>0.0984</td>\n","      <td>0.0328</td>\n","      <td>0.0</td>\n","      <td>0.0164</td>\n","      <td>0.0984</td>\n","      <td>0.0984</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>32372</th>\n","      <td>16.3125</td>\n","      <td>4.4375</td>\n","      <td>1.1875</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>26155</th>\n","      <td>20.0000</td>\n","      <td>6.8462</td>\n","      <td>1.1538</td>\n","      <td>0.0769</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>50441</th>\n","      <td>16.5000</td>\n","      <td>5.0000</td>\n","      <td>1.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>0.0000</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>45622</th>\n","      <td>19.1250</td>\n","      <td>9.8750</td>\n","      <td>1.7500</td>\n","      <td>0.0000</td>\n","      <td>0.1250</td>\n","      <td>0.0000</td>\n","      <td>0.0</td>\n","      <td>0.0000</td>\n","      <td>0.3750</td>\n","      <td>0.3750</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5 rows × 643 columns</p>\n","</div>"],"text/plain":["       tq_db11_mean  tq_db12_mean  tq_db13_mean  tq_db14_mean  tq_dt1_mean  \\\n","12437       15.9672        7.3934        1.4918        0.0164       0.0984   \n","32372       16.3125        4.4375        1.1875        0.0000       0.0000   \n","26155       20.0000        6.8462        1.1538        0.0769       0.0000   \n","50441       16.5000        5.0000        1.0000        0.0000       0.0000   \n","45622       19.1250        9.8750        1.7500        0.0000       0.1250   \n","\n","       tq_dt2_mean  tq_dt3_mean  tq_dt4_mean  tq_db15_mean  tq_db16_mean  ...  \\\n","12437       0.0328          0.0       0.0164        0.0984        0.0984  ...   \n","32372       0.0000          0.0       0.0000        0.0000        0.0000  ...   \n","26155       0.0000          0.0       0.0000        0.0000        0.0000  ...   \n","50441       0.0000          0.0       0.0000        0.0000        0.0000  ...   \n","45622       0.0000          0.0       0.0000        0.3750        0.3750  ...   \n","\n","       a_v7_8__97_431  a_v7_8__97_46  a_v7_8__97_647  a_v7_8__97_770  \\\n","12437               0              0               0               0   \n","32372               0              0               0               0   \n","26155               0              0               0               0   \n","50441               0              0               0               0   \n","45622               0              0               0               0   \n","\n","       a_v7_8__97_875  a_v7_8__97_909  a_v7_8__99_37  a_v7_8__99_995  \\\n","12437               0               0              0               0   \n","32372               0               0              0               0   \n","26155               0               0              0               0   \n","50441               0               0              0               0   \n","45622               0               0              0               0   \n","\n","       a_v7_8__99_996  a_v7_8_other  \n","12437               0             0  \n","32372               0             0  \n","26155               0             0  \n","50441               0             0  \n","45622               0             0  \n","\n","[5 rows x 643 columns]"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"markdown","metadata":{"id":"MDatZnQYhWpT","colab_type":"text"},"source":["### Scaling (numerical and histogram variables)"]},{"cell_type":"markdown","metadata":{"id":"R8W9zcIThWpU","colab_type":"text"},"source":["Numerical variables : normalize "]},{"cell_type":"code","metadata":{"id":"HPiEP8lohWpU","colab_type":"code","colab":{}},"source":["df_scaled = normalize(df_all[dict_['vars_numeric']])\n","df_scaled = pd.DataFrame(df_scaled, columns=df_all[dict_['vars_numeric']].columns)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"en7r-gFjhWpX","colab_type":"code","colab":{}},"source":["for col in dict_['vars_numeric']:\n","    df_all[col] = df_scaled[col]"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"CCjDp1MBhWpZ","colab_type":"text"},"source":["Histogram variables : divide by 100 so they sum to 1"]},{"cell_type":"code","metadata":{"id":"fGkrKLPahWpa","colab_type":"code","colab":{}},"source":["for col in dict_['vars_hist']:\n","    df_all[col] = df_all[col]/100"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"_ZLSZRI9hWpc","colab_type":"code","outputId":"69d89af2-d3e2-425f-aacf-6f3cdfeabad8","colab":{}},"source":["del df_scaled\n","gc.collect()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["40"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"markdown","metadata":{"id":"vd6EyO8hhWpe","colab_type":"text"},"source":["### Distance matrix components\n","\n","Create distance matrices for numerical, categorial and histogram varaibles separately, then later add together to get final distance matrix"]},{"cell_type":"markdown","metadata":{"id":"wUY8d3fwhWpf","colab_type":"text"},"source":["#### Set weights for numerical, categorical and histgoram"]},{"cell_type":"code","metadata":{"id":"Yu4d0sTyhWpf","colab_type":"code","colab":{}},"source":["weight_num = 1\n","weight_cat = 1\n","weight_his = 1"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"BmLDtUKehWph","colab_type":"text"},"source":["#### Downsample data"]},{"cell_type":"code","metadata":{"id":"8dPKeJ34hWph","colab_type":"code","colab":{}},"source":["# randomly sample 20% of the dataset\n","df_all = df_all.sample(frac=0.2, random_state=123)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"jtrSS63bhWpj","colab_type":"code","colab":{}},"source":["# Drop ID as is not useful for clustering\n","# Drop target \n","# Drop tq_v3 & a_v7_8\n","\n","vars_notToUse1 = ['id',\n"," 'tq_dt1_mean',\n"," 'tq_dt2_mean',\n"," 'tq_dt3_mean',\n"," 'tq_dt4_mean',\n"," 'tq_dt1_std',\n"," 'tq_dt2_std',\n"," 'tq_dt3_std',\n"," 'tq_dt4_std',\n"," 'tq_v3']\n","\n","vars_notToUse2 = [col for col in df_all.columns if col.startswith('a_v7_8')]\n","\n","vars_notToUse = vars_notToUse1 + vars_notToUse2\n","\n","vars_toUse = [var for var in df_all.columns if var not in vars_notToUse]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"CyMYQ5mUhWpl","colab_type":"code","colab":{}},"source":["df_ds = df_all[vars_toUse]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"qlws0sfRhWpn","colab_type":"code","outputId":"92cbce77-805b-4329-d40d-614e4ef6d66b","colab":{}},"source":["df_ds.shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(13039, 344)"]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"code","metadata":{"scrolled":false,"id":"-NQMkEz3hWpp","colab_type":"code","colab":{}},"source":["#check the variables used\n","#list(df_ds.columns) "],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"x7jzaXVphWpr","colab_type":"text"},"source":["### Numerical Variables\n","\n","Using the 'pdist' function and distance metric \"cityblock\"   \n","\n","Considering different types of numerical varaibles separately (mean, std, mode, NMiss, a_v6, a_v9, a_v10, a_v11, a_v12)"]},{"cell_type":"code","metadata":{"id":"hGM7xwWdhWpr","colab_type":"code","colab":{}},"source":["vars_num_prefix_1 = ('_mean','_Mean')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"uSWzEiRphWpu","colab_type":"code","colab":{}},"source":["vars_num_prefix_2 = ('_std','_StdDev')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"BFOLU-K9hWpx","colab_type":"code","colab":{}},"source":["vars_num_prefix_3 = ('_Mode', '_NMiss')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"1Zfsm4FvhWpz","colab_type":"code","colab":{}},"source":["vars_num_prefix_4 = ['a_v6', 'a_v9', 'a_v10', 'a_v11', 'a_v12']"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZHzBv5LehWp0","colab_type":"code","colab":{}},"source":["### exclude the target variables \n","\n","vars_num_separate = []\n","\n","vars_num_separate.append([col for col in dict_['vars_numeric'] if col.endswith(vars_num_prefix_1) and not col.startswith('tq_dt1') and not col.startswith ('tq_dt2') and not col.startswith('tq_dt3') and not col.startswith ('tq_dt4')])\n","\n","vars_num_separate.append([col for col in dict_['vars_numeric'] if col.endswith(vars_num_prefix_2) and not col.startswith('tq_dt1') and not col.startswith ('tq_dt2') and not col.startswith('tq_dt3') and not col.startswith ('tq_dt4')])\n","\n","for prefix in vars_num_prefix_3:\n","    vars_num_separate.append([col for col in dict_['vars_numeric'] if col.endswith(prefix)])\n","\n","vars_num_separate.append(vars_num_prefix_4)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"UQBuIc_shWp2","colab_type":"code","outputId":"91cfb5e5-5497-4220-9b6c-78cebac23a6d","colab":{}},"source":["# we have a total of 5 types of numerical variables\n","len(vars_num_separate)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["5"]},"metadata":{"tags":[]},"execution_count":21}]},{"cell_type":"code","metadata":{"id":"Ypfkx8PHhWp3","colab_type":"code","colab":{}},"source":["#vars_num_separate"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"2_gHGPIMhWp5","colab_type":"text"},"source":["Combine the 5 types of numerical varaibles distance matrices into one distance matrix 'num_dist_mat\" and add weights. Have chosen weight 1 so no weight for now. This can be adjusted"]},{"cell_type":"code","metadata":{"id":"kU97mAFZhWp5","colab_type":"code","outputId":"310f7101-fb6d-4491-8e15-b7f08b557214","colab":{}},"source":["num_dist_mat = None\n","weights = [1, 1, 1, 1, 1]\n","\n","for idx, number in enumerate(vars_num_separate):\n","    number_as_matrix = df_ds[number].values\n","    each_distance_matrix = pdist(number_as_matrix, metric='cityblock')\n","    del number_as_matrix\n","    gc.collect()\n","    print(each_distance_matrix)\n","    #print(sys.getsizeof(each_distance_matrix)/1e9)\n","    if num_dist_mat is None:\n","        num_dist_mat = each_distance_matrix * weights[idx]\n","    else:\n","        num_dist_mat = np.sum([num_dist_mat, each_distance_matrix * weights[idx]], axis=0)\n","    del each_distance_matrix\n","    gc.collect()        "],"execution_count":0,"outputs":[{"output_type":"stream","text":["[0.57837191 0.68884797 0.96088635 ... 1.7798353  2.65400556 0.99020731]\n","[0.29191958 0.31139068 1.23138086 ... 1.03733625 1.63922339 0.60378148]\n","[2.30695339 2.09398696 1.39625035 ... 2.01703892 2.34294503 0.80124663]\n","[0.21567934 0.30144201 0.3539212  ... 0.77338483 1.82176689 1.09711334]\n","[0.22803257 0.08236019 0.11309987 ... 0.10221587 0.14894312 0.1075923 ]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"F13NmtPqhWp7","colab_type":"code","outputId":"021dac29-adc6-42ad-fc6e-ec9f609afe9d","colab":{}},"source":["num_dist_mat"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([3.62095677, 3.47802781, 4.05553863, ..., 5.70981117, 8.60688398,\n","       3.59994106])"]},"metadata":{"tags":[]},"execution_count":24}]},{"cell_type":"markdown","metadata":{"id":"F6cWrXVPhWp8","colab_type":"text"},"source":["### Categorical Variables\n","\n","Using the 'pdist' function and distance metric \"hamming\"   \n","\n","Considering different types of categorical varaibles separately ('a_v1','a_v2','a_v3','a_v4','a_v5')"]},{"cell_type":"code","metadata":{"id":"qQowrFGEhWp9","colab_type":"code","colab":{}},"source":["vars_cat_prefix = ('a_v1','a_v2','a_v3','a_v4','a_v5')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"nPH1LEARhWp_","colab_type":"code","colab":{}},"source":["vars_cat_separate = []\n","for prefix in vars_cat_prefix:\n","    vars_cat_separate.append([col for col in dict_['vars_cat'] if col.startswith(prefix)])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ThIgOm55hWqB","colab_type":"code","outputId":"d529f19d-cd31-4800-8221-7e74164d068e","colab":{}},"source":["# we have a total of 6 categories\n","len(vars_cat_separate)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["5"]},"metadata":{"tags":[]},"execution_count":27}]},{"cell_type":"markdown","metadata":{"id":"vgxXscAwhWqD","colab_type":"text"},"source":["Combine the 5 types of categorical varaibles distance matrices into one distance matrix 'cat_dist_mat\" and add weights. Have chosen weight 1 so no weight for now. This can be adjusted"]},{"cell_type":"code","metadata":{"id":"vREPuRXhhWqE","colab_type":"code","outputId":"95ad436b-af75-4523-e2f0-c3e5bf1100b3","colab":{}},"source":["# consider each category in turn, calculating its distance matrix\n","# weights defined here for each category so that we can add as we go along\n","cat_dist_mat = None\n","weights = [1, 1, 1, 1, 1]\n","\n","for idx, category in enumerate(vars_cat_separate):\n","    #print(idx)\n","    category_as_matrix = df_ds[category].values\n","    each_distance_matrix = pdist(category_as_matrix, metric='hamming')\n","    print(each_distance_matrix)\n","    del category_as_matrix\n","    gc.collect()\n","    if cat_dist_mat is None:\n","        cat_dist_mat = each_distance_matrix * weights[idx]\n","    else:\n","        cat_dist_mat = np.sum([cat_dist_mat, each_distance_matrix * weights[idx]], axis=0)\n","    del each_distance_matrix\n","    gc.collect()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[0. 0. 0. ... 0. 0. 0.]\n","[0.         0.66666667 0.         ... 0.66666667 0.         0.66666667]\n","[0.66666667 0.         0.66666667 ... 0.         0.         0.        ]\n","[0.         0.28571429 0.28571429 ... 0.28571429 0.28571429 0.        ]\n","[0.4 0.4 0.4 ... 0.4 0.4 0. ]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"frXAMe_lhWqF","colab_type":"code","outputId":"deac041c-07b0-41d0-95e3-417836a69ba5","colab":{}},"source":["cat_dist_mat"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([1.06666667, 1.35238095, 1.35238095, ..., 1.35238095, 0.68571429,\n","       0.66666667])"]},"metadata":{"tags":[]},"execution_count":29}]},{"cell_type":"markdown","metadata":{"id":"lVEG7HElhWqI","colab_type":"text"},"source":["#### Combine num_dist_mat and cat_dist_mat, weights 1\n","Combine num_dist_mat and cat_dist_mat to avoid storing both"]},{"cell_type":"code","metadata":{"id":"BikzZVJKhWqI","colab_type":"code","outputId":"43d85a0f-cc5c-4199-846b-1ea14139cfc7","colab":{}},"source":["a, b = num_dist_mat.mean(), cat_dist_mat.mean()\n","print((a, b))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(4.723541367455769, 1.345706436047386)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"wWgNij3phWqJ","colab_type":"code","outputId":"ebbaf972-2489-407d-e3f7-be7af4edd56d","colab":{}},"source":["print(num_dist_mat)\n","print(cat_dist_mat)\n","dist_mat = np.sum([weight_num * num_dist_mat, \n","                   weight_cat * a / b * cat_dist_mat\n","                  ], axis=0)\n","print(dist_mat)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[3.62095677 3.47802781 4.05553863 ... 5.70981117 8.60688398 3.59994106]\n","[1.06666667 1.35238095 1.35238095 ... 1.35238095 0.68571429 0.66666667]\n","[ 7.36504537  8.22499728  8.8025081  ... 10.45678065 11.01379808\n","  5.93999643]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"rD_bZvU9hWqL","colab_type":"code","outputId":"7a0b8694-42ee-4543-975b-e32cac38bd91","colab":{}},"source":["#mean values are the same \n","(weight_cat * a / b * cat_dist_mat).mean()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["4.723541367455735"]},"metadata":{"tags":[]},"execution_count":32}]},{"cell_type":"code","metadata":{"id":"NnDqppPBhWqN","colab_type":"code","outputId":"4ab0b69c-c950-459b-80f8-8eec1dcc0103","colab":{}},"source":["(weight_num * num_dist_mat).mean()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["4.723541367455769"]},"metadata":{"tags":[]},"execution_count":33}]},{"cell_type":"code","metadata":{"id":"tuAm00s0hWqP","colab_type":"code","outputId":"d6a1028e-8a13-42b7-ceea-1b334972fe0c","colab":{}},"source":["del num_dist_mat, cat_dist_mat\n","gc.collect()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["120"]},"metadata":{"tags":[]},"execution_count":34}]},{"cell_type":"markdown","metadata":{"id":"_iNUDfBNhWqR","colab_type":"text"},"source":["### Histogram Variables\n","\n","Using the 'pdist' function and distance metric \"jensenshannon\"   \n","\n","Considering different histogram varaibles separately"]},{"cell_type":"code","metadata":{"id":"OjlyaeFzhWqT","colab_type":"code","colab":{}},"source":["vars_hist_prefix = ('tq_db24', 'tq_db25', 'tq_db27', 'tq_db28', 'tq_db29', 'tq_db30','tq_db31',\n","                           'tq_v4', 'tq_v5', 'tq_db32', 'tq_db33', 'tq_db34', \n","                           'tq_da12','tq_da13', 'tq_da14',\n","                           'tq_db35', 'tq_db36','tq_db37', 'tq_db38',\n","                           'tq_db39', 'tq_db40', 'tq_db41', 'tq_db42',\n","                           'tq_db43', 'tq_db45', 'tq_db46',\n","                           'tq_db47', 'tq_db48', 'tq_db49', 'tq_db50')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"iSvYKimUhWqV","colab_type":"code","colab":{}},"source":["vars_hist_separate = []\n","for prefix in vars_hist_prefix:\n","    vars_hist_separate.append([col for col in dict_['vars_hist'] if col.startswith(prefix)])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"scrolled":true,"id":"xmPDKvp7hWqX","colab_type":"code","colab":{}},"source":["#vars_hist_separate"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZMnTmSwAhWqY","colab_type":"code","outputId":"2ab1dad7-1f26-41b4-9efc-700fa81d4c56","colab":{}},"source":["# we have a total of 30 histograms\n","len(vars_hist_separate)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["30"]},"metadata":{"tags":[]},"execution_count":38}]},{"cell_type":"markdown","metadata":{"id":"qblNYebyhWqa","colab_type":"text"},"source":["Combine the 30 histogram varaibles distance matrices into one distance matrix 'hist_dist_mat\" and add weights. Have chosen weight 1 so no weight for now. This can be adjusted"]},{"cell_type":"code","metadata":{"id":"mDngCTJYhWqc","colab_type":"code","outputId":"4d81d1ad-5b81-4d5d-89ec-ffd96c410ac4","colab":{}},"source":["hist_dist_mat = None\n","weights = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n","\n","for idx, histogram in enumerate(vars_hist_separate):\n","    #print(idx)\n","    histogram_as_matrix = df_ds[histogram].values\n","    each_distance_matrix = pdist(histogram_as_matrix, metric='jensenshannon')\n","    print(each_distance_matrix)\n","    del histogram_as_matrix\n","    gc.collect()\n","    if hist_dist_mat is None:\n","        hist_dist_mat = each_distance_matrix * weights[idx]\n","    else:\n","        hist_dist_mat = np.sum([hist_dist_mat, each_distance_matrix * weights[idx]], axis=0)\n","    del each_distance_matrix\n","    gc.collect()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[0.32110426 0.02630462 0.7266391  ... 0.07540514 0.26874401 0.19678598]\n","[0.         0.         0.         ... 0.05273159 0.04166534 0.01325817]\n","[0.17577947 0.10663291 0.42821022 ... 0.19713337 0.2288473  0.04856455]\n","[0.11384739 0.07656987 0.36037072 ... 0.04116571 0.03323081 0.07431246]\n","[0.30412519 0.29689756 0.44052665 ... 0.2478376  0.24869964 0.0985484 ]\n","[0.65673232 0.64414187 0.78745939 ... 0.36314955 0.37851526 0.19271047]\n","[0.2189326  0.25761886 0.54726255 ... 0.17260587 0.14243383 0.04191737]\n","[0.30858726 0.30550598 0.72655756 ... 0.7069239  0.72178872 0.06520844]\n","[0.07468223 0.10663291 0.07468223 ... 0.         0.01861985 0.01861985]\n","[0.05780056 0.15466943 0.48424546 ... 0.30233582 0.26649434 0.0939827 ]\n","[0.10758877 0.06914959 0.10758877 ... 0.17298864 0.13224733 0.06930745]\n","[0.10758877 0.06914959 0.10758877 ... 0.15765851 0.10814969 0.08639448]\n","[0.49549307 0.5092181  0.70161271 ... 0.64971755 0.56336008 0.35952711]\n","[0.25468087 0.07946667 0.74024851 ... 0.29865737 0.01508642 0.28461528]\n","[0.15659199 0.06840507 0.62993708 ... 0.25150378 0.04413111 0.20914208]\n","[0.67384875 0.1993494  0.75457542 ... 0.13920959 0.17036885 0.07558425]\n","[0.67384875 0.1993494  0.75457542 ... 0.13920959 0.17130766 0.07783603]\n","[0.37153809 0.22844078 0.67554096 ... 0.60835242 0.53386878 0.12422574]\n","[0.43459014 0.23565165 0.66407262 ... 0.62779629 0.54530724 0.13336404]\n","[0.23534874 0.16818887 0.44970293 ... 0.10039537 0.1705865  0.15127789]\n","[0.11641324 0.1527803  0.45001383 ... 0.119355   0.15032675 0.17653903]\n","[0.14941096 0.         0.         ... 0.03226217 0.01861985 0.01619028]\n","[0.14941096 0.         0.         ... 0.03226217 0.01861985 0.01619028]\n","[0.04990181 0.0763181  0.00565707 ... 0.22517571 0.298467   0.07693193]\n","[0.10758877 0.10758877 0.10758877 ... 0.02049493 0.08151775 0.06142493]\n","[0.07088945 0.28316493 0.28316493 ... 0.27011981 0.32453484 0.28277816]\n","[0.37024069 0.26596371 0.28316493 ... 0.25886834 0.28716205 0.15501286]\n","[0.35409719 0.26596371 0.28316493 ... 0.29191556 0.29509842 0.24414725]\n","[0.         0.         0.         ... 0.00787552 0.01064789 0.00277561]\n","[0.         0.         0.         ... 0.         0.01350424 0.01350424]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3kxrifM9hWqe","colab_type":"code","outputId":"a8bbe0d7-abc9-4ef4-a118-fbde87bc2e6a","colab":{}},"source":["hist_dist_mat                                                                                                                                                                                                                                                                                                                                                                                                       "],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([ 7.11066234,  4.95312266, 11.57415155, ...,  6.56310686,\n","        6.30195136,  3.46067731])"]},"metadata":{"tags":[]},"execution_count":40}]},{"cell_type":"markdown","metadata":{"id":"tKMyOfcshWqf","colab_type":"text"},"source":["### Combining into final distance matrix\n","Combining the numerical, categorical and histogram distance matrices into one distance matrix D_3 (apply weights above)"]},{"cell_type":"code","metadata":{"scrolled":true,"id":"D2cMfejmhWqf","colab_type":"code","outputId":"aea5a8f2-9ce4-45b2-a20e-19a129ffbe13","colab":{}},"source":["c = hist_dist_mat.mean()\n","print((a, b, c))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(4.723541367455769, 1.345706436047386, 8.020125426935385)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"y6fHIEmohWqi","colab_type":"code","colab":{}},"source":["# rescale so so mean values are the same so each matrix has an equal weighting \n","# and no matrix is negligible (these weights can be adjusted)\n","dist_mat = np.sum([dist_mat, weight_his * a / c * hist_dist_mat], axis=0)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"kedStCq7hWql","colab_type":"code","outputId":"ef11a83a-ec9a-47ff-890e-0037473b7c41","colab":{}},"source":["# mean values are the same\n","(weight_his * a / c * hist_dist_mat).mean()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["4.7235413674557725"]},"metadata":{"tags":[]},"execution_count":43}]},{"cell_type":"code","metadata":{"id":"_unbtEkvhWqn","colab_type":"code","outputId":"0919f2a7-ce5e-40a2-e474-4cf11dbb7634","colab":{}},"source":["del hist_dist_mat\n","gc.collect()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["100"]},"metadata":{"tags":[]},"execution_count":44}]},{"cell_type":"code","metadata":{"id":"eeruI2BbhWqo","colab_type":"code","outputId":"10c833a9-79bb-4038-dc3a-57d33d6d0d03","colab":{}},"source":["D_3 = dist_mat\n","del dist_mat\n","gc.collect()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["20"]},"metadata":{"tags":[]},"execution_count":45}]},{"cell_type":"markdown","metadata":{"id":"tBTffawEhWqp","colab_type":"text"},"source":["### Save"]},{"cell_type":"code","metadata":{"id":"MJS3kB_vhWqp","colab_type":"code","outputId":"b94c480b-bcb4-4625-8b1c-add77cec97cc","colab":{}},"source":["import hickle as hkl \n","\n","dict_['df_all_2'] = df_all_2  # 20% of non scaled dataset\n","dict_['df_all'] = df_all  # 20% of scaled dataset\n","dict_['D_3'] = D_3 # a condensed distance matrix\n","\n","\n","fname = dirPData + '03_distancematrix_20%_exclude_target_tqv3_av78.hkl'\n","\n","with open(fname, 'w') as f:\n","    hkl.dump(dict_, f)\n","\n","del dict_, fname, df_all"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/Users/hujuiting/anaconda3/lib/python3.7/site-packages/hickle/hickle.py:403: SerializedWarning: <class 'pandas.core.frame.DataFrame'> type not understood, data have been serialized\n","  SerializedWarning)\n"],"name":"stderr"}]}]}