Project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import librosa as lr\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Based on theory referred from various papers, we have found that below analysis \n",
    "# are most important for detecting speech features which could be relevant to\n",
    "# detecting emotions. Tonal and pitch analysis are generally difficult to analyse as \n",
    "# they are very complex. We are using librosa library to extract below features which are\n",
    "# basically categorized into spectral (related to spectrum of audio), prosodic (stress and \n",
    "# inotation patterns of speech) and qualitative (representing characteristic of voice). \n",
    "# Based on this understanding, and through experimentation, we hope to find relevant features \n",
    "# to emotions in speech. \n",
    "\n",
    "# Extraction of spectral features \n",
    "def spectral_features(y,sr,stft):\n",
    "    # 1. MFCC (40 most important features)\n",
    "    mfccs = lr.feature.mfcc(y=y, sr=sr, n_mfcc=40)\n",
    "    mean_mfccs = np.mean(mfccs.T,axis=0) \n",
    "    # 2. Pitch Chroma\n",
    "    p_chroma = lr.feature.chroma_stft(S=stft, sr=sr)\n",
    "    mean_p_chroma = np.mean(p_chroma.T,axis=0)\n",
    "    # 3. Spectral Centroid\n",
    "    sp_centroid = lr.feature.spectral_centroid(y=y, sr=sr)\n",
    "    mean_sp_centroid = np.mean(sp_centroid.T,axis=0)\n",
    "    # 4. Spectral Skewness/Contrast\n",
    "    contrast = lr.feature.spectral_contrast(S=stft, sr=sr)\n",
    "    mean_contrast = np.mean(contrast.T,axis=0)\n",
    "    \n",
    "    return mean_mfccs,mean_p_chroma,mean_sp_centroid,mean_contrast\n",
    "\n",
    "# Extraction of prosodic features\n",
    "def prosodic_features(y):\n",
    "    # 1. Zero crossing rate\n",
    "    zcr = lr.feature.zero_crossing_rate(y)\n",
    "    mean_zcr = np.mean(zcr.T,axis=0)\n",
    "    # 2. Root mean square and energy\n",
    "    rmse = lr.feature.rmse(y=y)\n",
    "    mean_rmse = np.mean(rmse.T,axis=0)\n",
    "    \n",
    "    return mean_zcr,mean_rmse\n",
    "\n",
    "def qualitative_features(y,sr):\n",
    "    # 1. For tone analysis, we calculate mel frequencies\n",
    "    mel = lr.feature.melspectrogram(y, sr=sr)\n",
    "    mean_mel = np.mean(mel.T,axis=0)\n",
    "    \n",
    "    return mean_mel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to parse data file name and return the gender and emotion of the speaker\n",
    "def parse_filename(file):\n",
    "    emotion_dict = {'01': 'neutral', '02' : 'calm', '03' : 'happy', '04' : 'sad', '05' : 'angry', '06' : 'fearful', '07' : 'disgust', '08' : 'surprised'}\n",
    "    gender = 'female' if int(file[18]+file[19])%2 == 0 else 'male'\n",
    "    return gender,emotion_dict[file[6]+file[7]]\n",
    "\n",
    "def create_data_ravdess(data_dir_ravdess):\n",
    "    \n",
    "    data = []\n",
    "    \n",
    "    list_dir = os.listdir(data_dir_ravdess)\n",
    "    \n",
    "    emotion = {'happy':0,'sad':1,'fearful':2,'angry':3}\n",
    "    \n",
    "    for each in list_dir:\n",
    "        \n",
    "        file = data_dir_ravdess+'/'+each\n",
    "        \n",
    "        if file.endswith(\".wav\"):\n",
    "            #print(file)\n",
    "\n",
    "            gender,emo = parse_filename(each)\n",
    "                \n",
    "            \n",
    "            if emo in ['happy','sad','fearful','angry']:\n",
    "                \n",
    "\n",
    "                # Loading data file to librosa and calculate y and sample rate\n",
    "                y, sr = lr.load(file)\n",
    "                # Generate short term Fourier Transform (stft)\n",
    "                stft = np.abs(lr.stft(y)) \n",
    "\n",
    "                f_mfccs,f_p_chroma,f_sp_centroid,f_contrast = spectral_features(y,sr,stft)\n",
    "                f_zcr,f_rmse = prosodic_features(y)\n",
    "                f_mel = qualitative_features(y,sr)\n",
    "\n",
    "                row = np.concatenate((f_mfccs,f_p_chroma,f_sp_centroid,f_contrast,f_zcr,f_rmse,f_mel,[emotion[emo]]))\n",
    "\n",
    "                data.append(row)\n",
    "\n",
    "    return pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data_cremad(data_dir_cremad):\n",
    "    \n",
    "    emotion =  {'HAP':0,'SAD':1,'ANG':3,'FEA':2}\n",
    "    \n",
    "    data = []\n",
    "    \n",
    "    list_dir = os.listdir(data_dir_cremad)\n",
    "    \n",
    "    for file in list_dir:\n",
    "\n",
    "        emo = file.split('_')[2]\n",
    "        if emo in ['HAP','SAD','FEA','ANG']:\n",
    "            if file.endswith(\".wav\"):\n",
    "     \n",
    "                # Loading data file to librosa and calculate y and sample rate\n",
    "                y, sr = lr.load(data_dir_cremad+'/'+file)\n",
    "                # Generate short term Fourier Transform (stft)\n",
    "                stft = np.abs(lr.stft(y)) \n",
    "\n",
    "                f_mfccs,f_p_chroma,f_sp_centroid,f_contrast = spectral_features(y,sr,stft)\n",
    "                f_zcr,f_rmse = prosodic_features(y)\n",
    "                f_mel = qualitative_features(y,sr)\n",
    "\n",
    "                row = np.concatenate((f_mfccs,f_p_chroma,f_sp_centroid,f_contrast,f_zcr,f_rmse,f_mel,[emotion[emo]]))\n",
    "\n",
    "                data.append(row)\n",
    "\n",
    "    return pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data_others(data_dir):\n",
    "    \n",
    "    data = []\n",
    "    \n",
    "    for i in [0,1,2,3]:\n",
    "        path = data_dir+'/'+str(i)+'/'\n",
    "        list_dir =  os.listdir(path)\n",
    "\n",
    "        for file in list_dir:\n",
    "\n",
    "            if file.endswith(\".wav\"):\n",
    "                #print(file)\n",
    "\n",
    "                # Loading data file to librosa and calculate y and sample rate\n",
    "                y, sr = lr.load(path+file)\n",
    "                # Generate short term Fourier Transform (stft)\n",
    "                stft = np.abs(lr.stft(y)) \n",
    "\n",
    "                f_mfccs,f_p_chroma,f_sp_centroid,f_contrast = spectral_features(y,sr,stft)\n",
    "                f_zcr,f_rmse = prosodic_features(y)\n",
    "                f_mel = qualitative_features(y,sr)\n",
    "\n",
    "                row = np.concatenate((f_mfccs,f_p_chroma,f_sp_centroid,f_contrast,f_zcr,f_rmse,f_mel,[i]))\n",
    "\n",
    "                data.append(row)\n",
    "\n",
    "    return pd.DataFrame(data)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\utkar\\Anaconda3\\lib\\site-packages\\librosa\\core\\pitch.py:145: UserWarning: Trying to estimate tuning from empty frequency set.\n",
      "  warnings.warn('Trying to estimate tuning from empty frequency set.')\n"
     ]
    }
   ],
   "source": [
    "# Get the data directory which stores the audio files\n",
    "data_dir_ravdess = 'training_data/ravdess'\n",
    "data_dir_tess = 'training_data/tess'\n",
    "data_dir_savee = 'training_data/savee'\n",
    "data_dir_cremad = 'training_data/crema_d'\n",
    "data_dir_bdes = 'training_data/bdes'\n",
    "\n",
    "\n",
    "# Create data for all the data directories\n",
    "data_ravdess = create_data_ravdess(data_dir_ravdess)\n",
    "data_cremad = create_data_cremad(data_dir_cremad)\n",
    "data_tess = create_data_others(data_dir_tess)\n",
    "data_savee = create_data_others(data_dir_savee)\n",
    "data_bdes = create_data_others(data_dir_bdes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8021, 191)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Concatenate all data into a single Dataframe\n",
    "data = pd.concat([data_ravdess, data_cremad, data_tess, data_savee, data_bdes])\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>181</th>\n",
       "      <th>182</th>\n",
       "      <th>183</th>\n",
       "      <th>184</th>\n",
       "      <th>185</th>\n",
       "      <th>186</th>\n",
       "      <th>187</th>\n",
       "      <th>188</th>\n",
       "      <th>189</th>\n",
       "      <th>190</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-646.130257</td>\n",
       "      <td>63.436045</td>\n",
       "      <td>-4.705014</td>\n",
       "      <td>12.907815</td>\n",
       "      <td>3.926321</td>\n",
       "      <td>-2.585449</td>\n",
       "      <td>-5.982906</td>\n",
       "      <td>-6.311690</td>\n",
       "      <td>-11.837655</td>\n",
       "      <td>-2.075405</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>3.296748e-07</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-584.981918</td>\n",
       "      <td>43.497349</td>\n",
       "      <td>-21.613513</td>\n",
       "      <td>8.840687</td>\n",
       "      <td>-7.184710</td>\n",
       "      <td>-5.850826</td>\n",
       "      <td>-4.939766</td>\n",
       "      <td>-12.593667</td>\n",
       "      <td>-4.777969</td>\n",
       "      <td>-3.069792</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>7.735573e-07</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-541.433974</td>\n",
       "      <td>62.939903</td>\n",
       "      <td>-10.896681</td>\n",
       "      <td>5.023488</td>\n",
       "      <td>2.941020</td>\n",
       "      <td>-0.349534</td>\n",
       "      <td>-14.487152</td>\n",
       "      <td>-10.850216</td>\n",
       "      <td>-9.839632</td>\n",
       "      <td>-8.713090</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000203</td>\n",
       "      <td>0.000155</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.000139</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>5.344014e-06</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-650.659798</td>\n",
       "      <td>46.812761</td>\n",
       "      <td>-16.539898</td>\n",
       "      <td>6.699998</td>\n",
       "      <td>-8.078243</td>\n",
       "      <td>-8.253147</td>\n",
       "      <td>-8.076603</td>\n",
       "      <td>-13.949132</td>\n",
       "      <td>-8.494425</td>\n",
       "      <td>-4.106237</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>2.376553e-07</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-676.171373</td>\n",
       "      <td>65.932610</td>\n",
       "      <td>4.009063</td>\n",
       "      <td>15.863403</td>\n",
       "      <td>8.687657</td>\n",
       "      <td>2.067518</td>\n",
       "      <td>-0.919425</td>\n",
       "      <td>-0.706527</td>\n",
       "      <td>-10.634650</td>\n",
       "      <td>-2.900578</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>7.006203e-08</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 191 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0          1          2          3         4         5          6    \\\n",
       "0 -646.130257  63.436045  -4.705014  12.907815  3.926321 -2.585449  -5.982906   \n",
       "1 -584.981918  43.497349 -21.613513   8.840687 -7.184710 -5.850826  -4.939766   \n",
       "2 -541.433974  62.939903 -10.896681   5.023488  2.941020 -0.349534 -14.487152   \n",
       "3 -650.659798  46.812761 -16.539898   6.699998 -8.078243 -8.253147  -8.076603   \n",
       "4 -676.171373  65.932610   4.009063  15.863403  8.687657  2.067518  -0.919425   \n",
       "\n",
       "         7          8         9   ...        181       182       183  \\\n",
       "0  -6.311690 -11.837655 -2.075405 ...   0.000010  0.000006  0.000011   \n",
       "1 -12.593667  -4.777969 -3.069792 ...   0.000048  0.000044  0.000041   \n",
       "2 -10.850216  -9.839632 -8.713090 ...   0.000203  0.000155  0.000147   \n",
       "3 -13.949132  -8.494425 -4.106237 ...   0.000024  0.000031  0.000033   \n",
       "4  -0.706527 -10.634650 -2.900578 ...   0.000006  0.000007  0.000009   \n",
       "\n",
       "        184       185       186       187       188           189  190  \n",
       "0  0.000007  0.000010  0.000011  0.000006  0.000004  3.296748e-07  0.0  \n",
       "1  0.000036  0.000023  0.000032  0.000031  0.000008  7.735573e-07  0.0  \n",
       "2  0.000153  0.000108  0.000153  0.000139  0.000068  5.344014e-06  0.0  \n",
       "3  0.000022  0.000023  0.000016  0.000007  0.000003  2.376553e-07  0.0  \n",
       "4  0.000008  0.000007  0.000007  0.000005  0.000001  7.006203e-08  0.0  \n",
       "\n",
       "[5 rows x 191 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save data into a file\n",
    "data.to_csv('training_data/data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the correlation between the columns \n",
    "\n",
    "corr_matrix_train = data.corr().abs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns with high correlation to be removed:\n",
      "[169, 179, 180, 181, 182, 183, 187, 188, 189]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "## This code is based on Chris Albon's code to drop the correlated columns from a data frame\n",
    "upper = corr_matrix_train.where(np.triu(np.ones(corr_matrix_train.shape), k=1).astype(np.bool))\n",
    "to_drop = [int(column) for column in upper.columns if any(upper[column] > 0.95)]\n",
    "print(\"Columns with high correlation to be removed:\")\n",
    "print(to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the columns from data frame and create another data frame as df\n",
    "\n",
    "df = data.drop(data.columns[to_drop], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = range(df.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>172</th>\n",
       "      <th>173</th>\n",
       "      <th>174</th>\n",
       "      <th>175</th>\n",
       "      <th>176</th>\n",
       "      <th>177</th>\n",
       "      <th>178</th>\n",
       "      <th>179</th>\n",
       "      <th>180</th>\n",
       "      <th>181</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-646.130257</td>\n",
       "      <td>63.436045</td>\n",
       "      <td>-4.705014</td>\n",
       "      <td>12.907815</td>\n",
       "      <td>3.926321</td>\n",
       "      <td>-2.585449</td>\n",
       "      <td>-5.982906</td>\n",
       "      <td>-6.311690</td>\n",
       "      <td>-11.837655</td>\n",
       "      <td>-2.075405</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-584.981918</td>\n",
       "      <td>43.497349</td>\n",
       "      <td>-21.613513</td>\n",
       "      <td>8.840687</td>\n",
       "      <td>-7.184710</td>\n",
       "      <td>-5.850826</td>\n",
       "      <td>-4.939766</td>\n",
       "      <td>-12.593667</td>\n",
       "      <td>-4.777969</td>\n",
       "      <td>-3.069792</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.000047</td>\n",
       "      <td>0.000055</td>\n",
       "      <td>0.000065</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-541.433974</td>\n",
       "      <td>62.939903</td>\n",
       "      <td>-10.896681</td>\n",
       "      <td>5.023488</td>\n",
       "      <td>2.941020</td>\n",
       "      <td>-0.349534</td>\n",
       "      <td>-14.487152</td>\n",
       "      <td>-10.850216</td>\n",
       "      <td>-9.839632</td>\n",
       "      <td>-8.713090</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001259</td>\n",
       "      <td>0.001030</td>\n",
       "      <td>0.000504</td>\n",
       "      <td>0.000401</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>0.000345</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-650.659798</td>\n",
       "      <td>46.812761</td>\n",
       "      <td>-16.539898</td>\n",
       "      <td>6.699998</td>\n",
       "      <td>-8.078243</td>\n",
       "      <td>-8.253147</td>\n",
       "      <td>-8.076603</td>\n",
       "      <td>-13.949132</td>\n",
       "      <td>-8.494425</td>\n",
       "      <td>-4.106237</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000061</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000117</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-676.171373</td>\n",
       "      <td>65.932610</td>\n",
       "      <td>4.009063</td>\n",
       "      <td>15.863403</td>\n",
       "      <td>8.687657</td>\n",
       "      <td>2.067518</td>\n",
       "      <td>-0.919425</td>\n",
       "      <td>-0.706527</td>\n",
       "      <td>-10.634650</td>\n",
       "      <td>-2.900578</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 182 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0          1          2          3         4         5          6    \\\n",
       "0 -646.130257  63.436045  -4.705014  12.907815  3.926321 -2.585449  -5.982906   \n",
       "1 -584.981918  43.497349 -21.613513   8.840687 -7.184710 -5.850826  -4.939766   \n",
       "2 -541.433974  62.939903 -10.896681   5.023488  2.941020 -0.349534 -14.487152   \n",
       "3 -650.659798  46.812761 -16.539898   6.699998 -8.078243 -8.253147  -8.076603   \n",
       "4 -676.171373  65.932610   4.009063  15.863403  8.687657  2.067518  -0.919425   \n",
       "\n",
       "         7          8         9   ...        172       173       174  \\\n",
       "0  -6.311690 -11.837655 -2.075405 ...   0.000017  0.000023  0.000013   \n",
       "1 -12.593667  -4.777969 -3.069792 ...   0.000026  0.000022  0.000033   \n",
       "2 -10.850216  -9.839632 -8.713090 ...   0.001259  0.001030  0.000504   \n",
       "3 -13.949132  -8.494425 -4.106237 ...   0.000019  0.000061  0.000083   \n",
       "4  -0.706527 -10.634650 -2.900578 ...   0.000038  0.000046  0.000031   \n",
       "\n",
       "        175       176       177       178       179       180  181  \n",
       "0  0.000012  0.000009  0.000020  0.000007  0.000010  0.000011  0.0  \n",
       "1  0.000047  0.000055  0.000065  0.000036  0.000023  0.000032  0.0  \n",
       "2  0.000401  0.000554  0.000345  0.000153  0.000108  0.000153  0.0  \n",
       "3  0.000091  0.000075  0.000117  0.000022  0.000023  0.000016  0.0  \n",
       "4  0.000008  0.000006  0.000006  0.000008  0.000007  0.000007  0.0  \n",
       "\n",
       "[5 rows x 182 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8021, 182)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Features:  100\n",
      "Selected Features:  [False False False False False False False False False False False  True\n",
      " False False False False False False False False False False False False\n",
      " False False False False  True False False False False False False False\n",
      " False False False False  True  True  True  True  True  True  True  True\n",
      "  True  True  True  True False False  True  True  True  True  True False\n",
      "  True  True  True  True  True False False False False False False False\n",
      " False False False False False False False False False False False False\n",
      " False False False False False False False False False  True False False\n",
      "  True False  True  True False  True False  True  True  True False  True\n",
      "  True False  True  True  True  True False False  True False False False\n",
      "  True  True  True  True  True  True  True  True  True  True  True  True\n",
      "  True  True  True  True  True  True  True  True  True  True  True  True\n",
      "  True  True  True  True  True  True  True  True  True  True  True  True\n",
      "  True  True  True  True  True  True  True  True  True  True  True  True\n",
      "  True  True  True  True  True  True  True  True  True  True  True  True\n",
      "  True]\n",
      "Feature Ranking:  [73 63 65 43 28 17 60 42 52 37 38  1 59 48 29 27 14 15 25  3 47 23 33 34\n",
      "  6 40 39 50  1 35 36 21 12 13 61 31 22 24 26 44  1  1  1  1  1  1  1  1\n",
      "  1  1  1  1 82 16  1  1  1  1  1 18  1  1  1  1  1  5  9 11 66 54 53 55\n",
      " 68 69 56 58 57 76 77 80 75 79 81 74 64 67 78 62 71 49 51 46 72  1  2 70\n",
      "  1  7  1  1 32  1 45  1  1  1 41  1  1  4  1  1  1  1 30 10  1  8 19 20\n",
      "  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1\n",
      "  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1\n",
      "  1  1  1  1  1  1  1  1  1  1  1  1  1]\n"
     ]
    }
   ],
   "source": [
    "#Feature selection using Backward Elimination\n",
    "\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVR\n",
    "\n",
    "X = df[df.columns[0:181]]\n",
    "Y = df[181]\n",
    "\n",
    "# feature extraction\n",
    "model = LogisticRegression() #SVR(kernel='poly', C=100, gamma='auto', degree=3, epsilon=.1,coef0=1) #\n",
    "\n",
    "rfe = RFE(model, 100)\n",
    "fit = rfe.fit(X, Y)\n",
    "print(\"Num Features: \",fit.n_features_)\n",
    "print(\"Selected Features: \",fit.support_)\n",
    "print(\"Feature Ranking: \",fit.ranking_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Int64Index([ 11,  28,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,\n",
      "             51,  54,  55,  56,  57,  58,  60,  61,  62,  63,  64,  93,  96,\n",
      "             98,  99, 101, 103, 104, 105, 107, 108, 110, 111, 112, 113, 116,\n",
      "            120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,\n",
      "            133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145,\n",
      "            146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158,\n",
      "            159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171,\n",
      "            172, 173, 174, 175, 176, 177, 178, 179, 180],\n",
      "           dtype='int64')\n"
     ]
    }
   ],
   "source": [
    "print(X.columns[fit.support_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>11</th>\n",
       "      <th>28</th>\n",
       "      <th>40</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "      <th>43</th>\n",
       "      <th>44</th>\n",
       "      <th>45</th>\n",
       "      <th>46</th>\n",
       "      <th>47</th>\n",
       "      <th>...</th>\n",
       "      <th>171</th>\n",
       "      <th>172</th>\n",
       "      <th>173</th>\n",
       "      <th>174</th>\n",
       "      <th>175</th>\n",
       "      <th>176</th>\n",
       "      <th>177</th>\n",
       "      <th>178</th>\n",
       "      <th>179</th>\n",
       "      <th>180</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-6.506576</td>\n",
       "      <td>-4.230472</td>\n",
       "      <td>0.763436</td>\n",
       "      <td>0.790995</td>\n",
       "      <td>0.763995</td>\n",
       "      <td>0.774775</td>\n",
       "      <td>0.759068</td>\n",
       "      <td>0.709952</td>\n",
       "      <td>0.672845</td>\n",
       "      <td>0.702815</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-2.429099</td>\n",
       "      <td>2.132234</td>\n",
       "      <td>0.705350</td>\n",
       "      <td>0.653129</td>\n",
       "      <td>0.632511</td>\n",
       "      <td>0.629488</td>\n",
       "      <td>0.613461</td>\n",
       "      <td>0.670480</td>\n",
       "      <td>0.763453</td>\n",
       "      <td>0.745118</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.000047</td>\n",
       "      <td>0.000055</td>\n",
       "      <td>0.000065</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-7.089989</td>\n",
       "      <td>-2.473618</td>\n",
       "      <td>0.594999</td>\n",
       "      <td>0.580023</td>\n",
       "      <td>0.641304</td>\n",
       "      <td>0.752040</td>\n",
       "      <td>0.728397</td>\n",
       "      <td>0.653160</td>\n",
       "      <td>0.659680</td>\n",
       "      <td>0.655550</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000774</td>\n",
       "      <td>0.001259</td>\n",
       "      <td>0.001030</td>\n",
       "      <td>0.000504</td>\n",
       "      <td>0.000401</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>0.000345</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.000153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-8.504312</td>\n",
       "      <td>-2.049245</td>\n",
       "      <td>0.458457</td>\n",
       "      <td>0.471476</td>\n",
       "      <td>0.456127</td>\n",
       "      <td>0.464032</td>\n",
       "      <td>0.457922</td>\n",
       "      <td>0.464545</td>\n",
       "      <td>0.475808</td>\n",
       "      <td>0.516776</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000061</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000117</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.000016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-2.081819</td>\n",
       "      <td>-1.316368</td>\n",
       "      <td>0.671510</td>\n",
       "      <td>0.717728</td>\n",
       "      <td>0.744103</td>\n",
       "      <td>0.731981</td>\n",
       "      <td>0.739587</td>\n",
       "      <td>0.718110</td>\n",
       "      <td>0.722261</td>\n",
       "      <td>0.694046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000053</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000007</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        11        28        40        41        42        43        44   \\\n",
       "0 -6.506576 -4.230472  0.763436  0.790995  0.763995  0.774775  0.759068   \n",
       "1 -2.429099  2.132234  0.705350  0.653129  0.632511  0.629488  0.613461   \n",
       "2 -7.089989 -2.473618  0.594999  0.580023  0.641304  0.752040  0.728397   \n",
       "3 -8.504312 -2.049245  0.458457  0.471476  0.456127  0.464032  0.457922   \n",
       "4 -2.081819 -1.316368  0.671510  0.717728  0.744103  0.731981  0.739587   \n",
       "\n",
       "        45        46        47     ...          171       172       173  \\\n",
       "0  0.709952  0.672845  0.702815    ...     0.000016  0.000017  0.000023   \n",
       "1  0.670480  0.763453  0.745118    ...     0.000049  0.000026  0.000022   \n",
       "2  0.653160  0.659680  0.655550    ...     0.000774  0.001259  0.001030   \n",
       "3  0.464545  0.475808  0.516776    ...     0.000024  0.000019  0.000061   \n",
       "4  0.718110  0.722261  0.694046    ...     0.000053  0.000038  0.000046   \n",
       "\n",
       "        174       175       176       177       178       179       180  \n",
       "0  0.000013  0.000012  0.000009  0.000020  0.000007  0.000010  0.000011  \n",
       "1  0.000033  0.000047  0.000055  0.000065  0.000036  0.000023  0.000032  \n",
       "2  0.000504  0.000401  0.000554  0.000345  0.000153  0.000108  0.000153  \n",
       "3  0.000083  0.000091  0.000075  0.000117  0.000022  0.000023  0.000016  \n",
       "4  0.000031  0.000008  0.000006  0.000006  0.000008  0.000007  0.000007  \n",
       "\n",
       "[5 rows x 100 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df[df.columns[0:181]]\n",
    "y = df[181]\n",
    "df[X.columns[fit.support_]].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG25JREFUeJzt3XuQXOV55/HvD91AF5BBSNFtPcYhMiAWZKaUTSkQI9mYWzA2vkC8GGxSWrKwhi1YI5u4IjubWojXGDBeExZwYANYDkZlFmwWrYFgdrHsGTG6ecQCinCEBDIQwQgVEEnP/nHeGZp2z6X7nNM90L9P1dScPpf3PDrTeubM2+95XkUEZmbWXvZrdQBmZtZ8Tv5mZm3Iyd/MrA05+ZuZtSEnfzOzNuTkb2bWhpz8zczakJO/mVkbcvI3M2tDY1sdAMC0adOio6Oj1WGYmb2jdHd3vxgRhzZy7KhI/h0dHXR1dbU6DDOzdxRJzzZ6bMPdPpLmSnpYUq+kjZIuSev/UtI6ST2SHpQ0q9FzmJlZOdRoYTdJM4GZEbFG0hSgGzgT2BoRr6Z9vggcGREXDtXWvHkT4r99d05Dcdi7x5LFz7Q6BLN3FEndEdHZyLEN3/lHxPaIWJOW+4BeYHZ/4k8mAS4bamY2yhTS5y+pA1gArE6v/wr4HPAKcGIR5zAzs+LkHuopaTLwQ+DS/rv+iLgyIuYCdwAXD3LcUkldkrp27tyXNwwzM6tDruQvaRxZ4r8jIu6pscudwFm1jo2ImyKiMyI6p0714wZmZs2UZ7SPgFuA3oi4pmL94RW7nQFsajw8MzMrQ54+/0XAucB6ST1p3VeACyTNA/YBzwJDjvQBmDLlaJYs9jh/M7NmaTj5R8RjgGps+nHj4ZiZWTO4s93MrA05+ZuZtaGGu30kzQVuB36HrH//poi4TtIKYF7abSqwMyKOzR2pmZkVJs8HvnuAyyrLO0haFRGf6d9B0jfJHvQa0rZt21i+fHmOUOzdxO8Fs/Ll+cB3O7A9LfdJ6gVmA7+CgaGgnwYWFxCnmZkVqJA+/+ryDsnxwAsR8VQR5zAzs+KUUt4hOQe4a4jjBso77N69O28YZmZWh1LKO0gaC3wCWDHYsZXlHSZOnJgnDDMzq1Oe0T41yzskHwY2RcTWkbQ1a9Ysf8hnZtZEee78+8s7LE6zdvVIOjVtO5shunzMzKy1yijvQESc32i7ZmZWPj/ha2bWhpz8zczaUK5pHCVtAfqAvcCeiOiUdDDZKJ8OYAvw6Yj453xhmplZkYqYw/fEiHix4vUy4KcRcZWkZen1FUM18OZzu9i67GcFhGLvBnOuOr7VIZi965XR7fMx4La0fBtwZgnnMDOzHPIm/wAelNQtaWlaNyPV/emv/zM95znMzKxgebt9FkXENknTgVWSRjxfb/plsRRg9oEzcoZhZmb1yHXnHxHb0vcdwEpgIfCCpJkA6fuOQY4dKO9w8MSpecIwM7M65SnvMAnYL5VzngScBHwduBc4D7gqff/RcG2Nnz3ZH/KZmTVRnm6fGcDKrMQPY4E7I+IBSb8EfiDpAuDXwKfyh2lmZkXKU95hM3BMjfUvAUvyBGVmZuXyE75mZm3Iyd/MrA01nPwl7S/pF5LWStoo6Wtp/cWSnpYUkqYVF6qZmRVFEdHYgdknvZMiYlea0esx4BLgDeCfgUeAzqrSDzXNPXhqXPqRP2woDnv3umzFfa0OwWxUk9QdEZ2NHJvnA98AdqWX49JXRMQTKahGmzYzs5LlncN3jKQesge5VkXE6jqOHZjA/bU33swThpmZ1SnvE757I+JYYA6wUNL8Oo4deMJ30oTxecIwM7M6FTLaJyJ2kvXxn1xEe2ZmVq485R0OBf4lInZKOgD4MHB1I23NOOx3/eGemVkT5bnznwk8LGkd8EuyPv/7JH1R0layrqB1km4uIlAzMytOntE+64AFNdZfD1yfJygzMyuXn/A1M2tDTv5mZm0od/JPY/2fkHRfen1LKvmwTtLdkibnD9PMzIqUdxpHyEo69AIHptf/MSJeBZB0DXAx2cQug9rxbB/fufChAkKxd6OLblzc6hDM3nXyPuE7BzgNGBjRU5H4BRxANsm7mZmNInm7fa4FvgTsq1wp6XvA88AHgG/XOrCyvMOu13fmDMPMzOqRp6Tz6cCOiOiu3hYRnwdmkXUHfabW8ZXlHSbv7wnczcyaKc+d/yLgDElbgO8DiyX9Xf/GiNgLrADOyhWhmZkVruF6/m9rRPoQcDnwx8D7I+Lp1Of/DYCIuHyo4zs7O6Orqyt3HGZm7aQl9fwHiwW4TdKBaXkt8GcFn8PMzHIqJPlHxCNkVT0h6w4yM7NRzE/4mpm1oVx3/unD3j5gL7AnIjolHQPcCEwGtgCf7R/7b2Zmo0MR3T4nVk3SfjNweUT8g6QvAP8J+OpQDby+YSO9HziigFCsHRyxqbfVIZi945XR7TMPeDQtr8JDPc3MRp28yT+AByV1S1qa1m0AzkjLnwLm5jyHmZkVLG/yXxQRHwROAS6SdALwhbTcDUwB3qx1YGV5h5f37skZhpmZ1SNX8o+Iben7DmAlsDAiNkXESRFxHHAX8Mwgxw6Udzh4TNGPG5iZ2VDy1PaZJGlK/zJwErBB0vS0bj/gz8lG/piZ2SiS55Z7BrAyq+LAWODOiHhA0iWSLkr73AN8b7iG9p9/FEe4vIOZWdPkmcB9M3BMjfXXAdflCcrMzMrlJ3zNzNqQk7+ZWRvKO43jrZJ2SNpQse5TkjZK2iepoVKjZmZWrrxjLP8WuAG4vWLdBuATwN+MtJGNL23k6NuOzhmKtbv1561vdQhm7xi5kn9EPCqpo2pdL0AaBWRmZqOQ+/zNzNpQy5J/ZXmHvX17WxWGmVlbalnyryzvMGbKmFaFYWbWltztY2bWhvLO5HUX8CFgmqStwF8ALwPfBg4F7pfUExEfHaqdow45iq7zXN7BzKxZ8o72OWeQTSvztGtmZuVyt4+ZWRty8jcza0N56vnPlfSwpN5UzuGStP4bkjZJWidppaSpxYVrZmZFUEQ0dqA0E5gZEWvSpC7dwJnAHOChiNgj6WqAiLhiqLY6Z42JrqWTG4rD7G2Wv9LqCMyaRlJ3RDRUQ63hO/+I2B4Ra9JyH9ALzI6IByOif1Len5P9MjAzs1GkkD7/VN9nAbC6atMXgJ8UcQ4zMytO7uQvaTLwQ+DSiHi1Yv2VwB7gjkGOGyjv8JvdjXU9mZlZY/LW8x9HlvjviIh7KtafB5wOfDYG+VChsrzDoRNdAdTMrJkafshLWc3mW4DeiLimYv3JwBXAH0XE7vwhmplZ0fI84bsIOBdYL6knrfsKcD0wAViVavr/PCIuHLKlWQtgucs7mJk1S8PJPyIeA2r11/y48XDMzKwZ/ISvmVkbcvI3M2tDwyZ/SbdK2iFpQ8W6v0zlG3okPShpVsW2D6X1GyX9Q1mBm5lZ44Yt7yDpBGAXcHtEzE/rDuwf0y/pi8CREXFhquPzf4GTI+LXkqZHxI7hgpgw8/CYed61ef8tZmy56rRWh2DWNKWWd4iIR8kmaKlc92rFy0lA/2+QPwHuiYhfp/2GTfxmZtZ8ecb5/xXwOeAV4MS0+veAcZIeAaYA10XE7XmDNDOzYuUp7HZlRMwlK99wcVo9FjgOOA34KPBVSb9X6/jK8g57d7sSo5lZMxUx2udO4Ky0vBV4ICJei4gXgUeBY2odVFneYczEgwoIw8zMRqqhbh9Jh0fEU+nlGcCmtPwj4AZJY4HxwO8D3xquvaNnH0SXP6gzM2uaYZO/pLuADwHTJG0F/gI4VdI8YB/wLHAhQET0SnoAWJe23RwRG2o2bGZmLdPwTF5F6uzsjK4u1/YxM6tHS2byMjOzdy4nfzOzNlRK8pc0V9LDknpTmYdLyjiPmZk1Jk89/6HsAS6LiDWSpgDdklZFxK9q7bz+uVfoWHZ/SaGYZVz6wewtpdz5R8T2iFiTlvuAXmB2GecyM7P6ld7nL6kDWACsLvtcZmY2MqUmf0mTySZ4v7SqGJzLO5iZtVBpyV/SOLLEf0dE3FO93eUdzMxap5QPfJXN3H4L0BsR1wy3v8s7mJk1V1l3/ouAc4HFaVavHkmnlnQuMzOrUyl3/hHxGKAy2jYzs/z8hK+ZWRty8jcza0PDJn9Jt0raIem3SjNLulxSSJqWXr9H0kpJ6yT9QtL8MoI2M7N8RtLn/7fADcDb5uKVNBf4CPDritVfAXoi4uOSPgB8B1gy3AnW9u3mdx7uGWnMZrk8f+KxrQ7BrOWGvfOPiEeBl2ts+hbwJaByQoAjgZ+m4zYBHZJmFBCnmZkVqKE+f0lnAM9FxNqqTWuBT6R9FgLvBebkitDMzApX91BPSROBK4GTamy+CrhOUg+wHniCrMJnrXaWAksB9psxs94wzMwsh0bG+b8feB+wNnuQlznAGkkLI+J54PMw8JTvP6av3xIRNwE3AYybd2Tr55I0M2sjdSf/iFgPTO9/LWkL0BkRL0qaCuyOiDeBPwUerS7oVssxUybS5Q/hzMyaZiRDPe8CHgfmSdoq6YIhdj8C2ChpE3AK4Bm8zMxGoWHv/CPinGG2d1QsPw4cnj8sMzMrk5/wNTNrQ07+ZmZtqOHkL2mupIcl9UraKOmStP5T6fU+SZ3FhWpmZkXJU9J5D3BZRKyRNAXolrQK2ED2oNffjLShvr71/PSh9+cIxawxSxY/0+oQzFqi4eQfEduB7Wm5T1IvMDsiVgGkZwDMzGwUKqTPX1IHsABYXccxAxO479y5r4gwzMxshHInf0mTySZqv3QkD3T1q5zAfepUf+5sZtZMubKupHFkif+OiLinmJDMzKxsDff5p9o9twC9EXFNniCmTDmaJYu78jRhZmZ1yDPaZxFwLrA+VfGEbDKXCcC3gUOB+yX1RMRH84VpZmZFyjPa5zFgsCE9Kxtt18zMyudPWs3M2pCTv5lZG8o72udWSTskbahYt0JST/raUvF5gJmZjRKKaHwSLUknALuA2yNifo3t3wReiYivD9XOrFmzYunSpQ3HYdYMy5cvb3UIZm8jqTsiGqqhlme0DxHxaHq6t1ZQAj4NLM5zDjMzK16Zff7HAy9ExFO1NlaWd9i9e3eJYZiZWbUyk/85wF2Dbaws7zBx4sQSwzAzs2q5un0GI2ksWVnn48po38zM8ikl+QMfBjZFxNaR7Dxr1ix/mGZm1kR5h3reBTwOzJO0VdIFadPZDNHlY2ZmrZV3tM85g6w/P0+7ZmZWLj/ha2bWhvJ2+0yVdLekTWki9z/wBO5mZqNf3g98rwMeiIhPShoPTAR2UucE7m8+t4uty36WMxSz0W/OVce3OgQzIN9kLgcCJwDnA0TEm8CbZMnfE7ibmY1iebp9DgN+A3xP0hOSbpY0qaC4zMysRHmS/1jgg8B3I2IB8BqwbKQHV5Z3eHn3zhxhmJlZvfIk/63A1ohYnV7fTfbLYEQqyzscPHFqjjDMzKxeDSf/iHge+CdJ89KqJcCvConKzMxKlXe0z38A7kgjfTYDn5f0ceqcwH387MkeBWFm1kR5n/DtAarH8q/EE7ibmY1qfsLXzKwNOfmbmbWhXN0+krYAfcBeYE9EdEpaAfR/CDwV2BkRx+aK0szMClVEPf8TI+LF/hcR8Zn+5f4J3Idr4IXNT/PNz5xeQChm7wyXrbiv1SFYmytrMhdP4G5mNorl7fMP4EFJ3ZKWVm0bcgJ3MzNrnbx3/osiYpuk6cAqSZsi4tG0bcgJ3NMvi6UA75l4QM4wzMysHrnu/CNiW/q+g2xs/0J42wTuK4Y4dqC8w6QJ4/OEYWZmdWo4+UuaJGlK/zJwErAhba5rAnczM2uuPN0+M4CVqW7/WODOiHggbatrAvcZh/2uRz+YmTVRw8k/IjYDxwyy7fxG2zUzs/L5CV8zszbk5G9m1oZKS/6SxqTpHd2Zb2Y2ypT2hC9wCdALHDjcjjue7eM7Fz5UYihm7wwX3egH4q05SrnzlzQHOA24uYz2zcwsn7K6fa4FvgTsK6l9MzPLofDkL+l0YEdEdA+z31JJXZK6dr2+s+gwzMxsCGXc+S8Czki1/r8PLJb0d9U7VZZ3mLz/1BLCMDOzwRSe/CPiyxExJyI6yJ70fSgi/m3R5zEzs8aVOdpnxKa/d4pHOZiZNVGpyT8iHgEeKfMcZmZWPz/ha2bWhpz8zcza0LDJX9KtknZI2lCx7huSNklaJ2mlpKlVx/wrSbskXV5G0GZmlo8iYugdpBOAXcDtETE/rTuJbBTPHklXA0TEFRXH/JDsAa/VEfFfhwti/v4HxN93dDT8jzCz+h2xqbfVIVhOkrojorORY4e9809z8r5cte7BiNiTXv4cmFMRzJnAZmBjIwGZmVn5iujz/wLwExiYzvEK4GsFtGtmZiXJlfwlXQnsAe5Iq74GfCsido3g2IHyDi/v3TPc7mZmVqCGx/lLOg84HVgSb31w8PvAJyX9NTAV2Cfp9Yi4ofr4iLgJuAmyPv9G4zAzs/o1lPwlnUzWvfNHEbG7f31EHF+xz3JgV63EX23/+UdxRFdXI6GYmVkDRjLU8y7gcWCepK2SLgBuAKYAqyT1SLqx5DjNzKxAw975R8Q5NVbfMoLjljcSkJmZlc9P+JqZtSEnfzOzNlRaVc80mUsfsBfY0+hTaGZmVryy6/mfGBEvDrfTxpc2cvRtR5ccipkNZv1561sdgjWZu33MzNpQmck/gAcldUtaWuJ5zMysTmV2+yyKiG2SppM9D7ApFYkDsvIOwFKAcYeMKzEMMzOrVtqdf0RsS993ACuBhVXbb4qIzojoHDNlTFlhmJlZDaXc+afqnvtFRF9aPgn4+mD7H3XIUXSd5/IOZmbNUla3zwxgpaT+c9wZEQ+UdC4zM6tTKck/IjYDx5TRtpmZ5eehnmZmbcjJ38ysDZWS/CXtL+kXktZK2ijJ0zqamY0iZX3g+wawOCJ2SRoHPCbpJxHx85p7b3sClh9UUihmNuotf6XVEbSdsj7wDaB/Ht9x6ctTNZqZjRKl9flLGiOpB9gBrIqI1VXbByZw/81u/14wM2umMp/w3RsRxwJzgIWS5ldtH3jC99CJKisMMzOrofTRPhGxE3gEOLnsc5mZ2ciUVd7hUOBfImKnpAOADwNXD3rArAWw3OUdzMyapazRPjOB2ySNIfvr4gcRcV9J5zIzszqVNdpnHbCgjLbNzCw/P+FrZtaGnPzNzNpQmeP8T5b0pKSnJS0r6zxmZla/skb7jAG+A3wE2Ar8UtK9EfGrWvuvf+4VOpbdX0YoZmaj1parTmvZucu6818IPB0RmyPiTeD7wMdKOpeZmdWprOQ/G/initdb07oBleUd9u52USczs2YqK/nXqtfwtgI+b5vAfaIrepqZNVNZyX8rMLfi9RxgW0nnMjOzOpX1hO8vgcMlvQ94Djgb+JPBdj569kF0tfCDDzOzdlPWE757JF0M/C9gDHBrRGws41xmZla/su78iYgfAz8uq30zM2ucskm3WhyE1Ac82eo4hjENeLHVQQxjtMc42uMDx1iE0R4fvHtifG9EHNpI46Xd+dfpyYjobHUQQ5HU5RjzGe3xgWMswmiPDxwjuLaPmVlbcvI3M2tDoyX539TqAEbAMeY32uMDx1iE0R4fOMbR8YGvmZk112i58zczs2aKiEK+gJPJhms+DSyrsX0CsCJtXw10VGz7clr/JPDR4doE3pfaeCq1Ob7Z8ZGVr3gY6AU2ApdU7L+c7MnmnvR1aguv4RZgfYqjq2L9wcCqdA1XAe9pwTWcV3GNeoBXgUtbcQ2BQ9LPcxdwQ9Uxx6Vr+DRwPW/9xVz3NSwjRmAicD+wKb0Xr6rYdj7wm4rr+KctvI6PpDb7Y5k+3PumiddwStV78UXg2hZdw48A3ek91w0sLuO9OOw/YIT/yDHAM8BhwHhgLXBk1T7/HrgxLZ8NrEjLR6b9J5Al9WdSe4O2CfwAODst3wj8WQvimwl8sOKN8/8q4lsOXN7qa5i2bQGm1TjfX/e/IYFlwNWtiK+q/efJxi234hpOAv4QuJDfTlq/AP6ArGDhT4BTGrmGZcVIlvxPTMvjgZ9VxHh+9b+nhdfxEaCzxvlqttXs+KqO7wZOaNE1XADMSsvzgefKeC8W1e0zkvr9HwNuS8t3A0skKa3/fkS8ERH/SPYbbeFgbaZjFqc2SG2e2ez4ImJ7RKwBiIg+sr8AZtO4Mq7hUCrbask1rDp2CfBMRDw7TBylxBgRr0XEY8DrlTtLmgkcGBGPR/Y/63beulb1XsNSYoyI3RHxcFp+E1hDVkyxUYXHOIzB3jctiU/S4cB0sl+ijcoT4xMR0V8IcyOwv6QJRb8Xi0r+w9bvr9wnIvYAr5D9CTbYsYOtPwTYmdoY7FzNiG+ApA6y39arK1ZfLGmdpFslvWeY+MqMMYAHJXVLWlqxz4yI2J7a2k72Zm9FfP3OBu6qWtfMazhUm1sHabPea1hWjAMkTQX+GPhpxeqz0nW8W9LcQQ5tVozfk9Qj6asVCb6Rn0lp1xA4h+wuvHI0TKuu4VnAExHxBgW/F4tK/sPW7x9in6LWD6WM+LKDpMnAD8n6ql9Nq78LvB84FtgOfHOY+MqMcVFEfBA4BbhI0gkjiKWZ8SFpPHAG8PcV25t9DfO0WY8yYswOksaS/QK9PiI2p9X/k6wv+V8D/5u37g5bEeNnI+Jo4Pj0dW6DbZV2DZPqG5GWXENJRwFXA/+ujjZHrKjkP5L6/QP7pDfpQcDLQxw72PoXgampjcHO1Yz4kDSOLPHfERH39O8QES9ExN6I2Af8d4bvgiktxv4/HyNiB7CyIpYX0p+R/V0bO1oRX3IKsCYiXuhf0YJrOFSblV0olW3Wew3LirHfTcBTEXFt/4qIeCndNUJ2HY9rVYwR8Vz63gfcyVs/00Z+JqVcQ0nHAGMjorsi7qZfQ0lzyP6/fi4inqnYv7D3YlHJf6B+f7qLOxu4t2qfe4Hz0vIngYfSn1X3AmenPq33AYeTfahRs810zMOpDVKbP2p2fOlP1luA3oi4prKh/h9C8nFgwzDxlRXjJElTUkyTgJMqYqlsqyXXsOK4c6jq8mnBNawp/QndJ+nfpJ/553jrWtV7DUuJEUDSfyZLHpdWra+8jmeQfTbV9BgljZU0LS2PA06n9ntxJP/eUq5hMtx7sfRrmLru7ge+HBH/p3/nwt+Lg30SXO8XcCrZiJdngCvTuq8DZ6Tl/cn+rH+a7D/+YRXHXpmOe5L06fVgbab1h6U2nk5tTmh2fGQjBgJYR9VwROB/kA3HWpd+KDNbcQ3TdVqbvjZWXcNDyPqFn0rfD27Rz3gi8BJwUNW5WnENt5Ddee0iu8vqH73VSZaongFu4K3hdXVfwzJiJLsDDLKk9LbhiMB/ST/7tWQ3TR9oUYyTyEbQrEvxXMdbI9IGbauZP+e0bXP1NWr2NQT+HHiNtw897R8WW9h70U/4mpm1IT/ha2bWhpz8zczakJO/mVkbcvI3M2tDTv5mZm3Iyd/MrA05+ZuZtSEnfzOzNvT/AQruS/BWEV+/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the most important features for data\n",
    "\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "model = ExtraTreesClassifier()\n",
    "model.fit(X,y)\n",
    "\n",
    "imp_features = pd.Series(model.feature_importances_, index=X.columns)\n",
    "imp_features.nlargest(20).plot(kind='barh')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale and normalise the data for testing \n",
    "\n",
    "from sklearn import preprocessing\n",
    "\n",
    "X_normal = preprocessing.normalize(X, norm='l2') #preprocessing.scale(X_data)\n",
    "X_scaled = preprocessing.scale(X)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG8JJREFUeJzt3X+UHWWd5/H3h/wAOoT8ICQmkLEBMYOQJUgvjsuAIREEdMBfKDBHwllmW3ZkV+bokbgcjxl3ZxdFRGZxxShomAEGRHJghiGQFTAyItqJ+UmC/NiQ6SQmhmygA0fYJN/9o55Ortd7+96+datvcvN5ndPnVj9V9dS3i+bblaee+pYiAjMza1+HtDoAMzMrlhO9mVmbc6I3M2tzTvRmZm3Oid7MrM050ZuZtTknejOzNudEb2bW5pzozcza3PBWBwAwYcKE6OzsbHUYZmYHlKVLl26LiKNrbbdfJPrOzk56enpaHYaZ2QFF0sv1bJdr6EbSekmrJC2X1JPa5knamNqWS7owzzHMzCyfZlzRnxMR28rabo6Ir9fbQV/fKn78+AlNCMUOdrNnvdjqEMz2O74Za2bW5vIm+gAek7RUUndJ+zWSVkq6Q9K4nMcwM7Mc8ib6MyPi3cAFwGcknQ18GzgBmAFsBm6qtKOkbkk9knp27NiTMwwzM6smV6KPiE3pcyuwEDgjIrZExO6I2AN8Fzijyr7zI6IrIrrGjvUIkplZURrOsJJGSRrdvwycB6yWNLlks48Aq/OFaGZmeeSZdTMJWCipv5+7I2KRpL+TNINs/H498OlaHY0ePZ3ZszyP3sysCA0n+oh4CTi1QvunckVkZmZN5cFxM7M250RvZtbmcj8ZK2k90AfsBnZFRJek8cC9QCfZOP0nIuL/5j2WmZkNniIiXwdZou8qLYMg6WvA9oi4QdJcYFxEXFetjylTpkR3d3e11WZ1mzdvXqtDMBsykpZGRFet7YoaurkYWJCWFwAfLug4ZmZWQzMSfaUyCJMiYjNA+pzYhOOYmVkDmlG98syI2CRpIrBY0rp6dkp/FLoBxowZ04QwzMysktxX9JXKIABb+p+QTZ9bK+y3twRCR0dH3jDMzKyKXDdjU+mDQyKiLy0vBr4CzAZeKbkZOz4ivlCtn66urvAbpszMBqfem7F5h26qlUH4JXCfpKuADcAlOY9jZmYNypXoByiD8ArZVb2ZmbWYn4w1M2tzTvRmZm0uTz36qZKekLRW0hpJny1b/3lJIWlC/jDNzKxRecbodwGfi4hl6QUkSyUtjohnJU0FziW7EVvTWxt30jv3pzlCMavs2BvOanUIZi3X8BV9RGyOiGVpuQ9YCxyTVt8MfIHsqVkzM2uhpozRS+oETgOekXQRsDEiVjSjbzMzy6cZZYqPAH4EXEs2nHM92ftja+23twTCMUdOyhuGmZlVkeuKXtIIsiR/V0Q8AJwAHAesSOWLjwWWSXpb+b6lJRDGd4zNE4aZmQ2g4St6ZY/D3g6sjYhvAETEKkoqVVaqVV/JyGOO8E0zM7OC5LmiPxP4FDBL0vL0dWGT4jIzsyZp+Io+Ip4CVGObzkb7NzOz5vCTsWZmbc6J3syszeW5GTsVuBN4G7AHmB8Rt0i6F5iWNhsL7IiIGbkjNTOzhhRRAuGT/RtIugl4tVZHW156gZs++aEcoZhV97l7/6nVIZi1VJ6bsZuB/heA90nqL4HwLOydfvkJYFYT4jQzswY1vQRCSfNZwJaIeL7KPt2SeiT1vP7mW80Iw8zMKsid6EtLIETEayWrLgPuqbZf6ZOxow4dmTcMMzOrIletmwolEPrbhwMfBU7PF56ZmeXV1BIIJd4PrIuI3nr6mnT8O3zDzMysIEWVQLiUAYZtzMxs6BRSAiEirmy0XzMzay4/GWtm1uac6M3M2lzNRC/pDklbJa2usO7zkkLShJK2mWm8fo2knzQ7YDMzG5x6xuh/ANxKVtdmr1Tr5lxgQ0nbWOB/AedHxAZJE6nD1pf7+NbVj9cbs1lTfOY2P7RtB4eaV/QRsQTYXmHVzcAXgChpuxx4ICI2pH23NiNIMzNrXENj9JIuAjZGxIqyVe8Exkl6UtJSSVcM0MfeEgg7f7ejkTDMzKwOg55eKakDuB44r0p/pwOzgcOBpyX9PCJ+Xb5hRMwH5gP80dHTony9mZk1RyPz6E8AjgNWZA/HciywTNIZQC+wLSJeB16XtAQ4FfiDRG9mZkNj0Ik+IlYBe2+ySloPdEXENkkPAremWjcjgfeQjeUPaOLbR/vGmJlZQeqZXnkP8DQwTVKvpKuqbRsRa4FFwErgF8D3IuIPpmWamdnQqXlFHxGX1VjfWfb9jcCN+cIyM7Nm8ZOxZmZtLleir/TUrKRL0lOxeyR15Q/RzMzyyPXiESo/Nbua7KUj36m3k9+tXsPaPz4pZyhmg3fSurWtDsGscLkSfUQsSe+LLW1bC5CmXpqZWYt5jN7MrM21LNGXlkDYvntXq8IwM2t7LUv0ETE/Iroiomv8sLy3CszMrBoP3ZiZtblcl9LpqdmZwARJvcCXyUoa/0/gaOBhScsj4gMD9XPYKSdzUk9PnlDMzKyKvLNuqj01uzBPv2Zm1jweujEza3NO9GZmba6hl4NXK3MgaaSk70taJWmFpJkFxW1mZnVq9OXg1coc/AeAiJieXgz+iKR/GxF7BjrAmlfWMH3B9LqDNmuWVXNWtToEs8I19HLwiFgbEc9V2PxdwI/TNluBHYALm5mZtVCzx+hXABdLGi7pOLL3x05t8jHMzGwQmv1I6h3ASUAP8DLwM6BifQNJ3UA3wIijRjQ5DDMz69fURB8Ru4C/6v9e0s+A56tsOx+YD3D4cYdHM+MwM7N9mjp0I6lD0qi0fC6wKyKebeYxzMxscGpe0Q+yzMFE4FFJe4CNwKfqCeLko06mZ45LIJiZFSHPy8H/oMxBRKwHpuWMyczMmshPxpqZtTknejOzNtdwopc0VdITktamcgifTe3jJS2W9Hz6HNe8cM3MbLAU0djMRkmTgckRsUzSaGAp8GHgSmB7RNwgaS4wLiKuG6ivrinDoqf7iIbiMGuqea+2OgKzuklaGhE1qw80fEUfEZsjYlla7gPWAscAFwML0mYLyJK/mZm1SFPG6CV1AqcBzwCTImIzZH8MyKZcmplZi+RO9JKOAH4EXBsRrw1iv25JPZJ6fvuGH4w1MytKrkQvaQRZkr8rIh5IzVvS+H3/OP7WSvtGxPyI6IqIrqM7lCcMMzMbQJ5ZNwJuB9ZGxDdKVj0EzEnLc4AHGw/PzMzyyjPr5k+BnwKrgP4Xi/wXsnH6+4A/AjYAl0TE9oqdJF1dXdHT4xIIZmaDUe+sm4arV0bEU0C1MZfZjfZrZmbN5SdjzczanBO9mVmbq5noJd0haauk1SVt8yRtlLQ8fV2Y2s+VtFTSqvQ5q8jgzcystnrG6H8A3ArcWdZ+c0R8vaxtG/BnEbFJ0inAo2RPyw5o1cZX6Zz7cB2hmBVv/Q0fbHUIZk1VTz36JenJ15oi4lcl364BDpN0aES82Vh4ZmaWV54x+mskrUxDO5UqVH4M+JWTvJlZazWa6L8NnADMADYDN5WulHQy8FXg09U6KC2BsPsNVww0MytKQ4k+IrZExO6I2AN8Fzijf52kY8leM3hFRLw4QB97SyAM6xjTSBhmZlaHhh6YkjS5v0Il8BFgdWofCzwMfDEi/qXe/qYfM4Ye3wAzMytEzUQv6R5gJjBBUi/wZWCmpBlAAOvZN0RzDfAO4EuSvpTazouIioXNzMyseA3Xumkm17oxMxu8wt8wZWZmBwYnejOzNtdoCYQbJa1L8+gXppuwSBohaUEqgbBW0heLDN7MzGqrOUYv6WxgJ3BnRJyS2s4DHo+IXZK+ChAR10m6HLgoIi6V1AE8C8yMiPUDHePQySfG5DnfzP/TmLWASyZYqzRtjD4ilgDby9oei4hd6dufA8f2rwJGSRoOHA68BdT9HlkzM2u+ZozR/3vgkbR8P/A62dOyG4Cv13q7lJmZFSvvy8GvB3YBd6WmM4DdwBTgOOBzko6vsq9LIJiZDYE8LwefA3wI+PPYN9B/ObAoIv5fekjqX4CK40cugWBmNjQaLYFwPnAd8L6IeKNk1QZglqS/BzqAPwFq3mV1CQQzs+LUM73yHuBpYJqkXklXkb2IZDSwOL1h6ra0+beAI8hq3/wS+H5ErCwmdDMzq0c9Lx65rELz7VW23QlckjcoMzNrHj8Za2bW5pzozczaXO5EL2l9KnmwXFJP2brPSwpJE/Iex8zMGtPQrJsKzomIbaUNkqYC55LNxBnQir43eNsTy5sUitnQ+805M1odgllVRQ7d3Ax8gawsgpmZtUgzEn0Aj0laKqkbQNJFwMaIWNGE/s3MLIdmDN2cGRGbJE0km1e/DrgeOG+gndIfhW6AQyZNbkIYZmZWSe4r+ojYlD63AguB95HVuVkhaT1ZZctlkt5Wtt/eEgiHjBmbNwwzM6si1xW9pFHAIRHRl5bPA74SERNLtlkPdJXfrC116ugOenwzy8ysEHmHbiYBCyX193V3RCzKHZWZmTVNrkQfES8Bp9bYpjPPMczMLB8/GWtm1uac6M3M2lzeN0zdIWmrpNUlbfemcgjLU3kEP/JqZtZCeW/G/oCsNv2d/Q0R8cn+ZUk3ATXfE9jXt4ofP35CzlDM9i+zZ73Y6hDMgPw3Y5dI6qy0TtlUnE8As/Icw8zM8ilyjP4sYEtEPF9pZenLwXfs2FNgGGZmB7ciE/1lwD3VVpY+GTt2rO8Jm5kVpVllin+PpOHAR4HTi+jfzMzqV0iiB94PrIuI3no2Hj16OrNn9dTe0MzMBi3v9Mp7gKeBaZJ6JV2VVl3KAMM2ZmY2dPLOurmsSvuVefo1M7Pm8V1QM7M250RvZtbmaib6KmUO/quklanMwWOSppSsm5na10j6SVGBm5lZfRQx8Lu7JZ0N7ATujIhTUtuREfFaWv7PwLsi4mpJY4GfAedHxAZJE9ObpwY0ZcqU6O7uzvuzmO135s2b1+oQrI1JWhoRXbW2q3lFHxFLgO1lba+VfDuK7AXhAJcDD0TEhrRdzSRvZmbFanjWjaS/Aa4gK1p2Tmp+JzBC0pPAaOCWiLizyv57Xw4+ZsyYRsMwM7MaGr4ZGxHXR8RU4C7gmtQ8nOxp2A8CHwC+JOmdVfbfWwKho6Oj0TDMzKyGZsy6uRv4WFruBRZFxOvpZeBLqPGqQTMzK1ZDQzeSTiypSnkRsC4tPwjcmmrdjATeA9xcq78pU6b4ppWZWUFqJvpU5mAmMEFSL/Bl4EJJ04A9wMvA1QARsVbSImBlWve9iFhdsWMzMxsSNadXDoWurq7o6XFRMzOzwWja9EozMzuwFZLoJU0reUH4ckmvSbq2iGOZmdnACqlHHxHPATMAJA0DNgILq23/1sad9M79aRGhmO0Xjr3hrFaHYAexoRi6mQ28GBEvD8GxzMyszFAker+ExMyshQpN9JJGks2z/2GFdd2SeiT1bH9jR5FhmJkd1Iq+or8AWBYRW8pXlJZAGN8xtuAwzMwOXkUn+svwsI2ZWUsVMusGQFIHcC7w6VrbjjzmCM9KMDMrSGGJPiLeAI4qqn8zM6uPn4w1M2tzTvRmZm2uqBIIh0n6haQV6SXhf13EcczMrLaixujfBGZFxE5JI4CnJD0SET+vtPGWl17gpk9+qKBQzPZPn7v3n1odgh0kiqp1E8DO9O2I9NX6eshmZgehwsboJQ2TtBzYCiyOiGeKOpaZmVVXWKKPiN0RMQM4FjhD0iml60tLILz+5ltFhWFmdtArfNZNROwAngTOL2vfWwJh1KEjiw7DzOygVdSsm6MljU3LhwPvZ98LxM3MbAgVNetmMrAgvXTkEOC+iKg6xWDS8e/wDAQzs4IUNetmJXBaEX2bmdng+MlYM7M250RvZtbmiixTvB7oA3YDuyKiq6hjmZlZdYUl+uSciNhWa6OtL/fxrasfLzgUswPDZ26b1eoQrM146MbMrM0VmegDeEzSUkndBR7HzMwGUOTQzZkRsUnSRGCxpHURsaR/ZUr+3QDjjphYYBhmZge3ImvdbEqfW4GFwBll6/eWQDjisLFFhWFmdtArqgTCKEmj+5eB84DVRRzLzMwGVtTQzSRgoaT+Y9wdEYuqbTzx7aM908DMrCBFlUB4CTi1iL7NzGxwPL3SzKzNOdGbmbW5mole0h2StkpaXdI2XtJiSc+nz3El62ZKWi5pjaSfFBW4mZnVR9l7vAfYQDqb7EXfd0bEKanta8D2iLhB0lxgXERcl1428jPg/IjYIGliml45oFMOOzx+2NmZ92cxs+SkdWtbHYINAUlL66kjVvOKPj3ktL2s+WJgQVpeAHw4LV8OPBARG9K+NZO8mZkVq9Ex+kkRsRkgffY/2vpOYJykJ1PpgyuaEaSZmTWu2dMrhwOnA7OBw4GnJf08In5dvmFpCYTJw4suomlmdvBq9Ip+i6TJAOmzf4imF1gUEa+n8sRLqDKfvrQEwvhhTvRmZkVpNMM+BMwBbkifD6b2B4FbJQ0HRgLvAW6u1dlhp5zMST09DYZiZmYDqZnoJd0DzAQmSOoFvkyW4O+TdBWwAbgEICLWSloErAT2AN+LCNe4MTNroZqJPiIuq7JqdpXtbwRuzBOUmZk1j5+MNTNrc070ZmZtLvd0F0nrgT5gN7ArIrokzQBuAw4DdgF/GRG/yHssMzMbvGbNazwnTafs9zXgryPiEUkXpu9nVtt5zStrmL5gepNCMTOAVXNWtToE208UNXQTwJFpeQywqaDjmJlZDc24og/gMUkBfCci5gPXAo9K+jrZH5N/14TjmJlZA5qR6M+MiE2SJgKLJa0DPg78VUT8SNIngNuB95fuVFoCYcRRI5oQhpmZVZJ76CYiNqXPrcBC4Ayyp2UfSJv8MLWV77e3BMKw0cPyhmFmZlXkuqKXNAo4JCL60vJ5wFfIxuTfBzwJzAKeH6ifk486mZ45LoFgZlaEvEM3k4CFkvr7ujsiFknaCdySat78jjREY2ZmQy9Xoo+Il6hQnTIiniIrV2xmZi3mJ2PNzNqcE72ZWZvLlegljZV0v6R1ktZKeq+kSyStkbRHUs2X1pqZWbHy3oy9heyNUh+XNBLoAHYAHwW+U3cvm34F88bkDMXMrEDzXm11BA1rONFLOhI4G7gSICLeAt4iS/SkmThmZtZieYZujgd+C3xf0q8kfS/Npa+LpG5JPZJ6fvtG5AjDzMwGkifRDwfeDXw7Ik4DXgfm1rtz6ZOxR3f46t/MrCh5En0v0BsRz6Tv7ydL/GZmth9peIw+In4j6V8lTYuI58jeIftsQ51NOQ3muQSCmVkR8s6j/0/AXZJWAjOA/y7pI5J6gfcCD0t6NG+QZmbWuLwlEJYD5XPlF6YvMzPbD/jJWDOzNudEb2bW5gpL9JLOl/ScpBck1T3t0szMmqsZrxL8A5KGAd8CziWbhvlLSQ9FRMVZOas2vkrn3IeLCMXMbL+1/oYPDslxirqiPwN4ISJeSqUR/gG4uKBjmZnZAIpK9McA/1ryfW9q26u0BMLuNw7cYkFmZvu7ohJ9pZoGv1fQ5vdeDt7hypVmZkUpKtH3AlNLvj+W7IXhZmY2xAq5GQv8EjhR0nHARuBS4PJqG08/Zgw9Q3RTwszsYFNIoo+IXZKuAR4FhgF3RMSaIo5lZmYDK+qKnoj4Z+Cfi+rfzMzqo4jWv/RDUh/wXKvjGIQJwLZWBzEIB1K8B1KscGDFeyDFCo63Hm+PiKNrbVTYFf0gPRcRB8yLxCX1ON5iHEixwoEV74EUKzjeZnKtGzOzNudEb2bW5vaXRD+/1QEMkuMtzoEUKxxY8R5IsYLjbZr94masmZkVZ3+5ojczs4I0LdHXqj8v6VBJ96b1z0jqLFn3xdT+nKQP1OpT0nGpj+dTnyNbGaukqZKekLRW0hpJny3Zfp6kjZKWp68LBxNrEfGm9vWSVqWYekrax0tanM7tYknjWhmrpGkl5265pNckXZvWtezcSjoq/TffKenWsn1OT+f2BUl/K0mpPde5LSJeSR2SHpa0Lv3u3lCy7kpJvy05v3/RyljTuidTn/0xTRyor1bGK2l02e/uNknfTOtyndtBi4jcX2RPv74IHA+MBFYA7yrb5i+B29LypcC9afldaftDgeNSP8MG6hO4D7g0Ld8G/McWxzoZeHfaZjTw65JY5wGf35/ObVq3HphQ4XhfA+am5bnAV1sda1n/vyGbO9zqczsK+FPgauDWsn1+AbyXrLjfI8AFec9tUfECHcA5aXkk8NOSeK8s/9n2g3P7JNBV4XgV+2p1vGX7LwXOzntuG/lq1hV9PfXnLwYWpOX7gdnpSudi4B8i4s2I+D/AC6m/in2mfWalPkh9friVsUbE5ohYBhARfcBaysoy51DEuR1IaV8tP7dl+84GXoyIlwcRUyHxRsTrEfEU8LvSjSVNBo6MiKcj+z/6TvadwzzntpB4I+KNiHgiLb8FLCMrQphX02Otodrv1X4Rr6QTgYlkf0iHXLMSfc3686XbRMQu4FXgqAH2rdZ+FLAj9VHtWEMd617pn3OnAc+UNF8jaaWkOxr453pR8QbwmKSlkrpLtpkUEZtTX5vJfjlbHWu/S4F7ytpadW4H6rO3Sp95zm1R8e4laSzwZ8CPS5o/ls7v/ZKmVtl1qGP9fhru+FJJMm/45x6CeAEuI/sXQOnsl0bP7aA1K9HXrD8/wDbNaq9XEbFmO0lHAD8Cro2I11Lzt4ETgBnAZuCmQcRaZLxnRsS7gQuAz0g6e5BxVVLkuR0JXAT8sGR9K89tnj4bVUS82U7ScLI/on8bES+l5n8EOiPi3wD/m31Xs62M9c8jYjpwVvr6VI6+BhtLnmOUX6TkObeD1qxEX0/9+b3bpF+qMcD2Afat1r4NGJv6qHasoY4VSSPIkvxdEfFA/wYRsSUidkfEHuC71B46GZJ4I6L/cyuwsCSuLWn4oX8YYmurY00uAJZFxJb+hhaf24H6LB36KO0zz7ktKt5+84HnI+Kb/Q0R8UpEvJm+/S5weqtjjYiN6bMPuJt9/80b/bkLjTdteyowPCKWlvwcec7toDUr0e+tP5+uvC4FHirb5iFgTlr+OPB4+mfMQ8Cl6Y72ccCJZDezKvaZ9nki9UHq88FWxpr++Xg7sDYivlHaUf//2MlHgNWDiLWoeEdJGp3iGwWcVxJXaV8tP7cl+11G2bBNi89tRWlIpk/Sn6TfiyvYdw7znNtC4gWQ9N/Ikta1Ze2l5/cisntPLYtV0nBJE9LyCOBDVP69revnLjreErV+dwd7bgevWXd1gQvJZpu8CFyf2r4CXJSWDyP7Z/cLZP8DH1+y7/Vpv+dId/yr9Znaj099vJD6PLSVsZLdcQ9gJbA8fV2Y1v0dsCqtewiY3Opzm87fivS1puzcHkU2Rvt8+hy/H/wedACvAGPKjtXqc7ue7IpuJ9nVXv9Mqy6yBPQicCv7HkzMdW6LiJfsyjXIEk3/7+5fpO3/R/r9WEF2cfXHLY51FNnMlZUprlvYN4usal+t/F1I614qP3d5z+1gv/xkrJlZm/OTsWZmbc6J3syszTnRm5m1OSd6M7M250RvZtbmnOjNzNqcE72ZWZtzojcza3P/H0GjllAyXzNyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the most important features for scaled data\n",
    "\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "model = ExtraTreesClassifier()\n",
    "model.fit(X_scaled,y)\n",
    "\n",
    "imp_features = pd.Series(model.feature_importances_, index=X.columns)\n",
    "imp_features.nlargest(20).plot(kind='barh')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.23302180685358256\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC  \n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import hinge_loss\n",
    "\n",
    "svclassifier = SVC(degree=6)  \n",
    "\n",
    "svclassifier.fit(X_train_scaled, y_train)  \n",
    "\n",
    "predicted = svclassifier.predict(X_test_scaled)\n",
    "\n",
    "score = accuracy_score(predicted, y_test)\n",
    "\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7133956386292835\n"
     ]
    }
   ],
   "source": [
    "#Import Random Forest Model\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "#Create a Random Forest Classifier\n",
    "clf=RandomForestClassifier(n_estimators=150)\n",
    "\n",
    "#Train the model using the training sets y_pred=clf.predict(X_test)\n",
    "clf.fit(X_train,y_train)\n",
    "\n",
    "predicted=clf.predict(X_test)\n",
    "\n",
    "score = accuracy_score(predicted, y_test)\n",
    "\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.42679127725856697\n"
     ]
    }
   ],
   "source": [
    "# Create base model using Gaussian Naive Bayes \n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "model_nb = GaussianNB()\n",
    "\n",
    "model_nb.fit(X_train,y_train)\n",
    "\n",
    "predicted= model_nb.predict(X_test)\n",
    "\n",
    "score = accuracy_score(predicted, y_test)\n",
    "\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "onehotencoder = OneHotEncoder()\n",
    "\n",
    "X_train =np.expand_dims(X_train, axis=2)\n",
    "X_test= np.expand_dims(X_test, axis=2)\n",
    "\n",
    "y_train = onehotencoder.fit_transform(np.array(y_train).reshape(-1,1)).toarray()\n",
    "y_test = onehotencoder.fit_transform(np.array(y_test).reshape(-1,1)).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6416 samples, validate on 1605 samples\n",
      "Epoch 1/20\n",
      "6416/6416 [==============================] - 31s 5ms/step - loss: 1.7031 - acc: 0.3932 - val_loss: 1.0992 - val_acc: 0.5396\n",
      "Epoch 2/20\n",
      "6416/6416 [==============================] - 31s 5ms/step - loss: 1.0132 - acc: 0.5729 - val_loss: 0.9113 - val_acc: 0.6224\n",
      "Epoch 3/20\n",
      "6416/6416 [==============================] - 32s 5ms/step - loss: 0.9005 - acc: 0.6166 - val_loss: 0.8437 - val_acc: 0.6486\n",
      "Epoch 4/20\n",
      "6416/6416 [==============================] - 32s 5ms/step - loss: 0.8439 - acc: 0.6392 - val_loss: 0.8146 - val_acc: 0.6498\n",
      "Epoch 5/20\n",
      "6416/6416 [==============================] - 33s 5ms/step - loss: 0.8092 - acc: 0.6555 - val_loss: 0.8302 - val_acc: 0.6492\n",
      "Epoch 6/20\n",
      "6416/6416 [==============================] - 32s 5ms/step - loss: 0.7838 - acc: 0.6682 - val_loss: 0.7770 - val_acc: 0.6822\n",
      "Epoch 7/20\n",
      "6416/6416 [==============================] - 36s 6ms/step - loss: 0.7540 - acc: 0.6789 - val_loss: 0.7701 - val_acc: 0.6760\n",
      "Epoch 8/20\n",
      "6416/6416 [==============================] - 35s 5ms/step - loss: 0.7312 - acc: 0.6892 - val_loss: 0.7533 - val_acc: 0.6729\n",
      "Epoch 9/20\n",
      "6416/6416 [==============================] - 37s 6ms/step - loss: 0.7104 - acc: 0.7003 - val_loss: 0.7579 - val_acc: 0.6798\n",
      "Epoch 10/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.6934 - acc: 0.7109 - val_loss: 0.7351 - val_acc: 0.6891\n",
      "Epoch 11/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.6702 - acc: 0.7173 - val_loss: 0.7335 - val_acc: 0.6891\n",
      "Epoch 12/20\n",
      "6416/6416 [==============================] - 37s 6ms/step - loss: 0.6501 - acc: 0.7302 - val_loss: 0.7125 - val_acc: 0.7028\n",
      "Epoch 13/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.6368 - acc: 0.7357 - val_loss: 0.7249 - val_acc: 0.6978\n",
      "Epoch 14/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.6150 - acc: 0.7477 - val_loss: 0.7307 - val_acc: 0.6854\n",
      "Epoch 15/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.6012 - acc: 0.7526 - val_loss: 0.7449 - val_acc: 0.6953\n",
      "Epoch 16/20\n",
      "6416/6416 [==============================] - 39s 6ms/step - loss: 0.5806 - acc: 0.7642 - val_loss: 0.7557 - val_acc: 0.6673\n",
      "Epoch 17/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.5693 - acc: 0.7703 - val_loss: 0.7642 - val_acc: 0.6984\n",
      "Epoch 18/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.5533 - acc: 0.7729 - val_loss: 0.7209 - val_acc: 0.7059\n",
      "Epoch 19/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.5378 - acc: 0.7816 - val_loss: 0.7175 - val_acc: 0.6991\n",
      "Epoch 20/20\n",
      "6416/6416 [==============================] - 38s 6ms/step - loss: 0.5274 - acc: 0.7848 - val_loss: 0.7007 - val_acc: 0.7178\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1b8d757e7f0>"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv1D, Flatten,Activation, Dropout,MaxPooling1D\n",
    "import keras\n",
    "\n",
    "#create model\n",
    "model = Sequential()\n",
    "#add model layers\n",
    "\n",
    "model.add(Conv1D(200, 5, activation='relu', padding='same', input_shape=(181,1)))\n",
    "model.add(Conv1D(100, 5, activation='relu', padding='same'))\n",
    "model.add(Dropout(0.1))\n",
    "\n",
    "model.add(Conv1D(100, 5, activation='relu', padding='same'))\n",
    "\n",
    "model.add(Conv1D(100, 5, activation='relu', padding='same'))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(4, activation='softmax'))\n",
    "opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)\n",
    "\n",
    "#compile model using accuracy to measure model performance\n",
    "model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "#train the model\n",
    "model.fit(X_train, y_train, batch_size=8, epochs=20, validation_data=(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved trained model at saved_models\\Model_Emotion_Speech.h5 \n"
     ]
    }
   ],
   "source": [
    "model_name = 'Model_Emotion_Speech.h5'\n",
    "save_dir = 'saved_models'\n",
    "\n",
    "# Save model and weights\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "model_path = os.path.join(save_dir, model_name)\n",
    "model.save(model_path)\n",
    "print('Saved trained model at %s ' % model_path)\n",
    "\n",
    "import json\n",
    "model_json = model.to_json()\n",
    "with open(save_dir+\"/model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}