downstream_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python3.5/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, Embedding, Input, Flatten\n",
    "from keras.layers import Conv1D, MaxPooling1D, GlobalMaxPool1D, Dropout, concatenate\n",
    "from keras.preprocessing import text as keras_text, sequence as keras_seq\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "np.random.seed(512)\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn import preprocessing\n",
    "from keras.layers import LeakyReLU\n",
    "from keras import initializers\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.models import load_model\n",
    "from keras.utils import to_categorical\n",
    "import keras\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# categories = {'Usable Directly Categorical':    2, \n",
    "#               'Usable Directly Numeric':        0, \n",
    "#               'Usable With Extraction':         1, \n",
    "#               'Unusable':                       3, \n",
    "#               'Context_Specific':               4, \n",
    "#               }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = {'Usable Directly Categorical':    0, \n",
    "              'Usable Directly Numeric':        1, \n",
    "              'Usable With Extraction':         2, \n",
    "              'Unusable':                       3\n",
    "#               'Context_Specific':               4, \n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('results/train_raw_4.csv')\n",
    "labels = pd.read_csv('results/train_marginals_4.csv',header=None).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8691, 4), (8691, 16))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.shape, data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(labels.shape[1]):\n",
    "    data[str(i)] = labels[:,i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['y_act'] = data['y_act'].map(categories)\n",
    "data_LSTM = pd.concat([data['Attribute_name'], data['sample_1'], data['sample_2'], data['sample_3'], data['sample_4'], data['sample_5']], axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.rename(columns={'num of dist_val': 'num_of_dist_val'})\n",
    "data['p_dist_val'] = data.num_of_dist_val / data.Total_val\n",
    "data['p_nans'] = data['% nans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_castability(dat):\n",
    "    sample_values = dat['sample_1':'sample_5']\n",
    "    #make sure the value you are avaluating is not nan\n",
    "    castability = 0\n",
    "    for values in sample_values:\n",
    "        if pd.isnull(values) == False and values.__class__.__name__ == 'str':\n",
    "            try:\n",
    "                castability = str(type(eval(values)))\n",
    "                castability = 1\n",
    "                break\n",
    "            except:\n",
    "                #print('passing')\n",
    "                pass\n",
    "    return castability  \n",
    "\n",
    "def get_extractability(dat, cast):\n",
    "    sample_values = dat['sample_1':'sample_5']\n",
    "    #make sure the value you are avaluating is not nan\n",
    "    extractability = 0\n",
    "    for values in sample_values:\n",
    "        if pd.isnull(values) == False and values.__class__.__name__ == 'str' and cast == 0:\n",
    "            try:\n",
    "                #it will faile when you have no numbers or if you have two numbers seperated by space\n",
    "                float(re.sub('[^0-9\\. ]', ' ',values))\n",
    "                #print('yes')\n",
    "                extractability = 1\n",
    "                break\n",
    "            except:\n",
    "                pass\n",
    "    return extractability\n",
    "\n",
    "def get_len(dat):\n",
    "    sample_values = dat['sample_1':'sample_5']\n",
    "    #make sure the value you are avaluating is not nan\n",
    "    length = 0\n",
    "    for values in sample_values:\n",
    "        if pd.isnull(values) == False:\n",
    "            length = len(str(values).split()) + length\n",
    "    return length/5.0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8691/8691 [00:10<00:00, 793.39it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "castability = []\n",
    "extractability = []\n",
    "len_val = []\n",
    "for i in tqdm(range(len(data))):\n",
    "    castability.append(get_castability(data.loc[i]))\n",
    "    extractability.append(get_extractability(data.loc[i], castability[i]))\n",
    "    len_val.append(get_len(data.loc[i]))\n",
    "data['castability'] = castability\n",
    "data['extractability'] = extractability\n",
    "data['len_val'] = len_val  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Unnamed: 0',\n",
       " 'y_act',\n",
       " 'Reason',\n",
       " 'Attribute_name',\n",
       " 'Total_val',\n",
       " 'num_of_dist_val',\n",
       " '% nans',\n",
       " 'max_val',\n",
       " 'mean',\n",
       " 'min_val',\n",
       " 'std_dev',\n",
       " 'sample_1',\n",
       " 'sample_2',\n",
       " 'sample_3',\n",
       " 'sample_4',\n",
       " 'sample_5',\n",
       " '0',\n",
       " '1',\n",
       " '2',\n",
       " '3',\n",
       " 'p_dist_val',\n",
       " 'p_nans',\n",
       " 'castability',\n",
       " 'extractability',\n",
       " 'len_val']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num_of_dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>...</th>\n",
       "      <th>sample_5</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>p_dist_val</th>\n",
       "      <th>p_nans</th>\n",
       "      <th>castability</th>\n",
       "      <th>extractability</th>\n",
       "      <th>len_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.785551</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>Stratification2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.785551</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory3</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.785551</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>Stratification3</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.071483</td>\n",
       "      <td>0.785551</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>c</td>\n",
       "      <td>GeoLocation</td>\n",
       "      <td>403984</td>\n",
       "      <td>55</td>\n",
       "      <td>0.006357</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>(37.63864012300047, -120.99999953799971)</td>\n",
       "      <td>0.095930</td>\n",
       "      <td>0.027548</td>\n",
       "      <td>0.848974</td>\n",
       "      <td>0.027548</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>0.006357</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  y_act Reason           Attribute_name  Total_val  \\\n",
       "0           0      3      l  StratificationCategory2     403984   \n",
       "1           1      3      l          Stratification2     403984   \n",
       "2           2      3      l  StratificationCategory3     403984   \n",
       "3           3      3      l          Stratification3     403984   \n",
       "4           4      2      c              GeoLocation     403984   \n",
       "\n",
       "   num_of_dist_val    % nans  max_val  mean  min_val   ...     \\\n",
       "0                1  0.803648      0.0   0.0      0.0   ...      \n",
       "1                1  0.803648      0.0   0.0      0.0   ...      \n",
       "2                1  0.803648      0.0   0.0      0.0   ...      \n",
       "3                1  0.803648      0.0   0.0      0.0   ...      \n",
       "4               55  0.006357      0.0   0.0      0.0   ...      \n",
       "\n",
       "                                   sample_5         0         1         2  \\\n",
       "0                                       NaN  0.071483  0.071483  0.071483   \n",
       "1                                       NaN  0.071483  0.071483  0.071483   \n",
       "2                                       NaN  0.071483  0.071483  0.071483   \n",
       "3                                       NaN  0.071483  0.071483  0.071483   \n",
       "4  (37.63864012300047, -120.99999953799971)  0.095930  0.027548  0.848974   \n",
       "\n",
       "          3 p_dist_val    p_nans  castability  extractability  len_val  \n",
       "0  0.785551   0.000002  0.803648            0               0      0.0  \n",
       "1  0.785551   0.000002  0.803648            0               0      0.0  \n",
       "2  0.785551   0.000002  0.803648            0               0      0.0  \n",
       "3  0.785551   0.000002  0.803648            0               0      0.0  \n",
       "4  0.027548   0.000136  0.006357            1               0      2.0  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('processed_raw_4.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## After prepossessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, Embedding, Input, Flatten\n",
    "from keras.layers import Conv1D, MaxPooling1D, GlobalMaxPool1D, Dropout, concatenate\n",
    "from keras.preprocessing import text as keras_text, sequence as keras_seq\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "np.random.seed(512)\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn import preprocessing\n",
    "from keras.layers import LeakyReLU\n",
    "from keras import initializers\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.models import load_model\n",
    "from keras.utils import to_categorical\n",
    "import keras\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('processed_raw.csv')\n",
    "labels = np.loadtxt('train_marginals.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num_of_dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>...</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>p_dist_val</th>\n",
       "      <th>p_nans</th>\n",
       "      <th>castability</th>\n",
       "      <th>extractability</th>\n",
       "      <th>len_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.823504</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>Stratification2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.823504</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory3</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.823504</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>Stratification3</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.823504</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>c</td>\n",
       "      <td>GeoLocation</td>\n",
       "      <td>403984</td>\n",
       "      <td>55</td>\n",
       "      <td>0.635669</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.184907</td>\n",
       "      <td>0.040923</td>\n",
       "      <td>0.692323</td>\n",
       "      <td>0.040923</td>\n",
       "      <td>0.040923</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>0.635669</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1  y_act Reason           Attribute_name  Total_val  \\\n",
       "0           0             0      3      l  StratificationCategory2     403984   \n",
       "1           1             1      3      l          Stratification2     403984   \n",
       "2           2             2      3      l  StratificationCategory3     403984   \n",
       "3           3             3      3      l          Stratification3     403984   \n",
       "4           4             4      2      c              GeoLocation     403984   \n",
       "\n",
       "   num_of_dist_val     % nans  max_val  mean   ...            0         1  \\\n",
       "0                1  80.364816      0.0   0.0   ...     0.044124  0.044124   \n",
       "1                1  80.364816      0.0   0.0   ...     0.044124  0.044124   \n",
       "2                1  80.364816      0.0   0.0   ...     0.044124  0.044124   \n",
       "3                1  80.364816      0.0   0.0   ...     0.044124  0.044124   \n",
       "4               55   0.635669      0.0   0.0   ...     0.184907  0.040923   \n",
       "\n",
       "          2         3         4 p_dist_val     p_nans  castability  \\\n",
       "0  0.044124  0.823504  0.044124   0.000002  80.364816            0   \n",
       "1  0.044124  0.823504  0.044124   0.000002  80.364816            0   \n",
       "2  0.044124  0.823504  0.044124   0.000002  80.364816            0   \n",
       "3  0.044124  0.823504  0.044124   0.000002  80.364816            0   \n",
       "4  0.692323  0.040923  0.040923   0.000136   0.635669            1   \n",
       "\n",
       "   extractability  len_val  \n",
       "0               0      0.0  \n",
       "1               0      0.0  \n",
       "2               0      0.0  \n",
       "3               0      0.0  \n",
       "4               0      2.0  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def abs_limit(x):\n",
    "    if abs(x) > 10000:\n",
    "        return 10000*np.sign(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.fillna(0)\n",
    "data['scaled_mean'] = data['mean'].apply(abs_limit)\n",
    "data['scaled_min_val'] = data['min_val'].apply(abs_limit)\n",
    "data['scaled_max_val'] = data['max_val'].apply(abs_limit)\n",
    "data['scaled_std_dev'] = data['std_dev'].apply(abs_limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names_to_normalize = ['scaled_max_val', 'scaled_mean', 'scaled_min_val','scaled_std_dev']\n",
    "x = data[column_names_to_normalize].values\n",
    "x = np.nan_to_num(x)\n",
    "x_scaled = StandardScaler().fit_transform(x)\n",
    "df_temp = pd.DataFrame(x_scaled, columns=column_names_to_normalize, index = data.index)\n",
    "data[column_names_to_normalize] = df_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num_of_dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>...</th>\n",
       "      <th>4</th>\n",
       "      <th>p_dist_val</th>\n",
       "      <th>p_nans</th>\n",
       "      <th>castability</th>\n",
       "      <th>extractability</th>\n",
       "      <th>len_val</th>\n",
       "      <th>scaled_mean</th>\n",
       "      <th>scaled_min_val</th>\n",
       "      <th>scaled_max_val</th>\n",
       "      <th>scaled_std_dev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.366826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>Stratification2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.366826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory3</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.366826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>l</td>\n",
       "      <td>Stratification3</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.044124</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>80.364816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.366826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>c</td>\n",
       "      <td>GeoLocation</td>\n",
       "      <td>403984</td>\n",
       "      <td>55</td>\n",
       "      <td>0.635669</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040923</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>0.635669</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.366826</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1  y_act Reason           Attribute_name  Total_val  \\\n",
       "0           0             0      3      l  StratificationCategory2     403984   \n",
       "1           1             1      3      l          Stratification2     403984   \n",
       "2           2             2      3      l  StratificationCategory3     403984   \n",
       "3           3             3      3      l          Stratification3     403984   \n",
       "4           4             4      2      c              GeoLocation     403984   \n",
       "\n",
       "   num_of_dist_val     % nans  max_val  mean       ...               4  \\\n",
       "0                1  80.364816      0.0   0.0       ...        0.044124   \n",
       "1                1  80.364816      0.0   0.0       ...        0.044124   \n",
       "2                1  80.364816      0.0   0.0       ...        0.044124   \n",
       "3                1  80.364816      0.0   0.0       ...        0.044124   \n",
       "4               55   0.635669      0.0   0.0       ...        0.040923   \n",
       "\n",
       "   p_dist_val     p_nans castability extractability len_val scaled_mean  \\\n",
       "0    0.000002  80.364816           0              0     0.0   -0.300157   \n",
       "1    0.000002  80.364816           0              0     0.0   -0.300157   \n",
       "2    0.000002  80.364816           0              0     0.0   -0.300157   \n",
       "3    0.000002  80.364816           0              0     0.0   -0.300157   \n",
       "4    0.000136   0.635669           1              0     2.0   -0.300157   \n",
       "\n",
       "   scaled_min_val  scaled_max_val  scaled_std_dev  \n",
       "0        0.038284       -0.442185       -0.366826  \n",
       "1        0.038284       -0.442185       -0.366826  \n",
       "2        0.038284       -0.442185       -0.366826  \n",
       "3        0.038284       -0.442185       -0.366826  \n",
       "4        0.038284       -0.442185       -0.366826  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'Unnamed: 0.1', 'y_act', 'Reason', 'Attribute_name',\n",
       "       'Total_val', 'num_of_dist_val', '% nans', 'max_val', 'mean', 'min_val',\n",
       "       'std_dev', 'sample_1', 'sample_2', 'sample_3', 'sample_4', 'sample_5',\n",
       "       '0', '1', '2', '3', '4', 'p_dist_val', 'p_nans', 'castability',\n",
       "       'extractability', 'len_val', 'scaled_mean', 'scaled_min_val',\n",
       "       'scaled_max_val', 'scaled_std_dev'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8592, 31), (2149, 31))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train, data_test = train_test_split(data, test_size=0.2, random_state=100)\n",
    "data_train.shape, data_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8592, 15), (2149, 15), (8592, 5), (2149, 5), (2149,))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_columns =  ['Attribute_name',\n",
    "                  'sample_1',\n",
    "                  'sample_2',\n",
    "                  'sample_3',\n",
    "                  'sample_4',\n",
    "                  'sample_5',\n",
    "                  'p_nans',\n",
    "                  'p_dist_val',\n",
    "                  'scaled_max_val', \n",
    "                  'scaled_mean', \n",
    "                  'scaled_min_val',\n",
    "                  'scaled_std_dev',\n",
    "                  'castability',\n",
    "                  'extractability',\n",
    "                  'len_val']\n",
    "\n",
    "y_columns = ['0', '1', '2', '3', '4']\n",
    "\n",
    "X_train = data_train[x_columns].values\n",
    "X_test = data_test[x_columns].values\n",
    "y_train = data_train[y_columns].values.astype(float)\n",
    "y_test = data_test[y_columns].values.astype(float)\n",
    "y_act = data_test['y_act'].values.astype(int)\n",
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape, y_act.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_base = 'sample_'\n",
    "# samples_train = []\n",
    "# samples_test = []\n",
    "\n",
    "i = 1\n",
    "field_name = sample_base + str(i)\n",
    "field_idx = x_columns.index(field_name)\n",
    "\n",
    "samples1_train = X_train[:,field_idx].astype(str)\n",
    "samples1_test = X_test[:,field_idx].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "field_name = 'Attribute_name'\n",
    "atr_Xtrain = X_train[:,x_columns.index(field_name)].astype(str)\n",
    "atr_Xtest = X_test[:,x_columns.index(field_name)].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['pixel612', 's2p2c1seed', 'pixel291', ..., 'q25Scala',\n",
       "       'number_of_panels_s', 'tBodyAccJerk.max.X'], dtype='<U91')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "atr_Xtrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define network parameters\n",
    "max_features = 128\n",
    "maxlen = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_padded_tokens(train_list, test_list, maxlen=maxlen):\n",
    "    tokenizer = keras_text.Tokenizer(char_level = True)\n",
    "    tokenizer.fit_on_texts(train_list)\n",
    "    tokenized_train = tokenizer.texts_to_sequences(train_list)\n",
    "    pad_train = keras_seq.pad_sequences(tokenized_train, maxlen)\n",
    "\n",
    "    tokenized_test = tokenizer.texts_to_sequences(test_list)\n",
    "    pad_test = keras_seq.pad_sequences(tokenized_test, maxlen)\n",
    "    return tokenizer, pad_train, pad_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "atr_tokenizer, atr_pad_train, atr_pad_test = generate_padded_tokens(atr_Xtrain, atr_Xtest)\n",
    "\n",
    "# sample_tokenizers = list()\n",
    "# sample_pad_trains = list()\n",
    "# sample_pad_tests = list()\n",
    "# for i in range(5):\n",
    "#     tokenizer, pad_train, pad_test = generate_padded_tokens(samples_train[i], samples_test[i])\n",
    "#     sample_tokenizers.append(tokenizer)\n",
    "#     sample_pad_trains.append(pad_train)\n",
    "#     sample_pad_tests.append(pad_test)\n",
    "\n",
    "sam_tokenizer, sam_pad_train, sam_pad_test = generate_padded_tokens(samples1_train, samples1_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8592, 9), (2149, 9))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_left = X_train[:,6:]\n",
    "X_test_left = X_test[:,6:]\n",
    "X_train_left.shape, X_test_left.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# X_train_pro = np.concatenate([atr_pad_train]+sample_pad_trains+[X_train_left],axis=1)\n",
    "# X_test_pro = np.concatenate([atr_pad_test]+sample_pad_tests+[X_test_left],axis=1)\n",
    "# X_train_pro.shape, X_test_pro.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8592, 128), (8592, 128), (8592, 9))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_t = atr_pad_train\n",
    "X_t1 = sam_pad_train\n",
    "structured_data_train = X_train_left\n",
    "X_te = atr_pad_test\n",
    "X_te1 = sam_pad_test\n",
    "structured_data_test = X_test_left\n",
    "X_t.shape, X_t1.shape, structured_data_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(numfilters,embed_size, tokenizer1, tokenizer2):\n",
    "#     embed_size = 512\n",
    "    inp = Input(shape=(None, ))\n",
    "    x = Embedding(input_dim = len(tokenizer1.word_counts)+1, output_dim = embed_size)(inp)\n",
    "    prefilt_x = Dropout(0.25)(x)\n",
    "    out_conv = []\n",
    "\n",
    "    x = prefilt_x\n",
    "    for i in range(2):\n",
    "        x = Conv1D(numfilters*2**(i), kernel_size = 3, activation = 'relu')(x)\n",
    "    out_conv += [Dropout(0.5)(GlobalMaxPool1D()(x))]\n",
    "    out_conv += [GlobalMaxPool1D()(x)]\n",
    "    xy = concatenate(out_conv, axis = -1)  \n",
    "    print(xy.shape)\n",
    "    #########################################################\n",
    "    inp1 = Input(shape=(None, ))\n",
    "    x = Embedding(input_dim = len(tokenizer2.word_counts)+1, output_dim = embed_size)(inp1)\n",
    "    prefilt_x = Dropout(0.25)(x)\n",
    "    out_conv = []\n",
    "\n",
    "    x = prefilt_x\n",
    "    for i in range(2):\n",
    "        x = Conv1D(numfilters*2**(i), kernel_size = 3, activation = 'relu')(x)\n",
    "    out_conv += [Dropout(0.5)(GlobalMaxPool1D()(x))]\n",
    "    out_conv += [GlobalMaxPool1D()(x)]\n",
    "    x1 = concatenate(out_conv, axis = -1)\n",
    "    print(x1.shape)\n",
    "#     x1 = Flatten()(out_conv)\n",
    "    #########################################################\n",
    "#     inp2 = Input(shape=(None, ))\n",
    "#     x = Embedding(input_dim = len(tokenizer2.word_counts)+1, output_dim = embed_size)(inp1)\n",
    "# #     prefilt_x = Dropout(0.25)(x)\n",
    "#     out_conv = []\n",
    "#     # dilation rate lets us use ngrams and skip grams to process \n",
    "#     for dilation_rate in dilation_rates:\n",
    "# #         x = prefilt_x\n",
    "#         for i in range(2):\n",
    "#             if dilation_rate>0:\n",
    "#                 x = Conv1D(16*2**(i), \n",
    "#                            kernel_size = 3, \n",
    "#                            dilation_rate = dilation_rate,\n",
    "#                           activation = 'relu',\n",
    "#                           name = 'ngram_{}_cnn2_{}'.format(dilation_rate, i)\n",
    "#                           )(x)\n",
    "#             else:\n",
    "#                 x = Conv1D(16*2**(i), \n",
    "#                            kernel_size = 1,\n",
    "#                           activation = 'relu',\n",
    "#                           name = 'word_fcl2_{}'.format(i))(x)\n",
    "# #         out_conv += [Dropout(0.5)(GlobalMaxPool1D()(x))]\n",
    "#         out_conv += [GlobalMaxPool1D()(x)]\n",
    "#     x2 = concatenate(out_conv, axis = -1)\n",
    "#     print(x2.shape)\n",
    "#     x1 = Flatten()(out_conv)\n",
    "#     #########################################################\n",
    "    Str_input = Input(shape=(9,))\n",
    "    layersfin = keras.layers.concatenate([xy,x1,Str_input])\n",
    "    print(layersfin.shape)\n",
    "    x = Dense(1000, activation='relu')(layersfin)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(1000, activation='relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(5, activation='softmax')(x)\n",
    "    model = Model(inputs=[inp,inp1,Str_input], outputs=[x])\n",
    "#     model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "    model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "# model = build_model()\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 40)\n",
      "(?, 40)\n",
      "(?, 89)\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 5)      340         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 5)      545         input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, None, 5)      0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, None, 5)      0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, None, 10)     160         dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, None, 10)     160         dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, None, 20)     620         conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, None, 20)     620         conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 20)           0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 20)           0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 20)           0           global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 20)           0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 20)           0           global_max_pooling1d_3[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_4 (GlobalM (None, 20)           0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 40)           0           dropout_2[0][0]                  \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 40)           0           dropout_4[0][0]                  \n",
      "                                                                 global_max_pooling1d_4[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            (None, 9)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 89)           0           concatenate_1[0][0]              \n",
      "                                                                 concatenate_2[0][0]              \n",
      "                                                                 input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1000)         90000       concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 1000)         0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1000)         1001000     dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 1000)         0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 5)            5005        dropout_6[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 1,098,450\n",
      "Trainable params: 1,098,450\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dummy_model = build_model(10,5,atr_tokenizer, sam_tokenizer)\n",
    "dummy_model.metrics_names\n",
    "dummy_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "(5154, 5)\n",
      "(1719, 5)\n",
      "(5154, 9)\n",
      "(1719, 9)\n",
      "(?, 40)\n",
      "(?, 40)\n",
      "(?, 89)\n",
      "(?, 256)\n",
      "(?, 256)\n",
      "(?, 521)\n",
      "Train on 5154 samples, validate on 1719 samples\n",
      "Epoch 1/50\n",
      "5154/5154 [==============================] - 14s 3ms/step - loss: 0.0523 - acc: 0.6434 - val_loss: 0.0523 - val_acc: 0.6457\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.64572, saving model to best_weights64128.h5\n",
      "Epoch 2/50\n",
      "5154/5154 [==============================] - 13s 2ms/step - loss: 0.0496 - acc: 0.6903 - val_loss: 0.0429 - val_acc: 0.7504\n",
      "\n",
      "Epoch 00002: val_acc improved from 0.64572 to 0.75044, saving model to best_weights64128.h5\n",
      "Epoch 3/50\n",
      "5154/5154 [==============================] - 13s 2ms/step - loss: 0.0349 - acc: 0.7253 - val_loss: 0.0127 - val_acc: 0.7958\n",
      "\n",
      "Epoch 00003: val_acc improved from 0.75044 to 0.79581, saving model to best_weights64128.h5\n",
      "Epoch 4/50\n",
      "5154/5154 [==============================] - 13s 3ms/step - loss: 0.0127 - acc: 0.7918 - val_loss: 0.0102 - val_acc: 0.8133\n",
      "\n",
      "Epoch 00004: val_acc improved from 0.79581 to 0.81326, saving model to best_weights64128.h5\n",
      "Epoch 5/50\n",
      "5154/5154 [==============================] - 14s 3ms/step - loss: 0.0110 - acc: 0.8124 - val_loss: 0.0106 - val_acc: 0.8138\n",
      "\n",
      "Epoch 00005: val_acc improved from 0.81326 to 0.81385, saving model to best_weights64128.h5\n",
      "Epoch 6/50\n",
      "5154/5154 [==============================] - 14s 3ms/step - loss: 0.0090 - acc: 0.8388 - val_loss: 0.0118 - val_acc: 0.8121\n",
      "\n",
      "Epoch 00006: val_acc did not improve from 0.81385\n",
      "Epoch 7/50\n",
      "5154/5154 [==============================] - 13s 3ms/step - loss: 0.0088 - acc: 0.8454 - val_loss: 0.0092 - val_acc: 0.8278\n",
      "\n",
      "Epoch 00007: val_acc improved from 0.81385 to 0.82781, saving model to best_weights64128.h5\n",
      "Epoch 8/50\n",
      "5154/5154 [==============================] - 14s 3ms/step - loss: 0.0082 - acc: 0.8479 - val_loss: 0.0093 - val_acc: 0.8162\n",
      "\n",
      "Epoch 00008: val_acc did not improve from 0.82781\n",
      "Epoch 9/50\n",
      "5154/5154 [==============================] - 14s 3ms/step - loss: 0.0073 - acc: 0.8615 - val_loss: 0.0088 - val_acc: 0.8546\n",
      "\n",
      "Epoch 00009: val_acc improved from 0.82781 to 0.85457, saving model to best_weights64128.h5\n",
      "Epoch 10/50\n",
      "5154/5154 [==============================] - 13s 2ms/step - loss: 0.0067 - acc: 0.8677 - val_loss: 0.0086 - val_acc: 0.8482\n",
      "\n",
      "Epoch 00010: val_acc did not improve from 0.85457\n",
      "Epoch 11/50\n",
      "5154/5154 [==============================] - 13s 2ms/step - loss: 0.0062 - acc: 0.8704 - val_loss: 0.0084 - val_acc: 0.8284\n",
      "\n",
      "Epoch 00011: val_acc did not improve from 0.85457\n",
      "Epoch 12/50\n",
      "5154/5154 [==============================] - 13s 3ms/step - loss: 0.0057 - acc: 0.8799 - val_loss: 0.0090 - val_acc: 0.8365\n",
      "\n",
      "Epoch 00012: val_acc did not improve from 0.85457\n",
      "Epoch 13/50\n",
      "5154/5154 [==============================] - 13s 2ms/step - loss: 0.0055 - acc: 0.8865 - val_loss: 0.0095 - val_acc: 0.8330\n",
      "\n",
      "Epoch 00013: val_acc did not improve from 0.85457\n",
      "Epoch 14/50\n",
      "5154/5154 [==============================] - 12s 2ms/step - loss: 0.0052 - acc: 0.8904 - val_loss: 0.0088 - val_acc: 0.8325\n",
      "\n",
      "Epoch 00014: val_acc did not improve from 0.85457\n",
      "Epoch 15/50\n",
      "1920/5154 [==========>...................] - ETA: 9s - loss: 0.0046 - acc: 0.9026"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-3805a345600f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     51\u001b[0m                               \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m                               \u001b[0mshuffle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m                               callbacks=callbacks_list)\n\u001b[0m\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m             \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'best_weights'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mne\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/anaconda/lib/python3.5/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# n_filters_grid = [16,32,64,128]\n",
    "# embed_size = [32,64,128,512]\n",
    "\n",
    "\n",
    "n_filters_grid = [64]\n",
    "embed_size = [128]\n",
    "\n",
    "batch_size = 32 # large enough that some other labels come in\n",
    "epochs = 50\n",
    "\n",
    "k = 5\n",
    "kf = KFold(n_splits=k)\n",
    "# n_filters_grid = [32]\n",
    "# embed_size = [256]\n",
    "\n",
    "avgsc_lst,avgsc_train_lst = [],[]\n",
    "avgsc,avgsc_train = 0,0\n",
    "bestne,bestmd = 0,0\n",
    "i=0\n",
    "for train_index, test_index in kf.split(X_t):\n",
    "    print('\\n')\n",
    "    X_train_cur, X_test_cur = X_t[train_index], X_t[test_index]\n",
    "    X_train_cur1, X_test_cur1 = X_t1[train_index], X_t1[test_index]    \n",
    "    y_train_cur, y_test_cur = y_train[train_index], y_train[test_index]\n",
    "    structured_data_train_cur, structured_data_test_cur = structured_data_train[train_index],structured_data_train[test_index]\n",
    "    \n",
    "    X_train_train, X_val,y_train_train,y_val = train_test_split(X_train_cur,y_train_cur, test_size=0.25,random_state=100)\n",
    "    X_train_train1, X_val1 = train_test_split(X_train_cur1, test_size=0.25,random_state=100)    \n",
    "    structured_data_train_train,structured_data_val = train_test_split(structured_data_train_cur, test_size=0.25,random_state=100)\n",
    "    \n",
    "    print(y_train_train.shape)\n",
    "    print(y_val.shape)\n",
    "    print(structured_data_train_train.shape)\n",
    "    print(structured_data_val.shape)\n",
    "    \n",
    "    bestPerformingModel = build_model(10,5,atr_tokenizer, sam_tokenizer)\n",
    "    bestscore = 0\n",
    "    for ne in n_filters_grid:\n",
    "        for md in embed_size:\n",
    "            file_path= 'best_weights'+str(ne)+str(md)+'.h5'\n",
    "            checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "            early = EarlyStopping(monitor=\"val_acc\", mode=\"max\", patience=20)\n",
    "            callbacks_list = [checkpoint, early] #early            \n",
    "            \n",
    "#             clf = RandomForestClassifier(n_estimators=ne,max_depth=md)\n",
    "            clf = build_model(ne,md,atr_tokenizer, sam_tokenizer)\n",
    "            history = clf.fit([X_train_train,X_train_train1,structured_data_train_train],\n",
    "                              y_train_train,\n",
    "                              validation_data=([X_val,X_val1,structured_data_val], y_val),\n",
    "                              batch_size=batch_size, \n",
    "                              epochs=epochs, \n",
    "                              shuffle = True, \n",
    "                              callbacks=callbacks_list)\n",
    "        \n",
    "            clf.load_weights('best_weights'+str(ne)+str(md)+'.h5')\n",
    "            sc = clf.evaluate([X_val,X_val1,structured_data_val],y_val)\n",
    "            print(sc)\n",
    "        \n",
    "            if bestscore < sc:\n",
    "                bestscore = sc\n",
    "                bestPerformingModel = clf\n",
    "                bestne,bestmd = ne,md\n",
    "                print(bestPerformingModel)\n",
    "    \n",
    "#     bestPerformingModel.load_weights('best_weights.h5')\n",
    "    bestPerformingModel.load_weights('best_weights'+str(bestne)+str(bestmd)+'.h5')\n",
    "    loss, bscr_train = \\\n",
    "        bestPerformingModel.evaluate([X_train_train,X_train_train1,structured_data_train_train],y_train_train)\n",
    "    print(loss, bscr_train)\n",
    "    loss, bscr = \\\n",
    "        bestPerformingModel.evaluate([X_test_cur,X_test_cur1,structured_data_test_cur],y_test_cur)\n",
    "    print(loss, bscr)\n",
    "    \n",
    "    avgsc_lst.append(bscr)\n",
    "    avgsc_train_lst.append(bscr_train)      \n",
    "    \n",
    "    avgsc = avgsc + bscr\n",
    "    avgsc_train = avgsc_train + bscr_train\n",
    "    print('The training accuracy is:')\n",
    "    print(bscr_train)\n",
    "    print('The test accuracy is:')    \n",
    "    print(bscr)\n",
    "    print('\\n')\n",
    "    i=i+1\n",
    "\n",
    "\n",
    "print(avgsc_train/k)\n",
    "print(avgsc/k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8592, 512), (8592, 512), (8592, 9), (8592, 1))"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_t.shape, X_t1.shape, structured_data_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "8592/8592 [==============================] - 13s 2ms/step\n",
      "6.157857602757004 0.37639664804469275\n",
      "2149/2149 [==============================] - 3s 1ms/step\n",
      "5.93661403101286 0.37971149373187635\n"
     ]
    }
   ],
   "source": [
    "\n",
    "bestone = load_model('best_weights.h5')\n",
    "print('---')\n",
    "loss, acc = bestone.evaluate([X_t,X_t1,structured_data_train],to_categorical(y_train),verbose=1)\n",
    "print(loss, acc)\n",
    "loss, acc = bestone.evaluate([X_te,X_te1,structured_data_test],to_categorical(y_test),verbose=1)\n",
    "print(loss, acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8592, 512)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0,  0,  0, ...,  0,  0, 31],\n",
       "       [ 0,  0,  0, ...,  0,  0, 18],\n",
       "       [ 0,  0,  0, ..., 23, 26, 31],\n",
       "       ...,\n",
       "       [ 0,  0,  0, ...,  2,  4,  9],\n",
       "       [ 0,  0,  0, ...,  0,  0, 31],\n",
       "       [ 0,  0,  0, ..., 36, 38, 34]], dtype=int32)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_t1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p_nans</th>\n",
       "      <th>scaled_max_val</th>\n",
       "      <th>scaled_mean</th>\n",
       "      <th>scaled_min_val</th>\n",
       "      <th>p_dist_val</th>\n",
       "      <th>scaled_std_dev</th>\n",
       "      <th>castability</th>\n",
       "      <th>extractability</th>\n",
       "      <th>len_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.274532</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.004267</td>\n",
       "      <td>-0.336663</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.220381</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.018756</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.261691</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.004267</td>\n",
       "      <td>-0.336455</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.750000</td>\n",
       "      <td>-0.525882</td>\n",
       "      <td>-3.786845</td>\n",
       "      <td>-4.596337</td>\n",
       "      <td>0.960294</td>\n",
       "      <td>2.994287</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.495087</td>\n",
       "      <td>3.186532</td>\n",
       "      <td>0.052652</td>\n",
       "      <td>0.996703</td>\n",
       "      <td>2.994287</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23.371887</td>\n",
       "      <td>-0.438660</td>\n",
       "      <td>-0.299104</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000938</td>\n",
       "      <td>-0.366274</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.343201</td>\n",
       "      <td>-0.283824</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.080252</td>\n",
       "      <td>-0.339652</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.294500</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.025400</td>\n",
       "      <td>-0.350244</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.075026</td>\n",
       "      <td>-0.299079</td>\n",
       "      <td>-0.499332</td>\n",
       "      <td>0.124324</td>\n",
       "      <td>-0.315371</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>72.835261</td>\n",
       "      <td>-0.379327</td>\n",
       "      <td>-0.300054</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.001980</td>\n",
       "      <td>-0.365043</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>53.829322</td>\n",
       "      <td>-0.412812</td>\n",
       "      <td>-0.299625</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.002813</td>\n",
       "      <td>-0.364936</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.441970</td>\n",
       "      <td>-0.300329</td>\n",
       "      <td>0.037864</td>\n",
       "      <td>0.412719</td>\n",
       "      <td>-0.366760</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>83.328123</td>\n",
       "      <td>1.503758</td>\n",
       "      <td>-0.282654</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.003647</td>\n",
       "      <td>-0.272155</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300476</td>\n",
       "      <td>0.037821</td>\n",
       "      <td>0.223881</td>\n",
       "      <td>-0.366782</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.495087</td>\n",
       "      <td>3.186532</td>\n",
       "      <td>-4.596337</td>\n",
       "      <td>0.115731</td>\n",
       "      <td>2.994287</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.952263</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.038206</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.295832</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.004267</td>\n",
       "      <td>-0.352113</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.264698</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.004267</td>\n",
       "      <td>-0.335889</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000120</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442006</td>\n",
       "      <td>-0.300383</td>\n",
       "      <td>0.037821</td>\n",
       "      <td>0.376379</td>\n",
       "      <td>-0.366696</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.433582</td>\n",
       "      <td>-0.442352</td>\n",
       "      <td>-4.595874</td>\n",
       "      <td>0.090928</td>\n",
       "      <td>0.314767</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>99.338235</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.001471</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.039286</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.495087</td>\n",
       "      <td>0.847988</td>\n",
       "      <td>0.038386</td>\n",
       "      <td>0.415751</td>\n",
       "      <td>1.915823</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.980787</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.466008</td>\n",
       "      <td>0.186407</td>\n",
       "      <td>0.038748</td>\n",
       "      <td>0.002392</td>\n",
       "      <td>0.625028</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>92.706054</td>\n",
       "      <td>-0.424268</td>\n",
       "      <td>-0.299829</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.001771</td>\n",
       "      <td>-0.364977</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.773386</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>79.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.441891</td>\n",
       "      <td>-0.300031</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000735</td>\n",
       "      <td>-0.366665</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8562</th>\n",
       "      <td>79.076795</td>\n",
       "      <td>-0.441598</td>\n",
       "      <td>-0.299882</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000313</td>\n",
       "      <td>-0.366560</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8563</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.291933</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.004267</td>\n",
       "      <td>-0.347305</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8564</th>\n",
       "      <td>6.139706</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.095221</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8565</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.404294</td>\n",
       "      <td>-0.260238</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.021801</td>\n",
       "      <td>-0.359970</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8566</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442040</td>\n",
       "      <td>-0.300451</td>\n",
       "      <td>0.037821</td>\n",
       "      <td>0.299156</td>\n",
       "      <td>-0.366747</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8567</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.495087</td>\n",
       "      <td>3.186532</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.963736</td>\n",
       "      <td>2.994287</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8568</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.261644</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.025600</td>\n",
       "      <td>-0.334933</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8569</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.495087</td>\n",
       "      <td>3.186532</td>\n",
       "      <td>4.672906</td>\n",
       "      <td>0.666744</td>\n",
       "      <td>2.994287</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8570</th>\n",
       "      <td>97.488799</td>\n",
       "      <td>-0.441598</td>\n",
       "      <td>-0.299494</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000313</td>\n",
       "      <td>-0.366717</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8571</th>\n",
       "      <td>71.366052</td>\n",
       "      <td>2.495087</td>\n",
       "      <td>0.122399</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.008336</td>\n",
       "      <td>2.994287</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8572</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.441914</td>\n",
       "      <td>-0.300162</td>\n",
       "      <td>0.037907</td>\n",
       "      <td>0.673589</td>\n",
       "      <td>-0.366728</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8573</th>\n",
       "      <td>33.249974</td>\n",
       "      <td>-0.441891</td>\n",
       "      <td>-0.300142</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000208</td>\n",
       "      <td>-0.366759</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8574</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.998750</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8575</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442049</td>\n",
       "      <td>-0.300383</td>\n",
       "      <td>0.037821</td>\n",
       "      <td>0.390006</td>\n",
       "      <td>-0.366694</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8576</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000852</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8577</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000120</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8578</th>\n",
       "      <td>27.175159</td>\n",
       "      <td>-0.148458</td>\n",
       "      <td>-0.296708</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.010420</td>\n",
       "      <td>-0.357252</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8579</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.441891</td>\n",
       "      <td>-0.300156</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.001318</td>\n",
       "      <td>-0.366818</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8580</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000920</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8581</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.429261</td>\n",
       "      <td>-0.299368</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000512</td>\n",
       "      <td>-0.366331</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8582</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.273995</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.025600</td>\n",
       "      <td>-0.337070</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8583</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442002</td>\n",
       "      <td>-0.300382</td>\n",
       "      <td>0.037821</td>\n",
       "      <td>0.381570</td>\n",
       "      <td>-0.366695</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8584</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000080</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8585</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.367285</td>\n",
       "      <td>-0.273091</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.004267</td>\n",
       "      <td>-0.335825</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8586</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.441891</td>\n",
       "      <td>-0.300145</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.001318</td>\n",
       "      <td>-0.366766</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8587</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000492</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8588</th>\n",
       "      <td>36.135879</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8589</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442185</td>\n",
       "      <td>-0.300157</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000080</td>\n",
       "      <td>-0.366826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8590</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.495087</td>\n",
       "      <td>3.186532</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.889182</td>\n",
       "      <td>2.994287</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8591</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.442007</td>\n",
       "      <td>-0.300401</td>\n",
       "      <td>0.037821</td>\n",
       "      <td>0.368592</td>\n",
       "      <td>-0.366707</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8592 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          p_nans  scaled_max_val  scaled_mean  scaled_min_val  p_dist_val  \\\n",
       "0       0.000000       -0.367285    -0.274532        0.038284    0.004267   \n",
       "1       5.220381       -0.442185    -0.300157        0.038284    0.018756   \n",
       "2       0.000000       -0.367285    -0.261691        0.038284    0.004267   \n",
       "3       3.750000       -0.525882    -3.786845       -4.596337    0.960294   \n",
       "4       0.000000        2.495087     3.186532        0.052652    0.996703   \n",
       "5     100.000000       -0.442185    -0.300157        0.038284    0.000000   \n",
       "6      23.371887       -0.438660    -0.299104        0.038284    0.000938   \n",
       "7       0.000000        1.343201    -0.283824        0.038284    0.080252   \n",
       "8       0.000000       -0.367285    -0.294500        0.038284    0.025400   \n",
       "9       0.000000       -0.075026    -0.299079       -0.499332    0.124324   \n",
       "10     72.835261       -0.379327    -0.300054        0.038284    0.001980   \n",
       "11     53.829322       -0.412812    -0.299625        0.038284    0.002813   \n",
       "12      0.000000       -0.441970    -0.300329        0.037864    0.412719   \n",
       "13     83.328123        1.503758    -0.282654        0.038284    0.003647   \n",
       "14      0.000000       -0.442185    -0.300476        0.037821    0.223881   \n",
       "15      0.000000        2.495087     3.186532       -4.596337    0.115731   \n",
       "16      0.952263       -0.442185    -0.300157        0.038284    0.038206   \n",
       "17      0.000000       -0.367285    -0.295832        0.038284    0.004267   \n",
       "18      0.000000       -0.367285    -0.264698        0.038284    0.004267   \n",
       "19      0.000000       -0.442185    -0.300157        0.038284    0.000120   \n",
       "20      0.000000       -0.442006    -0.300383        0.037821    0.376379   \n",
       "21      0.000000       -0.433582    -0.442352       -4.595874    0.090928   \n",
       "22     99.338235       -0.442185    -0.300157        0.038284    0.001471   \n",
       "23      0.000000       -0.442185    -0.300157        0.038284    0.039286   \n",
       "24      0.000000        2.495087     0.847988        0.038386    0.415751   \n",
       "25      0.000000       -0.442185    -0.300157        0.038284    0.980787   \n",
       "26      0.000000        2.466008     0.186407        0.038748    0.002392   \n",
       "27     92.706054       -0.424268    -0.299829        0.038284    0.001771   \n",
       "28      0.000000       -0.442185    -0.300157        0.038284    0.773386   \n",
       "29      0.000000       -0.441891    -0.300031        0.038284    0.000735   \n",
       "...          ...             ...          ...             ...         ...   \n",
       "8562   79.076795       -0.441598    -0.299882        0.038284    0.000313   \n",
       "8563    0.000000       -0.367285    -0.291933        0.038284    0.004267   \n",
       "8564    6.139706       -0.442185    -0.300157        0.038284    0.095221   \n",
       "8565    0.000000       -0.404294    -0.260238        0.038284    0.021801   \n",
       "8566    0.000000       -0.442040    -0.300451        0.037821    0.299156   \n",
       "8567    0.000000        2.495087     3.186532        0.038284    0.963736   \n",
       "8568    0.000000       -0.367285    -0.261644        0.038284    0.025600   \n",
       "8569    0.000000        2.495087     3.186532        4.672906    0.666744   \n",
       "8570   97.488799       -0.441598    -0.299494        0.038284    0.000313   \n",
       "8571   71.366052        2.495087     0.122399        0.038284    0.008336   \n",
       "8572    0.000000       -0.441914    -0.300162        0.037907    0.673589   \n",
       "8573   33.249974       -0.441891    -0.300142        0.038284    0.000208   \n",
       "8574    0.000000       -0.442185    -0.300157        0.038284    0.998750   \n",
       "8575    0.000000       -0.442049    -0.300383        0.037821    0.390006   \n",
       "8576    0.000000       -0.442185    -0.300157        0.038284    0.000852   \n",
       "8577    0.000000       -0.442185    -0.300157        0.038284    0.000120   \n",
       "8578   27.175159       -0.148458    -0.296708        0.038284    0.010420   \n",
       "8579    0.000000       -0.441891    -0.300156        0.038284    0.001318   \n",
       "8580    0.000000       -0.442185    -0.300157        0.038284    0.000920   \n",
       "8581    0.000000       -0.429261    -0.299368        0.038284    0.000512   \n",
       "8582    0.000000       -0.367285    -0.273995        0.038284    0.025600   \n",
       "8583    0.000000       -0.442002    -0.300382        0.037821    0.381570   \n",
       "8584    0.000000       -0.442185    -0.300157        0.038284    0.000080   \n",
       "8585    0.000000       -0.367285    -0.273091        0.038284    0.004267   \n",
       "8586    0.000000       -0.441891    -0.300145        0.038284    0.001318   \n",
       "8587    0.000000       -0.442185    -0.300157        0.038284    0.000492   \n",
       "8588   36.135879       -0.442185    -0.300157        0.038284    0.000005   \n",
       "8589    0.000000       -0.442185    -0.300157        0.038284    0.000080   \n",
       "8590    0.000000        2.495087     3.186532        0.038284    0.889182   \n",
       "8591    0.000000       -0.442007    -0.300401        0.037821    0.368592   \n",
       "\n",
       "      scaled_std_dev  castability  extractability  len_val  \n",
       "0          -0.336663          1.0             0.0      1.0  \n",
       "1          -0.366826          1.0             0.0      1.0  \n",
       "2          -0.336455          1.0             0.0      1.0  \n",
       "3           2.994287          1.0             0.0      1.0  \n",
       "4           2.994287          1.0             0.0      1.0  \n",
       "5          -0.366826          0.0             0.0      0.0  \n",
       "6          -0.366274          1.0             0.0      0.8  \n",
       "7          -0.339652          1.0             0.0      1.0  \n",
       "8          -0.350244          1.0             0.0      1.0  \n",
       "9          -0.315371          1.0             0.0      1.0  \n",
       "10         -0.365043          1.0             0.0      0.8  \n",
       "11         -0.364936          1.0             0.0      0.8  \n",
       "12         -0.366760          1.0             0.0      1.0  \n",
       "13         -0.272155          1.0             0.0      0.8  \n",
       "14         -0.366782          1.0             0.0      1.0  \n",
       "15          2.994287          1.0             0.0      1.0  \n",
       "16         -0.366826          0.0             0.0      1.4  \n",
       "17         -0.352113          1.0             0.0      1.0  \n",
       "18         -0.335889          1.0             0.0      1.0  \n",
       "19         -0.366826          1.0             0.0      0.6  \n",
       "20         -0.366696          1.0             0.0      1.0  \n",
       "21          0.314767          1.0             0.0      1.0  \n",
       "22         -0.366826          0.0             0.0      1.8  \n",
       "23         -0.366826          0.0             0.0      1.0  \n",
       "24          1.915823          1.0             0.0      0.8  \n",
       "25         -0.366826          0.0             0.0     95.0  \n",
       "26          0.625028          1.0             0.0      1.0  \n",
       "27         -0.364977          1.0             0.0      0.8  \n",
       "28         -0.366826          0.0             0.0     79.4  \n",
       "29         -0.366665          1.0             0.0      1.0  \n",
       "...              ...          ...             ...      ...  \n",
       "8562       -0.366560          1.0             0.0      0.8  \n",
       "8563       -0.347305          1.0             0.0      1.0  \n",
       "8564       -0.366826          0.0             0.0      3.6  \n",
       "8565       -0.359970          1.0             0.0      1.0  \n",
       "8566       -0.366747          1.0             0.0      1.0  \n",
       "8567        2.994287          1.0             0.0      1.0  \n",
       "8568       -0.334933          1.0             0.0      1.0  \n",
       "8569        2.994287          1.0             0.0      0.8  \n",
       "8570       -0.366717          1.0             0.0      0.6  \n",
       "8571        2.994287          1.0             0.0      0.8  \n",
       "8572       -0.366728          1.0             0.0      1.0  \n",
       "8573       -0.366759          1.0             0.0      0.6  \n",
       "8574       -0.366826          0.0             0.0      1.2  \n",
       "8575       -0.366694          1.0             0.0      1.0  \n",
       "8576       -0.366826          1.0             0.0      3.6  \n",
       "8577       -0.366826          1.0             0.0      0.6  \n",
       "8578       -0.357252          1.0             0.0      1.0  \n",
       "8579       -0.366818          1.0             0.0      1.0  \n",
       "8580       -0.366826          1.0             0.0     12.2  \n",
       "8581       -0.366331          1.0             0.0      1.0  \n",
       "8582       -0.337070          1.0             0.0      1.0  \n",
       "8583       -0.366695          1.0             0.0      1.0  \n",
       "8584       -0.366826          0.0             0.0      0.6  \n",
       "8585       -0.335825          1.0             0.0      1.0  \n",
       "8586       -0.366766          1.0             0.0      1.0  \n",
       "8587       -0.366826          0.0             0.0      4.0  \n",
       "8588       -0.366826          0.0             0.0      0.6  \n",
       "8589       -0.366826          0.0             0.0      1.0  \n",
       "8590        2.994287          1.0             0.0      1.0  \n",
       "8591       -0.366707          1.0             0.0      1.0  \n",
       "\n",
       "[8592 rows x 9 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structured_data_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.5",
   "language": "python",
   "name": "python3.5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}