xg_boost_final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "31f74112f58de920b89eae99945e10d3e184c0aa"
   },
   "source": [
    "**XGBoost Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['trainpredict', 'testpredict']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "os.cpu_count()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "8405c12bf6745054bd75d9c9f19eb86c9139e477"
   },
   "outputs": [],
   "source": [
    "X = pd.read_csv(\"../input/trainpredict/train.csv\")\n",
    "X_1 = pd.read_csv(\"../input/testpredict/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "e7e1ab41c9c257d192650c280ba55b006f56a647"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID                   0\n",
      "datetime             0\n",
      "siteid         1212515\n",
      "offerid              0\n",
      "category             0\n",
      "merchant             0\n",
      "countrycode          0\n",
      "browserid       608327\n",
      "devid          1820299\n",
      "click                0\n",
      "dtype: int64\n",
      "ID                  0\n",
      "datetime            0\n",
      "siteid         370776\n",
      "offerid             0\n",
      "category            0\n",
      "merchant            0\n",
      "countrycode         0\n",
      "browserid      221906\n",
      "devid          704619\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "train = X\n",
    "test = X_1\n",
    "print(train.isnull().sum(axis=0))\n",
    "print(test.isnull().sum(axis = 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "894733397692f969117d157ba88462cc0db1cd95"
   },
   "source": [
    "**Data  Pre Processing**\n",
    "1. Add features of week day and hour\n",
    "2. Update hour as range. Here, 24 hour has been divided into 6 parts\n",
    "3. Label encode all features\n",
    "4. For the missing values keep a separate label as -1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "569dde48060f0901d214f499cfae76aba701684a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>siteid</th>\n",
       "      <th>offerid</th>\n",
       "      <th>category</th>\n",
       "      <th>merchant</th>\n",
       "      <th>countrycode</th>\n",
       "      <th>browserid</th>\n",
       "      <th>devid</th>\n",
       "      <th>click</th>\n",
       "      <th>day</th>\n",
       "      <th>hour1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4709696.0</td>\n",
       "      <td>887235</td>\n",
       "      <td>17714</td>\n",
       "      <td>20301556</td>\n",
       "      <td>e</td>\n",
       "      <td>Firefox</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5189467.0</td>\n",
       "      <td>178235</td>\n",
       "      <td>21407</td>\n",
       "      <td>9434818</td>\n",
       "      <td>b</td>\n",
       "      <td>Mozilla Firefox</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>98480.0</td>\n",
       "      <td>518539</td>\n",
       "      <td>25085</td>\n",
       "      <td>2050923</td>\n",
       "      <td>a</td>\n",
       "      <td>Edge</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8896401.0</td>\n",
       "      <td>390352</td>\n",
       "      <td>40339</td>\n",
       "      <td>72089744</td>\n",
       "      <td>c</td>\n",
       "      <td>Firefox</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5635120.0</td>\n",
       "      <td>472937</td>\n",
       "      <td>12052</td>\n",
       "      <td>39507200</td>\n",
       "      <td>d</td>\n",
       "      <td>Mozilla Firefox</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      siteid  offerid  category  merchant  ...      devid click day  hour1\n",
       "0  4709696.0   887235     17714  20301556  ...        NaN     0   5      1\n",
       "1  5189467.0   178235     21407   9434818  ...    Desktop     0   2      1\n",
       "2    98480.0   518539     25085   2050923  ...        NaN     0   2      0\n",
       "3  8896401.0   390352     40339  72089744  ...     Mobile     0   1      2\n",
       "4  5635120.0   472937     12052  39507200  ...    Desktop     0   5      0\n",
       "\n",
       "[5 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['datetime'] = pd.to_datetime(train['datetime'])\n",
    "train['day'] = train['datetime'].dt.weekday\n",
    "train['hour'] = train['datetime'].dt.hour\n",
    "\n",
    "test['datetime'] = pd.to_datetime(test['datetime'])\n",
    "test['day'] = test['datetime'].dt.weekday\n",
    "test['hour'] = test['datetime'].dt.hour\n",
    "\n",
    "train['hour1'] = train['hour']%4\n",
    "test['hour1'] = test['hour']%4\n",
    "\n",
    "train = train.drop(columns=['ID', 'datetime', 'hour',])\n",
    "test = test.drop(columns=['datetime', 'hour', ])\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "4bc02b94aae7bb8c0662a530a03af601c8e3e33d",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "train = train.astype('object')\n",
    "test = test.astype('object')\n",
    "col_with_null = ['siteid', 'browserid','devid']\n",
    "cols = ['offerid', 'category', 'merchant', 'countrycode']\n",
    "for col in col_with_null:\n",
    "    lbl = LabelEncoder()\n",
    "    temp_train = train[col]\n",
    "    temp_test = test[col]\n",
    "    mask = train[col].isnull()\n",
    "    mask1 = test[col].isnull()\n",
    "    train[col] = train[col].astype(str)\n",
    "    test[col] = test[col].astype(str)\n",
    "    lbl.fit(list(train[col].values) + list(test[col].values))\n",
    "    train[col] = lbl.transform(list(train[col].values))\n",
    "    test[col] = lbl.transform(list(test[col].values))\n",
    "    train[col] = train[col].where(~mask, temp_train)\n",
    "    test[col] = test[col].where(~mask1, temp_test)\n",
    "    \n",
    "for col in cols:\n",
    "    lbl = LabelEncoder()\n",
    "    lbl.fit(list(train[col].values) + list(test[col].values))\n",
    "    train[col] = lbl.transform(list(train[col].values))\n",
    "    test[col] = lbl.transform(list(test[col].values))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "f1fb1ee672d4d570bb3454d73415eb3d544d1c86"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>siteid</th>\n",
       "      <th>offerid</th>\n",
       "      <th>category</th>\n",
       "      <th>merchant</th>\n",
       "      <th>countrycode</th>\n",
       "      <th>browserid</th>\n",
       "      <th>devid</th>\n",
       "      <th>click</th>\n",
       "      <th>day</th>\n",
       "      <th>hour1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>112743.0</td>\n",
       "      <td>784773</td>\n",
       "      <td>48</td>\n",
       "      <td>127</td>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>127360.0</td>\n",
       "      <td>157563</td>\n",
       "      <td>59</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>269055.0</td>\n",
       "      <td>458279</td>\n",
       "      <td>69</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>240075.0</td>\n",
       "      <td>345067</td>\n",
       "      <td>117</td>\n",
       "      <td>507</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>140991.0</td>\n",
       "      <td>417948</td>\n",
       "      <td>36</td>\n",
       "      <td>276</td>\n",
       "      <td>3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     siteid  offerid  category  merchant  ...    devid  click  day  hour1\n",
       "0  112743.0   784773        48       127  ...     -1.0      0    5      1\n",
       "1  127360.0   157563        59        65  ...      0.0      0    2      1\n",
       "2  269055.0   458279        69        15  ...     -1.0      0    2      0\n",
       "3  240075.0   345067       117       507  ...      1.0      0    1      2\n",
       "4  140991.0   417948        36       276  ...      0.0      0    5      0\n",
       "\n",
       "[5 rows x 10 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.replace('nan', np.NAN)\n",
    "test = test.replace('nan',np.NAN)\n",
    "train['siteid'].fillna(-1, inplace = True)\n",
    "test['siteid'].fillna(-1, inplace = True)\n",
    "train['browserid'].fillna(-1, inplace = True)\n",
    "test['browserid'].fillna(-1, inplace = True)\n",
    "train['devid'].fillna(-1, inplace = True)\n",
    "test['devid'].fillna(-1, inplace = True)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "c89700d9c4b488854855342c88374f198cd310a8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        ID    siteid  offerid  category  ...    browserid  devid  day  hour1\n",
      "0  IDFDJVI  199198.0   715248        48  ...          8.0    0.0    6      1\n",
      "1  IDNWkTQ   82416.0   247936        36  ...          1.0    2.0    6      3\n",
      "2  ID9pRmM  102642.0   823274        91  ...          8.0   -1.0    5      2\n",
      "3  IDHaQaj   22971.0   541204        33  ...          1.0    2.0    6      2\n",
      "4  IDT2CrF  130703.0   463364       130  ...          7.0   -1.0    6      1\n",
      "\n",
      "[5 rows x 10 columns]\n",
      "     siteid  offerid  category  merchant  ...    devid  click  day  hour1\n",
      "0  112743.0   784773        48       127  ...     -1.0      0    5      1\n",
      "1  127360.0   157563        59        65  ...      0.0      0    2      1\n",
      "2  269055.0   458279        69        15  ...     -1.0      0    2      0\n",
      "3  240075.0   345067       117       507  ...      1.0      0    1      2\n",
      "4  140991.0   417948        36       276  ...      0.0      0    5      0\n",
      "\n",
      "[5 rows x 10 columns]\n"
     ]
    }
   ],
   "source": [
    "print(test.head())\n",
    "print(train.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "d015ea870f8c03f8fd962b0aee2960a3cc173c4e"
   },
   "outputs": [],
   "source": [
    "train = train.astype(int)\n",
    "for col in test.columns[1:]: test[col] = test[col].astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_uuid": "adf3b7c9f3a3160077b9660f0d1e1455a434f615"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "Y_t = train['click'].astype('float')\n",
    "X_t = train.drop(columns=['click'])\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_t, Y_t, test_size=0.25, random_state=69)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "ab53a7b64f4e4b099613836910a5d36a5c11c70d"
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "num_round = 1000\n",
    "param = {'objective': 'binary:logistic',  \n",
    "         'tree_method': 'hist', \n",
    "         'max_depth': 10,\n",
    "         'eta': 0.2,\n",
    "         'eval_metric': 'auc',\n",
    "         'early_stopping_rounds':50,\n",
    "         'nthread': 4,\n",
    "         'max_bin':512,\n",
    "         #'grow_policy':'lossguide',\n",
    "         'random_state':69,\n",
    "         }\n",
    "dtrain = xgb.DMatrix(X_train, label=y_train, missing = -1)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test, missing = -1)\n",
    "gbm = xgb.train(param, dtrain, num_round, evals=[(dtest, 'test')])\n",
    "predicted = gbm.predict(xgb.DMatrix(X_test, missing = -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_uuid": "57f1452bdd2a89466596735cdb36e7c723f7e3b1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2906408   18832]\n",
      " [  41522   67691]]\n",
      "0.8066847017209796\n",
      "0.980110418582855\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, roc_auc_score, accuracy_score as auc \n",
    "predicted = gbm.predict(xgb.DMatrix(X_test, missing = -1))\n",
    "print(confusion_matrix(y_test, np.round(predicted)))\n",
    "print(roc_auc_score(y_test, np.round(predicted)))\n",
    "print(auc(y_test, np.round(predicted)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_uuid": "5d1a1d74faee9b64ba74c9d8c3c9113bd6da5066"
   },
   "outputs": [],
   "source": [
    "\n",
    "preds = gbm.predict(xgb.DMatrix(test[test.columns[1:]], missing = -1))\n",
    "sub = pd.DataFrame({'ID':test['ID'], 'click':preds})\n",
    "sub.to_csv('submission_xg.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "a57049d6ca435a9cedfd66e78298b70e3e5070e9"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}