4c. k-Nearest Neighbors (average days).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preamble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Configure libraries\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import matplotlib as mpl\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "pd.set_option('display.width', 500)\n",
    "pd.set_option('display.max_columns', 100)\n",
    "pd.set_option('display.notebook_repr_html', True)\n",
    "import seaborn as sns\n",
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_context(\"poster\")\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "import geohash\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Funtion for cross-validation over a grid of parameters\n",
    "\n",
    "def cv_optimize(clf, parameters, X, y, n_jobs=1, n_folds=5, score_func=None):\n",
    "    if score_func:\n",
    "        gs = GridSearchCV(clf, param_grid=parameters, cv=n_folds, n_jobs=n_jobs, scoring=score_func)\n",
    "    else:\n",
    "        gs = GridSearchCV(clf, param_grid=parameters, n_jobs=n_jobs, cv=n_folds)\n",
    "    gs.fit(X, y)\n",
    "    print \"BEST\", gs.best_params_, gs.best_score_, gs.grid_scores_\n",
    "    best = gs.best_estimator_\n",
    "    return best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Reading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(312330, 11)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time_cat</th>\n",
       "      <th>time_num</th>\n",
       "      <th>time_cos</th>\n",
       "      <th>time_sin</th>\n",
       "      <th>day_cat</th>\n",
       "      <th>day_num</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>weekend</th>\n",
       "      <th>geohash</th>\n",
       "      <th>pickups</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22:00</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.923880</td>\n",
       "      <td>-0.382683</td>\n",
       "      <td>Monday</td>\n",
       "      <td>0.133929</td>\n",
       "      <td>0.666347</td>\n",
       "      <td>0.745642</td>\n",
       "      <td>0</td>\n",
       "      <td>dr5rst</td>\n",
       "      <td>5476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22:00</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.923880</td>\n",
       "      <td>-0.382683</td>\n",
       "      <td>Monday</td>\n",
       "      <td>0.133929</td>\n",
       "      <td>0.666347</td>\n",
       "      <td>0.745642</td>\n",
       "      <td>0</td>\n",
       "      <td>dr5xep</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02:00</td>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.793353</td>\n",
       "      <td>0.608761</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.300595</td>\n",
       "      <td>-0.312572</td>\n",
       "      <td>0.949894</td>\n",
       "      <td>0</td>\n",
       "      <td>dr5rek</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05:00</td>\n",
       "      <td>0.229167</td>\n",
       "      <td>0.130526</td>\n",
       "      <td>0.991445</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>0.461310</td>\n",
       "      <td>-0.970597</td>\n",
       "      <td>0.240712</td>\n",
       "      <td>0</td>\n",
       "      <td>dr72qr</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08:00</td>\n",
       "      <td>0.354167</td>\n",
       "      <td>-0.608761</td>\n",
       "      <td>0.793353</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>0.193452</td>\n",
       "      <td>0.347871</td>\n",
       "      <td>0.937542</td>\n",
       "      <td>0</td>\n",
       "      <td>dr70jn</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  time_cat  time_num  time_cos  time_sin    day_cat   day_num   day_cos   day_sin  weekend geohash  pickups\n",
       "0    22:00  0.937500  0.923880 -0.382683     Monday  0.133929  0.666347  0.745642        0  dr5rst     5476\n",
       "1    22:00  0.937500  0.923880 -0.382683     Monday  0.133929  0.666347  0.745642        0  dr5xep        3\n",
       "2    02:00  0.104167  0.793353  0.608761  Wednesday  0.300595 -0.312572  0.949894        0  dr5rek       18\n",
       "3    05:00  0.229167  0.130526  0.991445   Thursday  0.461310 -0.970597  0.240712        0  dr72qr       54\n",
       "4    08:00  0.354167 -0.608761  0.793353    Tuesday  0.193452  0.347871  0.937542        0  dr70jn        2"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Each line is of the format:\n",
    "# ((time_cat, time_num, time_cos, time_sin, day_cat, day_num, day_cos, day_sin, weekend, geohash), number of pickups)\n",
    "names = [\"time_cat\", \"time_num\", \"time_cos\", \"time_sin\", \"day_cat\", \"day_num\", \"day_cos\", \"day_sin\", \"weekend\", \"geohash\", \"pickups\"]\n",
    "dftaxi=pd.read_csv(\"./data/taxi_data.csv\", header=None, names = names)\n",
    "print dftaxi.shape\n",
    "\n",
    "# n = 50000\n",
    "# dftaxi = dftaxi.head(n)\n",
    "dftaxi.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False,  True,  True,  True, False, False,  True,  True,  True], dtype=bool)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "itrain, itest = train_test_split(xrange(dftaxi.shape[0]), train_size=0.8)\n",
    "mask=np.ones(dftaxi.shape[0], dtype='int')\n",
    "mask[itrain]=1\n",
    "mask[itest]=0\n",
    "mask = (mask==1)\n",
    "mask[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final preperation for machine learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Split off the features\n",
    "Xnames = [\"time_cat\", \"time_num\", \"time_cos\", \"time_sin\", \"day_cat\",\n",
    "          \"day_num\", \"day_cos\", \"day_sin\", \"weekend\", \"geohash\"]\n",
    "X = dftaxi[Xnames]\n",
    "\n",
    "# Split off the target (which will be the logarithm of the number of pickups (+1))\n",
    "y = np.log10(dftaxi['pickups']+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get the longitude and latitude from the geohash\n",
    "def decodegeo(geo, which):\n",
    "    if len(geo) == 6:\n",
    "        geodecoded = geohash.decode(geo)\n",
    "        return geodecoded[which]\n",
    "    else:\n",
    "        return 0\n",
    "X['latitude'] = X['geohash'].apply(lambda geo: decodegeo(geo, 0))\n",
    "X['longitude'] = X['geohash'].apply(lambda geo: decodegeo(geo, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time_cat</th>\n",
       "      <th>time_num</th>\n",
       "      <th>time_cos</th>\n",
       "      <th>time_sin</th>\n",
       "      <th>day_cat</th>\n",
       "      <th>day_num</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>weekend</th>\n",
       "      <th>geohash</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22:00</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.923880</td>\n",
       "      <td>-0.382683</td>\n",
       "      <td>Monday</td>\n",
       "      <td>0.133929</td>\n",
       "      <td>0.666347</td>\n",
       "      <td>0.745642</td>\n",
       "      <td>0</td>\n",
       "      <td>dr5rst</td>\n",
       "      <td>40.723572</td>\n",
       "      <td>-73.976440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22:00</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.923880</td>\n",
       "      <td>-0.382683</td>\n",
       "      <td>Monday</td>\n",
       "      <td>0.133929</td>\n",
       "      <td>0.666347</td>\n",
       "      <td>0.745642</td>\n",
       "      <td>0</td>\n",
       "      <td>dr5xep</td>\n",
       "      <td>40.734558</td>\n",
       "      <td>-73.690796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02:00</td>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.793353</td>\n",
       "      <td>0.608761</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>0.300595</td>\n",
       "      <td>-0.312572</td>\n",
       "      <td>0.949894</td>\n",
       "      <td>0</td>\n",
       "      <td>dr5rek</td>\n",
       "      <td>40.718079</td>\n",
       "      <td>-74.031372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05:00</td>\n",
       "      <td>0.229167</td>\n",
       "      <td>0.130526</td>\n",
       "      <td>0.991445</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>0.461310</td>\n",
       "      <td>-0.970597</td>\n",
       "      <td>0.240712</td>\n",
       "      <td>0</td>\n",
       "      <td>dr72qr</td>\n",
       "      <td>40.866394</td>\n",
       "      <td>-73.899536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08:00</td>\n",
       "      <td>0.354167</td>\n",
       "      <td>-0.608761</td>\n",
       "      <td>0.793353</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>0.193452</td>\n",
       "      <td>0.347871</td>\n",
       "      <td>0.937542</td>\n",
       "      <td>0</td>\n",
       "      <td>dr70jn</td>\n",
       "      <td>40.816956</td>\n",
       "      <td>-74.306030</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  time_cat  time_num  time_cos  time_sin    day_cat   day_num   day_cos   day_sin  weekend geohash   latitude  longitude\n",
       "0    22:00  0.937500  0.923880 -0.382683     Monday  0.133929  0.666347  0.745642        0  dr5rst  40.723572 -73.976440\n",
       "1    22:00  0.937500  0.923880 -0.382683     Monday  0.133929  0.666347  0.745642        0  dr5xep  40.734558 -73.690796\n",
       "2    02:00  0.104167  0.793353  0.608761  Wednesday  0.300595 -0.312572  0.949894        0  dr5rek  40.718079 -74.031372\n",
       "3    05:00  0.229167  0.130526  0.991445   Thursday  0.461310 -0.970597  0.240712        0  dr72qr  40.866394 -73.899536\n",
       "4    08:00  0.354167 -0.608761  0.793353    Tuesday  0.193452  0.347871  0.937542        0  dr70jn  40.816956 -74.306030"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create indicator variables for the hours and days of the week and drop the categorical values\n",
    "# g = 5\n",
    "X = X.join(pd.get_dummies(X['time_cat']))\\\n",
    "     .join(pd.get_dummies(X['day_cat']))\\\n",
    "     .drop(['time_cat','day_cat','geohash'], axis=1)\n",
    "#     .join(pd.get_dummies(X['geohash'].str[:g]))\\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 312330 entries, 0 to 312329\n",
      "Data columns (total 40 columns):\n",
      "time_num     312330 non-null float64\n",
      "time_cos     312330 non-null float64\n",
      "time_sin     312330 non-null float64\n",
      "day_num      312330 non-null float64\n",
      "day_cos      312330 non-null float64\n",
      "day_sin      312330 non-null float64\n",
      "weekend      312330 non-null int64\n",
      "latitude     312330 non-null float64\n",
      "longitude    312330 non-null float64\n",
      "00:00        312330 non-null float64\n",
      "01:00        312330 non-null float64\n",
      "02:00        312330 non-null float64\n",
      "03:00        312330 non-null float64\n",
      "04:00        312330 non-null float64\n",
      "05:00        312330 non-null float64\n",
      "06:00        312330 non-null float64\n",
      "07:00        312330 non-null float64\n",
      "08:00        312330 non-null float64\n",
      "09:00        312330 non-null float64\n",
      "10:00        312330 non-null float64\n",
      "11:00        312330 non-null float64\n",
      "12:00        312330 non-null float64\n",
      "13:00        312330 non-null float64\n",
      "14:00        312330 non-null float64\n",
      "15:00        312330 non-null float64\n",
      "16:00        312330 non-null float64\n",
      "17:00        312330 non-null float64\n",
      "18:00        312330 non-null float64\n",
      "19:00        312330 non-null float64\n",
      "20:00        312330 non-null float64\n",
      "21:00        312330 non-null float64\n",
      "22:00        312330 non-null float64\n",
      "23:00        312330 non-null float64\n",
      "Friday       312330 non-null float64\n",
      "Monday       312330 non-null float64\n",
      "Saturday     312330 non-null float64\n",
      "Sunday       312330 non-null float64\n",
      "Thursday     312330 non-null float64\n",
      "Tuesday      312330 non-null float64\n",
      "Wednesday    312330 non-null float64\n",
      "dtypes: float64(39), int64(1)\n",
      "memory usage: 97.7 MB\n"
     ]
    }
   ],
   "source": [
    "X.head()\n",
    "X.info() # http://pandas.pydata.org/pandas-docs/stable/faq.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time_num</th>\n",
       "      <th>time_cos</th>\n",
       "      <th>time_sin</th>\n",
       "      <th>day_num</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>weekend</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>00:00</th>\n",
       "      <th>01:00</th>\n",
       "      <th>02:00</th>\n",
       "      <th>03:00</th>\n",
       "      <th>04:00</th>\n",
       "      <th>05:00</th>\n",
       "      <th>06:00</th>\n",
       "      <th>07:00</th>\n",
       "      <th>08:00</th>\n",
       "      <th>09:00</th>\n",
       "      <th>10:00</th>\n",
       "      <th>11:00</th>\n",
       "      <th>12:00</th>\n",
       "      <th>13:00</th>\n",
       "      <th>14:00</th>\n",
       "      <th>15:00</th>\n",
       "      <th>16:00</th>\n",
       "      <th>17:00</th>\n",
       "      <th>18:00</th>\n",
       "      <th>19:00</th>\n",
       "      <th>20:00</th>\n",
       "      <th>21:00</th>\n",
       "      <th>22:00</th>\n",
       "      <th>23:00</th>\n",
       "      <th>Friday</th>\n",
       "      <th>Monday</th>\n",
       "      <th>Saturday</th>\n",
       "      <th>Sunday</th>\n",
       "      <th>Thursday</th>\n",
       "      <th>Tuesday</th>\n",
       "      <th>Wednesday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.923880</td>\n",
       "      <td>-0.382683</td>\n",
       "      <td>0.133929</td>\n",
       "      <td>0.666347</td>\n",
       "      <td>0.745642</td>\n",
       "      <td>0</td>\n",
       "      <td>40.723572</td>\n",
       "      <td>-73.976440</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.793353</td>\n",
       "      <td>0.608761</td>\n",
       "      <td>0.300595</td>\n",
       "      <td>-0.312572</td>\n",
       "      <td>0.949894</td>\n",
       "      <td>0</td>\n",
       "      <td>40.718079</td>\n",
       "      <td>-74.031372</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.229167</td>\n",
       "      <td>0.130526</td>\n",
       "      <td>0.991445</td>\n",
       "      <td>0.461310</td>\n",
       "      <td>-0.970597</td>\n",
       "      <td>0.240712</td>\n",
       "      <td>0</td>\n",
       "      <td>40.866394</td>\n",
       "      <td>-73.899536</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.354167</td>\n",
       "      <td>-0.608761</td>\n",
       "      <td>0.793353</td>\n",
       "      <td>0.193452</td>\n",
       "      <td>0.347871</td>\n",
       "      <td>0.937542</td>\n",
       "      <td>0</td>\n",
       "      <td>40.816956</td>\n",
       "      <td>-74.306030</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.020833</td>\n",
       "      <td>0.991445</td>\n",
       "      <td>0.130526</td>\n",
       "      <td>0.288690</td>\n",
       "      <td>-0.240712</td>\n",
       "      <td>0.970597</td>\n",
       "      <td>0</td>\n",
       "      <td>40.619202</td>\n",
       "      <td>-73.943481</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   time_num  time_cos  time_sin   day_num   day_cos   day_sin  weekend   latitude  longitude  00:00  01:00  02:00  03:00  04:00  05:00  06:00  07:00  08:00  09:00  10:00  11:00  12:00  13:00  14:00  15:00  16:00  17:00  18:00  19:00  20:00  21:00  22:00  23:00  Friday  Monday  Saturday  Sunday  Thursday  Tuesday  Wednesday\n",
       "0  0.937500  0.923880 -0.382683  0.133929  0.666347  0.745642        0  40.723572 -73.976440      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      1      0       0       1         0       0         0        0          0\n",
       "2  0.104167  0.793353  0.608761  0.300595 -0.312572  0.949894        0  40.718079 -74.031372      0      0      1      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0       0       0         0       0         0        0          1\n",
       "3  0.229167  0.130526  0.991445  0.461310 -0.970597  0.240712        0  40.866394 -73.899536      0      0      0      0      0      1      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0       0       0         0       0         1        0          0\n",
       "4  0.354167 -0.608761  0.793353  0.193452  0.347871  0.937542        0  40.816956 -74.306030      0      0      0      0      0      0      0      0      1      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0       0       0         0       0         0        1          0\n",
       "7  0.020833  0.991445  0.130526  0.288690 -0.240712  0.970597        0  40.619202 -73.943481      1      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0      0       0       0         0       0         0        0          1"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtrain, Xtest, ytrain, ytest = X[mask], X[~mask], y[mask], y[~mask]\n",
    "n_samples = Xtrain.shape[0]\n",
    "n_features = Xtrain.shape[1]\n",
    "Xtrain.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### k-Nearest Neighbors Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create a k-Nearest Neighbors Regression estimator\n",
    "knn_estimator = KNeighborsRegressor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normalize Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xtrain_mean = Xtrain.mean()\n",
    "Xtrain_std_dev = Xtrain.std()\n",
    "Xtrain_normalized = (Xtrain - Xtrain_mean)/Xtrain_std_dev\n",
    "Xtest_normalized = (Xtest - Xtrain_mean)/Xtrain_std_dev\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEST {'n_neighbors': 2} -0.184266248657 [mean: -0.25184, std: 0.00527, params: {'n_neighbors': 1}, mean: -0.18427, std: 0.00418, params: {'n_neighbors': 2}, mean: -0.18792, std: 0.00331, params: {'n_neighbors': 5}]\n",
      "CPU times: user 24min 46s, sys: 2.04 s, total: 24min 48s\n",
      "Wall time: 24min 48s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Define a grid of parameters over which to optimize the knn regressor\n",
    "# We will figure out which number of neighbors is optimal\n",
    "#knn_parameters = {\"n_neighbors\": [1,2,5,10,20,50,100]}\n",
    "knn_parameters = {\"n_neighbors\": [1,2,5]}\n",
    "knn_best = cv_optimize(knn_estimator, knn_parameters, Xtrain_normalized, ytrain, score_func='mean_squared_error')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### k-Nearest Neighbors Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############# based on standard predict ################\n",
      "R^2 on training data: 0.9480\n",
      "R^2 on test data:     0.8542\n"
     ]
    }
   ],
   "source": [
    "# Fit the best Random Forest and calculate R^2 values for training and test sets\n",
    "knn_reg=knn_best.fit(Xtrain_normalized, ytrain)\n",
    "knn_training_accuracy = knn_reg.score(Xtrain_normalized, ytrain)\n",
    "knn_test_accuracy = knn_reg.score(Xtest_normalized, ytest)\n",
    "print \"############# based on standard predict ################\"\n",
    "print \"R^2 on training data: %0.4f\" % (knn_training_accuracy)\n",
    "print \"R^2 on test data:     %0.4f\" % (knn_test_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  2,   3],\n",
       "       [  1,   1],\n",
       "       [ 10,  19],\n",
       "       ..., \n",
       "       [  1,   1],\n",
       "       [276, 870],\n",
       "       [  9,  43]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show some of the predictions vs. the real number of pickups\n",
    "# predictions vs. real number of pickups\n",
    "np.round(np.power(10,np.column_stack((knn_reg.predict(Xtest_normalized),ytest))) - 1,decimals=0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.40400787441828129"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the Root Mean Squared Error\n",
    "np.sqrt(mean_squared_error(knn_reg.predict(Xtest_normalized),ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}