Extremely Randomized Trees_tuning_hyper_Approach4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/javi_fdez_093/.local/lib/python3.5/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/home/javi_fdez_093/.local/lib/python3.5/site-packages/sklearn/grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LogisticRegression #logistic regression\n",
    "from sklearn import svm #support vector Machine\n",
    "from sklearn.ensemble import RandomForestClassifier #Random Forest\n",
    "from sklearn.model_selection import train_test_split #training and testing data split\n",
    "from sklearn import metrics #accuracy measure\n",
    "from sklearn.metrics import confusion_matrix #for confusion matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from time import time\n",
    "from operator import itemgetter\n",
    "from scipy.stats import randint\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.tree import export_graphviz\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from sklearn.cross_validation import  cross_val_score\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import subprocess\n",
    "from time import time\n",
    "from operator import itemgetter\n",
    "from scipy.stats import randint\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "print(__doc__)\n",
    "import numpy as np\n",
    "from time import time\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import export_graphviz\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from sklearn.cross_validation import  cross_val_score\n",
    "from pprint import pprint\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "\n",
    "filename = 'df_imputed_scaled_OHE_reducedv1.pkl'\n",
    "df = pd.read_pickle(filename)\n",
    "\n",
    "X = df[df.columns[:-1]]\n",
    "y = pd.Series(df['heartdisease'])\n",
    "\n",
    "\n",
    "train_X,test_X,train_y,test_y =train_test_split(X,y,test_size=0.33,shuffle = True, random_state=45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def report(grid_scores, n_top=3):\n",
    "    \"\"\"Report top n_top parameters settings, default n_top=3.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    grid_scores -- output from grid or random search\n",
    "    n_top -- how many to report, of top models\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] top parameter settings found in\n",
    "                  search\n",
    "    \"\"\"\n",
    "    top_scores = sorted(grid_scores,\n",
    "                        key=itemgetter(1),\n",
    "                        reverse=True)[:n_top]\n",
    "    for i, score in enumerate(top_scores):\n",
    "        print(\"Model with rank: {0}\".format(i + 1))\n",
    "        print((\"Mean validation score: \"\n",
    "               \"{0:.3f} (std: {1:.3f})\").format(\n",
    "               score.mean_validation_score,\n",
    "               np.std(score.cv_validation_scores)))\n",
    "        print(\"Parameters: {0}\".format(score.parameters))\n",
    "        print(\"\")\n",
    "\n",
    "    return top_scores[0].parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_gridsearch(X, y, clf, param_grid, cv=5):\n",
    "    \"\"\"Run a grid search for best Decision Tree parameters.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    X -- features\n",
    "    y -- targets (classes)\n",
    "    cf -- scikit-learn Decision Tree\n",
    "    param_grid -- [dict] parameter settings to test\n",
    "    cv -- fold of cross-validation, default 5\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] from report()\n",
    "    \"\"\"\n",
    "    grid_search = GridSearchCV(clf,\n",
    "                               param_grid=param_grid,\n",
    "                               cv=cv)\n",
    "    start = time()\n",
    "    grid_search.fit(X, y)\n",
    "\n",
    "    print((\"\\nGridSearchCV took {:.2f} \"\n",
    "           \"seconds for {:d} candidate \"\n",
    "           \"parameter settings.\").format(time() - start,\n",
    "                len(grid_search.grid_scores_)))\n",
    "\n",
    "    top_params = report(grid_search.grid_scores_, 3)\n",
    "    return  top_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_randomsearch(X, y, clf, para_dist, cv=5,\n",
    "                     n_iter_search=100):\n",
    "    \"\"\"Run a random search for best Decision Tree parameters.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    X -- features\n",
    "    y -- targets (classes)\n",
    "    cf -- scikit-learn Decision Tree\n",
    "    param_dist -- [dict] list, distributions of parameters\n",
    "                  to sample\n",
    "    cv -- fold of cross-validation, default 5\n",
    "    n_iter_search -- number of random parameter sets to try,\n",
    "                     default 20.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] from report()\n",
    "    \"\"\"\n",
    "    random_search = RandomizedSearchCV(clf,\n",
    "                        param_distributions=param_dist,\n",
    "                        n_iter=n_iter_search)\n",
    "\n",
    "    start = time()\n",
    "    random_search.fit(X, y)\n",
    "    print((\"\\nRandomizedSearchCV took {:.2f} seconds \"\n",
    "           \"for {:d} candidates parameter \"\n",
    "           \"settings.\").format((time() - start),\n",
    "                               n_iter_search))\n",
    "\n",
    "    top_params = report(random_search.grid_scores_, 3)\n",
    "    return  top_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- 10-fold cross-validation [using setup from previous post]\n",
      "mean: 0.767 (std: 0.063)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"-- 10-fold cross-validation \"\n",
    "      \"[using setup from previous post]\")\n",
    "\n",
    "rfc_old = ExtraTreesClassifier(n_estimators=20)\n",
    "rfc_old.fit(X, y)\n",
    "scores = cross_val_score(rfc_old, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores.mean(),\n",
    "                                          scores.std()),\n",
    "                                          end=\"\\n\\n\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- Best Parameters:\n",
      "parameters: min_samples_split    setting: 2\n",
      "parameters: max_features         setting: sqrt\n",
      "parameters: max_depth            setting: 30\n",
      "parameters: min_samples_leaf     setting: 4\n",
      "parameters: bootstrap            setting: True\n",
      "parameters: n_estimators         setting: 2000\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n-- Best Parameters:\")\n",
    "for k, v in ts_rs.items():\n",
    "    print(\"parameters: {:<20s} setting: {}\".format(k, v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- Random Parameter Search via 10-fold CV\n",
      "{'bootstrap': [True, False],\n",
      " 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None],\n",
      " 'max_features': ['auto', 'sqrt'],\n",
      " 'min_samples_leaf': [1, 2, 4],\n",
      " 'min_samples_split': [2, 5, 10],\n",
      " 'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000]}\n",
      "\n",
      "RandomizedSearchCV took 430.57 seconds for 100 candidates parameter settings.\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.803 (std: 0.024)\n",
      "Parameters: {'max_depth': 50, 'min_samples_split': 2, 'n_estimators': 1600, 'max_features': 'sqrt', 'bootstrap': True, 'min_samples_leaf': 2}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.802 (std: 0.026)\n",
      "Parameters: {'max_depth': 30, 'min_samples_split': 2, 'n_estimators': 600, 'max_features': 'sqrt', 'bootstrap': True, 'min_samples_leaf': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.802 (std: 0.030)\n",
      "Parameters: {'max_depth': None, 'min_samples_split': 2, 'n_estimators': 1600, 'max_features': 'sqrt', 'bootstrap': True, 'min_samples_leaf': 2}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"-- Random Parameter Search via 10-fold CV\")\n",
    "from sklearn.model_selection import KFold,StratifiedKFold \n",
    "\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=10, random_state=1)\n",
    "\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "# Create the random grid\n",
    "param_dist = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}\n",
    "pprint(param_dist)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "rfc = ExtraTreesClassifier()\n",
    "ts_rs = run_randomsearch(X, y, rfc, param_dist, cv=kfold, n_iter_search=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- Best Parameters:\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'ts_rs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-48606ad37f94>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n-- Best Parameters:\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mts_rs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"parameters: {:<20s} setting: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'ts_rs' is not defined"
     ]
    }
   ],
   "source": [
    "print(\"\\n-- Best Parameters:\")\n",
    "for k, v in ts_rs.items():\n",
    "    print(\"parameters: {:<20s} setting: {}\".format(k, v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "-- Testing best parameters [Random]...\n",
      "mean: 0.783 (std: 0.072)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# test the retuned best parameters\n",
    "print(\"\\n\\n-- Testing best parameters [Random]...\")\n",
    "dt_ts_rs = RandomForestClassifier(**ts_rs)\n",
    "scores = cross_val_score(dt_ts_rs, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores.mean(),\n",
    "                                          scores.std()),\n",
    "                                          end=\"\\n\\n\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "-- Testing best parameters [Random] CV=3 ...\n",
      "mean: 0.791 (std: 0.068)\n",
      "\n",
      "\n",
      "\n",
      "-- Testing best parameters [Random] CV=5 ...\n",
      "mean: 0.781 (std: 0.072)\n",
      "\n",
      "\n",
      "\n",
      "-- Testing best parameters [Random] CV=10 ...\n",
      "mean: 0.778 (std: 0.075)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# test the retuned best parameters\n",
    "print(\"\\n\\n-- Testing best parameters [Random] CV=3 ...\")\n",
    "dt_ts_rs = RandomForestClassifier(n_estimators= 400,\n",
    "               max_features= 'sqrt',\n",
    "               max_depth= 30,\n",
    "               min_samples_split= 2,\n",
    "               min_samples_leaf= 4,\n",
    "               bootstrap= True)\n",
    "\n",
    "scores_3fold = cross_val_score(dt_ts_rs, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores_3fold.mean(),\n",
    "                                          scores_3fold.std()),\n",
    "                                          end=\"\\n\\n\" )\n",
    "\n",
    "print(\"\\n\\n-- Testing best parameters [Random] CV=5 ...\")\n",
    "dt_ts_rs = RandomForestClassifier(n_estimators= 200,\n",
    "               max_features= 'auto',\n",
    "               max_depth= 20,\n",
    "               min_samples_split= 2,\n",
    "               min_samples_leaf= 4,\n",
    "               bootstrap= True)\n",
    "\n",
    "scores_5fold = cross_val_score(dt_ts_rs, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores_5fold.mean(),\n",
    "                                          scores_5fold.std()),\n",
    "                                          end=\"\\n\\n\" )\n",
    "\n",
    "print(\"\\n\\n-- Testing best parameters [Random] CV=10 ...\")\n",
    "\n",
    "dt_ts_rs = RandomForestClassifier(n_estimators= 400,\n",
    "               max_features= 'auto',\n",
    "               max_depth= 60,\n",
    "               min_samples_split= 5,\n",
    "               min_samples_leaf= 4,\n",
    "               bootstrap= True)\n",
    "\n",
    "scores_10fold = cross_val_score(dt_ts_rs, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores_10fold.mean(),\n",
    "                                          scores_10fold.std()),\n",
    "                                          end=\"\\n\\n\" )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    \n",
    "    errors = abs(predictions - test_labels)\n",
    "    pprint('this is errors ' + str(errors))\n",
    "    pprint('this is predictions ' + str(predictions))\n",
    "    mape = 100 * np.mean(abs(errors) / abs(test_labels))\n",
    "    \n",
    "    accuracy = 100 - mape\n",
    "    print('Model Performance')\n",
    "    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))\n",
    "    print('Accuracy = {:0.2f}%.'.format(accuracy))\n",
    "    \n",
    "    return accuracy\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy for RF base is  0.8256578947368421\n",
      "Test Accuracy for RF best is  0.819078947368421\n",
      "Training Accuracy for RF base is  1.0\n",
      "Training Accuracy for RF best is  0.8668831168831169\n"
     ]
    }
   ],
   "source": [
    "rfc_best = RandomForestClassifier(n_estimators= 400,\n",
    "               max_features= 'sqrt',\n",
    "               max_depth= 30,\n",
    "               min_samples_split= 2,\n",
    "               min_samples_leaf= 4,\n",
    "               bootstrap= True)\n",
    "rfc_base = RandomForestClassifier(n_estimators= 200, random_state = 42)\n",
    "\n",
    "\n",
    "\n",
    "rfc_base.fit(train_X, train_y)\n",
    "rfc_best.fit(train_X, train_y)\n",
    "prediction_base=rfc_base.predict(test_X)\n",
    "prediction_best=rfc_best.predict(test_X)\n",
    "\n",
    "\n",
    "print('Test Accuracy for RF base is ',metrics.accuracy_score(prediction_base,test_y))\n",
    "print('Test Accuracy for RF best is ',metrics.accuracy_score(prediction_best,test_y))\n",
    "\n",
    "\n",
    "prediction_base_train=rfc_base.predict(train_X)\n",
    "prediction_best_train=rfc_best.predict(train_X)\n",
    "print('Training Accuracy for RF base is ',metrics.accuracy_score(train_y,prediction_base_train))\n",
    "print('Training Accuracy for RF best is ',metrics.accuracy_score(train_y,prediction_best_train))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}