SVC_Linear_tuninghyper_Approach2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LogisticRegression #logistic regression\n",
    "from sklearn import svm #support vector Machine\n",
    "from sklearn.ensemble import RandomForestClassifier #Random Forest\n",
    "from sklearn.model_selection import train_test_split #training and testing data split\n",
    "from sklearn import metrics #accuracy measure\n",
    "from sklearn.metrics import confusion_matrix #for confusion matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from time import time\n",
    "from operator import itemgetter\n",
    "from scipy.stats import randint\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.tree import export_graphviz\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from sklearn.cross_validation import  cross_val_score\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import subprocess\n",
    "from time import time\n",
    "from operator import itemgetter\n",
    "from scipy.stats import randint\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "print(__doc__)\n",
    "import numpy as np\n",
    "from time import time\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import export_graphviz\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from sklearn.cross_validation import  cross_val_score\n",
    "from pprint import pprint\n",
    "\n",
    "\n",
    "filename = 'df_imputed__OHE_cat_std_num.pkl'\n",
    "df = pd.read_pickle(filename)\n",
    "\n",
    "X = df[df.columns[:-1]]\n",
    "y = pd.Series(df['heartdisease'])\n",
    "\n",
    "\n",
    "train_X,test_X,train_y,test_y =train_test_split(X,y,test_size=0.33,shuffle = True, random_state=45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def report(grid_scores, n_top=3):\n",
    "    \"\"\"Report top n_top parameters settings, default n_top=3.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    grid_scores -- output from grid or random search\n",
    "    n_top -- how many to report, of top models\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] top parameter settings found in\n",
    "                  search\n",
    "    \"\"\"\n",
    "    top_scores = sorted(grid_scores,\n",
    "                        key=itemgetter(1),\n",
    "                        reverse=True)[:n_top]\n",
    "    for i, score in enumerate(top_scores):\n",
    "        print(\"Model with rank: {0}\".format(i + 1))\n",
    "        print((\"Mean validation score: \"\n",
    "               \"{0:.3f} (std: {1:.3f})\").format(\n",
    "               score.mean_validation_score,\n",
    "               np.std(score.cv_validation_scores)))\n",
    "        print(\"Parameters: {0}\".format(score.parameters))\n",
    "        print(\"\")\n",
    "\n",
    "    return top_scores[0].parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_randomsearch(X, y, clf, para_dist, cv=5,\n",
    "                     n_iter_search=100):\n",
    "    \"\"\"Run a random search for best Decision Tree parameters.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    X -- features\n",
    "    y -- targets (classes)\n",
    "    cf -- scikit-learn Decision Tree\n",
    "    param_dist -- [dict] list, distributions of parameters\n",
    "                  to sample\n",
    "    cv -- fold of cross-validation, default 5\n",
    "    n_iter_search -- number of random parameter sets to try,\n",
    "                     default 20.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] from report()\n",
    "    \"\"\"\n",
    "    random_search = RandomizedSearchCV(clf,\n",
    "                        param_distributions=param_dist,\n",
    "                        n_iter=n_iter_search)\n",
    "\n",
    "    start = time()\n",
    "    random_search.fit(X, y)\n",
    "    print((\"\\nRandomizedSearchCV took {:.2f} seconds \"\n",
    "           \"for {:d} candidates parameter \"\n",
    "           \"settings.\").format((time() - start),\n",
    "                               n_iter_search))\n",
    "\n",
    "    top_params = report(random_search.grid_scores_, 3)\n",
    "    return  top_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- Random Parameter Search via 3-fold CV\n",
      "{'C': array([1.e-05, 1.e-04, 1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02,\n",
      "       1.e+03, 1.e+04, 1.e+05]),\n",
      " 'gamma': array([1.e-05, 1.e-04, 1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02,\n",
      "       1.e+03, 1.e+04, 1.e+05]),\n",
      " 'kernel': ['linear']}\n",
      "\n",
      "RandomizedSearchCV took 8491.06 seconds for 100 candidates parameter settings.\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.789 (std: 0.016)\n",
      "Parameters: {'kernel': 'linear', 'gamma': 10000.0, 'C': 0.01}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.789 (std: 0.016)\n",
      "Parameters: {'kernel': 'linear', 'gamma': 0.01, 'C': 0.01}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.789 (std: 0.016)\n",
      "Parameters: {'kernel': 'linear', 'gamma': 1e-05, 'C': 0.01}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"-- Random Parameter Search via 3-fold CV\")\n",
    "\n",
    "C_range = np.logspace(-5, 5, 11)\n",
    "gamma_range = np.logspace(-5, 5, 11)\n",
    "param_dist = {'kernel': ['linear'],\n",
    "              'gamma': gamma_range,\n",
    "               'C': C_range}\n",
    "\n",
    "pprint(param_dist)\n",
    "SVM_linear = svm.SVC()\n",
    "SVM_linear_rs = run_randomsearch(X, y, SVM_linear, param_dist, cv=3, n_iter_search=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "-- Testing best parameters [Random, CV = 3]...\n",
      "mean: 0.794 (std: 0.064)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# test the retuned best parameters\n",
    "print(\"\\n\\n-- Testing best parameters [Random, CV = 3]...\")\n",
    "SVM_linear_rs_model = svm.SVC(**SVM_linear_rs)\n",
    "scores = cross_val_score(SVM_linear_rs_model, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores.mean(),\n",
    "                                          scores.std()),\n",
    "                                          end=\"\\n\\n\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy for SVM_linear base is  0.819078947368421\n",
      "Test Accuracy for SVM_linear best is  0.8519736842105263\n",
      "Training Accuracy for SVM_linear base is  0.7840909090909091\n",
      "Training Accuracy for SVM_linear best is  0.7857142857142857\n"
     ]
    }
   ],
   "source": [
    "SVM_linear_rs_model = svm.SVC(**SVM_linear_rs)\n",
    "SVM_linear_base_model = svm.SVC(kernel='linear', C=0.05, gamma = 0.001)\n",
    "\n",
    "\n",
    "\n",
    "SVM_linear_base_model.fit(train_X, train_y)\n",
    "SVM_linear_rs_model.fit(train_X, train_y)\n",
    "prediction_base=SVM_linear_base_model.predict(test_X)\n",
    "prediction_best=SVM_linear_rs_model.predict(test_X)\n",
    "\n",
    "\n",
    "print('Test Accuracy for SVM_linear base is ',metrics.accuracy_score(prediction_base,test_y))\n",
    "print('Test Accuracy for SVM_linear best is ',metrics.accuracy_score(prediction_best,test_y))\n",
    "\n",
    "\n",
    "prediction_base_train=SVM_linear_base_model.predict(train_X)\n",
    "prediction_best_train=SVM_linear_rs_model.predict(train_X)\n",
    "print('Training Accuracy for SVM_linear base is ',metrics.accuracy_score(train_y,prediction_base_train))\n",
    "print('Training Accuracy for SVM_linear best is ',metrics.accuracy_score(train_y,prediction_best_train))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "default: ‘liblinear’ Algorithm to use in the optimization problem.\n",
    "For small datasets, ‘liblinear’ is a good choice, whereas ‘sag’ and\n",
    "‘saga’ are faster for large ones.\n",
    "For multiclass problems, only ‘newton-cg’, ‘sag’, ‘saga’ and ‘lbfgs’\n",
    "handle multinomial loss; ‘liblinear’ is limited to one-versus-rest schemes.\n",
    "‘newton-cg’, ‘lbfgs’ and ‘sag’ only handle L2 penalty, whereas\n",
    "‘liblinear’ and ‘saga’ handle L1 penalty.\n",
    "'''\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}