Data Classification framework.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import linear_model\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import cross_val_score\n",
    "pd.options.mode.chained_assignment = None\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.impute import SimpleImputer\n",
    "from impyute.imputation.cs import fast_knn\n",
    "\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk(r'E:\\Documents\\Kaggle Projects\\Categorical feature encoding'):\n",
    "    for filename in filenames:\n",
    "        file=os.path.join(dirname, filename)\n",
    "        if filename=='train.csv':\n",
    "            train= file\n",
    "        if filename=='test.csv':\n",
    "            test= file\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reduce memory usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Reduce memory usage'''\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "\n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "\n",
    "    return df\n",
    "#####################################\n",
    "#df_train=reduce_mem_usage(df_train)\n",
    "#df_test=reduce_mem_usage(df_test)\n",
    "#####################################\n",
    "'''Variable Description'''\n",
    "def description(df):\n",
    "    #print(f\"Dataset Shape: {df.shape}\")\n",
    "    summary = pd.DataFrame(df.dtypes,columns=['dtypes'])\n",
    "    summary = summary.reset_index()\n",
    "    summary['Name'] = summary['index']\n",
    "    summary = summary[['Name','dtypes']]\n",
    "    summary['Missing'] = df.isnull().sum().values    \n",
    "    summary['Uniques'] = df.nunique().values\n",
    "    summary['First Value'] = df.iloc[0].values\n",
    "    summary['Second Value'] = df.iloc[1].values\n",
    "    summary['Third Value'] = df.iloc[2].values\n",
    "    return summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_encoding(label_df,label_col):\n",
    "    if label_col == None:\n",
    "        label_df = label_df.apply(lambda series: pd.Series(\n",
    "        LabelEncoder().fit_transform(series[series.notnull()]),\n",
    "        index=series[series.notnull()].index))\n",
    "    else:\n",
    "        label_df[label_col] = label_df[label_col].apply(lambda series: pd.Series(\n",
    "        LabelEncoder().fit_transform(series[series.notnull()]),\n",
    "        index=series[series.notnull()].index))\n",
    "    return label_df\n",
    "            \n",
    "def one_hot_encoding(one_df,one_col):\n",
    "    one_df = pd.concat([one_df,pd.get_dummies(one_df[one_col], prefix=one_col)],axis=1).drop(one_col,axis=1)\n",
    "    return one_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logreg(x_train,y_train,x_val):\n",
    "    logreg = LogisticRegression(solver='lbfgs')\n",
    "    logreg.fit(x_train, y_train)\n",
    "    Y_pred = logreg.predict(x_test)\n",
    "    logreg.score(x_train, y_train)\n",
    "    acc_log = round(logreg.score(x_train, y_train) * 100, 2)\n",
    "    return Y_pred ,acc_log\n",
    "\n",
    "def sgd(x_train,y_train,x_val):\n",
    "    sgd = linear_model.SGDClassifier(max_iter=5, tol=None)\n",
    "    sgd.fit(x_train, y_train)\n",
    "    Y_pred = sgd.predict(x_val)\n",
    "    sgd.score(x_train, y_train)\n",
    "    acc_sgd = round(sgd.score(x_train, y_train) * 100, 2)\n",
    "    return Y_pred ,acc_sgd\n",
    "\n",
    "def rnd_forest(x_train,y_train,x_val):\n",
    "    random_forest = RandomForestClassifier(n_estimators=100)\n",
    "    random_forest.fit(x_train, y_train)\n",
    "    Y_pred = random_forest.predict(x_val)\n",
    "    random_forest.score(x_train, y_train)\n",
    "    acc_random_forest = round(random_forest.score(x_train,y_train) * 100, 2)\n",
    "    return Y_pred ,acc_random_forest\n",
    "\n",
    "def knn(x_train,y_train,x_val):\n",
    "    knn = KNeighborsClassifier(n_neighbors = 3) \n",
    "    knn.fit(x_train, y_train)  \n",
    "    Y_pred = knn.predict(x_val) \n",
    "    acc_knn = round(knn.score(x_train, y_train) * 100, 2)\n",
    "    return Y_pred ,acc_knn\n",
    "\n",
    "def gaussian(x_train,y_train,x_val):\n",
    "    gaussian = GaussianNB() \n",
    "    gaussian.fit(x_train, y_train)  \n",
    "    Y_pred = gaussian.predict(x_val)  \n",
    "    acc_gaussian = round(gaussian.score(x_train,y_train) * 100, 2)\n",
    "    return Y_pred ,acc_gaussian\n",
    "\n",
    "def perceptron (x_train,y_train,x_val):\n",
    "    perceptron.fit(x_train, y_train)\n",
    "    Y_pred = perceptron.predict(x_val)\n",
    "    acc_perceptron = round(perceptron.score(x_train, y_train) * 100, 2)\n",
    "    return Y_pred ,acc_perceptron\n",
    "\n",
    "def linearsvc(x_train,y_train,x_val):\n",
    "    linear_svc = LinearSVC()\n",
    "    linear_svc.fit(x_train, y_train)\n",
    "    Y_pred = linear_svc.predict(x_val)\n",
    "    acc_linear_svc = round(linear_svc.score(x_train, y_train) * 100, 2)\n",
    "    return Y_pred ,acc_linear_svc\n",
    "\n",
    "def decisiontree(x_train,y_train,x_val):\n",
    "    decision_tree = DecisionTreeClassifier() \n",
    "    decision_tree.fit(x_train, y_train)  \n",
    "    Y_pred = decision_tree.predict(x_val) \n",
    "    acc_decision_tree = round(decision_tree.score(x_train, y_train) * 100, 2)\n",
    "    return Y_pred ,acc_decision_tree\n",
    "\n",
    "\n",
    "def featureimportance(x_train, y_train):\n",
    "    random_forest = RandomForestClassifier(n_estimators=100)\n",
    "    random_forest.fit(x_train, y_train)\n",
    "    importances = pd.DataFrame({'feature':x_train.columns,'importance':np.round(random_forest.feature_importances_,3)})\n",
    "    importances = importances.sort_values('importance',ascending=False).set_index('feature')\n",
    "    return importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}