Modeling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Modeling.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "srnkYN3XtBG7",
        "0ZPB0nmx_nzT",
        "QWx3UslRs5_Z",
        "1KrfWuvMs8dP"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "74b38c71a0bf429b9410639ad163f05c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_527409c0cfc649f8a7c92e763144b08c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_26b137ad3f174023a302bddf59b81900",
              "IPY_MODEL_4146c4dd98fe415cb9026492d1ff22e2"
            ]
          }
        },
        "527409c0cfc649f8a7c92e763144b08c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "26b137ad3f174023a302bddf59b81900": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0ab27068e3434e97a119812517ac3c0b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6232b8b4f2d047218bd49978714a12be"
          }
        },
        "4146c4dd98fe415cb9026492d1ff22e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ab590566b5fa4726b27579b9049b17d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 253kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2a83c1ae880b4db1964b74643c778851"
          }
        },
        "0ab27068e3434e97a119812517ac3c0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6232b8b4f2d047218bd49978714a12be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab590566b5fa4726b27579b9049b17d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2a83c1ae880b4db1964b74643c778851": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJ-R3ooz_iOJ",
        "colab_type": "code",
        "outputId": "98115fdb-6226-402d-8b7e-8b59bfadb3c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import nltk\n",
        "import sklearn\n",
        "from sklearn import metrics\n",
        "import seaborn as sns\n",
        "from sklearn.metrics import mean_squared_error"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cU5BcOXW2S8V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "device = torch.device(\"cuda\") #set pytorch to be used on GPU"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMRZ6UXQ05GG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Get interface for working with PyTorch\n",
        "!pip install transformers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a0NDg6cule-R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import transformers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kAbDH75o3dI",
        "colab_type": "code",
        "outputId": "c361dee9-3879-4dce-fb49-da0d6baf4c46",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "source": [
        "import os\n",
        "from google.colab import drive\n",
        "\n",
        "# Mount google drive\n",
        "DRIVE_MOUNT='/content/gdrive'\n",
        "drive.mount(DRIVE_MOUNT)\n",
        "\n",
        "# create folder to write data to\n",
        "CIS545_FOLDER=os.path.join(DRIVE_MOUNT, 'My Drive', 'CIS545_2020')\n",
        "HOMEWORK_FOLDER=os.path.join(CIS545_FOLDER, 'Project')\n",
        "os.makedirs(HOMEWORK_FOLDER, exist_ok=True)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I6Z77Z2Jo37b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train = pd.read_csv('/content/gdrive/My Drive/CIS545_2020/Project/training_data.csv')\n",
        "x_test = pd.read_csv('/content/gdrive/My Drive/CIS545_2020/Project/testing_data.csv')\n",
        "y_train = pd.read_csv('/content/gdrive/My Drive/CIS545_2020/Project/train_labels.csv')\n",
        "y_test = pd.read_csv('/content/gdrive/My Drive/CIS545_2020/Project/test_labels.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZcyjEsEstPaH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train.drop(columns=['Unnamed: 0'], inplace = True)\n",
        "x_test.drop(columns=['Unnamed: 0'], inplace = True)\n",
        "y_train.drop(columns=['Unnamed: 0'], inplace = True)\n",
        "y_test.drop(columns=['Unnamed: 0'], inplace = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vCVIqBqh3ivU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fake = pd.read_csv('/content/gdrive/My Drive/CIS545_2020/Project/fakeForModeling.csv')\n",
        "true = pd.read_csv('/content/gdrive/My Drive/CIS545_2020/Project/trueForModeling.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z5wX1Nir_HbJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fake.drop(columns=['Unnamed: 0'], inplace = True)\n",
        "true.drop(columns=['Unnamed: 0'], inplace = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "srnkYN3XtBG7",
        "colab_type": "text"
      },
      "source": [
        "# Binomial Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pv31pm4A5EUg",
        "colab_type": "code",
        "outputId": "3f309499-da63-4578-9216-716c66327622",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# 2. instantiate a logistic regression model\n",
        "lr = LogisticRegression()\n",
        "\n",
        "lr.fit(x_train, np.ravel(y_train))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
              "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
              "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
              "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
              "                   warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XcFX6fyQ5JiI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred_lr = lr.predict(x_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8DYiAdLT5P6Z",
        "colab_type": "code",
        "outputId": "51e4dc23-a6ca-494f-c0c7-d99676cfcac3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "print (\"Accuracy is: \", metrics.accuracy_score(y_test, y_pred_lr))\n",
        "print(\"Mean Squared Error is:\", np.sqrt(mean_squared_error(y_test, y_pred_lr)))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy is:  0.8941226849373729\n",
            "Mean Squared Error is: 0.32538794547835836\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j4s3g8e6A0Fc",
        "colab_type": "code",
        "outputId": "3053ed46-e645-4671-b84e-ad227bfb90f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "lr_cm = metrics.confusion_matrix(y_test, y_pred_lr)\n",
        "print (lr_cm)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[4017  505]\n",
            " [ 484 4335]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HdtfsVL0B6Ef",
        "colab_type": "code",
        "outputId": "da67dae6-8b5a-433a-f40a-bc5583b4b7a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        }
      },
      "source": [
        "labels = np.array([['4017','505'],['484','4335']])\n",
        "sns.heatmap(lr_cm, annot=labels, fmt = '')\n",
        "plt.show()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAD4CAYAAAAw/yevAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAbCklEQVR4nO3de5xXVb3/8debQUwFBW9IAwoaZVAeMryVv0rzApQiXUw8Kcew0UIz8wfeOsdMLS+lJ/Pyi44kUop4B3+g8kM86kmuSiheYrzCAGICKtecmc/vj++CvuFcviPfYTbb95PHeszen7323mv7gI9r1l57b0UEZmaWLe3augFmZvZBTs5mZhnk5GxmlkFOzmZmGeTkbGaWQe1b+wTr597v6SD2AZ0OG9HWTbAMev/vNdriY/ztlZJzzna777vF52st7jmbmWVQq/eczcy2qvq6tm5BWTg5m1m+1NW2dQvKwsnZzHIlor6tm1AWTs5mli/1Ts5mZtnjnrOZWQb5hqCZWQa552xmlj3h2RpmZhnkG4JmZhnkYQ0zswzyDUEzswxyz9nMLIN8Q9DMLINyckPQrww1s1yJqCu5lEJShaRnJD2Y1ntJmimpWtKdkjqk+PZpvTpt71l0jAtT/CVJx5ZyXidnM8uXqC+9lOYc4IWi9auA6yLiE8BKYHiKDwdWpvh1qR6S+gAnAX2BAcBNkiqaO6mTs5nlS3196aUZkroDXwP+K60LOBK4O1UZC5yQlgenddL2r6b6g4HxEbEhIl4FqoGDmzu3k7OZ5UsLes6SqiTNKSpVmx3tP4FRwMZMvhuwKiI23nVcDFSm5UpgEUDa/k6qvynewD6N8g1BM8uXuvdLrhoRo4HRDW2T9HVgeUTMlfSV8jSudE7OZpYv5Zut8UXgeEmDgI8BOwO/ATpLap96x92BmlS/BugBLJbUHtgFeLsovlHxPo3ysIaZ5UuZbghGxIUR0T0ielK4ofdoRPwrMB34Vqo2DHggLU9M66Ttj0ZEpPhJaTZHL6A3MKu5y3DP2czypfXnOZ8PjJd0OfAMcEuK3wKMk1QNrKCQ0ImIBZImAM8DtcCIKGEen5OzmeVLKyTniHgMeCwtv0IDsy0iYj3w7Ub2vwK4oiXndHI2s1yJFtwQzDInZzPLF7/4yMwsg3Lybg0nZzPLF/eczcwyyD1nM7MMcs/ZzCyDav2yfTOz7HHP2cwsgzzmbGaWQe45m5llkHvOZmYZ5J6zmVkGebaGmVkGRbR1C8rCydnM8sVjzmZmGeTkbGaWQb4haGaWQXXNfgFqm+DkbGb5kpNhDX9928zypb6+9NIESR+TNEvSXyQtkHRpit8q6VVJ81Lpl+KSdL2kaknzJR1YdKxhkhamMqyxcxZzz9nM8qV8Y84bgCMjYrWk7YAnJU1J20ZGxN2b1R8I9E7lEOBm4BBJuwKXAP2BAOZKmhgRK5s6uXvOZpYrUR8llyaPU7A6rW6XSlM7DQZuS/vNADpL6gYcC0yNiBUpIU8FBjR3HU7OZpYvZRrWAJBUIWkesJxCgp2ZNl2Rhi6uk7R9ilUCi4p2X5xijcWb5ORsZvlSV1dykVQlaU5RqSo+VETURUQ/oDtwsKTPABcC+wMHAbsC57fGZTg5m1m+tKDnHBGjI6J/URnd0CEjYhUwHRgQEUvT0MUG4A/AwalaDdCjaLfuKdZYvElOzmaWL+WbrbGHpM5peQfgaODFNI6MJAEnAM+lXSYCp6ZZG4cC70TEUuBh4BhJXSR1AY5JsSZ5tkYZ1NXXM/Ti37Lnrjtzw8jTWLx8Bef/9nbeWb2WT/eq5Bc//A7btW/P3Bde4epxk1j4xjKuOnsoRx9yAACzFrzMr/44adPxXl3yFleddTJHHtS3rS7JymjhX2ewevVq6urqqa2t5dDDBtGlS2du/9PN7LNPD15/fRFDTz6TVave4UtfOox77xnDa68Vhijvu38yV1zxn218BduY8r34qBswVlIFhY7shIh4UNKjkvYABMwDzkz1JwODgGpgLXBaoTmxQtJlwOxU7+cRsaK5kzs5l8GfpjzJvpV7snrdegB+c8dkvjvwcAZ+oR+X3XIv902fzYlHH8Zeu3fmsjNPZOyDj//T/gf33Y8Jv/wxAO+sXsvXz72aww7ovdWvw1rPUUd/m7ff/sfMqVGjRvDo9Ce55pobGTlyBKNGjeCii34BwJNPzuKEISVNhbWGlOkhlIiYD3yugfiRjdQPYEQj28YAY1py/maHNSTtL+n8NLn6+rT86ZacJM/efHsVT8x7kSFHHARARDBrwcscfchnATj+f32eR+csAKByj1355N7daNdOjR5v6sxnOfxfPsUO23do/cZbmznuuGMZN+4uAMaNu4vjj292ZpWVqj5KLxnWZHKWdD4wnkL3fVYqAu6QdEHrNy/7rh43iXOHDqKdCgl31Xtr6bTTDrSvqACg6267sHzluyUf76Gn/sKAL/RrlbZa24gIpky+g5kzpnD68H8FoOueu7Ns2XIAli1bTtc9d99U/9BDP8/cOVOZNHEcffp8sk3avE1rwWyNLGtuWGM40Dci3i8OSroWWABc2dBOaTpKFcANF/2A4d84pgxNzZ7/fvoFdt25I3327c7s51/e4uO9tfJdqhct4wsH+B9knnzliCEsWbKMPfbYjYemjOfFl6o/UCfSOOkzzzzLfp84mDVr1jJgwJHcfdcY+vQ9fGs3eZsWOXm3RnPJuR74OPD6ZvFuaVuD0nSU0QDr596f7d8dtsC8v77GY08/z5PzXmLD+++zZt0Grr5tIu+tWUdtXR3tKyp48+132LPLziUd75EZ8zmyf1+2a1/Ryi23rWnJkmUAvPXW29z/wBQOOqgfby7/G3vttSfLli1nr732ZPlbbwPw3nurN+330EOP8tvrf8Fuu3X5p/Fqa0bGhytK1dyY84+BaZKmSBqdykPANOCc1m9etp1z0kCm3nAxU66/gKvOPpmD+u7HL88aykF99mPqzGcBmPjEXI7oX9qsiylPzfOQRs7suOMOdOy406blo4/6MgsWvMSDkx7hlFO+DcApp3ybSZMKM6u6dt1j074H9e9Hu3btnJhbKupLLxnWZM85Ih6S9EkKk6w3Pm5YA8yOiGwP2LShHw8dyKjf3s6Ndz3C/vt8nCFfKdwsfO7lRZx73W28u2Yd//30C9x091Tuu+Y8AGreWsGyt9+h/6d7tWXTrcy6dt2Du++6BYCK9hWMH38/jzzyGHPm/IU7bv8/nPZvQ3njjcUMPbkwG+ub3/gaVWecSl1tHevWree73/1hWzZ/25STnrOilT+GmOdhDfvwOh3W4Iwj+4h7/+81jU9lKtGa/zip5Jyz08/Hb/H5WovnOZtZvmR8uKJUTs5mli85GdZwcjazXPmoTKUzM9u2uOdsZpZBTs5mZhmU8ceyS+XkbGa50ty3AbcVTs5mli9OzmZmGeTZGmZmGeSes5lZBjk5m5llT9R5WMPMLHty0nNu9huCZmbbkqiPkktTJH1M0ixJf5G0QNKlKd5L0kxJ1ZLulNQhxbdP69Vpe8+iY12Y4i9JOraU63ByNrN8Kd8HXjcAR0bEvwD9gAGSDgWuAq6LiE8AKyl8zo/0c2WKX5fqIakPcBLQFxgA3CSp2c8dOTmbWb7Ut6A0IQo2fjdsu1QCOBK4O8XHAiek5cFpnbT9q5KU4uMjYkNEvApUU/iASZOcnM0sV6K2vuQiqUrSnKJSVXwsSRWS5gHLganAy8CqiKhNVRbzj69EVQKLANL2d4DdiuMN7NMo3xA0s3xpwWSN4o9RN7K9DugnqTNwH7D/ljavVE7OZpYrrfFujYhYJWk6cBjQWVL71DvuTuG7qqSfPYDFktoDuwBvF8U3Kt6nUR7WMLN8KdOYs6Q9Uo8ZSTsARwMvANOBb6Vqw4AH0vLEtE7a/mgUPtI6ETgpzeboBfQGZjV3Ge45m1mulLHn3A0Ym2ZWtAMmRMSDkp4Hxku6HHgGuCXVvwUYJ6kaWEFhhgYRsUDSBOB5oBYYkYZLmuTkbGb5UqYHBCNiPvC5BuKv0MBsi4hYD3y7kWNdAVzRkvM7OZtZrmyaR7GNc3I2s1yJfLxaw8nZzHLGydnMLHvcczYzyyAnZzOzDIo6tXUTysLJ2cxyxT1nM7MMinr3nM3MMsc9ZzOzDIpwz9nMLHPcczYzy6B6z9YwM8se3xA0M8sgJ2czswyK8n8IpU04OZtZrrjnbGaWQZ5KZ2aWQXWerWFmlj156Tn769tmlitRr5JLUyT1kDRd0vOSFkg6J8V/JqlG0rxUBhXtc6GkakkvSTq2KD4gxaolXVDKdbjnbGa5UsbZGrXAeRHxtKROwFxJU9O26yLiV8WVJfWh8MXtvsDHgf8n6ZNp843A0cBiYLakiRHxfFMnd3I2s1wp12yNiFgKLE3L70l6AahsYpfBwPiI2AC8Kqmaf3yluzp9tRtJ41PdJpOzhzXMLFfq6tuVXCRVSZpTVKoaOqaknsDngJkpdJak+ZLGSOqSYpXAoqLdFqdYY/EmOTmbWa5EtKTE6IjoX1RGb348SR2Be4AfR8S7wM3AfkA/Cj3rX7fGdXhYw8xypb6MszUkbUchMf8pIu4FiIg3i7b/HngwrdYAPYp2755iNBFvlHvOZpYrESq5NEWSgFuAFyLi2qJ4t6JqQ4Dn0vJE4CRJ20vqBfQGZgGzgd6SeknqQOGm4cTmrsM9ZzPLlTLO1vgicArwrKR5KXYRMFRSPyCA14AzCueNBZImULjRVwuMiIg6AElnAQ8DFcCYiFjQ3MkVrfyWkPYdKnPyGhIrp3VLnmjrJlgGbbf7vls8JjGn+wkl55z+i+/P7BMr7jmbWa7U1edjtNbJ2cxyJS+/qjs5m1mulHO2RltycjazXMnLi4+cnM0sV3Ly8W0nZzPLl8A9ZzOzzKn1sIaZWfa452xmlkEeczYzyyD3nM3MMsg9ZzOzDKpzz9nMLHvK9JWqNufkbGa5Uu+es5lZ9vjFR2ZmGeQbgmZmGVQvD2uYmWVOXVs3oEycnM0sV/IyWyMf33MxM0vqUcmlKZJ6SJou6XlJCySdk+K7SpoqaWH62SXFJel6SdWS5ks6sOhYw1L9hZKGlXIdTs5mlivRgtKMWuC8iOgDHAqMkNQHuACYFhG9gWlpHWAg0DuVKuBmKCRz4BLgEOBg4JKNCb0pTs5mliv1Kr00JSKWRsTTafk94AWgEhgMjE3VxgInpOXBwG1RMAPoLKkbcCwwNSJWRMRKYCowoLnrcHI2s1ypb0GRVCVpTlGpauiYknoCnwNmAl0jYmnatAzompYrgUVFuy1OscbiTfINQTPLlboW3BCMiNHA6KbqSOoI3AP8OCLeVdFUvYgISa3y3It7zmaWKy3pOTdH0nYUEvOfIuLeFH4zDVeQfi5P8RqgR9Hu3VOssXiTnJzNLFfKlZxV6CLfArwQEdcWbZoIbJxxMQx4oCh+apq1cSjwThr+eBg4RlKXdCPwmBRrkoc1zCxXyvgJwS8CpwDPSpqXYhcBVwITJA0HXgdOTNsmA4OAamAtcBpARKyQdBkwO9X7eUSsaO7kTs5mlivlerdGRDwJjU6G/moD9QMY0cixxgBjWnJ+J2czyxU/vm1mlkF5eXzbydnMcsWvDDUzyyAnZzOzDPKXUMzMMshjzmZmGeTZGmZmGVSfk4ENJ2czyxXfEDQzy6B89JudnM0sZ9xzNjPLoNrWeb3yVufkbGa5ko/U7ORsZjnjYQ0zswzyVDozswzKR2p2cjaznPGwhplZBtXlpO/s5GxmuZKXnrO/vm1muRIt+NMcSWMkLZf0XFHsZ5JqJM1LZVDRtgslVUt6SdKxRfEBKVYt6YJSrsPJ2cxypb4FpQS3AgMaiF8XEf1SmQwgqQ9wEtA37XOTpApJFcCNwECgDzA01W2ShzXKrF27dsycMYUlNcsYPGQYRx5xOFde+VPatWvHmtVr+N7p5/Lyy69tqj9kyCDuuvP3HHLoQOY+Pb/tGm5lV1dXx3eG/4g999idm665lH//5XUseHEhEUHPHpVccfF57LjjDtx53/9l/L0P0q5dO3bc8WP8bNSP2K/XPtQsfZPjT66i597dATig7/5cMursNr6q7CvnVLqIeFxSzxKrDwbGR8QG4FVJ1cDBaVt1RLwCIGl8qvt8Uwdzci6zH519Oi++uJCdO3UC4IYbfsk3vnkaL75YzZlnDOOiC89h+OnnAtCx40786KzhzJz5dFs22VrJH+96gH177s3qNWsBOP9HVXTcaScArr5+NLffM4nTTzmRrx3zFb4z5GsATH9iBlf/9vf87trLAehR2Y17xt7YNhewjWpJapZUBVQVhUZHxOgSdj1L0qnAHOC8iFgJVAIziuosTjGARZvFD2nuBB7WKKPKym4MGvhVxoy5Y1MsIjYl6l126cTSpW9u2nbpz0Zxza9uYv369Vu9rda6li1/i8f/PItvHrdp2HFTYo4I1m/YgPTPcYB169cj5eRTHm2klii5RMToiOhfVEpJzDcD+wH9gKXAr1vjOtxzLqNrf30pF1x4OZ06ddwUO+OM/82kieNYt2497773Hl88/DgAPtfvM/To0Y3JU6Zx3k/ObKsmWyu56je/4yc/HM6atev+Kf7TK67l8adms1/PvRl59vc3xe+4ZxJjx9/L+7W1jLn+yk3xmqXL+Na/jaDjTjty9veH8fl+n9lq17CtKuVG3xYdP2JTD0vS74EH02oN0KOoavcUo4l4oz50z1nSaU1sq5I0R9Kc+vo1H/YU25SvDTqK5cv/xtPPPPtP8XPO+T7HHX8KPfftz9ixd/Kray5BEr+65hJGjvp5G7XWWtNj/zOTXbt0pu/+vT+w7fKLf8L0B/7Ivj178NC0xzfFh37zOB666w/85Aff43e3Fn7z2mO3Lky99zbuvvVGRp5dxahLr2L1mo/Gv6ctUeYbgh8gqVvR6hBg40yOicBJkraX1AvoDcwCZgO9JfWS1IHCTcOJzZ1nS4Y1Lm1sQ/GvCu3a7dRYtVz5whf6c9zXj6H6rzP40x9v4ogjvsjE+2/jgM/2YdbsZwCYcNdEDjusP506daRv3/2ZNvVuqv86g0MOOZD77v0Dnz/wgDa+CiuHZ+Y/z2NPzuCYbw5j5CVXMmvuXzj/0qs3ba+oqGDgUV9m6mP/84F9Bx71ZR594ikAOnToQOdddgag7/696VHZjdfeaLbD9ZFX5ql0dwBPAZ+StFjScOBqSc9Kmg8cAZwLEBELgAkUbvQ9BIyIiLqIqAXOAh4GXgAmpLpNanJYI528wU1A12av7CPk4p9eycU/Lfw6+uUvHcZPzj2Tb3zre9Qsmkfv3vuycOErHPXVL/Hiiwt599332Ovjn92077SpdzHq/Ms8WyMnzv3BaZz7g8IvlrOens+td9zDlf8xkjcWL2Hv7h8nIpj+5Ax67VOYhfH6ohr26VG4b/T4n2exd/fC8oqVq9hl505UVFSwqGYpbyxaQo/Kbg2f1DYp50MoETG0gfAtTdS/AriigfhkYHJLzt3cmHNX4Fhg5WZxAX9uyYk+iurq6jjjByOZcOdo6uuDVStXcXrVeW3dLGsDEcFFl/+aNWvWEhF86hO9+PeRZwFw+z2TmDH7Gdq3b8/OnTryi58W/o7MnfccN/zXONq3b0+7duI/Rp7FLjt3asvL2CbURT4e31Y0cSGSbgH+EBFPNrDt9og4ubkTtO9QmY//UlZW65Y80dZNsAzabvd9t3iqysn7DCk559z++n2ZnRrTZM85IoY3sa3ZxGxmtrW19myNrcVT6cwsV/Ly4iMnZzPLFX8JxcwsgzysYWaWQXmZreHkbGa54mENM7MM8g1BM7MM8pizmVkGeVjDzCyDmnrqeVvi5GxmuVLnnrOZWfZ4WMPMLIM8rGFmlkHuOZuZZZCn0pmZZZAf3zYzyyAPa5iZZVBekvOWfH3bzCxzIqLk0hxJYyQtl/RcUWxXSVMlLUw/u6S4JF0vqVrSfEkHFu0zLNVfKGlYKdfh5GxmuVJPlFxKcCswYLPYBcC0iOgNTEvrAAOB3qlUATdDIZkDlwCHAAcDl2xM6E1xcjazXIkW/Gn2WBGPAys2Cw8GxqblscAJRfHbomAG0FlSN+BYYGpErIiIlcBUPpjwP8BjzmaWK3VR+ktDJVVR6OVuNDoiRjezW9eIWJqWlwFd03IlsKio3uIUayzeJCdnM8uVljwhmBJxc8m4qf1DUqvcgfSwhpnlSpnHnBvyZhquIP1cnuI1QI+iet1TrLF4k5yczSxXyjnm3IiJwMYZF8OAB4rip6ZZG4cC76Thj4eBYyR1STcCj0mxJnlYw8xypb6MTwhKugP4CrC7pMUUZl1cCUyQNBx4HTgxVZ8MDAKqgbXAaQARsULSZcDsVO/nEbH5TcYPnru13+DUvkNlPmaEW1mtW/JEWzfBMmi73ffVlh6jb9dDSs45C96cucXnay3uOZtZrrRktkaWOTmbWa6Uc1ijLTk5m1mu+JWhZmYZ5J6zmVkGuedsZpZBdVHX1k0oCydnM8sVf+DVzCyD8vKyfSdnM8sV95zNzDLIszXMzDLIszXMzDLIj2+bmWWQx5zNzDLIY85mZhnknrOZWQZ5nrOZWQa552xmlkGerWFmlkF5uSHor2+bWa5ERMmlOZJek/SspHmS5qTYrpKmSlqYfnZJcUm6XlK1pPmSDtyS63ByNrNciRb8KdEREdEvIvqn9QuAaRHRG5iW1gEGAr1TqQJu3pLrcHI2s1wpZ8+5EYOBsWl5LHBCUfy2KJgBdJbU7cOexMnZzHKlPqLkIqlK0pyiUrXZ4QJ4RNLcom1dI2JpWl4GdE3LlcCion0Xp9iH0uo3BGv/XqPWPse2QlJVRIxu63ZYtvjvRXl9iJzT1H/7wyOiRtKewFRJLxZvjIiQ1Cp3IN1z3ro2/7+yGfjvRWZFRE36uRy4DzgYeHPjcEX6uTxVrwF6FO3ePcU+FCdnM7MGSNpJUqeNy8AxwHPARGBYqjYMeCAtTwROTbM2DgXeKRr+aDHPczYza1hX4D5JUMiVt0fEQ5JmAxMkDQdeB05M9ScDg4BqYC1w2pacXHl51HFb4LFFa4j/XlhDnJzNzDLIY85mZhnk5GxmlkFOzluJpAGSXkrP3V/Q/B6Wd5LGSFou6bm2botlj5PzViCpAriRwrP3fYChkvq0bassA24FBrR1IyybnJy3joOB6oh4JSL+Doyn8By+fYRFxOPAirZuh2WTk/PWUdZn7s0s/5yczcwyyMl56yjrM/dmln9OzlvHbKC3pF6SOgAnUXgO38ysQU7OW0FE1AJnAQ8DLwATImJB27bK2pqkO4CngE9JWpze1WAG+PFtM7NMcs/ZzCyDnJzNzDLIydnMLIOcnM3MMsjJ2cwsg5yczcwyyMnZzCyD/j+NFGRCSa9YBAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w2IIbRmpk6Nm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "f9a1198a-26c8-453b-b73d-bc7400744e02"
      },
      "source": [
        "print(metrics.classification_report(y_test, y_pred_lr))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.89      0.89      4522\n",
            "           1       0.90      0.90      0.90      4819\n",
            "\n",
            "    accuracy                           0.89      9341\n",
            "   macro avg       0.89      0.89      0.89      9341\n",
            "weighted avg       0.89      0.89      0.89      9341\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9CY7X2FnCTdn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Calculate predicted probabilities for test data \n",
        "y_pred_prob_lr = lr.predict_proba(x_test)[:, 1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fz7AE2gP5bFE",
        "colab_type": "code",
        "outputId": "f2b14c73-08b5-4aee-b738-35e062f4c8ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "metrics.roc_auc_score(y_test, y_pred_prob_lr)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9554417686734811"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0ZPB0nmx_nzT",
        "colab_type": "text"
      },
      "source": [
        "# Naive Bayes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N0fdHCwj_qsi",
        "colab_type": "code",
        "outputId": "4c8d2a2f-89b2-4fa6-c0b6-93acd8316c2e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "from nltk import classify\n",
        "from nltk import NaiveBayesClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "nb = MultinomialNB()# one of the two classic naive Bayes variants used in text classification\n",
        "\n",
        "%time nb.fit(x_train, np.ravel(y_train))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 42.1 ms, sys: 1.46 ms, total: 43.6 ms\n",
            "Wall time: 39.6 ms\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YU82695MNHjV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred_class = nb.predict(x_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHOh_T8MCX8t",
        "colab_type": "code",
        "outputId": "a7d07552-b40f-449b-c5c9-459db655ceeb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "print(\"Accuracy is:\", metrics.accuracy_score(y_test, y_pred_class))\n",
        "print(\"Mean Squared Error is:\", np.sqrt(mean_squared_error(y_test, y_pred_class)))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy is: 0.8610427149127502\n",
            "Mean Squared Error is: 0.3727697480848597\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Y5GcLuCNPQK",
        "colab_type": "code",
        "outputId": "e24172fb-3223-4bae-ec78-bf646ef1397b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "nb_cm = metrics.confusion_matrix(y_test, y_pred_class)\n",
        "print(nb_cm)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[3876  646]\n",
            " [ 652 4167]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5-_gA6i4CjX0",
        "colab_type": "code",
        "outputId": "81a44919-343c-4134-d2f2-57581c4705d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        }
      },
      "source": [
        "labels =  np.array([['3876','646'],['652','4167']])\n",
        "sns.heatmap(nb_cm, annot=labels, fmt = '')\n",
        "plt.show()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAD4CAYAAAAw/yevAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAa3klEQVR4nO3deZgU1b3/8feXYVhcWNwQBxRU0IAiKLK4RIPK5s3FLQaTJyIhmUTh/twedyPicmPiQqImBhQUV8RolGtUJC6JJrIpCA5KmIjIjCPbIIgIMt3f3x99wEZmenpCz3RN5fPiOc90nTpVdcpn+PL11KnT5u6IiEi0NMl3B0REZGcKziIiEaTgLCISQQrOIiIRpOAsIhJBTev7ApvfflbTQWQne/Qfne8uSARVfVVuu3qOrWs+zDrmFO5z8C5fr74ocxYRiaB6z5xFRBpUMpHvHuSEgrOIxEuiKt89yAkFZxGJFfdkvruQEwrOIhIvSQVnEZHoUeYsIhJBMXkgqKl0IhIvnsy+ZMHMCsxsvpk9H7Y7m9lsMys1syfNrFmobx62S8P+TmnnuCbULzGzQdlcV8FZRGLFE1VZlyxdDLyftv0rYLy7HwqsA0aF+lHAulA/PrTDzLoBw4HuwGDg92ZWUNtFFZxFJF6SyexLLcysA3A68EDYNmAA8MfQZApwRvg8LGwT9p8S2g8Dprr7FndfBpQCfWq7toKziMRLHYY1zKzYzOalleJvnO03wJXAtki+N/CZu29Lu8uAovC5CFgBEPavD+2311dzTI30QFBE4qUODwTdfSIwsbp9ZvZfwCp3f9vMTs5N57Kn4Cwi8ZK7qXTHA/9tZkOBFkAr4LdAGzNrGrLjDkB5aF8OdATKzKwp0BpYm1a/TfoxNdKwhojES6Iq+5KBu1/j7h3cvROpB3qvuvsPgdeAc0KzEcBz4fP0sE3Y/6qnvqR1OjA8zOboDHQB5tR2G8qcRSRe6v8NwauAqWZ2CzAfmBTqJwGPmFkpUEkqoOPuJWY2DVgMVAGj3b3WsRcFZxGJlSzi3r9xTn8deD18/pBqZlu4+2bgezUcfytwa12uqeAsIvGi17dFRCJICx+JiESQMmcRkQhKbM13D3JCwVlE4kXDGiIiEaRhDRGRCFLmLCISQQrOIiLR43ogKCISQRpzFhGJIA1riIhEkDJnEZEIUuYsIhJBypxFRCKoKutv1Y40BWcRiRdlziIiEaQxZxGRCFLmLCISQcqcRUQiSJmziEgExWS2RpN8d0BEJKfcsy8ZmFkLM5tjZu+aWYmZjQv1D5nZMjNbEErPUG9mdreZlZrZQjM7Ou1cI8xsaSgjsrkNZc4iEi+5G3PeAgxw941mVgi8aWYvhn1XuPsfv9F+CNAllL7AfUBfM9sLGAv0Bhx428ymu/u6TBdX5iwi8ZJMZl8y8JSNYbMwlEzp9jDg4XDcLKCNmbUHBgEz3b0yBOSZwODabkPBWUTixZPZl1qYWYGZLQBWkQqws8OuW8PQxXgzax7qioAVaYeXhbqa6jNScBaReEkksi5mVmxm89JKcfqp3D3h7j2BDkAfMzsCuAY4HDgW2Au4qj5uQ8FZROKlDsMa7j7R3XunlYnVndLdPwNeAwa7e0UYutgCPAj0Cc3KgY5ph3UIdTXVZ6TgLCLxkqMxZzPb18zahM8tgdOAD8I4MmZmwBnAe+GQ6cD5YdZGP2C9u1cAM4CBZtbWzNoCA0NdRpqtISLxkruXUNoDU8ysgFQiO83dnzezV81sX8CABcDPQ/sXgKFAKbAJGAng7pVmdjMwN7S7yd0ra7u4grOIxIonM89fzvo87guBXtXUD6ihvQOja9g3GZhcl+srOItIvGhtDRGRCEok8t2DnFBwFpF4UeYsIhJBCs6y5autjLzpD2ytSlCVSHBa3yO56JyBzH6vlLse/zPuTsvmzbn559/jwP334fZH/o+5i/8FwJdbtrJuw0befGAcABVr1nHj/U+zcu1nmBn3XjmSon33yuftSY60bt2KiRPuoHv3w3B3fvrTy5k1+20ALr3kZ9z+6xto1/4I1q5NLbVw0rf7c+ed4ygsbMraNZUMOPWcfHa/8allQaPGQsF5FzQrbMoD1xezW4vmbK1KcMG4+zjhqMO4ZfKf+O3l53NwUTuenPkW9z/7Kjf//Fyu+NF3tx/7+Iy/88FHn2zfvv6+afzkjO/Q/8iubNq8hdQUSomD8XfdxIwZr/H94cUUFhay224tAejQ4QBOO/XbLF9etr1t69atuOee/+X0//ohK1Z8wr777p2vbjdeMcmca30JxcwON7OrwlJ4d4fP32qIzkWdmbFbi9Rr9VWJVPaMGWaw8cstAGzctJl927Ta6diX/rGAIccdBcC/ylZSlUjS/8iuAOzWojktmzdroLuQ+tSq1Z6ceEJfJj/4BABbt25l/foNANx5x41cfe2teFqmd97wM3n22RdZsSL1D/fq1WsbvtONXdKzLxGWMXM2s6uA84CpwJxQ3QF4wsymuvtt9dy/yEskk5x33d18/Olavj+wPz0OPZAbf3oOY379IM2bNWWPli14ZNyOUx8/Wb2O8tXr6NP9UACWV6xhz91bcOn4hylftY5+RxzKxecNoaCJXuBs7Dp3PpA1a9Yy6YHx9OjRjXfeWcill93AKaecSHl5BQsXLt6hfZcuB1NY2JRXZj7FHnvuwT33TuLRR7+5MqVkFJPZGrX97R8FHOvut7n7o6HcRupd8lE1HZS+mMikZ17OZX8jp6BJE6b98hJevvda3vvXCpau+JRHXnyDe68cycx7r2PYt3tzx6PP73DMS2+9y6l9jtwefBPJBPM/WMblPzidx28ZQ9mqSp7767x83I7kWNOCAnr1OpIJEx7m2D6D+OKLTYz9xeVcc9X/cOO4O3Zu37SAY47uwXeHnc/Q03/AdddcQpcuB+eh542XJ5NZlyirLTgngQOqqW8f9lUrfTGRUWcN3JX+NRqtdm/Jsd0O4e8LlvDP5RX0OPRAAAb178G7S5fv0Palt97dPqQB0G6v1hx20AF0aLc3TQsK+E7v7juMR0vjVVZeQVlZBXPmzgfgmWf+TK9eR9Kp04G8M28mpf+cRYcO7Zk7ewbt2u1LeXkFL898nU2bvmTt2nW88eYsevTolue7aGRiMqxRW3C+BHjFzF40s4mhvAS8Alxc/92LtsoNG9nwxZcAbP5qK7MWLaVz0X5s3LSZjypWA/DWoqV0PmC/7ccsK1/F5198yVFdDtpe1/2Qjny+aTOVG1Lres8pKeXgov2Qxm/lytWUlX1C166HADBgwAnMn7+IAzocxaFd+3Fo136UlVVwbN9BrFy5mun/N4Pjj+tDQUEBLVu2oE+fXnzwwdI830Ujk8P1nPMp45izu79kZl1JDWNsWxy6HJjr7vEY2NkFaz77nOvvm0YymSTpzsB+PTjp6G9xw0/P5vLfPEoTM1rt3pJxxV9PhXrprXcZ1P+oHWZjFDRpwmU/HErxrffjQLfORZw9oE81V5TG6OJLf8HDU+6hWbNCli37mFE/uazGth98UMqMl19j/jt/IZlMMnnyE5SULGnA3sZAxDPibJnX85zAzW8/G4//UpJTe/Svdn0Y+Q9X9VX5Ls8h/eKG4VnHnN1vmhrZOaua5ywi8RLx4YpsKTiLSLzEZFhDwVlEYiXqU+SypeAsIvGizFlEJIIUnEVEIigmr28rOItIrOTqOwTzTcFZROJFwVlEJIJiMltDa1KKSLzkaOEjM2thZnPM7F0zKzGzcaG+s5nNNrNSM3vSzJqF+uZhuzTs75R2rmtC/RIzG5TNbSg4i0i85G5Vui3AAHc/CugJDDazfsCvgPHufiiwjq+XTx4FrAv140M7zKwbMBzoDgwGfm9mBbVdXMFZRGLFE8msS8bzpGwMm4WhODAA2PYNCFOAM8LnYWGbsP8US61wNgyY6u5b3H0ZUEpqMbmMFJxFJF7qkDmnfzFIKMXppzKzAjNbAKwCZgL/Aj5z96rQpIyvV+wsAlYAhP3rgb3T66s5pkZ6ICgisVKXqXTuPhGYmGF/AuhpZm2APwGH73IHs6TMWUTipR6+CcXdPwNeA/oDbcxsW2LbgdQa94SfHQHC/tbA2vT6ao6pkYKziMRLsg4lAzPbN2TMmFlL4DTgfVJBets3aIwAngufp4dtwv5XPbVg/nRgeJjN0RnowtdfmF0jDWuISKx4Vc7mObcHpoSZFU2Aae7+vJktBqaa2S3AfGBSaD8JeMTMSoFKUjM0cPcSM5sGLAaqgNHZfJOUgrOIxEuOYrO7LwR6VVP/IdXMtnD3zcD3ajjXrcCtdbm+grOIxIrW1hARiaJ4vL2t4Cwi8aLMWUQkipQ5i4hEz/Z39xo5BWcRiRVX5iwiEkEKziIi0aPMWUQkghScRUQiyBOW7y7khIKziMSKMmcRkQjypDJnEZHIUeYsIhJB7sqcRUQiR5mziEgEJTVbQ0QkevRAUEQkghScRUQiyOOxnLOCs4jEizJnEZEI0lQ6EZEISsRktkaTfHdARCSX3C3rkomZdTSz18xssZmVmNnFof5GMys3swWhDE075hozKzWzJWY2KK1+cKgrNbOrs7kPZc4iEis5HHOuAi5393fMbE/gbTObGfaNd/c70hubWTdgONAdOAD4i5l1Dbt/B5wGlAFzzWy6uy/OdHEFZxGJlVzN1nD3CqAifP7czN4HijIcMgyY6u5bgGVmVgr0CftK3f1DADObGtpmDM4a1hCRWPGkZV3MrNjM5qWV4urOaWadgF7A7FA1xswWmtlkM2sb6oqAFWmHlYW6muozUnAWkVhJJJtkXdx9orv3TisTv3k+M9sDeBq4xN03APcBhwA9SWXWd9bHfWhYQ0RiJZcvoZhZIanA/Ji7P5M6v69M238/8HzYLAc6ph3eIdSRob5GypxFJFaSblmXTMzMgEnA++5+V1p9+7RmZwLvhc/TgeFm1tzMOgNdgDnAXKCLmXU2s2akHhpOr+0+lDmLSKzk8CWU44EfAYvMbEGouxY4z8x6Ag58BPwsdV0vMbNppB70VQGj3T0BYGZjgBlAATDZ3Utqu7iCs4jESg5na7wJVBfpX8hwzK3ArdXUv5DpuOrUe3Deo//o+r6ENEJffvJGvrsgMVXbcEVjocxZRGIlkYzHozQFZxGJlZisGKrgLCLxomENEZEI0pKhIiIRFJMv31ZwFpF48WpnvzU+Cs4iEitVGtYQEYkeZc4iIhGkMWcRkQhS5iwiEkHKnEVEIiihzFlEJHpy9/2u+aXgLCKxklTmLCISPVr4SEQkgvRAUEQkgpKmYQ0RkchJ5LsDOaLgLCKxotkaIiIRpNkaIiIRFJfZGvH4JkQRkSBp2ZdMzKyjmb1mZovNrMTMLg71e5nZTDNbGn62DfVmZnebWamZLTSzo9PONSK0X2pmI7K5DwVnEYmVZB1KLaqAy929G9APGG1m3YCrgVfcvQvwStgGGAJ0CaUYuA9SwRwYC/QF+gBjtwX0TBScRSRWEpZ9ycTdK9z9nfD5c+B9oAgYBkwJzaYAZ4TPw4CHPWUW0MbM2gODgJnuXunu64CZwODa7kPBWURipS6Zs5kVm9m8tFJc3TnNrBPQC5gNtHP3irDrU6Bd+FwErEg7rCzU1VSfkR4Iikis1OUNQXefCEzM1MbM9gCeBi5x9w2W9pKLu7uZ1cszSGXOIhIrbtmX2phZIanA/Ji7PxOqV4bhCsLPVaG+HOiYdniHUFdTfUYKziISK7l6IGipFHkS8L6735W2azqwbcbFCOC5tPrzw6yNfsD6MPwxAxhoZm3Dg8CBoS4jDWuISKzk8PXt44EfAYvMbEGouxa4DZhmZqOA5cC5Yd8LwFCgFNgEjARw90ozuxmYG9rd5O6VtV1cwVlEYiVXr2+7+5tQ4+uGp1TT3oHRNZxrMjC5LtdXcBaRWNGSoSIiEaTgLCISQXFZW0PBWURiRUuGiohEkBbbFxGJoGRMBjYUnEUkVvRAUEQkguKRNys4i0jMKHMWEYmgqvpZJK7BKTiLSKzEIzQrOItIzGhYQ0QkgjSVTkQkguIRmhWcRSRmNKwhIhJBiZjkzgrOIhIrypxFRCLIlTmLiERPXDJnfft2DrVu3Yonp07kvUV/ZdHC1+nX9xhu+MVlLF82j3lzX2be3JcZMngAAKeeciKzZ73I/Hf+wuxZL/Kdk4/Pc+8l1xKJBOdcMJqLrhgLwON/nM6Qc3/MEccPYd1n63doO+edhZw9YjTDfvgzLhh9BQDLlpdx9ojR20vf087ikSf/1OD30dgk8axLlClzzqHxd93EjBmv8f3hxRQWFrLbbi0ZOPAkfnv3/dw1fsIObdesreSMMy+gomIl3bsfxgvPP8ZBnXvnqedSHx596jkO7nQgG7/YBECvHt046fi+jBxz5Q7tNny+kVvuvJcJd95C+/33Y+26zwDofFAHnp7yOyAV6Aec8SNOOem4hr2JRijaITd7ypxzpFWrPTnxhL5MfvAJALZu3cr69RtqbL9gQQkVFSsBKClZQsuWLWjWrFmD9FXq36erVvO3f8zh7O8O2l73ra6HUtS+3U5tX5j5OqeedDzt998PgL3bttmpzax5C+hY1J4D9t/5eNlRFZ51iTIF5xzp3PlA1qxZy6QHxjN3zgwm/OF2dtutJQAXXTiSd96eyf0T76RNm9Y7HXvWWaczf/57fPXVVw3dbaknv/rtBC67aBRmtf8V++jjMjZ8vpELxlzJuT/+H5578S87tXnxlb8y9NST6qOrseN1+FMbM5tsZqvM7L20uhvNrNzMFoQyNG3fNWZWamZLzGxQWv3gUFdqZldncx//dnA2s5EZ9hWb2Twzm5dMfvHvXqJRaVpQQK9eRzJhwsMc22cQX3yxiauuHMMfJjxM18OP45jeA/n001Xc/usbdjiuW7eu/PLWa7lw9FV56rnk2ut/n81ebdvQ/fAuWbVPJJIs/mApv7/9JibcdQsTHnqCjz4u275/69atvP7mbAYOOLG+uhwryTqULDwEDK6mfry79wzlBQAz6wYMB7qHY35vZgVmVgD8DhgCdAPOC20z2pXMeVxNO9x9orv3dvfeTZrsvguXaDzKyisoK6tgztz5ADzzzJ/p1fNIVq1aQzKZxN15YNJjHHtsz+3HFBW1549PTWLkjy/mww+X56vrkmPzFy7m9TdnMfDsEVwx9jbmvP0uV437dY3t2+23D8f1PYbdWragbZvWHNPzCJaULtu+/41Z8/hW10PYZ6+2DdH9Ri+XmbO7/w2ozPLSw4Cp7r7F3ZcBpUCfUErd/UN3/wqYGtpmlDE4m9nCGsoiQINfaVauXE1Z2Sd07XoIAAMGnMD77/+T/cM4IsAZw4ZQUrIESM3smP7cw1x73f/yj7fm5aXPUj8uvXAkrzz7KC8/PYXbx11Nn2OO4ldjr6yx/XdO7Mf8hSVUVSX4cvNmFpUs4eBOHbfvf2Hm6ww97eQG6Hk81CVzTv+//FCKs7zMmBALJ5vZtn81i4AVaW3KQl1N9RnVNlujHTAIWPeNegP+UdvJ/9NcfOkveHjKPTRrVsiyZR8z6ieX8ZvxN3PUUd1wd5YvL+PCi1LDF6MvGsmhh3Ti+usu5frrLgVgyNDzWL16bT5vQerRo089x4OPPcWaynWcdf5FnNj/WG665hIO6XQgx/ftzVkjLqSJNeHs7w6iy8GdANj05WbemjufsVf+v/x2vhFJePYP+tx9IjCxjpe4D7iZ1MSQm4E7gR/X8Ry1Ms9wI2Y2CXjQ3d+sZt/j7v6D2i7QtFlRtB+JSl58+ckb+e6CRFDhPgfbrp7jBwedmXXMeXz5n2q9npl1Ap539yMy7TOzawDc/Zdh3wzgxtD0RncfFOp3aFeTjMMa7j6qusAc9tUamEVEGloux5yrY2bt0zbPBLbN5JgODDez5mbWGegCzAHmAl3MrLOZNSP10HB6bdfRSygiEiu5fH3bzJ4ATgb2MbMyYCxwspn1JDWs8RHwMwB3LzGzacBioAoY7e6JcJ4xwAygAJjs7iW1XjvTsEYuaFhDqqNhDalOLoY1vnfQsKxjzlPLn9vl69UXZc4iEitalU5EJILqMlsjyhScRSRWor7aXLYUnEUkVuKynrOCs4jEisacRUQiSMMaIiIRVN/TgxuKgrOIxEpCmbOISPRoWENEJII0rCEiEkHKnEVEIkhT6UREIkivb4uIRJCGNUREIkjBWUQkgjRbQ0QkgpQ5i4hEkGZriIhEUMLjsWiogrOIxIrGnEVEIkhjziIiERSXMecm+e6AiEguJd2zLrUxs8lmtsrM3kur28vMZprZ0vCzbag3M7vbzErNbKGZHZ12zIjQfqmZjcjmPhScRSRWvA5/svAQMPgbdVcDr7h7F+CVsA0wBOgSSjFwH6SCOTAW6Av0AcZuC+iZKDiLSKwkPJl1qY27/w2o/Eb1MGBK+DwFOCOt/mFPmQW0MbP2wCBgprtXuvs6YCY7B/ydaMxZRGIlm+GKbcysmFSWu81Ed59Yy2Ht3L0ifP4UaBc+FwEr0tqVhbqa6jNScBaRWKnLA8EQiGsLxpmOdzOrlyeQGtYQkVjJ5QPBGqwMwxWEn6tCfTnQMa1dh1BXU31GCs4iEis5fiBYnenAthkXI4Dn0urPD7M2+gHrw/DHDGCgmbUNDwIHhrqMNKwhIrGS8ETOzmVmTwAnA/uYWRmpWRe3AdPMbBSwHDg3NH8BGAqUApuAkQDuXmlmNwNzQ7ub3P2bDxl3vnZ9v+rYtFlRPGaES059+ckb+e6CRFDhPgfbrp7jwL2OzDrmfFy5aJevV1+UOYtIrOj1bRGRCNLCRyIiEbQLszAiRcFZRGIlLgsfKTiLSKxosX0RkQjSmLOISARpzFlEJIKUOYuIRJDmOYuIRJAyZxGRCNJsDRGRCNIDQRGRCNKwhohIBOkNQRGRCFLmLCISQXEZc673xfbla2ZWnMU3+8p/GP1eSHX0HYINq7j2JvIfSL8XshMFZxGRCFJwFhGJIAXnhqVxRamOfi9kJ3ogKCISQcqcRUQiSMFZRCSCFJwbiJkNNrMlZlZqZlfnuz+Sf2Y22cxWmdl7+e6LRI+CcwMwswLgd8AQoBtwnpl1y2+vJAIeAgbnuxMSTQrODaMPUOruH7r7V8BUYFie+yR55u5/Ayrz3Q+JJgXnhlEErEjbLgt1IiLVUnAWEYkgBeeGUQ50TNvuEOpERKql4Nww5gJdzKyzmTUDhgPT89wnEYkwBecG4O5VwBhgBvA+MM3dS/LbK8k3M3sCeAs4zMzKzGxUvvsk0aHXt0VEIkiZs4hIBCk4i4hEkIKziEgEKTiLiESQgrOISAQpOIuIRJCCs4hIBP1/4uFNF8ncG5EAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "01Z_Wcs_kvND",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "3618dc1f-2c85-497d-e8b3-6b7bfe799d3d"
      },
      "source": [
        "print(metrics.classification_report(y_test, y_pred_class))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.86      0.86      0.86      4522\n",
            "           1       0.87      0.86      0.87      4819\n",
            "\n",
            "    accuracy                           0.86      9341\n",
            "   macro avg       0.86      0.86      0.86      9341\n",
            "weighted avg       0.86      0.86      0.86      9341\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xL-5v2B65jkB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Calculate predicted probabilities for test data \n",
        "y_pred_prob_nb = nb.predict_proba(x_test)[:, 1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_SjyxT3O5nFy",
        "colab_type": "code",
        "outputId": "f84f5098-ea4d-4945-b823-a326e5b0f4f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "metrics.roc_auc_score(y_test, y_pred_prob_nb)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9307757036476302"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kl8bID4V8pFL",
        "colab_type": "text"
      },
      "source": [
        "See which words have the biggest indication of Fake news and which words have the biggest indication of Real News"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HsZQ4C1B3B2r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Number of times token appears in each True article\n",
        "true_token = nb.feature_count_[0, :]\n",
        "\n",
        "#Number of times token appears in each Fake article\n",
        "fake_token = nb.feature_count_[1, :]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHp9b7239CYJ",
        "colab_type": "code",
        "outputId": "e45c04ad-bfa0-4a46-85c9-d4dc82e9772c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 228
        }
      },
      "source": [
        "tokens = pd.DataFrame({'token':x_train.columns, 'true':true_token, 'fake':fake_token}).set_index('token')\n",
        "tokens.head()"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>true</th>\n",
              "      <th>fake</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>token</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>according</th>\n",
              "      <td>391.543167</td>\n",
              "      <td>407.032347</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>act</th>\n",
              "      <td>221.037092</td>\n",
              "      <td>281.428699</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>action</th>\n",
              "      <td>285.273320</td>\n",
              "      <td>278.263287</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>added</th>\n",
              "      <td>315.779870</td>\n",
              "      <td>164.326046</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>administration</th>\n",
              "      <td>498.607789</td>\n",
              "      <td>352.136160</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      true        fake\n",
              "token                                 \n",
              "according       391.543167  407.032347\n",
              "act             221.037092  281.428699\n",
              "action          285.273320  278.263287\n",
              "added           315.779870  164.326046\n",
              "administration  498.607789  352.136160"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lDS8pCQM9dvf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# add 1 to true and fake counts to avoid dividing by 0\n",
        "tokens['true'] = tokens.true + 1\n",
        "tokens['fake'] = tokens.fake + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-KObV8Y9huu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# convert the true and fake counts into frequencies\n",
        "tokens['true'] = tokens.true / nb.class_count_[0]\n",
        "tokens['fake'] = tokens.fake / nb.class_count_[1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r62ygt3A9rLq",
        "colab_type": "code",
        "outputId": "2536e16a-b829-4188-f567-cd36347f2b41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 378
        }
      },
      "source": [
        "# calculate the ratio of fake to true for each token\n",
        "tokens['fake/true ratio'] = tokens.fake / tokens.true\n",
        "tokens.sort_values('fake/true ratio', ascending=False).head(10)"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>true</th>\n",
              "      <th>fake</th>\n",
              "      <th>fake/true ratio</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>token</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>video</th>\n",
              "      <td>0.005782</td>\n",
              "      <td>0.040030</td>\n",
              "      <td>6.923142</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>fact</th>\n",
              "      <td>0.004353</td>\n",
              "      <td>0.025581</td>\n",
              "      <td>5.876533</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>hillary</th>\n",
              "      <td>0.008762</td>\n",
              "      <td>0.047256</td>\n",
              "      <td>5.393014</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>really</th>\n",
              "      <td>0.005104</td>\n",
              "      <td>0.025209</td>\n",
              "      <td>4.938970</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>got</th>\n",
              "      <td>0.004439</td>\n",
              "      <td>0.021494</td>\n",
              "      <td>4.842391</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>know</th>\n",
              "      <td>0.008978</td>\n",
              "      <td>0.039264</td>\n",
              "      <td>4.373402</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>america</th>\n",
              "      <td>0.008584</td>\n",
              "      <td>0.037274</td>\n",
              "      <td>4.342246</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>story</th>\n",
              "      <td>0.005492</td>\n",
              "      <td>0.022700</td>\n",
              "      <td>4.133175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>went</th>\n",
              "      <td>0.004823</td>\n",
              "      <td>0.019103</td>\n",
              "      <td>3.960777</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>thing</th>\n",
              "      <td>0.007939</td>\n",
              "      <td>0.031229</td>\n",
              "      <td>3.933433</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             true      fake  fake/true ratio\n",
              "token                                       \n",
              "video    0.005782  0.040030         6.923142\n",
              "fact     0.004353  0.025581         5.876533\n",
              "hillary  0.008762  0.047256         5.393014\n",
              "really   0.005104  0.025209         4.938970\n",
              "got      0.004439  0.021494         4.842391\n",
              "know     0.008978  0.039264         4.373402\n",
              "america  0.008584  0.037274         4.342246\n",
              "story    0.005492  0.022700         4.133175\n",
              "went     0.004823  0.019103         3.960777\n",
              "thing    0.007939  0.031229         3.933433"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "phoTGMVpxZlw",
        "colab_type": "code",
        "outputId": "9807f2e3-f766-4222-ea3d-54fba35d54aa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 378
        }
      },
      "source": [
        "tokens.sort_values('fake/true ratio', ascending=True).head(10)"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>true</th>\n",
              "      <th>fake</th>\n",
              "      <th>fake/true ratio</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>token</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>minister</th>\n",
              "      <td>0.049143</td>\n",
              "      <td>0.004579</td>\n",
              "      <td>0.093168</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>wednesday</th>\n",
              "      <td>0.039187</td>\n",
              "      <td>0.008761</td>\n",
              "      <td>0.223561</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>thursday</th>\n",
              "      <td>0.037885</td>\n",
              "      <td>0.009197</td>\n",
              "      <td>0.242759</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tuesday</th>\n",
              "      <td>0.036921</td>\n",
              "      <td>0.009034</td>\n",
              "      <td>0.244693</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>monday</th>\n",
              "      <td>0.034633</td>\n",
              "      <td>0.008822</td>\n",
              "      <td>0.254719</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>friday</th>\n",
              "      <td>0.036306</td>\n",
              "      <td>0.009464</td>\n",
              "      <td>0.260674</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>foreign</th>\n",
              "      <td>0.034608</td>\n",
              "      <td>0.010393</td>\n",
              "      <td>0.300297</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>international</th>\n",
              "      <td>0.023358</td>\n",
              "      <td>0.007386</td>\n",
              "      <td>0.316202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>military</th>\n",
              "      <td>0.036022</td>\n",
              "      <td>0.011809</td>\n",
              "      <td>0.327837</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>senate</th>\n",
              "      <td>0.034323</td>\n",
              "      <td>0.011941</td>\n",
              "      <td>0.347889</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   true      fake  fake/true ratio\n",
              "token                                             \n",
              "minister       0.049143  0.004579         0.093168\n",
              "wednesday      0.039187  0.008761         0.223561\n",
              "thursday       0.037885  0.009197         0.242759\n",
              "tuesday        0.036921  0.009034         0.244693\n",
              "monday         0.034633  0.008822         0.254719\n",
              "friday         0.036306  0.009464         0.260674\n",
              "foreign        0.034608  0.010393         0.300297\n",
              "international  0.023358  0.007386         0.316202\n",
              "military       0.036022  0.011809         0.327837\n",
              "senate         0.034323  0.011941         0.347889"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1KrfWuvMs8dP",
        "colab_type": "text"
      },
      "source": [
        "# Support Vector Machines"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ycDtArtw1bhG",
        "colab_type": "code",
        "outputId": "9683fc1c-8197-4867-e69d-a5931db7bc27",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "from sklearn.svm import SVC\n",
        "\n",
        "#Linear kernel fits decently well, decided not to use another kernel because of parsimony \n",
        "#and because linear has the lowest risk of overfitting\n",
        "svc = SVC(kernel='linear', random_state=1) \n",
        "svc.fit(x_train, np.ravel(y_train))"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SVC(C=1.0, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
              "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
              "    max_iter=-1, probability=False, random_state=1, shrinking=True, tol=0.001,\n",
              "    verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i470dNxu1ZTM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred_svm = svc.predict(x_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hIwOCcr6y_9",
        "colab_type": "code",
        "outputId": "9c492288-38f9-454b-da52-50d02b19e778",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "print(\"Accuracy is:\", metrics.accuracy_score(y_test, y_pred_svm))\n",
        "print(\"Mean Squared Error is:\", np.sqrt(mean_squared_error(y_test, y_pred_svm)))\n",
        "print(metrics.classification_report(y_test, y_pred_svm))"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy is: 0.8942297398565464\n",
            "Mean Squared Error is: 0.3252234003626639\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.89      0.89      4522\n",
            "           1       0.89      0.90      0.90      4819\n",
            "\n",
            "    accuracy                           0.89      9341\n",
            "   macro avg       0.89      0.89      0.89      9341\n",
            "weighted avg       0.89      0.89      0.89      9341\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cAVX3Kdo1jVo",
        "colab_type": "code",
        "outputId": "8ad93fef-a5c5-4b47-fc86-ccc83f926cd2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "svm_cm = metrics.confusion_matrix(y_test, y_pred_svm)\n",
        "print(svm_cm)"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[4005  517]\n",
            " [ 471 4348]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8sIAX7E66E_",
        "colab_type": "code",
        "outputId": "f16dc21e-b087-484a-cd40-21ea81c3ed08",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        }
      },
      "source": [
        "labels =  np.array([['4005','517'],['471','4348']])\n",
        "sns.heatmap(svm_cm, annot=labels, fmt = '')\n",
        "plt.show()"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAD4CAYAAAAw/yevAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAazUlEQVR4nO3dfZxVZbn/8c+XAVTkWZEfAR0xScM6kS9Ty9PxWYHU8SERrSSjMx4T86EHNfV4NDX1lJRlnh8dSSx1xEfISCXFn3JKARFJQI5zNBNEMUESEGRmX78/5sZ2MLNnD+yZWSy/b173a9a+1r2eXuLFNfe691qKCMzMLFs6dfQJmJnZlpyczcwyyMnZzCyDnJzNzDLIydnMLIM6t/UB1s+519NBbAu9PndeR5+CZdCG9a9qW/ex8S8vlZ1zuuy6xzYfr624cjYzy6A2r5zNzNpVoaGjz6AinJzNLF8a6jv6DCrCydnMciWi0NGnUBFOzmaWLwUnZzOz7HHlbGaWQb4haGaWQa6czcyyJzxbw8wsg3xD0MwsgzysYWaWQb4haGaWQa6czcwyyDcEzcwyyDcEzcyyJyIfY85+nrOZ5UsUym9lkFQl6VlJD6bPQyQ9LalO0l2Suqb4DulzXVq/e9E+Lk7xJZKOLue4Ts5mli+FQvmtPOcCi4s+XwdMiIg9gVXAuBQfB6xK8QmpH5KGAWOAfYARwM8kVbV0UCdnM8uXClbOkgYBnwf+K30WcBhwT+oyGTg+LVenz6T1h6f+1UBtRGyIiJeBOmD/lo7t5Gxm+dKwsewmqUbS3KJWs9nefgR8B9iUyXcB3o6ITVNClgID0/JA4FWAtH516v9+vIltmuUbgmaWL62YrRERE4GJTa2TdAywIiKekXRIZU6ufE7OZpYvlfsSykHAcZJGATsCPYEfA70ldU7V8SBgWeq/DBgMLJXUGegFvFUU36R4m2Z5WMPM8qVCNwQj4uKIGBQRu9N4Q++xiPgiMBP4Quo2Fpialqelz6T1j0VEpPiYNJtjCDAUmN3SZbhyNrN8afsvoVwI1Eq6CngWuCXFbwF+KakOWEljQiciFkqaAiwC6oGzo4zJ2E7OZpYr0bCx8vuMeBx4PC2/RBOzLSJiPXByM9tfDVzdmmM6OZtZvvjBR2ZmGeRna5iZZZArZzOzDHLlbGaWQa6czcwyqN4P2zczyx5XzmZmGeQxZzOzDHLlbGaWQa6czcwyyJWzmVkGebaGmVkGRXT0GVSEk7OZ5YvHnM3MMsjJ2cwsg3xD0MwsgxpafMnIdsHJ2czyxcMaZmYZlJPk7Ldvm1m+RKH8VoKkHSXNlvScpIWSrkjxWyW9LGl+asNTXJJulFQnaYGkfYv2NVbSi6mNbe6YxVw5m1muRKFi85w3AIdFxBpJXYBZkn6b1n07Iu7ZrP9IYGhqBwA3AwdI6gtcDuwHBPCMpGkRsarUwV05m1m+FArltxKi0Zr0sUtqpTJ/NXBb2u4poLekAcDRwIyIWJkS8gxgREuX4eRsZvnS0FB2k1QjaW5RqynelaQqSfOBFTQm2KfTqqvT0MUESTuk2EDg1aLNl6ZYc/GSPKxhZvnSihuCETERmFhifQMwXFJv4H5JHwcuBl4HuqZtLwSu3JZTboorZzPLlwoNaxSLiLeBmcCIiFiehi42AL8A9k/dlgGDizYblGLNxUty5VwBDYUCp152E7v16clPvzWWpStWcuFNtax+Zx0fGzKQa846mS6dO/Pexnou+c+7WfzyMnr16Mb1409lYL8+LHtzFSd8ZwK7D+gHwCf2HMxlXz2+g6/KKmXJkt+z5p21NDQ0UF/fwGcP+jwnnvh5Lrv0fPbeeygH/dOxzJu3AIAxY47ngvP/9f1tP/GJj3HAgSNZsGBRR53+9qdCDz6S1A/YGBFvS9oJOBK4TtKAiFguScDxwPNpk2nAeEm1NN4QXJ36PQxcI6lP6ncUjdV3SU7OFXD7Q79njw/1Y827GwD4ce1DfGnEQYz8zCf53qQHuP/xuYw+4kDuf3wuPXfeiQdv+Ba//cNz/Kj2If7jnFMBGNS/L1OuOacjL8Pa0FFHj+att/52c37RwiWcckoNP73p2r/rV1v7ALW1DwCwzz57c8/d/+XE3FqVm+c8AJgsqYrGUYYpEfGgpMdS4hYwH9j0r+l0YBRQB6wDzgCIiJWSvgfMSf2ujIiVLR28xeQsaW8a70JuGsBeBkyLiMVlXmCuvfHWap6c/wJfqz6UX/52FhHB7EUv8f2zTwHguM/ty833PcroIw5k5rzFnHXi4QAcuf/HuXbyr4mcPN7QWueFJXUt9jnllGqm3D2tHc4mZyo0lS4iFgCfaiJ+WDP9Azi7mXWTgEmtOX7JMWdJFwK1NP4LMTs1AXdKuqg1B8qr63/1IOefOpJOEgBvr1lHj2470rmqCoD+fXuyYtVqAFasWs3/6dsLgM5VVXTvtiNvr1kHwLI3VzH6kp/w1asmMu+FlzvgSqzNRPCbB2/nD7//DePGnVb2Zid/4VjuumtqG55YTrVitkaWtVQ5jwP2iYiNxUFJNwALgWub2ihNR6kB+OnFZzLuhCMrcKrZ8/+efYG+PbszbMhA5ix6aav30693Dx7+0YX07tGNRS8v47wJv+K+a8+le7cdK3i21lEOPewkXnvtdfr124Xpv7mDJUv+l1mzni65zac/PZx1695l0aIl7XSW+RE5+fp2S8m5AHwIeGWz+IC0rknF01PWz7k3t7+3z/+fV3h83mJmPbeEDRvrWfvuBq7/5YO8s2499Q0NdK6q4o2Vf2W3Po3V8m59evH6ytX036UX9Q0NrFm3nt7duyGJrl0a/1MMGzKQwbv15ZXX/8I+ewzqyMuzCnnttdcBePPNt5g67SE+vd/wFpPz6JOruWuKq+atUrlvCHaolpLzecCjkl7kb5OoPwzsCYxvyxPbHpx7ytGce8rRAMxZ9BKTpz/J979+Ct+68Q5mzH6ekZ/5JNOenMeh+34MgEP23ZtpT87jk0M/zIzZz7P/sD2QxMq/rqFX925UderE0hUreeWNtxi0W9+OvDSrkG7ddqJTp06sWbOWbt124ojD/5lrrvlxyW0kcdJJx3D4ESe101nmzAfhec4R8ZCkj9I4j6/4huCcNDnbmnDemBF856e13HT3DPbe/UOccMh+AJxw8H5c8p93c8wFP6Bn925cP34MAPNe+BM33fs7ulRVIYlLz6imV/duHXkJViH9+/djyl0/B6Bz5ypq75rKIzMe57jjRjDhhivp168vD9x/KwsWLOKYY78EwOc+dwBLl77Gyy//uSNPffuVk8pZbT1bIM/DGrb1en3uvI4+BcugDetf1bbuY+2/jSk75+x8Ze02H6+teJ6zmeXLB2FYw8xsu5OTYQ0nZzPLlQ/KVDozs+2LK2czswxycjYzy6CMfy27XE7OZpYrFXyHYIdycjazfHFyNjPLIM/WMDPLIFfOZmYZ5ORsZpY90eBhDTOz7MlJ5VzyNVVmZtubKETZrRRJO0qaLek5SQslXZHiQyQ9LalO0l2Suqb4DulzXVq/e9G+Lk7xJZKOLuc6nJzNLF8KUX4rbQNwWER8EhgOjJB0IHAdMCEi9gRW0fg6P9LPVSk+IfVD0jBgDLAPMAL4WXqjd0lOzmaWL4VWtBKi0Zr0sUtqARwG3JPik4Hj03J1+kxaf7gkpXhtRGyIiJeBOhpfYFKSk7OZ5UrUF8pukmokzS1qNcX7klQlaT6wApgB/C/wdkTUpy5L+dtbogaSXueX1q8GdimON7FNs3xD0MzypRWTNYpfRt3M+gZguKTewP3A3tt6euVycjazXGmLZ2tExNuSZgKfAXpL6pyq40E0vleV9HMwsFRSZ6AX8FZRfJPibZrlYQ0zy5cKjTlL6pcqZiTtBBwJLAZmAl9I3cYCU9PytPSZtP6xaHxJ6zRgTJrNMQQYCsxu6TJcOZtZrlSwch4ATE4zKzoBUyLiQUmLgFpJVwHPArek/rcAv5RUB6ykcYYGEbFQ0hRgEVAPnJ2GS0pycjazfKnQFwQjYgHwqSbiL9HEbIuIWA+c3My+rgaubs3xnZzNLFfen0exnXNyNrNciXw8WsPJ2cxyxsnZzCx7XDmbmWWQk7OZWQZFgzr6FCrCydnMcsWVs5lZBkXBlbOZWea4cjYzy6AIV85mZpnjytnMLIMKnq1hZpY9viFoZpZBTs5mZhkUlX8RSodwcjazXHHlbGaWQZ5KZ2aWQQ2erWFmlj15qZz99m0zy5UoqOxWiqTBkmZKWiRpoaRzU/zfJS2TND+1UUXbXCypTtISSUcXxUekWJ2ki8q5DlfOZpYrFZytUQ98MyLmSeoBPCNpRlo3ISJ+UNxZ0jAa37i9D/Ah4HeSPppW3wQcCSwF5kiaFhGLSh3cydnMcqVSszUiYjmwPC2/I2kxMLDEJtVAbURsAF6WVMff3tJdl97ajaTa1LdkcvawhpnlSkOhU9lNUo2kuUWtpql9Stod+BTwdAqNl7RA0iRJfVJsIPBq0WZLU6y5eElOzmaWKxGtaTExIvYrahM335+k7sC9wHkR8VfgZuAjwHAaK+sftsV1eFjDzHKlUMHZGpK60JiYb4+I+wAi4o2i9T8HHkwflwGDizYflGKUiDfLlbOZ5UqEym6lSBJwC7A4Im4oig8o6nYC8HxangaMkbSDpCHAUGA2MAcYKmmIpK403jSc1tJ1uHI2s1yp4GyNg4AvA3+UND/FvgucKmk4EMCfgDMbjxsLJU2h8UZfPXB2RDQASBoPPAxUAZMiYmFLB1e08VNCOncdmJPHkFglvfvakx19CpZBXXbdY5vHJOYOOr7snLPf0gcy+40VV85mlisNhXyM1jo5m1mu5OVXdSdnM8uVSs7W6EhOzmaWK3l58JGTs5nlSk5evu3kbGb5ErhyNjPLnHoPa5iZZY8rZzOzDPKYs5lZBrlyNjPLIFfOZmYZ1ODK2cwseyr0lqoO5+RsZrlScOVsZpY9fvCRmVkG+YagmVkGFeRhDTOzzGno6BOoECdnM8uVvMzWyMf7XMzMkgIqu5UiabCkmZIWSVoo6dwU7ytphqQX088+KS5JN0qqk7RA0r5F+xqb+r8oaWw51+HkbGa5Eq1oLagHvhkRw4ADgbMlDQMuAh6NiKHAo+kzwEhgaGo1wM3QmMyBy4EDgP2Byzcl9FKcnM0sVwoqv5USEcsjYl5afgdYDAwEqoHJqdtk4Pi0XA3cFo2eAnpLGgAcDcyIiJURsQqYAYxo6TqcnM0sVwqtaJJqJM0tajVN7VPS7sCngKeB/hGxPK16HeiflgcCrxZttjTFmouX5BuCZpYrDa24IRgRE4GJpfpI6g7cC5wXEX9V0VS9iAhJbfK9F1fOZpYrramcWyKpC42J+faIuC+F30jDFaSfK1J8GTC4aPNBKdZcvCQnZzPLlUolZzWWyLcAiyPihqJV04BNMy7GAlOL4qenWRsHAqvT8MfDwFGS+qQbgUelWEke1jCzXKngKwQPAr4M/FHS/BT7LnAtMEXSOOAVYHRaNx0YBdQB64AzACJipaTvAXNSvysjYmVLB3dyNrNcqdSzNSJiFjQ7GfrwJvoHcHYz+5oETGrN8Z2czSxX/PVtM7MMysvXt52czSxX/MhQM7MMcnI2M8sgvwnFzCyDPOZsZpZBnq1hZpZBhZwMbDg5m1mu+IagmVkG5aNudnI2s5xx5WxmlkH1bfN45Xbn5GxmuZKP1OzkbGY542ENM7MM8lQ6M7MMykdqdnI2s5zxsIaZWQY15KR2dnI2s1zJS+Xst2+bWa5EK/60RNIkSSskPV8U+3dJyyTNT21U0bqLJdVJWiLp6KL4iBSrk3RROdfh5GxmuVJoRSvDrcCIJuITImJ4atMBJA0DxgD7pG1+JqlKUhVwEzASGAacmvqW5ORcYZ06dWLO7IeZev9kAB5/7D7mznmEuXMe4c9/eoZ777kFgL32+giznpjG2nde4oLzz+zIU7Y20tDQwBe+cjZf//blAFz2/QmcOPbrnHD6WZx/yVWsW/fu3/WfMXMWHz9oJM8v/h8ANtbX893v/YATvnwWx55Ww89vu6vdr2F7VCDKbi2JiCeAlWUeuhqojYgNEfEyUAfsn1pdRLwUEe8BtalvSR5zrrBvnPM1XnjhRXr26AHAIYed+P66KXdNZNqvHwFg5cq3Oe/8y6iubuofZcuDX909lT12/zBr1q4D4MJv1NB9550BuP7Gidxx76/52pdHA7B27Tp+dfdU/nHYXu9v/8hjT/Lexo3c/8ubeXf9eqq/eCajjjyEgQP6t//FbEdacztQUg1QUxSaGBETy9h0vKTTgbnANyNiFTAQeKqoz9IUA3h1s/gBLR3AlXMFDRw4gFEjD2fSpDu3WNejR3cOPeQgpk59CIA333yLuc88x8aNG9v7NK0dvL7iTZ74/WxOOvb9Ycf3E3NEsH7DBlT0xo6f/Pw2vvqlk+m6Q9f3Y5J4d/166usb2LDhPbp06UL3nbu12zVsr+qJsltETIyI/YpaOYn5ZuAjwHBgOfDDtrgOJ+cKuuGHV3DRxVdRKGw5mlVdPYLHZv4377yzpgPOzNrbdT/+v1zw9XFIf/+/2KVX38DBx57Gy68s5bQvHAfAoiV1vL7iLxz82f3/ru+Rh/4TO+24I4dWn8aRJ57OV049kV49e7TbNWyvKnlDsMn9R7wREQ0RUQB+TuOwBcAyYHBR10Ep1ly8pK1OzpLOKLGuRtJcSXMLhbVbe4jtyudHHcGKFX9h3rN/bHL9mNHV1N71QDuflXWEx//7afr26c0+ew/dYt1Vl1zAzKm/Yo/dB/PQo09QKBS4/icT+fY5/7JF3z8uWkJVp048NvV2HrrnVibfeR+vLlveHpewXavwDcEtSBpQ9PEEYNNMjmnAGEk7SBoCDAVmA3OAoZKGSOpK403DaS0dZ1vGnK8AftHUivSrwUSAzl0H5mNGeAs++9n9OPaYoxg54jB23HEHevbsweRbb2TsV77BLrv04dOf/hQnnfy1jj5NawfPLljE47Oe4sk/zGHDextZu3YdF15xPddd/h0AqqqqGHnEwUy6/R6OOPgg6l56hTPGN677y8pVnHPhFfzkusuZPuNxDjpwP7p07swufXoz/B+HsfCFFxk8cECpw3/gbW1F3BRJdwKHALtKWgpcDhwiaTiNw9t/As4EiIiFkqYAi4B64OyIaEj7GQ88DFQBkyJiYUvHLpmcJS1obhXguxJFLrn0Wi659FoADv7nz3DB+f/K2K98A4CTTjyG30z/HRs2bOjIU7R2cv5ZZ3D+WY2/WM6et4Bb77yXa//t2/x56Wt8eNCHiAhmznqKIf8wiB7dd2bW9L/NwvjK+O/wrbO/xsc/9lGefmY+s595juNGHM66d9ezYOELfHn0CR11WduNSn4JJSJObSJ8S4n+VwNXNxGfDkxvzbFbqpz7A0cDqzaLC/h9aw70QXbK6OO4/j9u+rtY//79ePoPv6Vnz+4UCgW+cc6/8IlPHuIx6ZyKCL571Q9Zu3YdEcFeew7hsm+PL7nNqScey6XX3ED1F88kCI4fdRR77Tmknc54+9UQ+fhlXVHiQiTdAvwiImY1se6OiDitpQN8UIY1rHXefe3Jjj4Fy6Auu+6hlnuVdto/nFB2zrnjlfu3+XhtpWTlHBHjSqxrMTGbmbW3So45dyR/CcXMciUvDz5ycjazXPGbUMzMMsjDGmZmGZSX2RpOzmaWKx7WMDPLIN8QNDPLII85m5llkIc1zMwyqNS3nrcnTs5mlisNrpzNzLLHwxpmZhnkYQ0zswxy5WxmlkGeSmdmlkH++raZWQblZVhjq9++bWaWRQWi7NYSSZMkrZD0fFGsr6QZkl5MP/ukuCTdKKlO0gJJ+xZtMzb1f1HS2HKuw8nZzHIlIspuZbgVGLFZ7CLg0YgYCjyaPgOMBIamVgPcDI3JnMa3dh8A7A9cvimhl+LkbGa5UsnKOSKeAFZuFq4GJqflycDxRfHbotFTQG9JA2h8SfaMiFgZEauAGWyZ8Lfg5GxmuRKt+COpRtLcolZTxiH6R8TytPw60D8tDwReLeq3NMWai5fkG4JmlisNUf5DQyNiIjBxa48VESGpTe5AunI2s1yp8JhzU95IwxWknytSfBkwuKjfoBRrLl6Sk7OZ5Uolx5ybMQ3YNONiLDC1KH56mrVxILA6DX88DBwlqU+6EXhUipXkYQ0zy5VKfkNQ0p3AIcCukpbSOOviWmCKpHHAK8Do1H06MAqoA9YBZwBExEpJ3wPmpH5XRsTmNxm3PHZbPySkc9eB+ZgRbhX17mtPdvQpWAZ12XUPbes+Pt7/wLJzzvNvPLXNx2srrpzNLFf8bA0zswxqzWyNLHNyNrNcKfjBR2Zm2eNhDTOzDHLlbGaWQa6czcwyqCEaOvoUKsLJ2cxyxS94NTPLoLy8CcXJ2cxyxZWzmVkGebaGmVkGebaGmVkG+evbZmYZ5DFnM7MM8pizmVkGuXI2M8sgz3M2M8sgV85mZhmUl9kafvu2meVKIaLs1hJJf5L0R0nzJc1Nsb6SZkh6Mf3sk+KSdKOkOkkLJO27Ldfh5GxmuRIRZbcyHRoRwyNiv/T5IuDRiBgKPJo+A4wEhqZWA9y8Ldfh5GxmuRKt+LOVqoHJaXkycHxR/LZo9BTQW9KArT2Ik7OZ5UprKmdJNZLmFrWazXcHPCLpmaJ1/SNieVp+HeiflgcCrxZtuzTFtopvCJpZrrTmSygRMRGYWKLLP0XEMkm7ATMkvbDZ9iGpTaaHtHlyrn9vmdr6GNsLSTXpL4PZ+/z3orIqmXMiYln6uULS/cD+wBuSBkTE8jRssSJ1XwYMLtp8UIptFQ9rtK/Nf2UyA/+9yCRJO0vqsWkZOAp4HpgGjE3dxgJT0/I04PQ0a+NAYHXR8EereVjDzKxp/YH7JUFjrrwjIh6SNAeYImkc8AowOvWfDowC6oB1wBnbcnDl5ds02wNJc4um45gB/nthTfOwRvvyuKI1xX8vbAuunM3MMsiVs5lZBjk5m5llkJNzO5E0QtKS9FCUi1rewvJO0iRJKyQ939HnYtnj5NwOJFUBN9H4YJRhwKmShnXsWVkG3AqM6OiTsGxycm4f+wN1EfFSRLwH1NL4kBT7AIuIJ4CVHX0elk1Ozu2jog9EMbP8c3I2M8sgJ+f2UdEHophZ/jk5t485wFBJQyR1BcbQ+JAUM7MmOTm3g4ioB8YDDwOLgSkRsbBjz8o6mqQ7gT8Ae0lamh6kYwb469tmZpnkytnMLIOcnM3MMsjJ2cwsg5yczcwyyMnZzCyDnJzNzDLIydnMLIP+P2VxJb3vr2BQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h55e3NG36sGa",
        "colab_type": "code",
        "outputId": "3a2a751a-4ca7-47bb-df76-c1839df1d2ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print (metrics.roc_auc_score(y_test, y_pred_svm))"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8939659687773932\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QWx3UslRs5_Z",
        "colab_type": "text"
      },
      "source": [
        "# Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFzTAw2nvSF4",
        "colab_type": "code",
        "outputId": "bcd4a66a-37c2-4db6-cbd6-db85a83ada61",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "#Tried depths greater and less than 60, 60 was the best. \n",
        "#Tried estimators greater and less than 200 but 200 was the best\n",
        "\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "\n",
        "rf = RandomForestClassifier(random_state = 1)\n",
        "\n",
        "param_grid = {\n",
        "    'n_estimators': [200],\n",
        "    'max_depth': [50, 60, 70]\n",
        "    }\n",
        "\n",
        "%time grid_search_rf = GridSearchCV(estimator = rf, param_grid = param_grid, cv = 5)\n",
        "grid_search_rf.fit(x_train, np.ravel(y_train))\n",
        "grid_search_rf.best_params_"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 224 µs, sys: 6.99 ms, total: 7.22 ms\n",
            "Wall time: 7.14 ms\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'max_depth': 50, 'n_estimators': 200}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ApxMOK9HYmr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred_rf = grid_search_rf.predict(x_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "irFVTe9xw1y1",
        "colab_type": "code",
        "outputId": "c379514f-fd76-4f42-e36f-1a23cd2746f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "print(\"Accuracy is:\", metrics.accuracy_score(y_test, y_pred_rf))\n",
        "print(\"Mean Squared Error is:\", np.sqrt(mean_squared_error(y_test, y_pred_rf)))"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy is: 0.9283802590729044\n",
            "Mean Squared Error is: 0.2676186483171447\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R9-6FRDhw8gW",
        "colab_type": "code",
        "outputId": "eb7710c6-9486-4f72-c63c-7447afd26cac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "rf_cm = metrics.confusion_matrix(y_test, y_pred_rf)\n",
        "print(rf_cm)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[4097  425]\n",
            " [ 244 4575]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XYFr1uXmkq8A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "9c1926d7-f6dc-452e-9eaf-eeb1ff0dfcaa"
      },
      "source": [
        "print(metrics.classification_report(y_test, y_pred_rf))"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.91      0.92      4522\n",
            "           1       0.92      0.95      0.93      4819\n",
            "\n",
            "    accuracy                           0.93      9341\n",
            "   macro avg       0.93      0.93      0.93      9341\n",
            "weighted avg       0.93      0.93      0.93      9341\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ieZMUfdTC0Og",
        "colab_type": "code",
        "outputId": "18e38c72-e58c-4a5f-c115-a4ac967d19d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 266
        }
      },
      "source": [
        "labels = np.array([['4097','425'],['244','4575']])\n",
        "sns.heatmap(rf_cm, annot=labels, fmt = '')\n",
        "plt.show()"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAD5CAYAAAD7o/QKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAcXUlEQVR4nO3de5xWVb3H8c8XUFS8AKIEDCl2UENP4g2vJWkiUoaezNRKNDpjpqVlqdgFNe1o3tJOWqMQeDQRywvlFVG8HC8wKqGIlzkWwYhiXEwE0Zn5nT+exfSAzDzPyFw22++713qx92+tfSvO7yzWXs9eigjMzCxbOnX0DZiZ2Qc5OZuZZZCTs5lZBjk5m5llkJOzmVkGOTmbmWVQl7a+wMpHJniunn1A3xEXdvQtWAYtXV6j9T3H+/94teycs1GvHUpeT1JnoBqojYgvSJoAHAS8lZqcGBGzJAm4ChgBrEjxZ9I5RgE/Tu0vjIiJpa7b5snZzGwDdzowF9iyKPbDiPjDWu0OBwamsg9wLbCPpJ7AWGAvIICnJU2JiKXNXdTDGmaWLw315ZcSJFUAnweuL+PKI4EbouBJoLukPsBhwNSIWJIS8lRgeKmTOTmbWb7U15VfSvslcBbQsFb8IkmzJV0pqWuK9QPmF7VZkGJNxZvl5GxmuRLRUHaRVCmpuqhUrj6PpC8AiyLi6bUuMQbYGdgb6Amc3RbP4TFnM8uXhrU7uU2LiCqgqonqA4AvShoBbAJsKenGiPhaql8l6XfAD9J+LdC/6PiKFKsFhq4Vn17q3txzNrN8iYbyS3OniRgTERURsT1wLPBgRHwtjSOTZmccCTyfDpkCnKCCfYG3ImIhcB8wTFIPST2AYSnWLPeczSxfynjRt55ukrQNIGAW8K0Uv5vCNLoaClPpTgKIiCWSfgbMTO0uiIglpS7i5Gxm+VKiR/yhThkxnTQUEREHN9EmgFObqBsPjG/JNZ2czSxXorxZGJnn5Gxm+dKCF4JZ5uRsZvnSBsMaHcHJ2czype1fCLYLJ2czyxf3nM3MMsgvBM3MMsgvBM3MsifCY85mZtnjMWczswzysIaZWQa552xmlkH173f0HbQKJ2czyxcPa5iZZZCHNczMMsg9ZzOzDMpJcvYyVWaWK1H/ftmlHJI6S3pW0p/T/gBJT0mqkXSLpI1TvGvar0n12xedY0yKvyTpsHKu6+RsZvnSSmsIFjkdmFu0fwlwZUT8G7AUGJ3io4GlKX5laoekQRTWINwFGA5cI6lzqYs6OZtZvjQ0lF9KkFQBfB64Pu0LOBj4Q2oykcIirwAj0z6p/pDUfiQwKSJWRcRfKawxOKTUtZ2czSxfWrfn/EvgLGB1462BZRGx+tN3C4B+absfMB8g1b+V2jfG13FMk5yczSxfWtBzllQpqbqoVK4+jaQvAIsi4umOeAzP1jCzfGnBPOeIqAKqmqg+APiipBHAJsCWwFVAd0ldUu+4AqhN7WuB/sACSV2ArYDFRfHVio9pknvOZpYvdXXll2ZExJiIqIiI7Sm80HswIr4KPAQcnZqNAu5M21PSPqn+wYiIFD82zeYYAAwEZpR6DPeczSxf2v4XgmcDkyRdCDwLjEvxccD/SKoBllBI6ETEHEmTgReAOuDUKOOj007OZpYvbfAjlIiYDkxP26+yjtkWEfEu8OUmjr8IuKgl13RyNrN88bc1zMwyKCc/33ZyNrN8cc/ZzCyDSszC2FA4OZtZvkR09B20CidnM8sXjzmbmWWQk7OZWQb5haCZWQbVl/zx3QbBydnM8sXDGmZmGeTkbGaWQR5zNjPLnmjwPGczs+zxsIaZWQZ5toaZWQa552yr1Tc0cPyFE9i2++b86rvHUPvmMs6+7g7eWr6ST27Xh4tGH8FGXTrz2uK3OG/CXSx9ewVbdtuUn48+gt49t2Tmi/O49JYHGs/3t9cXc3HlkRy8+44d+FTWmjp16sRDj97Bwtde59gvV1I17nIG7/7v1NXV8XT1X/jed39CXV0dB3x6H34/6TfMm1dYrPlPU+7n0ov/u4PvfgPTSslZ0ibAI0BXCrnyDxExVtIE4CAKq2sDnBgRsySJwhqDI4AVKf5MOtco4Mep/YURMbHU9b2GYCv4/QPVDOizdeP+L//4EF/73BD+9PNT2HKzTbj9sb8AcMWtD/KF/Xbl1vO+yclfOICrb58OwN47b8fksaOZPHY01/3geDbZeCP2GzSgIx7F2si3vn0iL79U07h/6y1TGLLHMPYfMoJNN92EE048prHuicdn8pn9v8hn9v+iE/OHEVF+ad4q4OCI2A0YDAyXtG+q+2FEDE5lVoodTmF9wIFAJXAtgKSewFhgHworqIyV1KPUxUsmZ0k7Szpb0tWpnC3pk6WO+6h4Y8k/efS5Gv7jwN0AiAhmvjSPz+25MwBH7L8rDz37MgCvvvYPhuy8PVBIyNNnvfKB8019+kUO2HUHNu26Ufs8gLW5vn0/xrDhQ7lh4uTG2NT7H27cfrp6Nn37fawjbi2fGhrKL82IguVpd6NUmsvoI4Eb0nFPUliluw9wGDA1IpZExFJgKjC81GM0m5wlnQ1MAkRhtdgZaftmSeeUOvlHwaW3PMAZR38WdRIAy5avZItNu9Klc+G/2t49tmTRsrcB2LH/tkx75iUAHnz2Zd559z2WLV+xxvnumzGXw4cMascnsLb281/8mLE/voSGdUzx6tKlC1857kimTX2kMbb3kN159Ik/cett49j5kwPb81bzoSHKLyVI6ixpFrCIQoJ9KlVdJGm2pCsldU2xfsD8osMXpFhT8WaV6jmPBvaOiIsj4sZULqbQNR9d8sly7pG/vEKPLTdj0HZ9ymr//S8fzNMv/52vXDCe6pf/zrbdt6BTp3/9T/DmsuXU1C5iv112aKtbtnZ22PDP8o83F/OXWXPWWX/Zlefz+P/O4InHqwGYPWsOnxp0EJ/e7wiqfnMDN958bXvebj7U15ddJFVKqi4qlcWnioj6iBgMVABDJO0KjAF2BvYGelJYjbvVlXoh2AD0BeatFe+T6tYpPWAlwK/OHMXoLw5dj1vMrln/V8vDs2p47LlreO/9Ot55dxW/mDSVt1euoq6+gS6dO/HG0n+ybfctANi2+xZc8e0vAbDi3feY9vRLbLnZJo3nu796Lp/dfSc26tK5Q57HWt8+++7J8BGHcOiwg+i6SVe22GJzfnv95Zz8zTM5a8x36NWrJ18//seN7d9+e3nj9tT7H+ayK8+n59Y9WLJ4aUfc/gYpWvBCMCKqgKoy2i2T9BAwPCIuS+FVkn4H/CDt1wL9iw6rSLFaYOha8emlrlmq53wGME3SPZKqUrkXmAac3syDVEXEXhGxV14TM8B3/2Mo9196Gvdc/G0urhzJ3jttx3/950j22mk7Hnj6RQD+9PjzDB1c+Kfp0rdXNP7Tdtw9T3DkgZ9a43z3znjBQxo5c8F5l7HrTgey2y5DGX3iGTz68BOc/M0z+fqoYzjkkE/zzZPOIIpeTG27ba/G7T32/BSdOnVyYm6pVhrWkLSNpO5pe1PgUODFNI5Mmp1xJPB8OmQKcIIK9gXeioiFwH3AMEk90ovAYSnWrGZ7zhFxr6QdKQxjrB4jqQVmRkQ+Znq3gTO+NJSzq+7k13c8zE4f/xhHpZeF1S//natvm46APXf8OGOOH9Z4TO0/lvH60n+y544f76C7tvZ0xVUXMP/vr3H/g7cC/5oyN/Kowznpm8dTX1fHypWrGH1ik30ga0rrfVujDzBRUmcKHdnJEfFnSQ9K2obC+7dZwLdS+7spTKOroTCV7iSAiFgi6WfAzNTugohYUuriijZeb2vlIxPy8UN3a1V9R1zY0bdgGbR0eY3W9xzvXPDVsnNOt5/etN7Xayv+EYqZ5UtdPv5R7+RsZvniT4aamWWQPxlqZpY9LZlKl2VOzmaWL+45m5llkJOzmVkG+WP7ZmbZ4zUEzcyyyMnZzCyDPFvDzCyD3HM2M8sgJ2czs+yJeg9rmJllj3vOZmbZ46l0ZmZZ5ORsZpZB+RhyLrmGoJnZBiXqGsouzZG0iaQZkv4iaY6k81N8gKSnJNVIukXSxineNe3XpPrti841JsVfknRYOc/h5Gxm+dLQgtK8VcDBEbEbMBgYnhZuvQS4MiL+DVgKjE7tRwNLU/zK1A5Jg4BjgV2A4cA1aV3CZjk5m1muREOUXZo9T8HytLtRKgEcDPwhxSdSWIEbYGTaJ9UfklboHglMiohVEfFXCgvADin1HE7OZpYvLeg5S6qUVF1UKotPJamzpFnAImAq8H/AsoioS00WAP3Sdj9gPkCqfwvYuji+jmOa5BeCZpYrLZlKFxFVQFUz9fXAYEndgduBndf7BsvknrOZ5UvrjTk3iohlwEPAfkB3Sas7thVAbdquBfoDpPqtgMXF8XUc0yQnZzPLlagrvzRH0japx4ykTYFDgbkUkvTRqdko4M60PSXtk+ofjIhI8WPTbI4BwEBgRqnn8LCGmeVKtN485z7AxDSzohMwOSL+LOkFYJKkC4FngXGp/TjgfyTVAEsozNAgIuZImgy8ANQBp6bhkmY5OZtZvrRSco6I2cDu64i/yjpmW0TEu8CXmzjXRcBFLbm+k7OZ5Uor9pw7lJOzmeWKk7OZWQZFvTr6FlqFk7OZ5Yp7zmZmGRQN7jmbmWWOe85mZhkU4Z6zmVnmuOdsZpZBDZ6tYWaWPX4haGaWQU7OZmYZFPlYfNvJ2czyxT1nM7MM8lQ6M7MMqvdsDTOz7MlLz9nLVJlZrkSDyi7NkdRf0kOSXpA0R9LpKX6epFpJs1IZUXTMGEk1kl6SdFhRfHiK1Ug6p5zncM/ZzHKlFWdr1AFnRsQzkrYAnpY0NdVdGRGXFTeWNIjC0lS7AH2BByTtmKp/TWENwgXATElTIuKF5i7u5GxmudJaszUiYiGwMG2/LWku0K+ZQ0YCkyJiFfDXtJbg6uWsatLyVkialNo2m5w9rGFmuVLf0KnsUi5J21NYT/CpFDpN0mxJ4yX1SLF+wPyiwxakWFPxZjk5m1muRJRfJFVKqi4qlWufT9LmwB+BMyLin8C1wCeAwRR61pe3xXN4WMPMcqWhBbM1IqIKqGqqXtJGFBLzTRFxWzrmjaL664A/p91aoH/R4RUpRjPxJrnnbGa5EqGyS3MkCRgHzI2IK4rifYqaHQU8n7anAMdK6ippADAQmAHMBAZKGiBpYwovDaeUeg73nM0sV1pxtsYBwNeB5yTNSrFzgeMkDQYC+BtwcuG6MUfSZAov+uqAUyOiHkDSacB9QGdgfETMKXVxRRt/JaTLxv1y8hkSa00rX3u0o2/BMmijXjus91SL6oojy845ey24I7O/WHHP2cxypSWzMLLMydnMciUv/1R3cjazXGnJbI0sc3I2s1zJy4ePnJzNLFdysvi2k7OZ5UvgnrOZWebUeVjDzCx73HM2M8sgjzmbmWWQe85mZhnknrOZWQbVu+dsZpY9rbRKVYdzcjazXGlwz9nMLHv84SMzswzKywvBfHz41MwsaZDKLs2R1F/SQ5JekDRH0ukp3lPSVEmvpD97pLgkXS2pJq3MvUfRuUal9q9IGlXOczg5m1mu1LeglFAHnBkRg4B9gVMlDQLOAaZFxEBgWtoHOJzCuoEDgUoKq3QjqScwFtgHGAKMXZ3Qm+PkbGa50qDyS3MiYmFEPJO23wbmAv2AkcDE1GwicGTaHgncEAVPAt3TYrCHAVMjYklELAWmAsNLPYfHnM0sV9pitoak7YHdgaeA3hGxMFW9DvRO2/2A+UWHLUixpuLNcs/ZzHIlWlAkVUqqLiqVa59P0ubAH4EzIuKfa1yrsEJ2m0wQcc/ZzHKlJT9CiYgqoKqpekkbUUjMN0XEbSn8hqQ+EbEwDVssSvFaoH/R4RUpVgsMXSs+vdS9uedsZrnS0ILSHEkCxgFzI+KKoqopwOoZF6OAO4viJ6RZG/sCb6Xhj/uAYZJ6pBeBw1KsWe45m1mu1LfekPMBwNeB5yTNSrFzgYuByZJGA/OAY1Ld3cAIoAZYAZwEEBFLJP0MmJnaXRARS0pd3MnZzHKltX6EEhGPQZNvFw9ZR/sATm3iXOOB8S25vpOzmeVKXn4h6ORsZrmSkyUEnZzNLF/cczYzy6Ayfpa9QXByNrNc8cf2zcwyyMMaZmYZ5ORsZpZBXgnFzCyDPOZsZpZBnq1hZpZBDTkZ2HByNrNc8QtBM7MMyke/2cnZzHLGPWczswyqUz76zk7OZpYr+UjNXqbKzHKmtZapApA0XtIiSc8Xxc6TVCtpViojiurGSKqR9JKkw4riw1OsRtI55TyHk7OZ5UoDUXYpwwRg+DriV0bE4FTuBpA0CDgW2CUdc42kzpI6A78GDgcGAcelts3ysIaZ5UprDmtExCOSti+z+UhgUkSsAv4qqQYYkupqIuJVAEmTUtsXmjuZe85mliutOazRjNMkzU7DHj1SrB8wv6jNghRrKt4sJ2czy5V6ouwiqVJSdVGpLOMS1wKfAAYDC4HL2+I5PKxhZrnSkh5xRFQBVS05f0S8sXpb0nXAn9NuLdC/qGlFitFMvEnuOZtZrkQL/vNhSOpTtHsUsHomxxTgWEldJQ0ABgIzgJnAQEkDJG1M4aXhlFLXcc/ZzHKlNX8hKOlmYCjQS9ICYCwwVNJgCu8e/wacDBARcyRNpvCirw44NSLq03lOA+4DOgPjI2JOqWs7ObeSioq+TBh/Fdv27kVEcP31N/Gr/x7XWP+9M07m0l/8lN59dmXx4qWN8b323I3HHp3C8V/7NrfddldH3Lq1kfr6er4y+rtsu00vrrn0fH504eVUz3qOzbt1A+CiH32fnXf8BONv+gN33f9Q4zGvzpvPo3dNYqstt2DYl0bRbbPN6NSpE507d2by+Ks78pE2CK35VbqIOG4d4XHriK1ufxFw0TridwN3t+TaTs6tpK6ujh+edT7PznqezTfvxoyn7uWBaY8wd+4rVFT05dDPfYZ58xascUynTp34r5//iKlTH+6gu7a2dOOtd7LD9h9n+TsrGmNnnjqaYZ/99BrtvvHVo/nGV48GYPpjT3LDLXew1ZZbNNaP/9XF9Oi+VfvcdA74F4K2htdfX8SzswpDT8uXv8OLL75Cv74fA+Dyy87jnHMvImLNvzannfoNbrv9Lha9ubjd79fa1uuL3uSRx2fwpSMOK924yN0PPMyIQw9qo7v6aKgjyi5Z9qGTs6STWvNG8mS77SoYvNuuPDXjWY44Yhi1tQuZPXvN+eZ9+36MI0cO5ze/vaGD7tLa0iVX/Zbvf3s00pr/J3b1bydy1AmncMlVv+W9995bo27lu+/y2JPVHDr0wMaYJCq/9yOO+cZ3uPXOFv2r+COrrV8Itpf1GdY4H/jduirSXMFKAHXeik6duq3HZTYs3bptxuRbruP7PxhLXV0dY87+DsNHHP+Bdldcfj5jzv35B3rTtuGb/r9P0bNHd3bZeSAznpndGD/jWyfRa+sevP/++5x3ydWMu/FWTvnGV/913GNPsfunBq0xpHHDtZfRe5teLF66jP8841wGbNefvQb/e7s+z4bmI/HJUEmzm6oCejd1XPHcwS4b9/vIZJ8uXbpw6y3XcfPNt3PHHfew6647s/32H+eZ6qkAVFT0YeZT97HfAZ9nzz0+xU03XgNAr149OXz4wdTV1TFlyn0d+QjWCp6d/QLTH3uSR5+Yyar33uedd1Zw9vm/4JKxZwGw8cYbc+TnhzHh5j+ucdw90x5mxOeGrhHrvU0vALbu0Z1DPrM/z73wkpNzCVnvEZerVM+5N3AYsHStuIDH2+SONmDXVV3O3Bdr+OVVhTntzz//In0rdmusr3n5SfbZ73AWL17KwJ32a4yPu/5K7rr7ASfmnPjeKSfxvVMKo34znpnNhJv/yCVjz+LNfyxhm149iQgefORxBu6wXeMxby9/h+pnn+Pin57VGFux8l2ioYFu3TZjxcp3eXzGM5xy0gf/FWZr+kj0nCn88mXziJi1doWk6W1yRxuoA/bfm69/7WhmP/cC1TPvB+AnP7mYe+59sIPvzLLi7PN/wdJlbxER7DRwB8b+8DuNddMefpz9h+zBZptu0hhbvGQpp5/7MwDq6+oZMWwoB+67V7vf94amPidDhWrrMc+P0rCGlW/la4929C1YBm3Uawet7zmO3+6osnPO7+fdvt7Xayue52xmufJRGXM2M9ugfFTGnM3MNiit+fPtjuTkbGa54mENM7MMystsDSdnM8sVD2uYmWWQXwiamWVQXsac/clQM8uVBqLsUkpaXXuRpOeLYj0lTZX0SvqzR4pL0tWSatLK3HsUHTMqtX9F0qhynsPJ2cxyJSLKLmWYAAxfK3YOMC0iBgLT0j7A4RTWDRxI4auc10IhmVNY3mofYAgwdnVCb46Ts5nlSj1RdiklIh4BlqwVHglMTNsTgSOL4jdEwZNA97QY7GHA1IhYEhFLgal8MOF/gMeczSxX2mG2Ru+IWJi2X+dfn0/uB8wvarcgxZqKN8s9ZzPLlZYMa0iqlFRdVCpbeK2gjZYtdM/ZzHKlJT3n4oVBWuANSX0iYmEatliU4rVA/6J2FSlWCwxdKz691EXcczazXGmHNQSnAKtnXIwC7iyKn5BmbewLvJWGP+4DhknqkV4EDkuxZrnnbGa50po/35Z0M4Veby9JCyjMurgYmCxpNDAPOCY1vxsYAdQAK4CTACJiiaSfATNTuwsiYu2XjB/g5GxmudKaLwQj4rgmqg5ZR9sATm3iPOOB8S25tpOzmeWKv61hZpZBbb30XntxcjazXHHP2cwsg/Ly4SMnZzPLlfrIx0dDnZzNLFc85mxmlkEeczYzyyCPOZuZZVCDhzXMzLLHPWczswzybA0zswzysIaZWQZ5WMPMLIPcczYzyyD3nM3MMqg+6jv6FlqFl6kys1xpyQKvpUj6m6TnJM2SVJ1iPSVNlfRK+rNHikvS1ZJqJM2WtMf6PIeTs5nlSgNRdinTZyNicETslfbPAaZFxEBgWtoHOBwYmEolcO36PIeTs5nlSmv2nJswEpiYticCRxbFb4iCJ4HuaXXuD8XJ2cxypSGi7FKGAO6X9LSkyhTrnVbVBngd6J22+wHzi45dkGIfil8ImlmutGS2Rkq4lUWhqoioKto/MCJqJW0LTJX04hrXighJbTI9xMnZzHKlJT/fTom4qpn62vTnIkm3A0OANyT1iYiFadhiUWpeC/QvOrwixT4UD2uYWa601pizpG6Stli9DQwDngemAKNSs1HAnWl7CnBCmrWxL/BW0fBHi7nnbGa50oq/EOwN3C4JCrny9xFxr6SZwGRJo4F5wDGp/d3ACKAGWAGctD4Xd3I2s1xprWWqIuJVYLd1xBcDh6wjHsCprXJxnJzNLGe8TJWZWQZ5gVczswzyx/bNzDLInww1M8sgD2uYmWWQv+dsZpZB7jmbmWVQXsaclZf/L7MhkFS51kdVzPz3wtbJ39ZoX5Wlm9hHkP9e2Ac4OZuZZZCTs5lZBjk5ty+PK9q6+O+FfYBfCJqZZZB7zmZmGeTk3E4kDZf0kqQaSeeUPsLyTtJ4SYskPd/R92LZ4+TcDiR1Bn4NHA4MAo6TNKhj78oyYAIwvKNvwrLJybl9DAFqIuLViHgPmASM7OB7sg4WEY8ASzr6PiybnJzbRz9gftH+ghQzM1snJ2czswxycm4ftUD/ov2KFDMzWycn5/YxExgoaYCkjYFjgSkdfE9mlmFOzu0gIuqA04D7gLnA5IiY07F3ZR1N0s3AE8BOkhZIGt3R92TZ4V8ImpllkHvOZmYZ5ORsZpZBTs5mZhnk5GxmlkFOzmZmGeTkbGaWQU7OZmYZ5ORsZpZB/w/4qB3xZYBecgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5vOI13Gy5rzG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Calculate predicted probabilities for test data \n",
        "y_pred_prob_rf = grid_search_rf.predict_proba(x_test)[:, 1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mI7wpYf_5wSE",
        "colab_type": "code",
        "outputId": "18829a89-00ff-4564-90a2-00db7d25f187",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "metrics.roc_auc_score(y_test, y_pred_prob_rf)"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9786879234388352"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i4eujoJV6VQf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "outputId": "5c226c59-de41-4b90-9dbc-12ba6f6e881d"
      },
      "source": [
        "feature_importances = pd.Series(grid_search_rf.best_estimator_.feature_importances_, index=x_train.columns)\n",
        "feature_importances.nlargest(10).plot(kind='barh')"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f8b887b6cc0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAD4CAYAAADYU1DBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAar0lEQVR4nO3de5QV5Z3u8e8TQlQEYSKaIcRjx4iiiBLZOho1x1FjosZbwngjjmCWHKITZ5KlE6JmRbNmziGTdU6M8cyFcbwl3uI1rhBvY0JEBGE3Ag0CmgBJRCdeYjoqERV+5496+7ht+gbs/e69u5/PWr2ofuutt361dfv4VhVVigjMzMxq7X31LsDMzAYGB46ZmWXhwDEzsywcOGZmloUDx8zMsnh/vQtoZCNHjoyWlpZ6l2Fm1lRaW1tfjojdOrc7cHrQ0tJCuVyudxlmZk1F0q+7avcpNTMzy8KBY2ZmWThwzMwsCweOmZll4ZsGetC2vp2WGbPrXYYNcOtmnlTvEsyqwjMcMzPLol8EjqQrJV1S7zrMzKx7/SJwzMys8dU8cCS1SFol6UZJz0i6RdJxkuZJelbSoZI+KOk+ScskLZB0YNr2SknXS5ojaY2kiyvGvTyN9ziwb0X7BZIWSVoq6W5JQyQNk7RW0uDUZ5fK383MrPZyzXD2Bv43MDb9nAMcCVwCXAZcBTwVEQem32+u2HYs8GngUOCbkgZLmgicBUwATgQOqeh/T0QcEhEHASuBL0bEa8AcoOPq61mp39udC5U0TVJZUnnThvaqHLyZmeULnLUR0RYRm4EVwKNRvGq0DWihCJ8fAETEz4BdJe2Stp0dERsj4mXgReBDwFHAvRGxISL+CNxfsa8DJM2V1AZMBsal9uuAqWl5KnBDV4VGxKyIKEVEadCQ4VU5eDMzyxc4GyuWN1f8vpneb82u3HZTH/rfCPxNRIynmDntCBAR84AWSUcDgyJieZ8qNzOzqmiUmwbmUsxGSIHwcpq5dOcx4DRJO0kaBpxcsW4Y8EK6PjO503Y3A7fSzezGzMxqp1H+4ueVwPWSlgEbgPN66hwRiyXdASylOM22qGL1N4AngZfSn8Mq1t0C/ANwW9UqNzOzPlFxKWVgkDQJODUizu1L/x1GjYlR511d46rMeuYnDVizkdQaEaXO7Y0yw6k5Sd8HTqC4q61Pxo8eTtlfdjOzqhgwgRMRX653DWZmA1mj3DRgZmb9nAPHzMyycOCYmVkWDhwzM8vCgWNmZlk4cMzMLAsHjpmZZeHAMTOzLBw4ZmaWxYB50sC2aFvfTsuM2fUuw6xbfs6aNRPPcMzMLIt+EziSnqh3DWZm1r2mDxxJ7weIiE/UuxYzM+telsCRdJ+kVkkrJE1Lba9L+k5q+09Jh0qaI2mNpFNSn0GpzyJJyyT9j9R+tKS5ku4Hnu4Yr2J/X5PUJmmppJmp7YI0zlJJd0sakuPYzcyskGuGc35ETARKwMWSdgV2Bn4WEeOA1yjexPkp4HTgW2m7LwLtEXEIcAhwgaSPpnUHA38bEftU7kjSCcCpwF9ExEHAP6VV90TEIaltZRp7C5KmSSpLKm/a0F6Vgzczs3x3qV0s6fS0vAcwBngLeDC1tQEbI+JtSW1AS2o/HjgwvakTYHjFtgsjYm0X+zoOuCEiNgBExO9T+wGS/gEYAQwFHuqq0IiYBcyC4o2f23CsZmbWhZoHjqSjKULg8IjYIGkOsCPwdrz7fuvNwEaAiNjccV0GEPDliHioizHf2MpSbgROi4ilkqYAR2/tsZiZ2bbLcUptOPBqCpuxwGFbse1DwJckDQaQtI+knXvZ5hFgasc1GkkfTO3DgBfSWJO36gjMzGy75Til9iAwXdJKYDWwYCu2vY7i9NpiSQJeAk7raYOIeFDSBKAs6S3gp8BlwDeAJ9MYT1IEkJmZZaJ3z2pZZ6VSKcrlcr3LMDNrKpJaI6LUub3p/x6OmZk1BweOmZll4cAxM7MsHDhmZpaFA8fMzLJw4JiZWRYOHDMzy8KBY2ZmWThwzMwsCweOmZllkev1BE2pbX07LTNm17sMs7pYN/Okepdg/YxnOGZmloUDx8zMsmiawJH0YUl3dbNujqQtnkxqZmaNo2mu4UTE88CkXjuamVlDasgZjqSZki6q+P1KSZdIWp5+30nS7ZJWSroX2Kmi7/GS5ktaLOlOSUNT+7GSnpLUJul6STtkPzAzswGsIQMHuAM4o+L3Myje0tnhS8CGiNgP+CYwEUDSSOAK4LiIOBgoA1+VtCNwI3BmRIynmNl9qasdS5omqSypvGlDe3WPysxsAGvIwImIp4Dd03Wbg4BXgd9WdPkk8MPUdxmwLLUfBuwPzJO0BDgP2BPYF1gbEc+kfjelMbra96yIKEVEadCQ4VU+MjOzgauRr+HcSXHN5s8pZjx9IeCRiDj7PY1FaJmZWR015AwnuQM4iyJ07uy07jHgHABJBwAHpvYFwBGS9k7rdpa0D7AaaOloB84FflHb8s3MrFLDBk5ErACGAesj4oVOq/8FGCppJfAtoDVt8xIwBbhN0jJgPjA2It4EpgJ3SmoDNgP/muVAzMwMAEVEvWtoWKVSKcrlcr3LMDNrKpJaI2KLvxvZsDMcMzPrXxw4ZmaWhQPHzMyycOCYmVkWDhwzM8vCgWNmZlk4cMzMLAsHjpmZZeHAMTOzLBw4ZmaWRSM/Lbru2ta30zJjdr3LMGsY62aeVO8SrIl5hmNmZlk4cMzMLAsHjpmZZdEUgSPpC5IWSloi6d8kXSTpOxXrp0i6tpu+g1L765L+UdJSSQskfahex2NmNhA1fOBI2g84EzgiIiYAm4DXgdMrup0J3N5N38mpz87Agog4iOKNoRd0s79pksqSyps2tNfkmMzMBqJmuEvtWGAisEgSwE7Ai8AaSYcBzwJjgXnARd30BXgL+ElabgU+1dXOImIWMAtgh1Fj/HY6M7MqaYbAEXBTRHz9PY3S+cAZwCrg3ogIFSmzRd/k7Xj39aabaI5jNzPrNxr+lBrwKDBJ0u4Akj4oaU/gXuBU4Gzg9l76mplZnTV84ETE08AVwMOSlgGPAKMi4lVgJbBnRCzsqW99Kjczs0pNcVopIu4A7uii/bNb0XdoxfJdwF1VLtPMzHrQFIFTL+NHD6fsR3mYmVVFw59SMzOz/sGBY2ZmWThwzMwsCweOmZll4cAxM7MsHDhmZpaFA8fMzLJw4JiZWRYOHDMzy8KBY2ZmWfjRNj1oW99Oy4zZ9S7DrOGt8yOgrA88wzEzsyxqHjiSRki6sMb7WCdpZC33YWZm2yfHDGcEUNPAMTOzxpcjcGYCH5O0RNIiST/pWCHpWklT0vJESb+Q1CrpIUmjUvvFkp6WtEzS7altV0kPS1oh6TqK11B3jHlfGmOFpGmp7XxJV1f0uUDSdzMcu5mZJTkCZwbwq4iYAFzaVQdJg4HvA5MiYiJwPfCPFdt/PCIOBKantm8Cj0fEOIpXTf+3iuHOT2OUgIsl7Qr8CDg57QdgatpHV7VMk1SWVN60oX3bjtjMzLbQKHep7QscADwiCWAQ8EJatwy4RdJ9wH2p7ZPA5wAiYrakVyvGuljS6Wl5D2BMRCyQ9DPgs5JWAoMjoq2rQiJiFjALYIdRY6JaB2hmNtDlDpx3eO+sasf0p4AVEXF4F9ucRBEwJwOXSxrf3eCSjgaOAw6PiA2S5lTs4zrgMmAVcMN2HIOZmW2DHKfUXgOGpeVfA/tL2kHSCODY1L4a2E3S4VCcYpM0TtL7gD0i4ufA14DhwFDgMeCc1PcE4M/SOMOBV1PYjAUO6ygiIp6kmPGcA9xWs6M1M7Mu1XyGExGvSJonaTnwAMX1lOXAWuCp1OctSZOAayQNT3VdDTwD/DC1CbgmIv4g6SrgNkkrgCeA36TdPQhMT6fNVgMLOpXzI2BCRLyKmZllleWUWkSc06np77vos4Ti1FlnR3bR9xXg+G52d0IPpRwJ+O40M7M6aJSbBmoqnb5bCCyNiEf7ut340cMp+5EdZmZVMSACJyL+AOxT7zrMzAYyP0vNzMyycOCYmVkWDhwzM8vCgWNmZlk4cMzMLAsHjpmZZeHAMTOzLBw4ZmaWhQPHzMyyGBBPGthWbevbaZkxu95lmDW8dX4ElPWBZzhmZpZF1QNH0ghJF6bloyX9pNr76GKfV0q6pNb7MTOzbVeLGc4I4MJqDCTJp/zMzPqJWgTOTOBjkpYA3wGGSrpL0ipJt0gSgKR1kkam5VJ6HXTHbOUHkuYBP0hv/lwoaYmkZZLGpH6XS3pG0uPAvh07l3SBpEWSlkq6W9IQScMkrZU0OPXZpfJ3MzOrvVoEzgzgVxExAbgU+Djwd8D+wF7AEX0YY3/guIg4G5gOfC+NVwKekzQROAuYAJwIHFKx7T0RcUhEHASsBL4YEa8Bc4COK5tnpX5vd96xpGmSypLKmza0b+Whm5lZd3LcNLAwIp6LiM3AEqClD9vcHxF/SsvzgcskfQ3YM7UfBdwbERsi4o/A/RXbHiBprqQ2YDIwLrVfB0xNy1OBG7racUTMiohSRJQGDRm+FYdpZmY9yRE4GyuWN/HurdjvVOx/x07bvNGxEBG3AqcAfwJ+KumYXvZ3I/A3ETEeuKpj7IiYB7RIOhoYFBHLt/pIzMxsm9UicF4DhvWh3zpgYlr+fHedJO0FrImIa4AfAwcCjwGnSdpJ0jDg5IpNhgEvpOszkzsNdzNwK93MbszMrHaqHjgR8QowT9JyipsGunMV8D1JZYqZT3fOAJanmxAOAG6OiMXAHcBS4AFgUUX/bwBPAvOAVZ3GugX4M+C2vh+RmZlVgyKi3jVkI2kScGpEnNuX/qVSKcrlco2rMjPrXyS1RkSpc/uA+Xsukr4PnEBxV5uZmWU2YAInIr5c7xrMzAYyP0vNzMyycOCYmVkWDhwzM8vCgWNmZlk4cMzMLAsHjpmZZeHAMTOzLBw4ZmaWxYD5i5/bom19Oy0zZte7DLMBZ93Mk3rvZE3HMxwzM8vCgWNmZlk0dOBIakmvOTAzsybX0IFjZmb9R9MEjqS9JD0l6VJJ90h6UNKzkv6pos/ZktokLZf07dT2V5L+T1r+W0lrKsabV5+jMTMbeJriLjVJ+wK3A1OAjwMT0p8bgdXpXTebgG9TvLb6VeBhSacBc4G/T0MdBbwiaXRafqyLfU0DpgEM2mW32h2UmdkA0wwznN2AHwOTI2Jpans0Itoj4k3gaWBP4BBgTkS8FBHvULxO+pMR8V/AUEnDgD2AW4FPUgTO3M47i4hZEVGKiNKgIcNrfnBmZgNFMwROO/Ab4MiKto0Vy5vofab2BDAVWE0RMkcBhwM+pWZmlkkzBM5bwOnAX0s6p4d+C4H/LmmkpEHA2cAv0rq5wCUUp9CeAv4S2BgR7bUr28zMKjVD4BARbwCfBb4C7NJNnxeAGcDPgaVAa0T8OK2eS3E67bGI2AT8Fni81nWbmdm7FBH1rqFhlUqlKJfL9S7DzKypSGqNiFLn9qaY4ZiZWfNz4JiZWRYOHDMzy8KBY2ZmWThwzMwsCweOmZll4cAxM7MsHDhmZpaFA8fMzLJw4JiZWRZN8T6cemlb307LjNn1LsPMtsK6mSfVuwTrhmc4ZmaWRZbAkXS0pJ/UcPwWSctrNb6ZmW0/z3DMzCyLXgNH0qWSLk7L35X0s7R8jKRbJB0vab6kxZLulDQ0rf+MpFWSFgOfqxjvSknXS5ojaU3H2GndFyQtlLRE0r9JGpR+bpS0XFKbpK+kvhMlLZW0FLioYowWSXNTPYslfSK13yzptIp+t0g6dXs/QDMz65u+zHA6XskMUAKGShqc2pYBVwDHRcTBQBn4qqQdgX8HTgYmAn/eacyxwKeBQ4FvShosaT/gTOCIiJhA8eroycAEYHREHBAR44Eb0hg3AF+OiIM6jf0i8KlUz5nANan9P4ApAJKGA58AtrgjQNI0SWVJ5U0b/EJQM7Nq6UvgtAITJe0CbATmUwTPUcCfgP2BeZKWAOcBe1IEytqIeDaKN7z9sNOYsyNiY0S8TBEQHwKOpQinRWmsY4G9gDXAXpK+L+kzwB8ljQBGRMRjabwfVIw9GPh3SW3Anak+IuIXwBhJu1G8fvruiHin88FGxKyIKEVEadCQ4X34eMzMrC96vS06It6WtJZidvAExazmL4G9gbXAIxFxduU2kib0MuzGiuVNqQ4BN0XE1zt3lnQQxYxoOnAG8NUexv4K8DvgIIpAfbNi3c3AF4CzgKm91GhmZlXU15sG5gKXAI+l5enAU8AC4AhJewNI2lnSPsAqoEXSx9L2Z2855BYeBSZJ2j2N9UFJe0oaCbwvIu6mOH13cET8AfiDpCPTtpMrxhkOvBARm4FzgUEV624E/g4gIp7u47GbmVkVbE3gjALmR8TvKGYNcyPiJYqZz22SllGcbhsbEW8C04DZ6aaBF3vbQQqAK4CH01iPpH2OBuak02w/BDpmQFOB/5vaVTHUPwPnpZsJxgJvVOzjd8BK3r0OZGZmmai4xDIwSBoCtFHMknq9I2CHUWNi1HlX174wM6saP2mg/iS1RkSpc/uAebSNpOMo7lT7bl/CBmD86OGU/S+vmVlVDJjAiYj/pLiDzszM6sBPGjAzsywcOGZmloUDx8zMsnDgmJlZFg4cMzPLwoFjZmZZOHDMzCwLB46ZmWXhwDEzsywGzJMGtkXb+nZaZmzxjjYzs4bWqM+T8wzHzMyyaIjAkXSKpBm99PmwpLt6WD9C0oXVr87MzKqhIQInIu6PiJm99Hk+Iib10GUEsFWBo0JDfAZmZv1dzf9jK6lF0ipJN0p6RtItko6TNE/Ss5IOlTRF0rWp/42SrpH0hKQ1kiZVjLM8LY+TtFDSEknLJI0BZgIfS23fSf0ulbQo9bmqYpzVkm4GlgN71PozMDOzfDcN7A38FXA+sAg4BzgSOAW4DLivU/9Raf1Y4H6g86m06cD3IuIWSR+geI30DOCAiJgAIOl4YAxwKMUbQe+X9EngN6n9vIhY0LlQSdMo3lbKoF12276jNjOz/y9X4KyNiDYASSuARyMiJLUBLV30vy8iNgNPS/pQF+vnA5dL+ghwT0Q8K6lzn+PTz1Pp96EUQfMb4NddhQ1ARMwCZkHxxs+tOEYzM+tBrusXGyuWN1f8vpmuQ6+y/xZJEhG3UsyO/gT8VNIxXYwh4H9FxIT0s3dE/Eda98bWHoCZmW2fprxgLmkvYE1EXAP8GDgQeA0YVtHtIeB8SUPTNqMl7Z69WDMzA5r3L36eAZwr6W3gv4D/GRG/TzciLAceiIhLJe0HzE+n214HvgBsqlvVZmYDmCJ8maI7O4waE6POu7reZZiZbZV6P2lAUmtElDq3N+sMJ4vxo4dTbtBHRJiZNZumvIZjZmbNx4FjZmZZOHDMzCwLB46ZmWXhwDEzsywcOGZmloUDx8zMsnDgmJlZFg4cMzPLwk8a6EHb+nZaZsyudxlmZlnV6tE4nuGYmVkWDhwzM8ui6QNH0uvpzw9LuistT5F0bX0rMzOzSv3mGk5EPA9MqncdZmbWtaaf4XSQ1JJevta5/SRJ8yWNlHR8Wl4s6c6Ot4GamVnt9ZvA6Yqk04EZwImp6QrguIg4GCgDX+1im2mSypLKmza05yvWzKyf6zen1LpwDFACjo+IP0r6LLA/MC+9cvoDwPzOG0XELGAWFG/8zFeumVn/1p8D51fAXsA+FLMZAY9ExNl1rcrMbIDqz6fUfg18HrhZ0jhgAXCEpL0BJO0saZ96FmhmNpD058AhIlYBk4E7gV2AKcBtkpZRnE4bW7/qzMwGFkX4MkV3SqVSlMvlepdhZtZUJLVGRKlze7+e4ZiZWeNw4JiZWRYOHDMzy8KBY2ZmWThwzMwsC9+l1gNJrwGr611HH4wEXq53EX3gOqvLdVaX66yePSNit86N/flJA9Wwuqtb+xqNpLLrrB7XWV2us7qapc6u+JSamZll4cAxM7MsHDg9m1XvAvrIdVaX66wu11ldzVLnFnzTgJmZZeEZjpmZZeHAMTOzLAZM4Ej6jKTVkn4paUYX63eQdEda/6Sklop1X0/tqyV9uq9jNlCd6yS1SVoiqSqPv97WOiXtKunnkl6XdG2nbSamOn8p6RqlV7M2YJ1z0phL0s/udazzU5Ja0+fWKumYim0a6fPsqc5G+jwPrahjqYrX1PdpzAaqs+rf96qJiH7/Awzi3TeAfgBYCuzfqc+FwL+m5bOAO9Ly/qn/DsBH0ziD+jJmI9SZ1q0DRjbI57kzcCQwHbi20zYLgcMo3s76AHBCg9Y5Byg1yOf5ceDDafkAYH2Dfp491dlIn+cQ4P1peRTwIsXfV2y073uXdabf11HF73s1fwbKDOdQ4JcRsSYi3gJuB07t1OdU4Ka0fBdwbPo/wlOB2yNiY0SsBX6ZxuvLmI1QZy1sc50R8UZEPA68WdlZ0ihgl4hYEMW35mbgtEars0a2p86nIuL51L4C2Cn9X3GjfZ5d1rmd9dSizg0R8U5q3xHouKuqob7vPdTZ0AZK4IwGflvx+3Oprcs+6R9kO7BrD9v2ZcxGqBOKfxkfTqcypm1njdtbZ09jPtfLmI1QZ4cb0imLb1ThVFW16vw8sDgiNtLYn2dlnR0a5vOU9BeSVgBtwPS0vtG+793VCdX/vleNH20zMBwZEevTufFHJK2KiMfqXVQTm5w+z2HA3cC5FDOIupE0Dvg2cHw96+hNN3U21OcZEU8C4yTtB9wk6YF61dKTruqMiDdp4O/7QJnhrAf2qPj9I6mtyz6S3g8MB17pYdu+jNkIdRIRHX++CNzL9p9q2546exrzI72M2Qh1Vn6erwG3UufPU9JHKP65/nVE/Kqif0N9nt3U2XCfZ0VdK4HXSdec+jBmI9RZi+979dT7IlKOH4qZ3BqKi+kdF+fGdepzEe+9OPejtDyO916MX0Nxsa/XMRukzp2BYanPzsATwGfqVWfF+in0ftPAiY1WZxpzZFoeTHFefXod/7mPSP0/18W4DfN5dldnA36eH+Xdi+97As9TPJ250b7v3dVZ9e97NX/qXkC2A4UTgWco7gq5PLV9CzglLe8I3ElxsX0hsFfFtpen7VZTcadPV2M2Wp0Ud8AsTT8rGqTOdcDvKf6v7DnSnTlACViexryW9CSMRqozfYlbgWXp8/we6W7AetQJXAG8ASyp+Nm90T7P7upswM/z3FTHEmAxcFojft+7q5Mafd+r9eNH25iZWRYD5RqOmZnVmQPHzMyycOCYmVkWDhwzM8vCgWNmZlk4cMzMLAsHjpmZZfH/ADxnOPIibXihAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CN3og0p2tDah",
        "colab_type": "text"
      },
      "source": [
        "# Neural Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GE1WUXj7jtlu",
        "colab_type": "text"
      },
      "source": [
        "BERT can be used in two ways: 1) to produce a vector representation of each word as the output or 2) fine-tune the original model and train it on data with a classification as the output.\n",
        "\n",
        "I use the basic bert-base-uncased model. The maximum number of tokens allowed for BERT is 512, so articles are truncated. This could pose a possible issue."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fNRHGMe5vygb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sklearn.model_selection as ms\n",
        "\n",
        "#Create new train-test set\n",
        "X = pd.concat([true[\"text\"], fake[\"text\"]])\n",
        "true[\"label\"] = [0 for x in true[\"label\"]]\n",
        "fake[\"label\"] = [1 for x in fake[\"label\"]]\n",
        "y = pd.concat([true[\"label\"], fake[\"label\"]])\n",
        "\n",
        "X_train, X_test, y_train, y_test = ms.train_test_split(X, y, test_size = 0.2, random_state=1)\n",
        "train_data = pd.concat([X_train, y_train],axis = 1).reset_index(drop=True)\n",
        "test_data = pd.concat([X_test, y_test],axis = 1).reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ed0WqarkjE95",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = pd.DataFrame(X)\n",
        "articles = X[\"text\"].tolist()\n",
        "y = pd.DataFrame(y)\n",
        "labels = y[\"label\"].tolist()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_SECJbbClVhy",
        "colab_type": "code",
        "outputId": "3feed53d-18e2-4612-b44b-6e92c09c12c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "74b38c71a0bf429b9410639ad163f05c",
            "527409c0cfc649f8a7c92e763144b08c",
            "26b137ad3f174023a302bddf59b81900",
            "4146c4dd98fe415cb9026492d1ff22e2",
            "0ab27068e3434e97a119812517ac3c0b",
            "6232b8b4f2d047218bd49978714a12be",
            "ab590566b5fa4726b27579b9049b17d6",
            "2a83c1ae880b4db1964b74643c778851"
          ]
        }
      },
      "source": [
        "#Create new tokenizer because it's best to tokenize with the same tokenizer \n",
        "#that BERT was trained on instead of self-defined one \n",
        "from transformers import BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "74b38c71a0bf429b9410639ad163f05c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MJDsBDwkl-1x",
        "colab_type": "text"
      },
      "source": [
        "Note: attention_masks adds 1 to the scores for positions that are taken up by real tokens and adds 0 to the positions outside current sequence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XtG0vtCYjAv5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Tokenize all of the articles and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "for article in articles:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        article,                      #Article to encode\n",
        "                        add_special_tokens = True, #Add [CLS] to start, [SEP] to end\n",
        "                        max_length = 64,           #Pad & truncate all sentences\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   #Create masks\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors\n",
        "                   )\n",
        "    \n",
        "    #Add the encoded sentence to the list    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    \n",
        "    # Add attention mask (simply differentiates padding from non-padding)\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "# Convert the lists into tensors\n",
        "input_ids = torch.cat(input_ids, dim=0) #encoded sentence\n",
        "attention_masks = torch.cat(attention_masks, dim=0) #mask\n",
        "labels = torch.tensor(labels) #labels (fake/true)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W4LQN86bmUgd",
        "colab_type": "code",
        "outputId": "877e8c37-a593-4c30-9b58-2a6419def766",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "# Combine the training inputs into a TensorDataset\n",
        "dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "# Create a 70-10-20 train-validation-test split\n",
        "\n",
        "# Calculate the number of samples to include in each set.\n",
        "train_size = int(0.7 * len(dataset))\n",
        "val_size = int(0.1 * len(dataset)) + 1 #add 1 because int() causes 1 row to be left out\n",
        "test_size = int(0.2 * len(dataset)) \n",
        "\n",
        "train_dataset, val_dataset, test_dataset = random_split(dataset, [train_size, val_size, test_size])\n",
        "\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print('{:>5,} validation samples'.format(val_size))\n",
        "print('{:>5,} test samples'.format(test_size))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "32,692 training samples\n",
            "4,671 validation samples\n",
            "9,340 test samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y8L4W_Smmst0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "#BERT developers recommend 16 or 32 batch_size\n",
        "batch_size = 16\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  \n",
        "            sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "            batch_size = batch_size)\n",
        "\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset, \n",
        "            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially, since order doesn't matter here\n",
        "            batch_size = batch_size)\n",
        "\n",
        "test_dataloader = DataLoader(\n",
        "            test_dataset, \n",
        "            sampler = SequentialSampler(test_dataset), # Pull out batches sequentially, since order doesn't matter here\n",
        "            batch_size = batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "42qLMvZamyRa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer  \n",
        "# Don't output all layers due to memory limits\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab\n",
        "    num_labels = 2, #binary classification  \n",
        "    output_attentions = False, \n",
        "    output_hidden_states = False)\n",
        "\n",
        "#run model on the GPU\n",
        "model.cuda()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VkZeFvllm8dA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Use Adam optimizer\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 0.00002,\n",
        "                  eps = 1e-8) # default is 1e-8"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Po1WlwAkm_rA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# BERT authors recommend between 2 and 4 epochs\n",
        "epochs = 4\n",
        "\n",
        "# number of batches x number of epochs\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler to reduce the learning rate as training epochs increase\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, #default\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PrTJNJqzm_fk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return metrics.accuracy_score(pred_flat, labels_flat)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VLE7gsDP8o2d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "#Takes a time in seconds and returns a string hh:mm:ss\n",
        "def format_time(elapsed):\n",
        "    \n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tp1DuKV08kOZ",
        "colab_type": "code",
        "outputId": "a3b520c8-a6b9-4f10-bbb7-588d7249512f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "import random\n",
        "\n",
        "#Use a seed to ensure reproductibility\n",
        "seed_val = 22\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "\n",
        "\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "\n",
        "    # Measure how long the training epoch takes\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch\n",
        "    total_train_loss = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        b_input_ids = batch[0].to(device) #copy each tensor to the GPU\n",
        "        b_input_mask = batch[1].to(device) #copy each tensor to the GPU\n",
        "        b_labels = batch[2].to(device) #copy each tensor to the GPU\n",
        "\n",
        "        model.zero_grad()        \n",
        "\n",
        "        # Perform forward pass \n",
        "        # Returns the loss (because we provided labels) and the \"logits\" - the model\n",
        "        # outputs prior to activation\n",
        "        loss, logits = model(b_input_ids, \n",
        "                             token_type_ids=None, \n",
        "                             attention_mask=b_input_mask, \n",
        "                             labels=b_labels)\n",
        "\n",
        "        total_train_loss += loss.item()\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0\n",
        "        # This is to help prevent the \"exploding gradients\" problem\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "        optimizer.step()\n",
        "        scheduler.step() # Update the learning rate\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Measure how long this epoch took\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        b_input_ids = batch[0].to(device) #copy each tensor to the GPU\n",
        "        b_input_mask = batch[1].to(device) #copy each tensor to the GPU\n",
        "        b_labels = batch[2].to(device) #copy each tensor to the GPU\n",
        "        \n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():        \n",
        "            (loss, logits) = model(b_input_ids, \n",
        "                                   token_type_ids=None, \n",
        "                                   attention_mask=b_input_mask,\n",
        "                                   labels=b_labels)         \n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += accuracy(logits, label_ids)\n",
        "        \n",
        "    # Report the final accuracy for this validation run\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "    \n",
        "    # Measure how long the validation run took\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    \n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/pytorch/torch/csrc/utils/python_arg_parser.cpp:756: UserWarning: This overload of add_ is deprecated:\n",
            "\tadd_(Number alpha, Tensor other)\n",
            "Consider using one of the following signatures instead:\n",
            "\tadd_(Tensor other, *, Number alpha)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch    40  of  2,044.    Elapsed: 0:00:11.\n",
            "  Batch    80  of  2,044.    Elapsed: 0:00:21.\n",
            "  Batch   120  of  2,044.    Elapsed: 0:00:32.\n",
            "  Batch   160  of  2,044.    Elapsed: 0:00:42.\n",
            "  Batch   200  of  2,044.    Elapsed: 0:00:53.\n",
            "  Batch   240  of  2,044.    Elapsed: 0:01:03.\n",
            "  Batch   280  of  2,044.    Elapsed: 0:01:14.\n",
            "  Batch   320  of  2,044.    Elapsed: 0:01:24.\n",
            "  Batch   360  of  2,044.    Elapsed: 0:01:35.\n",
            "  Batch   400  of  2,044.    Elapsed: 0:01:45.\n",
            "  Batch   440  of  2,044.    Elapsed: 0:01:56.\n",
            "  Batch   480  of  2,044.    Elapsed: 0:02:06.\n",
            "  Batch   520  of  2,044.    Elapsed: 0:02:17.\n",
            "  Batch   560  of  2,044.    Elapsed: 0:02:27.\n",
            "  Batch   600  of  2,044.    Elapsed: 0:02:37.\n",
            "  Batch   640  of  2,044.    Elapsed: 0:02:48.\n",
            "  Batch   680  of  2,044.    Elapsed: 0:02:59.\n",
            "  Batch   720  of  2,044.    Elapsed: 0:03:09.\n",
            "  Batch   760  of  2,044.    Elapsed: 0:03:19.\n",
            "  Batch   800  of  2,044.    Elapsed: 0:03:30.\n",
            "  Batch   840  of  2,044.    Elapsed: 0:03:40.\n",
            "  Batch   880  of  2,044.    Elapsed: 0:03:51.\n",
            "  Batch   920  of  2,044.    Elapsed: 0:04:01.\n",
            "  Batch   960  of  2,044.    Elapsed: 0:04:12.\n",
            "  Batch 1,000  of  2,044.    Elapsed: 0:04:22.\n",
            "  Batch 1,040  of  2,044.    Elapsed: 0:04:32.\n",
            "  Batch 1,080  of  2,044.    Elapsed: 0:04:43.\n",
            "  Batch 1,120  of  2,044.    Elapsed: 0:04:53.\n",
            "  Batch 1,160  of  2,044.    Elapsed: 0:05:04.\n",
            "  Batch 1,200  of  2,044.    Elapsed: 0:05:14.\n",
            "  Batch 1,240  of  2,044.    Elapsed: 0:05:25.\n",
            "  Batch 1,280  of  2,044.    Elapsed: 0:05:35.\n",
            "  Batch 1,320  of  2,044.    Elapsed: 0:05:45.\n",
            "  Batch 1,360  of  2,044.    Elapsed: 0:05:56.\n",
            "  Batch 1,400  of  2,044.    Elapsed: 0:06:06.\n",
            "  Batch 1,440  of  2,044.    Elapsed: 0:06:17.\n",
            "  Batch 1,480  of  2,044.    Elapsed: 0:06:27.\n",
            "  Batch 1,520  of  2,044.    Elapsed: 0:06:38.\n",
            "  Batch 1,560  of  2,044.    Elapsed: 0:06:48.\n",
            "  Batch 1,600  of  2,044.    Elapsed: 0:06:58.\n",
            "  Batch 1,640  of  2,044.    Elapsed: 0:07:09.\n",
            "  Batch 1,680  of  2,044.    Elapsed: 0:07:19.\n",
            "  Batch 1,720  of  2,044.    Elapsed: 0:07:30.\n",
            "  Batch 1,760  of  2,044.    Elapsed: 0:07:40.\n",
            "  Batch 1,800  of  2,044.    Elapsed: 0:07:51.\n",
            "  Batch 1,840  of  2,044.    Elapsed: 0:08:01.\n",
            "  Batch 1,880  of  2,044.    Elapsed: 0:08:11.\n",
            "  Batch 1,920  of  2,044.    Elapsed: 0:08:22.\n",
            "  Batch 1,960  of  2,044.    Elapsed: 0:08:32.\n",
            "  Batch 2,000  of  2,044.    Elapsed: 0:08:42.\n",
            "  Batch 2,040  of  2,044.    Elapsed: 0:08:53.\n",
            "\n",
            "  Average training loss: 0.10\n",
            "  Training epcoh took: 0:08:54\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation Loss: 0.04\n",
            "  Validation took: 0:00:19\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of  2,044.    Elapsed: 0:00:10.\n",
            "  Batch    80  of  2,044.    Elapsed: 0:00:21.\n",
            "  Batch   120  of  2,044.    Elapsed: 0:00:31.\n",
            "  Batch   160  of  2,044.    Elapsed: 0:00:41.\n",
            "  Batch   200  of  2,044.    Elapsed: 0:00:52.\n",
            "  Batch   240  of  2,044.    Elapsed: 0:01:02.\n",
            "  Batch   280  of  2,044.    Elapsed: 0:01:13.\n",
            "  Batch   320  of  2,044.    Elapsed: 0:01:23.\n",
            "  Batch   360  of  2,044.    Elapsed: 0:01:33.\n",
            "  Batch   400  of  2,044.    Elapsed: 0:01:44.\n",
            "  Batch   440  of  2,044.    Elapsed: 0:01:54.\n",
            "  Batch   480  of  2,044.    Elapsed: 0:02:04.\n",
            "  Batch   520  of  2,044.    Elapsed: 0:02:15.\n",
            "  Batch   560  of  2,044.    Elapsed: 0:02:25.\n",
            "  Batch   600  of  2,044.    Elapsed: 0:02:35.\n",
            "  Batch   640  of  2,044.    Elapsed: 0:02:46.\n",
            "  Batch   680  of  2,044.    Elapsed: 0:02:56.\n",
            "  Batch   720  of  2,044.    Elapsed: 0:03:06.\n",
            "  Batch   760  of  2,044.    Elapsed: 0:03:17.\n",
            "  Batch   800  of  2,044.    Elapsed: 0:03:27.\n",
            "  Batch   840  of  2,044.    Elapsed: 0:03:37.\n",
            "  Batch   880  of  2,044.    Elapsed: 0:03:48.\n",
            "  Batch   920  of  2,044.    Elapsed: 0:03:58.\n",
            "  Batch   960  of  2,044.    Elapsed: 0:04:09.\n",
            "  Batch 1,000  of  2,044.    Elapsed: 0:04:19.\n",
            "  Batch 1,040  of  2,044.    Elapsed: 0:04:29.\n",
            "  Batch 1,080  of  2,044.    Elapsed: 0:04:40.\n",
            "  Batch 1,120  of  2,044.    Elapsed: 0:04:50.\n",
            "  Batch 1,160  of  2,044.    Elapsed: 0:05:00.\n",
            "  Batch 1,200  of  2,044.    Elapsed: 0:05:11.\n",
            "  Batch 1,240  of  2,044.    Elapsed: 0:05:21.\n",
            "  Batch 1,280  of  2,044.    Elapsed: 0:05:31.\n",
            "  Batch 1,320  of  2,044.    Elapsed: 0:05:42.\n",
            "  Batch 1,360  of  2,044.    Elapsed: 0:05:52.\n",
            "  Batch 1,400  of  2,044.    Elapsed: 0:06:02.\n",
            "  Batch 1,440  of  2,044.    Elapsed: 0:06:13.\n",
            "  Batch 1,480  of  2,044.    Elapsed: 0:06:23.\n",
            "  Batch 1,520  of  2,044.    Elapsed: 0:06:33.\n",
            "  Batch 1,560  of  2,044.    Elapsed: 0:06:44.\n",
            "  Batch 1,600  of  2,044.    Elapsed: 0:06:54.\n",
            "  Batch 1,640  of  2,044.    Elapsed: 0:07:05.\n",
            "  Batch 1,680  of  2,044.    Elapsed: 0:07:15.\n",
            "  Batch 1,720  of  2,044.    Elapsed: 0:07:25.\n",
            "  Batch 1,760  of  2,044.    Elapsed: 0:07:36.\n",
            "  Batch 1,800  of  2,044.    Elapsed: 0:07:46.\n",
            "  Batch 1,840  of  2,044.    Elapsed: 0:07:56.\n",
            "  Batch 1,880  of  2,044.    Elapsed: 0:08:07.\n",
            "  Batch 1,920  of  2,044.    Elapsed: 0:08:17.\n",
            "  Batch 1,960  of  2,044.    Elapsed: 0:08:27.\n",
            "  Batch 2,000  of  2,044.    Elapsed: 0:08:38.\n",
            "  Batch 2,040  of  2,044.    Elapsed: 0:08:48.\n",
            "\n",
            "  Average training loss: 0.02\n",
            "  Training epcoh took: 0:08:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation Loss: 0.04\n",
            "  Validation took: 0:00:19\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of  2,044.    Elapsed: 0:00:10.\n",
            "  Batch    80  of  2,044.    Elapsed: 0:00:21.\n",
            "  Batch   120  of  2,044.    Elapsed: 0:00:31.\n",
            "  Batch   160  of  2,044.    Elapsed: 0:00:41.\n",
            "  Batch   200  of  2,044.    Elapsed: 0:00:52.\n",
            "  Batch   240  of  2,044.    Elapsed: 0:01:02.\n",
            "  Batch   280  of  2,044.    Elapsed: 0:01:12.\n",
            "  Batch   320  of  2,044.    Elapsed: 0:01:23.\n",
            "  Batch   360  of  2,044.    Elapsed: 0:01:33.\n",
            "  Batch   400  of  2,044.    Elapsed: 0:01:43.\n",
            "  Batch   440  of  2,044.    Elapsed: 0:01:54.\n",
            "  Batch   480  of  2,044.    Elapsed: 0:02:04.\n",
            "  Batch   520  of  2,044.    Elapsed: 0:02:14.\n",
            "  Batch   560  of  2,044.    Elapsed: 0:02:25.\n",
            "  Batch   600  of  2,044.    Elapsed: 0:02:35.\n",
            "  Batch   640  of  2,044.    Elapsed: 0:02:45.\n",
            "  Batch   680  of  2,044.    Elapsed: 0:02:56.\n",
            "  Batch   720  of  2,044.    Elapsed: 0:03:06.\n",
            "  Batch   760  of  2,044.    Elapsed: 0:03:16.\n",
            "  Batch   800  of  2,044.    Elapsed: 0:03:27.\n",
            "  Batch   840  of  2,044.    Elapsed: 0:03:37.\n",
            "  Batch   880  of  2,044.    Elapsed: 0:03:47.\n",
            "  Batch   920  of  2,044.    Elapsed: 0:03:58.\n",
            "  Batch   960  of  2,044.    Elapsed: 0:04:08.\n",
            "  Batch 1,000  of  2,044.    Elapsed: 0:04:18.\n",
            "  Batch 1,040  of  2,044.    Elapsed: 0:04:29.\n",
            "  Batch 1,080  of  2,044.    Elapsed: 0:04:39.\n",
            "  Batch 1,120  of  2,044.    Elapsed: 0:04:49.\n",
            "  Batch 1,160  of  2,044.    Elapsed: 0:05:00.\n",
            "  Batch 1,200  of  2,044.    Elapsed: 0:05:10.\n",
            "  Batch 1,240  of  2,044.    Elapsed: 0:05:20.\n",
            "  Batch 1,280  of  2,044.    Elapsed: 0:05:31.\n",
            "  Batch 1,320  of  2,044.    Elapsed: 0:05:41.\n",
            "  Batch 1,360  of  2,044.    Elapsed: 0:05:51.\n",
            "  Batch 1,400  of  2,044.    Elapsed: 0:06:02.\n",
            "  Batch 1,440  of  2,044.    Elapsed: 0:06:12.\n",
            "  Batch 1,480  of  2,044.    Elapsed: 0:06:22.\n",
            "  Batch 1,520  of  2,044.    Elapsed: 0:06:33.\n",
            "  Batch 1,560  of  2,044.    Elapsed: 0:06:43.\n",
            "  Batch 1,600  of  2,044.    Elapsed: 0:06:53.\n",
            "  Batch 1,640  of  2,044.    Elapsed: 0:07:04.\n",
            "  Batch 1,680  of  2,044.    Elapsed: 0:07:14.\n",
            "  Batch 1,720  of  2,044.    Elapsed: 0:07:24.\n",
            "  Batch 1,760  of  2,044.    Elapsed: 0:07:35.\n",
            "  Batch 1,800  of  2,044.    Elapsed: 0:07:45.\n",
            "  Batch 1,840  of  2,044.    Elapsed: 0:07:56.\n",
            "  Batch 1,880  of  2,044.    Elapsed: 0:08:06.\n",
            "  Batch 1,920  of  2,044.    Elapsed: 0:08:16.\n",
            "  Batch 1,960  of  2,044.    Elapsed: 0:08:27.\n",
            "  Batch 2,000  of  2,044.    Elapsed: 0:08:37.\n",
            "  Batch 2,040  of  2,044.    Elapsed: 0:08:47.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:08:48\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation Loss: 0.06\n",
            "  Validation took: 0:00:19\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of  2,044.    Elapsed: 0:00:10.\n",
            "  Batch    80  of  2,044.    Elapsed: 0:00:21.\n",
            "  Batch   120  of  2,044.    Elapsed: 0:00:31.\n",
            "  Batch   160  of  2,044.    Elapsed: 0:00:41.\n",
            "  Batch   200  of  2,044.    Elapsed: 0:00:52.\n",
            "  Batch   240  of  2,044.    Elapsed: 0:01:02.\n",
            "  Batch   280  of  2,044.    Elapsed: 0:01:12.\n",
            "  Batch   320  of  2,044.    Elapsed: 0:01:23.\n",
            "  Batch   360  of  2,044.    Elapsed: 0:01:33.\n",
            "  Batch   400  of  2,044.    Elapsed: 0:01:43.\n",
            "  Batch   440  of  2,044.    Elapsed: 0:01:54.\n",
            "  Batch   480  of  2,044.    Elapsed: 0:02:04.\n",
            "  Batch   520  of  2,044.    Elapsed: 0:02:14.\n",
            "  Batch   560  of  2,044.    Elapsed: 0:02:25.\n",
            "  Batch   600  of  2,044.    Elapsed: 0:02:35.\n",
            "  Batch   640  of  2,044.    Elapsed: 0:02:45.\n",
            "  Batch   680  of  2,044.    Elapsed: 0:02:56.\n",
            "  Batch   720  of  2,044.    Elapsed: 0:03:06.\n",
            "  Batch   760  of  2,044.    Elapsed: 0:03:16.\n",
            "  Batch   800  of  2,044.    Elapsed: 0:03:27.\n",
            "  Batch   840  of  2,044.    Elapsed: 0:03:37.\n",
            "  Batch   880  of  2,044.    Elapsed: 0:03:47.\n",
            "  Batch   920  of  2,044.    Elapsed: 0:03:58.\n",
            "  Batch   960  of  2,044.    Elapsed: 0:04:08.\n",
            "  Batch 1,000  of  2,044.    Elapsed: 0:04:18.\n",
            "  Batch 1,040  of  2,044.    Elapsed: 0:04:29.\n",
            "  Batch 1,080  of  2,044.    Elapsed: 0:04:39.\n",
            "  Batch 1,120  of  2,044.    Elapsed: 0:04:49.\n",
            "  Batch 1,160  of  2,044.    Elapsed: 0:05:00.\n",
            "  Batch 1,200  of  2,044.    Elapsed: 0:05:10.\n",
            "  Batch 1,240  of  2,044.    Elapsed: 0:05:20.\n",
            "  Batch 1,280  of  2,044.    Elapsed: 0:05:31.\n",
            "  Batch 1,320  of  2,044.    Elapsed: 0:05:41.\n",
            "  Batch 1,360  of  2,044.    Elapsed: 0:05:51.\n",
            "  Batch 1,400  of  2,044.    Elapsed: 0:06:02.\n",
            "  Batch 1,440  of  2,044.    Elapsed: 0:06:12.\n",
            "  Batch 1,480  of  2,044.    Elapsed: 0:06:22.\n",
            "  Batch 1,520  of  2,044.    Elapsed: 0:06:33.\n",
            "  Batch 1,560  of  2,044.    Elapsed: 0:06:43.\n",
            "  Batch 1,600  of  2,044.    Elapsed: 0:06:53.\n",
            "  Batch 1,640  of  2,044.    Elapsed: 0:07:04.\n",
            "  Batch 1,680  of  2,044.    Elapsed: 0:07:14.\n",
            "  Batch 1,720  of  2,044.    Elapsed: 0:07:24.\n",
            "  Batch 1,760  of  2,044.    Elapsed: 0:07:35.\n",
            "  Batch 1,800  of  2,044.    Elapsed: 0:07:45.\n",
            "  Batch 1,840  of  2,044.    Elapsed: 0:07:55.\n",
            "  Batch 1,880  of  2,044.    Elapsed: 0:08:06.\n",
            "  Batch 1,920  of  2,044.    Elapsed: 0:08:16.\n",
            "  Batch 1,960  of  2,044.    Elapsed: 0:08:26.\n",
            "  Batch 2,000  of  2,044.    Elapsed: 0:08:37.\n",
            "  Batch 2,040  of  2,044.    Elapsed: 0:08:47.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:08:48\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation Loss: 0.05\n",
            "  Validation took: 0:00:19\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:36:36 (h:mm:ss)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oPo7PBk9Poxn",
        "colab_type": "code",
        "outputId": "f7ca4147-60e2-4819-ab6c-85c9ca717201",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        }
      },
      "source": [
        "pd.set_option('precision', 2)\n",
        "\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "\n",
        "df_stats"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.03e-01</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.99</td>\n",
              "      <td>0:08:54</td>\n",
              "      <td>0:00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2.45e-02</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.99</td>\n",
              "      <td>0:08:49</td>\n",
              "      <td>0:00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7.39e-03</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.99</td>\n",
              "      <td>0:08:48</td>\n",
              "      <td>0:00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.06e-03</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.99</td>\n",
              "      <td>0:08:48</td>\n",
              "      <td>0:00:19</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Training Loss  Valid. Loss  Valid. Accur. Training Time Validation Time\n",
              "epoch                                                                         \n",
              "1           1.03e-01         0.04           0.99       0:08:54         0:00:19\n",
              "2           2.45e-02         0.04           0.99       0:08:49         0:00:19\n",
              "3           7.39e-03         0.06           0.99       0:08:48         0:00:19\n",
              "4           1.06e-03         0.05           0.99       0:08:48         0:00:19"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "toBHWDqLPwoR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sns.set(style='darkgrid')\n",
        "\n",
        "sns.set(font_scale=0.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QViYaMjJQTsO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.eval()\n",
        "\n",
        "predictions , true_labels = [], []\n",
        "\n",
        "for batch in test_dataloader:\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  \n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  \n",
        "  # Telling the model not to compute or store gradients, saving memory and \n",
        "  # speeding up prediction\n",
        "  with torch.no_grad():\n",
        "      # Forward pass, calculate logit predictions\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs[0]\n",
        "\n",
        "  # Move logits and labels to CPU\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  # Store predictions and true labels\n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SkbnfmDQlTBC",
        "colab_type": "code",
        "outputId": "d5c0eedb-0d37-45a8-dce2-85a343ca41ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "flat_predictions = np.concatenate(predictions, axis=0)\n",
        "\n",
        "# For each sample, pick the label (0 or 1) with the higher score.\n",
        "flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "\n",
        "# Combine the correct labels for each batch into a single list.\n",
        "flat_true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "print(\"Accuracy is:\", metrics.accuracy_score(flat_predictions,flat_true_labels))\n",
        "print(\"Mean Squared Error is:\", np.sqrt(mean_squared_error(flat_predictions, flat_true_labels)))\n",
        "b_cm = metrics.confusion_matrix(flat_predictions,flat_true_labels)\n",
        "print(b_cm)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy is: 0.99796573875803\n",
            "Mean Squared Error is: 0.04510278530168643\n",
            "[[4519   12]\n",
            " [   7 4802]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XLX1Vyd3m-O5",
        "colab_type": "code",
        "outputId": "2860543f-9fdf-4e06-8e19-ec7047048c13",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 385
        }
      },
      "source": [
        "labels = np.array([['4519','12'],['7','4802']])\n",
        "sns.heatmap(b_cm, annot=labels, fmt = '')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAFwCAYAAABq9LkwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3SU1d328YtADiQQjhNEEAErAQkJAasEELVQDRQ5aILYEohY1AIuhUd9QNRaxReFyAMWsIoWo1JELBCREhSwHgqVihxERiyIggaSAZRASGYCmfcPmqnjTOAeks3cge+nK6vNvvfs2elaXb387cNdx+v1egUAAACEWUS4JwAAAABIBFMAAADYBMEUAAAAtkAwBQAAgC0QTAEAAGALBFMAAADYQr1wT+DH3J+vDfcUANQCcakjwz0FALXECc934Z6CJKn84FfV+nxk8/Y1NBN7o2IKAAAAW7BVxRQAAOC8VHEy3DOoFQimAAAApnkrwj2DWoFgCgAAYFoFwdQK9pgCAADAFqiYAgAAGOZlKd8SgikAAIBpLOVbQjAFAAAwjYqpJewxBQAAgC1QMQUAADCNe0wtIZgCAACYxlK+JQRTAAAA0zj8ZAnBFAAAwDCui7KGw08AAACwBSqmAAAAprGUbwnBFAAAwDSW8i0hmAIAAJjGdVGWEEwBAABMo2JqCYefAAAAYAtUTAEAAEzj8JMlBFMAAADTWMq3hGAKAABgGhVTS9hjCgAAAFugYgoAAGCY18t1UVYQTAEAAExjj6klBFMAAADT2GNqCcEUAADANCqmlnD4CQAAALZAxRQAAMC0Cg4/WUEwBQAAMI2lfEsIpgAAAKZx+MkSgikAAIBpVEwt4fATAAAAbIGKKQAAgGks5VtCMAUAADCNYGoJwRQAAMAwr5froqxgjykAAABsgYopAACAaSzlW0IwBQAAMI3roiwhmAIAAJhGxdQSgikAAIBpVEwt4fATAAAAbIGKKQAAgGks5VtCMAUAADCNpXxLCKYAAACmUTG1hD2mAAAAsAUqpgAAAKZRMbWEYAoAAGAae0wtYSkfAADAtIqK6v1Uw/z585WYmKjBgwcHPPv000912223KSUlRb169dLUqVNVWloa0M/j8WjGjBnq3bu3kpOTNWzYMG3YsCHo91kdMxiCKQAAgGneiur9nCWXy6XnnntOsbGxAc+cTqeys7Pldrs1adIkZWRkaPHixZowYUJA30mTJik3N1eDBg3SlClTFBERoTFjxmjz5s1nPWYwLOUDAACcp5555hklJSXJ6/WquLjY79nMmTPVuHFjvfrqq4qLi5MktW7dWg8//LA2bNigtLQ0SdK2bdu0cuVKTZ48WdnZ2ZKkIUOGaODAgcrJydHChQtDHrMqVEwBAABMC8NS/rZt2/TWW29p8uTJAc+OHTum9evXa8iQIb4AKUmDBw9WbGysVq1a5WvLz89XZGSkMjMzfW3R0dHKyMjQpk2bVFRUFPKYVSGYAgAAmHaOl/K9Xq+eeOIJDRkyRJ06dQp4vnPnTp04cUJJSUl+7VFRUerUqZOcTqevzel0ql27dn5hU5KSk5Pl9Xp9fUMZsyos5QMAAJhWzQNMxcXFAUvxkhQfH6/4+PiA9uXLl2vXrl2aO3du0PFcLpckyeFwBDxzOBzasmWLX98WLVoE7SfJVzENZcyqEEwBAABMq2Ywzc3N1Zw5cwLax48fr3vuucev7dixY3rmmWd05513KiEhIeh4ZWVlkk5VM38qOjra97yyb2RkZNB+kuR2u0MesyoEUwAAAJsbNWqUhg4dGtAerFr63HPPKTIyUrfffnuV48XExEg6dQ3UT7ndbt/zyr7l5eVB+0n/DaihjFkVgikAAIBpXm+1Pl7Vkv1PFRUVKTc3V/fee68OHjzoa3e73SovL9e3336rhg0b+pbbK5fff8zlcvlVWh0Oh2+5/qf9JPn6hjJmVTj8BAAAYNo5OpV/6NAhlZeXKycnR3379vX9bN26Vbt371bfvn01f/58dejQQfXq1dP27dv9Pu/xeOR0Ov0OTHXs2FF79uxRSUmJX9+tW7f6nksKacyqUDEFAAAwrZp7TK1q3bp10ANPs2bN0vHjx/XQQw+pbdu2atiwodLS0pSXl6e77rrLd+I+Ly9Px48fV3p6uu+z6enp+vOf/6wlS5b47jH1eDxaunSpunXr5jsYFcqYVSGYAgAAnCcaNmyofv36BbTn5uaqbt26fs8mTJig4cOHKysrS5mZmTpw4IAWLFigPn36qGfPnr5+KSkpSk9PV05Ojlwul9q0aaNly5apoKBA06ZN8/seq2NWhaV8AAAA08L0StLT6dy5sxYsWKCoqChNmzZNS5Ys0bBhwzR79uyAvtOnT1dWVpby8vI0depUnThxQi+88IK6d+9+1mMGU8frreZu3Brk/nxtuKcAoBaISx0Z7ikAqCVOeL4L9xQkSaWvBL59KRT1R047c6fzAEv5AAAAptmnDmhrBFMAAADTztHhp9qOPaYAAACwBSqmAAAAplExtYRgCgAAYJqhk/XnG4IpAACAYd4KDj9ZQTAFAAAwjaV8Szj8BAAAAFugYgoAAGAae0wtIZgCAACYxh5TSwimAAAAprHH1BKCKYwqdXt0831P6LvCQxre/1o9NOZW37N5r7+tP73xt6CfmzhyqLKH/NKv7cW/5sv51T7t+Gqvvis8pIsdTZX//NSgn/d6vVryzod6852PtOe7QkXVq6cuHdrpd7f+SimJ7WruDwRwTvzvg+OVmtpF3VK7qH37S/X11/v0sw49AvpFR0drxG9u0a9+1U/JXa5QixbNtX9/kTb+a7OmPvl/+uKLXWGYPQCrCKYwau6iFfr+yLHT9nng9gw1iY/za7visjYB/Z5d+JYaNYhTp/aX6GhJ6WnHnPrC61qy+kP9POlyTcgaojJ3ud589yONfuT/9KdHx+vnSR1C/2MAhM2TUyfr0KHvtXnzZ2rcOL7Kfm3bttbzf5qhjz76WAtefl0FBQfUvv2luuvOLA0d0l+/GjhCf39//TmcOfAfVEwtIZjCmB2792rh2+9pwsihynn5r1X2+8XVKWqV0OyM4/1t3uNqfVFzSdLQe59QaZk7aL8v9uzTktUfqlfqFZr38DjVqVNHkpRxY28NvudxPf7cX5T3x0cVEcGlFEBtcXlimvbs2StJ2rJ5rRrExQXt53IdVvef36CtWz/3a//LoqX6ZONqPfXUw+qRNsD4fIEAXvaYWhFSMD148KCcTqeKiopUVlammJgYJSQkqGPHjnI4HKbmiFro5MkK/eG5heqVeoX69uh62mAqSceOlyomOkr16tatsk9lKD2Tf23/UpI06PoevlAqSfFxsbr+58l6892PtPmLr9T9ip9ZGg9A+FWG0jM5fPh7HT78fUC70/lvbf98p5I6J9b01ABrqJhaYimYbt26VTk5Odq0aZO8Xq+8P0n9derUUffu3XX//fera9euRiaK2uXVFWu157tCzXzwzjP2zZjwpEpKy1Q3IkJJl1+qOzMH6Jpunc/6uz3lJyRJMVFRAc9iok+1bftyD8EUuIDUqVNHLS9KUGHhwXBPBRcqTuVbcsZgumHDBo0ZM0YXX3yx7rvvPnXp0kUJCQmKioqSx+NRUVGRtm7dqmXLlikrK0vz589Xjx6BG9Jx4fi28KDmLV6puzMHqFVCM31XdChov4Zxscr4ZW+ldGyv+LhYfV1QqIVvr9P4J+fp8XEjNPgXaWf1/Zdd0lKStPGznbr+qmRfu9fr1Sef/1uSVHgwsKIC4Px1150jdfHFF2nqk/8X7qkAOI0zBtNZs2apS5cuys3NVVSQCtRll12mtLQ0jR49WiNHjtTMmTP1xhtvGJksaoepzy9S6xbNlTWo72n7Zd30i4C2oX3TdPO9UzV9wZv6ZVqqYuvHhPz9vbt11mWXtNTi1R/I0bSR+vXoqjK3R6+sWKdd+woknbotAMCFIa3HlcqZ8ai2bP1c0576Y7ingwsVF+xbcsbTH1988YVuvvnmoKH0x6KionTzzTdr586dNTY51D5vv/+xNmz9Qg/fNVyR9areL1qVxg0bKPPGa3S0pFRbdn51VnOoV7eu5j08Tqkd22vWq8s1cNxjypj4//TFV/t034ghkqQGsaEHXgC1T7fULnorL1cFBYUaNHik3O7ghyYB4yq81fu5QJyxYhofH6+9e61tOt+7d6/i46u+xgPnN095uWYs+Kuu6dZZzRvHa+/+IklS0eEfJJ064LR3f5EaxzdQfFxsleNc/J8T+t8Xl5z1XFo6muqlxydov+uwvis6pMYN4/SzNhfr9VXvS5LatbrorMcGUDukdk1S/qpFOnLkqPrdkKmCggPhnhIuYF4OP1lyxmA6aNAgvfzyy0pISFBGRobq168f0Ke0tFRLlixRbm6uRo4caWSisL8yT7m+Lz6mDzZt1webtgc8f/v9jXr7/Y1BL8//scpA26xxw2rPqaWjqVo6mvp+/+jTzxURUUc9u3aq9tgA7Cu1a5JW57+uo0dL1O+GTO3d+124p4QL3QVU9ayOMwbTe++9V/v379eTTz6p6dOnq3379nI4HL7DTy6XS1999ZXKy8uVnp6ue++991zMGzZUPzpaOff/NqD9++JjevKF19Ur9QoN7dtTHdq20omTJ1Va5lHDOP9/0Dlw8LDeyP9QjRvGqWti+xqd33sbt+mDTds16PoevqosgPNP166dlb/qdR07dlz9bsjU11/vC/eUAFh0xmAaFRWlmTNnKjs7W/n5+friiy9UWFjou8fU4XCoV69eSk9PV3Jy8pmGw3kssl5d3dCzW0B75an8Sy5y+J4XlxxX/7sf0S+uSlG71hcpvkGsvv6uUMvWrNfxMreenjjad7VTpRV//1j7XYclnQq75SdO6IUlqySdqozedN3Vvr6/n/uqvF4psV1rRUdFarNzt/72wb+U9LNL9b+jM438/QDM+c1vbtGlbVpLkhzNmykqKlIPTT5VCPlm77dauPDUXclt2rRS/t9eV5MmjTRn7ktKS7tSaWlX+o21fPkqHT9++rfHATWOw0+WWL5gPzk5meCJGhMTFal+PVL12b+/1rqNW1Va5lbjhg10dXJH3T70l+pyeduAzyxbu9533VOlOYtWSJKu7Hy5XzBN+llbvfnuR1rzz80qP3FSl1zk0NjhA5V10y8CAi8A+xudPVzXXtvTr+3xPzwoSXr//fW+YNqubRs1b35q+87vH70/6FiXXX61vvnmW4OzBYJgKd+SOt6f3pYfRu7P14Z7CgBqgbhU9rIDsOaExx77i0seu61an497bFENzcTeeFk4AAAAbMHyUj4AAADOEkv5lhBMAQAATOPwkyUEUwAAANOomFpCMAUAADCMNz9Zw+EnAAAA2AIVUwAAANNYyreEYAoAAGAawdQSgikAAIBpnMq3hGAKAABgGhVTSzj8BAAAAFugYgoAAGCYl4qpJQRTAAAA0wimlhBMAQAATOOCfUvYYwoAAABboGIKAABgGkv5lhBMAQAATCOYWkIwBQAAMMzrJZhaQTAFAAAwjYqpJRx+AgAAgC1QMQUAADCNiqklBFMAAADDePOTNQRTAAAA0wimlhBMAQAATOPFT5Zw+AkAAAC2QMUUAADAMPaYWkMwBQAAMI1gagnBFAAAwDT2mFrCHlMAAADYAsEUAADAMG+Ft1o/ofjss880btw4XX/99UpOTlavXr10xx136NNPPw3o++mnn+q2225TSkqKevXqpalTp6q0tDSgn8fj0YwZM9S7d28lJydr2LBh2rBhQ9DvtzpmMARTAAAA0yqq+ROCffv26eTJk8rMzNQjjzyiO+64Q4cPH9aIESP0j3/8w9fP6XQqOztbbrdbkyZNUkZGhhYvXqwJEyYEjDlp0iTl5uZq0KBBmjJliiIiIjRmzBht3rzZr18oYwZTx+v12mY3rvvzteGeAoBaIC51ZLinAKCWOOH5LtxTkCQdHnpttT7fdNn71fp8aWmp+vXrp6SkJD3//POSpDFjxmjnzp1atWqV4uLiJElLlizRww8/rJdffllpaWmSpG3btikzM1OTJ09Wdna2JMntdmvgwIFKSEjQwoULfd9jdcyqUDEFAAAw7RxWTIOpX7++mjZtquLiYknSsWPHtH79eg0ZMsQXICVp8ODBio2N1apVq3xt+fn5ioyMVGZmpq8tOjpaGRkZ2rRpk4qKikIesyoEUwAAgPPQsWPHdPjwYX311VeaOXOmvvzyS1/FcufOnTpx4oSSkpL8PhMVFaVOnTrJ6XT62pxOp9q1a+cXNiUpOTlZXq/X1zeUMavCdVEAAACGeatZ9SwuLvZVO38sPj5e8fHxQT/z0EMPafXq1ZKkyMhIDR8+XHfffbckyeVySZIcDkfA5xwOh7Zs2eL73eVyqUWLFkH7SfJVTEMZsyoEUwAAANOqGUxzc3M1Z86cgPbx48frnnvuCfqZcePG6dZbb9WBAweUl5cnj8ej8vJyRUVFqaysTNKpauZPRUdH+55LUllZmSIjI4P2k07tN63sZ3XMqhBMAQAADKtuxXTUqFEaOnRoQHtV1VJJSkxMVGJioiRp0KBBuuWWWzR58mQ9++yziomJkXTqGqifcrvdvueSFBMTo/Ly8qD9pP8G1FDGrArBFAAAwLRqBtPTLdlbERkZqb59++q5555TWVmZb7m9cvn9x1wulxISEny/OxwO33L9T/tJ8vUNZcyqcPgJAADgAlBWViav16uSkhJ16NBB9erV0/bt2/36eDweOZ1OderUydfWsWNH7dmzRyUlJX59t27d6nsuKaQxq0IwBQAAMMxbUb2fUBw+fDig7dixY1q9erVatmypZs2aqWHDhkpLS1NeXp5f4MzLy9Px48eVnp7ua0tPT1d5ebmWLFnia/N4PFq6dKm6devmOxgVyphVYSkfAADAsOruMQ3Ffffdp+joaKWmpsrhcGj//v1aunSpDhw4oJkzZ/r6TZgwQcOHD1dWVpYyMzN14MABLViwQH369FHPnj19/VJSUpSenq6cnBy5XC61adNGy5YtU0FBgaZNm+b33VbHrApvfgJQ6/DmJwBW2eXNT4XXV+/NTy3es/7mpzfffFN5eXnatWuXiouL1bBhQ3Xt2lWjR4/WVVdd5df3k08+UU5Ojnbs2KEGDRpowIABmjhxomJjY/36ud1uzZo1SytWrNCRI0eUmJioiRMnBg2bVscMhmAKoNYhmAKw6kIMprUZS/kAAACmeeuEewa1AsEUAADAsHO5x7Q2I5gCAAAY5q2gYmoFwRQAAMAwKqbWcI8pAAAAbIGKKQAAgGFeDj9ZQjAFAAAwjKV8awimAAAAhnH4yRr2mAIAAMAWqJgCAAAYZp/3bNobwRQAAMAwlvKtIZgCAAAYRjC1hmAKAABgGEv51nD4CQAAALZAxRQAAMAwlvKtIZgCAAAYxpufrCGYAgAAGMabn6whmAIAABhWQcXUEg4/AQAAwBaomAIAABjGHlNrCKYAAACGcSrfGoIpAACAYVywbw17TAEAAGALVEwBAAAMYynfGoIpAACAYVwXZQ3BFAAAwDBO5VtDMAUAADCMw0/WcPgJAAAAtkDFFAAAwDD2mFpDMAUAADCMPabWEEwBAAAMY4+pNQRTAAAAw1jKt8ZWwTQudWS4pwCgFigt+DDcUwAAGGCrYAoAAHA+Yo+pNQRTAAAAw1jKt4ZgCgAAYBhnn6zhgn0AAADYAhVTAAAAw1jKt4ZgCgAAYBiHn6whmAIAABhWEe4J1BIEUwAAAMO8omJqBYefAAAAYAtUTAEAAAyr4L4oSwimAAAAhlWwlG8JwRQAAMAw9phaQzAFAAAwjFP51nD4CQAAALZAxRQAAMAwlvKtIZgCAAAYxlK+NQRTAAAAwwim1rDHFAAAALZAxRQAAMAw9phaQzAFAAAwrIJcagnBFAAAwDDe/GQNe0wBAAAM81bzJxTbtm3TH/7wBw0YMEBdu3bVddddpwkTJuibb74J6Pvpp5/qtttuU0pKinr16qWpU6eqtLQ0oJ/H49GMGTPUu3dvJScna9iwYdqwYUPQ77c6ZjAEUwAAgPPIiy++qHfffVc9e/bUlClTNGzYMG3cuFFDhgzR7t27ff2cTqeys7Pldrs1adIkZWRkaPHixZowYULAmJMmTVJubq4GDRqkKVOmKCIiQmPGjNHmzZv9+oUyZjAs5QMAABh2Lq+Lys7OVk5OjqKionxtAwYM0E033aT58+frqaeekiTNnDlTjRs31quvvqq4uDhJUuvWrfXwww9rw4YNSktLk3SqArty5UpNnjxZ2dnZkqQhQ4Zo4MCBysnJ0cKFC33fY3XMqlAxBQAAMKyiTp1q/YSiW7dufqFUktq2bavLL7/cVzE9duyY1q9fryFDhvgCpCQNHjxYsbGxWrVqla8tPz9fkZGRyszM9LVFR0crIyNDmzZtUlFRUchjVoVgCgAAYNi53GMa9Pu9Xh08eFBNmjSRJO3cuVMnTpxQUlKSX7+oqCh16tRJTqfT1+Z0OtWuXTu/sClJycnJ8nq9vr6hjFkVlvIBAAAMq+5SfnFxsYqLiwPa4+PjFR8ff8bPv/XWWyosLPTt9XS5XJIkh8MR0NfhcGjLli2+310ul1q0aBG0nyRfxTSUMatCMAUAALC53NxczZkzJ6B9/Pjxuueee0772d27d+vxxx9X9+7dNXjwYElSWVmZJAUs+Uunlukrn1f2jYyMDNpPktxud8hjVoVgCgAAYFh1L9gfNWqUhg4dGtB+pmqpy+XSXXfdpUaNGmn27NmKiDi1izMmJkbSqWugfsrtdvueV/YtLy8P2k/6b0ANZcyqEEwBAAAMq+4F+1aX7H/s6NGjGjNmjI4ePapFixb5LbFX/ufK5fcfc7lcSkhI8OtbuVz/036SfH1DGbMqHH4CAAAw7FwffnK73br77rv19ddf6/nnn1f79u39nnfo0EH16tXT9u3b/do9Ho+cTqc6derka+vYsaP27NmjkpISv75bt271PQ91zKoQTAEAAM4jJ0+e1H333actW7Zo9uzZ6tq1a0Cfhg0bKi0tTXl5eX6BMy8vT8ePH1d6erqvLT09XeXl5VqyZImvzePxaOnSperWrZvvYFQoY1aFpXwAAADDqrvHNBRPPfWU1q1bp+uvv14//PCD8vLyfM/i4uLUr18/SdKECRM0fPhwZWVlKTMzUwcOHNCCBQvUp08f9ezZ0/eZlJQUpaenKycnRy6XS23atNGyZctUUFCgadOm+X231TGrUsfr9dbE9Vg1ol5Uq3BPAUAtUFrwYbinAKCWiGze/sydzoGXW42o1uezv3vNct+srCxt3Lgx6LNWrVpp3bp1vt8/+eQT5eTkaMeOHWrQoIEGDBigiRMnKjY21u9zbrdbs2bN0ooVK3TkyBElJiZq4sSJQcOm1TGDIZgCqHUIpgCsskswXVDNYHp7CMG0NmMpHwAAwLBzuZRfm3H4CQAAALZAxRQAAMCw6r6S9EJBMAUAADCMYGoNwRQAAMAwL3tMLSGYAgAAGEbF1BoOPwEAAMAWqJgCAAAYRsXUGoIpAACAYbZ5m5HNEUwBAAAM44J9a9hjCgAAAFugYgoAAGAYe0ytIZgCAAAYRjC1hmAKAABgGIefrCGYAgAAGMbhJ2s4/AQAAABboGIKAABgGHtMrSGYAgAAGMYeU2sIpgAAAIZVEE0tYY8pAAAAbIGKKQAAgGHsMbWGYAoAAGAYC/nWEEwBAAAMo2JqDcEUAADAMC7Yt4bDTwAAALAFKqYAAACGcV2UNQRTAAAAw4il1hBMAQAADOPwkzUEUwAAAMNYyreGw08AAACwBSqmAAAAhlEvtYZgCgAAYBh7TK0hmAIAABjGHlNr2GMKAAAAW6BiCgAAYBj1UmsIpgAAAIaxx9QagikAAIBhXmqmlhBMAQAADKNiag3BFLby6CMT9egj/1Pl8/LyctWPa3vuJgTgnCktK9PQrN/p24IDuu3mmzTlf8b6Pd+y3akXX1ks55e79MORo3I0b6qru3fVb7OG6ZJWLf36ejwevZD7ulasXqeig4fUwtFcQ391g0aPyFRkvf/+X1+h66DeWrVW//j4E3297zuVlBzXxRe10DVpP9dvs4apcaP4c/K3AziFYApbWbZ8lXbt/jqgPblLJ93/P2P19sp3z/2kAJwTc158VYe/PxL02Uf//ERjH/i9LmnVUrfdMkiNG8Vr955v9OZbq/Tu3z/SslefUwtHc1//+x99Sus+3KChA29QSudO2vq5U3+c/4r2flugJx/+7z/8/v2jjzXvz6+pT9pVuv3XaYqLra/Pdnyp195YrlVr39fiF2erebOmxv92nP+4Lsoagils5bPPnPrsM2dAe++5T0uSFix4/VxPCcA5sGPnLr32xnJNHHuHZvxxfsDzVxYvU0REhF770zNq0riRr/2ydpfqsadn6511Hyrr1qGSpA/Wb9S6Dzdo1PCb9cA9YyRJGYPSFd+ggXJfX6qMwf2V2uUKSVL3lCS9+9dcv/CZMai/ulyRqMeenq0Fi/6qB8aPMfmn4wJBLLWGe0xhe7Gx9XXrsEHat69A+avfC/d0ANSwkydP6rGnZ6vX1Veq37W9gvYpKTmu6KhIxTds4Nee0PxUoKxfP8bXtvLdv0uSRgwb4te38ve3V6/ztf2s/aVBK6L9+/aRJO366psQ/xoguAp5q/VzoajxYLpw4UL17du3pofFBSzjlpvUqFG8Xnn1DVVUsH0cON+8sniZ9nyzT1Mmjq2yT8+ru6vkeKkemvqMvvj3Vyp0HdQ/Pt6kGXNeVPu2l6h/32t9fT93fqkWjmZq2cLhN0bLFg4lNG+m7c4vzzinA66DkqRmTRqf5V8F+Kuo5s+FosaX8ouLi1VQUFDTw+ICNvr24aqoqNCCl1nGB8433xYc0LyXXtPdt/9arVq20Hf7C4P2G5M1TIe//0HLVr6jle/8d+XkmrSfa8Yf/ldxcbG+tqKDh3VZuzZBx0lwNFNh0cEzzmveS69JkgYP6BfKnwOgmiwF03/961+WB/z222/PejLAT3XocJl6975aa9d+qK+/3hfu6QCoYY/P+KNaX9xSI4fffNp+ERF1leBopqcDd3EAAAypSURBVB5Xpqpvn55qFN9Amz/bob+8+ZYeePQpPfv0732n7cvcbkVFRgYdJzoqSmVu92m/6+VFf9XqdR8qc3B/Xd2969n9YcBPcI+pNZaCaVZWlurUqWNpQK/Xa7kvcCajbx8uSXppwV/CPBMANW3F6nXa8K/Nyp073e8Kp2CmPPmMtm53avlrf1JMdLQkqd+1vdSm1cV6ImeO8v62RhmD0iVJMdHR8pSXBx3H7fH4Ph/Mm2/l65m5L6lPz6v00Gm2FgChupCW46vDUjCNjY1Vx44dNXr06DP2zc/P18qVK6s9MaBu3boa8ZsMHTx4WMuX54d7OgBqkMfj0YxnX9A1aT9X82ZNtffbU1vACv+zt/NoSYn2flugxo3iVVJyXCvfeU+/vuWmgFB54y+u0RM5c/TJls98wTSheVMVuYIv1xe5DinB0Szos6Vvr9Yfpj+rnld106wnp5wxLAOhoGJqjaX/1SUlJamwsFD9+p15r82///3vak8KkKSBA3+piy5K0OxnX5TH4wn3dADUoDK3R4d/OKIP1m/UB+s3Bjx/e/U6vb16nf5n3B1KTe4sSToZ5PDjiZMnTz37z79LUudOHbTynfe0v9DldwBqf6FLRQcP6breVweMs/Tt1fr9U7PV48quenbao4qKiqr23wggdJaCaXJysl566SUdOXJEjRo1Om1fr9crr5d/KkD1jc6+TZK04OVFYZ4JgJpWv36MZk59KKD98A9HNDVnrnr3uFI3D7xBHS5rpyaNG6lu3Qit+2CD7r0r2+/KqLy/nXrpRlKnDr62Ab+8TivfeU+vvbHcd4+pJL32xnJJ0sAbrvf7zuUr39VjTz+rq7un6I9P/17R0YRS1DyW8q2xFExHjRqlPn36KLKKzeQ/NnbsWI0dy74cVE/Lli10443XaePGT7V9+xfhng6AGhZZr55uuP6agPbKU/mXXNzS7/mIYUOUu2ipMm8fr1tuSlej+Iba/NkOrXznPV3SqqVuuelGX99re16la3tdpdzXl+rosRKlJHXS1u1OLX17tQbe+At1S0ny9X3vw3/q0admqUFcrNL79tG7f/+H33xi68eob5+eNf3n4wJUQdHOEkvB1OFwyOFwnLkjUENGjRymevXq6aU/Uy0FIN0/7rdq16a1/rpitea/uljlnnIlOJrp1qG/0tjRv1GDuDi//jOfmKLncxfp7dXrtGL1WrVwNNf432bpjqxhfv12fLlLFRUVKj56TI89/WzA9158UQLBFDWCWGpNHa+N1t3rRbUK9xQA1AKlBR+GewoAaonI5u3DPQVJ0q8vHVqtz//lm2U1NBN745WkAAAAsAXuwgAAADCM66KsoWIKAABgWEU1f0JRVFSknJwcZWVlKTU1VYmJifr444+D9l27dq2GDh2qLl266LrrrtOcOXN04sSJgH7FxcV65JFH1KNHD3Xt2lUjR46U0+ms1pjBEEwBAAAMq5C3Wj+h2LNnj+bPn6/CwkIlJiZW2e/999/XuHHj1KhRIz3yyCPq16+f5s6dq2nTpvnPvaJCd955p1auXKkRI0bogQce0KFDh5SVlaW9e/ee1ZhVYSkfAADAsHO5lN+5c2f985//VJMmTbRmzRqNGzcuaL/p06friiuu0EsvvaS6detKkuLi4vTCCy8oKytLbdu2lXTqrZ6bN2/W3LlzfS9b6t+/v2688UbNmTNH06dPD3nMqlAxBQAAOI80aNBATZo0OW2fXbt2adeuXbr11lt9AVKSfv3rX6uiokLvvPOOr2316tVKSEhQ3759fW1NmzZV//79tWbNGpWXl4c8ZlUIpgAAAIadyz2mVuzYsUPSqdfO/1iLFi100UUX+Z5LktPpVOfOnVWnTh2/vl26dFFJSYlvOT+UMavCUj4AAIBh1b02vri4WMXFxQHt8fHxio+PD3k8l8slSUFfoORwOFRUVOTXt0ePHgH9EhISJJ06bHXZZZeFNGZVCKYAAACGhXqA6adyc3M1Z86cgPbx48frnnvuCXm8srIySVJUVFTAs+joaJWWlvr1Ddavsq1yrFDGrArBFAAAwOZGjRqloUMD3x51NtVSSYqJiZEkeTyegGdut9v3vLJvsH6VbZV9QxmzKgRTAAAAw6q7T/Rsl+yrUrnc7nK5fEvylVwul1JTU/36BluGr2yr/HwoY1aFw08AAACGeav5r5rWqVMnSdL27dv92gsLC3XgwAHfc0nq2LGjPv/884B9stu2bVNsbKzatGkT8phVIZgCAAAYdi4v2Lfi8ssvV/v27bV48WKdPHnS175o0SJFRETohhtu8LWlp6erqKhIa9eu9bUdPnxY+fn56tu3ryIjI0Mesyos5QMAABhW3VP5oZo3b54kaffu3ZKkvLw8bdq0SfHx8RoxYoQk6cEHH9Tvfvc73XHHHRowYIC+/PJLLVy4ULfeeqvatWvnG+vGG29U165d9eCDD2r06NFq0qSJFi1apIqKioCDV1bHrEod77n+b+o06kW1CvcUANQCpQUfhnsKAGqJyObtwz0FSVL/S/pX6/Or9q0KqX9VryJt1aqV1q1b5/t9zZo1mjNnjnbv3q2mTZvqlltu0dixY1Wvnn/t8siRI5o+fbrWrFkjt9utLl26aNKkSercuXPAd1gdMxiCKYBah2AKwCq7BNMbqxlMV4cYTGsrlvIBAAAMM3GA6XxEMAUAADDMxAGm8xHBFAAAwDAb7Zy0Na6LAgAAgC1QMQUAADCMpXxrCKYAAACGcfjJGoIpAACAYRXsMbWEPaYAAACwBSqmAAAAhlEvtYZgCgAAYBiHn6whmAIAABhGMLWGYAoAAGAYF+xbw+EnAAAA2AIVUwAAAMNYyreGYAoAAGAYF+xbQzAFAAAwjD2m1hBMAQAADGMp3xoOPwEAAMAWqJgCAAAYxlK+NQRTAAAAw1jKt4ZgCgAAYBin8q1hjykAAABsgYopAACAYRXsMbWEYAoAAGAYS/nWEEwBAAAMo2JqDcEUAADAMCqm1nD4CQAAALZAxRQAAMAwlvKtIZgCAAAYxlK+NQRTAAAAw6iYWkMwBQAAMIyKqTUcfgIAAIAtUDEFAAAwzOutCPcUagWCKQAAgGEVLOVbQjAFAAAwzMvhJ0vYYwoAAABboGIKAABgGEv51hBMAQAADGMp3xqCKQAAgGFcsG8NwRQAAMAwLti3hsNPAAAAsAUqpgAAAIaxx9QagikAAIBhnMq3hmAKAABgGBVTa9hjCgAAAFugYgoAAGAY10VZQzAFAAAwjKV8awimAAAAhnH4yRqCKQAAgGFUTK3h8BMAAABsgYopAACAYRx+soZgCgAAYJiXPaaWEEwBAAAMo2JqDcEUAADAMA4/WcPhJwAAANgCFVMAAADD2GNqDcEUAADAMJbyrSGYAgAAGEYwtYY9pgAAALCFOl4iPAAAAGyAiikAAABsgWAKAAAAWyCYAgAAwBYIpgAAALAFgikAAABsgWAKAAAAWyCYAgAAwBYIpgAAALAFgikAAABsgWAKAAAAWyCYwnY8Ho9mzJih3r17Kzk5WcOGDdOGDRvCPS0ANlRUVKScnBxlZWUpNTVViYmJ+vjjj8M9LQBniWAK25k0aZJyc3M1aNAgTZkyRRERERozZow2b94c7qkBsJk9e/Zo/vz5KiwsVGJiYrinA6Ca6ni9Xm+4JwFU2rZtmzIzMzV58mRlZ2dLktxutwYOHKiEhAQtXLgwvBMEYCvHjh1TeXm5mjRpojVr1mjcuHF65ZVXdPXVV4d7agDOAhVT2Ep+fr4iIyOVmZnpa4uOjlZGRoY2bdqkoqKiMM4OgN00aNBATZo0Cfc0ANQQgilsxel0ql27doqLi/NrT05OltfrldPpDNPMAACAaQRT2IrL5VJCQkJAu8PhkCQqpgAAnMcIprCVsrIyRUZGBrRHR0dLOrXfFAAAnJ8IprCVmJgYlZeXB7RXBtLKgAoAAM4/BFPYisPhCLpc73K5JCnoMj8AADg/EExhKx07dtSePXtUUlLi175161bfcwAAcH4imMJW0tPTVV5eriVLlvjaPB6Pli5dqm7duqlFixZhnB0AADCpXrgnAPxYSkqK0tPTlZOTI5fLpTZt2mjZsmUqKCjQtGnTwj09ADY0b948SdLu3bslSXl5edq0aZPi4+M1YsSIcE4NQIh48xNsx+12a9asWVqxYoWOHDmixMRETZw4UT179gz31ADYUFWvIm3VqpXWrVt3jmcDoDoIpgAAALAF9pgCAADAFgimAAAAsAWCKQAAAGyBYAoAAABbIJgCAADAFgimAAAAsAWCKQAAAGyBYAoAAABbIJgCAADAFgimAAAAsIX/D8GMuSk2MxWVAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8eSz1FqVpYEI",
        "colab_type": "code",
        "outputId": "318b7e69-e18d-4bc4-e1ec-ccf121d35e7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "print (metrics.roc_auc_score(flat_predictions, flat_true_labels))\n",
        "print(metrics.classification_report(flat_predictions, flat_true_labels))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9979479869712031\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      1.00      1.00      4531\n",
            "           1       1.00      1.00      1.00      4809\n",
            "\n",
            "    accuracy                           1.00      9340\n",
            "   macro avg       1.00      1.00      1.00      9340\n",
            "weighted avg       1.00      1.00      1.00      9340\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}