Outliers_Detection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Outliers Detection.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/roshank1605A04/SECOM-Detecting-Defected-Items/blob/master/Outliers_Detection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "-prZHuYiOhs1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        },
        "outputId": "82a7db72-82ea-490a-83c1-b444f1b8cec1"
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "e4gtsF8XOhmU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np \n",
        "import pandas as pd \n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "N_GjUr1iPIT0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "93554f1f-47b2-40a4-ddef-21d41282cb18"
      },
      "cell_type": "code",
      "source": [
        "!ls 'drive/My Drive/Projects/practice/SECOM'"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "'Outliers Detection.ipynb'   secom.csv\t SECOM.ipynb\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "0wLy4PTKOgyA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c8b9ba92-e4d6-477f-bf8b-09bc1f073ccb"
      },
      "cell_type": "code",
      "source": [
        "# reading the data\n",
        "data = pd.read_csv('drive/My Drive/Projects/practice/SECOM/secom.csv')\n",
        "\n",
        "# getting the shape of the data\n",
        "print(data.shape)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1567, 592)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MtN9TaafQJVr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 420
        },
        "outputId": "d67d0f74-cbb4-4b5a-8bef-525dfffc0a63"
      },
      "cell_type": "code",
      "source": [
        "data.head()"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Time</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>...</th>\n",
              "      <th>581</th>\n",
              "      <th>582</th>\n",
              "      <th>583</th>\n",
              "      <th>584</th>\n",
              "      <th>585</th>\n",
              "      <th>586</th>\n",
              "      <th>587</th>\n",
              "      <th>588</th>\n",
              "      <th>589</th>\n",
              "      <th>Pass/Fail</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2008-07-19 11:55:00</td>\n",
              "      <td>3030.93</td>\n",
              "      <td>2564.00</td>\n",
              "      <td>2187.7333</td>\n",
              "      <td>1411.1265</td>\n",
              "      <td>1.3602</td>\n",
              "      <td>100.0</td>\n",
              "      <td>97.6133</td>\n",
              "      <td>0.1242</td>\n",
              "      <td>1.5005</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.5005</td>\n",
              "      <td>0.0118</td>\n",
              "      <td>0.0035</td>\n",
              "      <td>2.3630</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2008-07-19 12:32:00</td>\n",
              "      <td>3095.78</td>\n",
              "      <td>2465.14</td>\n",
              "      <td>2230.4222</td>\n",
              "      <td>1463.6606</td>\n",
              "      <td>0.8294</td>\n",
              "      <td>100.0</td>\n",
              "      <td>102.3433</td>\n",
              "      <td>0.1247</td>\n",
              "      <td>1.4966</td>\n",
              "      <td>...</td>\n",
              "      <td>208.2045</td>\n",
              "      <td>0.5019</td>\n",
              "      <td>0.0223</td>\n",
              "      <td>0.0055</td>\n",
              "      <td>4.4447</td>\n",
              "      <td>0.0096</td>\n",
              "      <td>0.0201</td>\n",
              "      <td>0.0060</td>\n",
              "      <td>208.2045</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2008-07-19 13:17:00</td>\n",
              "      <td>2932.61</td>\n",
              "      <td>2559.94</td>\n",
              "      <td>2186.4111</td>\n",
              "      <td>1698.0172</td>\n",
              "      <td>1.5102</td>\n",
              "      <td>100.0</td>\n",
              "      <td>95.4878</td>\n",
              "      <td>0.1241</td>\n",
              "      <td>1.4436</td>\n",
              "      <td>...</td>\n",
              "      <td>82.8602</td>\n",
              "      <td>0.4958</td>\n",
              "      <td>0.0157</td>\n",
              "      <td>0.0039</td>\n",
              "      <td>3.1745</td>\n",
              "      <td>0.0584</td>\n",
              "      <td>0.0484</td>\n",
              "      <td>0.0148</td>\n",
              "      <td>82.8602</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2008-07-19 14:43:00</td>\n",
              "      <td>2988.72</td>\n",
              "      <td>2479.90</td>\n",
              "      <td>2199.0333</td>\n",
              "      <td>909.7926</td>\n",
              "      <td>1.3204</td>\n",
              "      <td>100.0</td>\n",
              "      <td>104.2367</td>\n",
              "      <td>0.1217</td>\n",
              "      <td>1.4882</td>\n",
              "      <td>...</td>\n",
              "      <td>73.8432</td>\n",
              "      <td>0.4990</td>\n",
              "      <td>0.0103</td>\n",
              "      <td>0.0025</td>\n",
              "      <td>2.0544</td>\n",
              "      <td>0.0202</td>\n",
              "      <td>0.0149</td>\n",
              "      <td>0.0044</td>\n",
              "      <td>73.8432</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2008-07-19 15:22:00</td>\n",
              "      <td>3032.24</td>\n",
              "      <td>2502.87</td>\n",
              "      <td>2233.3667</td>\n",
              "      <td>1326.5200</td>\n",
              "      <td>1.5334</td>\n",
              "      <td>100.0</td>\n",
              "      <td>100.3967</td>\n",
              "      <td>0.1235</td>\n",
              "      <td>1.5031</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.4800</td>\n",
              "      <td>0.4766</td>\n",
              "      <td>0.1045</td>\n",
              "      <td>99.3032</td>\n",
              "      <td>0.0202</td>\n",
              "      <td>0.0149</td>\n",
              "      <td>0.0044</td>\n",
              "      <td>73.8432</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 592 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                  Time        0        1          2          3       4      5  \\\n",
              "0  2008-07-19 11:55:00  3030.93  2564.00  2187.7333  1411.1265  1.3602  100.0   \n",
              "1  2008-07-19 12:32:00  3095.78  2465.14  2230.4222  1463.6606  0.8294  100.0   \n",
              "2  2008-07-19 13:17:00  2932.61  2559.94  2186.4111  1698.0172  1.5102  100.0   \n",
              "3  2008-07-19 14:43:00  2988.72  2479.90  2199.0333   909.7926  1.3204  100.0   \n",
              "4  2008-07-19 15:22:00  3032.24  2502.87  2233.3667  1326.5200  1.5334  100.0   \n",
              "\n",
              "          6       7       8    ...           581     582     583     584  \\\n",
              "0   97.6133  0.1242  1.5005    ...           NaN  0.5005  0.0118  0.0035   \n",
              "1  102.3433  0.1247  1.4966    ...      208.2045  0.5019  0.0223  0.0055   \n",
              "2   95.4878  0.1241  1.4436    ...       82.8602  0.4958  0.0157  0.0039   \n",
              "3  104.2367  0.1217  1.4882    ...       73.8432  0.4990  0.0103  0.0025   \n",
              "4  100.3967  0.1235  1.5031    ...           NaN  0.4800  0.4766  0.1045   \n",
              "\n",
              "       585     586     587     588       589  Pass/Fail  \n",
              "0   2.3630     NaN     NaN     NaN       NaN         -1  \n",
              "1   4.4447  0.0096  0.0201  0.0060  208.2045         -1  \n",
              "2   3.1745  0.0584  0.0484  0.0148   82.8602          1  \n",
              "3   2.0544  0.0202  0.0149  0.0044   73.8432         -1  \n",
              "4  99.3032  0.0202  0.0149  0.0044   73.8432         -1  \n",
              "\n",
              "[5 rows x 592 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "metadata": {
        "id": "VAkNSLdlQcPF",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# droping the first column of the data\n",
        "\n",
        "data = data.drop(['Time'], axis = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kJP_piv_Qh4F",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9c66d60a-876e-4d52-d01f-5e1509bfdc35"
      },
      "cell_type": "code",
      "source": [
        "# filling missing values\n",
        "\n",
        "data.fillna(0, inplace = True)\n",
        "\n",
        "data.isnull().sum().sum()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "metadata": {
        "id": "6LSqmdUQObs9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "c51752f0-87fd-4666-9fd0-6f1f5d5678b8"
      },
      "cell_type": "code",
      "source": [
        "# again creating x and y from the dataset\n",
        "\n",
        "x = data.iloc[:, :-1]\n",
        "y = data.iloc[:, -1]\n",
        "\n",
        "# getting the shapes\n",
        "print(\"Shape of x:\", x.shape)\n",
        "print(\"Shape of y:\", y.shape)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of x: (1567, 590)\n",
            "Shape of y: (1567,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "x1PuviLxOZi2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "outputId": "541a285c-2095-4f3b-992a-3d07dccd5bdd"
      },
      "cell_type": "code",
      "source": [
        "# splittng the dataset\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)\n",
        "\n",
        "# print the shapes\n",
        "print(\"Shape of x_train :\", x_train.shape)\n",
        "print(\"Shape of x_test :\", x_test.shape)\n",
        "print(\"Shape of y_train :\", y_train.shape)\n",
        "print(\"Shape of y_test :\", y_test.shape)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of x_train : (1253, 590)\n",
            "Shape of x_test : (314, 590)\n",
            "Shape of y_train : (1253,)\n",
            "Shape of y_test : (314,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "CxEPhGXROZDG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4bf540bf-0a3d-4d59-9574-0e8476cd01b3"
      },
      "cell_type": "code",
      "source": [
        "# defining outlier fraction\n",
        "\n",
        "Fraud = data[data['Pass/Fail']==1]\n",
        "Valid = data[data['Pass/Fail']==-1]\n",
        "\n",
        "outlier_fraction = len(Fraud)/float(len(Valid))\n",
        "print(\"Outlier Fraction :\", outlier_fraction)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Outlier Fraction : 0.0710868079289132\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "a_mNc8fWOHnW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# importing the libraries\n",
        "from sklearn.ensemble import IsolationForest\n",
        "from sklearn.neighbors import LocalOutlierFactor\n",
        "from sklearn.svm import OneClassSVM\n",
        "\n",
        "classifiers = {\n",
        "    \"Isolation Forest\":IsolationForest(n_estimators=100, max_samples=len(x_train), \n",
        "                                       contamination=outlier_fraction,random_state=0, verbose=0),\n",
        "    \"Local Outlier Factor\":LocalOutlierFactor(n_neighbors=20, algorithm='auto', \n",
        "                                              leaf_size=30, metric='minkowski',\n",
        "                                              p=2, metric_params=None, contamination=outlier_fraction),\n",
        "    \"Support Vector Machine\":OneClassSVM(kernel='rbf', degree=3, gamma=0.1,nu=0.05, \n",
        "                                         max_iter=-1, random_state=0)\n",
        "   \n",
        "}\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FgkQYp_6OOM5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1074
        },
        "outputId": "e9f66246-4779-4ea9-e401-d86c6e1fc970"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "n_outliers = len(Fraud)\n",
        "for i, (clf_name,clf) in enumerate(classifiers.items()):\n",
        "    #Fit the data and tag outliers\n",
        "    if clf_name == \"Local Outlier Factor\":\n",
        "        y_pred = clf.fit_predict(x_test)\n",
        "        scores_prediction = clf.negative_outlier_factor_\n",
        "    elif clf_name == \"Support Vector Machine\":\n",
        "        clf.fit(x_train)\n",
        "        y_pred = clf.predict(x_test)\n",
        "    else:    \n",
        "        clf.fit(x_train)\n",
        "        scores_prediction = clf.decision_function(x_test)\n",
        "        y_pred = clf.predict(x_test)\n",
        "    #Reshape the prediction values to 0 for Valid transactions , 1 for Fraud transactions\n",
        "    y_pred[y_pred == 1] = 0\n",
        "    y_pred[y_pred == -1] = 1\n",
        "    n_errors = (y_pred != y_test).sum()\n",
        "    # Run Classification Metrics\n",
        "    print(\"{}: {}\".format(clf_name,n_errors))\n",
        "    print(\"Accuracy Score :\")\n",
        "    print(accuracy_score(y_test,y_pred))\n",
        "    print(\"Classification Report :\")\n",
        "    print(classification_report(y_test,y_pred))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/ensemble/iforest.py:223: FutureWarning: behaviour=\"old\" is deprecated and will be removed in version 0.22. Please use behaviour=\"new\", which makes the decision_function change to match other anomaly detection algorithm API.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/ensemble/iforest.py:417: DeprecationWarning: threshold_ attribute is deprecated in 0.20 and will be removed in 0.22.\n",
            "  \" be removed in 0.22.\", DeprecationWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
            "  'recall', 'true', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
            "  'recall', 'true', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/svm/classes.py:1177: DeprecationWarning: The random_state parameter is deprecated and will be removed in version 0.22.\n",
            "  \" be removed in version 0.22.\", DeprecationWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Isolation Forest: 312\n",
            "Accuracy Score :\n",
            "0.006369426751592357\n",
            "Classification Report :\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "          -1       0.00      0.00      0.00       301\n",
            "           0       0.00      0.00      0.00         0\n",
            "           1       0.11      0.15      0.12        13\n",
            "\n",
            "   micro avg       0.01      0.01      0.01       314\n",
            "   macro avg       0.04      0.05      0.04       314\n",
            "weighted avg       0.00      0.01      0.01       314\n",
            "\n",
            "Local Outlier Factor: 312\n",
            "Accuracy Score :\n",
            "0.006369426751592357\n",
            "Classification Report :\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "          -1       0.00      0.00      0.00       301\n",
            "           0       0.00      0.00      0.00         0\n",
            "           1       0.09      0.15      0.11        13\n",
            "\n",
            "   micro avg       0.01      0.01      0.01       314\n",
            "   macro avg       0.03      0.05      0.04       314\n",
            "weighted avg       0.00      0.01      0.00       314\n",
            "\n",
            "Support Vector Machine: 301\n",
            "Accuracy Score :\n",
            "0.041401273885350316\n",
            "Classification Report :\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "          -1       0.00      0.00      0.00       301\n",
            "           1       0.04      1.00      0.08        13\n",
            "\n",
            "   micro avg       0.04      0.04      0.04       314\n",
            "   macro avg       0.02      0.50      0.04       314\n",
            "weighted avg       0.00      0.04      0.00       314\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "4MRdoqfCQrb-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}