Elie_MulambaGDA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# GDA Implementation.\n",
        "\n",
        "Implement the Gaussian Discriminant Analysis (GDA) learning algorithm following the steps as discussed in class.\n",
        "\n",
        "INSTRUCTION: Rename your notebook as: <br>\n",
        "`firstName_LastName_Live_coding_GDA.ipynb`.\n",
        "\n",
        "Notes: \n",
        "* Do not use any built-in functions to complete a task;\n",
        "* Do not import additional libraries."
      ],
      "metadata": {
        "id": "g17Z46tmw2oZ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "aT5nlL-QTKwv"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.datasets import make_classification"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate data\n",
        "def generate_data():\n",
        "  x, y = make_classification(n_samples= 1000, n_features=3, n_redundant=0, \n",
        "                           n_informative=3, random_state=1, \n",
        "                           n_clusters_per_class=1)\n",
        "  \n",
        "  return x,y\n",
        "\n",
        "x,y= generate_data() # get data\n",
        "print(x.shape, y.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_-lL4Yq9Tbzn",
        "outputId": "7f8e4f03-db3b-4d22-aa46-1a42e9e3debd"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1000, 3) (1000,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def split_data(x,y, train_size= 0.8):\n",
        "    # shuffle the data to randomize the train/test split\n",
        "    n=x.shape[0]\n",
        "    data=np.concatenate([x,y.reshape(-1,1)], axis=1)\n",
        "    np.random.shuffle(data)\n",
        "    X_train, X_test=data[:int(n*train_size),:-1], data[int(n*train_size):,:-1]\n",
        "    y_train, y_test=data[:int(n*train_size),-1], data[int(n*train_size):,-1]\n",
        "    \n",
        "    return X_train, X_test, y_train, y_test"
      ],
      "metadata": {
        "id": "EUgiWLDhUAAK"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_test, y_train, y_test= split_data(x,y,0.8) # split your data into x_train, x_test, y_train, y_test\n",
        "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cr2Akm_A_FcJ",
        "outputId": "a4d1ac57-4d0a-49bb-8767-66f0ac635388"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(800, 3) (800,) (200, 3) (200,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def covariance(x, mu):\n",
        "  mu=np.mean(x, axis=0)\n",
        "  n,d=x.shape\n",
        "  cov=np.zeros((d,d))\n",
        "  print(cov.shape)\n",
        "  # Easy way: cov= np.cov(x, rowvar=0) but do not use it. One can use it to assess his/her result.\n",
        "  for i in range(d):\n",
        "    for j in range(d):\n",
        "      cov[i,j]=np.sum((x[:,i]-mu[i])*(x[:,j]-mu[j]))/(len(x)-1)\n",
        "  return cov"
      ],
      "metadata": {
        "id": "FrJ7GfXBHNxD"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mu=np.mean(x, axis=0)\n",
        "covariance(X_train,mu )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R9AtIpJ43aZ2",
        "outputId": "6583b138-a184-48be-9165-573203a642ca"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(3, 3)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1.83174045, 0.04449634, 1.00795144],\n",
              "       [0.04449634, 1.04155271, 0.05734296],\n",
              "       [1.00795144, 0.05734296, 1.80997036]])"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class GDA:\n",
        "  def __init__(self):\n",
        "    ## set mu, phi and sigma to None\n",
        "    self.mu=None\n",
        "    self.phi=None\n",
        "    self.sigma=None\n",
        "    \n",
        "  def fit(self,x,y):\n",
        "    k=  2 # Number of class.\n",
        "    d= x.shape[1]  # input dim\n",
        "    m= x.shape[0] # Number of examples.\n",
        "    \n",
        "    ## Initialize mu, phi and sigma\n",
        "    self.mu= np.zeros((k,d))#: kxd, i.e., each row contains an individual class mu.\n",
        "    self.sigma= np.zeros((k,d,d))#: kxdxd, i.e., each row contains an individual class sigma.\n",
        "    self.phi= np.zeros(k)# d-dimension\n",
        "\n",
        "    ## START THE LEARNING: estimate mu, phi and sigma.\n",
        "   \n",
        "    for idx in range(k): \n",
        "      self.phi[idx]=np.sum(idx==y)/m\n",
        "      self.mu[idx]=np.mean(x[idx==y], axis=0)\n",
        "      self.sigma[idx]=covariance(x[idx==y], self.mu[idx])\n",
        "\n",
        "    return self.phi, self.mu, self.sigma\n",
        "\n",
        "  def predict_proba(self,x):\n",
        "    \n",
        "    # reshape or flatt x.\n",
        "    #x= np.reshape(-1,1)\n",
        "    d= x.shape[0]\n",
        "    k_class=self.mu.shape[0] # Number of classes we have in our case it's k = 2\n",
        "    proba=np.zeros((d,k_class))\n",
        "    \n",
        "    ## START THE LEARNING: estimate mu, phi and sigma.\n",
        "    for i in range(k_class):\n",
        "      sigma_det=np.linalg.det(self.sigma[i])\n",
        "      sigma_inv=np.linalg.inv(self.sigma[i])\n",
        "      for j in range(x.shape[0]):\n",
        "        t=1/((2*np.pi)**(d/2)*(sigma_det**0.5))\n",
        "        exp=-0.5*((x[j]-self.mu[i]).T)@(sigma_inv)@(x[j]-self.mu[i])\n",
        "        t2=np.exp(exp)\n",
        "        proba[j, i]= t*t2*self.phi[i]\n",
        "    return proba\n",
        "\n",
        "  def predict(self,x):\n",
        "    pred=self.predict_proba(x)\n",
        "    ypred=np.argmax(pred, axis=1)\n",
        "    return ypred\n",
        "  \n",
        "  def accuracy(self, y, ypreds):\n",
        "    reslt=np.mean(y==ypreds)\n",
        "    return reslt*100"
      ],
      "metadata": {
        "id": "1ocKLDAfceF0"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model= GDA()\n",
        "model.fit(X_train,y_train)"
      ],
      "metadata": {
        "id": "l_qO0Yp1c3Is",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fb17ee3d-e5d4-4971-de8d-a28a3f939b08"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(3, 3)\n",
            "(3, 3)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([0.49, 0.51]), array([[ 0.98623346,  1.04799839,  0.99955429],\n",
              "        [-1.01063579,  0.96511727, -0.96654476]]), array([[[ 0.86014661, -0.38710893, -0.06176344],\n",
              "         [-0.38710893,  1.74880995,  0.10050906],\n",
              "         [-0.06176344,  0.10050906,  0.04009008]],\n",
              " \n",
              "        [[ 0.81097036,  0.37794805,  0.10960257],\n",
              "         [ 0.37794805,  0.36128408, -0.06402819],\n",
              "         [ 0.10960257, -0.06402819,  1.61594903]]]))"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "yproba= model.predict_proba(X_test)\n",
        "yproba"
      ],
      "metadata": {
        "id": "NKY1eojY1l4e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f9ae73c-83cf-4470-b317-d9f9a159e788"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1.66415108e-080, 2.32988605e-081],\n",
              "       [2.02653476e-081, 2.18328044e-101],\n",
              "       [1.52154211e-082, 4.89726369e-081],\n",
              "       [1.65320589e-125, 5.34797614e-081],\n",
              "       [2.98157435e-080, 1.51750741e-081],\n",
              "       [7.87000033e-106, 3.82463115e-081],\n",
              "       [3.11528886e-102, 2.36884361e-081],\n",
              "       [1.58786345e-080, 8.95560547e-082],\n",
              "       [6.05960755e-081, 9.03310887e-086],\n",
              "       [6.36434080e-081, 2.58044623e-082],\n",
              "       [1.66340310e-080, 1.35825850e-089],\n",
              "       [3.01334469e-080, 1.10325424e-082],\n",
              "       [9.33293323e-151, 7.24185001e-081],\n",
              "       [1.82887924e-081, 1.84708886e-084],\n",
              "       [2.00800279e-111, 1.56778369e-080],\n",
              "       [9.37552439e-087, 8.88948998e-081],\n",
              "       [2.65102764e-082, 3.93778433e-100],\n",
              "       [5.12005331e-081, 6.39532210e-089],\n",
              "       [9.58595968e-081, 2.29429948e-081],\n",
              "       [7.88191558e-093, 1.31314281e-080],\n",
              "       [6.34058096e-126, 1.48388453e-080],\n",
              "       [5.09242609e-081, 1.05518604e-081],\n",
              "       [3.56361261e-144, 4.97003090e-081],\n",
              "       [2.66409050e-108, 9.37341469e-081],\n",
              "       [2.19614240e-080, 3.75237408e-086],\n",
              "       [2.67638458e-080, 1.57580380e-081],\n",
              "       [2.15077839e-085, 2.14856290e-081],\n",
              "       [8.92369570e-099, 9.24008858e-081],\n",
              "       [5.14942379e-083, 1.15771903e-081],\n",
              "       [2.50173812e-116, 5.10444928e-081],\n",
              "       [3.00461920e-123, 1.23440214e-081],\n",
              "       [1.40514767e-086, 3.46885490e-081],\n",
              "       [9.78676214e-081, 1.86195280e-082],\n",
              "       [3.41875451e-090, 8.64762941e-081],\n",
              "       [9.27930382e-081, 1.24864822e-083],\n",
              "       [9.72165476e-172, 5.81131274e-081],\n",
              "       [6.03104255e-099, 1.42699605e-080],\n",
              "       [2.55630002e-080, 8.71081694e-082],\n",
              "       [1.32302805e-081, 8.45587771e-098],\n",
              "       [1.19487056e-080, 9.05490355e-083],\n",
              "       [6.92023981e-126, 1.15262309e-080],\n",
              "       [5.24512952e-083, 1.26910327e-081],\n",
              "       [2.10085400e-103, 9.98880148e-081],\n",
              "       [1.68539354e-080, 3.20270639e-087],\n",
              "       [1.64156733e-170, 3.65002284e-081],\n",
              "       [1.63879297e-080, 4.03069564e-090],\n",
              "       [1.36754992e-081, 7.29688810e-086],\n",
              "       [2.22002936e-087, 1.37591251e-081],\n",
              "       [3.07675505e-080, 1.71142004e-083],\n",
              "       [7.24427572e-081, 2.65871924e-087],\n",
              "       [2.64723450e-080, 4.47789264e-086],\n",
              "       [1.61306796e-087, 7.40743093e-081],\n",
              "       [2.86440874e-080, 1.10914659e-081],\n",
              "       [1.38310159e-080, 4.07983012e-082],\n",
              "       [3.15003720e-081, 3.78121913e-084],\n",
              "       [2.37026496e-101, 5.57540686e-081],\n",
              "       [1.34458274e-080, 2.35406680e-087],\n",
              "       [2.50433988e-080, 1.84751474e-081],\n",
              "       [2.28866033e-081, 2.92473625e-084],\n",
              "       [3.11538213e-118, 1.41291466e-080],\n",
              "       [4.20913939e-082, 3.92920866e-085],\n",
              "       [1.17423334e-080, 1.57331166e-088],\n",
              "       [1.54420445e-146, 7.08819514e-081],\n",
              "       [2.95620824e-080, 1.07582971e-085],\n",
              "       [1.17335647e-080, 1.99657205e-087],\n",
              "       [4.16294243e-081, 2.62154432e-096],\n",
              "       [1.09828473e-080, 2.92386814e-087],\n",
              "       [4.55201778e-081, 2.68441573e-084],\n",
              "       [2.32216022e-093, 2.96715453e-081],\n",
              "       [1.20486313e-080, 2.42128433e-092],\n",
              "       [1.09880850e-080, 6.20462349e-083],\n",
              "       [2.19228712e-080, 2.63993023e-083],\n",
              "       [2.41473533e-132, 1.29468392e-080],\n",
              "       [4.73482419e-097, 7.61025878e-081],\n",
              "       [7.85612995e-085, 9.41966857e-081],\n",
              "       [1.50068765e-105, 9.32617524e-081],\n",
              "       [1.01518203e-080, 1.13903669e-088],\n",
              "       [4.84456725e-081, 2.14479459e-085],\n",
              "       [5.41008428e-081, 7.45511016e-084],\n",
              "       [5.93594899e-128, 1.45327600e-080],\n",
              "       [2.66424542e-235, 2.33861332e-082],\n",
              "       [1.10489992e-080, 1.89577488e-081],\n",
              "       [2.78396147e-117, 1.56817416e-080],\n",
              "       [1.51706978e-153, 1.14773675e-081],\n",
              "       [2.87709826e-080, 7.68427918e-083],\n",
              "       [1.60894279e-092, 3.12896450e-081],\n",
              "       [2.23390179e-080, 2.15532829e-081],\n",
              "       [3.99696733e-081, 7.67791067e-087],\n",
              "       [2.82454080e-080, 1.88866017e-082],\n",
              "       [9.77218171e-081, 4.76506831e-085],\n",
              "       [9.87633601e-081, 4.82256816e-084],\n",
              "       [9.93662780e-081, 6.54023032e-093],\n",
              "       [1.47691913e-133, 5.04568502e-081],\n",
              "       [3.46535956e-103, 4.43101080e-081],\n",
              "       [2.67761533e-147, 9.96831469e-081],\n",
              "       [1.41725027e-080, 4.62503084e-082],\n",
              "       [5.53537607e-085, 4.93678766e-081],\n",
              "       [6.05960159e-082, 1.46767039e-084],\n",
              "       [1.82815942e-080, 8.04228468e-083],\n",
              "       [2.55677720e-080, 1.16688429e-085],\n",
              "       [5.27299137e-081, 2.38901325e-085],\n",
              "       [1.30969226e-080, 7.71259119e-091],\n",
              "       [3.11123920e-080, 6.44082075e-084],\n",
              "       [2.47400025e-080, 1.95297238e-087],\n",
              "       [7.42237991e-119, 1.48776996e-080],\n",
              "       [3.30766283e-080, 4.32675116e-082],\n",
              "       [1.82440166e-105, 1.21257038e-080],\n",
              "       [2.88264940e-080, 9.89016743e-082],\n",
              "       [8.53872768e-081, 4.00155677e-083],\n",
              "       [1.50646023e-080, 1.83886337e-082],\n",
              "       [1.75813961e-080, 2.96970477e-084],\n",
              "       [2.08530177e-080, 2.76339801e-086],\n",
              "       [4.76840757e-146, 7.38552639e-081],\n",
              "       [7.60670288e-093, 6.19855524e-081],\n",
              "       [2.10645443e-080, 7.34504710e-088],\n",
              "       [9.85942963e-081, 1.05359356e-082],\n",
              "       [5.22364424e-101, 5.58732308e-081],\n",
              "       [3.02936592e-080, 1.57750240e-082],\n",
              "       [5.11234118e-081, 7.75121418e-082],\n",
              "       [3.06458684e-135, 1.92411118e-081],\n",
              "       [1.50425815e-080, 1.13050066e-082],\n",
              "       [9.94387704e-081, 8.26281091e-094],\n",
              "       [8.80177807e-086, 8.07772316e-081],\n",
              "       [5.32468496e-234, 7.65201368e-082],\n",
              "       [6.20967306e-084, 4.54065287e-081],\n",
              "       [7.50475032e-107, 3.92387215e-081],\n",
              "       [9.56546043e-141, 2.53310643e-081],\n",
              "       [2.99260611e-080, 2.22091599e-085],\n",
              "       [4.14857680e-082, 1.65950072e-081],\n",
              "       [7.68884531e-090, 1.81763245e-081],\n",
              "       [2.58215792e-080, 5.34781222e-082],\n",
              "       [9.69643114e-082, 4.80509070e-081],\n",
              "       [1.71245945e-163, 8.28864181e-081],\n",
              "       [7.65375538e-109, 1.59872526e-080],\n",
              "       [1.15360751e-086, 9.67721299e-081],\n",
              "       [9.90206185e-081, 3.50050439e-091],\n",
              "       [1.17633627e-103, 1.11534951e-080],\n",
              "       [1.96759026e-080, 1.27083154e-085],\n",
              "       [3.74037236e-084, 8.31759428e-081],\n",
              "       [3.34814337e-080, 4.60994948e-082],\n",
              "       [9.18340249e-081, 1.99662703e-083],\n",
              "       [1.91067556e-081, 3.49134397e-086],\n",
              "       [7.06720757e-081, 5.05036523e-090],\n",
              "       [7.02573691e-081, 9.25544376e-083],\n",
              "       [7.12696112e-082, 1.83138794e-082],\n",
              "       [2.03222736e-086, 7.34883438e-082],\n",
              "       [1.73763266e-126, 1.82514035e-081],\n",
              "       [6.34148099e-081, 1.19071139e-082],\n",
              "       [1.10684523e-084, 2.00191458e-081],\n",
              "       [1.80048755e-080, 5.61831351e-089],\n",
              "       [2.92635818e-195, 2.71375926e-081],\n",
              "       [2.58556512e-123, 2.54903800e-081],\n",
              "       [1.51189476e-102, 1.25976828e-080],\n",
              "       [1.85478902e-080, 1.65185098e-082],\n",
              "       [1.78628521e-080, 1.55591181e-081],\n",
              "       [2.30371118e-094, 5.50336189e-081],\n",
              "       [6.13799308e-084, 2.90284194e-082],\n",
              "       [2.84583413e-094, 2.17484303e-081],\n",
              "       [7.07522856e-096, 6.99231348e-081],\n",
              "       [3.10645722e-081, 7.01751053e-092],\n",
              "       [1.15238083e-084, 5.61657746e-082],\n",
              "       [2.68389951e-117, 1.12233081e-080],\n",
              "       [4.19214273e-208, 2.43778026e-082],\n",
              "       [2.31521485e-132, 4.83510638e-081],\n",
              "       [2.72226393e-080, 3.42236293e-082],\n",
              "       [9.65922059e-081, 4.16284324e-083],\n",
              "       [5.61729782e-081, 1.72474148e-086],\n",
              "       [2.85744106e-081, 1.30525481e-084],\n",
              "       [9.85642706e-082, 4.20659975e-084],\n",
              "       [3.48859007e-099, 1.12290787e-080],\n",
              "       [3.12386926e-081, 5.46239211e-082],\n",
              "       [2.77116318e-080, 1.23340738e-081],\n",
              "       [2.55604462e-144, 5.79196446e-081],\n",
              "       [1.88329567e-080, 1.06146943e-082],\n",
              "       [2.67674216e-080, 7.90807348e-083],\n",
              "       [3.78295850e-081, 3.35374728e-102],\n",
              "       [3.61821439e-106, 9.12414051e-081],\n",
              "       [8.84088701e-115, 1.08167473e-080],\n",
              "       [1.05820417e-175, 4.58373018e-081],\n",
              "       [1.07884977e-173, 5.00166723e-081],\n",
              "       [2.50107718e-104, 1.06138541e-080],\n",
              "       [9.81026965e-081, 4.57860816e-085],\n",
              "       [2.32112488e-080, 8.52419789e-083],\n",
              "       [1.81855441e-148, 2.26782172e-081],\n",
              "       [6.72102578e-198, 3.71643092e-081],\n",
              "       [1.83794172e-080, 2.89291224e-082],\n",
              "       [1.65275344e-081, 2.07575012e-081],\n",
              "       [1.21300905e-080, 2.31778251e-088],\n",
              "       [1.83491565e-080, 2.54578915e-082],\n",
              "       [8.25659343e-111, 3.26343092e-081],\n",
              "       [4.47172991e-115, 3.69922613e-081],\n",
              "       [5.53179614e-082, 5.50599553e-081],\n",
              "       [1.02434391e-080, 1.21020320e-083],\n",
              "       [1.47983970e-086, 9.46984480e-081],\n",
              "       [3.28197291e-080, 2.32036148e-083],\n",
              "       [2.63827081e-091, 6.14187716e-081],\n",
              "       [1.39426339e-080, 1.51902835e-092],\n",
              "       [4.12506977e-081, 2.03115202e-085],\n",
              "       [6.63562191e-084, 9.56141343e-082],\n",
              "       [2.05453506e-126, 5.14567503e-081]])"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ypreds= model.predict(X_test)\n",
        "ypreds"
      ],
      "metadata": {
        "id": "D4clV6PK1UJK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f991a4b0-dac9-4eb9-9915-1acf640841e2"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0,\n",
              "       1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0,\n",
              "       1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0,\n",
              "       0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0,\n",
              "       0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0,\n",
              "       0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1,\n",
              "       1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0,\n",
              "       0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0,\n",
              "       1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0,\n",
              "       1, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.accuracy(y_test, ypreds)"
      ],
      "metadata": {
        "id": "QgG1xPUg1ULw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "22ad9886-cf85-4906-c993-cbc743079438"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "98.5"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Logistic Regression"
      ],
      "metadata": {
        "id": "YeQfcfjsryX8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LogisticRegression:\n",
        "  '''\n",
        "  The goal of this class is to create a LogisticRegression class, \n",
        "  that we will use as our model to classify data point into a corresponding class\n",
        "  '''\n",
        "  def __init__(self,lr,n_epochs):\n",
        "    self.lr = lr\n",
        "    self.n_epochs = n_epochs\n",
        "    self.train_losses = []\n",
        "    self.w = None\n",
        "    self.weight = []\n",
        "\n",
        "  def add_ones(self, x):\n",
        "    ##### WRITE YOUR CODE HERE #####\n",
        "    one = np.ones((x.shape[0],1))\n",
        "    return np.hstack((one,x))\n",
        "    #### END CODE ####\n",
        "\n",
        "  def sigmoid(self, x):\n",
        "    return 1/(1+np.exp(-x@self.w))\n",
        "\n",
        "\n",
        "  def cross_entropy(self, x, y_true):\n",
        "    ##### WRITE YOUR CODE HERE #####\n",
        "    y_pred = self.sigmoid(x)\n",
        "    loss = -np.mean(y_true*np.log(y_pred)+(1-y_true)*np.log(1-y_pred))\n",
        "    return loss\n",
        "    #### END CODE ####\n",
        "  \n",
        "  def predict_proba(self,x):  #This function will use the sigmoid function to compute the probalities\n",
        "    ##### WRITE YOUR CODE HERE #####\n",
        "    x= self.add_ones(x)\n",
        "    proba = self.sigmoid(x)\n",
        "    return proba\n",
        "    #### END CODE ####\n",
        "\n",
        "  def predict(self,x):\n",
        "    ##### WRITE YOUR CODE HERE #####\n",
        "    probas = self.predict_proba(x)\n",
        "    output = [0 if p<0.5 else 1 for p in probas]#np.where(probas>=0.5, 1, 0)      #convert the probalities into 0 and 1 by using a treshold=0.5\n",
        "    return output\n",
        "    #### END CODE ####\n",
        "\n",
        "  def fit(self,x,y):\n",
        "    # Add ones to x\n",
        "    x=self.add_ones(x)\n",
        "\n",
        "    # reshape y if needed\n",
        "    y=y.reshape(-1,1)\n",
        "\n",
        "    # Initialize w to zeros vector >>> (x.shape[1])\n",
        "    self.w=np.zeros((x.shape[1],1))\n",
        "\n",
        "    for epoch in range(self.n_epochs):\n",
        "      # make predictions\n",
        "      ypred = self.sigmoid(x)\n",
        "\n",
        "      #compute the gradient\n",
        "      dl = (-1/x.shape[0])*(x.T@(y-ypred))\n",
        "\n",
        "      #update rule\n",
        "      self.w=self.w-self.lr*dl\n",
        "\n",
        "      #Compute and append the training loss in a list\n",
        "      loss = self.cross_entropy(x,y)\n",
        "      self.train_losses.append(loss)\n",
        "\n",
        "      if epoch%1000 == 0:\n",
        "        print(f'loss for epoch {epoch}  : {loss}')\n",
        "\n",
        "  def accuracy(self,y_true, y_pred):\n",
        "    ##### WRITE YOUR CODE HERE #####\n",
        "    acc = np.mean(y_true==y_pred)*100\n",
        "    return acc\n",
        "    #### END CODE ####"
      ],
      "metadata": {
        "id": "OpXYY-yj1UOj"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = LogisticRegression(0.01,n_epochs=10000)\n",
        "model.fit(X_train,y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VwDvGqoomJ0H",
        "outputId": "aae870f2-9481-4fbf-b6b7-62a22d137051"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "loss for epoch 0  : 0.6882546546861045\n",
            "loss for epoch 1000  : 0.19487474396451476\n",
            "loss for epoch 2000  : 0.1708657974798757\n",
            "loss for epoch 3000  : 0.16187807712262728\n",
            "loss for epoch 4000  : 0.1570005916192161\n",
            "loss for epoch 5000  : 0.15385948502076208\n",
            "loss for epoch 6000  : 0.15163299851168463\n",
            "loss for epoch 7000  : 0.1499592016785418\n",
            "loss for epoch 8000  : 0.1486518169813554\n",
            "loss for epoch 9000  : 0.1476036002821396\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ypred_train = model.predict(X_train)\n",
        "acc = model.accuracy(y_train,ypred_train)\n",
        "print(f\"The training accuracy is: {acc}\")\n",
        "print(\" \")\n",
        "\n",
        "ypred_test = model.predict(X_test)\n",
        "acc = model.accuracy(y_test,ypred_test)\n",
        "print(f\"The test accuracy is: {acc}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UdLkVG7dmJxR",
        "outputId": "e714e4b2-a44f-4fa9-952e-f7b4907a7bc7"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The training accuracy is: 95.75\n",
            " \n",
            "The test accuracy is: 95.0\n"
          ]
        }
      ]
    }
  ]
}