Basic_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from numba import vectorize\n",
    "\n",
    "class Learn_model:\n",
    "    \"\"\"The class trains a 3 layer network based on the given data using backpropagation. It takes the data and prepares it by\n",
    "    normalizing the numbers, taking out the answer column and adding ones for bias. It generates random weights(depends if \n",
    "    the seed is set). Then it splits the data into mini-batches and trains using backpropagation. The class main function\n",
    "    is to return(the method returns) the cost and success rate with specific parameters.\"\"\"\n",
    "    \n",
    "    np.random.seed(42)\n",
    "    \n",
    "    def __init__(self, wLayer1, wLayer2, trainData, testData, position_of_expected_output, over_fit, num_iterations,\n",
    "                 learning_rate, batch_size, rate_decreasing_speed):\n",
    "        \"\"\"The class takes these inputs:\n",
    "        1. the size of the weight of the first layer.\n",
    "        2. the size of the weight of the second layer.\n",
    "        3. the training data in numpy array.\n",
    "        4. testing data in numpy array\n",
    "        5. position of expected output.\n",
    "        6. a variable to protect from overfitting\n",
    "        7. number of iterations the training will repeat\n",
    "        8. learning rate\n",
    "        9. the batch size\n",
    "        10. how much overtime does the learning rate will decrease\"\"\"\n",
    "        self.wLayer1 = wLayer1\n",
    "        self.wLayer2 = wLayer2\n",
    "        self.trainData = trainData\n",
    "        self.testData = testData\n",
    "        self.position_of_expected_output = position_of_expected_output\n",
    "        self.over_fit = over_fit\n",
    "        self.num_iterations = num_iterations\n",
    "        self.learning_rate = learning_rate\n",
    "        self.batch_size = batch_size\n",
    "        self.rate_decreasing_speed = rate_decreasing_speed\n",
    "    \n",
    "    def prepare_data_for_training(self):\n",
    "        \"\"\"regularizes the data and removes expected outputs from the data set, adds ones for the bias unit.\n",
    "        returns training output is 2d array and 1d array of testing output\"\"\"\n",
    "        \n",
    "        expected_output1D = self.trainData[:, self.position_of_expected_output].copy()#list of expected Output\n",
    "        expected_output1D_test = self.testData[:, self.position_of_expected_output].copy()\n",
    "        #array of expected outputs. 1s are placed in rows that correspond with expected_output1D\n",
    "        expected_output2D = np.zeros((len(expected_output1D), self.wLayer2[1]))\n",
    "        for i in range(len(expected_output1D)):\n",
    "            expected_output2D[i, expected_output1D[i]] = 1\n",
    "        \n",
    "        #regularizes the data and removes expected outputs from the training set\n",
    "        self.trainData = np.delete(self.trainData, self.position_of_expected_output, 1)\n",
    "        self.trainData = (self.trainData - np.average(self.trainData)) / np.std(self.trainData)\n",
    "        self.trainData = np.append(np.ones((len(self.trainData), 1)), self.trainData, axis=1)\n",
    "        \n",
    "        self.testData = np.delete(self.testData, self.position_of_expected_output, 1)\n",
    "        self.testData = (self.testData - np.average(self.testData)) / np.std(self.testData)\n",
    "        self.testData = np.append(np.ones((len(self.testData), 1)), self.testData, axis=1)\n",
    "        \n",
    "        return expected_output2D, expected_output1D_test\n",
    "    \n",
    "    def generate_random_weights(self):\n",
    "        \"\"\"Creates random weights for network. Returns first and second layers weights\"\"\"\n",
    "        w1 = np.random.rand(self.wLayer1[0], self.wLayer1[1]) - 1\n",
    "        w2 = np.random.rand(self.wLayer2[0], self.wLayer2[1]) - 1\n",
    "        return w1, w2\n",
    "    \n",
    "    def sigmoid(self, prediction):\n",
    "        \"\"\"takes each layers predictions and maps it between 0 and 1\"\"\"\n",
    "        #prediction = prediction.astype(float)\n",
    "        prediction = prediction * -1\n",
    "        sigmoid = 1 / (1 + np.exp(prediction))\n",
    "        return sigmoid\n",
    "    \n",
    "    def feedforward(self, w1, w2, input_layer):\n",
    "        \"\"\"Takes in weights of the layer 1 and 2 and the inputs and outputs 1 and 2 layers calculations\"\"\"\n",
    "        layer2 = self.sigmoid(np.dot(input_layer, w1))#multiplies input layer with weights\n",
    "        #add ones because the bias is in the weights in w[:, 0]. By multiplying by one it is the same as adding the bias\n",
    "        z2 = np.append(np.ones((len(layer2), 1)), layer2, axis=1)\n",
    "        layer3 = self.sigmoid(np.dot(z2, w2))\n",
    "        #a3 -= 1e-15\n",
    "        return layer2, layer3\n",
    "    \n",
    "    def backpropagation(self, w2, layer3, layer2, input_layer, expected_output):\n",
    "        \"\"\"Uses partial derivatives to calculate each layers partial derivative.\n",
    "        Takes in second layers weights, layers 2 and 3 predictions, input layer,\n",
    "        expected 2d array output. Returns the gradient of weights of layers 1 and 2.\"\"\"\n",
    "        delta3 = (layer3 - expected_output) #the derivative of the cost function\n",
    "        delta2 = np.dot(delta3, w2.T) #the partial derivative of the second layer sigmoid\n",
    "        #removing the bias layer. The partial derivative of bias is equal to delta3 because in layer 2 it is equal to 1.\n",
    "        delta2 = np.delete(delta2, 0, 1)\n",
    "        sigmoid_layer2_derivative = (layer2 * (1 - layer2))#the partial derivative of the sigmoid function\n",
    "        delta2 = delta2 * sigmoid_layer2_derivative #the partial derivative of the second layer\n",
    "        w1_gradient = (np.dot(input_layer.T, delta2)) / len(input_layer)#the layer 1 weights gradient\n",
    "        layer2 = np.append(np.ones((len(layer2), 1)), layer2, axis=1) #this is needed to calculate with the bias gradient\n",
    "        w2_gradient = (np.dot(layer2.T, delta3)) / len(input_layer)#the layer 2 weights gradient\n",
    "        return w1_gradient, w2_gradient\n",
    "    \n",
    "    def cost(self, w1, w2, prediction, expected_output, over_fit):\n",
    "        \"\"\"Calculate the error in network prediction. Takes weights of layer 1 and 2, models prediction, 2d expected output\n",
    "        and overfit unit. Returns the error of the function\"\"\"\n",
    "        #temp_w_all is for calculating the sum of all weights, this helps to avoid overfitting\n",
    "        temp_w_all = w1[:, 1:].ravel()\n",
    "        temp_w_all = np.append(w2[:, 1:].ravel(), temp_w_all)\n",
    "        #some number is too small or too big and overflow happens and it turns out either 1 or 0. The cost function has log()\n",
    "        #and log(0) is invalid so the for loop adds or subtracts a very small number if it is 1 or 0.\n",
    "        for i in range(len(prediction)):\n",
    "            for j in range(len(prediction[0])):\n",
    "                if prediction[i][j] == 1.:\n",
    "                    #print(prediction[i][j])\n",
    "                    prediction[i][j] -= 1e-15\n",
    "                elif prediction[i][j] == 0.:\n",
    "                    #print(prediction[i][j])\n",
    "                    prediction[i][j] += 1e-15\n",
    "                    \n",
    "        #the cost function. The log punishes more if the answer is incorrect.\n",
    "        cost = -(sum(sum(expected_output * np.log(prediction) + (1 - expected_output) * np.log(1 - prediction))) /\n",
    "                 len(expected_output)) + (sum(np.power(temp_w_all, 2))) * over_fit / (2 * len(expected_output))\n",
    "        return cost\n",
    "    \n",
    "    def succes_rate(self, layer3, expected_output):\n",
    "        \"\"\"cheeks how well the model is doing on the test dataset, returns the success rate.\"\"\"\n",
    "        #converts to a list for easy comparison\n",
    "        expected_output = expected_output.reshape((1, len(expected_output))).tolist()\n",
    "        predictions = np.argmax(layer3, axis=1)#takes the biggest models prediction for each example\n",
    "        succes_rate = (expected_output == predictions).astype(int)\n",
    "        succes_rate = sum(sum(succes_rate)) / len(expected_output[0])\n",
    "        return succes_rate\n",
    "    \n",
    "    def shuffling_data(self, expected_output2D):\n",
    "        \"\"\"Combines train data and train datas expected output arrays in one and then shuffles the columns. Then restores\n",
    "        data into seperate arrays. Takes in train data 2d array expected output. Returns train data expected_output2D\"\"\"\n",
    "        combined_data = np.c_[self.trainData.reshape(len(self.trainData), - 1), #combines the arrays\n",
    "                  expected_output2D.reshape(len(expected_output2D), - 1)]\n",
    "        np.random.shuffle(combined_data)\n",
    "        #splits the data back into separate arrays\n",
    "        self.trainData = combined_data[:,0:len(self.trainData[0])]\n",
    "        expected_output2D = combined_data[:, (len(combined_data[0]) - len(expected_output2D[0])):]\n",
    "        return expected_output2D\n",
    "        \n",
    "    def update_weights(self, expected_output2D, w1, w2, mini_batch):\n",
    "        \"\"\"using feedforward and backpropagation methods updates the weights. update_weights takes train expected output\n",
    "        2d array, 1 and 2 weights to change and the mini-batch. Returns modified weights of layers 1 and 2.\"\"\"\n",
    "        [layer2, layer3] = self.feedforward(w1, w2, mini_batch)\n",
    "        [w1_gradient, w2_gradient] = self.backpropagation(w2, layer3, layer2, mini_batch, expected_output2D)\n",
    "        w1 = (w1 * (1 - (self.over_fit * self.learning_rate / len(expected_output2D)))) - (w1_gradient * self.learning_rate)\n",
    "        w2 = (w2 * (1 - (self.over_fit * self.learning_rate / len(expected_output2D)))) - (w2_gradient * self.learning_rate)\n",
    "        return w1, w2\n",
    "    \n",
    "    def training(self):\n",
    "        \"\"\"Combines all of the methods together. Prepares the data, generates new weights, shuffles the data and splits\n",
    "        it to mini batches. Returns how the model did on the train and test data set.\"\"\"\n",
    "        original_lerning_rate = self.learning_rate\n",
    "        [expected_output2D, expected_output1D_test] = self.prepare_data_for_training()\n",
    "        [w1, w2] = self.generate_random_weights()\n",
    "        \n",
    "        for i in range(self.num_iterations):\n",
    "            expected_output2D = self.shuffling_data(expected_output2D)\n",
    "            #his decreases the learning rate as iterations increase. Over time the gradient gets smaller so the steps also have\n",
    "            #to get smaller, this helps to not overshoot.\n",
    "            self.learning_rate = original_lerning_rate / (1 + (i / self.rate_decreasing_speed)) \n",
    "            for j in range(int(len(self.trainData) / self.batch_size)):\n",
    "                #splitting data into mini batches\n",
    "                mini_batch = self.trainData[j * self.batch_size : j * self.batch_size + self.batch_size, :]\n",
    "                batch_output2D = expected_output2D[j * self.batch_size : j * self.batch_size + self.batch_size, :]\n",
    "                [w1, w2] = self.update_weights(batch_output2D, w1, w2, mini_batch)\n",
    "        \n",
    "        #how well did the model does\n",
    "        [layer2, layer3] = self.feedforward(w1, w2, self.trainData)\n",
    "        cost_result = self.cost(w1, w2, layer3, expected_output2D, self.over_fit)\n",
    "        [layer2, layer3] = self.feedforward(w1, w2, self.testData)\n",
    "        succes_rate_result = self.succes_rate(layer3, expected_output1D_test)\n",
    "        \n",
    "        return cost_result, succes_rate_result  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "%matplotlib notebook\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_test = pd.read_csv(\"mnist_test.csv\")\n",
    "mnist_test_np = mnist_test.to_numpy()\n",
    "mnist_train = pd.read_csv(\"mnist_train.csv\")\n",
    "mnist_train = mnist_train.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this will test a wide range of different parameters\n",
    "overfit = 0\n",
    "learning_rate = 1.1\n",
    "batch_size = 2\n",
    "learning_rate_decreasing_speed = 70\n",
    "\n",
    "different_parameters_tracker = np.zeros((2*5*6+1, 6))\n",
    "index = 0\n",
    "for i in range(2): #learning_rate\n",
    "    for j in range(5): # overfit\n",
    "        \n",
    "        for k in range(6): # batch_size\n",
    "            mnist = Learn_model([785, 78], [79, 10], mnist_train, mnist_test_np, 0, overfit, 50, \n",
    "                                learning_rate, batch_size, learning_rate_decreasing_speed)\n",
    "                \n",
    "            [cost, succes_rate] = mnist.training()\n",
    "            #saving the parameters and models performance\n",
    "            different_parameters_tracker[index] = [cost, succes_rate, overfit, learning_rate, batch_size, learning_rate_decreasing_speed]\n",
    "            index += 1\n",
    "            #tweking parameters\n",
    "            batch_size *= 2\n",
    "            \n",
    "        batch_size = 2\n",
    "        overfit += 0.5\n",
    "        \n",
    "    overfit = 0\n",
    "    learning_rate += 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "succes_results = different_parameters_tracker[:, 1]\n",
    "result = np.where(succes_results == np.amax(succes_results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost: [[0.14153398]] succes rate: [[0.9606]] overfit value: [[0.]] training rate: [[1.1]] batch size: [[8.]] learning rate decreasing speed [[70.]]\n"
     ]
    }
   ],
   "source": [
    "print(\"cost:\", different_parameters_tracker[result, 0], \"succes rate:\", different_parameters_tracker[result, 1],\n",
    "     \"overfit value:\", different_parameters_tracker[result, 2], \"training rate:\", different_parameters_tracker[result, 3],\n",
    "     \"batch size:\", different_parameters_tracker[result, 4], \"learning rate decreasing speed\", different_parameters_tracker[result, 5])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}