Training_a_PyTorch_network.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Training a PyTorch network.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/paulgureghian/Google_Colab_Notebooks/blob/master/Training_a_PyTorch_network.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "U8zIDwQqFxBq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Created by Paul A. Gureghian on 11/24/18.**\n",
        "\n",
        "**This notebook has the code to train a PyTorch deep learning network,**\n",
        "\n",
        "**which makes it a 'smart' network.**\n",
        "\n",
        "**I will show it real data, then adjust the network parameters.**"
      ]
    },
    {
      "metadata": {
        "id": "ClNkEiwV9k_Z",
        "colab_type": "code",
        "outputId": "4c3b1c83-714d-45ea-b575-1577efe73378",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "### Install PyTorch and Torchvision \n",
        "!pip3 install -U -q torchvision \n",
        "!pip3 install -U -q http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tcmalloc: large alloc 1073750016 bytes == 0x583d4000 @  0x7efe1a1062a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "laYvMkqzNNDz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### Import packages\n",
        "import torch\n",
        "import numpy as np\n",
        "from torch import nn\n",
        "from torch import optim\n",
        "import torch.nn.functional as F\n",
        "import matplotlib.pyplot as plt\n",
        "from torchvision import  datasets, transforms"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1DIlAZYbOuev",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### Define a transform to normalize the data\n",
        "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),])\n",
        "\n",
        "### Download and load the training data\n",
        "trainset = datasets.MNIST('~/.pytorch/MNIST_data/', download=True, train=True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Jox_kTMlP0XE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "outputId": "ba08ede1-2443-4112-97ca-7324ee165f68"
      },
      "cell_type": "code",
      "source": [
        "### Build a feed-forward network\n",
        "model = nn.Sequential(nn.Linear(784, 128),\n",
        "                      nn.ReLU(),\n",
        "                      nn.Linear(128, 64),\n",
        "                      nn.ReLU(),\n",
        "                      nn.Linear(64, 10),\n",
        "                      nn.LogSoftmax(dim=1))\n",
        "\n",
        "criterion = nn.NLLLoss()\n",
        "images, labels = next(iter(trainloader))\n",
        "images = images.view(images.shape[0], -1)\n",
        "\n",
        "logps = model(images)\n",
        "loss = criterion(logps, labels)\n",
        "print(\"Loss: \", loss)\n",
        "print('')\n",
        "print(\"Before backward pass:\\n \", model[0].weight.grad)\n",
        "print('')\n",
        "loss.backward()\n",
        "print(\"After backward pass:\\n \", model[0].weight.grad)"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loss:  tensor(2.3109, grad_fn=<NllLossBackward>)\n",
            "\n",
            "Before backward pass:\n",
            "  None\n",
            "\n",
            "After backward pass:\n",
            "  tensor([[-0.0019, -0.0019, -0.0019,  ..., -0.0019, -0.0019, -0.0019],\n",
            "        [-0.0022, -0.0022, -0.0022,  ..., -0.0022, -0.0022, -0.0022],\n",
            "        [ 0.0022,  0.0022,  0.0022,  ...,  0.0022,  0.0022,  0.0022],\n",
            "        ...,\n",
            "        [-0.0033, -0.0033, -0.0033,  ..., -0.0033, -0.0033, -0.0033],\n",
            "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
            "        [-0.0023, -0.0023, -0.0023,  ..., -0.0023, -0.0023, -0.0023]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-aLtbjYF7qOv",
        "colab_type": "code",
        "outputId": "4f35acda-3278-4f11-831e-beffb3492c76",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "cell_type": "code",
      "source": [
        "### Print initial weights before training\n",
        "print(\"Initial weights:\\n \", model[0].weight)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Initial weights:\n",
            "  Parameter containing:\n",
            "tensor([[-0.0024,  0.0289,  0.0081,  ..., -0.0298,  0.0322,  0.0336],\n",
            "        [-0.0073, -0.0066,  0.0160,  ..., -0.0236, -0.0068, -0.0120],\n",
            "        [ 0.0217, -0.0166,  0.0234,  ..., -0.0029, -0.0205,  0.0110],\n",
            "        ...,\n",
            "        [-0.0184, -0.0003,  0.0324,  ...,  0.0317, -0.0014, -0.0170],\n",
            "        [-0.0013,  0.0272,  0.0309,  ...,  0.0091, -0.0249, -0.0026],\n",
            "        [ 0.0287, -0.0345,  0.0188,  ..., -0.0343, -0.0223, -0.0139]],\n",
            "       requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ppJLl63yzrqK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Now I will put this algorithm into a loop.**"
      ]
    },
    {
      "metadata": {
        "id": "vkAlelzaSRle",
        "colab_type": "code",
        "outputId": "55b9f0d5-fce1-4213-9055-201531f26c77",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "cell_type": "code",
      "source": [
        "### Iterate over all the dataset images\n",
        "epochs = 5\n",
        "### Define an optimizer and a learning rate\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.003)\n",
        "\n",
        "for e in range(epochs):\n",
        "    running_loss = 0\n",
        "    for images, labels in trainloader:\n",
        "      \n",
        "        ### flatten images into a 784 long vector\n",
        "        images = images.view(images.shape[0], -1)\n",
        "\n",
        "        ### clear the gradients\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        ### forward pass through the network\n",
        "        ### get the network output\n",
        "        output = model(images)\n",
        "\n",
        "        ### calculate the loss with the logps and labels\n",
        "        loss = criterion(output, labels)       \n",
        "              \n",
        "        ### backward pass through the network \n",
        "        loss.backward()\n",
        "      \n",
        "        ### update the weights\n",
        "        optimizer.step()\n",
        "\n",
        "        running_loss += loss.item()\n",
        "    else:\n",
        "        print(f\"Training loss: {running_loss/len(trainloader)}\") "
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training loss: 1.8310978416440837\n",
            "Training loss: 0.8072923103184588\n",
            "Training loss: 0.5230883436321195\n",
            "Training loss: 0.43013259770074636\n",
            "Training loss: 0.3854167649645541\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "2aD4Ep8Tew5E",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "d6db137f-8342-4f4f-85da-16b51b85c0ae"
      },
      "cell_type": "code",
      "source": [
        "### After optimizer step\n",
        "print(\"Updated weights:\\n \", model[0].weight)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Updated weights:\n",
            "  Parameter containing:\n",
            "tensor([[-0.0017,  0.0295,  0.0087,  ..., -0.0291,  0.0329,  0.0343],\n",
            "        [-0.0114, -0.0107,  0.0119,  ..., -0.0277, -0.0109, -0.0161],\n",
            "        [ 0.0150, -0.0233,  0.0167,  ..., -0.0096, -0.0272,  0.0043],\n",
            "        ...,\n",
            "        [-0.0207, -0.0027,  0.0301,  ...,  0.0293, -0.0038, -0.0194],\n",
            "        [-0.0036,  0.0248,  0.0286,  ...,  0.0067, -0.0273, -0.0050],\n",
            "        [ 0.0238, -0.0394,  0.0139,  ..., -0.0392, -0.0272, -0.0188]],\n",
            "       requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "qCJPdXHmq2Pq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### Define a function 'view_classify'\n",
        "### for viewing an image and its predicted class\n",
        "def view_classify(img, ps, version=\"MNIST\"):\n",
        "    ''' Function for viewing an image and it's predicted classes.\n",
        "    '''\n",
        "    ps = ps.data.numpy().squeeze()\n",
        "\n",
        "    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)\n",
        "    ax1.imshow(img.resize_(1, 28, 28).numpy().squeeze())\n",
        "    ax1.axis('off')\n",
        "    ax2.barh(np.arange(10), ps)\n",
        "    ax2.set_aspect(0.1)\n",
        "    ax2.set_yticks(np.arange(10))\n",
        "    if version == \"MNIST\":\n",
        "        ax2.set_yticklabels(np.arange(10))\n",
        "    elif version == \"Fashion\":\n",
        "        ax2.set_yticklabels(['T-shirt/top',\n",
        "                            'Trouser',\n",
        "                            'Pullover',\n",
        "                            'Dress',\n",
        "                            'Coat',\n",
        "                            'Sandal',\n",
        "                            'Shirt',\n",
        "                            'Sneaker',\n",
        "                            'Bag',\n",
        "                            'Ankle Boot'], size='small');\n",
        "    ax2.set_title('Class Probability')\n",
        "    ax2.set_xlim(0, 1.1)\n",
        "\n",
        "    plt.tight_layout()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tABKjHVrsIwf",
        "colab_type": "code",
        "outputId": "a2f2c53d-a9ba-4415-8236-97429006b945",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 242
        }
      },
      "cell_type": "code",
      "source": [
        "### Get the predictions of the trained network\n",
        "images, labels = next(iter(trainloader))\n",
        "\n",
        "img = images[0].view(1, 784)\n",
        "\n",
        "with torch.no_grad():\n",
        "  output = model(img)\n",
        "\n",
        "### Get the exponential of the network output   \n",
        "ps = torch.exp(output)  \n",
        "view_classify(img.view(1 ,28, 28), ps)\n"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAADhCAYAAACHrsFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGS9JREFUeJzt3Xl8FdXh/vEnC4GwhQRIFBVZLAVC\nqSDaymIgTUhYfgoWSdjLIlAFxWoFEUkshQIvKG2VCkW0GKRiMShYSMC8wDVsQqEJUjB9SdkMIQtb\nAiY38/vDer+id0KiycwhfN5/Jefcc8+DRB5m7jDjZ1mWJQAAXObvdgAAACQKCQBgCAoJAGAECgkA\nYAQKCQBgBAoJAGAECglAjbAsSy+//LIGDhyouLg4xcTEKDk5WefPn5ckzZgxQ3/+859rNMOMGTP0\n05/+VPHx8YqLi1N8fLyWLFkij8dTpfc5fvy4OnbsWOX9R40apbfeeutb47m5uRo4cKAk6bnnntPT\nTz8tSRozZoyys7MlSa+//nqV97vWUUgAasSiRYu0adMmrVy5Uunp6dqwYYNKS0s1adIkOfnPH0eP\nHq20tDSlp6dr3bp1+uijj1z/wz4iIkJvv/32t8ZXrVqlyMhIeTweLVy40IVk7qKQAFS7oqIipaSk\naP78+YqIiJAk1a9fX7Nnz9aECRO+VUj79u3T/fffr/j4ePXv318fffSRJKmsrExPP/204uLiFBsb\nqylTpujChQu241fTsGFDDRo0SB9++KGkL49glixZon79+mnv3r0qKirSo48+qri4OPXv319/+ctf\nrlj/8ssvq1+/foqOjtY777wjSSovL9ezzz6ruLg4RUdH69e//rVKS0u9aw4fPqwhQ4YoKipKs2bN\nksfjsT3iio6O1p49ezR27FidP39e8fHxevXVVzVp0iTva8rLy9W9e3d98sknlfmtuKZQSACq3f79\n+3XDDTeobdu2V4zXrVtX0dHR8ve/8o+e2bNna/z48UpLS9PEiROVlJQkSfrggw90/PhxpaWlacuW\nLbrtttu0b98+2/HKKCsrU1BQkPf7rKws/eMf/1DXrl31+9//XiEhIUpPT9eaNWv0t7/9TXv27JEk\neTweeTwebd68WXPmzNEzzzyj0tJSbd26VXv27NHbb7+tzZs3Kzs7W5s2bfK+/86dO5WSkqK0tDTt\n3r1b27Ztu2rGefPmKSAgQGlpaYqPj9eOHTtUWFgoSdq7d68aN26sDh06VOrXey2hkABUu6KiIjVt\n2rTSr3/zzTfVr18/SdIdd9yhY8eOSZLCwsKUk5OjrVu3qqSkRNOmTVOvXr1sx68mPz9fb7zxhmJj\nY71jUVFR3oJ89913NXz4cElSkyZNFBsb6z2akqTBgwdLknr06KGysjL997//VVxcnN544w3VqVNH\ndevW1Y9+9CNvfkmKi4tTcHCwgoODFRUVpX/+85+V/u8iSU2bNlW3bt2Unp4uSdq6dav69+9fpfe4\nVlBIAKpdaGiocnNzK/36jRs3asiQIYqLi9O4ceO8p/Q6d+6sWbNmKSUlRT169NDjjz+uc+fO2Y77\n8sorryg+Pl7x8fGaMGGCHnjgAW/5SVJISIj364KCAjVu3Nj7fePGjZWfn3/Fr+srjRo10rlz51RQ\nUKDp06d7L5rIyMi44pRkWFjYt9ZU1YABA7yfOWVkZFBIAFBZt99+u/Lz871XjH2ltLRUS5YsUUlJ\niXcsNzdXs2bN0ty5c5Wenq4VK1ZcsSY+Pl4pKSnatm2bSkpKtHLlygrHv+mrixrS0tK0fv16jRo1\nyjZ3s2bNVFRU5P2+qKhIzZo1835/9uzZK74OCQnRkiVLFBgYqI0bNyotLU1RUVFXvKevNVUVGxur\nrKwsvfvuuwoODtZtt91W5fe4FlBIAKpd48aNNWHCBE2fPl1Hjx6VJJWUlGj27Nk6ePCggoODva8t\nKChQ/fr11aZNG5WVlWnt2rWSpIsXL+qNN97Q0qVLJX15Cq1NmzaSZDv+ffXu3du7f0FBgbZu3are\nvXt75zdu3ChJ+vDDDxUcHKyWLVsqPz9f7dq1U1BQkA4dOqR9+/apuLjYu2bLli26fPmyiouL9f77\n76tbt25XzVGnTh2Vl5d7L9Ro1KiRevXqpWefffaKo7vaJtDtAABqp6lTpyokJES//OUv5fF45O/v\nr5/97GdKTk6+4nXt27fXPffco7i4ODVt2lQzZszQ3r17NWrUKL300kuaOXOm+vbtq4CAAN16662a\nP3++JNmOfx/Tpk1TcnKy4uPj5e/vr4kTJ6pz5846fvy46tevr/Lycg0cOFCXLl3S3LlzFRgYqHHj\nxmn69OlKTU1Vt27dNH36dD399NPq3LmzJKl79+4aPXq0cnNz1bt3b/Xq1UsnT56sMEfz5s11xx13\nqE+fPlq+fLm6du2qAQMGaMuWLbX2dJ0k+fE8JAAw34EDB/Sb3/xG69atcztKjeGUHQAYrqysTEuX\nLq3w86/agEICAIMdPHhQsbGxCg8P17333ut2nBrFKTsAgBE4QgIAGIGr7ACDlZV5VFhYfPUX1qDQ\n0PpkIEO1ZWjevJHtHEdIgMECAwPcjkAGMjiWgUICABiBQgIAGIFCAgAYgUICABiBQgIAGIFCAgAY\ngUICABiBQgIAGIFCAgAYgUICABiBe9kBDiovL1dSUpKOHDmiOnXqKDk5WW3btnU7FmAEjpAAB2Vk\nZOj8+fN67bXXNHfuXC1cuNDtSIAxKCTAQZ999pk6d+4sSWrZsqVOnjwpj8fjcirADJyyAxzUrl07\nrVq1SmPGjNHRo0d17NgxFRYWqlmzZrZrKrpdv1PIQAYnMlBIgIOioqK0d+9ejRgxQj/84Q/Vpk0b\nXe2hzXl55x1K51vz5o3IQIZqy1BRmVFI16B///vftnMffPCBz3E/P78q7zNs2DDbueDg4Cq/H770\n2GOPeb+OiYlR06ZNXUwDmIPPkAAHHTp0SE899ZQk6b333lPHjh3l78//hoDEERLgqHbt2smyLA0Z\nMkR169bVokWL3I4EGINCAhzk7++v+fPnux0DMBLnCgAARqCQAABGoJAAAEbgMySXffHFF7Zz69at\n8zk+YcIE2zWXLl3yOf5dLvueNm2a7dwrr7zic3zQoEFV3gcAJI6QAACGoJAAAEagkAAARuAzJMBB\nFy9e1PTp03X27FmVlpbq4YcfVq9evdyOBRiBQgIctH79erVu3VqPP/64cnNzNWbMGKWlpbkdCzAC\nheSyy5cv286NGjXKwSTfdvHiRdu5ESNG+By3u/pOkn7+859/70zXutDQUO/Ncc+dO6fQ0FCXEwHm\noJAABw0YMECpqamKjY3VuXPntHz5crcjAcagkAAHvfXWW2rRooVWrlypQ4cOaebMmUpNTa1wTW1+\nIBsZyPB1FBLgoL1796pnz56SpPbt2+v06dPyeDwKCAiwXXOtP5CNDGT45no7XPYNOOjWW2/V/v37\nJUknTpxQgwYNKiwj4HrCERLgoISEBM2cOVMjR45UWVmZkpOT3Y4EGINCAhzUoEED/fGPf3Q7BmAk\nCsllgYH2vwXjxo3zOX7ffffZrrEsy+f4ggULbNdkZmbaztmxu4nrihUrbNdw2TeAivAZEgDACBQS\nAMAIFBIAwAgUEgDACBQSAMAIXGXnsuDgYNu5iq5Yq6ro6GjbObsbok6ZMqXa9geAq+EICQBgBI6Q\nAAf9/e9/14YNG7zfZ2Vlad++fS4mAsxBIQEOeuCBB/TAAw9Iknbt2qXNmze7nAgwB6fsAJcsXbpU\nDz30kNsxAGNwhAS44MCBA7rxxhvVvHnzq762Nj//hgxk+DoKCXDBunXrNHjw4Eq99lp//g0ZyPDN\n9XYopOtEgwYNbOfat29fbfvs3r3bdi4nJ8fneNu2batt/2vFzp07NWvWLLdjAEbhMyTAYbm5uWrQ\noIGCgoLcjgIYhUICHJaXl6ewsDC3YwDGoZAAh3Xq1Ekvvvii2zEA41BIAAAjUEgAACNwlR2qVVFR\nke1cQUGBz/Hr8So7AN/GERIAwAgUEgDACBQSAMAIFBIAwAgUEuCwDRs26N5779X999+v7du3ux0H\nMAaFBDiosLBQS5cu1Zo1a7Rs2TJlZGS4HQkwBpd9Aw7KzMzU3XffrYYNG6phw4aaM2eO25EAY3CE\nBDjo+PHjunTpkiZPnqzhw4crMzPT7UiAMThCAhxWVFSk559/XidPntTo0aO1bds2+fn52b6+Nj+Q\njQxk+DoKCXBQ06ZN1aVLFwUGBqply5Zq0KCBCgoK1LRpU9s11/oD2chAhm+ut8MpO8BBPXv21I4d\nO1ReXq7CwkIVFxcrNDTU7ViAEThCAhwUERGhuLg4DR06VJI0a9Ys+fvz90JAopCuGx6Px3ZuzZo1\n1bbPbbfd9p3mrieJiYlKTEx0OwZgHP5qBgAwAoUEADAChQQAMAKFBAAwAoUEADAChQQAMAKXfV8n\nxo8fbzuXkpJSbfu0bt3ado5/AAqgIhwhAQCMwBES4KCdO3fq0Ucf1Q9+8ANJUrt27fTMM8+4nAow\nA4UEOOyuu+7Sn/70J7djAMbhlB0AwAgcIQEO+/TTTzV58mSdPXtWU6ZMUY8ePSp8fW1+/g0ZyPB1\nfpZlWTXyznDFf/7zH5/jkZGRtmsuX75c5X3sbpS6Y8cO2zVhYWFV3qe2yc3N1ccff6x+/frp2LFj\nGj16tLZs2aKgoCDbNdf682/IQIZvrrfDKTvAQREREerfv7/8/PzUsmVLNWvWTLm5uW7HAoxAIQEO\n2rBhg1auXClJysvLU35+viIiIlxOBZiBz5AAB0VHR+uJJ55QRkaGSktLlZycXOHpOuB6QiEBDmrY\nsKGWLVvmdgzASJyyAwAYgUICABiBU3Y23nvvPdu5X/3qVz7Hc3JyqrzPxIkTbed+8pOf+BzfsGGD\n7Zq1a9f6HP/iiy9s1/j5+dnO2dm8ebPPcS7tBvBdcYQEADAChQQAMAKFBAAwAoUEADAChQS44NKl\nS4qJiVFqaqrbUQBjXBdX2eXl5dnOxcXF+RzPzs62XVNWVva9M31l0aJF1fZekmR3r9zvciXd8OHD\nbedatWpV5ffD/3nhhRcUEhLidgzAKBwhAQ7LycnRp59+qt69e7sdBTDKdXGEBJhkwYIFeuaZZ/Tm\nm29W6vW1+fk3ZCDD11FIgIPefPNN3X777brlllsqveZaf/4NGcjwzfV2KCTAQdu3b9exY8e0fft2\nff755woKCtINN9yg7t27ux0NcB2FBDjoD3/4g/fr5557TjfddBNlBPwPFzUAAIxQq46Qfvvb3/oc\nr+j5M6dOnaq2/bt162Y7t2fPnmrbxylvv/227ZzdB/IVfTbSoUMHn+ONGrn/Ia0bpk6d6nYEwCgc\nIQEAjEAhAQCMQCEBAIxAIQEAjEAhAQCMUKuusps9e7bP8e9yY9GKDBs2zOd4ZmZmte7jtnPnztnO\nDR06tMrvFxkZ6XM8IiLCdk39+vV9jq9atcp2TZMmTaoWDIAROEICABiBQgIAGKFWnbIDTFdSUqIZ\nM2YoPz9fly9f1kMPPaQ+ffq4HQswAoUEOGjbtm3q1KmTHnzwQZ04cULjxo2jkID/oZAAB/Xv39/7\n9alTpyq8oAO43lBIgAsSExP1+eefV3ifxa/U5geykYEMX+dnWZZVI+/sAn9/39doVPdl325r1aqV\n7VybNm18jgcEBNiuOXbsmM/xQ4cOVSmXkzp27Gg7969//cvBJN/dJ598oieffFIbNmyo8Gf0Wn8g\nGxnI8M31drjKDnBQVlaW9w7zHTp0kMfjUUFBgcupADNQSICD9uzZo5deekmSdObMGRUXFys0NNTl\nVIAZKCTAQYmJiSooKNDw4cM1ceJEzZ492/ZUM3C94aIGwEH16tXT4sWL3Y4BGIm/mgEAjFCrjpDs\nLhg0+Sq7ih75/cgjj/gcHz16tO2aZs2aVTnD2bNnfY6vW7fOds0LL7zgc/zw4cO2ay5evFi1YBXg\nQgCg9uEICQBgBAoJAGCEWnXKDqht/t/jb7kdAbjCxsX31dh7c4QEADAChQQAMAKn7ACHLVy4UB9/\n/LHKyso0adIk9e3b1+1IgBEopO+gXr16PscrepTA1KlTfY7/4he/sF3j1C1lQkJCfI6PHz/edo3d\nXHZ2tu2avLw8n+OrVq2yXbNr1y6f4xkZGbZrTLZjxw4dOXJEa9euVWFhoQYPHkwhAf9DIQEOuvPO\nO9W5c2dJUuPGjVVSUiKPx1Ph3diB6wWFBDgoICBA9evXl/TlPzy+5557KCNcc2rqeUgUEuCCd955\nR+vWrfPe+Ru4ltTU85AoJMBh77//vpYtW6YXX3xRjRq5//RPwBQUEuCg8+fPa+HChfrrX/+qJk2a\nuB0HMEqtKqT169f7HH/ooYds13Tv3t3n+MiRI23X2N0Q9Y477qgg3fUhMjKyymt69+5d/UEMtWnT\nJhUWFmratGnesQULFqhFixYupgLMUKsKCTBdQkKCEhIS3I4BGIk7NQAAjMAREmCwjYvv+15XNFWH\n5s0bkYEMjuAICQBgBAoJAGAECgkAYIRa9RnSoEGDqjQOADAHR0gAACNQSAAAI1BIgMMOHz6smJgY\nrV692u0ogFEoJMBBxcXFmjNnju6++263owDGoZAABwUFBWnFihUKDw93OwpgnFp1lR1gusDAQAUG\nVu1/u5p6GBoZyGBaBgoJMJzbt4ox4XY1ZKg9GSoqM07ZAQCMQCEBAIzAKTvAQVlZWVqwYIFOnDih\nwMBApaen67nnnuPpsYAoJMBRnTp1UkpKitsxACNxyg4AYAQKCQBgBAoJAGAECgkAYAQKCQBgBAoJ\nAGAECgkAYAT+HRLgsHnz5mn//v3y8/PTzJkz1blzZ7cjAUagkAAH7dq1S0ePHtXatWuVk5OjmTNn\nau3atW7HAozAKTvAQZmZmYqJiZEktW3bVmfPntWFCxdcTgWYgSMkwEFnzpxRZGSk9/uwsDDl5eWp\nYcOGtmtq8/NvyECGr6OQABdZlnXV11zrz78hAxm+ud4Op+wAB4WHh+vMmTPe70+fPq3mzZu7mAgw\nB4UEOKhHjx5KT0+XJGVnZys8PLzC03XA9YRTdoCDunbtqsjISCUmJsrPz09JSUluRwKMQSEBDnvi\niSfcjgAYiVN2AAAjUEgAACNQSAAAI1BIAAAjUEgAACNQSAAAI1BIAAAjUEgAACNQSAAAI/hZlbnd\nMAAANYwjJACAESgkAIARKCQAgBEoJACAESgkAIARKCQAgBEoJACAESgkwBDz5s1TQkKCEhMTdeDA\ngSvmPvroIw0ZMkQJCQlaunSp4/vv2LFDQ4cOVWJiop566imVl5c7nuErixcv1qhRo2pk/6tlOHXq\nlIYNG6YhQ4Zo9uzZrmR49dVXlZCQoGHDhmnu3Lk1lkGSDh8+rJiYGK1evfpbczXyM2kBcN3OnTut\niRMnWpZlWZ9++qk1dOjQK+b79etnnTx50vJ4PNawYcOsI0eOOLp/bGysderUKcuyLGvq1KnW9u3b\nq3X/ymSwLMs6cuSIlZCQYI0cObLa969MhkceecTasmWLZVmWlZycbJ04ccLRDOfPn7f69OljlZaW\nWpZlWWPHjrX27dtX7Rksy7IuXrxojRw50po1a5aVkpLyrfma+JnkCAkwQGZmpmJiYiRJbdu21dmz\nZ3XhwgVJ0rFjxxQSEqIbb7xR/v7+ioqKUmZmpmP7S1JqaqpuuOEGSVJYWJgKCwurdf/KZJCk+fPn\n67HHHqv2vSuToby8XB9//LGio6MlSUlJSWrRooWjGerUqaM6deqouLhYZWVlKikpUUhISLVnkKSg\noCCtWLFC4eHh35qrqZ9JCgkwwJkzZxQaGur9PiwsTHl5eZKkvLw8hYWF+ZxzYn9JatiwoSTp9OnT\n+vDDDxUVFVWt+1cmQ2pqqu666y7ddNNN1b53ZTIUFBSoQYMG+t3vfqdhw4Zp8eLFjmeoW7euHn74\nYcXExKhPnz768Y9/rNatW9dIjsDAQNWrV8/nXE39TFJIgIEsl28x6Wv//Px8TZ48WUlJSVf8gelE\nhqKiIqWmpmrs2LE1vq9dBsuylJubq9GjR2v16tU6ePCgtm/f7miGCxcuaPny5UpLS1NGRob279+v\nQ4cO1XgGp1BIgAHCw8N15swZ7/enT59W8+bNfc7l5ub6PI1SU/tLX/5B+OCDD2ratGnq2bNnte5d\nmQw7duxQQUGBRowYoSlTpig7O1vz5s1zNENoaKhatGihli1bKiAgQHfffbeOHDniaIacnBzdcsst\nCgsLU1BQkLp166asrKxqz1DVjNX1M0khAQbo0aOH0tPTJUnZ2dkKDw/3nia7+eabdeHCBR0/flxl\nZWXatm2bevTo4dj+0pef3YwZM0b33HNPte5b2Qzx8fHatGmTXn/9dT3//POKjIzUzJkzHc0QGBio\nW265RZ999pl3viZOl1WU4aabblJOTo4uXbokScrKylKrVq2qPcPV1NTPJI+fAAyxaNEi7dmzR35+\nfkpKStLBgwfVqFEjxcbGavfu3Vq0aJEkqW/fvho/frxj+/fs2VN33nmnunTp4n3twIEDlZCQ4FiG\n2NhY72uOHz+up556SikpKdW+/9UyHD16VDNmzJBlWWrXrp2Sk5Pl71/9f6+vKMNrr72m1NRUBQQE\nqEuXLnryySerfX/py7JbsGCBTpw4ocDAQEVERCg6Olo333xzjf1MUkgAACNwyg4AYAQKCQBgBAoJ\nAGAECgkAYAQKCQBgBAoJAGAECgkAYAQKCQBgBAoJAGCE/w+lO7zIVAKxpwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7faedc4c1080>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "oCA3MPO4q-Ki",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Autograd.** "
      ]
    },
    {
      "metadata": {
        "id": "A70GZm7ZrGIp",
        "colab_type": "code",
        "outputId": "964dabc6-9839-42fc-dc00-68751feed6ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "### some autograd examples\n",
        "\n",
        "x = torch.randn(2, 2, requires_grad=True)\n",
        "print(x)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.9551,  0.3717],\n",
            "        [-0.5094,  0.1403]], requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "6D4NCx-Pv2uY",
        "colab_type": "code",
        "outputId": "a176f9e8-36a9-49e3-db8c-9b44b8b26e98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "y = x**2\n",
        "print(y)  "
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.9122, 0.1381],\n",
            "        [0.2595, 0.0197]], grad_fn=<PowBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "f1U0rt6DyO3-",
        "colab_type": "code",
        "outputId": "c6a312b3-def9-40f2-d9d8-b6f23ffc78a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "### grad_fn shows the function that generated the 'y' variable\n",
        "\n",
        "print(y.grad_fn)"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<PowBackward0 object at 0x7fae901a15f8>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "0uzMWUkjzDF8",
        "colab_type": "code",
        "outputId": "e2a13c6f-98cd-4304-ebb6-e3df98a68ce6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "### scale the 'y' tensor to a scalar\n",
        "\n",
        "z = y.mean()\n",
        "print(z)"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.3324, grad_fn=<MeanBackward1>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "BaVQgjVSzpNG",
        "colab_type": "code",
        "outputId": "f3984467-d5ae-4a9c-d1d4-28ffed24b136",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "### check the gradient for 'x'\n",
        "\n",
        "print(x.grad)"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LGR9r9_10ANc",
        "colab_type": "code",
        "outputId": "08775207-f886-43aa-ce66-597faef36f22",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "cell_type": "code",
      "source": [
        "### calculate the gradient for 'z' with respect to 'x'\n",
        "\n",
        "z.backward()\n",
        "print(x.grad)\n",
        "print(x/2) "
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.4776,  0.1858],\n",
            "        [-0.2547,  0.0702]])\n",
            "tensor([[ 0.4776,  0.1858],\n",
            "        [-0.2547,  0.0702]], grad_fn=<DivBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}