D2L_Conv_Basics.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "D2L_Conv_Basics.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPqjZY3qIYoM7eCRfbavsqu",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rajlm10/D2L-Torch/blob/main/D2L_Conv_Basics.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 106,
      "metadata": {
        "id": "cOPup5g6Ta4i"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "from torch.utils import data\n",
        "import torchvision\n",
        "from torchvision import transforms"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Basic Operations in a CNN "
      ],
      "metadata": {
        "id": "_v8mglcOTixG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def corr2d(X,K):\n",
        "  h,w=K.shape\n",
        "  Y=torch.zeros(X.shape[0]-h+1,X.shape[1]-w+1)\n",
        "\n",
        "  for i in range(Y.shape[0]):\n",
        "    for j in range(Y.shape[1]):\n",
        "      Y[i,j]=(X[i:i+h,j:j+w]*K).sum()\n",
        "\n",
        "  return Y"
      ],
      "metadata": {
        "id": "M7yi7dRcTlzA"
      },
      "execution_count": 107,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = torch.tensor([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]) \n",
        "K = torch.tensor([[0.0, 1.0], [2.0, 3.0]])\n",
        "print(X.shape,K.shape)\n",
        "corr2d(X, K)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "euH7WmrpULJe",
        "outputId": "943da6c0-50bc-421a-c649-ed7ec5976114"
      },
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([3, 3]) torch.Size([2, 2])\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[19., 25.],\n",
              "        [37., 43.]])"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Conv2D(nn.Module):\n",
        "  def __init__(self,kernel_size):\n",
        "    super().__init__()\n",
        "    self.weight=nn.Parameter(torch.rand(kernel_size))\n",
        "    self.bias=nn.Parameter(torch.zeros(1))\n",
        "\n",
        "  def forward(self,X):\n",
        "    return corr2d(X,self.weight)+self.bias"
      ],
      "metadata": {
        "id": "ltv7TUaZUkGa"
      },
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = torch.ones((6, 8)) \n",
        "X[:, 2:6] = 0\n",
        "X"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qc2wpV9RVoTi",
        "outputId": "1f89470b-b639-48a2-8b8d-b8b9dbe8077a"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1., 0., 0., 0., 0., 1., 1.],\n",
              "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
              "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
              "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
              "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
              "        [1., 1., 0., 0., 0., 0., 1., 1.]])"
            ]
          },
          "metadata": {},
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "K = torch.tensor([[1.0, -1.0]])"
      ],
      "metadata": {
        "id": "Th0otQYNWz94"
      },
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Y = corr2d(X, K)\n",
        "Y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BrHjMODaVp1N",
        "outputId": "72b80144-6755-46e6-8123-823ebfa8f8e1"
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
              "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
              "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
              "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
              "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
              "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Learning a kernel through gradient descent\n",
        "\n",
        "# Construct a two-dimensional convolutional layer with 1 input channel,1 output channel and a kernel of shape (1, 2). \n",
        "#For the sake of simplicity, we ignore the bias here \n",
        "conv2d = nn.Conv2d(1,1, kernel_size=(1, 2), bias=False)\n",
        "# The two-dimensional convolutional layer uses four-dimensional input and\n",
        "# output in the format of (num_example, channel, height, width), where the batch\n",
        "# size (number of examples in the batch) and the number of channels are both 1 \n",
        "X = X.reshape((1, 1, 6, 8))\n",
        "Y = Y.reshape((1, 1, 6, 7))\n",
        "lr = 3e-2 # Learning rate\n",
        "\n",
        "\n",
        "for i in range(10):\n",
        "  Y_hat=conv2d(X)\n",
        "  l=(Y_hat-Y)**2\n",
        "  conv2d.zero_grad()\n",
        "  l.sum().backward()\n",
        "\n",
        "  # Update the kernel\n",
        "  conv2d.weight.data[:] -= lr * conv2d.weight.grad \n",
        "  print(f'epoch {i + 1}, loss {l.sum():.3f}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lai1A6oXVZos",
        "outputId": "e54d3742-9500-47da-be76-d6a78571245c"
      },
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch 1, loss 27.388\n",
            "epoch 2, loss 13.092\n",
            "epoch 3, loss 6.561\n",
            "epoch 4, loss 3.455\n",
            "epoch 5, loss 1.906\n",
            "epoch 6, loss 1.095\n",
            "epoch 7, loss 0.650\n",
            "epoch 8, loss 0.395\n",
            "epoch 9, loss 0.244\n",
            "epoch 10, loss 0.153\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def corr2d_multi_in(X, K):\n",
        "  # First, iterate through the 0th dimension (channel dimension) of `X` and # `K`. Then, add them together\n",
        "  #Zip iterates through first channel\n",
        "  return sum(corr2d(x, k) for x, k in zip(X, K))"
      ],
      "metadata": {
        "id": "VRet2XuCX9XT"
      },
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = torch.tensor([[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]], [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]])\n",
        "K = torch.tensor([[[0.0, 1.0], [2.0, 3.0]], [[1.0, 2.0], [3.0, 4.0]]])\n",
        "print(X.shape,K.shape) #Note X and K shapes\n",
        "corr2d_multi_in(X, K)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A0p1VvnfW6lJ",
        "outputId": "8dac2dc6-1dbe-4207-d2b6-7549b0840dd6"
      },
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([2, 3, 3]) torch.Size([2, 2, 2])\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 56.,  72.],\n",
              "        [104., 120.]])"
            ]
          },
          "metadata": {},
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def corr2d_multi_in_out(X, K):\n",
        "  # Iterate through the 0th dimension of `K`, and each time, perform\n",
        "  # cross-correlation operations with input `X`. All of the results are stacked together\n",
        "  return torch.stack([corr2d_multi_in(X, k) for k in K], 0)"
      ],
      "metadata": {
        "id": "vrDSOed4YtW9"
      },
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "K = torch.stack((K, K + 1, K + 2), 0) \n",
        "K.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zc0NfqgxZNDa",
        "outputId": "85372fd6-6a2a-4ff4-8bda-ebd00b375d06"
      },
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([3, 2, 2, 2])"
            ]
          },
          "metadata": {},
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "corr2d_multi_in_out(X, K)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LlaGdk0VZSTc",
        "outputId": "94304994-83db-4b01-9f07-bbc3791536e2"
      },
      "execution_count": 118,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[ 56.,  72.],\n",
              "         [104., 120.]],\n",
              "\n",
              "        [[ 76., 100.],\n",
              "         [148., 172.]],\n",
              "\n",
              "        [[ 96., 128.],\n",
              "         [192., 224.]]])"
            ]
          },
          "metadata": {},
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#1X1 convolution\n",
        "X = torch.normal(0, 1, (3, 3, 3)) # C X H X W\n",
        "K = torch.normal(0, 1, (2, 3, 1, 1)) # N X C X Hk X Wk\n",
        "\n",
        "corr2d_multi_in_out(X, K),corr2d_multi_in_out(X, K).shape\n",
        "\n",
        "#Reduced depth from 3 to 2 spatial dimensions unchanged"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-XMqqWFTZoNv",
        "outputId": "48e9c7ec-f487-4a4a-8528-2348efdb7e8d"
      },
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[[-4.7996, -2.9807, -1.5040],\n",
              "          [-0.3910,  0.8925,  0.9623],\n",
              "          [-2.1591, -1.5493,  1.4726]],\n",
              " \n",
              "         [[ 2.6735,  0.6189, -0.7479],\n",
              "          [-0.0580, -0.2176, -1.2712],\n",
              "          [ 0.6884,  0.3185, -0.5092]]]), torch.Size([2, 3, 3]))"
            ]
          },
          "metadata": {},
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Pooling\n",
        "\n",
        "def pool2d(X, pool_size, mode='max'):\n",
        "  h, w = pool_size\n",
        "  Y = torch.zeros((X.shape[0] - h + 1, X.shape[1] - w + 1)) \n",
        "  for i in range(Y.shape[0]):\n",
        "    for j in range(Y.shape[1]): \n",
        "      if mode == 'max':\n",
        "        Y[i, j] = X[i: i + h, j: j + w].max() \n",
        "      elif mode == 'avg':\n",
        "        Y[i,j]=X[i:i+h,j:j+w].mean()\n",
        "\n",
        "  return Y"
      ],
      "metadata": {
        "id": "CwnXxDJ0aCUt"
      },
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = torch.tensor([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]) \n",
        "print(X.shape)\n",
        "pool2d(X, (2, 2))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-PYfk6i5a2sb",
        "outputId": "652accee-e354-41c2-db44-ea25dc66dc1a"
      },
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([3, 3])\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[4., 5.],\n",
              "        [7., 8.]])"
            ]
          },
          "metadata": {},
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "When processing multi-channel input data, the pooling layer pools each input channel separately, rather than summing the inputs up over channels as in a convolutional layer. This means that the number of output channels for the pooling layer is the same as the number of input channels."
      ],
      "metadata": {
        "id": "_CRuNajBbZIP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# LeNet "
      ],
      "metadata": {
        "id": "F-IzX0gfbfIZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "net = nn.Sequential(\n",
        "  nn.Conv2d(1, 6, kernel_size=5, padding=2), #in channels, out channels\n",
        "  nn.Sigmoid(), \n",
        "  nn.AvgPool2d(kernel_size=2, stride=2),\n",
        "  nn.Conv2d(6, 16, kernel_size=5), \n",
        "  nn.Sigmoid(), \n",
        "  nn.AvgPool2d(kernel_size=2, stride=2),\n",
        "  nn.Flatten(),\n",
        "  nn.Linear(16 * 5 * 5, 120), \n",
        "  nn.Sigmoid(),\n",
        "  nn.Linear(120, 84), \n",
        "  nn.Sigmoid(),\n",
        "  nn.Linear(84, 10))"
      ],
      "metadata": {
        "id": "lAkmPL3lbg9r"
      },
      "execution_count": 122,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = torch.rand(size=(1, 1, 28, 28), dtype=torch.float32) \n",
        "for layer in net: #Iterate through nets layers\n",
        "  X = layer(X)\n",
        "  print(layer.__class__.__name__,'output shape: \\t',X.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ez1gVrz9b-lx",
        "outputId": "5fe1cbe5-87a8-4786-83b3-9cb1589b49f3"
      },
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Conv2d output shape: \t torch.Size([1, 6, 28, 28])\n",
            "Sigmoid output shape: \t torch.Size([1, 6, 28, 28])\n",
            "AvgPool2d output shape: \t torch.Size([1, 6, 14, 14])\n",
            "Conv2d output shape: \t torch.Size([1, 16, 10, 10])\n",
            "Sigmoid output shape: \t torch.Size([1, 16, 10, 10])\n",
            "AvgPool2d output shape: \t torch.Size([1, 16, 5, 5])\n",
            "Flatten output shape: \t torch.Size([1, 400])\n",
            "Linear output shape: \t torch.Size([1, 120])\n",
            "Sigmoid output shape: \t torch.Size([1, 120])\n",
            "Linear output shape: \t torch.Size([1, 84])\n",
            "Sigmoid output shape: \t torch.Size([1, 84])\n",
            "Linear output shape: \t torch.Size([1, 10])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import multiprocessing\n",
        "def get_workers():\n",
        "  return multiprocessing.cpu_count()"
      ],
      "metadata": {
        "id": "8KC8H2DzdCgk"
      },
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Accumulator: \n",
        "  \"\"\"For accumulating sums over `n` variables.\"\"\" \n",
        "  def __init__(self, n):\n",
        "    self.data = [0.0] * n \n",
        "    \n",
        "  def add(self, *args):\n",
        "    self.data = [a + float(b) for a, b in zip(self.data, args)] \n",
        "    \n",
        "  def reset(self):\n",
        "    self.data = [0.0] * len(self.data)\n",
        "  \n",
        "  def __getitem__(self, idx): \n",
        "    return self.data[idx]"
      ],
      "metadata": {
        "id": "Wna7nqFcdmuK"
      },
      "execution_count": 125,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 256\n",
        "\n",
        "def load_fashion_mnist(batch_size,resize=None):\n",
        "  trans=[transforms.ToTensor()]\n",
        "  if resize:\n",
        "    trans.insert(0,transforms.Resize(resize))\n",
        "  trans=transforms.Compose(trans)\n",
        "\n",
        "  mnist_train=torchvision.datasets.FashionMNIST(root=\"../data\", train=True, transform=trans, download=True)\n",
        "  mnist_test=torchvision.datasets.FashionMNIST(root=\"../data\", train=False, transform=trans, download=True)\n",
        "\n",
        "  return data.DataLoader(mnist_train,batch_size,shuffle=True,num_workers=get_workers()),data.DataLoader(mnist_test,batch_size,shuffle=False,num_workers=get_workers())\n",
        "\n",
        "train_iter, test_iter = load_fashion_mnist(batch_size=batch_size)"
      ],
      "metadata": {
        "id": "K_b8s8AkcLyE"
      },
      "execution_count": 126,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def accuracy(y_hat,y):\n",
        "  \"\"\"Compute the number of correct predictions.\"\"\" \n",
        "  if y_hat.shape[0]>1 and y_hat.shape[1]>1:\n",
        "    y_hat=y_hat.argmax(axis=1)\n",
        "  cmp=y_hat.type(y.dtype)==y\n",
        "  return float(cmp.type(y.dtype).sum())"
      ],
      "metadata": {
        "id": "rDnglGT3ecLp"
      },
      "execution_count": 127,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_accuracy_gpu(net,test_iter,device=None):\n",
        "  \"\"\"Compute the accuracy for a model on a dataset using a GPU.\"\"\"\n",
        "  if isinstance (net,nn.Module):\n",
        "    net.eval()\n",
        "    if not device:\n",
        "      device=next(iter(net.parameters())).device\n",
        "\n",
        "    # No. of correct predictions, no. of predictions \n",
        "    metric =Accumulator(2)\n",
        "\n",
        "    with torch.no_grad():\n",
        "      for X,y in test_iter:\n",
        "        if isinstance(X, list):\n",
        "          # Required for BERT Fine-tuning (to be covered later) \n",
        "          X = [x.to(device) for x in X]\n",
        "\n",
        "        else:\n",
        "          X=X.to(device)\n",
        "        \n",
        "        y=y.to(device)\n",
        "        metric.add(accuracy(net(X), y), y.numel())\n",
        "      \n",
        "    return metric[0] / metric[1]\n",
        "\n",
        "  "
      ],
      "metadata": {
        "id": "bFmN525GdR_y"
      },
      "execution_count": 128,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train(net,train_iter,test_iter,loss,optimizer,num_epochs,learning_rate,device):\n",
        "  def init_weights(layer):\n",
        "    if isinstance(layer,nn.Linear) or isinstance(layer,nn.Conv2d):\n",
        "      nn.init.xavier_uniform_(layer.weight)\n",
        "  net.apply(init_weights)\n",
        "  print('training on', device)\n",
        "  net.to(device)\n",
        "\n",
        "  for epoch in range(num_epochs):\n",
        "    metric=Accumulator(3) # Sum of training loss, sum of training accuracy, no. of examples\n",
        "    net.train()\n",
        "\n",
        "    for X,y in train_iter:\n",
        "      X,y=X.to(device), y.to(device)\n",
        "      optimizer.zero_grad()\n",
        "      y_hat=net(X)\n",
        "      l=loss(y_hat,y)\n",
        "      l.backward()\n",
        "      optimizer.step()\n",
        "\n",
        "      with torch.no_grad():\n",
        "        metric.add(l * X.shape[0], accuracy(y_hat, y), X.shape[0])\n",
        "\n",
        "      train_l = metric[0] / metric[2]\n",
        "      train_acc = metric[1] / metric[2]\n",
        "\n",
        "    test_acc = evaluate_accuracy_gpu(net, test_iter)\n",
        "\n",
        "    print(f'epcoch:{epoch+1} 'f'loss {train_l:.3f}, train acc {train_acc:.3f},'f'test acc {test_acc:.3f}')\n",
        "  \n",
        "\n"
      ],
      "metadata": {
        "id": "fQeo-k1hekRa"
      },
      "execution_count": 129,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def try_gpu(i=0):\n",
        "  \"\"\"Return gpu(i) if exists, otherwise return cpu().\"\"\" \n",
        "  if torch.cuda.device_count() >= i + 1:\n",
        "    return torch.device(f'cuda:{i}') \n",
        "  return torch.device('cpu')"
      ],
      "metadata": {
        "id": "vV62GNFvgupa"
      },
      "execution_count": 130,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lr, num_epochs = 0.9, 10\n",
        "loss=nn.CrossEntropyLoss()\n",
        "optimizer=torch.optim.SGD(net.parameters(),lr)\n",
        "_=train(net, train_iter, test_iter,loss,optimizer, num_epochs, lr, try_gpu())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TFfzXnBjgVcT",
        "outputId": "bd5a2d0e-46f8-45a5-fee7-69b82123a4ae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training on cuda:0\n"
          ]
        }
      ]
    }
  ]
}