00_pytorch_fundamentals.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## 00. PyTorch Fundamentals\n",
        "\n",
        "Resource notebook: https://www.learnpytorch.io/00_pytorch_fundamentals/"
      ],
      "metadata": {
        "id": "u3a27Ly0oI-R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "print(torch.__version__)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Ff0ebOGoVEd",
        "outputId": "c00a455a-f10d-46a8-90b5-c206b46702c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.4.0+cu121\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Introduction to Tensors\n",
        "\n",
        "### Creating tensors\n",
        "\n",
        "Pytorch tensors are created using `torch.Tensor()`\n"
      ],
      "metadata": {
        "id": "TdQBkQygqVzB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# scalar\n",
        "scalar = torch.tensor(7)\n",
        "scalar"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XfhT9MUYrDqA",
        "outputId": "616ea7fd-73c6-46b0-b6dd-34b767be03b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(7)"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scalar.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bdf4TOQor6Jo",
        "outputId": "d4fbec5e-d02d-4ec6-c936-9993d9bc786b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get tensor back as Python int\n",
        "scalar.item()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7CLUiWikr72J",
        "outputId": "e046efef-5978-47de-e1e2-e6941a80629b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Vector\n",
        "vector = torch.tensor([7,7])\n",
        "vector"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-2Qg9AZIr_7s",
        "outputId": "f7f329b6-ff62-4bf2-911f-e7a657f48fca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([7, 7])"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vector.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HdT4FPgSseJ8",
        "outputId": "6d16bd74-c4b1-4250-814b-e2925e133a75"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vector.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q5_0uzlRsmiY",
        "outputId": "365250c1-bf84-4f27-a288-edbd1ab9440a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# MATRIX\n",
        "MATRIX = torch.tensor([[7,8],\n",
        "                       [9,10]])\n",
        "MATRIX"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oog1oVdPstUP",
        "outputId": "3e7f30d1-6922-4dd2-9ac6-a64d5640e262"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 7,  8],\n",
              "        [ 9, 10]])"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MATRIX.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oHvCWUKvs3ER",
        "outputId": "b9735c0a-5f7f-4d3f-f15a-2f77fd4cdfce"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MATRIX[0]\n",
        "MATRIX[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S7O5RdC1tGbO",
        "outputId": "fbd08c8b-622a-4518-9021-dcf678554739"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 9, 10])"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MATRIX.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4KnyosXDtR-R",
        "outputId": "8e1c00d4-40ec-4c71-a084-fa24a026092f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 2])"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TENSOR = torch.tensor([[[1,2,3],\n",
        "                        [3,6,9],\n",
        "                        [2,4,5]],[[3,5,6],\n",
        "                                  [4,6,7],\n",
        "                                  [2,4,6]]])\n",
        "TENSOR"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2OcIQGeltctj",
        "outputId": "8bb08125-1815-4d7d-b8a1-7d2b664db976"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[1, 2, 3],\n",
              "         [3, 6, 9],\n",
              "         [2, 4, 5]],\n",
              "\n",
              "        [[3, 5, 6],\n",
              "         [4, 6, 7],\n",
              "         [2, 4, 6]]])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TENSOR.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-oJipj1ktvnu",
        "outputId": "b3c362fb-835a-43ce-90aa-984690506288"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TENSOR.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4goDr7zltx2L",
        "outputId": "cd7146bb-c4a8-4b6f-a8ff-3e678f867106"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 3, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TENSOR[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bNxBU7LDt6Wu",
        "outputId": "9f6b45f7-e214-4b22-dd98-832ca8f05c83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 2, 3],\n",
              "        [3, 6, 9],\n",
              "        [2, 4, 5]])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Random Tensors\n",
        "\n",
        "Why random tensors?\n",
        "\n",
        "Random tensors are important becuase the way many neural netowrks learn is that they start with tensors full of random numbers and then adjust those random numbers to better represent the data.\n",
        "\n",
        "`Start with random numbers -> look at data -> update random numbers -> look at data -> update random numbers`"
      ],
      "metadata": {
        "id": "tm_jLSX4MPMU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a random tensor of size (3, 4)\n",
        "random_tensor = torch.rand(2,4)\n",
        "random_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HN4F2UaDMU7t",
        "outputId": "2e9da6ac-cb64-4485-a4bb-40d6df8921b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.5222, 0.0654, 0.9544, 0.2479],\n",
              "        [0.3610, 0.3384, 0.6869, 0.4256]])"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a random tensor with similar shape to an image tensor\n",
        "random_image_size_tensor = torch.rand(size = (224, 224, 3)) # height, width, color channels (R, G, B)\n",
        "random_image_size_tensor.shape, random_image_size_tensor.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cJxctnBMNACx",
        "outputId": "dcf2e1e5-2592-485a-905b-4b82533b566b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([224, 224, 3]), 3)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_tensor_temp = torch.rand(size = (2,4,6,7))\n",
        "random_tensor_temp"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QQin0Yt8NDym",
        "outputId": "50bca49a-42b0-4dad-8711-817f5849318e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[[0.1645, 0.2004, 0.1237, 0.4482, 0.8988, 0.9859, 0.6788],\n",
              "          [0.9459, 0.0237, 0.2481, 0.0721, 0.4401, 0.1386, 0.9733],\n",
              "          [0.1241, 0.8595, 0.2326, 0.8207, 0.2614, 0.7442, 0.6392],\n",
              "          [0.9975, 0.1292, 0.0580, 0.2057, 0.2199, 0.9383, 0.8959],\n",
              "          [0.7447, 0.5390, 0.7116, 0.4016, 0.3766, 0.5383, 0.6536],\n",
              "          [0.3086, 0.7682, 0.7715, 0.8636, 0.5569, 0.1051, 0.0755]],\n",
              "\n",
              "         [[0.7126, 0.7299, 0.1840, 0.2318, 0.1786, 0.7649, 0.9337],\n",
              "          [0.9214, 0.1463, 0.5563, 0.0884, 0.3733, 0.7470, 0.5292],\n",
              "          [0.7005, 0.9911, 0.8825, 0.9082, 0.5545, 0.8701, 0.8869],\n",
              "          [0.0185, 0.7404, 0.3455, 0.6742, 0.0434, 0.0369, 0.4428],\n",
              "          [0.4978, 0.3958, 0.3516, 0.9215, 0.1216, 0.6558, 0.4472],\n",
              "          [0.4549, 0.0552, 0.6733, 0.0426, 0.4889, 0.7503, 0.7173]],\n",
              "\n",
              "         [[0.1303, 0.3371, 0.3002, 0.1375, 0.3011, 0.4520, 0.4527],\n",
              "          [0.0532, 0.1455, 0.9816, 0.3909, 0.3771, 0.3441, 0.9606],\n",
              "          [0.4934, 0.1514, 0.0125, 0.1216, 0.3226, 0.3286, 0.7916],\n",
              "          [0.0890, 0.0804, 0.0217, 0.9443, 0.2898, 0.9169, 0.0039],\n",
              "          [0.9638, 0.2247, 0.2747, 0.9089, 0.5404, 0.4290, 0.2423],\n",
              "          [0.6910, 0.3245, 0.9792, 0.4681, 0.0128, 0.4988, 0.1152]],\n",
              "\n",
              "         [[0.4741, 0.1927, 0.7146, 0.0758, 0.4990, 0.2461, 0.1033],\n",
              "          [0.0229, 0.1383, 0.6753, 0.2575, 0.9011, 0.0881, 0.7145],\n",
              "          [0.7303, 0.6769, 0.9578, 0.1717, 0.2794, 0.6316, 0.4850],\n",
              "          [0.6878, 0.6097, 0.1869, 0.3343, 0.8840, 0.1108, 0.7414],\n",
              "          [0.2852, 0.4751, 0.0886, 0.6423, 0.4556, 0.3601, 0.6891],\n",
              "          [0.2301, 0.4032, 0.3186, 0.7682, 0.2835, 0.5716, 0.9631]]],\n",
              "\n",
              "\n",
              "        [[[0.0176, 0.1450, 0.7647, 0.0434, 0.6390, 0.2906, 0.2898],\n",
              "          [0.2490, 0.4800, 0.1045, 0.1480, 0.5310, 0.8453, 0.6588],\n",
              "          [0.4574, 0.5750, 0.6209, 0.0171, 0.0157, 0.5927, 0.8351],\n",
              "          [0.5217, 0.4211, 0.6852, 0.7252, 0.2118, 0.6337, 0.9453],\n",
              "          [0.5226, 0.4776, 0.8745, 0.6196, 0.9848, 0.8338, 0.2892],\n",
              "          [0.0289, 0.3322, 0.2795, 0.5201, 0.1268, 0.9370, 0.2265]],\n",
              "\n",
              "         [[0.5006, 0.4596, 0.2959, 0.2756, 0.0211, 0.9171, 0.4560],\n",
              "          [0.7336, 0.9319, 0.3099, 0.2862, 0.6953, 0.3133, 0.6759],\n",
              "          [0.6567, 0.2153, 0.1472, 0.4406, 0.0732, 0.3148, 0.3773],\n",
              "          [0.6968, 0.3130, 0.7178, 0.7491, 0.3259, 0.4325, 0.6071],\n",
              "          [0.3789, 0.0069, 0.4679, 0.8263, 0.2294, 0.8834, 0.6477],\n",
              "          [0.8680, 0.3158, 0.0921, 0.8895, 0.2342, 0.6909, 0.7448]],\n",
              "\n",
              "         [[0.3059, 0.8731, 0.6264, 0.3670, 0.6766, 0.8135, 0.6341],\n",
              "          [0.8676, 0.6939, 0.2825, 0.1287, 0.1362, 0.9117, 0.1568],\n",
              "          [0.1111, 0.1453, 0.5139, 0.1186, 0.1247, 0.5367, 0.1202],\n",
              "          [0.8936, 0.3316, 0.4546, 0.9844, 0.3818, 0.3286, 0.7020],\n",
              "          [0.3515, 0.5215, 0.3827, 0.1545, 0.1806, 0.9007, 0.0474],\n",
              "          [0.8507, 0.3473, 0.3122, 0.0650, 0.1020, 0.7390, 0.3091]],\n",
              "\n",
              "         [[0.5632, 0.1250, 0.4770, 0.5036, 0.3100, 0.3397, 0.5024],\n",
              "          [0.7976, 0.6618, 0.8523, 0.4382, 0.5444, 0.2665, 0.0769],\n",
              "          [0.8388, 0.4528, 0.0075, 0.2819, 0.2188, 0.1493, 0.2120],\n",
              "          [0.7516, 0.4173, 0.3001, 0.7973, 0.4302, 0.6429, 0.6691],\n",
              "          [0.2416, 0.6981, 0.1381, 0.7137, 0.3885, 0.2789, 0.5899],\n",
              "          [0.9845, 0.8037, 0.4376, 0.7211, 0.7087, 0.5122, 0.2535]]]])"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Zeros and ones"
      ],
      "metadata": {
        "id": "cSsnhRxPNDto"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor of all zeros (useful for masking a column)\n",
        "zeros = torch.zeros(size = (2,4))\n",
        "zeros"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cRnUEEryNDpK",
        "outputId": "00282f91-fd2f-4f1e-fc83-30170a6c0315"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "zeros * random_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kDV0S9tINDoB",
        "outputId": "8bf4fe13-6081-41aa-c3f5-0ad858c59a60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Creates a tensor of all ones\n",
        "ones = torch.ones(size = (3,4))\n",
        "ones"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RAFmn-sWNDis",
        "outputId": "fe36499c-9d23-437b-8673-d43b1eb6458e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1., 1., 1.],\n",
              "        [1., 1., 1., 1.],\n",
              "        [1., 1., 1., 1.]])"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ones.dtype # data type"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ew2aU7NDNDW_",
        "outputId": "46c925e3-b7b1-4c83-d81c-8cbd385fb48d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float32"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_tensor.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QSH34wXpNDHX",
        "outputId": "863d90bd-aea4-4b3d-f730-bc400b900196"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float32"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Creating a range of tensors and tensors-like"
      ],
      "metadata": {
        "id": "CUYuY90kVwLv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Use torch.range() and get depcrecated msg, use torch.arange() instead\n",
        "one_to_ten = torch.arange(start = 1,end = 11, step = 1)\n",
        "one_to_ten"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BmaAVO5RV1TV",
        "outputId": "ebbedd14-acc8-4415-fc5c-235daaf99642"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10])"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating tensors like (create the same tensor size)\n",
        "ten_zeros = torch.zeros_like(input = one_to_ten)\n",
        "ten_zeros"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OH__el3BWeFK",
        "outputId": "52730af0-60e0-4cd1-d076-2582d6e964d7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Tensor datatypes\n",
        "\n",
        "**Note:** Tensor datatypes is one of the 3 big errors you will run into with PyTorch & deep learning\n",
        "\n",
        "1. Tensors not right datatype (might use float 16 with float 32 and get error)\n",
        "2. Tensors not right shape (shapes must match)\n",
        "3. Tensors not on the right device (might do operations on two tensors on diff devices)"
      ],
      "metadata": {
        "id": "P0zPVIIoV25c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Precision of number is a measure of the detail in how it's expressed in memory\n",
        "# Float 32 tensor; 32 bits (single precision)\n",
        "float_32_tensor = torch.tensor([3.0,6.0,9.0],dtype = None)\n",
        "float_32_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9f1SiE8oV24O",
        "outputId": "423842ef-ffdc-465e-d8ae-be61d796ab54"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3., 6., 9.])"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_32_tensor.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F2Z5hd4aV227",
        "outputId": "892a170b-f43f-49c2-c73f-588a321e746e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float32"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Float 16 tensor; 16 bits (half precision); use if can use less space to compute faster\n",
        "\n",
        "float_16_tensor = torch.tensor([3.0,6.0,9.0],dtype = torch.float16)\n",
        "float_16_tensor.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TGhHCgIuV2wy",
        "outputId": "d984b8ba-cbdd-495a-eb79-4a269e427943"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float16"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_32_tensor = torch.tensor([3.0,6.0,9.0],\n",
        "                               dtype = None, # what datatype is the tensor (e.g., float32 or float16)\n",
        "                               device = None, # what device is your tensor on (e.g., cpu, gpu)\n",
        "                               requires_grad = False) # if you want PyTorch to track gradients with this tensor's operations\n",
        "float_32_tensor"
      ],
      "metadata": {
        "id": "aF9YiYM6V2r4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dcd3e26d-b728-444f-c9e1-11719dfebcd3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3., 6., 9.])"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_16_tensor = float_32_tensor.type(torch.float16)\n",
        "float_16_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_DXOL5NOV2qg",
        "outputId": "7e36b79e-6d7b-4c0a-c872-3e8acdfc8fd7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3., 6., 9.], dtype=torch.float16)"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_16_tensor * float_32_tensor # sometimes it works -- just be aware of diff datatypes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k2CptO5XV2mi",
        "outputId": "7ac33df7-969e-4cb2-92fb-990b27dd799a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 9., 36., 81.])"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "int_32_tensor = torch.tensor([3, 6, 9], dtype = torch.int32)\n",
        "int_32_tensor"
      ],
      "metadata": {
        "id": "3JQgRFezV2lO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3409293e-53bc-4899-a3d5-5cc507c35247"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3, 6, 9], dtype=torch.int32)"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_32_tensor * int_32_tensor"
      ],
      "metadata": {
        "id": "bd_DyO_fV2hR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93dcfe44-8c52-4e9b-c2ad-b0037332fc72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 9., 36., 81.])"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Getting information from Tensors (tensor attributes)\n",
        "\n",
        "1. Tensors not right datatype - to do get datatype from a tensor, can use `tensor.dtype`\n",
        "2. Tensors not right shape - to get shape from a tensor, can use `tensor.shape`\n",
        "3. Tensors not on the right device - to get device from a tensor, can use `tensor.device`"
      ],
      "metadata": {
        "id": "Kxx-7ctpV2dN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor\n",
        "some_tensor = torch.rand(3, 4)\n",
        "some_tensor"
      ],
      "metadata": {
        "id": "Urks9b28V2V9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6efa0232-e446-4d6d-8c9c-7bd9ec4b5837"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.8363, 0.2137, 0.0454, 0.1175],\n",
              "        [0.3148, 0.0681, 0.1867, 0.0586],\n",
              "        [0.6771, 0.3164, 0.7980, 0.2300]])"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find out details about some tensor\n",
        "print(some_tensor)\n",
        "print(f'Datatype of tensor: {some_tensor.dtype}')\n",
        "print(f'Shape of tensor: {some_tensor.shape} or {some_tensor.size()}')\n",
        "print(f'Device tensor is on: {some_tensor.device}')"
      ],
      "metadata": {
        "id": "azCpIx4OV2TS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c251b95c-119f-4151-af92-582835ed1c0f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.8363, 0.2137, 0.0454, 0.1175],\n",
            "        [0.3148, 0.0681, 0.1867, 0.0586],\n",
            "        [0.6771, 0.3164, 0.7980, 0.2300]])\n",
            "Datatype of tensor: torch.float32\n",
            "Shape of tensor: torch.Size([3, 4]) or torch.Size([3, 4])\n",
            "Device tensor is on: cpu\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_tensor1 = torch.rand(3,4, dtype = torch.float16, device = 'cuda')\n",
        "print(random_tensor)\n",
        "print(f'Datatype of tensor: {random_tensor.dtype}')\n",
        "print(f'Shape of tensor: {random_tensor.shape} or {random_tensor.size()}')\n",
        "print(f'Device tensor is on: {random_tensor.device}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 358
        },
        "id": "7eVTi8cvXBDe",
        "outputId": "433c1c35-b43b-4354-8292-22a1d41d1502"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-38-b2935130302e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrandom_tensor1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat16\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'cuda'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Datatype of tensor: {random_tensor.dtype}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Shape of tensor: {random_tensor.shape} or {random_tensor.size()}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Device tensor is on: {random_tensor.device}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/cuda/__init__.py\u001b[0m in \u001b[0;36m_lazy_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m    312\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m\"CUDA_MODULE_LOADING\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m             \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"CUDA_MODULE_LOADING\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"LAZY\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 314\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cuda_init\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    315\u001b[0m         \u001b[0;31m# Some of the queued calls may reentrantly call _lazy_init();\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    316\u001b[0m         \u001b[0;31m# we need to just return without initializing in that case.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Manipulating Tensors (tensor operations)\n",
        "\n",
        "Tensor operations include:\n",
        "* Addition\n",
        "* Subtraction\n",
        "* Multiplication (element-wise)\n",
        "* Division\n",
        "* Matrix multiplication"
      ],
      "metadata": {
        "id": "XXChXAULX35-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor\n",
        "tensor = torch.tensor([1, 2, 3])\n",
        "tensor + 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9E4glKBRYL18",
        "outputId": "ef318659-8c3d-4479-a8e7-815d0f6713ac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([11, 12, 13])"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tensor = torch.tensor([1, 2, 3])\n",
        "tensor + 100"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "31JU1z7tYajF",
        "outputId": "4b523343-55fe-4f4f-f362-1148d82ea998"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([101, 102, 103])"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Multiply tensor by 10\n",
        "tensor * 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qWXxSpNDYdDp",
        "outputId": "d61d4348-199b-4f51-c7e9-4784a157f27c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([10, 20, 30])"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Subtract 10\n",
        "tensor - 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a-hOGiazYsGc",
        "outputId": "c726840d-daa6-4f69-b4e3-2b17ed4ad3a8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([-9, -8, -7])"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Try out PyTorch in-built functions\n",
        "torch.mul(tensor, 10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gghvhp4cYw0b",
        "outputId": "86e3b195-2cd0-4269-9c3a-1480351047ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([10, 20, 30])"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.add(tensor, 10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2vBZZ7YpYwyV",
        "outputId": "d89d97eb-ad60-4cc6-9664-9a7d0b50f2be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([11, 12, 13])"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Matrix multiplication\n",
        "\n",
        "Two main ways of performing multiplication in neural networks and deep learning:\n",
        "\n",
        "1. Element-wise multiplication\n",
        "2. Matrix multiplication (dot product)\n",
        "\n",
        "There are two main rules that performing matrix multiplication needs to satisfy:\n",
        "1. The **inner dimensions** must match:\n",
        "* `(3,2) @ (3,2)` won't work\n",
        "* `(2,3) @ (3,2)` will work\n",
        "* `(3,2) @ (2,3)` will work\n",
        "2. The resulting matrix has the shape of the **outer dimensions**:\n",
        "* `(2,3) @ (3,2)` -> `(2,2)`\n",
        "* `(3,2) @ (2,3)` -> `(3,3)`"
      ],
      "metadata": {
        "id": "h8ESS-8HYwwN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Element wise multiplication\n",
        "print(f\"{tensor} * {tensor} = {tensor * tensor}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GxBc5edXYwtO",
        "outputId": "a5ae7607-2722-4ab6-e3be-b0060c90f643"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([1, 2, 3]) * tensor([1, 2, 3]) = tensor([1, 4, 9])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Matrix multiplication\n",
        "torch.matmul(tensor, tensor)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rMYulO7KYwFR",
        "outputId": "f4769f30-5f06-400d-a671-561c3a1afab8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(14)"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Matrix multiplication by hand\n",
        "1 * 1 + 2 * 2 + 3 * 3"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gIQwhKKDYwCO",
        "outputId": "5fdcbc2a-697e-4548-f983-e26bc06ad0d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "14"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "value = 0\n",
        "for i in range(len(tensor)):\n",
        "  value = tensor[i] * tensor[i]\n",
        "print(value)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WjDKxx9oYv_W",
        "outputId": "aa259740-b736-4e4d-fa65-7dfbf697f2f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(9)\n",
            "CPU times: user 1.8 ms, sys: 0 ns, total: 1.8 ms\n",
            "Wall time: 1.71 ms\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "torch.matmul(tensor,tensor) # use pytorch functions for complex bc more efficient"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gmqP2YBaYv8v",
        "outputId": "0c9c5ec9-349b-48b7-8ea5-10d8adbf7c2d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 174 µs, sys: 0 ns, total: 174 µs\n",
            "Wall time: 104 µs\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(14)"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tensor @ tensor # another way to mat multiply -- matmul is more clear tho"
      ],
      "metadata": {
        "id": "SQOLc0R3Yv6E",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "76009cb7-1985-4965-c06d-cda078a1a90d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(14)"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.matmul(torch.rand(2,3),torch.rand(2,3)) # produces error"
      ],
      "metadata": {
        "id": "gHsq37XnYv3b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 144
        },
        "outputId": "9c16dada-4d33-4daf-c6d0-9aafcefe8ee3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "mat1 and mat2 shapes cannot be multiplied (2x3 and 2x3)",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-52-6bd8034735d3>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# produces error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (2x3 and 2x3)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.matmul(torch.rand(2,3),torch.rand(3,2)) # Uses outer for dimension"
      ],
      "metadata": {
        "id": "vrMqyu-NYv0y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## One of the most common errors in deep learning: shape errors"
      ],
      "metadata": {
        "id": "pxIxN5ylYvyS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Shapes for matrix multiplication\n",
        "tensor_A = torch.tensor([[1,2],\n",
        "                         [3,4],\n",
        "                         [5,6]])\n",
        "tensor_B = torch.tensor([[7,10],\n",
        "                         [8,11],\n",
        "                         [9,12]])\n",
        "# torch.mm(tensor_A, tensor_B) # torch.mm is the same as torch.matmul\n",
        "torch.matmul(tensor_A,tensor_B)"
      ],
      "metadata": {
        "id": "lgMzS70fYvva"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tensor_A.shape, tensor_B.shape"
      ],
      "metadata": {
        "id": "FSUg5DjvYvsY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "To fix our tensor shape issues, we can manipulate the shape of one of our tensors using a **transpose**\n",
        "\n",
        "A **transpose** switches the axes or dimensions of a given tensor"
      ],
      "metadata": {
        "id": "VZvYrnGMYvp1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tensor_B.T, tensor_B.T.shape # transposes"
      ],
      "metadata": {
        "id": "gI9dsrVVYvmq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tensor_B, tensor_B.shape"
      ],
      "metadata": {
        "id": "1dK0xct9Yvjw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The matrix multiplication operation works when tensor_B is transposed\n",
        "print(f\"Original shapes: tensor_A = {tensor_A.shape}, tensor_B = {tensor_B.shape}\")\n",
        "print(f\"New shapes: tensor_A = {tensor_A.shape} (same shape as above), tensor_B.T = {tensor_B.T.shape}\")\n",
        "print(f\"Multiplying: {tensor_A.shape} @ {tensor_B.T.shape} <- inner dimensions must match\")\n",
        "print(\"Output:\\n\")\n",
        "\n",
        "output = torch.matmul(tensor_A,tensor_B.T)\n",
        "print(output)\n",
        "print(f\"\\nOutput shape: {output.shape}\")"
      ],
      "metadata": {
        "id": "yVT-sD72YvhU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.mm(tensor_A.T,tensor_B)"
      ],
      "metadata": {
        "id": "NCDmqKTMYve1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Finding the min, max, mean, sum, etc (tensor aggregation)"
      ],
      "metadata": {
        "id": "NefPKBaQYvcQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor\n",
        "x = torch.arange(1, 100, 10)\n",
        "x"
      ],
      "metadata": {
        "id": "Vpbuy-7fYvZt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33b0d437-a730-45ec-f761-4fac769bfc76"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 1, 11, 21, 31, 41, 51, 61, 71, 81, 91])"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the min\n",
        "torch.min(x), x.min()"
      ],
      "metadata": {
        "id": "lHSCoVWsYvU0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d5c4b9f8-d472-46bc-b4a3-7942428cfcbe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(1), tensor(1))"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the max\n",
        "torch.max(x), x.max()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qq_xyA0rNl2N",
        "outputId": "34d9b6dd-9b7b-471b-b6ff-798e85233bf9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(91), tensor(91))"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the mean\n",
        "torch.mean(x) # error bc uses long/int but needs float"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "id": "M5-_IvqXOfaS",
        "outputId": "25f689d6-060c-4433-8066-4836aa7d7277"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "mean(): could not infer output dtype. Input dtype must be either a floating point or complex dtype. Got: Long",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-58-8143f3645d43>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Find the mean\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# error bc uses long/int but needs float\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m: mean(): could not infer output dtype. Input dtype must be either a floating point or complex dtype. Got: Long"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the mean with no error\n",
        "torch.mean(x.type(torch.float32)), x.type(torch.float32).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-vXLPb77OvLW",
        "outputId": "1194ab0e-aace-48fd-f047-8ac0184fb6c3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(46.), tensor(46.))"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the sum\n",
        "torch.sum(x), x.sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NhuMzPmlPN92",
        "outputId": "a2b77069-e946-4434-db76-046781780285"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(460), tensor(460))"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Finding the positional min and max"
      ],
      "metadata": {
        "id": "MQnMOi0VPp7g"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the position in tensor that has the minimum value with argmin() -> returns index position of target tensor where the minimum value occurs\n",
        "x.argmin()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MrO-Wtw0PxA3",
        "outputId": "22261e63-9c19-4dff-aa74-d76240b5ea05"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0)"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cwIGIK_yP-d9",
        "outputId": "bc911c1c-62ac-4c30-9afd-64961caf50bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(1)"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the position in tensor that has the maximum value with argmax()\n",
        "x.argmax()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lJxRZ1yQQLIH",
        "outputId": "7d2795df-2763-470c-84d2-475da0132388"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(9)"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x[9]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fi4Xj3liQPib",
        "outputId": "dc537d09-fc91-4da0-8b17-6a1f30e58ca4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(91)"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Reshaping, stacking, squeezing, and unsqueezing tensors\n",
        "\n",
        "* Reshaping - reshapes an input tensor to a defined shape\n",
        "* View - Return a view of an input tensor of certain shape but keep the same memory as the original tensor\n",
        "* Stacking - combine multiple tensors on top of each other (vstack) or side by side (hstack)\n",
        "* Squeeze - removes all `1` dimensions from a tensor\n",
        "* Unsqueeze - add a `1` dimension to a target tensor\n",
        "* Permute - Return a view of the input with dimensions purmuted (swapped) in a certain way"
      ],
      "metadata": {
        "id": "g-SqQGjE9nK7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = torch.arange (1., 10.)\n",
        "x, x.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cHsIFJPlYgLI",
        "outputId": "9f86e9f8-a612-446e-b9d1-e08e5952e0e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([1., 2., 3., 4., 5., 6., 7., 8., 9.]), torch.Size([9]))"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Add an extra dimension\n",
        "x_reshaped = x.reshape (1,7) # error because can't squeeze 9 into 7\n",
        "x_reshaped, x_reshaped.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "id": "bq5YB_UyYemg",
        "outputId": "7ffdea07-e38f-48aa-de26-2a5f782abb47"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "shape '[1, 7]' is invalid for input of size 9",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-67-6fc03b15f8e8>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Add an extra dimension\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mx_reshaped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# error because can't squeeze 9 into 7\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mx_reshaped\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_reshaped\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: shape '[1, 7]' is invalid for input of size 9"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_reshaped = x.reshape (1,9)\n",
        "x_reshaped, x_reshaped.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O32teQTvYek3",
        "outputId": "227de520-b30b-443c-8dfa-6d6a7928fdc3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.]]), torch.Size([1, 9]))"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_reshaped = x.reshape (2,9) # can't do 18 elements\n",
        "x_reshaped, x_reshaped.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162
        },
        "id": "jixGFdIQYedu",
        "outputId": "5ef07dbf-c6ea-41c2-f15a-2951adbc4875"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "shape '[2, 9]' is invalid for input of size 9",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-80-7e1167e0e13b>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx_reshaped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m9\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# can't do 18 elements\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mx_reshaped\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_reshaped\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: shape '[2, 9]' is invalid for input of size 9"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_reshaped = x.reshape (9, 1)\n",
        "x_reshaped, x_reshaped.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uXXFsPLbYeb7",
        "outputId": "b3e6e11f-d162-43c3-b798-edf2c770974e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[5.],\n",
              "         [2.],\n",
              "         [3.],\n",
              "         [4.],\n",
              "         [5.],\n",
              "         [6.],\n",
              "         [7.],\n",
              "         [8.],\n",
              "         [9.]]),\n",
              " torch.Size([9, 1]))"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Change the view\n",
        "z = x.view(1,9)\n",
        "z, z.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eph0li9rYeW1",
        "outputId": "25acb210-0768-4155-ac6a-c97174037ae4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.]]), torch.Size([1, 9]))"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Changing z changes x (because a view of a tensor shares the same memory as the original input)\n",
        "z[:, 0] = 5\n",
        "z,x"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b3_fRmgEYeVM",
        "outputId": "2483c0cf-6750-4423-b06f-81b2937082e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.]]),\n",
              " tensor([5., 2., 3., 4., 5., 6., 7., 8., 9.]))"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Stack tensors on top of each other\n",
        "x_stacked = torch.stack([x, x, x, x], dim = 0)\n",
        "x_stacked"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "olXcLnr0YeP2",
        "outputId": "db20c9dc-8944-4aac-fc92-4af5a001c222"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.],\n",
              "        [5., 2., 3., 4., 5., 6., 7., 8., 9.],\n",
              "        [5., 2., 3., 4., 5., 6., 7., 8., 9.],\n",
              "        [5., 2., 3., 4., 5., 6., 7., 8., 9.]])"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Stack tensors on top of each other\n",
        "x_stacked = torch.stack([x, x, x, x], dim = 1)\n",
        "x_stacked"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tlBQ1ccIaa-y",
        "outputId": "25fb81f5-d0db-45c9-e57f-decfc3b2f0a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[5., 5., 5., 5.],\n",
              "        [2., 2., 2., 2.],\n",
              "        [3., 3., 3., 3.],\n",
              "        [4., 4., 4., 4.],\n",
              "        [5., 5., 5., 5.],\n",
              "        [6., 6., 6., 6.],\n",
              "        [7., 7., 7., 7.],\n",
              "        [8., 8., 8., 8.],\n",
              "        [9., 9., 9., 9.]])"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.squeeze() - removes all single dimensions from a target tensor\n",
        "print(f'Previous tensor: {x_reshaped}')\n",
        "print(f'Previous tensor: {x_reshaped.shape}')\n",
        "\n",
        "# Remove extra dimenions from x_reshaped\n",
        "x_squeezed = x_reshaped.squeeze()\n",
        "print(f'\\nNew tensor: {x_squeezed}')\n",
        "print(f'New shape: {x_squeezed.shape}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rb5wtohVa8UQ",
        "outputId": "cc76ef47-0b7a-454e-b544-69924e21c07e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Previous tensor: tensor([[5.],\n",
            "        [2.],\n",
            "        [3.],\n",
            "        [4.],\n",
            "        [5.],\n",
            "        [6.],\n",
            "        [7.],\n",
            "        [8.],\n",
            "        [9.]])\n",
            "Previous tensor: torch.Size([9, 1])\n",
            "\n",
            "New tensor: tensor([5., 2., 3., 4., 5., 6., 7., 8., 9.])\n",
            "New shape: torch.Size([9])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.unsqueeze() - adds a single dimension to a target tensor at a specific dim (dimension)\n",
        "print(f'Previous target: {x_squeezed}')\n",
        "print(f'Previous shape: {x_squeezed.shape}')\n",
        "\n",
        "# add an extra dimension with unsqueeze\n",
        "x_unsqueezed = x_squeezed.unsqueeze(dim=0)\n",
        "print(f'\\nNew tensor:{x_unsqueezed}')\n",
        "print(f'\\nNew shape:{x_unsqueezed.shape}')\n",
        "\n",
        "x_unsqueezed = x_squeezed.unsqueeze(dim=1)\n",
        "print(f'\\nNew tensor dim = 1:{x_unsqueezed}')\n",
        "print(f'\\nNew shape dim = 1:{x_unsqueezed.shape}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NBxotXefa-_B",
        "outputId": "7a31e01f-63ae-4f88-8981-b82908295f23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Previous target: tensor([5., 2., 3., 4., 5., 6., 7., 8., 9.])\n",
            "Previous shape: torch.Size([9])\n",
            "\n",
            "New tensor:tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.]])\n",
            "\n",
            "New shape:torch.Size([1, 9])\n",
            "\n",
            "New tensor dim = 1:tensor([[5.],\n",
            "        [2.],\n",
            "        [3.],\n",
            "        [4.],\n",
            "        [5.],\n",
            "        [6.],\n",
            "        [7.],\n",
            "        [8.],\n",
            "        [9.]])\n",
            "\n",
            "New shape dim = 1:torch.Size([9, 1])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.permute - rearrange the dimensions of a target in a specified order\n",
        "x_original = torch.rand(size = (224, 224, 3)) # [height, width, color_channels]\n",
        "\n",
        "# Permute the original tensor to rearrange the axis (or dim) order\n",
        "x_permuted = x_original.permute(2, 0, 1) # shifts axis 0->1, 1->2, 2->0\n",
        "\n",
        "x_original[0, 0, 0] = 100\n",
        "print(f'Previous shape: {x_original.shape}')\n",
        "print(f'Previous tensor: {x_original}')\n",
        "print(f'New shape: {x_permuted.shape}') # [color_channels, height, width]\n",
        "print(f'New tensor: {x_permuted}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QqCagw9Qa-6v",
        "outputId": "964f6df9-14b8-47c1-ab79-720c19ee9e29"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Previous shape: torch.Size([224, 224, 3])\n",
            "Previous tensor: tensor([[[1.0000e+02, 1.0002e-02, 9.8424e-01],\n",
            "         [2.8824e-01, 1.0991e-01, 9.7688e-01],\n",
            "         [4.4340e-03, 8.2603e-01, 4.9447e-01],\n",
            "         ...,\n",
            "         [8.2938e-01, 8.2615e-01, 5.8477e-01],\n",
            "         [3.6573e-01, 9.5326e-01, 3.7119e-02],\n",
            "         [7.8377e-01, 4.2833e-01, 8.0613e-01]],\n",
            "\n",
            "        [[3.8531e-01, 7.1479e-01, 8.3523e-01],\n",
            "         [2.7440e-01, 6.6139e-01, 5.6268e-01],\n",
            "         [2.8653e-01, 4.9152e-01, 6.9015e-01],\n",
            "         ...,\n",
            "         [9.7334e-01, 5.6220e-01, 7.2450e-01],\n",
            "         [9.2460e-01, 5.6414e-01, 1.7210e-01],\n",
            "         [7.4007e-01, 6.1905e-01, 5.9924e-01]],\n",
            "\n",
            "        [[9.5858e-01, 8.0007e-01, 9.1482e-01],\n",
            "         [8.6825e-02, 1.2561e-01, 4.0383e-01],\n",
            "         [5.3951e-01, 4.6580e-01, 5.9039e-02],\n",
            "         ...,\n",
            "         [9.3861e-01, 7.8242e-01, 9.4497e-01],\n",
            "         [2.3532e-01, 4.6360e-03, 7.8693e-01],\n",
            "         [6.5992e-01, 8.3528e-01, 1.9401e-01]],\n",
            "\n",
            "        ...,\n",
            "\n",
            "        [[6.4503e-01, 4.7824e-01, 5.9621e-01],\n",
            "         [3.4702e-01, 9.0344e-01, 7.0612e-02],\n",
            "         [6.6454e-01, 9.7427e-01, 1.4426e-01],\n",
            "         ...,\n",
            "         [4.9761e-01, 7.8533e-01, 1.4359e-01],\n",
            "         [9.1761e-01, 8.5027e-01, 4.9467e-01],\n",
            "         [3.3187e-02, 8.5715e-01, 6.2965e-01]],\n",
            "\n",
            "        [[6.6413e-01, 2.4878e-01, 2.8428e-01],\n",
            "         [1.9581e-01, 3.3131e-02, 9.3873e-01],\n",
            "         [2.3582e-01, 7.1014e-01, 3.2525e-01],\n",
            "         ...,\n",
            "         [5.2050e-01, 7.8183e-01, 7.4698e-01],\n",
            "         [8.2367e-01, 5.3696e-01, 9.9085e-01],\n",
            "         [5.9462e-02, 8.2728e-01, 6.1717e-02]],\n",
            "\n",
            "        [[7.0758e-01, 1.6495e-02, 9.8957e-02],\n",
            "         [5.1137e-01, 2.8662e-01, 4.0553e-02],\n",
            "         [9.5389e-01, 3.1474e-01, 5.4602e-01],\n",
            "         ...,\n",
            "         [4.7442e-01, 4.0714e-01, 5.4213e-01],\n",
            "         [6.8617e-01, 3.3780e-01, 7.6277e-01],\n",
            "         [5.0434e-01, 5.6666e-01, 1.5130e-03]]])\n",
            "New shape: torch.Size([3, 224, 224])\n",
            "New tensor: tensor([[[1.0000e+02, 2.8824e-01, 4.4340e-03,  ..., 8.2938e-01,\n",
            "          3.6573e-01, 7.8377e-01],\n",
            "         [3.8531e-01, 2.7440e-01, 2.8653e-01,  ..., 9.7334e-01,\n",
            "          9.2460e-01, 7.4007e-01],\n",
            "         [9.5858e-01, 8.6825e-02, 5.3951e-01,  ..., 9.3861e-01,\n",
            "          2.3532e-01, 6.5992e-01],\n",
            "         ...,\n",
            "         [6.4503e-01, 3.4702e-01, 6.6454e-01,  ..., 4.9761e-01,\n",
            "          9.1761e-01, 3.3187e-02],\n",
            "         [6.6413e-01, 1.9581e-01, 2.3582e-01,  ..., 5.2050e-01,\n",
            "          8.2367e-01, 5.9462e-02],\n",
            "         [7.0758e-01, 5.1137e-01, 9.5389e-01,  ..., 4.7442e-01,\n",
            "          6.8617e-01, 5.0434e-01]],\n",
            "\n",
            "        [[1.0002e-02, 1.0991e-01, 8.2603e-01,  ..., 8.2615e-01,\n",
            "          9.5326e-01, 4.2833e-01],\n",
            "         [7.1479e-01, 6.6139e-01, 4.9152e-01,  ..., 5.6220e-01,\n",
            "          5.6414e-01, 6.1905e-01],\n",
            "         [8.0007e-01, 1.2561e-01, 4.6580e-01,  ..., 7.8242e-01,\n",
            "          4.6360e-03, 8.3528e-01],\n",
            "         ...,\n",
            "         [4.7824e-01, 9.0344e-01, 9.7427e-01,  ..., 7.8533e-01,\n",
            "          8.5027e-01, 8.5715e-01],\n",
            "         [2.4878e-01, 3.3131e-02, 7.1014e-01,  ..., 7.8183e-01,\n",
            "          5.3696e-01, 8.2728e-01],\n",
            "         [1.6495e-02, 2.8662e-01, 3.1474e-01,  ..., 4.0714e-01,\n",
            "          3.3780e-01, 5.6666e-01]],\n",
            "\n",
            "        [[9.8424e-01, 9.7688e-01, 4.9447e-01,  ..., 5.8477e-01,\n",
            "          3.7119e-02, 8.0613e-01],\n",
            "         [8.3523e-01, 5.6268e-01, 6.9015e-01,  ..., 7.2450e-01,\n",
            "          1.7210e-01, 5.9924e-01],\n",
            "         [9.1482e-01, 4.0383e-01, 5.9039e-02,  ..., 9.4497e-01,\n",
            "          7.8693e-01, 1.9401e-01],\n",
            "         ...,\n",
            "         [5.9621e-01, 7.0612e-02, 1.4426e-01,  ..., 1.4359e-01,\n",
            "          4.9467e-01, 6.2965e-01],\n",
            "         [2.8428e-01, 9.3873e-01, 3.2525e-01,  ..., 7.4698e-01,\n",
            "          9.9085e-01, 6.1717e-02],\n",
            "         [9.8957e-02, 4.0553e-02, 5.4602e-01,  ..., 5.4213e-01,\n",
            "          7.6277e-01, 1.5130e-03]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Indexing (selecting data from tensors)\n",
        "\n",
        "Indexing with PyTorch is similar to indexing with NumPy"
      ],
      "metadata": {
        "id": "M6SSPOHCa-0L"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "x = torch.arange(1, 10).reshape (1, 3, 3)\n",
        "x, x.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dy8xnr6-a-qC",
        "outputId": "18012782-9f4c-4d39-a644-d3b2860b47d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[[1, 2, 3],\n",
              "          [4, 5, 6],\n",
              "          [7, 8, 9]]]),\n",
              " torch.Size([1, 3, 3]))"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's index on our new tensor\n",
        "x[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EBzrE4XYa-gk",
        "outputId": "720ff93c-4e5a-4c81-c391-6e0736d9fb66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 2, 3],\n",
              "        [4, 5, 6],\n",
              "        [7, 8, 9]])"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's index on the middle bracket (dim = 1)\n",
        "x[0][0], x[0,0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2xR1D-l7a-Xu",
        "outputId": "acca02e0-410d-4320-f060-a6cf69b18749"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([1, 2, 3]), tensor([1, 2, 3]))"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's index on the most inner bracket (last dimension)\n",
        "x[0][0][0], x[0,2,2]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CAQy1qSga-Mm",
        "outputId": "51da9cd7-f074-4772-fe8c-1cf8b7512f70"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(1), tensor(9))"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# You can also use \":\" to select \"all\" of. atarget dimension\n",
        "x[:, 0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pv60nqn3a-BI",
        "outputId": "49e2613a-98f5-4193-ad05-126adeb81129"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 2, 3]])"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all values of 0th and 1st dimension but only index 1 of 2nd dimension\n",
        "x[:, :, 1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fcS27A7Aa94B",
        "outputId": "68b0591a-5ca0-4f66-da46-765503608304"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[2, 5, 8]])"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all values of the 0 dimension but only the 1 index value of 1st and 2nd dimension\n",
        "x[:, 1, 1], x[0, 1, 1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-BT-DIO5a9r7",
        "outputId": "26582344-699e-4ce4-e373-28baa73b1b64"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([5]), tensor(5))"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get index 0 of 0th and 1st dimension and all values of 2nd dimension\n",
        "x[0, 0, :]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VdhRXIO-a9mZ",
        "outputId": "fde12f4d-e2d8-4a20-8b5b-dc965c7826b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1, 2, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Index on x to return 9\n",
        "print(x[0, 2, 2])\n",
        "\n",
        "# Index on x to return 3, 6, 9\n",
        "print(x[:, :, 2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CVszdr4Sa9hX",
        "outputId": "608b8cbe-2370-4564-dae9-be718984e4ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(9)\n",
            "tensor([[3, 6, 9]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## PyTorch tensors & NumPy\n",
        "\n",
        "Numpy is. apopular scientific Python numerical computing library.\n",
        "\n",
        "And because of this, PyTorch has functinoality to interact with it.\n",
        "\n",
        "* Data in NumPy, want in PyTorch tensor -> `torch.from_numpy(ndarray)`\n",
        "* PyTorch tensor -> NumPy -> `torch.Tensor.numpy()`"
      ],
      "metadata": {
        "id": "HOo7HlV2kQON"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# NumPy array to tensor\n",
        "import torch\n",
        "import numpy as np\n",
        "\n",
        "array = np.arange(1.0, 8.0)\n",
        "tensor = torch.from_numpy(array) # warning: when converting from numpy -> pytorch, pytorch reflects numpy's default datatype of float 64 unless specified otherwise\n",
        "array, tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jCVM5qBHknr-",
        "outputId": "6792adae-4f77-40a5-9f0c-32ca3bed60ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([1., 2., 3., 4., 5., 6., 7.]),\n",
              " tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float64))"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "array.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JqNuLQClloie",
        "outputId": "aaf07c70-3e90-4e1e-fe24-bde7ca8fd700"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dtype('float64')"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.arange(1.0, 8.0).dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l_DXMDbRlumE",
        "outputId": "67640146-b4d6-427b-adf5-237ca520ec4f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float32"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tensor = torch.from_numpy(array).type(torch.float16)\n",
        "tensor.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d50ZrrfTmHGU",
        "outputId": "60ce1ef4-5937-4293-a7f1-f9975ba6f19f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float16"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# change the value of array, what will this do to `tensor`\n",
        "array = array + 1\n",
        "array, tensor\n",
        "# tensor does not change"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CMutcCuJhXzK",
        "outputId": "fbc365ac-d7fe-41b1-dad7-ecc73323a8e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([3., 4., 5., 6., 7., 8., 9.]),\n",
              " tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float16))"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Tensor to NumPy array\n",
        "tensor = torch.ones(7)\n",
        "numpy_tensor = tensor.numpy()\n",
        "tensor, numpy_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zS8z8isChnBp",
        "outputId": "8bf27e6e-775a-43a8-84e5-f51f12695913"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([1., 1., 1., 1., 1., 1., 1.]),\n",
              " array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "numpy_tensor.dtype # uses og datatype (PyTorch tensor)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Gz2UTKZhnAD",
        "outputId": "0306fd07-3400-418f-a4f9-2f010f760149"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dtype('float32')"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Change the tensor, what happens to `numpy_tensor`?\n",
        "tensor = tensor + 1\n",
        "tensor, numpy_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nxa-4nI4hm-N",
        "outputId": "6b612d24-39e8-42f5-e195-9cfd0591d7ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([3., 3., 3., 3., 3., 3., 3.]),\n",
              " array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reproducibility (trying to take random out of random)\n",
        "\n",
        "In short how a neural network learns:\n",
        "\n",
        "`start with random numbers -> tensor operations -> update random numbers to try and make them better representations of the data -> again -> again -> again...`\n",
        "\n",
        "To reduce the randomness in neural networks and PyTorch comes the concept of a **random seed**.\n",
        "\n",
        "Essentially what the random seed does is \"flavor\" the randomness"
      ],
      "metadata": {
        "id": "LFE81t3ehm8W"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "torch.rand(3,3)"
      ],
      "metadata": {
        "id": "uSajy7r6hm6m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5ba6cfcc-d529-4137-aeed-070e75dabfc3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.5880, 0.8483, 0.0398],\n",
              "        [0.5786, 0.4296, 0.2010],\n",
              "        [0.0445, 0.1475, 0.5301]])"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch"
      ],
      "metadata": {
        "id": "_LHQus-5hmyv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create two random tensors\n",
        "random_tensor_A = torch.rand(3,4)\n",
        "random_tensor_B = torch.rand(3,4)\n",
        "\n",
        "print(random_tensor_A)\n",
        "print(random_tensor_B)\n",
        "print(random_tensor_A == random_tensor_B)\n",
        "# if someone else were to run this cell, they would get different values"
      ],
      "metadata": {
        "id": "37jlrGZGhmtV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "525aece3-4e95-450e-8c57-b62edf3d4ec0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.6151, 0.8887, 0.4512, 0.8033],\n",
            "        [0.8893, 0.8203, 0.4154, 0.0226],\n",
            "        [0.6743, 0.8020, 0.8396, 0.9593]])\n",
            "tensor([[0.6348, 0.0764, 0.3503, 0.1952],\n",
            "        [0.9550, 0.3166, 0.8565, 0.5438],\n",
            "        [0.6000, 0.0322, 0.1333, 0.3995]])\n",
            "tensor([[False, False, False, False],\n",
            "        [False, False, False, False],\n",
            "        [False, False, False, False]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's make some random but reproducible tensors\n",
        "import torch\n",
        "\n",
        "# Set the random seed\n",
        "RANDOM_SEED = 42 # other common ones are 0, 1234, 77, 100\n",
        "\n",
        "torch.manual_seed(RANDOM_SEED) # only works for one block of code\n",
        "random_tensor_C = torch.rand(3,4)\n",
        "\n",
        "torch.manual_seed(RANDOM_SEED)\n",
        "random_tensor_D = torch.rand(3,4)\n",
        "\n",
        "random_tensor_E = torch.rand(3,4)\n",
        "\n",
        "print(random_tensor_C)\n",
        "print(random_tensor_D)\n",
        "print(random_tensor_C == random_tensor_D)\n",
        "print(random_tensor_E)"
      ],
      "metadata": {
        "id": "sLpkrdZZhmrF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e79ea57c-ec8c-41df-f33e-3efabb6e2ecc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
            "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
            "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
            "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
            "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
            "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
            "tensor([[True, True, True, True],\n",
            "        [True, True, True, True],\n",
            "        [True, True, True, True]])\n",
            "tensor([[0.8694, 0.5677, 0.7411, 0.4294],\n",
            "        [0.8854, 0.5739, 0.2666, 0.6274],\n",
            "        [0.2696, 0.4414, 0.2969, 0.8317]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Extra resources for Reproducibility:\n",
        "* https://pytorch.org/docs/stable/notes/randomness.html\n",
        "* https://en.wikipedia.org/wiki/Random_seed"
      ],
      "metadata": {
        "id": "K0fonlMyhmpW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Running tensors and PyTorch objects on the GPUs (and making faster computations)\n",
        "\n",
        "GPUS = faster computations on numbers, thanks to CUDE + NVIDIA hardware + PyTorch working behind the scenes to make everything hunky dory (good)"
      ],
      "metadata": {
        "id": "F5U-rvf6hmn2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1. Getting a GPU\n",
        "\n",
        "1. Easiest - Use Google Colab for a free GPU (options to upgrade as well)\n",
        "2. Use your own GPU - takes a little bit of setup and requires the investment of purchasing a GPU, there's a lot of options..\n",
        "3. Use cloud computing - GCP, AWS, Azure, these services allow you to rent computers on the cloud and access them\n",
        "\n",
        "For 2, 3 PyTorch + GPU drives (CUDA) takes a little bit of setting up, to do this, refer to PyTorch setup documentation"
      ],
      "metadata": {
        "id": "mLV7vxikhmk4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "id": "untAau7Lhmep",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "37e483c8-1afd-4823-f7db-2d21141870ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Thu Sep 12 05:21:46 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla T4                       Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   61C    P8              10W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 2. Check for GPU access with PyTorch"
      ],
      "metadata": {
        "id": "Yl4gdWOwhmdJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Check for GPU access with PyTorch\n",
        "import torch\n",
        "torch.cuda.is_available()"
      ],
      "metadata": {
        "id": "-Noo69jThmXW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1aaaa8c2-cb00-47c2-fcad-ac0fc7585edc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup device agnostic code\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "device"
      ],
      "metadata": {
        "id": "5UGYgGgjhl5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0db5dbaf-6f94-4355-9882-23f7a45af3ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'cuda'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "For PyTorch since it's capable of running compute on the GPU or CPU, it's best to practice to setup device agnostic code: https://pytorch.org/docs/stable/notes/cuda.html#best-practices\n",
        "\n",
        "E.g., run on GPU if available, else default to CPU"
      ],
      "metadata": {
        "id": "fhztMIbHf1lb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Count number of devices\n",
        "torch.cuda.device_count() # might want to check devices to run different models on different gpus"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bJ-OAD6rfQaa",
        "outputId": "d02b9f75-598a-4680-d945-c0aa7a2c0cc3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 3. Putting tensors (and models) on the GPU\n",
        "\n",
        "The reason we want tensors/models on the GPU is because using a GPU results in faster computations"
      ],
      "metadata": {
        "id": "bQsh7J-LgZhz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor (default on the CPU)\n",
        "tensor = torch.tensor([1, 2, 3])\n",
        "\n",
        "# Tensor not on GPU\n",
        "print(tensor, tensor.device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ESB1EawKgfN0",
        "outputId": "ecaff339-f6b3-4e13-f085-4b26f7f156d7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([1, 2, 3]) cpu\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Move tensor to GPU (if available)\n",
        "tensor_on_gpu = tensor.to(device)\n",
        "tensor_on_gpu # also gives index of gpu (which gpu it's on)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_K5ANzTNg92T",
        "outputId": "013ed054-8fbf-4d64-8bb4-f2cee38fea76"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1, 2, 3], device='cuda:0')"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 4. Moving tensors back to the CPU"
      ],
      "metadata": {
        "id": "1y04xBlhhcRL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# If tensor is on GPU, can't transform it to NumPy\n",
        "tensor_on_gpu.numpy() # gives error"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159
        },
        "id": "x8FNg5T_hhMm",
        "outputId": "dec2f6f2-f311-4a54-f017-c6939083b986"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-37-b461ec6c9c4f>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# If tensor is on GPU, can't transform it to NumPy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtensor_on_gpu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# gives error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# To fix the GPU tensor with NumPy issue, we can first set it to the CPU\n",
        "tensor_back_on_cpu = tensor_on_gpu.cpu()\n",
        "tensor_back_on_cpu.numpy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1ZH-3fVphkbn",
        "outputId": "28a214e0-93ae-405d-957d-3b5f78887809"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 2, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tensor_on_gpu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_tRt4PMiaRS",
        "outputId": "1f109bf9-85ad-4bcd-e76e-11a93f9b4317"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1, 2, 3], device='cuda:0')"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Exercises & Extra-curriculum\n",
        "\n",
        "See exercises for this notebook here: https://www.learnpytorch.io/00_pytorch_fundamentals/#exercises"
      ],
      "metadata": {
        "id": "CFnO8lBKieMO"
      }
    }
  ]
}