04_intro_to_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a546191e",
   "metadata": {},
   "source": [
    "# Intro to PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0a2fa47",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d7e1443b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.,  2.],\n",
       "        [ 3.,  4.],\n",
       "        [ 5.,  6.],\n",
       "        [ 7.,  8.],\n",
       "        [ 9., 10.],\n",
       "        [11., 12.]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a tensor:\n",
    "example_tensor = torch.Tensor(\n",
    "    [\n",
    "        [1,2], [3,4],\n",
    "        [5,6], [7,8],\n",
    "        [9,10], [11,12]\n",
    "    ]\n",
    ")\n",
    "\n",
    "example_tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fbb6a80",
   "metadata": {},
   "source": [
    "## Tensor properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1afed002",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check whether your tensor is on the cpu, or the gpu:\n",
    "example_tensor.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "595835b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shift tensor to the gpu:\n",
    "# new_tensor = example_tensor.to(device)  # where device = 'cpu', or 'cuda'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6b13cf86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([6, 2])\n",
      "6 2\n"
     ]
    }
   ],
   "source": [
    "# Print shape and size:\n",
    "print(example_tensor.shape)\n",
    "print(example_tensor.size(0), example_tensor.size(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b0b4054b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rank:\n",
    "len(example_tensor.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "15130f6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of elements:\n",
    "example_tensor.numel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "29ba775e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(4.)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Index a tensor:\n",
    "example_tensor[1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "018299f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.,  3.,  5.,  7.,  9., 11.])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_tensor[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d31d725d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 2.])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_tensor[0,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8839bd4f",
   "metadata": {},
   "source": [
    "## Initialize tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5c9e2ebd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1.],\n",
       "        [1., 1.],\n",
       "        [1., 1.],\n",
       "        [1., 1.],\n",
       "        [1., 1.],\n",
       "        [1., 1.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.ones_like(example_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c1f7c214",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0.],\n",
       "        [0., 0.],\n",
       "        [0., 0.],\n",
       "        [0., 0.],\n",
       "        [0., 0.],\n",
       "        [0., 0.]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.zeros_like(example_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ac239fc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.8430, -1.4126],\n",
       "        [-0.6290,  0.2915],\n",
       "        [ 0.4129, -1.0184],\n",
       "        [-1.7985,  1.0083],\n",
       "        [-0.2738, -0.3596],\n",
       "        [ 0.2478, -0.9492]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.randn_like(example_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0ea395e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.5614, -0.8446],\n",
       "        [ 0.5287,  0.3602]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.randn(2, 2, device='cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d88c6f3",
   "metadata": {},
   "source": [
    "## Basic functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f06db59f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-8., -6.],\n",
       "        [-4., -2.],\n",
       "        [ 0.,  2.],\n",
       "        [ 4.,  6.],\n",
       "        [ 8., 10.],\n",
       "        [12., 14.]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(example_tensor-5) * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2024547a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6.5000)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_tensor.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7f16343e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3.6056)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_tensor.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6d52ec51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([6., 7.])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean along a particular dimension:\n",
    "example_tensor.mean(0)  # 0th dimension"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7a26d88",
   "metadata": {},
   "source": [
    "## Neural network module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "282f95dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.7766,  0.7931],\n",
       "        [-0.6373, -0.2534],\n",
       "        [ 0.1682, -0.7006]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "linear = nn.Linear(10,2)  # input, output\n",
    "linear_input = torch.randn(3,10)\n",
    "linear_output = linear(linear_input)\n",
    "linear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2c46c333",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.7766, 0.7931],\n",
       "        [0.0000, 0.0000],\n",
       "        [0.1682, 0.0000]], grad_fn=<ReluBackward0>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relu = nn.ReLU()\n",
    "relu_output = relu(linear_output)\n",
    "relu_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a233cb55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.3839,  1.4142],\n",
       "        [-0.9441, -0.7071],\n",
       "        [-0.4398, -0.7071]], grad_fn=<NativeBatchNormBackward0>)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batchnorm = nn.BatchNorm1d(2)  # input dimensions\n",
    "batchnorm_output = batchnorm(relu_output)\n",
    "batchnorm_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "56965cfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.0668],\n",
       "        [0.0000, 0.1340],\n",
       "        [1.1970, 1.5653],\n",
       "        [0.8587, 0.0000],\n",
       "        [0.2530, 0.0000]], grad_fn=<ReluBackward0>)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp_layer = nn.Sequential(\n",
    "    nn.Linear(5,2),\n",
    "    nn.BatchNorm1d(2),\n",
    "    nn.ReLU()\n",
    ")\n",
    "\n",
    "mlp_input = torch.randn(5,5) + 1\n",
    "mlp_layer(mlp_input)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f73a7867",
   "metadata": {},
   "source": [
    "## Optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e44be04e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "adam_opt = optim.Adam(mlp_layer.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77c15c32",
   "metadata": {},
   "source": [
    "## Training loop\n",
    "\n",
    "1. Set gradients to zero: `opt.zero_grad()`\n",
    "2. Compute loss.\n",
    "3. Back propagate: `loss.backward()`\n",
    "4. Update parameters: `opt.step()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "903c259e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.4742, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_input = torch.randn(100,5) + 1\n",
    "adam_opt.zero_grad()\n",
    "cur_loss = torch.abs(1 - mlp_layer(train_input)).mean()\n",
    "cur_loss.backward()\n",
    "adam_opt.step()\n",
    "cur_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71279afc",
   "metadata": {},
   "source": [
    "## Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8d2ebd57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate the gradient with respect to a particular tensor if PyTorch wouldn't\n",
    "# have normally stored the gradient for that tensor. (Changes it in place).\n",
    "\n",
    "# example_tensor.requres_grad_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3ed6d8d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To prevent gradients from being calculated for a block of code:\n",
    "# with torch.no_grad():\n",
    "#     ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a6688623",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use a tensor's value without calculating its gradients:\n",
    "# detach()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b80f9a13",
   "metadata": {},
   "source": [
    "## Extending the `nn` module\n",
    "\n",
    "Whatever is in that class, becomes a parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "016e2ac4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyNet(nn.Module):\n",
    "    def __init__(self, input_dims, output_dims):\n",
    "        super(MyNet, self).__init__()\n",
    "        self.linear = nn.Linear(input_dims, output_dims)\n",
    "        self.exponent = nn.Parameter(torch.tensor(1.))\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.linear(x)\n",
    "        x = x ** self.exponent\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "44ad6ddf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Parameter containing:\n",
       " tensor(1., requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([[-0.1318, -0.0488,  0.1173, -0.0171, -0.0805,  0.2370, -0.0581, -0.0773,\n",
       "          -0.1309, -0.3126],\n",
       "         [ 0.0907, -0.0775,  0.1297, -0.0939,  0.2540,  0.2680, -0.2161,  0.1853,\n",
       "          -0.1456, -0.1704]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([-0.0323,  0.1736], requires_grad=True)]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_mynet = MyNet(10,2)\n",
    "list(example_mynet.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "16f89adb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('exponent',\n",
       "  Parameter containing:\n",
       "  tensor(1., requires_grad=True)),\n",
       " ('linear.weight',\n",
       "  Parameter containing:\n",
       "  tensor([[-0.1318, -0.0488,  0.1173, -0.0171, -0.0805,  0.2370, -0.0581, -0.0773,\n",
       "           -0.1309, -0.3126],\n",
       "          [ 0.0907, -0.0775,  0.1297, -0.0939,  0.2540,  0.2680, -0.2161,  0.1853,\n",
       "           -0.1456, -0.1704]], requires_grad=True)),\n",
       " ('linear.bias',\n",
       "  Parameter containing:\n",
       "  tensor([-0.0323,  0.1736], requires_grad=True))]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(example_mynet.named_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "dac2177f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.2511,  0.5484],\n",
       "        [-0.3156, -0.7913]], grad_fn=<PowBackward1>)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_mynet(torch.randn(2,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf5bb1c0",
   "metadata": {},
   "source": [
    "## 2D operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5a53085",
   "metadata": {},
   "outputs": [],
   "source": [
    "# nn.Conv2D: input, output, kernel_size\n",
    "# nn.ConvTranspose2d: input, output, kernel_size\n",
    "# nn.BatchNorm2d: input\n",
    "# nn.Upsample: final_size or scale_factor\n",
    "# nn.functional.interpolate: final_size or scale_factor"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}