Tail_Siamese.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One Shot Learning with Siamese Networks\n",
    "\n",
    "[Modified from Harshvardhan Gupta's work] (https://github.com/harveyslash/Facial-Similarity-with-Siamese-Networks-in-Pytorch/blob/master/Siamese-networks-medium.ipynb)  \n",
    "\n",
    "[for the Kaggle whale tail competition]\n",
    "(https://www.kaggle.com/c/humpback-whale-identification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "import glob\n",
    "import random\n",
    "import numpy as np\n",
    "import PIL.ImageOps    \n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torchvision.utils\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader,Dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions\n",
    "Set of helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(img,text=None,should_save=False):\n",
    "    npimg = img.numpy()\n",
    "    plt.axis(\"off\")\n",
    "    if text:\n",
    "        plt.text(75, 8, text, style='italic',fontweight='bold',\n",
    "            bbox={'facecolor':'white', 'alpha':0.8, 'pad':10})\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()    \n",
    "\n",
    "def show_plot(iteration,loss):\n",
    "    plt.plot(iteration,loss)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Path to your whale data\n",
    "pather = '/home/eagle/whale/data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_csv = pd.read_csv(f'{pather}train.csv')\n",
    "classes=[str(i) for i in (set(train_csv['Id']))]\n",
    "samp = pd.read_csv(f'{pather}sample_submission.csv')\n",
    "sub = samp.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Returns dict of enumerated classes\n",
    "def find_classes(classes):\n",
    "    classes.sort()\n",
    "    class_to_idx = {classes[i]: i for i in range(len(classes))}\n",
    "    return classes, class_to_idx\n",
    "\n",
    "def make_dataset(train, csv_file, class_to_idx):\n",
    "    images = []\n",
    "    trn_csv = pd.read_csv(csv_file)\n",
    "    for target in range(len(trn_csv)):\n",
    "        patho = os.path.join(train, trn_csv.loc[target][0])\n",
    "        lip = (patho, class_to_idx[trn_csv.loc[target][1]])\n",
    "        images.append(lip)\n",
    "    return images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creates a dataset with the files\n",
    "class Whale_csv_Dataset(Dataset):\n",
    "    def __init__(self, csv_file, train, classes, test_path=None, transform=None):\n",
    "\n",
    "        classes, class_to_idx = find_classes(classes)\n",
    "        imgs = make_dataset(train, csv_file, class_to_idx)\n",
    "        test_path = sorted(glob.glob(f'{test_path}/*'))\n",
    "\n",
    "        self.imgs = imgs\n",
    "        self.class_to_idx = class_to_idx\n",
    "\n",
    "        self.test_path = test_path\n",
    "        self.whale_frame = pd.read_csv(csv_file)\n",
    "        self.train = train\n",
    "        self.classes = classes\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.whale_frame)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        path, target = self.imgs[idx]\n",
    "        img_name = os.path.join(self.train,\n",
    "                                self.whale_frame.iloc[idx, 0])\n",
    "        image = io.imread(img_name)\n",
    "        landmarks = self.whale_frame.iloc[idx, 1:].as_matrix()\n",
    "        landmarks = landmarks.astype('float').reshape(-1, 2)\n",
    "        sample = {'image': image, 'landmarks': landmarks}\n",
    "        if self.transform:\n",
    "            sample = self.transform(sample)\n",
    "            \n",
    "        return sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Trywhale = Whale_csv_Dataset(csv_file=f'{pather}train.csv',\n",
    "                                train=f'{pather}train/',\n",
    "                                classes=classes,\n",
    "                                test_path=f'{pather}test/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creates a Pandas Dataframe of the set NOT including class 0 ('new_whale')\n",
    "df = pd.DataFrame(Trywhale.imgs)\n",
    "for i in range(len(df)):\n",
    "    if df.loc[i][1] == 0:\n",
    "        df.drop(i, axis=0, inplace=True)\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "label_to_indices = {label: np.where(np.array(df[1]) == label)[0]\n",
    "                  for label in set(df[1])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Selects images for the Siamese network\n",
    "def grab_two(df):\n",
    "    #Select a class\n",
    "    dex = (random.choice(range(len(label_to_indices))))\n",
    "    #If the class is 'class 0' we don't want it \n",
    "    if dex == 0:\n",
    "        dex += 1\n",
    "    one_class = label_to_indices[dex]\n",
    "    #Some classes have only one image\n",
    "    if len(one_class) > 1:\n",
    "        #If there is more than one image in the class we select an image\n",
    "        w0 = np.random.choice(one_class)\n",
    "        other_class = np.random.choice(2)\n",
    "        if other_class > 1:\n",
    "            #If > 1 - 1/2 of the time we'll select a second image of a different class\n",
    "            w1 = np.random.choice(df.index)\n",
    "        else:\n",
    "            #1/2 of the time we'll select an image in the same class\n",
    "            w1 = w0\n",
    "            #Use 'while it's not itself' to force it to not grab a duplicate\n",
    "            while w0 == w1:\n",
    "                w1 = np.random.choice(one_class)\n",
    "    else:\n",
    "        #if the class contains only one photo\n",
    "        w0 = np.random.choice(one_class)\n",
    "        w1 = np.random.choice(df.index)\n",
    "\n",
    "    return df.loc[w0][0], df.loc[w1][0], df.loc[w0][1], df.loc[w1][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SiameseNetworkDataset(Dataset):\n",
    "    def __init__(self,pic_set,transform=None,should_invert=True, show_mode=False):\n",
    "        self.pic_set = pic_set    \n",
    "        self.transform = transform\n",
    "        self.should_invert = should_invert\n",
    "        self.show_mode = show_mode\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        #Show_mode will be used later for predictions. (Returns one image and label)\n",
    "        if self.show_mode:\n",
    "            img0, label_0 = df.loc[index]\n",
    "            img1, label_1 = img0, label_0\n",
    "        else:\n",
    "            img0, img1, label_0, label_1 = grab_two(df)\n",
    "\n",
    "        img0 = Image.open(img0)\n",
    "        img1 = Image.open(img1)\n",
    "        img0 = img0.convert(\"RGB\")\n",
    "        img1 = img1.convert(\"RGB\")\n",
    "        \n",
    "        if self.should_invert:\n",
    "            img0 = PIL.ImageOps.invert(img0)\n",
    "            img1 = PIL.ImageOps.invert(img1)\n",
    "\n",
    "        if self.transform is not None:\n",
    "            img0 = self.transform(img0)\n",
    "            img1 = self.transform(img1)\n",
    "    \n",
    "        return img0, img1 , torch.from_numpy(\n",
    "            np.array([int(label_0!=label_1)],dtype=np.float32)), label_0\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualising some of the data\n",
    "The top row and the bottom row of any column is one pair. The 0s and 1s correspond to the column of the image.\n",
    "1 indiciates dissimilar, and 0 indicates similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_dataset = SiameseNetworkDataset(Trywhale,\n",
    "                                        transform=transforms.Compose([transforms.Resize((170,300)),\n",
    "                                                                     transforms.ToTensor()\n",
    "                                                                      ])\n",
    "                                       ,should_invert=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vis_dataloader = DataLoader(siamese_dataset,\n",
    "                        shuffle=True,\n",
    "                        num_workers=8,\n",
    "                        batch_size=1)\n",
    "dataiter = iter(vis_dataloader)\n",
    "\n",
    "\n",
    "example_batch = next(dataiter)\n",
    "concatenated = torch.cat((example_batch[0],example_batch[1]),0)\n",
    "imshow(torchvision.utils.make_grid(concatenated))\n",
    "print(example_batch[2].numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Neural Net Definition\n",
    "We will use a standard convolutional neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SiameseNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SiameseNetwork, self).__init__()\n",
    "\n",
    "        self.cnn1 = nn.Sequential(\n",
    "            nn.ReflectionPad2d(1),\n",
    "            nn.Conv2d(3, 4, kernel_size=3),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.BatchNorm2d(4),\n",
    "            nn.ReflectionPad2d(1),\n",
    "            nn.Conv2d(4, 8, kernel_size=3),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.BatchNorm2d(8))\n",
    "\n",
    "        self.fc1 = nn.Sequential(\n",
    "            nn.Linear(8*170*300, 700),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(700, 700),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(700, 500),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(500, 1))\n",
    "\n",
    "    def forward_once(self, x):\n",
    "        output = self.cnn1(x)\n",
    "        output = output.view(output.size()[0], -1)\n",
    "        output = self.fc1(output)\n",
    "        return output\n",
    "\n",
    "    def forward(self, input1, input2):\n",
    "        output1 = self.forward_once(input1)\n",
    "        output2 = self.forward_once(input2)\n",
    "        return output1, output2\n",
    "    \n",
    "    def get_embedding(self, x):\n",
    "        return self.forward_once(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Contrastive Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ContrastiveLoss(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, margin=1):\n",
    "        super(ContrastiveLoss, self).__init__()\n",
    "        self.margin = margin\n",
    "\n",
    "    def forward(self, output1, output2, label):\n",
    "        euclidean_distance = F.pairwise_distance(output1, output2)\n",
    "        loss_contrastive = torch.mean((1-label) * torch.pow(euclidean_distance, 2) +\n",
    "                                    (label) * torch.pow(\n",
    "                                    torch.clamp(self.margin - euclidean_distance, min=0.0), 2))\n",
    "\n",
    "        return loss_contrastive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Time!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader = DataLoader(siamese_dataset,\n",
    "                        shuffle=True,\n",
    "                        num_workers=4,\n",
    "                        batch_size=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = SiameseNetwork().cuda()\n",
    "criterion = ContrastiveLoss()\n",
    "optimizer = optim.Adam(net.parameters(),lr = 0.001 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = []\n",
    "loss_history = [] \n",
    "iteration_number = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for epoch in range(1):\n",
    "    for i, data in enumerate(train_dataloader):\n",
    "        img0, img1, label, _ = data\n",
    "        img0, img1, label = img0.cuda(), img1.cuda() , label.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        output1,output2 = net(img0,img1)\n",
    "        loss_contrastive = criterion(output1,output2,label)\n",
    "        loss_contrastive.backward()\n",
    "        optimizer.step()\n",
    "        if i %25 == 0 :\n",
    "            print(\"Epoch number {}\\n Current loss {}\\n\".format(i,loss_contrastive.item()))\n",
    "            iteration_number +=10\n",
    "            counter.append(iteration_number)\n",
    "            loss_history.append(loss_contrastive.item())\n",
    "        if i > 5000:\n",
    "            break\n",
    "show_plot(counter,loss_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the Test Images for the model to predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SiameseNet_TEST_set(Dataset):\n",
    "    def __init__(self,pic_set,transform=None,should_invert=False):\n",
    "        self.pic_set = pic_set\n",
    "        self.transform = transform\n",
    "        self.should_invert = should_invert\n",
    "        \n",
    "    def __getitem__(self,index):\n",
    "        img0 = self.pic_set.test_path[index]\n",
    "        \n",
    "        img0 = Image.open(img0)\n",
    "        img0 = img0.convert(\"RGB\")        \n",
    "        \n",
    "        if self.should_invert:\n",
    "            img0 = PIL.ImageOps.invert(img0)\n",
    "\n",
    "        if self.transform is not None:\n",
    "            img0 = self.transform(img0)\n",
    "\n",
    "        return img0, index\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.pic_set.test_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loads the training images into the model just to get average class embeddings\n",
    "siamese_dataset = SiameseNetworkDataset(Trywhale,\n",
    "                                        transform=transforms.Compose([transforms.Resize((170,300)),\n",
    "                                                                     transforms.ToTensor()]),\n",
    "                                       should_invert=False,\n",
    "                                       show_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_testset = SiameseNet_TEST_set(Trywhale,\n",
    "                                        transform=transforms.Compose([transforms.Resize((170,300)),\n",
    "                                                                     transforms.ToTensor()\n",
    "                                                                      ])\n",
    "                                       ,should_invert=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trained_dataloader = DataLoader(siamese_dataset,num_workers=6,batch_size=1,shuffle=False)\n",
    "dataiter = iter(trained_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    net.eval()\n",
    "    df_tr = pd.DataFrame([1, 2]).T\n",
    "    k = 0\n",
    "    for i, _,_, x in (tqdm(dataiter)):\n",
    "        i = i.cuda()\n",
    "        df_tr.loc[k] = float(net.get_embedding(i).data.cpu().numpy()), int(x)\n",
    "        k += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Here we have each < 0 class image by index number, embedding (0)\n",
    "df_tr.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Associates every embedding with it's class in a dict \n",
    "s = df_tr.groupby(1)[0].apply(lambda x: x.tolist()).to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Averages every embedding for each class\n",
    "avg_emb = {}\n",
    "for k,v in s.items():\n",
    "    avg_emb[k] = np.mean(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make a dataframe with class set as the index with its class embedding\n",
    "avg_emb2class = pd.DataFrame([i for i in enumerate(avg_emb.values())])\n",
    "avg_emb2class[0] += 1\n",
    "avg_emb2class.sort_values(1, inplace=True)\n",
    "avg_emb2class.set_index(1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "### Send the test images through the model to get embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "siamese_testset = DataLoader(siamese_testset,num_workers=6,batch_size=1,shuffle=False)\n",
    "unknown_tails = iter(siamese_testset)\n",
    "with torch.no_grad():\n",
    "    net.eval()\n",
    "    df_T = pd.DataFrame([1]).T\n",
    "    for i, x in tqdm(unknown_tails):\n",
    "        i = i.cuda()\n",
    "        df_T.loc[int(x)] = float(net.get_embedding(i).data.cpu().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to grab the closest number in an array then remove to find next closest\n",
    "def closest(num, arr):\n",
    "    curr = arr[0]\n",
    "    for val in whip:\n",
    "        if abs (num - val) < abs (num - curr):\n",
    "            curr = val\n",
    "    whip.remove(curr)\n",
    "    return curr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for aa, i in enumerate(tqdm(df_T[0])):\n",
    "    new4 = {}\n",
    "    whip = [ic for ic in avg_emb2class.index]\n",
    "    for x in range(4):\n",
    "        vk = closest(i, whip)\n",
    "        w_label = avg_emb2class.loc[vk][0]\n",
    "        wn = list([str(a) for a, z in Trywhale.class_to_idx.items()if z == (w_label)])\n",
    "        new4[x] = wn\n",
    "        new44 = pd.DataFrame(new4)\n",
    "        new44.at[0, 4] = 'new_whale'\n",
    "        sub.at[aa, 'Id'] = ' '.join([e for e in new44.loc[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv(f'{pather}DEC27_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}