nb_train.best_practices.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp train.best_practices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best practices\n",
    "\n",
    "> Best practices from papers and myself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "# hide\n",
    "from nbdev.showdoc import *\n",
    "from fastcore.test import *\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "import torch\n",
    "from pl_bolts.optimizers.lr_scheduler import LinearWarmupCosineAnnealingLR\n",
    "import torch.nn.functional as F\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "class LiuProcedure():\n",
    "    '''Liu et al. (https://arxiv.org/abs/2010.05351)\n",
    "\n",
    "    Paper: Identifying Melanoma Images using EfficientNet Ensemble:\n",
    "    Winning Solution to the SIIM-ISIC Melanoma Classification Challenge\n",
    "    \n",
    "    Strategy\n",
    "    --------\n",
    "    1. Train-Val Set\n",
    "    5-fold cross-validation on data from ISIC's 2018, 2019 and 2020 challenge. This leads to \n",
    "    enormous class-imbalance, however they do no do any balancing. They also do not mention \n",
    "    leakage between train and val set which must have occured but can probably be ignored. \n",
    "    \n",
    "    2. Target and metadata\n",
    "    Even though the 2020 challenge was binary, they used a multiclass (n=9) approach. They\n",
    "    also had some models which used metadata. \n",
    "    \n",
    "    3. Architectures\n",
    "    They used a variety of architecture backbones (EfficientNet B3, B4, B5, B6, B7,\n",
    "    SE-ResNeXt-101 and ResNeSt-101) and image sizes. \n",
    "    \n",
    "    4. Ensmeble\n",
    "    They ensembled a total of 18 models. Each of these 18 models is itself an ensemble\n",
    "    of 5 models which were obtained through 5-fold cross-validation. \n",
    "    \n",
    "    Model notes\n",
    "    -----------\n",
    "    EfficientNet-B0: 5.3M\n",
    "    EfficientNet-B1: 7.8M\n",
    "    EfficientNet-B2: 9.2M\n",
    "    EfficientNet-B3: 12M\n",
    "    EfficientNet-B4: 19M\n",
    "    EfficientNet-B5: 30M\n",
    "    EfficientNet-B6: 43M\n",
    "    EfficientNet-B7: 66M\n",
    "    SE-ResNeXt-101 : ??M\n",
    "    ResNeSt-101    : 48M\n",
    "    \n",
    "    Class labels\n",
    "    ------------\n",
    "    Good to know for future classifications tasks aimed to be similar.\n",
    "    For four classes: {'BKL': 0, 'melanoma': 1, 'nevus': 2, 'unknown': 3}\n",
    "    For nine classes: {'AK': 0, 'BCC': 1, 'BKL': 2, 'DF': 3, 'SCC': 4,\n",
    "                       'VASC': 5, 'melanoma': 6, 'nevus': 7, 'unknown': 8}\n",
    "    \n",
    "    seborrheic keratosis -> BKL\n",
    "    lichenoid keratosis -> BKL\n",
    "    solar lentigo -> BKL\n",
    "    lentigo NOS -> BKL\n",
    "    cafe-au-lait macule -> unknown\n",
    "    atypical melanocytic proliferation -> unknown\n",
    "    DF -> unknown\n",
    "    AK -> unknown\n",
    "    SCC -> unknown\n",
    "    VASC -> unknown\n",
    "    BCC -> unknown\n",
    "    '''\n",
    "    @staticmethod    \n",
    "    def get_transforms(img_size):\n",
    "        '''Get transforms \n",
    "        \n",
    "        As they used models with different image sizes, image size is parameterized here. \n",
    "        Their image sizes included 384, 448, 512, 576, 640, 768 and 896. Transforms were\n",
    "        fixed across all architectures and image sizes. \n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        img_size : int\n",
    "            Size image will be resized to\n",
    "            \n",
    "        Returns\n",
    "        -------\n",
    "        A list of albumentation transforms.\n",
    "        '''\n",
    "        return [\n",
    "            A.Transpose(p=0.5),\n",
    "            A.VerticalFlip(p=0.5),\n",
    "            A.HorizontalFlip(p=0.5),\n",
    "            A.RandomBrightness(limit=0.2, p=0.75),\n",
    "            A.RandomContrast(limit=0.2, p=0.75),\n",
    "            A.OneOf([\n",
    "                A.MotionBlur(blur_limit=5),\n",
    "                A.MedianBlur(blur_limit=5),\n",
    "                A.GaussianBlur(blur_limit=5),\n",
    "                A.GaussNoise(var_limit=(5.0, 30.0)),\n",
    "            ], p=0.7),\n",
    "            A.OneOf([\n",
    "                A.OpticalDistortion(distort_limit=1.0),\n",
    "                A.GridDistortion(num_steps=5, distort_limit=1.),\n",
    "                A.ElasticTransform(alpha=3),\n",
    "            ], p=0.7),\n",
    "            A.CLAHE(clip_limit=4.0, p=0.7),\n",
    "            A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n",
    "            A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),\n",
    "            A.Resize(img_size, img_size),\n",
    "            A.Cutout(max_h_size=int(img_size * 0.375), max_w_size=int(img_size * 0.375), num_holes=1, p=0.7),\n",
    "            A.Normalize(),\n",
    "            ToTensorV2(),\n",
    "        ]\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_criterion():\n",
    "        '''Get loss function'''\n",
    "        return F.cross_entropy\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_optimizer(params, lr=3e-4, epochs=15, warmup_epochs=1, discriminative_lr=False):\n",
    "        '''Get optimizer(s) and scheduler(s).\n",
    "        \n",
    "        They used cosine annealing with one warm-up epoch and mostly 15 total epochs (except 18 once). \n",
    "        Learning rated ranged from 1e-4 to 3e-4 and was tuned for every architecture. During warm-up, \n",
    "        the learning rate is one thenth of the initial learning rate of the cosine cycle (i.e. lowest \n",
    "        lr can be 3e-5 at warm-up). Batch size was 64 for all models. \n",
    "        \n",
    "        Note: learning rate was mostly 3e-4\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        params : iterable of torch Tensor\n",
    "            Model's parameters\n",
    "        \n",
    "        lr : float; optinal\n",
    "            Learning rate \n",
    "            \n",
    "        epochs : int; optional\n",
    "            Number of training epochs\n",
    "            \n",
    "        warmup_epochs : int; optional\n",
    "            Number of warmup epochs\n",
    "            \n",
    "        discriminative_lr : bool; optional\n",
    "            If True, a discriminative lr is used and each layer/group in ´params´\n",
    "            will have a different lr associated with it. Thus, the warmup lr for the\n",
    "            linear warmup should also be discriminative which is however not possible. \n",
    "            Therefore, the initial warmup lr is set to 0 instead of lr/10 which is\n",
    "            the default in the paper. \n",
    "        '''\n",
    "        warmup_start_lr = lr/10 # default \n",
    "        if discriminative_lr:\n",
    "            warmup_start_lr = 0. # special case\n",
    "        print(warmup_start_lr)\n",
    "        optimizer = torch.optim.Adam(params, lr=lr)\n",
    "        scheduler = LinearWarmupCosineAnnealingLR(\n",
    "            optimizer=optimizer,\n",
    "            warmup_epochs=warmup_epochs,\n",
    "            max_epochs=epochs, \n",
    "            warmup_start_lr=warmup_start_lr\n",
    "        )\n",
    "        \n",
    "        return {\"optimizer\": optimizer, \"lr_scheduler\": scheduler}\n",
    "    \n",
    "class LasserProcedure():\n",
    "    '''Lasser et al. ()\n",
    "\n",
    "    Paper: FusionM4Net: A multi-stage multi-modal learning algorithm \n",
    "    for multi-label skin lesion classification\n",
    "\n",
    "    Notes\n",
    "    -----\n",
    "    They did their resize when loading images i.e. before applying tfms\n",
    "    using cv2. They also wrapped their tfms in a ´A.Compose´ with an\n",
    "    associated proba\n",
    "    '''\n",
    "    @staticmethod\n",
    "    def get_transforms(img_size):\n",
    "        '''Get transforms\n",
    "\n",
    "        They did their resize when loading images i.e. before applying tfms\n",
    "        using cv2. They also wrapped their tfms in a ´A.Compose´ with an\n",
    "        associated probability of p=0.5\n",
    "        '''\n",
    "        return [\n",
    "            A.Resize(img_size, img_size),\n",
    "            A.VerticalFlip(p=0.5),\n",
    "            A.HorizontalFlip(p=0.5),\n",
    "            A.ShiftScaleRotate(shift_limit=0.0625,scale_limit=0.5,rotate_limit=45,p=0.5),\n",
    "            A.RandomRotate90(p=0.5),\n",
    "            A.RandomBrightnessContrast(p=0.5),\n",
    "            A.Normalize(),\n",
    "            ToTensorV2(),\n",
    "        ]\n",
    "\n",
    "class ValleProcedure():\n",
    "    '''Valle et al. (https://arxiv.org/abs/1809.01442)\n",
    "\n",
    "    Paper: Data Augmentation for Skin Lesion Analysis      \n",
    "    '''\n",
    "    @staticmethod\n",
    "    def get_transforms(img_size):\n",
    "        '''Get transforms'''\n",
    "        return [\n",
    "            A.Resize(img_size, img_size),\n",
    "            A.Normalize(),\n",
    "            ToTensorV2(),\n",
    "        ]\n",
    "    \n",
    "    \n",
    "class XuProcedure():\n",
    "    '''Xu et al. (https://arxiv.org/ftp/arxiv/papers/2101/2101.02353)\n",
    "\n",
    "    Paper: Low-cost and high-performance data augmentation for\n",
    "    deep-learningbased skin lesion classification\n",
    "    '''\n",
    "    @staticmethod\n",
    "    def get_transforms(img_size):\n",
    "        '''Get transforms'''\n",
    "        return [\n",
    "            A.Resize(img_size, img_size),\n",
    "            A.Normalize(),\n",
    "            ToTensorV2(),\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def get_inference_tfms(img_size, normalize=True):\n",
    "    '''Get basic albumentation transforms for inference'''\n",
    "    tfms = [\n",
    "            A.Resize(img_size, img_size),\n",
    "            A.Normalize(),\n",
    "            ToTensorV2(),\n",
    "        ]\n",
    "    \n",
    "    if not normalize:\n",
    "        tfms.pop(1)\n",
    "        \n",
    "    return tfms\n",
    "\n",
    "def get_train_tfms(img_size, tfm_name):\n",
    "    '''Get train transforms'''\n",
    "    if tfm_name == \"liu\":\n",
    "        return LiuProcedure.get_transforms(img_size)\n",
    "    elif tfm_name == \"lasser\":\n",
    "        return LasserProcedure.get_transforms(img_size)\n",
    "    elif tfm_name == \"valle\":\n",
    "        return ValleProcedure.get_transforms(img_size)\n",
    "    elif tfm_name == \"xu\":\n",
    "        return XuProcedure.get_transforms(img_size)\n",
    "    elif tfm_name == \"default\":\n",
    "        return get_inference_tfms(img_size) \n",
    "    else:\n",
    "        raise ValueError(f\"Unknown transform name '{tfm_name}'\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}