srgan_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorboard_logger==0.1.0 in /usr/local/lib/python3.5/dist-packages\n",
      "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.5/dist-packages (from tensorboard_logger==0.1.0)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.5/dist-packages (from tensorboard_logger==0.1.0)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.5/dist-packages (from tensorboard_logger==0.1.0)\n",
      "Requirement already satisfied: protobuf in /usr/local/lib/python3.5/dist-packages (from tensorboard_logger==0.1.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.5/dist-packages (from tensorboard_logger==0.1.0)\n",
      "Requirement already satisfied: olefile in /usr/local/lib/python3.5/dist-packages (from pillow>=4.1.1->tensorboard_logger==0.1.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.5/dist-packages (from protobuf->tensorboard_logger==0.1.0)\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 10.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 install tensorboard_logger==0.1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import os\n",
    "import pathlib\n",
    "import traceback\n",
    "from collections import OrderedDict\n",
    "\n",
    "import shutil\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "from tensorboard_logger import Logger\n",
    "from torch import optim\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataset import Dataset\n",
    "from torchvision import transforms\n",
    "from torchvision.datasets import ImageFolder\n",
    "from tqdm import tqdm, trange\n",
    "\n",
    "from settings.paths import FILTERED_MS_CELEB_IMAGES_DIR, \\\n",
    "                           SRGAN_MSE_LOSS_LOGS_DIR, \\\n",
    "                           ERRORS_LOGS_DIR, \\\n",
    "                           LIGHT_CNN_9_WEIGHT, \\\n",
    "                           SRGAN_VGG_LOSS_3_1_LOGS_DIR, \\\n",
    "                           SRGAN_VGG_LOSS_3_1_NO_ADVERSARIAL_LOGS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_MFM4_LOGS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_LOGS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_FC_NO_ADVERSARIAL_LOGS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_NO_IMAGE_LOGS_DIR, \\\n",
    "                           SRGAN_MSE_LOSS_WEIGHTS_DIR, \\\n",
    "                           SRGAN_MSE_LOSS_BEST_WEIGHT, \\\n",
    "                           SRGAN_VGG_LOSS_3_1_GENERATOR_WEIGHTS_DIR, \\\n",
    "                           SRGAN_VGG_LOSS_3_1_DISCRIMINATOR_WEIGHTS_DIR, \\\n",
    "                           SRGAN_VGG_LOSS_3_1_NO_ADVERSARIAL_WEIGHTS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_MFM4_GENERATOR_WEIGHTS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_MFM4_DISCRIMINATOR_WEIGHTS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_WEIGHTS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_FC_NO_ADVERSARIAL_WEIGHTS_DIR, \\\n",
    "                           SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_NO_IMAGE_WEIGHTS_DIR\n",
    "                        \n",
    "                        \n",
    "from utils import maybe_mkdir, remove_if_exists, get_last_file\n",
    "\n",
    "from src.light_cnn import LightCNN_9Layers\n",
    "from src.srgan import Generator, Discriminator, FeatureExtractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '7'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rgb_to_gray(x):\n",
    "    return 0.21 * x[:,0:1] + 0.72 * x[:,1:2] + 0.07 * x[:,2:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Grayscale2RGBModule(nn.Module):\n",
    "    def __init__(self, submodule, result_extractor=lambda x: x):\n",
    "        super().__init__()\n",
    "        self.submodule = submodule\n",
    "        self.result_extractor = result_extractor\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = rgb_to_gray(x)\n",
    "        result = self.submodule.forward(x)\n",
    "        return self.result_extractor(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MSCeleb(ImageFolder):\n",
    "    def __init__(self, root):\n",
    "        super().__init__(root)\n",
    "        \n",
    "        self._to_tensor = transforms.Compose(\n",
    "            [\n",
    "                transforms.ToTensor(),\n",
    "            ]\n",
    "        )\n",
    "        \n",
    "        self._downscale = transforms.Compose(\n",
    "            [\n",
    "                transforms.ToPILImage(),\n",
    "                transforms.Resize(32, 0),\n",
    "                transforms.ToTensor(),\n",
    "            ]\n",
    "        )\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        image, _ = super().__getitem__(index)\n",
    "        \n",
    "        image_hr = self._to_tensor(image)\n",
    "        image_lr = self._downscale(image_hr)\n",
    "        \n",
    "        return image_hr, image_lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = MSCeleb(FILTERED_MS_CELEB_IMAGES_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset_loader = DataLoader(\n",
    "    dataset, batch_size=16, shuffle=True,\n",
    "    num_workers=4, drop_last=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SRGANManager:\n",
    "    def __init__(self,\n",
    "                 dataset_loader,\n",
    "                 n_resblocks=16, n_upsample=2,\n",
    "                 generator_lr=0.0001, discriminator_lr=0.0001,\n",
    "                 batch_size=16,\n",
    "                 cuda=True):\n",
    "        self._batch_size = batch_size\n",
    "        \n",
    "        self._generator = Generator(n_resblocks, n_upsample)\n",
    "        self._discriminator = Discriminator()\n",
    "        \n",
    "        self._content_criterion = nn.MSELoss()\n",
    "        self._adversarial_criterion = nn.BCELoss()\n",
    "        self._ones_const = Variable(torch.ones(BATCH_SIZE, 1))\n",
    "        \n",
    "        self._cuda = cuda\n",
    "        if self._cuda is True:\n",
    "            self._generator = nn.DataParallel(self._generator).cuda()\n",
    "            self._discriminator = nn.DataParallel(self._discriminator).cuda()\n",
    "            self._content_criterion = self._content_criterion.cuda()\n",
    "            self._adversarial_criterion = self._adversarial_criterion.cuda()\n",
    "            self._ones_const = self._ones_const.cuda()\n",
    "        \n",
    "        self._optim_generator = optim.Adam(\n",
    "            self._generator.parameters(), lr=generator_lr\n",
    "        )\n",
    "        self._optim_discriminator = optim.Adam(\n",
    "            self._discriminator.parameters(), lr=discriminator_lr\n",
    "        )\n",
    "        \n",
    "        self._dataset_loader = dataset_loader\n",
    "        \n",
    "    def load_generator_weights(self, path):\n",
    "        self._generator.load_state_dict(torch.load(path))\n",
    "        \n",
    "    def load_discriminator_weights(self, path):\n",
    "        self._discriminator.load_state_dict(torch.load(path))\n",
    "\n",
    "    def train_mse_only(self,\n",
    "                       log_dir=SRGAN_MSE_LOSS_LOGS_DIR,\n",
    "                       epoch_count=3,\n",
    "                       generator_weights_dir=SRGAN_MSE_LOSS_WEIGHTS_DIR,\n",
    "                       save_frequency=15000,\n",
    "                       values_log_frequency=300,\n",
    "                       images_log_frequency=3000,\n",
    "                       start_log=100,\n",
    "                       train_type='careful'):\n",
    "        self._train(\n",
    "            log_dir=log_dir,\n",
    "            epoch_count=epoch_count,\n",
    "            train_function=self._mse_only_train_function,\n",
    "            generator_weights_dir=generator_weights_dir,\n",
    "            save_frequency=save_frequency,\n",
    "            values_log_frequency=values_log_frequency,\n",
    "            images_log_frequency=images_log_frequency,\n",
    "            start_log=start_log,\n",
    "            train_type=train_type\n",
    "        )\n",
    "        \n",
    "    def _mse_only_train_function(self, high_res_real, high_res_fake):\n",
    "        self._generator.zero_grad()\n",
    "\n",
    "        generator_content_loss = self._content_criterion(\n",
    "            high_res_fake, high_res_real,\n",
    "        )\n",
    "\n",
    "        generator_content_loss.backward()\n",
    "        self._optim_generator.step()\n",
    "        \n",
    "        return {\n",
    "            'generator_mse_loss': generator_content_loss,\n",
    "        }\n",
    "    \n",
    "    def train_light_cnn_9_loss_mfm4(self,\n",
    "                                    log_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_LOGS_DIR,\n",
    "                                    generator_weights_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_GENERATOR_WEIGHTS_DIR,\n",
    "                                    epoch_count=3,\n",
    "                                    save_frequency=5000,\n",
    "                                    values_log_frequency=100,\n",
    "                                    images_log_frequency=1000,\n",
    "                                    start_log=50,\n",
    "                                    train_type='careful',\n",
    "                                    load_pretrained_generator=SRGAN_MSE_LOSS_BEST_WEIGHT,\n",
    "                                    features_weight=0.002,\n",
    "                                    image_weight=1,\n",
    "                                    content_weight=1,\n",
    "                                    adversarial_weight=1e-3,\n",
    "                                    discriminator_weights_dir=None):\n",
    "        \n",
    "        light_cnn = self._init_light_cnn(\n",
    "            weight_path=LIGHT_CNN_9_WEIGHT,\n",
    "            name='LightCNN_9'\n",
    "        )\n",
    "        \n",
    "        grayscale_feature_extractor = FeatureExtractor(\n",
    "            light_cnn, 6,\n",
    "        )\n",
    "        \n",
    "        feature_extractor = Grayscale2RGBModule(\n",
    "            grayscale_feature_extractor\n",
    "        )\n",
    "        \n",
    "        self._train_perceptual_loss(\n",
    "            log_dir=log_dir,\n",
    "            epoch_count=epoch_count,\n",
    "            generator_weights_dir=generator_weights_dir,\n",
    "            discriminator_weights_dir=discriminator_weights_dir,\n",
    "            feature_extractor=feature_extractor,\n",
    "            save_frequency=save_frequency,\n",
    "            values_log_frequency=values_log_frequency,\n",
    "            images_log_frequency=images_log_frequency,\n",
    "            start_log=start_log,\n",
    "            train_type=train_type,\n",
    "            load_pretrained_generator=load_pretrained_generator,\n",
    "            features_weight=features_weight,\n",
    "            image_weight=image_weight,\n",
    "            content_weight=content_weight,\n",
    "            adversarial_weight=adversarial_weight,\n",
    "        )\n",
    "    \n",
    "    def train_light_cnn_9_loss_fc(self,\n",
    "                                  log_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_LOGS_DIR,\n",
    "                                  generator_weights_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_GENERATOR_WEIGHTS_DIR,\n",
    "                                  epoch_count=3,\n",
    "                                  save_frequency=5000,\n",
    "                                  values_log_frequency=100,\n",
    "                                  images_log_frequency=1000,\n",
    "                                  start_log=50,\n",
    "                                  train_type='careful',\n",
    "                                  load_pretrained_generator=SRGAN_MSE_LOSS_BEST_WEIGHT,\n",
    "                                  features_weight=7e-5,\n",
    "                                  image_weight=1,\n",
    "                                  content_weight=1,\n",
    "                                  adversarial_weight=1e-3,\n",
    "                                  discriminator_weights_dir=None):\n",
    "        \n",
    "        light_cnn = self._init_light_cnn(\n",
    "            weight_path=LIGHT_CNN_9_WEIGHT,\n",
    "            name='LightCNN_9'\n",
    "        )\n",
    "        \n",
    "        feature_extractor = Grayscale2RGBModule(\n",
    "            light_cnn,\n",
    "            lambda x: x[1],\n",
    "        )\n",
    "        \n",
    "        self._train_perceptual_loss(\n",
    "            log_dir=log_dir,\n",
    "            epoch_count=epoch_count,\n",
    "            generator_weights_dir=generator_weights_dir,\n",
    "            discriminator_weights_dir=discriminator_weights_dir,\n",
    "            feature_extractor=feature_extractor,\n",
    "            save_frequency=save_frequency,\n",
    "            values_log_frequency=values_log_frequency,\n",
    "            images_log_frequency=images_log_frequency,\n",
    "            start_log=start_log,\n",
    "            train_type=train_type,\n",
    "            load_pretrained_generator=load_pretrained_generator,\n",
    "            features_weight=features_weight,\n",
    "            image_weight=image_weight,\n",
    "            content_weight=content_weight,\n",
    "            adversarial_weight=adversarial_weight,\n",
    "        )\n",
    "    \n",
    "    def _init_light_cnn(self, weight_path, name='LightCNN_9'):\n",
    "        if name is 'LightCNN_9':\n",
    "            model_class = LightCNN_9Layers\n",
    "            num_classes=79077\n",
    "        else:\n",
    "            raise ValueError('No such model {}'.format(name))\n",
    "        \n",
    "        model = model_class(num_classes=num_classes)\n",
    "        model.eval()\n",
    "        \n",
    "        checkpoint = torch.load(weight_path)\n",
    "        \n",
    "        # remove DataParallel dependence\n",
    "        new_checkpoint = OrderedDict()\n",
    "        for layer_name, value in checkpoint['state_dict'].items():\n",
    "            new_layer_name = layer_name[7:] # remove `module.`\n",
    "            new_checkpoint[new_layer_name] = value\n",
    "        \n",
    "        model.load_state_dict(new_checkpoint)\n",
    "\n",
    "        return model\n",
    "    \n",
    "    def train_vgg_loss(self,\n",
    "                       log_dir,\n",
    "                       generator_weights_dir,\n",
    "                       discriminator_weights_dir=None,\n",
    "                       epoch_count=3,\n",
    "                       save_frequency=5000,\n",
    "                       values_log_frequency=100,\n",
    "                       images_log_frequency=1000,\n",
    "                       start_log=50,\n",
    "                       train_type='careful',\n",
    "                       load_pretrained_generator=SRGAN_MSE_LOSS_BEST_WEIGHT,\n",
    "                       features_weight=0.006,\n",
    "                       image_weight=1,\n",
    "                       content_weight=1,\n",
    "                       adversarial_weight=1e-3):\n",
    "        \n",
    "        feature_extractor = FeatureExtractor(\n",
    "            torchvision.models.vgg19(pretrained=True)\n",
    "        )\n",
    "        \n",
    "        self._train_perceptual_loss(\n",
    "            log_dir=log_dir,\n",
    "            epoch_count=epoch_count,\n",
    "            generator_weights_dir=generator_weights_dir,\n",
    "            discriminator_weights_dir=discriminator_weights_dir,\n",
    "            feature_extractor=feature_extractor,\n",
    "            save_frequency=save_frequency,\n",
    "            values_log_frequency=values_log_frequency,\n",
    "            images_log_frequency=images_log_frequency,\n",
    "            start_log=start_log,\n",
    "            train_type=train_type,\n",
    "            load_pretrained_generator=load_pretrained_generator,\n",
    "            features_weight=features_weight,\n",
    "            image_weight=image_weight,\n",
    "            content_weight=content_weight,\n",
    "            adversarial_weight=adversarial_weight,\n",
    "        )\n",
    "    \n",
    "    def _train_perceptual_loss(self,\n",
    "                               log_dir,\n",
    "                               generator_weights_dir,\n",
    "                               discriminator_weights_dir,\n",
    "                               epoch_count=3,\n",
    "                               feature_extractor=None,\n",
    "                               save_frequency=5000,\n",
    "                               values_log_frequency=100,\n",
    "                               images_log_frequency=1000,\n",
    "                               start_log=50,\n",
    "                               train_type='careful',\n",
    "                               load_pretrained_generator=SRGAN_MSE_LOSS_BEST_WEIGHT,\n",
    "                               features_weight=0.002,\n",
    "                               image_weight=1,\n",
    "                               content_weight=1,\n",
    "                               adversarial_weight=1e-3):\n",
    "        \n",
    "        assert feature_extractor is not None\n",
    "        \n",
    "        self._features_weight = features_weight\n",
    "        self._image_weight = image_weight\n",
    "        self._content_weight = content_weight\n",
    "        \n",
    "        if discriminator_weights_dir is not None:\n",
    "            self._adversarial_weight = adversarial_weight\n",
    "        \n",
    "        self._feature_extractor = feature_extractor\n",
    "        \n",
    "        if self._cuda is True:\n",
    "            self._feature_extractor = nn.DataParallel(\n",
    "                self._feature_extractor\n",
    "            ).cuda()\n",
    "        \n",
    "        if load_pretrained_generator is not None:\n",
    "            self.load_generator_weights(load_pretrained_generator)\n",
    "        \n",
    "        self._train(\n",
    "            log_dir=log_dir,\n",
    "            epoch_count=epoch_count,\n",
    "            train_function=self._perceptual_loss_train_function,\n",
    "            generator_weights_dir=generator_weights_dir,\n",
    "            discriminator_weights_dir=discriminator_weights_dir,\n",
    "            save_frequency=save_frequency,\n",
    "            values_log_frequency=values_log_frequency,\n",
    "            images_log_frequency=images_log_frequency,\n",
    "            start_log=start_log,\n",
    "            train_type=train_type\n",
    "        )\n",
    "        \n",
    "    def _perceptual_loss_train_function(self, high_res_real, high_res_fake):\n",
    "        # Additional variables\n",
    "        target_real = Variable(torch.rand(self._batch_size, 1) * 0.5 + 0.7)\n",
    "        target_fake = Variable(torch.rand(self._batch_size, 1) * 0.3)\n",
    "        \n",
    "        if self._cuda is True:\n",
    "            target_real = target_real.cuda()\n",
    "            target_fake = target_fake.cuda()\n",
    "        \n",
    "        # Train discriminator\n",
    "        if self._discriminator_weights_dir is not None:\n",
    "            self._discriminator.zero_grad()\n",
    "\n",
    "            discriminator_loss = self._adversarial_criterion(\n",
    "                self._discriminator(high_res_real), target_real\n",
    "            ) + self._adversarial_criterion(\n",
    "                self._discriminator(high_res_fake.detach()), target_fake\n",
    "            )\n",
    "\n",
    "            discriminator_loss.backward()\n",
    "            self._optim_discriminator.step()\n",
    "        \n",
    "        # Train generator \n",
    "        self._generator.zero_grad()\n",
    "        \n",
    "        real_features = self._feature_extractor(high_res_real).detach()\n",
    "        fake_features = self._feature_extractor(high_res_fake)\n",
    "        \n",
    "        generator_content_loss_image = self._content_criterion(\n",
    "            high_res_fake, high_res_real\n",
    "        )\n",
    "        generator_content_loss_features = self._content_criterion(\n",
    "            fake_features, real_features\n",
    "        )\n",
    "        \n",
    "        generator_content_loss_total = \\\n",
    "            self._image_weight * generator_content_loss_image + \\\n",
    "            self._features_weight * generator_content_loss_features\n",
    "        \n",
    "        if self._discriminator_weights_dir is not None:\n",
    "            generator_adversarial_loss = self._adversarial_criterion(\n",
    "                self._discriminator(high_res_fake), self._ones_const\n",
    "            )\n",
    "            \n",
    "            generator_total_loss = \\\n",
    "                self._content_weight * generator_content_loss_total + \\\n",
    "                self._adversarial_weight * generator_adversarial_loss\n",
    "        else:\n",
    "            generator_total_loss = generator_content_loss_total\n",
    "\n",
    "        generator_total_loss.backward()\n",
    "        self._optim_generator.step()\n",
    "        \n",
    "        log_dict = {\n",
    "            'generator_content_loss_image': generator_content_loss_image,\n",
    "            'generator_content_loss_features': generator_content_loss_features,\n",
    "            'generator_content_loss_total': generator_content_loss_total,\n",
    "        }\n",
    "        \n",
    "        if self._discriminator_weights_dir is not None:\n",
    "            log_dict.update(\n",
    "                {\n",
    "                    'discriminator_loss': discriminator_loss,\n",
    "                    'generator_adversarial_loss': generator_adversarial_loss,\n",
    "                    'generator_total_loss': generator_total_loss,\n",
    "                }\n",
    "            )\n",
    "        \n",
    "        return log_dict\n",
    "        \n",
    "    def _train(self,\n",
    "               log_dir,\n",
    "               epoch_count,\n",
    "               train_function,\n",
    "               generator_weights_dir,\n",
    "               save_frequency=15000,\n",
    "               values_log_frequency=300,\n",
    "               images_log_frequency=3000,\n",
    "               start_log=100,\n",
    "               train_type='careful',\n",
    "               discriminator_weights_dir=None):\n",
    "        \n",
    "        self._generator_weights_dir = generator_weights_dir\n",
    "        self._discriminator_weights_dir = discriminator_weights_dir\n",
    "        \n",
    "        step_num = self._prepare_train_type(\n",
    "            train_type, log_dir,\n",
    "            generator_weights_dir, discriminator_weights_dir\n",
    "        )\n",
    "        \n",
    "        try:\n",
    "            logger = Logger(log_dir)\n",
    "            progress_bar = tqdm(\n",
    "                total=len(self._dataset_loader),\n",
    "                initial=step_num % len(self._dataset_loader),\n",
    "            )\n",
    "            \n",
    "            while True:\n",
    "                stopped = False\n",
    "                for image_hr, image_lr in self._dataset_loader:\n",
    "                    epoch_num = step_num // len(self._dataset_loader)\n",
    "                    if epoch_num == epoch_count:\n",
    "                        stopped = True\n",
    "                        break\n",
    "                    progress_bar.desc = 'Epoch {}:'.format(epoch_num + 1)\n",
    "                    progress_bar.n = step_num % len(self._dataset_loader) + 1\n",
    "                    progress_bar.refresh()\n",
    "                    \n",
    "                    high_res_real = Variable(image_hr)\n",
    "                    low_res = Variable(image_lr)\n",
    "                    high_res_fake = self._generator(low_res)\n",
    "                    if self._cuda is True:\n",
    "                        high_res_real = high_res_real.cuda()\n",
    "                        high_res_fake = high_res_fake.cuda()\n",
    "\n",
    "                    values_dict = train_function(high_res_real, high_res_fake)\n",
    "\n",
    "                    if step_num >= start_log:\n",
    "                        if step_num % values_log_frequency == 0:\n",
    "                            for key, value in values_dict.items():\n",
    "                                logger.log_value(\n",
    "                                    key,\n",
    "                                    value,\n",
    "                                    step_num\n",
    "                                )\n",
    "\n",
    "                        if step_num % images_log_frequency == 0:\n",
    "                            logger.log_images(\n",
    "                                'real_images',\n",
    "                                high_res_real.data.cpu()[:1],\n",
    "                                step_num\n",
    "                            )\n",
    "\n",
    "                            logger.log_images(\n",
    "                                'lr_images',\n",
    "                                low_res.data.cpu()[:1],\n",
    "                                step_num\n",
    "                            )\n",
    "\n",
    "                            logger.log_images(\n",
    "                                'fake_images',\n",
    "                                high_res_fake.data.cpu()[:1],\n",
    "                                step_num\n",
    "                            )\n",
    "\n",
    "                    if step_num % save_frequency == 0 and step_num != 0:\n",
    "                        self._save_all(step_num, generator_weights_dir, discriminator_weights_dir)\n",
    "                    \n",
    "                    step_num += 1\n",
    "                    \n",
    "                self._save_all(step_num, generator_weights_dir, discriminator_weights_dir)\n",
    "                if stopped is True:\n",
    "                    break\n",
    "        except KeyboardInterrupt:\n",
    "            print('Stopped.')\n",
    "        except Exception:\n",
    "            print(traceback.format_exc())\n",
    "            self._log_error(traceback)\n",
    "        finally:\n",
    "            self._save_all(step_num, generator_weights_dir, discriminator_weights_dir)\n",
    "            progress_bar.close()\n",
    "        \n",
    "    def _save_all(self, step_num, generator_weights_dir, discriminator_weights_dir=None):\n",
    "        torch.save(\n",
    "            self._generator.state_dict(), self._make_weight_path(generator_weights_dir, step_num)\n",
    "        )\n",
    "        \n",
    "        if discriminator_weights_dir is not None:\n",
    "            torch.save(\n",
    "                self._discriminator.state_dict(), self._make_weight_path(discriminator_weights_dir, step_num)\n",
    "            )\n",
    "        \n",
    "    def _prepare_train_type(self, train_type, log_dir,\n",
    "                            generator_weights_dir, discriminator_weights_dir=None):\n",
    "        step = 0\n",
    "        \n",
    "        if train_type is 'careful':\n",
    "            if os.path.exists(log_dir) or os.path.exists(generator_weights_dir) or \\\n",
    "               (discriminator_weights_dir is not None and os.path.exists(discriminator_weights_dir)):\n",
    "                raise AssertionError(\n",
    "                    'There are previous train files:\\nlog_dir: {}\\ngenerator_weights_dir: {}\\ndiscriminator_weights_dir: {}'.format(\n",
    "                        log_dir,\n",
    "                        generator_weights_dir,\n",
    "                        discriminator_weights_dir,\n",
    "                    )\n",
    "                )\n",
    "        elif train_type is 'clean':\n",
    "            print(\n",
    "                'Clean all train data for (Y/N):\\nlog_dir: {}\\ngenerator_weights_dir: {}\\ndiscriminator_weights_dir: {}'.format(\n",
    "                    log_dir,\n",
    "                    generator_weights_dir,\n",
    "                    discriminator_weights_dir,\n",
    "                )\n",
    "            )\n",
    "            answer = input()\n",
    "            if answer == 'Y':\n",
    "                remove_if_exists(log_dir)\n",
    "                remove_if_exists(generator_weights_dir)\n",
    "                if discriminator_weights_dir is not None:\n",
    "                    remove_if_exists(discriminator_weights_dir)\n",
    "            else:\n",
    "                raise Exception('Bad answer')\n",
    "        elif train_type is 'continue':\n",
    "            try:\n",
    "                filename = get_last_file(generator_weights_dir)\n",
    "                self.load_generator_weights(os.path.join(generator_weights_dir, filename))\n",
    "\n",
    "                if discriminator_weights_dir is not None:\n",
    "                    filename = get_last_file(discriminator_weights_dir)\n",
    "                    self.load_discriminator_weights(os.path.join(discriminator_weights_dir, filename))\n",
    "                \n",
    "                step = int(os.path.splitext(filename)[0])\n",
    "            except Exception:\n",
    "                raise Exception('Nothing to load')\n",
    "        else:\n",
    "            raise ValueError('No such train type')\n",
    "            \n",
    "        pathlib.Path(generator_weights_dir).mkdir(parents=True, exist_ok=True)\n",
    "        if discriminator_weights_dir is not None:\n",
    "            pathlib.Path(discriminator_weights_dir).mkdir(parents=True, exist_ok=True)\n",
    "        pathlib.Path(log_dir).mkdir(parents=True, exist_ok=True)\n",
    "        \n",
    "        return step\n",
    "        \n",
    "    def _make_weight_path(self, folder, step):\n",
    "        return os.path.join(folder, '{:010d}.pth'.format(step))\n",
    "    \n",
    "    def _log_error(self, traceback, diirectory=ERRORS_LOGS_DIR):\n",
    "        error_path = os.path.join(\n",
    "            ERRORS_LOGS_DIR,\n",
    "            datetime.datetime.now().strftime(\n",
    "                '%Y-%m-%d-%H-%M-%S-%f'\n",
    "            )\n",
    "        )\n",
    "        with open(error_path, 'w') as file:\n",
    "            file.write(traceback.format_exc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "srgan_manager = SRGANManager(dataset_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      "  0%|          | 0/287187 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 1/287187 [00:00<49:35:23,  1.61it/s]\u001b[A\n",
      "  0%|          | 2/287187 [00:00<37:06:53,  2.15it/s]\u001b[A\n",
      "  0%|          | 4/287187 [00:00<28:11:47,  2.83it/s]\u001b[A\n",
      "  0%|          | 6/287187 [00:01<21:36:46,  3.69it/s]\u001b[A\n",
      "  0%|          | 8/287187 [00:01<16:57:47,  4.70it/s]\u001b[A\n",
      "  0%|          | 10/287187 [00:01<14:07:16,  5.65it/s]\u001b[A\n",
      "  0%|          | 12/287187 [00:01<11:51:25,  6.73it/s]\u001b[A\n",
      "  0%|          | 14/287187 [00:01<10:12:19,  7.82it/s]\u001b[A\n",
      "  0%|          | 16/287187 [00:01<9:15:05,  8.62it/s] \u001b[A\n",
      "  0%|          | 18/287187 [00:02<8:36:18,  9.27it/s]\u001b[A\n",
      " 21%|██        | 59307/287187 [1:34:13<6:04:02, 10.43it/s] Process Process-13:\n",
      "Process Process-15:\n",
      "Process Process-16:\n",
      "Process Process-14:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "Exception ignored in: <bound method DataLoaderIter.__del__ of <torch.utils.data.dataloader.DataLoaderIter object at 0x7ff224748d68>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 333, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 319, in _shutdown_workers\n",
      "    self.data_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "srgan_manager.train_mse_only(epoch_count=3, train_type='careful')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3  9%|▉         | 25967/287187 [1:43:21<17:19:49,  4.19it/s]Process Process-12:\n",
      "Epoch 3  9%|▉         | 25968/287187 [1:43:22<17:19:48,  4.19it/s]Exception ignored in: <bound method DataLoaderIter.__del__ of <torch.utils.data.dataloader.DataLoaderIter object at 0x7f3889126630>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 333, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 319, in _shutdown_workers\n",
      "    self.data_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "KeyboardInterrupt\n",
      "Process Process-9:\n",
      "Traceback (most recent call last):\n",
      "Epoch 3  9%|▉         | 25968/287187 [1:43:22<17:19:51,  4.19it/s]Traceback (most recent call last):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-17-fd41f254d670>\", line 309, in _train\n",
      "    values_dict = train_function(high_res_real, high_res_fake)\n",
      "  File \"<ipython-input-17-fd41f254d670>\", line 245, in _perceptual_loss_train_function\n",
      "    self._optim_generator.step()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/optim/adam.py\", line 69, in step\n",
      "    exp_avg.mul_(beta1).add_(1 - beta1, grad)\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-11:\n",
      "\n",
      "Process Process-10:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n"
     ]
    }
   ],
   "source": [
    "srgan_manager.train_vgg_loss(\n",
    "    log_dir=SRGAN_VGG_LOSS_3_1_LOGS_DIR,\n",
    "    generator_weights_dir=SRGAN_VGG_LOSS_3_1_GENERATOR_WEIGHTS_DIR,\n",
    "    discriminator_weights_dir=SRGAN_VGG_LOSS_3_1_DISCRIMINATOR_WEIGHTS_DIR,\n",
    "    epoch_count=3, train_type='careful'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clean all train data for (Y/N):\n",
      "log_dir: /home/data/alpus/clean_fsr/data/logs/srgan/vgg_loss_3.1_no_adversarial\n",
      "generator_weights_dir: /home/data/alpus/clean_fsr/data/weights/srgan/vgg_loss_3.1_no_adversarial\n",
      "discriminator_weights_dir: None\n",
      "Y\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 1:   0%|          | 0/287187 [00:00<?, ?it/s]\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Epoch 1:   0%|          | 1/287187 [00:00<60:32:36,  1.32it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 2/287187 [00:00<45:40:47,  1.75it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 3/287187 [00:01<35:12:57,  2.27it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 4/287187 [00:01<27:37:36,  2.89it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 5/287187 [00:01<22:11:59,  3.59it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 6/287187 [00:01<18:22:27,  4.34it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 7/287187 [00:01<15:40:45,  5.09it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 8/287187 [00:01<13:51:48,  5.75it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 9/287187 [00:01<12:35:50,  6.33it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 10/287187 [00:01<11:42:30,  6.81it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 11/287187 [00:02<11:22:07,  7.02it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 12/287187 [00:02<11:07:07,  7.17it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 13/287187 [00:02<10:56:46,  7.29it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 14/287187 [00:02<10:33:59,  7.55it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 15/287187 [00:02<10:14:42,  7.79it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 16/287187 [00:02<10:03:23,  7.93it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 17/287187 [00:02<10:06:46,  7.89it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 18/287187 [00:02<10:03:20,  7.93it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 19/287187 [00:03<10:00:09,  7.97it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 20/287187 [00:03<9:52:10,  8.08it/s] \u001b[A\n",
      "Epoch 1:   0%|          | 21/287187 [00:03<9:45:25,  8.18it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 22/287187 [00:03<9:36:35,  8.30it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 23/287187 [00:03<9:37:28,  8.29it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 24/287187 [00:03<16:39:07,  4.79it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 25/287187 [00:04<14:48:54,  5.38it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 26/287187 [00:04<13:29:31,  5.91it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 27/287187 [00:04<12:30:22,  6.38it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 28/287187 [00:04<11:36:04,  6.88it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 29/287187 [00:04<11:00:25,  7.25it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 30/287187 [00:04<10:31:20,  7.58it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 31/287187 [00:04<10:11:59,  7.82it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 32/287187 [00:04<9:55:08,  8.04it/s] \u001b[A\n",
      "Epoch 1:   0%|          | 33/287187 [00:04<9:46:14,  8.16it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 34/287187 [00:05<9:41:21,  8.23it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 35/287187 [00:05<9:36:33,  8.30it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 36/287187 [00:05<9:34:24,  8.33it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 37/287187 [00:05<9:27:00,  8.44it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 38/287187 [00:05<9:23:00,  8.50it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 39/287187 [00:05<9:23:21,  8.50it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 40/287187 [00:05<9:34:48,  8.33it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 41/287187 [00:05<9:45:58,  8.17it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 42/287187 [00:06<9:44:15,  8.19it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 43/287187 [00:06<9:38:13,  8.28it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 44/287187 [00:06<9:51:51,  8.09it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 45/287187 [00:06<9:54:23,  8.05it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 46/287187 [00:06<9:49:41,  8.12it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 47/287187 [00:06<9:50:32,  8.10it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 48/287187 [00:06<9:45:30,  8.17it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 49/287187 [00:06<9:53:38,  8.06it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 50/287187 [00:07<9:47:42,  8.14it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 51/287187 [00:07<9:47:48,  8.14it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 52/287187 [00:07<10:00:38,  7.97it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 53/287187 [00:07<10:07:08,  7.88it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 54/287187 [00:07<10:02:50,  7.94it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 55/287187 [00:07<9:59:46,  7.98it/s] \u001b[A\n",
      "Epoch 1:   0%|          | 56/287187 [00:07<10:06:20,  7.89it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 57/287187 [00:07<10:10:51,  7.83it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 58/287187 [00:08<10:09:30,  7.85it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 59/287187 [00:08<10:00:14,  7.97it/s]\u001b[A\n",
      "Epoch 1:   0%|          | 60/287187 [00:08<9:52:00,  8.08it/s] \u001b[A\n",
      "Epoch 1:   0%|          | 61/287187 [00:08<9:49:14,  8.12it/s]\u001b[A\n",
      "Epoch 1: 100%|██████████| 287187/287187 [10:31:26<00:00,  5.90it/s] \n",
      "Epoch 3:  10%|▉         | 27932/287187 [1:08:47<9:04:36,  7.93it/s] Process Process-16:\n",
      "Process Process-15:\n",
      "Process Process-14:\n",
      "Process Process-13:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-9-ad068de2b895>\", line 298, in _train\n",
      "    values_dict = train_function(high_res_real, high_res_fake)\n",
      "  File \"<ipython-input-9-ad068de2b895>\", line 244, in _perceptual_loss_train_function\n",
      "    generator_total_loss.backward()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/autograd/variable.py\", line 167, in backward\n",
      "    torch.autograd.backward(self, gradient, retain_graph, create_graph, retain_variables)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/autograd/__init__.py\", line 99, in backward\n",
      "    variables, grad_variables, retain_graph)\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "Exception ignored in: <bound method DataLoaderIter.__del__ of <torch.utils.data.dataloader.DataLoaderIter object at 0x7f5cf4aa3080>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 333, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 319, in _shutdown_workers\n",
      "    self.data_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "srgan_manager.train_vgg_loss(\n",
    "    log_dir=SRGAN_VGG_LOSS_3_1_NO_ADVERSARIAL_LOGS_DIR,\n",
    "    generator_weights_dir=SRGAN_VGG_LOSS_3_1_NO_ADVERSARIAL_WEIGHTS_DIR,\n",
    "    epoch_count=3,\n",
    "    save_frequency=10000,\n",
    "    values_log_frequency=200,\n",
    "    images_log_frequency=2000,\n",
    "    train_type='careful'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 55%|█████▍    | 156619/287186 [4:44:37<3:57:16,  9.17it/s]Process Process-19:\n",
      "Process Process-18:\n",
      "Process Process-17:\n",
      "Process Process-20:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Epoch 3: 55%|█████▍    | 156620/287186 [4:44:37<3:57:16,  9.17it/s]Exception ignored in: <bound method DataLoaderIter.__del__ of <torch.utils.data.dataloader.DataLoaderIter object at 0x7fc639f84b70>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 333, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 319, in _shutdown_workers\n",
      "    self.data_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "Epoch 3: 55%|█████▍    | 156620/287186 [4:44:37<3:57:16,  9.17it/s]  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopped.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "srgan_manager.train_light_cnn_9_loss_mfm4(\n",
    "    log_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_LOGS_DIR,\n",
    "    generator_weights_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_GENERATOR_WEIGHTS_DIR,\n",
    "    discriminator_weights_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_DISCRIMINATOR_WEIGHTS_DIR,\n",
    "    epoch_count=3, train_type='careful'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 81%|████████▏ | 233469/287186 [51:12<11:46, 75.99it/s]]]]]Process Process-16:\n",
      "Process Process-15:\n",
      "Process Process-13:\n",
      "Process Process-14:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 55, in _worker_loop\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 55, in <listcomp>\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"<ipython-input-6-ee4a06afb3b6>\", line 20, in __getitem__\n",
      "    image, _ = super().__getitem__(index)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "Epoch 3: 81%|████████▏ | 233471/287186 [51:12<11:46, 75.98it/s]Exception ignored in: <bound method DataLoaderIter.__del__ of <torch.utils.data.dataloader.DataLoaderIter object at 0x7fc639f950f0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 333, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 319, in _shutdown_workers\n",
      "    self.data_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "Epoch 3: 81%|████████▏ | 233471/287186 [51:12<11:46, 75.98it/s]KeyboardInterrupt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopped.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/usr/local/lib/python3.5/dist-packages/torchvision/datasets/folder.py\", line 122, in __getitem__\n",
      "    img = self.loader(path)\n",
      "\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torchvision/datasets/folder.py\", line 69, in default_loader\n",
      "    return pil_loader(path)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torchvision/datasets/folder.py\", line 52, in pil_loader\n",
      "    return img.convert('RGB')\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/PIL/Image.py\", line 860, in convert\n",
      "    self.load()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/PIL/ImageFile.py\", line 234, in load\n",
      "    n, err_code = decoder.decode(b)\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "srgan_manager.train_light_cnn_9_loss_mfm4(\n",
    "    log_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_LOGS_DIR,\n",
    "    generator_weights_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_WEIGHTS_DIR,\n",
    "    epoch_count=3,\n",
    "    save_frequency=10000,\n",
    "    values_log_frequency=200,\n",
    "    images_log_frequency=2000,\n",
    "    train_type='careful',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clean all train data for (Y/N):\n",
      "log_dir: /home/data/alpus/clean_fsr/data/logs/srgan/light_cnn_9_loss_fc_no_adversarial\n",
      "generator_weights_dir: /home/data/alpus/clean_fsr/data/weights/srgan/light_cnn_9_loss_fc_no_adversarial\n",
      "discriminator_weights_dir: None\n",
      "Y\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 43%|████▎     | 122744/287186 [28:59:04<38:49:52,  1.18it/s]]]]Process Process-15:\n",
      "Process Process-13:\n",
      "Process Process-14:\n",
      "Process Process-16:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Epoch 3: 43%|████▎     | 122745/287186 [28:59:05<38:49:50,  1.18it/s]Exception ignored in: <bound method DataLoaderIter.__del__ of <torch.utils.data.dataloader.DataLoaderIter object at 0x7f7ced0b0ef0>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 333, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 319, in _shutdown_workers\n",
      "    self.data_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "Epoch 3: 43%|████▎     | 122745/287186 [28:59:05<38:49:51,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopped.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "srgan_manager.train_light_cnn_9_loss_fc(\n",
    "    log_dir=SRGAN_LIGHT_CNN_9_LOSS_FC_NO_ADVERSARIAL_LOGS_DIR,\n",
    "    generator_weights_dir=SRGAN_LIGHT_CNN_9_LOSS_FC_NO_ADVERSARIAL_WEIGHTS_DIR,\n",
    "    epoch_count=3,\n",
    "    save_frequency=10000,\n",
    "    values_log_frequency=200,\n",
    "    images_log_frequency=2000,\n",
    "    train_type='careful',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 50%|████▉     | 142170/287186 [23:35:53<24:04:13,  1.67it/s]]]]Process Process-9:\n",
      "Traceback (most recent call last):\n",
      "Process Process-11:\n",
      "Process Process-10:\n",
      "Process Process-12:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "Exception ignored in: <bound method DataLoaderIter.__del__ of <torch.utils.data.dataloader.DataLoaderIter object at 0x7fec909b25f8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 333, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 319, in _shutdown_workers\n",
      "    self.data_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "Epoch 3: 50%|████▉     | 142170/287186 [23:35:53<24:04:13,  1.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopped.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "srgan_manager.train_light_cnn_9_loss_mfm4(\n",
    "    log_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_NO_IMAGE_LOGS_DIR,\n",
    "    generator_weights_dir=SRGAN_LIGHT_CNN_9_LOSS_MFM4_NO_ADVERSARIAL_NO_IMAGE_WEIGHTS_DIR,\n",
    "    epoch_count=3,\n",
    "    save_frequency=10000,\n",
    "    values_log_frequency=200,\n",
    "    images_log_frequency=2000,\n",
    "    image_weight=0,\n",
    "    train_type='careful',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}