ENTAR_Eng.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "ENTAR Eng",
      "provenance": [],
      "collapsed_sections": [
        "c7VjlYb2H9Ph",
        "ECm1EEXpPaTZ",
        "ZjPqTBNoohK9",
        "4AVBicY8T8XY",
        "148dzAWOTw29",
        "a165mYFXow8I",
        "lEhNSwi50Wa5",
        "hCmtWECtAE8X",
        "m0S8PshiV_aq",
        "Uu22Uwc3V_aw",
        "rDF6s4jYYF7I"
      ],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tg-bomze/ENTAR/blob/master/ENTAR_Eng.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c7VjlYb2H9Ph",
        "colab_type": "text"
      },
      "source": [
        "# <b><font color=\"black\" size=\"-5\">.</font></b>\n",
        "![alt text](https://raw.githubusercontent.com/tg-bomze/ENTAR/master/entar.png)\n",
        "\n",
        "<b><font color=\"black\" size=\"+3\">Ensemble of Neural Tools for Animations Restoration</font></b>\n",
        "\n",
        "<b><font color=\"red\">P.S. Sorry for my bad English. I translate using Google Translator</font></b>\n",
        "\n",
        "*↓ Open me ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VP3Be3bDIaNW",
        "colab_type": "text"
      },
      "source": [
        "##**Put it all together:**\n",
        "GitHub: [@tg-bomze](https://github.com/tg-bomze)\n",
        "\n",
        "Telegram: [@bomze](https://t.me/bomze) \n",
        "\n",
        "Twitter: [@tg_bomze](https://twitter.com/tg_bomze)\n",
        "\n",
        "*For all questions, Twitter.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OT-te5mSVW9-",
        "colab_type": "text"
      },
      "source": [
        "**Next are the blocks, divided into sections. The zero section is required. The rest is your choice.**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ECm1EEXpPaTZ",
        "colab_type": "text"
      },
      "source": [
        "# <b><font color=\"gree\" size=\"+3\">0. Preparation for processing</b>\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_-MnB32R4kZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Install all the necessary libraries** { display-mode: \"form\" }\n",
        "!pip install youtube_dl\n",
        "!pip install ffmpeg\n",
        "!pip install ffmpeg-python\n",
        "!pip install torchvision==0.5\n",
        "!pip install torch==1.4\n",
        "\n",
        "from IPython.display import clear_output\n",
        "from google.colab import files\n",
        "import imageio\n",
        "import youtube_dl\n",
        "import cv2\n",
        "import os\n",
        "import torch\n",
        "import fastai\n",
        "import ffmpeg\n",
        "import os.path as osp\n",
        "import logging\n",
        "import shutil\n",
        "import re\n",
        "import gc\n",
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "from os import path\n",
        "import numpy as np\n",
        "import moviepy.editor as mpy\n",
        "from moviepy.video.io.ffmpeg_writer import FFMPEG_VideoWriter\n",
        "from pathlib import Path\n",
        "import sys\n",
        "import glob\n",
        "from IPython import display as ipythondisplay\n",
        "from IPython.display import Image as ipythonimage\n",
        "torch.backends.cudnn.benchmark=True\n",
        "%matplotlib inline\n",
        "\n",
        "!rm -rf sample_data\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Bhwpq1R38FPw",
        "colab": {}
      },
      "source": [
        "#@title ##**Upload video** { display-mode: \"form\" }\n",
        "#@markdown *Enter a link to the video below (for example, YouTube or Twitter), or leave the **source_url** field blank (in this case, you will be asked to upload the video from your computer).*\n",
        "source_url = '' #@param {type:\"string\"}\n",
        "\n",
        "if source_url == '':\n",
        "  uploaded = files.upload()\n",
        "  for fn in uploaded.keys():\n",
        "    print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "        name=fn, length=len(uploaded[fn])))\n",
        "  os.rename(fn, fn.replace(\" \", \"\"))\n",
        "  fn = fn.replace(\" \", \"\")\n",
        "  file_name = \"downloaded_video.\" + fn.split(\".\")[-1]\n",
        "  !mv -f $fn $file_name\n",
        "\n",
        "else:\n",
        "  try:\n",
        "    ydl_opts = {\n",
        "        'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4',\n",
        "        'outtmpl': 'downloaded_video.mp4',\n",
        "        }\n",
        "    with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n",
        "      ydl.download([source_url])\n",
        "    file_name = 'downloaded_video.mp4'\n",
        "  \n",
        "  except BaseException:\n",
        "    !wget $source_url\n",
        "    fn = source_url.split('/')[-1]\n",
        "    os.rename(fn, fn.replace(\" \", \"\"))\n",
        "    fn = fn.replace(\" \", \"\")\n",
        "    file_name = \"downloaded_video.\" + fn.split(\".\")[-1]\n",
        "    !mv -f $fn $file_name\n",
        "\n",
        "!cp -r downloaded_video.mp4 video.mp4\n",
        "clear_output()\n",
        "fps_of_video = int(cv2.VideoCapture(file_name).get(cv2.CAP_PROP_FPS))\n",
        "frames_of_video = int(cv2.VideoCapture(file_name).get(cv2.CAP_PROP_FRAME_COUNT))\n",
        "#@markdown *Downloading videos lasting longer than one minute is not recommended. In addition, don't upload video that have a \"space\" or a \"dot\" in the title.*\n",
        "\n",
        "#@markdown *If there is an error during execution, then run this block again*"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_uPcd5NA30J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Crop video (Optional block)** { display-mode: \"form\" }\n",
        "#@markdown *Here you can crop the video if it is too long.*\n",
        "\n",
        "#@markdown **Enter the time frame for croping the video**\n",
        "target_start = '00:00:00' #@param {type:\"string\"}\n",
        "target_end = '00:00:05' #@param {type:\"string\"}\n",
        "\n",
        "#@markdown Enter a new resolution of the video file for rescaling (*for example, 640 and 480*), or leave it blank to work with the original resolution\n",
        "width = '' #@param {type:\"string\"}\n",
        "height = '' #@param {type:\"string\"}\n",
        "\n",
        "rescale = \"\"\n",
        "if width != '' and height != '':\n",
        "  rescale = f\"-s {width}x{height}\"\n",
        "\n",
        "!ffmpeg -i downloaded_video.mp4 $rescale -ss $target_start -to $target_end new_target.mp4\n",
        "clear_output()\n",
        "!rm video.mp4\n",
        "!mv new_target.mp4 video.mp4\n",
        "#@markdown *Display video:*\n",
        "display_video = False #@param {type:\"boolean\"}\n",
        "if display_video == True:\n",
        "  display(mpy.ipython_display(\"video.mp4\", height=400))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NwIlL8MOXfHN",
        "colab_type": "text"
      },
      "source": [
        "**You can execute the following sections in any order (the main thing is to execute each block inside a section one by one).**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ZjPqTBNoohK9"
      },
      "source": [
        "---\n",
        "# <b><font color=\"gree\" size=\"+3\">1. Colorize video ([DeOldify](https://github.com/jantic/DeOldify))\n",
        "*↓ Click to open section ↓*\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JPGpjSn70T-N",
        "colab_type": "text"
      },
      "source": [
        "Creator: [Jason Antic](https://twitter.com/citnaj)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-T-svuHytJ-8",
        "colab": {}
      },
      "source": [
        "#@title ##**Clone the repository and install the necessary dependencies** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "!git clone https://github.com/jantic/DeOldify.git\n",
        "%cd /content/DeOldify\n",
        "!pip install -r colab_requirements.txt\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XHjidueePj9g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Upload the pre-trained model** { display-mode: \"form\" }\n",
        "from deoldify.visualize import *\n",
        "!mkdir 'models'\n",
        "try:\n",
        "  !rm -rf models/ColorizeVideo_gen.pth\n",
        "  !gdown https://drive.google.com/uc?id=1-3ONnPYcX9fOqnY-pUKYGJOd6XeFon9X -O ./models/ColorizeVideo_gen.pth\n",
        "except BaseException:\n",
        "  !rm -rf models/ColorizeVideo_gen.pth\n",
        "  !wget https://www.dropbox.com/s/336vn9y4qwyg9yz/ColorizeVideo_gen.pth?dl=0 -O ./models/ColorizeVideo_gen.pth\n",
        "colorizer = get_video_colorizer()\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4spoDDLfPj90",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Colorize the video** { display-mode: \"form\" }\n",
        "#@markdown **21, 35** *- optimal values*\n",
        "!rm -rf video\n",
        "!mkdir 'video'\n",
        "!mkdir 'video/source'\n",
        "!cp -r /content/video.mp4 video/source/video.mp4\n",
        "\n",
        "render_factor = 21  #@param {type: \"slider\", min: 5, max: 44}\n",
        "video_path = colorizer.colorize_from_file_name('video.mp4', render_factor)\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DQa3ZcUIVm3F",
        "colab_type": "text"
      },
      "source": [
        "If an error occurs while executing a block from above, then convert the video [HERE](https://convert-video-online.com/ru/) and start the entire script again."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rMgsqYUKuF7U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Get result** { display-mode: \"form\" }\n",
        "!cp -r video/result/video.mp4 /content/colorized_video.mp4\n",
        "what_next = 'play' #@param [\"play\", \"download\"]\n",
        "if what_next == \"play\":\n",
        "  display(mpy.ipython_display(\"/content/colorized_video.mp4\", height=400, autoplay=1, loop=1, maxduration=600))\n",
        "else:\n",
        "  files.download('/content/colorized_video.mp4')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rGEPpJX3Pj95",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Optional block!** { display-mode: \"form\" }\n",
        "#@markdown *If it seems to you that the quality of coloring is insufficient, then run this block, wait until it is complete and look at what value of **render_factor** you like the picture more. Next, run the previous coloring block again, setting the desired value.*\n",
        "for i in range(10,45,2):\n",
        "    colorizer.vis.plot_transformed_image('video/bwframes/video/00001.jpg', render_factor=i, display_render_factor=True, figsize=(8,8))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QUN-VylMiIxT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Exit DeOldify** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "!rm -rf /content/video.mp4\n",
        "!cp -r video/result/video.mp4 /content/video.mp4\n",
        "!cp -r video/result/video.mp4 /content/colorized_video.mp4\n",
        "%cd /content\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4AVBicY8T8XY",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "# <b><font color=\"gree\" size=\"+3\">2. Enhancing video ([EDVR](https://github.com/xinntao/EDVR))\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NdtGbSHC0LU0",
        "colab_type": "text"
      },
      "source": [
        "Creator: [Xintao Wang](https://xinntao.github.io)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_8udPfjMUJHZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Clone the repository and upload the pre-trained model** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "!pip install numpy opencv-python lmdb pyyaml\n",
        "!pip install tb-nightly future\n",
        "!git clone https://github.com/xinntao/EDVR\n",
        "%cd /content/EDVR/codes/models/archs/dcn\n",
        "!python setup.py develop\n",
        "%cd /content/EDVR/codes\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1SGVehpZt4WL_X8Jh6blyqmHpc8DdImgv' -O /content/EDVR/experiments/pretrained_models/EDVR_REDS_deblurcomp_L.pth\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RPdZEJiFVoPf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Initialize the necessary functions** { display-mode: \"form\" }\n",
        "import utils.util as util\n",
        "import data.util as data_util\n",
        "import models.archs.EDVR_arch as EDVR_arch\n",
        "\n",
        "workfolder = Path('./video')\n",
        "source_folder = workfolder / \"source\"\n",
        "inframes_root = workfolder / \"inframes\"\n",
        "audio_root = workfolder / \"audio\"\n",
        "outframes_root = workfolder / \"outframes\"\n",
        "result_folder = workfolder / \"result\"\n",
        "pretrained_models = Path('../experiments/pretrained_models')\n",
        "\n",
        "def clean_mem():\n",
        "    # torch.cuda.empty_cache()\n",
        "    gc.collect()\n",
        "\n",
        "def get_fps(source_path: Path) -> str:\n",
        "    print(source_path)\n",
        "    probe = ffmpeg.probe(str(source_path))\n",
        "    stream_data = next(\n",
        "        (stream for stream in probe['streams'] if stream['codec_type'] == 'video'),\n",
        "        None,\n",
        "    )\n",
        "    return stream_data['avg_frame_rate']\n",
        "\n",
        "def preProcess(imag_path_l, multiple):\n",
        "  '''Need to resize images for blurred model (needs to be multiples of 16)'''\n",
        "  for img_path in imag_path_l:\n",
        "    im = Image.open(img_path)\n",
        "    h, w = im.size\n",
        "    # resize so they are multiples of 4 or 16 (for blurred)\n",
        "    h = h - h % multiple\n",
        "    w = w - w % multiple\n",
        "    im = im.resize((h,w))\n",
        "    im.save(img_path)\n",
        "\n",
        "def purge_images(dir):\n",
        "  for f in os.listdir(dir):\n",
        "    if re.search('.*?\\.jpg', f):\n",
        "      os.remove(os.path.join(dir, f))\n",
        "\n",
        "def extract_raw_frames(source_path: Path):\n",
        "    inframes_folder = inframes_root / (source_path.stem)\n",
        "    inframe_path_template = str(inframes_folder / '%5d.jpg')\n",
        "    inframes_folder.mkdir(parents=True, exist_ok=True)\n",
        "    purge_images(inframes_folder)\n",
        "    ffmpeg.input(str(source_path)).output(\n",
        "        str(inframe_path_template), format='image2', vcodec='mjpeg', qscale=0\n",
        "    ).run(capture_stdout=True)\n",
        "\n",
        "def make_subfolders(img_path_l, chunk_size):\n",
        "  i = 0\n",
        "  subFolderList = []\n",
        "  source_img_path = Path('/content/EDVR/codes/video/inframes/video_subfolders')\n",
        "  source_img_path.mkdir(parents=True, exist_ok=True)\n",
        "  for img in img_path_l:\n",
        "    if i % chunk_size == 0:\n",
        "      img_path = source_img_path / str(i)\n",
        "      img_path.mkdir(parents=True, exist_ok=True)\n",
        "      subFolderList.append(str(img_path))\n",
        "    i+=1\n",
        "    img_name = osp.basename(img)\n",
        "    img_path_name = img_path / img_name\n",
        "    shutil.copyfile(img, img_path_name)\n",
        "\n",
        "  return subFolderList\n",
        "\n",
        "def remove_subfolders():\n",
        "  shutil.rmtree('/content/EDVR/codes/video/inframes/video_subfolders', ignore_errors=True, onerror=None)\n",
        "\n",
        "def edvrPredict(data_mode, chunk_size, stage):\n",
        "  device = torch.device('cuda')\n",
        "  os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
        "  data_mode = \"blur_comp\"\n",
        "  stage = stage  # 1 or 2, use two stage strategy for REDS dataset.\n",
        "  flip_test = False\n",
        "  model_path = pretrained_models / 'EDVR_REDS_deblurcomp_L.pth'   \n",
        "\n",
        "  print('Model Used: ', model_path)\n",
        "  \n",
        "  if data_mode == 'Vid4':\n",
        "      N_in = 7  # use N_in images to restore one HR image\n",
        "  else:\n",
        "      N_in = 5\n",
        "\n",
        "  predeblur, HR_in = False, False\n",
        "  back_RBs = 40\n",
        "  if data_mode == 'blur' or data_mode == 'blur_comp':\n",
        "      predeblur, HR_in = True, True\n",
        "  if stage == 2:\n",
        "      HR_in = True\n",
        "      back_RBs = 20\n",
        "  if data_mode == 'TOF':\n",
        "    model = TOF_arch.TOFlow(adapt_official=True)\n",
        "  else:\n",
        "    model = EDVR_arch.EDVR(128, N_in, 8, 5, back_RBs, predeblur=predeblur, HR_in=HR_in)\n",
        "\n",
        "  #### dataset\n",
        "  test_dataset_folder = '/content/EDVR/codes/video/inframes'\n",
        "\n",
        "  #### evaluation\n",
        "  crop_border = 0\n",
        "  border_frame = N_in // 2  # border frames when evaluate\n",
        "  # temporal padding mode\n",
        "  if data_mode in ('Vid4','sharp_bicubic'):\n",
        "      padding = 'new_info'\n",
        "  else:\n",
        "      padding = 'replicate'\n",
        "  save_imgs = True\n",
        "\n",
        "  save_folder = '/content/EDVR/codes/video/outframes'\n",
        "  util.mkdirs(save_folder)\n",
        "\n",
        "  #### set up the models\n",
        "  model.load_state_dict(torch.load(model_path), strict=True)\n",
        "  model.eval()\n",
        "  model = model.to(device)\n",
        "\n",
        "  avg_psnr_l, avg_psnr_center_l, avg_psnr_border_l = [], [], []\n",
        "  subfolder_name_l = []\n",
        "  # remove old video_subfolder if exists\n",
        "  remove_subfolders()\n",
        "  subfolder_l = sorted(glob.glob(osp.join(test_dataset_folder, '*')))\n",
        "\n",
        "  # for each subfolder\n",
        "  for subfolder in subfolder_l:\n",
        "      subfolder_name = osp.basename(subfolder)\n",
        "      subfolder_name_l.append(subfolder_name)\n",
        "      save_subfolder = osp.join(save_folder, subfolder_name)\n",
        "\n",
        "      img_path_l = sorted(glob.glob(osp.join(subfolder, '*')))\n",
        "      if save_imgs:\n",
        "          util.mkdirs(save_subfolder)\n",
        "          purge_images(save_subfolder)\n",
        "\n",
        "      # preprocess images (needed for blurred models)\n",
        "      if predeblur:\n",
        "        preProcess(img_path_l, 16)\n",
        "      else:\n",
        "        preProcess(img_path_l, 4)\n",
        "      # make even more subfolders\n",
        "      subFolderList = make_subfolders(img_path_l, chunk_size)\n",
        "\n",
        "      #### read LQ and GT images in chunks of 1000\n",
        "      for subSubFolder in subFolderList:\n",
        "        clean_mem()\n",
        "        imgs_LQ = data_util.read_img_seq(subSubFolder)\n",
        "        subSubFolder_l = sorted(glob.glob(osp.join(subSubFolder, '*')))\n",
        "        max_idx = len(subSubFolder_l)\n",
        "        avg_psnr, avg_psnr_border, avg_psnr_center, N_border, N_center = 0, 0, 0, 0, 0\n",
        "\n",
        "        # process each image\n",
        "        for img_idx, img_path in tqdm(enumerate(subSubFolder_l)):\n",
        "            img_name = osp.splitext(osp.basename(img_path))[0]\n",
        "            select_idx = data_util.index_generation(img_idx, max_idx, N_in, padding=padding)\n",
        "            imgs_in = imgs_LQ.index_select(0, torch.LongTensor(select_idx)).unsqueeze(0).to(device)\n",
        "\n",
        "            if flip_test:\n",
        "                output = util.flipx4_forward(model, imgs_in)\n",
        "            else:\n",
        "                output = util.single_forward(model, imgs_in)\n",
        "            output = util.tensor2img(output.squeeze(0))\n",
        "\n",
        "            if save_imgs:\n",
        "                cv2.imwrite(osp.join(save_subfolder, '{}.jpg'.format(img_name)), output)\n",
        "                # print('Saved Image:', str(osp.join(save_subfolder, '{}.jpg'.format(img_name))))\n",
        "\n",
        "def moveProcessedFrames():\n",
        "  shutil.rmtree('/content/EDVR/codes/video/inframes')\n",
        "  os.rename('/content/EDVR/codes/video/outframes', '/content/EDVR/codes/video/inframes')\n",
        "\n",
        "def build_video(source_path: Path) -> Path:\n",
        "        out_path = result_folder / (\n",
        "            source_path.name.replace('.mp4', '_no_audio.mp4')\n",
        "        )\n",
        "        outframes_folder = outframes_root / (source_path.stem)\n",
        "        outframes_path_template = str(outframes_folder / '%5d.jpg')\n",
        "        out_path.parent.mkdir(parents=True, exist_ok=True)\n",
        "        if out_path.exists():\n",
        "            out_path.unlink()\n",
        "        fps = get_fps(source_path)\n",
        "        print('Original FPS is: ', fps)\n",
        "\n",
        "        ffmpeg.input(\n",
        "            str(outframes_path_template),\n",
        "            format='image2',\n",
        "            vcodec='mjpeg',\n",
        "            framerate=fps,\n",
        "        ).output(str(out_path), crf=17, vcodec='libx264').run(capture_stdout=True)\n",
        "\n",
        "        result_path = result_folder / source_path.name\n",
        "        if result_path.exists():\n",
        "            result_path.unlink()\n",
        "        # making copy of non-audio version in case adding back audio doesn't apply or fails.\n",
        "        shutil.copyfile(str(out_path), str(result_path))\n",
        "\n",
        "        # adding back sound here\n",
        "        audio_file = Path(str(source_path).replace('.mp4', '.aac'))\n",
        "        if audio_file.exists():\n",
        "            audio_file.unlink()\n",
        "\n",
        "        os.system(\n",
        "            'ffmpeg -y -i \"'\n",
        "            + str(source_path)\n",
        "            + '\" -vn -acodec copy \"'\n",
        "            + str(audio_file)\n",
        "            + '\"'\n",
        "        )\n",
        "\n",
        "        if audio_file.exists:\n",
        "            os.system(\n",
        "                'ffmpeg -y -i \"'\n",
        "                + str(out_path)\n",
        "                + '\" -i \"'\n",
        "                + str(audio_file)\n",
        "                + '\" -shortest -c:v copy -c:a aac -b:a 256k \"'\n",
        "                + str(result_path)\n",
        "                + '\"'\n",
        "            )\n",
        "        print('Video created here: ' + str(result_path))\n",
        "        return result_path\n",
        "\n",
        "def edvr_video(source_path: Path, chunk_size: int):\n",
        "    # extract frames\n",
        "    extract_raw_frames(source_path)\n",
        "\n",
        "    # process frames\n",
        "    edvrPredict(\"blur_comp\", chunk_size, 1)\n",
        "\n",
        "    # build back video\n",
        "    build_video(source_path)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGEngN6UXAcZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Enhancing the quality of frames** { display-mode: \"form\" }\n",
        "%%time\n",
        "edvr_video(Path('/content/video.mp4'), 100)\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pwwhj7Rf4v0k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Get result** { display-mode: \"form\" }\n",
        "!rm -rf /content/video.mp4\n",
        "!cp -r /content/EDVR/codes/video/result/video.mp4 /content/video.mp4\n",
        "!cp -r /content/EDVR/codes/video/result/video.mp4 /content/enhanced_video.mp4\n",
        "what_next = 'play' #@param [\"play\", \"download\"]\n",
        "if what_next == \"play\":\n",
        "  display(mpy.ipython_display(\"/content/enhanced_video.mp4\", height=400, autoplay=1, loop=1, maxduration=600))\n",
        "else:\n",
        "  files.download('/content/enhanced_video.mp4')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EWgDUMFN3_wh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Exit EDVR** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "!rm -rf /content/video.aac\n",
        "!cp -r /content/EDVR/codes/video/result/video.mp4 /content/video.mp4\n",
        "!cp -r /content/EDVR/codes/video/result/video.mp4 /content/enhanced_video.mp4\n",
        "%cd /content\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "148dzAWOTw29",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "# <b><font color=\"gree\" size=\"+3\">3. Frames interpolation ([VFIASC](https://github.com/sniklaus/sepconv-slomo))\n",
        "\n",
        "*(You can find another interpolation method below in the \"Advanced sections\" section. It runs faster but less qualitatively.)*\n",
        "\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8TCVYz2wz_RG",
        "colab_type": "text"
      },
      "source": [
        "Creator: [Simon Niklaus](http://sniklaus.com/about/welcome)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2eIC3CwoYbe6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Clone the repository and download the necessary components** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "!git clone https://github.com/sniklaus/sepconv-slomo.git VFIASC\n",
        "%cd /content/VFIASC\n",
        "!rm -rf network-l1.pytorch\n",
        "!rm -rf network-lf.pytorch\n",
        "try:\n",
        "  !gdown https://drive.google.com/uc?id=1v77wNU8sYh0hBmljeGt5LzY9ehQU0elv\n",
        "  !gdown https://drive.google.com/uc?id=1na11Ey0TB1KEDps4uEwyTBsDcmpo2kr6\n",
        "except BaseException:\n",
        "  !bash download.bash\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ekZ7p6JCdPqW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Install the necessary dependencies** { display-mode: \"form\" }\n",
        "#@markdown **This block is required. Execution time: ~ 12 minutes**\n",
        "print('You can go for a coffee while.\\n')\n",
        "!pip install cupy\n",
        "!pip install moviepy\n",
        "clear_output()\n",
        "print(\"It's time to do the next block.\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8rXNbVbVmz4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Interpolate frames** { display-mode: \"form\" }\n",
        "#@markdown **How many times increase FPS:**\n",
        "fps_boosting = 'x4' #@param [\"x4\", \"x16\"]\n",
        "%env CUDA_VISIBLE_DEVICES=0\n",
        "if fps_boosting == 'x4':\n",
        "  !python run.py --model lf --video /content/video.mp4 --out ./video_x4.mp4\n",
        "  video_name = \"video_x4.mp4\"\n",
        "elif fps_boosting == 'x16':\n",
        "  !python run.py --model lf --video /content/video.mp4 --out ./video_x4.mp4\n",
        "  !python run.py --model lf --video ./video_x4.mp4 --out ./video_x16.mp4\n",
        "  video_name = \"video_x16.mp4\"\n",
        "clear_output()\n",
        "\n",
        "mfps_of_video = int(cv2.VideoCapture(video_name).get(cv2.CAP_PROP_FPS))\n",
        "mframes_of_video = int(cv2.VideoCapture(video_name).get(cv2.CAP_PROP_FRAME_COUNT))\n",
        "#@markdown *I recommend installing* **x4**. *Otherwise, the execution of this block may be delayed or lead to an error. On top of that, the effect of SlowMo will be unnatural*"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "rdqfZhO0m0MT",
        "colab": {}
      },
      "source": [
        "#@title ##**Get result** { display-mode: \"form\" }\n",
        "!rm -rf /content/video.mp4\n",
        "!cp -r $video_name /content/video.mp4\n",
        "!cp -r $video_name /content/fps_boosted_video.mp4\n",
        "what_next = 'play' #@param [\"play\", \"download\"]\n",
        "if what_next == \"play\":\n",
        "  display(mpy.ipython_display(\"/content/fps_boosted_video.mp4\", height=400, autoplay=1, loop=1, maxduration=600))\n",
        "else:\n",
        "  files.download(\"/content/fps_boosted_video.mp4\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "z6g_E1rDmyXW",
        "colab": {}
      },
      "source": [
        "#@title ##**Convert SlowMo video to high-frequency and automatically suggest downloading it to a computer** { display-mode: \"form\" }\n",
        "!rm -rf output.mp4\n",
        "!rm -rf slowed_movie_frames\n",
        "!rm -rf /content/video.mp4\n",
        "!rm -rf /content/fps_boosted_video.mp4\n",
        "\n",
        "fr = int(fps_of_video*int(fps_boosting.split(\"x\")[1]))\n",
        "!mkdir 'slowed_movie_frames'\n",
        "vidcap = cv2.VideoCapture(video_name)\n",
        "success,image = vidcap.read()\n",
        "count = 0\n",
        "success = True\n",
        "while success:\n",
        "  cv2.imwrite(\"slowed_movie_frames/frame%09d.jpg\" % count, image)\n",
        "  success,image = vidcap.read()\n",
        "  count += 1\n",
        "\n",
        "staffs = []\n",
        "img = os.listdir(\"slowed_movie_frames/\")\n",
        "img.sort()\n",
        "clear_output()\n",
        "for i in img:\n",
        "  staffs.append(\"slowed_movie_frames/\"+i)\n",
        "\n",
        "staff = cv2.imread(staffs[0])  # get size from the 1st frame\n",
        "writer = cv2.VideoWriter(\n",
        "    'output.mp4',\n",
        "    cv2.VideoWriter_fourcc(*'MP4V'),   # codec (*'DIVX', *'MP4V', *'FMP4', *'MJPG', *'XVID', *'MP4S')\n",
        "    fr,  # fps\n",
        "    (staff.shape[1], staff.shape[0]),  # width, height\n",
        "    isColor=len(staff.shape) > 2)\n",
        "for staff in map(cv2.imread, staffs):\n",
        "    writer.write(staff)\n",
        "writer.release()\n",
        "clear_output()\n",
        "\n",
        "pfps_of_video = int(cv2.VideoCapture(\"output.mp4\").get(cv2.CAP_PROP_FPS))\n",
        "pframes_of_video = int(cv2.VideoCapture(\"output.mp4\").get(cv2.CAP_PROP_FRAME_COUNT))\n",
        "\n",
        "!cp -r output.mp4 /content/fps_boosted_video.mp4\n",
        "!cp -r output.mp4 /content/video.mp4\n",
        "files.download(\"/content/fps_boosted_video.mp4\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hvua74ctfoE2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ##**Exit VFIASC** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "%cd /content/\n",
        "clear_output()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "FEdP82Zc9zqB"
      },
      "source": [
        "---\n",
        "# <b><font color=\"gree\" size=\"+3\">4. Upscale Resolution ([ESRGAN](https://github.com/xinntao/ESRGAN))\n",
        "\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I3rNZnZwNkJY"
      },
      "source": [
        "Creator: [Xintao Wang](https://xinntao.github.io)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eBnba1MwNkJt",
        "colab": {}
      },
      "source": [
        "#@title ##**Clone the repository and download the necessary components** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "!git clone https://github.com/xinntao/ESRGAN.git\n",
        "!cp -r video.mp4 /content/ESRGAN/\n",
        "%cd /content/ESRGAN\n",
        "!git checkout tags/old-arch\n",
        "model_url = \"https://www.dropbox.com/s/vouc15j8jjp2o5n/RRDB_ESRGAN_x4_old_arch.pth?dl=0\"\n",
        "!wget $model_url --content-disposition -P models\n",
        "import architecture as arch\n",
        "import os.path\n",
        "!mkdir frames\n",
        "!rm -rf results/baboon_ESRGAN.png\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8YhZFMoqNkJ8",
        "colab": {}
      },
      "source": [
        "#@title ##**Split the video into frames** { display-mode: \"form\" }\n",
        "frames_of_video = int(cv2.VideoCapture(\"video.mp4\").get(cv2.CAP_PROP_FRAME_COUNT))\n",
        "fps_of_video = int(cv2.VideoCapture(\"video.mp4\").get(cv2.CAP_PROP_FPS))\n",
        "vidcap = cv2.VideoCapture('video.mp4')\n",
        "success,image = vidcap.read()\n",
        "count = 0\n",
        "success = True\n",
        "while success:\n",
        "  cv2.imwrite(\"frames/frame%09d.jpg\" % count, image)\n",
        "  success,image = vidcap.read()\n",
        "  count += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "AOZHY0UxNkKA",
        "colab": {}
      },
      "source": [
        "#@title ##**Upscale resolution** { display-mode: \"form\" }\n",
        "#@markdown **How many times to upscale the resolution:**\n",
        "upscale = 4 #@param {type: \"slider\", min: 4, max: 8}\n",
        "%env CUDA_VISIBLE_DEVICES=0\n",
        "device = torch.device('cuda')\n",
        "model = arch.RRDB_Net(3, 3, 64, 23, gc=32, upscale=upscale, norm_type=None, act_type='leakyrelu', \\\n",
        "                        mode='CNA', res_scale=1, upsample_mode='upconv')\n",
        "model.load_state_dict(torch.load('models/{:s}'.format('RRDB_ESRGAN_x4_old_arch.pth')), strict=True)\n",
        "model.eval()\n",
        "for k, v in model.named_parameters():\n",
        "    v.requires_grad = False\n",
        "model = model.to(device)\n",
        "\n",
        "count_frames = 0\n",
        "\n",
        "for path in glob.glob('frames/*'):\n",
        "    base = os.path.splitext(os.path.basename(path))[0]\n",
        "    img = cv2.imread(path, cv2.IMREAD_COLOR)\n",
        "    img = img * 1.0 / 255\n",
        "    img = torch.from_numpy(np.transpose(img[:, :, [2, 1, 0]], (2, 0, 1))).float()\n",
        "    img_LR = img.unsqueeze(0)\n",
        "    img_LR = img_LR.to(device)\n",
        "\n",
        "    output = model(img_LR).data.squeeze().float().cpu().clamp_(0, 1).numpy()\n",
        "    output = np.transpose(output[[2, 1, 0], :, :], (1, 2, 0))\n",
        "    output = (output * 255.0).round()\n",
        "    path = 'results/{:s}_rlt.png'.format(base)\n",
        "    cv2.imwrite(path, output)\n",
        "    count_frames += 1\n",
        "    clear_output()\n",
        "    print(\"Processed: {} из {}\".format(str(count_frames), str(frames_of_video)))\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "O2efZq9yNkKF",
        "colab": {}
      },
      "source": [
        "#@title ##**Collecting a video file** { display-mode: \"form\" }\n",
        "frames = []\n",
        "img = os.listdir(\"results/\")\n",
        "img.sort()\n",
        "for i in img:\n",
        "  frames.append(imageio.imread(\"results/\"+i))\n",
        "frames = np.array(frames)\n",
        "imageio.mimsave(\"upscaled_video.mp4\", frames, fps=fps_of_video)\n",
        "\n",
        "print('Сборка завершена')\n",
        "!cp -r upscaled_video.mp4 /content/upscaled_video.mp4\n",
        "!cp -r upscaled_video.mp4 /content/video.mp4\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sE1tifS_NkKJ",
        "colab": {}
      },
      "source": [
        "#@title ##**Get result** { display-mode: \"form\" }\n",
        "what_next = 'play' #@param [\"play\", \"download\"]\n",
        "if what_next == \"play\":\n",
        "  display(mpy.ipython_display(\"/content/upscaled_video.mp4\", height=400, autoplay=1, loop=1, maxduration=600))\n",
        "else:\n",
        "  files.download(\"/content/upscaled_video.mp4\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "VRtjLek5NkKN",
        "colab": {}
      },
      "source": [
        "#@title ##**Exit ESRGAN** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "%cd /content\n",
        "clear_output()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a165mYFXow8I",
        "colab_type": "text"
      },
      "source": [
        "# <b><font color=\"black\" size=\"+3\">Advanced sections\n",
        "<font color=\"grey\">These sections contain neural networks that are not included in the main list of video processing tools for various reasons.\n",
        "\n",
        "*↓ Click to open additional sections. ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "rCXaxlrnLCJi"
      },
      "source": [
        "---\n",
        "## <b><font color=\"grey\" size=\"+2\">Delete noise ([DeepRemaster](https://github.com/satoshiiizuka/siggraphasia2019_remastering))\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "9TvD3W60LCJ3"
      },
      "source": [
        "Creator: [Satoshi Iizuka](http://iizuka.cs.tsukuba.ac.jp/index_eng.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "3x3EUwPSLCJ6",
        "colab": {}
      },
      "source": [
        "#@title ##**Clone the repository** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "!git clone https://github.com/satoshiiizuka/siggraphasia2019_remastering.git DeepRemaster\n",
        "!cp -r /content/video.mp4 /content/DeepRemaster/\n",
        "%cd /content/DeepRemaster\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "USZ4_jo4LCKI",
        "colab": {}
      },
      "source": [
        "#@title ##**Upload the pre-trained model** { display-mode: \"form\" }\n",
        "try:\n",
        "  !rm -rf model/remasternet.pth.tar\n",
        "  !gdown https://drive.google.com/uc?id=1pB8C7c2EDUXNL0Ytk0NZ8q8bAj_nFykO\n",
        "  !gdown https://drive.google.com/uc?id=1fJJK6i6jVGJmWhoHssuHMZFmfnULxe8S -O model/remasternet.pth.tar\n",
        "  !bash ./checking_model.sh\n",
        "except BaseException:\n",
        "  !rm -rf model/remasternet.pth.tar\n",
        "  !bash ./download_model.sh\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LmffmF1MLCKN",
        "colab": {}
      },
      "source": [
        "#@title ##**Remove frame noise** { display-mode: \"form\" }\n",
        "!python remaster.py --input video.mp4 --disable_colorization --gpu\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cVmA3bT6LCKS",
        "colab": {}
      },
      "source": [
        "#@title ##**Get result** { display-mode: \"form\" }\n",
        "!rm -rf /content/video.mp4\n",
        "!cp -r video_out.mp4 /content/video.mp4\n",
        "!cp -r video_out.mp4 /content/denoise_video.mp4\n",
        "what_next = 'play' #@param [\"play\", \"download\"]\n",
        "if what_next == \"play\":\n",
        "  display(mpy.ipython_display(\"/content/denoise_video.mp4\", height=400, autoplay=1, loop=1, maxduration=600))\n",
        "else:\n",
        "  files.download('/content/denoise_video.mp4')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "T4e7kNxzLCKX",
        "colab": {}
      },
      "source": [
        "#@title ##**Exit DeepRemaster** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "!cp -r video_out.mp4 /content/video.mp4\n",
        "!cp -r video_out.mp4 /content/denoise_video.mp4\n",
        "%cd /content\n",
        "clear_output()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lEhNSwi50Wa5",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "## <b><font color=\"grey\" size=\"+2\">Upscale resolution 2.0 ([Fast-SRGAN](https://github.com/HasnainRaz/Fast-SRGAN))\n",
        "*If you decide to execute this block, it is recommended that you do this to the ambassador of section* \"Delete noise\"\n",
        "\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "48zbpTtCzTcQ",
        "colab_type": "text"
      },
      "source": [
        "Creator: [Hasnain Raza](https://github.com/HasnainRaz)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P7JfV7DX0YJ3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Clone the repository and install the necessary dependencies** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "!git clone https://github.com/HasnainRaz/Fast-SRGAN.git\n",
        "!cp -r /content/video.mp4 /content/Fast-SRGAN/\n",
        "%cd /content/Fast-SRGAN\n",
        "!pip install -r requirements.txt\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ykB8cTVG03Rk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Generate high-resolution frames** { display-mode: \"form\" }\n",
        "#@markdown *This block works very slowly, and the quality of the resulting images is very controversial, which is why I put this section into Advanced.*\n",
        "!rm -rf hr_frames\n",
        "!rm -rf frames\n",
        "!mkdir 'hr_frames'\n",
        "!mkdir 'frames'\n",
        "\n",
        "vidcap = cv2.VideoCapture('video.mp4')\n",
        "success,image = vidcap.read()\n",
        "count = 0\n",
        "success = True\n",
        "while success:\n",
        "  cv2.imwrite(\"frames/frame%09d.jpg\" % count, image)\n",
        "  success,image = vidcap.read()\n",
        "  count += 1\n",
        "\n",
        "!python infer.py --image_dir 'frames/' --output_dir 'hr_frames/'\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5dnoDuDLWq0-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Putting frames in a single video** { display-mode: \"form\" }\n",
        "\n",
        "!rm -rf HR_video.mp4\n",
        "!rm -rf ../HR_video.mp4\n",
        "frames = []\n",
        "img = os.listdir(\"hr_frames/\")\n",
        "img.sort()\n",
        "for i in img:\n",
        "  frames.append(imageio.imread(\"hr_frames/\"+i))\n",
        "frames = np.array(frames)\n",
        "imageio.mimsave(\"HR_video.mp4\", frames)\n",
        "\n",
        "print('Assembly completed')\n",
        "!cp -r HR_video.mp4 ../HR_video.mp4\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XkkXV9zWSY3A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Get result** { display-mode: \"form\" }\n",
        "what_next = 'play' #@param [\"play\", \"download\"]\n",
        "if what_next == \"play\":\n",
        "  display(mpy.ipython_display(\"HR_video.mp4\", height=400, autoplay=1, loop=1, maxduration=600))\n",
        "else:\n",
        "  files.download(\"HR_video.mp4\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SLS-IdH7cHFw",
        "colab": {}
      },
      "source": [
        "#@title ###**Convert SlowMo video to high-frequency and automatically suggest downloading it to a computer** { display-mode: \"form\" }\n",
        "fps_of_video = int(cv2.VideoCapture(\"video.mp4\").get(cv2.CAP_PROP_FPS))\n",
        "\n",
        "!rm -rf HR_video.mp4\n",
        "!rm -rf ../HR_video.mp4\n",
        "frames = []\n",
        "img = os.listdir(\"hr_frames/\")\n",
        "img.sort()\n",
        "clear_output()\n",
        "for i in img:\n",
        "  frames.append(\"hr_frames/\"+i)\n",
        "frames = np.array(frames)\n",
        "\n",
        "frame = cv2.imread(frames[0])  # get size from the 1st frame\n",
        "writer = cv2.VideoWriter(\n",
        "    'HR_video.mp4',\n",
        "    cv2.VideoWriter_fourcc(*'MP4V'),   # codec (*'DIVX', *'MP4V', *'FMP4', *'MJPG', *'XVID', *'MP4S')\n",
        "    fps_of_video,  # fps\n",
        "    (frame.shape[1], frame.shape[0]),  # width, height\n",
        "    isColor=len(frame.shape) > 2)\n",
        "for frame in map(cv2.imread, frames):\n",
        "    writer.write(frame)\n",
        "writer.release()\n",
        "\n",
        "clear_output()\n",
        "print('Конвертирование завершено. Ожидайте начала загрузки (при появлении можно нажать Отмена).')\n",
        "!cp -r HR_video.mp4 /content/HR_video.mp4\n",
        "files.download(\"HR_video.mp4\")\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OU8cRFef3BvA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Exit Fast-SRGAN** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "!rm -rf /content/video.mp4\n",
        "!cp -r HR_video.mp4 /content/video.mp4\n",
        "%cd /content/\n",
        "clear_output()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "hCmtWECtAE8X"
      },
      "source": [
        "---\n",
        "## <b><font color=\"grey\" size=\"+2\">Faster but lower quality frame interpolation ([VFI-CFT](https://github.com/MortenHannemose/pytorch-vfi-cft))\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dhF9d-XDzEGY",
        "colab_type": "text"
      },
      "source": [
        "Creator: [Morten Hannemose](https://scholar.google.ru/citations?user=AH58CjUAAAAJ&hl=ru)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WNPIlvbrAE8n",
        "colab": {}
      },
      "source": [
        "#@title ###**Clone a repository and download a pre-trained model** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "!git clone https://github.com/MortenHannemose/pytorch-vfi-cft.git VFI-CFT\n",
        "%cd /content/VFI-CFT\n",
        "video = \"video_2x_slow.mp4\"\n",
        "try:\n",
        "  !rm -rf VFI_CFT_weights.pt.gz\n",
        "  !rm -rf slow_movie.py\n",
        "  !gdown https://drive.google.com/uc?id=1-5zS1XOhwVXJhD4JDZWmG6plC_IVVg8b\n",
        "  !gdown https://drive.google.com/uc?id=1YinJeeUbTbDynD6kw6f6h76CAkRqFdHK\n",
        "except BaseException:\n",
        "  !rm -rf VFI_CFT_weights.pt.gz\n",
        "  !gdown https://drive.google.com/uc?id=1rQAi0UXcaMcEo8_l4oSd8vJUVzp2dIIF\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ezRAPiKv_nar",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Interpolate frames** { display-mode: \"form\" }\n",
        "#@markdown **How many times increase FPS:**\n",
        "!rm -rf slowed_movie_frames\n",
        "!rm -rf $video\n",
        "!rm -rf video.mp4\n",
        "!cp -r /content/video.mp4 /content/VFI-CFT\n",
        "fps_boosting = 2 #@param {type:\"slider\", min:2, max:8, step:1}\n",
        "\n",
        "!python slow_movie.py -m video.mp4 -f $fps_boosting\n",
        "\n",
        "video = \"video_\"+str(fps_boosting)+\"x_slow.mp4\"\n",
        "frames = []\n",
        "img = os.listdir(\"slowed_movie_frames/\")\n",
        "img.sort()\n",
        "clear_output()\n",
        "for i in img:\n",
        "  frames.append(imageio.imread(\"slowed_movie_frames/\"+i))\n",
        "frames = np.array(frames)\n",
        "imageio.mimsave(\"fps_boosted_video.mp4\", frames)\n",
        "#@markdown *More than x4 is not recommended. SlowMo effect will be unnatural*\n",
        "clear_output()\n",
        "\n",
        "nfps_of_video = int(cv2.VideoCapture(\"fps_boosted_video.mp4\").get(cv2.CAP_PROP_FPS))\n",
        "nframes_of_video = int(cv2.VideoCapture(\"fps_boosted_video.mp4\").get(cv2.CAP_PROP_FRAME_COUNT))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yg5GYF1DBevE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Get result** { display-mode: \"form\" }\n",
        "!rm -rf /content/video.mp4\n",
        "!cp -r fps_boosted_video.mp4 /content/video.mp4\n",
        "!cp -r fps_boosted_video.mp4 /content/fps_boosted_video.mp4\n",
        "what_next = 'play' #@param [\"play\", \"download\"]\n",
        "if what_next == \"play\":\n",
        "  display(mpy.ipython_display(\"/content/fps_boosted_video.mp4\", height=400, autoplay=1, loop=1, maxduration=600))\n",
        "else:\n",
        "  files.download(\"/content/fps_boosted_video.mp4\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vhDZe6OFAE9V",
        "colab": {}
      },
      "source": [
        "#@title ###**Exit VFI-CFT** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "%cd /content/\n",
        "clear_output()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "m0S8PshiV_aq"
      },
      "source": [
        "---\n",
        "## <b><font color=\"grey\" size=\"+2\">Bonus. Colorize photo and parallax effect ([DeOldify](https://github.com/jantic/DeOldify) + [3D Ken Burns](https://github.com/sniklaus/3d-ken-burns))</b>\n",
        "This section can be performed very first. At the end, the resulting video can be restored in other sections. It is important that before going through other sections, you need to go to **\"0. Preparation for processing\"** and execute the block **\"Install all the necessary libraries\"**. Further, all other sections will function normally.\n",
        "\n",
        "*↓ **Click to open section** ↓*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "86qN9siLYNix",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Upload photo** { display-mode: \"form\" }\n",
        "#@markdown *Enter a link to the photo below, or leave the **source_url** field blank (in this case, you will be prompted to download the video from your computer).*\n",
        "from IPython.display import clear_output\n",
        "from google.colab import files\n",
        "import os\n",
        "%cd /content\n",
        "!rm -rf sample_data\n",
        "source_url = 'http://www.worldfocus.ru/_ph/1/444433832.jpg' #@param {type:\"string\"}\n",
        "\n",
        "if source_url == '':\n",
        "  uploaded = files.upload()\n",
        "  for fn in uploaded.keys():\n",
        "    print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "        name=fn, length=len(uploaded[fn])))\n",
        "  pic_name = \"photo.\" + fn.split(\".\")[1]\n",
        "  !mv -f $fn $pic_name\n",
        "\n",
        "else:\n",
        "  try:\n",
        "    pic_name = \"photo.\" + source_url.split('.')[-1]\n",
        "    !wget $source_url -O $pic_name\n",
        "  except BaseException:\n",
        "    print('Something wrong. Try uploading photos from your computer.')\n",
        "\n",
        "!cp -r photo.jpg downloaded_photo.jpg\n",
        "clear_output()\n",
        "#@markdown *If an error occurs during execution, then run this block again*"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Uu22Uwc3V_aw"
      },
      "source": [
        "### <b><font color=\"aquamarine\" size=\"+1\">1) DeOldify - Colorize photo</b>\n",
        "Creator: [Jason Antic](https://twitter.com/citnaj)\n",
        "\n",
        "*↓ Click to open subsection ↓*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "37_1nFPhaDNn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Clone the repository and install the necessary dependencies** { display-mode: \"form\" }\n",
        "import torch\n",
        "import fastai\n",
        "%cd /content\n",
        "!git clone https://github.com/jantic/DeOldify.git DeOldifyPhoto\n",
        "%cd /content/DeOldifyPhoto\n",
        "from deoldify import device\n",
        "from deoldify.device_id import DeviceId\n",
        "device.set(device=DeviceId.GPU0)\n",
        "if not torch.cuda.is_available():\n",
        "    print('GPU not available.')\n",
        "!pip install -r colab_requirements.txt\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oVztY-tcbZP_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Upload the pre-trained model** { display-mode: \"form\" }\n",
        "from deoldify.visualize import *\n",
        "torch.backends.cudnn.benchmark = True\n",
        "!mkdir 'models'\n",
        "!rm -rf models/ColorizeStable_gen.pth\n",
        "!wget https://www.dropbox.com/s/mwjep3vyqk5mkjc/ColorizeStable_gen.pth?dl=0 -O ./models/ColorizeStable_gen.pth\n",
        "!wget https://media.githubusercontent.com/media/jantic/DeOldify/master/resource_images/watermark.png -O ./resource_images/watermark.png\n",
        "colorizer = get_image_colorizer(artistic=False)\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z4KPSH8HdYtM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Colorize photo** { display-mode: \"form\" }\n",
        "#@markdown **21** *- optimal value*\n",
        "!rm -rf ../colorized_photo.jpg\n",
        "from google.colab import files\n",
        "render_factor = 35  #@param {type: \"slider\", min: 5, max: 44}\n",
        "image_path = colorizer.plot_transformed_image(path='../photo.jpg', render_factor=render_factor, compare=True)\n",
        "!cp -r result_images/photo.jpg ../colorized_photo.jpg\n",
        "\n",
        "#@markdown **Download:**\n",
        "download_image = False #@param {type:\"boolean\"}\n",
        "if download_image == True:\n",
        "  files.download(\"result_images/photo.jpg\")\n",
        "else:\n",
        "  show_image_in_notebook(image_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5CDSDkFgNUu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Exit DeOldify** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "!cp -r result_images/photo.jpg ../photo.jpg\n",
        "%cd /content\n",
        "clear_output()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rDF6s4jYYF7I",
        "colab_type": "text"
      },
      "source": [
        "### <b><font color=\"aquamarine\" size=\"+1\">2) 3D Ken Burns - Parallax effect</b>\n",
        "\n",
        "Creator: [Simon Niklaus](http://sniklaus.com/about/welcome)\n",
        "\n",
        "*↓ Click to open subsection ↓*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WnE5sxJDiX3u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Сlone the repository and download the necessary components** { display-mode: \"form\" }\n",
        "import os\n",
        "import imageio\n",
        "import numpy as np\n",
        "import moviepy.editor as mpy\n",
        "from moviepy.video.io.ffmpeg_writer import FFMPEG_VideoWriter\n",
        "%matplotlib inline\n",
        "%cd /content\n",
        "os.environ['CUDA_HOME'] = \"/usr/local/cuda\"\n",
        "!pip install moviepy\n",
        "!pip install gevent\n",
        "!sudo apt install python-gevent\n",
        "!git clone https://github.com/sniklaus/3d-ken-burns.git 3D-Ken-Burns\n",
        "%cd /content/3D-Ken-Burns\n",
        "!mkdir 'results'\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R65yOrv_iYG1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Upload the pre-trained models** { display-mode: \"form\" }\n",
        "!rm -rf models/disparity-estimation.pytorch\n",
        "!rm -rf models/disparity-refinement.pytorch\n",
        "!rm -rf models/pointcloud-inpainting.pytorch\n",
        "try:\n",
        "  !gdown https://drive.google.com/uc?id=1-0M3GAZkiOexs7gqgWuORzqYAp00DeE- -O ./models/disparity-estimation.pytorch\n",
        "  !gdown https://drive.google.com/uc?id=1-11KRkOKxqLmKOQk69PDMYbaem-3mH04 -O ./models/disparity-refinement.pytorch\n",
        "  !gdown https://drive.google.com/uc?id=1-1-VjUQHVDwxtLY9xn65c3LpKCeIW6jH -O ./models/pointcloud-inpainting.pytorch\n",
        "except BaseException:\n",
        "  !wget -O ./3d-ken-burns/models/disparity-estimation.pytorch http://content.sniklaus.com/kenburns/network-disparity.pytorch\n",
        "  !wget -O ./3d-ken-burns/models/disparity-refinement.pytorch http://content.sniklaus.com/kenburns/network-refinement.pytorch\n",
        "  !wget -O ./3d-ken-burns/models/pointcloud-inpainting.pytorch http://content.sniklaus.com/kenburns/network-inpainting.pytorch\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vIwOeaZnka8j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Create Parallax Effect** { display-mode: \"form\" }\n",
        "!cp -r  ../photo.jpg ./images/photo.jpg\n",
        "!rm images/README.md\n",
        "!rm images/doublestrike.jpg\n",
        "!for image in ./images/*; do python autozoom.py --in $image --out ./results/$(basename $image | cut -f1 -d '.').mp4; done\n",
        "clear_output()\n",
        "#@markdown *To save a video, right-click on it and select* **Save**\n",
        "display(mpy.ipython_display(\"results/photo.mp4\", height=400, autoplay=1, loop=1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_3S6pFCXmw4H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Exit 3D Ken Burns** { display-mode: \"form\" }\n",
        "#@markdown *If you are going to continue to carry out the following points, then first of all, be sure to complete this block!*\n",
        "!cp -r results/photo.mp4 ../parallax_video.mp4\n",
        "!cp -r results/photo.mp4 ../video.mp4\n",
        "%cd /content\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UbuoqcxNGcmu"
      },
      "source": [
        "---\n",
        "## <b><font color=\"grey\" size=\"+2\">Bonus. Synthesizing audio using voice ([DDSP](https://github.com/magenta/ddsp))\n",
        "*↓ Click to open section ↓*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "J6p5zYzMGcm8"
      },
      "source": [
        "Creator: [Magenta](https://github.com/magenta)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UugbxH7AGcnG",
        "colab": {}
      },
      "source": [
        "#@title ###**Clone the repository and install all requirements** { display-mode: \"form\" }\n",
        "%cd /content\n",
        "%tensorflow_version 2.x\n",
        "!pip install -qU ddsp\n",
        "\n",
        "from IPython.display import clear_output\n",
        "from __future__ import absolute_import\n",
        "from __future__ import division\n",
        "from __future__ import print_function\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "import copy\n",
        "import os\n",
        "import time\n",
        "import crepe\n",
        "import ddsp\n",
        "import ddsp.training\n",
        "from ddsp.colab.colab_utils import (download, play, record, specplot, upload,\n",
        "                                    DEFAULT_SAMPLE_RATE)\n",
        "import gin\n",
        "from google.colab import files\n",
        "import librosa\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import tensorflow.compat.v2 as tf\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "sample_rate = DEFAULT_SAMPLE_RATE  # 16000\n",
        "clear_output()\n",
        "print('Done!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lWWe-70oHARB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Record or upload audio** { display-mode: \"form\" }\n",
        "\n",
        "#@markdown * Attention! Audio should be monotonous (*one voice or one instrument*)\n",
        "#@markdown * If you chosen \"**Record**\", the recording will begin as soon as the inscription appears \"**Starting recording for X seconds...**\"\n",
        "#@markdown * Sound recording doesn't work on all browsers. Testing was in Google Chrome and Opera.\n",
        "record_or_upload = \"Record\"  #@param [\"Record\", \"Upload (.mp3 or .wav)\"]\n",
        "\n",
        "record_seconds =   10  #@param {type:\"number\", min:1, max:10, step:1}\n",
        "\n",
        "if record_or_upload == \"Record\":\n",
        "  audio = record(seconds=record_seconds)\n",
        "else:\n",
        "  # Load audio sample here (.mp3 or .wav3 file)\n",
        "  # Just use the first file.\n",
        "  filenames, audios = upload()\n",
        "  audio = audios[0]\n",
        "audio = audio[np.newaxis, :]\n",
        "clear_output()\n",
        "print('\\nExtracting audio features...')\n",
        "\n",
        "# Plot.\n",
        "specplot(audio)\n",
        "play(audio)\n",
        "\n",
        "# Setup the session.\n",
        "ddsp.spectral_ops.reset_crepe()\n",
        "\n",
        "# Compute features.\n",
        "start_time = time.time()\n",
        "audio_features = ddsp.training.eval_util.compute_audio_features(audio)\n",
        "audio_features['loudness_db'] = audio_features['loudness_db'].astype(np.float32)\n",
        "audio_features_mod = None\n",
        "print('Audio features took %.1f seconds' % (time.time() - start_time))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZeiNh3hgHLqT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Choose a musical instrument** { display-mode: \"form\" }\n",
        "\n",
        "model = 'Violin' #@param ['Violin', 'Flute', 'Flute2', 'Trumpet', 'Tenor_Saxophone','Upload your own (checkpoint folder as .zip)']\n",
        "MODEL = model\n",
        "\n",
        "GCS_CKPT_DIR = 'gs://ddsp/models/tf2'\n",
        "\n",
        "def find_model_dir(dir_name):\n",
        "  # Iterate through directories until model directory is found\n",
        "  for root, dirs, filenames in os.walk(dir_name):\n",
        "    for filename in filenames:\n",
        "      if filename.endswith(\".gin\") and not filename.startswith(\".\"):\n",
        "        model_dir = root\n",
        "        break\n",
        "  return model_dir \n",
        "\n",
        "\n",
        "if model in ('Violin', 'Flute', 'Flute2', 'Trumpet', 'Tenor_Saxophone'):\n",
        "  # Pretrained models.\n",
        "  PRETRAINED_DIR = '/content/pretrained'\n",
        "  # Copy over from gs:// for faster loading.\n",
        "  !rm -r $PRETRAINED_DIR &> /dev/null\n",
        "  !mkdir $PRETRAINED_DIR &> /dev/null\n",
        "  model_dir = os.path.join(GCS_CKPT_DIR, 'solo_%s_ckpt' % model.lower())\n",
        "  !gsutil cp $model_dir/* $PRETRAINED_DIR &> /dev/null\n",
        "  model_dir = PRETRAINED_DIR\n",
        "  gin_file = os.path.join(model_dir, 'operative_config-0.gin')\n",
        "\n",
        "else:\n",
        "  # User models.\n",
        "  UPLOAD_DIR = '/content/uploaded'\n",
        "  !mkdir $UPLOAD_DIR\n",
        "  uploaded_files = files.upload()\n",
        "\n",
        "  for fnames in uploaded_files.keys():\n",
        "    print(\"Unzipping... {}\".format(fnames))\n",
        "    !unzip -o \"/content/$fnames\" -d $UPLOAD_DIR &> /dev/null\n",
        "  model_dir = find_model_dir(UPLOAD_DIR)\n",
        "  gin_file = os.path.join(model_dir, 'operative_config-0.gin')\n",
        "\n",
        "# Parse gin config,\n",
        "with gin.unlock_config():\n",
        "  gin.parse_config_file(gin_file, skip_unknown=True)\n",
        "\n",
        "# Assumes only one checkpoint in the folder, 'ckpt-[iter]`.\n",
        "ckpt_files = [f for f in tf.io.gfile.listdir(model_dir) if 'ckpt' in f]\n",
        "ckpt_name = ckpt_files[0].split('.')[0]\n",
        "ckpt = os.path.join(model_dir, ckpt_name)\n",
        "\n",
        "# Ensure dimensions and sampling rates are equal\n",
        "time_steps_train = gin.query_parameter('DefaultPreprocessor.time_steps')\n",
        "n_samples_train = gin.query_parameter('Additive.n_samples')\n",
        "hop_size = int(n_samples_train / time_steps_train)\n",
        "\n",
        "time_steps = int(audio.shape[1] / hop_size)\n",
        "n_samples = time_steps * hop_size\n",
        "\n",
        "# print(\"===Trained model===\")\n",
        "# print(\"Time Steps\", time_steps_train)\n",
        "# print(\"Samples\", n_samples_train)\n",
        "# print(\"Hop Size\", hop_size)\n",
        "# print(\"\\n===Resynthesis===\")\n",
        "# print(\"Time Steps\", time_steps)\n",
        "# print(\"Samples\", n_samples)\n",
        "# print('')\n",
        "\n",
        "gin_params = [\n",
        "    'Additive.n_samples = {}'.format(n_samples),\n",
        "    'FilteredNoise.n_samples = {}'.format(n_samples),\n",
        "    'DefaultPreprocessor.time_steps = {}'.format(time_steps),\n",
        "]\n",
        "\n",
        "with gin.unlock_config():\n",
        "  gin.parse_config(gin_params)\n",
        "\n",
        "\n",
        "# Trim all input vectors to correct lengths \n",
        "for key in ['f0_hz', 'f0_confidence', 'loudness_db']:\n",
        "  audio_features[key] = audio_features[key][:time_steps]\n",
        "audio_features['audio'] = audio_features['audio'][:, :n_samples]\n",
        "\n",
        "\n",
        "# Set up the model just to predict audio given new conditioning\n",
        "model = ddsp.training.models.Autoencoder()\n",
        "model.restore(ckpt)\n",
        "\n",
        "# Build model by running a batch through it.\n",
        "start_time = time.time()\n",
        "_ = model(audio_features, training=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QZIyC2icHVc2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title ###**Modify conditioning** { display-mode: \"form\" }\n",
        "\n",
        "#@markdown These models were not explicitly trained to perform timbre transfer, so they may sound unnatural if the incoming loudness and frequencies are very different then the training data (which will always be somewhat true). \n",
        "\n",
        "#@markdown This button will at least adjusts the average loudness and pitch to be similar to the training data (although not for user trained models).\n",
        "\n",
        "auto_adjust = True #@param{type:\"boolean\"}\n",
        "\n",
        "#@markdown You can also make additional manual adjustments:\n",
        "#@markdown * Shift the fundmental frequency to a more natural register.\n",
        "#@markdown * Silence audio below a threshold on f0_confidence.\n",
        "#@markdown * Adjsut the overall loudness level.\n",
        "f0_octave_shift =  0 #@param {type:\"slider\", min:-2, max:2, step:1}\n",
        "f0_confidence_threshold =  0 #@param {type:\"slider\", min:0.0, max:1.0, step:0.05}\n",
        "loudness_db_shift = 0 #@param {type:\"slider\", min:-20, max:20, step:1}\n",
        "\n",
        "#@markdown You might get more realistic sounds by shifting a few dB, or try going extreme and see what weird sounds you can make...\n",
        "\n",
        "audio_features_mod = {k: v.copy() for k, v in audio_features.items()}\n",
        "\n",
        "\n",
        "## Helper functions.\n",
        "def shift_ld(audio_features, ld_shift=0.0):\n",
        "  \"\"\"Shift loudness by a number of ocatves.\"\"\"\n",
        "  audio_features['loudness_db'] += ld_shift\n",
        "  return audio_features\n",
        "\n",
        "\n",
        "def shift_f0(audio_features, f0_octave_shift=0.0):\n",
        "  \"\"\"Shift f0 by a number of ocatves.\"\"\"\n",
        "  audio_features['f0_hz'] *= 2.0 ** (f0_octave_shift)\n",
        "  audio_features['f0_hz'] = np.clip(audio_features['f0_hz'], \n",
        "                                    0.0, \n",
        "                                    librosa.midi_to_hz(110.0))\n",
        "  return audio_features\n",
        "\n",
        "\n",
        "def mask_by_confidence(audio_features, confidence_level=0.1):\n",
        "  \"\"\"For the violin model, the masking causes fast dips in loudness. \n",
        "  This quick transient is interpreted by the model as the \"plunk\" sound.\n",
        "  \"\"\"\n",
        "  mask_idx = audio_features['f0_confidence'] < confidence_level\n",
        "  audio_features['f0_hz'][mask_idx] = 0.0\n",
        "  # audio_features['loudness_db'][mask_idx] = -ddsp.spectral_ops.LD_RANGE\n",
        "  return audio_features\n",
        "\n",
        "\n",
        "def smooth_loudness(audio_features, filter_size=3):\n",
        "  \"\"\"Smooth loudness with a box filter.\"\"\"\n",
        "  smoothing_filter = np.ones([filter_size]) / float(filter_size)\n",
        "  audio_features['loudness_db'] = np.convolve(audio_features['loudness_db'], \n",
        "                                           smoothing_filter, \n",
        "                                           mode='same')\n",
        "  return audio_features\n",
        "\n",
        "if auto_adjust:\n",
        "  if MODEL in ['Violin', 'Flute', 'Flute2', 'Trumpet', 'Saxophone', 'Tenor_Saxophone']:\n",
        "    # Adjust the peak loudness.\n",
        "    l = audio_features['loudness_db']\n",
        "    model_ld_avg_max = {\n",
        "        'Violin': -34.0,\n",
        "        'Flute': -45.0,\n",
        "        'Flute2': -44.0,\n",
        "        'Trumpet': -52.3,\n",
        "        'Tenor_Saxophone': -31.2\n",
        "    }[MODEL]\n",
        "    ld_max = np.max(audio_features['loudness_db'])\n",
        "    ld_diff_max = model_ld_avg_max - ld_max\n",
        "    audio_features_mod = shift_ld(audio_features_mod, ld_diff_max)\n",
        "\n",
        "    # Further adjust the average loudness above a threshold.\n",
        "    l = audio_features_mod['loudness_db']\n",
        "    model_ld_mean = {\n",
        "        'Violin': -44.0,\n",
        "        'Flute': -51.0,\n",
        "        'Flute2': -53.0,\n",
        "        'Trumpet': -69.2,\n",
        "        'Tenor_Saxophone': -50.8\n",
        "    }[MODEL]\n",
        "    ld_thresh = -70.0\n",
        "    ld_mean = np.mean(l[l > ld_thresh])\n",
        "    ld_diff_mean = model_ld_mean - ld_mean\n",
        "    audio_features_mod = shift_ld(audio_features_mod, ld_diff_mean)\n",
        "\n",
        "    # Shift the pitch register.\n",
        "    model_p_mean = {\n",
        "        'Violin': 73.0,\n",
        "        'Flute': 81.0,\n",
        "        'Flute2': 74.0,\n",
        "        'Trumpet': 65.8,\n",
        "        'Tenor_Saxophone': 57.8\n",
        "    }[MODEL]\n",
        "    p = librosa.hz_to_midi(audio_features['f0_hz'])\n",
        "    p[p == -np.inf] = 0.0\n",
        "    p_mean = p[l > ld_thresh].mean()\n",
        "    p_diff = model_p_mean - p_mean\n",
        "    p_diff_octave = p_diff / 12.0\n",
        "    round_fn = np.floor if p_diff_octave > 1.5 else np.ceil\n",
        "    p_diff_octave = round_fn(p_diff_octave)\n",
        "    audio_features_mod = shift_f0(audio_features_mod, p_diff_octave)\n",
        "\n",
        "  else:\n",
        "    print('\\nUser uploaded model: disabling auto-adjust.')\n",
        "\n",
        "  \n",
        "audio_features_mod = shift_ld(audio_features_mod, loudness_db_shift)\n",
        "audio_features_mod = shift_f0(audio_features_mod, f0_octave_shift)\n",
        "audio_features_mod = mask_by_confidence(audio_features_mod, f0_confidence_threshold)\n",
        "\n",
        "# Resynthesize Audio.\n",
        "af = audio_features if audio_features_mod is None else audio_features_mod\n",
        "\n",
        "# Run a batch of predictions.\n",
        "start_time = time.time()\n",
        "audio_gen = model(af, training=False)\n",
        "\n",
        "print('-------------------------------------------------------------------------------')\n",
        "print('--- To download audio, right-click on the player and select \"Save audio as\" ---')\n",
        "print('-------------------------------------------------------------------------------')\n",
        "print('\\n')\n",
        "\n",
        "# Audio.\n",
        "print('Resynthesis')\n",
        "play(audio_gen)\n",
        "\n",
        "print('Original')\n",
        "play(audio)\n",
        "\n",
        "# Plot\n",
        "specplot(audio_gen)\n",
        "plt.title(\"Resynthesis\")\n",
        "\n",
        "specplot(audio)\n",
        "_ = plt.title(\"Original\")"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}